diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d34774700654a0..78198176ba88d6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,6 +5,7 @@ "--security-opt", "seccomp=unconfined", "--network=host", + "--privileged", "-v", "/dev/bus/usb:/dev/bus/usb:ro", "--device-cgroup-rule=a 189:* rmw", diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c0080186a959b8..dbabc1ff3b9763 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -84,13 +84,15 @@ jobs: # actually succeeded, because that just wastes space. rsync -a out/debug/standalone/ objdir-clone || true - name: Run Tests - timeout-minutes: 20 + timeout-minutes: 30 run: | - scripts/tests/test_suites.sh -n - - name: Run TV Tests - timeout-minutes: 10 - run: | - scripts/tests/test_suites.sh -n -a tv + ./scripts/run_in_build_env.sh \ + "./scripts/tests/run_test_suite.py run \ + --iterations 2 \ + --chip-tool ./out/debug/standalone/chip-tool \ + --all-clusters-app ./out/debug/standalone/chip-all-clusters-app \ + --tv-app ./out/debug/standalone/chip-tv-app \ + " - name: Uploading core files uses: actions/upload-artifact@v2 if: ${{ failure() }} && ${{ !env.ACT }} @@ -167,17 +169,18 @@ jobs: # The idea is to not upload our objdir unless builds have # actually succeeded, because that just wastes space. rsync -a out/debug/standalone/ objdir-clone || true - - name: Run Test Suites - timeout-minutes: 35 + - name: Run Tests + timeout-minutes: 45 run: | - scripts/tests/test_suites.sh - - name: Uploading application logs - uses: actions/upload-artifact@v2 - if: ${{ failure() }} && ${{ !env.ACT }} - with: - name: test-suite-app-logs-${{ matrix.type }}-${{ matrix.eventloop }} - path: /tmp/test_suites_app_logs/ - retention-days: 5 + ./scripts/run_in_build_env.sh \ + "./scripts/tests/run_test_suite.py \ + --target-skip-glob 'tv-*' \ + run \ + --iterations 2 \ + --chip-tool ./out/debug/standalone/chip-tool \ + --all-clusters-app ./out/debug/standalone/chip-all-clusters-app \ + --tv-app ./out/debug/standalone/chip-tv-app \ + " - name: Uploading core files uses: actions/upload-artifact@v2 if: ${{ failure() }} && ${{ !env.ACT }} diff --git a/.restyled.yaml b/.restyled.yaml index e4092a0ccda7e6..176c2498fedc5b 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -67,7 +67,6 @@ exclude: - "third_party/nanopb/repo/**/*" - "src/android/CHIPTool/gradlew" # gradle wrapper generated file - "third_party/android_deps/gradlew" # gradle wrapper generated file - - "scripts/tests/test_suites.sh" # overly agressive shell harden changed_paths: diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py index 2806e811225dfc..c818bb2628660a 100644 --- a/scripts/build/build/targets.py +++ b/scripts/build/build/targets.py @@ -106,9 +106,11 @@ def HostTargets(): app_targets = [] - # RPC console compilation only for native + # Don't cross compile some builds app_targets.append( targets[0].Extend('rpc-console', app=HostApp.RPC_CONSOLE)) + app_targets.append( + targets[0].Extend('tv-app', app=HostApp.TV_APP)) for target in targets: app_targets.append(target.Extend( diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py index 548358092d48e0..c5887cd2310199 100644 --- a/scripts/build/builders/host.py +++ b/scripts/build/builders/host.py @@ -26,6 +26,7 @@ class HostApp(Enum): THERMOSTAT = auto() RPC_CONSOLE = auto() MIN_MDNS = auto() + TV_APP = auto() def ExamplePath(self): if self == HostApp.ALL_CLUSTERS: @@ -38,6 +39,8 @@ def ExamplePath(self): return 'common/pigweed/rpc_console' if self == HostApp.MIN_MDNS: return 'minimal-mdns' + if self == HostApp.TV_APP: + return 'tv-app/linux' else: raise Exception('Unknown app type: %r' % self) @@ -60,6 +63,9 @@ def OutputNames(self): yield 'minimal-mdns-client.map' yield 'minimal-mdns-server' yield 'minimal-mdns-server.map' + elif self == HostApp.TV_APP: + yield 'chip-tv-app' + yield 'chip-tv-app.map' else: raise Exception('Unknown app type: %r' % self) diff --git a/scripts/build/testdata/build_linux_on_x64.txt b/scripts/build/testdata/build_linux_on_x64.txt index 99c49ec1d5ddeb..71c5facfb23cb4 100644 --- a/scripts/build/testdata/build_linux_on_x64.txt +++ b/scripts/build/testdata/build_linux_on_x64.txt @@ -68,6 +68,12 @@ gn gen --check --fail-on-unused-args --root={root}/examples/thermostat/linux {ou # Generating linux-x64-thermostat-ipv6only gn gen --check --fail-on-unused-args --root={root}/examples/thermostat/linux --args=chip_inet_config_enable_ipv4=false {out}/linux-x64-thermostat-ipv6only +# Generating linux-x64-tv-app +gn gen --check --fail-on-unused-args --root={root}/examples/tv-app/linux {out}/linux-x64-tv-app + +# Generating linux-x64-tv-app-ipv6only +gn gen --check --fail-on-unused-args --root={root}/examples/tv-app/linux --args=chip_inet_config_enable_ipv4=false {out}/linux-x64-tv-app-ipv6only + # Building linux-arm64-all-clusters ninja -C {out}/linux-arm64-all-clusters @@ -118,3 +124,9 @@ ninja -C {out}/linux-x64-thermostat # Building linux-x64-thermostat-ipv6only ninja -C {out}/linux-x64-thermostat-ipv6only + +# Building linux-x64-tv-app +ninja -C {out}/linux-x64-tv-app + +# Building linux-x64-tv-app-ipv6only +ninja -C {out}/linux-x64-tv-app-ipv6only diff --git a/scripts/tests/chiptest/__init__.py b/scripts/tests/chiptest/__init__.py new file mode 100644 index 00000000000000..0e7223e01d4af9 --- /dev/null +++ b/scripts/tests/chiptest/__init__.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pathlib import Path +import os +import logging + +import chiptest.linux +import chiptest.runner + +from .test_definition import TestTarget, TestDefinition, ApplicationPaths + + +def AllTests(root: str): + """Gets all the tests that can be found in the ROOT directory based on + yaml file names. + """ + for path in Path(os.path.join(root, 'src', 'app', 'tests', 'suites')).rglob("*.yaml"): + logging.debug('Found YAML: %s' % path) + + # grab the name without the extension + name = path.stem.lower() + + if 'simulated' in name: + continue + + if name.startswith('tv_'): + target = TestTarget.TV + name = 'tv-' + name[3:] + elif name.startswith('test_'): + target = TestTarget.ALL_CLUSTERS + name = 'app-' + name[5:] + else: + continue + + yield TestDefinition(yaml_file=path, run_name=path.stem, name=name, target=target) + + +__all__ = ['TestTarget', 'TestDefinition', 'AllTests', 'ApplicationPaths'] diff --git a/scripts/tests/chiptest/glob_matcher.py b/scripts/tests/chiptest/glob_matcher.py new file mode 120000 index 00000000000000..6935d35f9015ef --- /dev/null +++ b/scripts/tests/chiptest/glob_matcher.py @@ -0,0 +1 @@ +../../build/glob_matcher.py \ No newline at end of file diff --git a/scripts/tests/chiptest/linux.py b/scripts/tests/chiptest/linux.py new file mode 100644 index 00000000000000..79fdff88d10b80 --- /dev/null +++ b/scripts/tests/chiptest/linux.py @@ -0,0 +1,138 @@ +# +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Handles linux-specific functionality for running test cases +""" + +import logging +import os +import subprocess +import sys +import time + +from .test_definition import ApplicationPaths + +test_environ = os.environ.copy() + + +def EnsureNetworkNamespaceAvailability(): + if os.getuid() == 0: + logging.debug("Current user is root") + logging.warn("Running as root and this will change global namespaces.") + return + + os.execvpe("unshare", ["unshare", "--map-root-user", "-n", "-m", "python3", + sys.argv[0], '--internal-inside-unshare'] + sys.argv[1:], test_environ) + + +def EnsurePrivateState(): + logging.info("Ensuring /run is privately accessible") + + logging.debug("Making / private") + if os.system("mount --make-private /") != 0: + logging.error("Failed to make / private") + logging.error("Are you using --privileged if running in docker?") + sys.exit(1) + + logging.debug("Remounting /run") + if os.system("mount -t tmpfs tmpfs /run") != 0: + logging.error("Failed to mount /run as a temporary filesystem") + logging.error("Are you using --privileged if running in docker?") + sys.exit(1) + + +def CreateNamespacesForAppTest(): + """ + Creates appropriate namespaces for a tool and app binaries in a simulated + isolated network. + """ + COMMANDS = [ + # 2 virtual hosts: for app and for the tool + "ip netns add app", + "ip netns add tool", + + # create links for switch to net connections + "ip link add eth-app type veth peer name eth-app-switch", + "ip link add eth-tool type veth peer name eth-tool-switch", + + # link the connections together + "ip link set eth-app netns app", + "ip link set eth-tool netns tool", + + "ip link add name br1 type bridge", + "ip link set br1 up", + "ip link set eth-app-switch master br1", + "ip link set eth-tool-switch master br1", + + # mark connections up + "ip netns exec app ip addr add 10.10.10.1/24 dev eth-app", + "ip netns exec app ip link set dev eth-app up", + "ip netns exec app ip link set dev lo up", + "ip link set dev eth-app-switch up", + + "ip netns exec tool ip addr add 10.10.10.2/24 dev eth-tool", + "ip netns exec tool ip link set dev eth-tool up", + "ip netns exec tool ip link set dev lo up", + "ip link set dev eth-tool-switch up", + + # Force IPv6 to use ULAs that we control + "ip netns exec tool ip -6 addr flush eth-tool", + "ip netns exec app ip -6 addr flush eth-app", + "ip netns exec tool ip -6 a add fd00:0:1:1::2/64 dev eth-tool", + "ip netns exec app ip -6 a add fd00:0:1:1::3/64 dev eth-app", + ] + + for command in COMMANDS: + logging.debug("Executing '%s'" % command) + if os.system(command) != 0: + logging.error("Failed to execute '%s'" % command) + logging.error("Are you using --privileged if running in docker?") + sys.exit(1) + + # IPv6 does Duplicate Address Detection even though + # we know ULAs provided are isolated. Wait for 'tenative' address to be gone + + logging.info('Waiting for IPv6 DaD to complete (no tentative addresses)') + for i in range(100): # wait at most 10 seconds + output = subprocess.check_output(['ip', 'addr']) + if b'tentative' not in output: + logging.info('No more tentative addresses') + break + time.sleep(0.1) + else: + logging.warn("Some addresses look to still be tentative") + + +def PrepareNamespacesForTestExecution(in_unshare: bool): + if not in_unshare: + EnsureNetworkNamespaceAvailability() + elif in_unshare: + EnsurePrivateState() + + CreateNamespacesForAppTest() + + +def PathsWithNetworkNamespaces(paths: ApplicationPaths) -> ApplicationPaths: + """ + Returns a copy of paths with updated command arrays to invoke the + commands in an appropriate network namespace. + """ + return ApplicationPaths( + chip_tool='ip netns exec tool'.split() + paths.chip_tool, + all_clusters_app='ip netns exec app'.split() + paths.all_clusters_app, + tv_app='ip netns exec app'.split() + paths.tv_app, + ) diff --git a/scripts/tests/chiptest/runner.py b/scripts/tests/chiptest/runner.py new file mode 100644 index 00000000000000..4c6a37c00c2be9 --- /dev/null +++ b/scripts/tests/chiptest/runner.py @@ -0,0 +1,103 @@ +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import subprocess +import sys +import threading +import time +import pty + +from dataclasses import dataclass + + +class LogPipe(threading.Thread): + + def __init__(self, level, capture_delegate=None, name=None): + """Setup the object with a logger and a loglevel + + and start the thread + """ + threading.Thread.__init__(self) + + self.daemon = False + self.level = level + if sys.platform == 'darwin': + self.fd_read, self.fd_write = pty.openpty() + else: + self.fd_read, self.fd_write = os.pipe() + + self.pipeReader = os.fdopen(self.fd_read) + self.captured_logs = [] + self.capture_delegate = capture_delegate + self.name = name + + self.start() + + def CapturedLogContains(self, txt: str): + return any(txt in l for l in self.captured_logs) + + def fileno(self): + """Return the write file descriptor of the pipe""" + return self.fd_write + + def run(self): + """Run the thread, logging everything.""" + for line in iter(self.pipeReader.readline, ''): + logging.log(self.level, line.strip('\n')) + self.captured_logs.append(line) + if self.capture_delegate: + self.capture_delegate.Log(self.name, line) + + self.pipeReader.close() + + def close(self): + """Close the write end of the pipe.""" + os.close(self.fd_write) + + +class Runner: + def __init__(self, capture_delegate=None): + self.capture_delegate = capture_delegate + + def RunSubprocess(self, cmd, name, wait=True, dependencies=[]): + outpipe = LogPipe( + logging.DEBUG, capture_delegate=self.capture_delegate, name=name + ' OUT') + errpipe = LogPipe( + logging.INFO, capture_delegate=self.capture_delegate, name=name + ' ERR') + + if self.capture_delegate: + self.capture_delegate.Log(name, 'EXECUTING %r' % cmd) + + s = subprocess.Popen(cmd, stdout=outpipe, stderr=errpipe) + outpipe.close() + errpipe.close() + + if not wait: + return s, outpipe, errpipe + + while s.poll() is None: + # dependencies MUST NOT be done + for dependency in dependencies: + if dependency.poll() is not None: + s.kill() + raise Exception("Unexpected return %d for %r", + dependency.poll(), dependency) + + code = s.wait() + if code != 0: + raise Exception('Command %r failed: %d' % (cmd, code)) + else: + logging.debug('Command %r completed with error code 0', cmd) diff --git a/scripts/tests/chiptest/test_definition.py b/scripts/tests/chiptest/test_definition.py new file mode 100644 index 00000000000000..9d6e22ba6c3109 --- /dev/null +++ b/scripts/tests/chiptest/test_definition.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import time +from datetime import datetime +import typing +import threading + +from enum import Enum, auto +from dataclasses import dataclass + +TEST_NODE_ID = '0x12344321' + + +class TestTarget(Enum): + ALL_CLUSTERS = auto() + TV = auto() + + +@dataclass +class ApplicationPaths: + chip_tool: typing.List[str] + all_clusters_app: typing.List[str] + tv_app: typing.List[str] + + +@dataclass +class CaptureLine: + when: datetime + source: str + line: str + + +class ExecutionCapture: + """ + Keeps track of output lines in a process, to help debug failures. + """ + + def __init__(self): + self.lock = threading.Lock() + self.captures = [] + + def Log(self, source, line): + with self.lock: + self.captures.append(CaptureLine( + when=datetime.now(), + source=source, + line=line.strip('\n') + )) + + def LogContents(self): + logging.error('================ CAPTURED LOG START ==================') + with self.lock: + for entry in self.captures: + logging.error('%02d:%02d:%02d.%03d - %-10s: %s', + entry.when.hour, + entry.when.minute, + entry.when.second, + entry.when.microsecond/1000, + entry.source, + entry.line + ) + logging.error('================ CAPTURED LOG END ====================') + + +@dataclass +class TestDefinition: + yaml_file: str + name: str + run_name: str + target: TestTarget + + def Run(self, runner, paths: ApplicationPaths): + """Executes the given test case using the provided runner for execution.""" + app_process = None + runner.capture_delegate = ExecutionCapture() + + try: + if self.target == TestTarget.ALL_CLUSTERS: + app_cmd = paths.all_clusters_app + elif self.target == TestTarget.TV: + app_cmd = paths.tv_app + else: + raise Exception( + "Unknown test target - don't know which application to run") + + tool_cmd = paths.chip_tool + if os.path.exists('/tmp/chip_tool_config.ini'): + os.unlink('/tmp/chip_tool_config.ini') + + logging.debug('Executing application under test.') + app_process, outpipe, errpipe = runner.RunSubprocess( + app_cmd, name='APP ', wait=False) + + logging.debug('Waiting for server to listen.') + start_time = time.time() + server_is_listening = outpipe.CapturedLogContains( + "Server Listening") + while not server_is_listening: + if time.time() - start_time > 10: + raise Exception('Timeout for server listening') + time.sleep(0.1) + server_is_listening = outpipe.CapturedLogContains( + "Server Listening") + logging.debug('Server is listening. Can proceed.') + + runner.RunSubprocess(tool_cmd + ['pairing', 'qrcode', TEST_NODE_ID, 'MT:D8XA0CQM00KA0648G00'], + name='PAIR', dependencies=[app_process]) + + runner.RunSubprocess(tool_cmd + ['tests', self.run_name, TEST_NODE_ID], + name='TEST', dependencies=[app_process]) + except: + logging.error("!!!!!!!!!!!!!!!!!!!! ERROR !!!!!!!!!!!!!!!!!!!!!!") + runner.capture_delegate.LogContents() + raise + finally: + if app_process: + app_process.kill() diff --git a/scripts/tests/run_test_suite.py b/scripts/tests/run_test_suite.py new file mode 100755 index 00000000000000..6022d6d50e5dcd --- /dev/null +++ b/scripts/tests/run_test_suite.py @@ -0,0 +1,211 @@ +#!/usr/bin/env -S python3 -B + +# Copyright (c) 2021 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import coloredlogs +import click +import logging +import os +import shutil +import sys +import typing +import time + +from pathlib import Path +from dataclasses import dataclass + +sys.path.append(os.path.abspath(os.path.dirname(__file__))) + +import chiptest # noqa: E402 +from chiptest.glob_matcher import GlobMatcher # noqa: E402 + +DEFAULT_CHIP_ROOT = os.path.abspath( + os.path.join(os.path.dirname(__file__), '..', '..')) + + +def FindBinaryPath(name: str): + for path in Path(DEFAULT_CHIP_ROOT).rglob(name): + if not path.is_file(): + continue + if path.name != name: + continue + return str(path) + + return 'NOT_FOUND_IN_OUTPUT_' + name + + +# Supported log levels, mapping string values required for argument +# parsing into logging constants +__LOG_LEVELS__ = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warn': logging.WARN, + 'fatal': logging.FATAL, +} + + +@dataclass +class RunContext: + root: str + tests: typing.List[chiptest.TestDefinition] + in_unshare: bool + + +@click.group(chain=True) +@click.option( + '--log-level', + default='info', + type=click.Choice(__LOG_LEVELS__.keys(), case_sensitive=False), + help='Determines the verbosity of script output.') +@click.option( + '--target', + default=['all'], + multiple=True, + help='Test to run (use "all" to run all tests)' +) +@click.option( + '--target-glob', + default='', + help='What targets to accept (glob)' +) +@click.option( + '--target-skip-glob', + default='', + help='What targets to skip (glob)' +) +@click.option( + '--no-log-timestamps', + default=False, + is_flag=True, + help='Skip timestaps in log output') +@click.option( + '--root', + default=DEFAULT_CHIP_ROOT, + help='Default directory path for CHIP. Used to determine what tests exist') +@click.option( + '--internal-inside-unshare', + hidden=True, + is_flag=True, + default=False, + help='Internal flag for running inside a unshared environment' +) +@click.pass_context +def main(context, log_level, target, target_glob, target_skip_glob, no_log_timestamps, root, internal_inside_unshare): + # Ensures somewhat pretty logging of what is going on + log_fmt = '%(asctime)s.%(msecs)03d %(levelname)-7s %(message)s' + if no_log_timestamps: + log_fmt = '%(levelname)-7s %(message)s' + coloredlogs.install(level=__LOG_LEVELS__[log_level], fmt=log_fmt) + + # Figures out selected test that match the given name(s) + tests = [test for test in chiptest.AllTests(root)] + if 'all' not in target: + target = set([name.lower() for name in target]) + tests = [test for test in tests if test.name in target] + + if target_glob: + matcher = GlobMatcher(target_glob) + tests = [test for test in tests if matcher.matches(test.name)] + + if target_skip_glob: + matcher = GlobMatcher(target_skip_glob) + tests = [test for test in tests if not matcher.matches(test.name)] + + tests.sort(key=lambda x: x.name) + + context.obj = RunContext(root=root, tests=tests, + in_unshare=internal_inside_unshare) + + +@main.command( + 'list', help='List available test suites') +@click.pass_context +def cmd_generate(context): + for test in context.obj.tests: + print(test.name) + + +@main.command( + 'run', help='Execute the tests') +@click.option( + '--iterations', + default=1, + help='Number of iterations to run') +@click.option( + '--chip-tool', + default=FindBinaryPath('chip-tool'), + help='What chip tool app to use to run the test') +@click.option( + '--all-clusters-app', + default=FindBinaryPath('chip-all-clusters-app'), + help='what all clusters app to use') +@click.option( + '--tv-app', + default=FindBinaryPath('chip-tv-app'), + help='what tv app to use') +@click.pass_context +def cmd_run(context, iterations, chip_tool, all_clusters_app, tv_app): + runner = chiptest.runner.Runner() + + # Command execution requires an array + paths = chiptest.ApplicationPaths( + chip_tool=[chip_tool], + all_clusters_app=[all_clusters_app], + tv_app=[tv_app] + ) + + if sys.platform == 'linux': + chiptest.linux.PrepareNamespacesForTestExecution( + context.obj.in_unshare) + paths = chiptest.linux.PathsWithNetworkNamespaces(paths) + + # Testing prerequisites: tv app requires a config. Copy it just in case + shutil.copyfile( + os.path.join( + context.obj.root, 'examples/tv-app/linux/include/endpoint-configuration/chip_tv_config.ini'), + '/tmp/chip_tv_config.ini' + ) + + logging.info("Each test will be executed %d times" % iterations) + + for i in range(iterations): + logging.info("Starting iteration %d" % (i+1)) + for test in context.obj.tests: + test_start = time.time() + try: + test.Run(runner, paths) + test_end = time.time() + logging.info('%-20s - Completed in %0.2f seconds' % + (test.name, (test_end - test_start))) + except: + test_end = time.time() + logging.exception('%s - FAILED in %0.2f seconds' % + (test.name, (test_end - test_start))) + sys.exit(2) + + +# On linux, allow an execution shell to be prepared +if sys.platform == 'linux': + @main.command( + 'shell', help='Execute a bash shell in the environment (useful to test network namespaces)') + @click.pass_context + def cmd_run(context): + chiptest.linux.PrepareNamespacesForTestExecution( + context.obj.in_unshare) + os.execvpe("bash", ["bash"], os.environ.copy()) + + +if __name__ == '__main__': + main() diff --git a/scripts/tests/test_suites.sh b/scripts/tests/test_suites.sh deleted file mode 100755 index e482e9b76e4e55..00000000000000 --- a/scripts/tests/test_suites.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env bash - -# -# Copyright (c) 2020 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Fail if one of our sub-commands fails. -set -e - -# Fail if anything in a pipeline fails, not just the last command (which for -# us tends to be 'tee'). -set -o pipefail - -declare INPUT_ARGS=$* - -declare -i iterations=2 -declare -i delay=0 -declare -i node_id=0x12344321 -declare -i background_pid=0 -declare -i use_netns=0 -declare -i root_remount=0 -declare -i clean_netns=0 -declare test_case_wrapper=() - -usage() { - echo "test_suites.sh [-a APPLICATION] [-i ITERATIONS] [-h] [-s CASE_NAME] [-w COMMAND] [-d DELAY]" - echo " -a APPLICATION: runs chip-tool against 'chip--app' (default: all-clusters)" - echo " -d DELAY: milliseconds to wait before running an individual test step (default: $delay)" - echo " -h: this help message" - echo " -i ITERATIONS: number of iterations to run (default: $iterations)" - echo " -s CASE_NAME: runs single test case name (e.g. Test_TC_OO_2_2" - echo " for Test_TC_OO_2_2.yaml) (by default, all are run)" - echo " -w COMMAND: prefix all instantiations with a command (e.g. valgrind) (default: '')" - echo " -n Use linux netns to isolate app and tool executables" - echo " -c execute a netns cleanup and exit" - echo " -r Execute a remount (INTERNAL USE for netns)" - echo "" - exit 0 -} - -declare app_run_prefix="" -declare tool_run_prefix="" - -netns_setup() { - # 2 virtual hosts: for app and for the tool - ip netns add app - ip netns add tool - - # create links for switch to net connections - ip link add eth-app type veth peer name eth-app-switch - ip link add eth-tool type veth peer name eth-tool-switch - - # link the connections together - ip link set eth-app netns app - ip link set eth-tool netns tool - - ip link add name br1 type bridge - ip link set br1 up - ip link set eth-app-switch master br1 - ip link set eth-tool-switch master br1 - - # mark connections up - ip netns exec app ip addr add 10.10.10.1/24 dev eth-app - ip netns exec app ip link set dev eth-app up - ip netns exec app ip link set dev lo up - ip link set dev eth-app-switch up - - ip netns exec tool ip addr add 10.10.10.2/24 dev eth-tool - ip netns exec tool ip link set dev eth-tool up - ip netns exec tool ip link set dev lo up - ip link set dev eth-tool-switch up - - # Force IPv6 to use ULAs that we control - ip netns exec tool ip -6 addr flush eth-tool - ip netns exec app ip -6 addr flush eth-app - ip netns exec tool ip -6 a add fd00:0:1:1::2/64 dev eth-tool - ip netns exec app ip -6 a add fd00:0:1:1::3/64 dev eth-app - - # TODO(andy314): IPv6 does Duplicate Address Detection even though - # we know these addresses are isolated. For a while IPv6 addresses - # will be in 'transitional' state and cannot be used. - # - # This sleep waits for the addresses to become 'global'. Ideally - # we should loop/wait here instead. - sleep 2 -} - -netns_cleanup() { - ip netns del app || true - ip netns del tool || true - ip link del br1 || true - - # attempt to delete orphaned items just in case - ip link del eth-tool || true - ip link del eth-tool-switch || true - ip link del eth-app || true - ip link del eth-app-switch || true -} - -while getopts a:d:i:hs:w:ncr flag; do - case "$flag" in - a) application=$OPTARG ;; - d) delay=$OPTARG ;; - h) usage ;; - i) iterations=$OPTARG ;; - s) single_case=$OPTARG ;; - w) test_case_wrapper=("$OPTARG") ;; - n) use_netns=1 ;; - c) clean_netns=1 ;; - r) root_remount=1 ;; - esac -done - -if [[ $clean_netns != 0 ]]; then - echo "Cleaning network namespaces" - netns_cleanup - exit 0 -fi - -if [[ $root_remount != 0 ]]; then - echo 'Creating a separate mount' - mount --make-private / - mount -t tmpfs tmpfs /run -fi - -if [[ $use_netns != 0 ]]; then - echo "Using network namespaces" - - if [[ `id -u` -ne 0 ]]; then - echo 'Executing in a new namespace: ' $0 -r $INPUT_ARGS - unshare --map-root-user -n -m $0 -r $INPUT_ARGS - exit 0 - else - if [[ $root_remount -eq 0 ]]; then - # Running as root may be fine in docker/vm however this is not advised - # on workstations as changes are global and harder to undo - echo 'Running as root: this changes global network namespaces, not ideal' - fi - fi - - netns_setup - - app_run_prefix="ip netns exec app" - tool_run_prefix="ip netns exec tool" - - trap netns_cleanup EXIT -fi - -if [[ $application == "tv" ]]; then - declare test_filenames="${single_case-TV_*}.yaml" - cp examples/tv-app/linux/include/endpoint-configuration/chip_tv_config.ini /tmp/chip_tv_config.ini -# in case there's no application argument -# always default to all-cluters app -else - application="all-clusters" - declare test_filenames="${single_case-Test*}.yaml" -fi -declare -a test_array="($(find src/app/tests/suites -type f -name "$test_filenames" -not -name "*Simulated*" -exec basename {} .yaml \;))" - -if [[ $iterations == 0 ]]; then - echo "Invalid iteration count: '$1'" - exit 1 -fi - -echo "Running tests for application: $application, with iterations set to: $iterations and delay set to $delay" - -cleanup() { - if [[ $background_pid != 0 ]]; then - # In case we died on a failure before we cleaned up our background task. - kill -9 "$background_pid" || true - fi - - if [[ $use_netns != 0 ]]; then - netns_cleanup - fi -} -trap cleanup EXIT - -echo "Found tests:" -for i in "${test_array[@]}"; do - echo " $i" -done -echo "" -echo "" - -ulimit -c unlimited || true - -rm -rf /tmp/test_suites_app_logs -mkdir -p /tmp/test_suites_app_logs - -declare -a iter_array="($(seq "$iterations"))" -for j in "${iter_array[@]}"; do - echo " ===== Iteration $j starting" - for i in "${test_array[@]}"; do - echo " ===== Running test: $i" - echo " * Starting cluster server" - rm -rf /tmp/chip_tool_config.ini - # This part is a little complicated. We want to - # 1) Start chip-app in the background - # 2) Pipe its output through tee so we can wait until it's ready for a - # PASE handshake. - # 3) Save its pid off so we can kill it. - # - # The subshell with echoing of $! to a file descriptor and - # then reading things out of there accomplishes item 3; - # otherwise $! would be the last-started command which would - # be the tee. This part comes from https://stackoverflow.com/a/3786955 - # and better ideas are welcome. - # - # The stdbuf -o0 is to make sure our output is flushed through - # tee expeditiously; otherwise it will buffer things up and we - # will never see the string we want. - - application_log_file=/tmp/test_suites_app_logs/"$application-$i-$j"-log - pairing_log_file=/tmp/test_suites_app_logs/pairing-"$application-$i-$j"-log - chip_tool_log_file=/tmp/test_suites_app_logs/chip-tool-"$application-$i-$j"-log - touch "$application_log_file" - touch "$pairing_log_file" - touch "$chip_tool_log_file" - rm -rf /tmp/pid - ( - ${app_run_prefix} stdbuf -o0 "${test_case_wrapper[@]}" out/debug/standalone/chip-"$application"-app & - echo $! >&3 - ) 3>/tmp/pid | tee "$application_log_file" & - while ! grep -q "Server Listening" "$application_log_file"; do - : - done - # Now read $background_pid from /tmp/pid; presumably it's - # landed there by now. If we try to read it immediately after - # kicking off the subshell, sometimes we try to do it before - # the data is there yet. - background_pid="$(/dev/null; then - echo " * [CI DEBUG] Looking for commissionable Nodes" - # Ignore the error that timeout generates - cat <(timeout 1 dns-sd -B _matterc._udp) - fi - echo " * Pairing to device" - ${tool_run_prefix} "${test_case_wrapper[@]}" out/debug/standalone/chip-tool pairing qrcode "$node_id" MT:D8XA0CQM00KA0648G00 | tee "$pairing_log_file" - echo " * Starting test run: $i" - ${tool_run_prefix} "${test_case_wrapper[@]}" out/debug/standalone/chip-tool tests "$i" "$node_id" --delayInMs "$delay" | tee "$chip_tool_log_file" - # Prevent cleanup trying to kill a process we already killed. - temp_background_pid=$background_pid - background_pid=0 - kill -9 "$temp_background_pid" - echo " ===== Test complete: $i" - done - echo " ===== Iteration $j completed" -done