-
Notifications
You must be signed in to change notification settings - Fork 2.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support running test suites via a python script (#11996)
* Start adding a file that is able to name possible tests * Move test logic into separate python module, for easier code logic split * Add support for linux netns in tests including a validation shell. tested that this works in host. Updated vscode container to also allow running these * Add milliseconds for logging, add sleep for tentative ipv6 addresses * Restyle fixes * Prepare for test execution * more work to make runners functional * This can now run unit tests (at least in linux) * Restyle fixes * Add TV app as a supported application on host builds * Finished reporting on error only * Restyle and run new tests on linux using the new method * Fix the test yaml file * Remove run tv tests step - now all tests are run in one * Remove todo - it is done * Fix name of the subcommand * Run tests on darwin as well using the updated python scripts * Add glob support for running tests (hackish implmementation though through symlinks) and avoid tv tests for darwin * Add missing link file * Fix target skip glob argument for darwin * Minor updates from darwin testing: start thread after full constructor run, cleanup code a bit * Add some locking th definition logging captures * Use PTY since that removes buffering on darwin * Only use pty on darwin for IPC * Restyle fixes * Update scripts/tests/chiptest/linux.py Co-authored-by: Victor Morales <[email protected]> * Update scripts/tests/chiptest/runner.py Co-authored-by: Victor Morales <[email protected]> * Restyle fixes * Run 2 iterations of test suites to preserve previous behaviour, remove test_suites.sh * Add extra validation: if chip-all-clusters-app fails, make the test runner fail the test Co-authored-by: Victor Morales <[email protected]>
- Loading branch information
1 parent
23fad89
commit 13de723
Showing
13 changed files
with
678 additions
and
280 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| # | ||
| # Copyright (c) 2021 Project CHIP Authors | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| from pathlib import Path | ||
| import os | ||
| import logging | ||
|
|
||
| import chiptest.linux | ||
| import chiptest.runner | ||
|
|
||
| from .test_definition import TestTarget, TestDefinition, ApplicationPaths | ||
|
|
||
|
|
||
| def AllTests(root: str): | ||
| """Gets all the tests that can be found in the ROOT directory based on | ||
| yaml file names. | ||
| """ | ||
| for path in Path(os.path.join(root, 'src', 'app', 'tests', 'suites')).rglob("*.yaml"): | ||
| logging.debug('Found YAML: %s' % path) | ||
|
|
||
| # grab the name without the extension | ||
| name = path.stem.lower() | ||
|
|
||
| if 'simulated' in name: | ||
| continue | ||
|
|
||
| if name.startswith('tv_'): | ||
| target = TestTarget.TV | ||
| name = 'tv-' + name[3:] | ||
| elif name.startswith('test_'): | ||
| target = TestTarget.ALL_CLUSTERS | ||
| name = 'app-' + name[5:] | ||
| else: | ||
| continue | ||
|
|
||
| yield TestDefinition(yaml_file=path, run_name=path.stem, name=name, target=target) | ||
|
|
||
|
|
||
| __all__ = ['TestTarget', 'TestDefinition', 'AllTests', 'ApplicationPaths'] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| ../../build/glob_matcher.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| # | ||
| # Copyright (c) 2021 Project CHIP Authors | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| """ | ||
| Handles linux-specific functionality for running test cases | ||
| """ | ||
|
|
||
| import logging | ||
| import os | ||
| import subprocess | ||
| import sys | ||
| import time | ||
|
|
||
| from .test_definition import ApplicationPaths | ||
|
|
||
| test_environ = os.environ.copy() | ||
|
|
||
|
|
||
| def EnsureNetworkNamespaceAvailability(): | ||
| if os.getuid() == 0: | ||
| logging.debug("Current user is root") | ||
| logging.warn("Running as root and this will change global namespaces.") | ||
| return | ||
|
|
||
| os.execvpe("unshare", ["unshare", "--map-root-user", "-n", "-m", "python3", | ||
| sys.argv[0], '--internal-inside-unshare'] + sys.argv[1:], test_environ) | ||
|
|
||
|
|
||
| def EnsurePrivateState(): | ||
| logging.info("Ensuring /run is privately accessible") | ||
|
|
||
| logging.debug("Making / private") | ||
| if os.system("mount --make-private /") != 0: | ||
| logging.error("Failed to make / private") | ||
| logging.error("Are you using --privileged if running in docker?") | ||
| sys.exit(1) | ||
|
|
||
| logging.debug("Remounting /run") | ||
| if os.system("mount -t tmpfs tmpfs /run") != 0: | ||
| logging.error("Failed to mount /run as a temporary filesystem") | ||
| logging.error("Are you using --privileged if running in docker?") | ||
| sys.exit(1) | ||
|
|
||
|
|
||
| def CreateNamespacesForAppTest(): | ||
| """ | ||
| Creates appropriate namespaces for a tool and app binaries in a simulated | ||
| isolated network. | ||
| """ | ||
| COMMANDS = [ | ||
| # 2 virtual hosts: for app and for the tool | ||
| "ip netns add app", | ||
| "ip netns add tool", | ||
|
|
||
| # create links for switch to net connections | ||
| "ip link add eth-app type veth peer name eth-app-switch", | ||
| "ip link add eth-tool type veth peer name eth-tool-switch", | ||
|
|
||
| # link the connections together | ||
| "ip link set eth-app netns app", | ||
| "ip link set eth-tool netns tool", | ||
|
|
||
| "ip link add name br1 type bridge", | ||
| "ip link set br1 up", | ||
| "ip link set eth-app-switch master br1", | ||
| "ip link set eth-tool-switch master br1", | ||
|
|
||
| # mark connections up | ||
| "ip netns exec app ip addr add 10.10.10.1/24 dev eth-app", | ||
| "ip netns exec app ip link set dev eth-app up", | ||
| "ip netns exec app ip link set dev lo up", | ||
| "ip link set dev eth-app-switch up", | ||
|
|
||
| "ip netns exec tool ip addr add 10.10.10.2/24 dev eth-tool", | ||
| "ip netns exec tool ip link set dev eth-tool up", | ||
| "ip netns exec tool ip link set dev lo up", | ||
| "ip link set dev eth-tool-switch up", | ||
|
|
||
| # Force IPv6 to use ULAs that we control | ||
| "ip netns exec tool ip -6 addr flush eth-tool", | ||
| "ip netns exec app ip -6 addr flush eth-app", | ||
| "ip netns exec tool ip -6 a add fd00:0:1:1::2/64 dev eth-tool", | ||
| "ip netns exec app ip -6 a add fd00:0:1:1::3/64 dev eth-app", | ||
| ] | ||
|
|
||
| for command in COMMANDS: | ||
| logging.debug("Executing '%s'" % command) | ||
| if os.system(command) != 0: | ||
| logging.error("Failed to execute '%s'" % command) | ||
| logging.error("Are you using --privileged if running in docker?") | ||
| sys.exit(1) | ||
|
|
||
| # IPv6 does Duplicate Address Detection even though | ||
| # we know ULAs provided are isolated. Wait for 'tenative' address to be gone | ||
|
|
||
| logging.info('Waiting for IPv6 DaD to complete (no tentative addresses)') | ||
| for i in range(100): # wait at most 10 seconds | ||
| output = subprocess.check_output(['ip', 'addr']) | ||
| if b'tentative' not in output: | ||
| logging.info('No more tentative addresses') | ||
| break | ||
| time.sleep(0.1) | ||
| else: | ||
| logging.warn("Some addresses look to still be tentative") | ||
|
|
||
|
|
||
| def PrepareNamespacesForTestExecution(in_unshare: bool): | ||
| if not in_unshare: | ||
| EnsureNetworkNamespaceAvailability() | ||
| elif in_unshare: | ||
| EnsurePrivateState() | ||
|
|
||
| CreateNamespacesForAppTest() | ||
|
|
||
|
|
||
| def PathsWithNetworkNamespaces(paths: ApplicationPaths) -> ApplicationPaths: | ||
| """ | ||
| Returns a copy of paths with updated command arrays to invoke the | ||
| commands in an appropriate network namespace. | ||
| """ | ||
| return ApplicationPaths( | ||
| chip_tool='ip netns exec tool'.split() + paths.chip_tool, | ||
| all_clusters_app='ip netns exec app'.split() + paths.all_clusters_app, | ||
| tv_app='ip netns exec app'.split() + paths.tv_app, | ||
| ) |
Oops, something went wrong.