From 035b067e285c106ad8b6796809df4a30fa30195c Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Wed, 23 Mar 2016 11:11:43 -0700 Subject: [PATCH 01/55] Fix: Dev: changing network mode to bridge --- lib/topology_docker_openswitch/openswitch.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 02c8ba8..ddfa5e5 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -77,6 +77,16 @@ def create_interfaces(): in_swns = check_output(shsplit( 'ip netns exec swns ls /sys/class/net/' )).split() + logging.info( + ' - Not in swns {not_in_swns} '.format( + **locals() + ) + ) + logging.info( + ' - In swns {in_swns} '.format( + **locals() + ) + ) create_cmd_tpl = 'ip tuntap add dev {hwport} mode tap' netns_cmd_tpl = 'ip link set {hwport} netns swns' @@ -87,7 +97,12 @@ def create_interfaces(): # Map the port with the labels for portlbl in not_in_swns: - if portlbl in ['lo', 'oobm', 'bonding_masters']: + logging.info( + ' - Port {portlbl} found'.format( + **locals() + ) + ) + if portlbl in ['lo', 'oobm', 'eth0', 'bonding_masters']: continue hwport = hwports.pop(0) mapping_ports[portlbl] = hwport @@ -224,7 +239,7 @@ def __init__( super(OpenSwitchNode, self).__init__( identifier, image=image, command='/sbin/init', binds=';'.join(container_binds), hostname='switch', - **kwargs + network_mode='bridge', **kwargs ) # Save location of the shared dir in host From e9ea1ed5c5a15022d6fbe7726cf7870b7477b9dd Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Tue, 21 Jun 2016 12:14:16 -0700 Subject: [PATCH 02/55] fix: dev: Adding attribute for compatibility. --- lib/topology_docker_openswitch/openswitch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index fc41757..0fef7e1 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -263,6 +263,9 @@ def __init__( network_mode='bridge', **kwargs ) + # FIXME: Remove this attribute to merge with version > 1.6.0 + self.shared_dir_mount = '/tmp' + # Add vtysh (default) shell # FIXME: Create a subclass to handle better the particularities of # vtysh, like prompt setup etc. From c7a2f3811bfc15f5e3a4b70a66d7b10c302761dc Mon Sep 17 00:00:00 2001 From: Sergio Barahona Date: Wed, 22 Jun 2016 16:46:07 -0700 Subject: [PATCH 03/55] fix: dev: Change dict validation method to avoid keys errors --- lib/topology_docker_openswitch/plugin/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/plugin/plugin.py b/lib/topology_docker_openswitch/plugin/plugin.py index 3637709..c0ed317 100644 --- a/lib/topology_docker_openswitch/plugin/plugin.py +++ b/lib/topology_docker_openswitch/plugin/plugin.py @@ -34,7 +34,7 @@ def pytest_runtest_teardown(item): logs_path = '/var/log/messages' for node in topology.nodes: node_obj = topology.get(node) - if node_obj.metadata['type'] == 'openswitch': + if node_obj.metadata.get('type', 'none') == 'openswitch': shared_dir = node_obj.shared_dir try: node_obj.send_command( From 0d790ca38e108d868311946cc8ebe4e0cb51ff29 Mon Sep 17 00:00:00 2001 From: Vasanth Viswanathan Date: Tue, 5 Jul 2016 10:59:11 -0700 Subject: [PATCH 04/55] chg: dev: Add check for ops-switchd to be active * Added a check to ensure ops-switchd has become active. This will ensure that the OpenSwitch docker image is in a usable state. Signed-off-by: Vasanth Viswanathan --- lib/topology_docker_openswitch/openswitch.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 0fef7e1..753a850 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -38,7 +38,7 @@ from os.path import exists, split from json import dumps, loads from shlex import split as shsplit -from subprocess import check_call, check_output +from subprocess import check_call, check_output, call from socket import AF_UNIX, SOCK_STREAM, socket, gethostname import yaml @@ -152,6 +152,10 @@ def cur_cfg_is_set(): except IndexError: return 0 +def ops_switchd_is_active(): + is_active = call(["systemctl", "is-active", "switchd.service"]) + return is_active == 0 + def main(): if '-d' in argv: @@ -196,6 +200,16 @@ def main(): else: raise Exception('Timed out while waiting for switchd pid.') + logging.info('Waiting for ops-switchd to become active...') + for i in range(0, config_timeout): + if not ops_switchd_is_active(): + sleep(0.1) + else: + break + else: + raise Exception('Timed out while waiting for ops-switchd ' + 'to become active.') + logging.info('Wait for final hostname...') for i in range(0, config_timeout): if gethostname() != 'switch': From 61ba028bdc3777cb3b84cf28a0d121544568b36b Mon Sep 17 00:00:00 2001 From: Vasanth Viswanathan Date: Tue, 19 Jul 2016 12:11:07 -0700 Subject: [PATCH 05/55] chg: dev: Increase ops-switchd active timeout * In legacy frameworks, there is a wait of 60 seconds for ops-switchd to become active. Making this change to stay in sync and ensure we dont timeout sooner. Signed-off-by: Vasanth Viswanathan --- lib/topology_docker_openswitch/openswitch.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 753a850..2717140 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -44,6 +44,7 @@ import yaml config_timeout = 300 +ops_switchd_active_timeout = 60 swns_netns = '/var/run/netns/swns' hwdesc_dir = '/etc/openswitch/hwdesc' db_sock = '/var/run/openvswitch/db.sock' @@ -201,9 +202,9 @@ def main(): raise Exception('Timed out while waiting for switchd pid.') logging.info('Waiting for ops-switchd to become active...') - for i in range(0, config_timeout): + for i in range(0, ops_switchd_active_timeout): if not ops_switchd_is_active(): - sleep(0.1) + sleep(1) else: break else: From e91ec721e7f2bab8adf6b645090f8c88a28d73fd Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 22 Jul 2016 12:23:58 -0600 Subject: [PATCH 06/55] chg: dev: Improving logging. --- lib/topology_docker_openswitch/openswitch.py | 165 ++++++++++++++---- .../plugin/plugin.py | 124 ++++++++----- 2 files changed, 207 insertions(+), 82 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 2717140..ce5063a 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -25,11 +25,19 @@ from __future__ import print_function, division from json import loads -from subprocess import check_call +from subprocess import check_output, CalledProcessError +from platform import system, linux_distribution +from logging import StreamHandler, getLogger, INFO, Formatter +from sys import stdout from topology_docker.node import DockerNode from topology_docker.shell import DockerShell, DockerBashShell +# When a failure happens during boot time, logs and other information is +# collected to help with the debugging. The path of this collection is to be +# stored here at module level to be able to import it in the pytest teardown +# hook later. Non-failing containers will append their log paths here also. +LOG_PATHS = [] SETUP_SCRIPT = """\ import logging @@ -233,20 +241,49 @@ def main(): main() """ +LOG = getLogger(__name__) +LOG_HDLR = StreamHandler(stream=stdout) +LOG_HDLR.setFormatter(Formatter('%(asctime)s %(message)s')) +LOG_HDLR.setLevel(INFO) +LOG.addHandler(LOG_HDLR) +LOG.setLevel(INFO) + + +def log_commands( + commands, location, function, escape=True, + prefix=None, suffix=None, **kwargs +): + if prefix is None: + prefix = '' + if suffix is None: + suffix = '' + + for command in commands: + log_path = ' >> {} 2>&1'.format(location) + args = [ + r'{prefix}echo \"Output of:' + r' {command}{log_path}\"{log_path}{suffix}'.format( + prefix=prefix, command=command, + log_path=log_path, suffix=suffix + ), + r'{}{}{}{}'.format( + prefix, command, log_path, suffix + ), + r'{}echo \"\"{}{}'.format(prefix, log_path, suffix) + ] -PROCESS_LOG = """ -#!/bin/bash -ovs-vsctl list Daemon >> /tmp/logs -echo "Coredump -->" >> /tmp/logs -coredumpctl gdb >> /tmp/logs -echo "All the running processes:" >> /tmp/logs -ps -aef >> /tmp/logs - -systemctl status >> /tmp/systemctl -systemctl --state=failed --all >> /tmp/systemctl + for arg in args: + try: + if not escape: + arg = arg.replace('\\', '') + function(arg, **kwargs) -ovsdb-client dump >> /tmp/ovsdb_dump -""" + except CalledProcessError as error: + LOG.warning( + '{} failed with error {}.'.format( + command, error.returncode + ) + ) class OpenSwitchNode(DockerNode): @@ -324,43 +361,101 @@ def _setup_system(self): #. Create remaining interfaces. """ - # Write the log gathering script - process_log = '{}/process_log.sh'.format(self.shared_dir) - with open(process_log, "w") as fd: - fd.write(PROCESS_LOG) - check_call('chmod 755 {}/process_log.sh'.format(self.shared_dir), - shell=True) - # Write and execute setup script setup_script = '{}/openswitch_setup.py'.format(self.shared_dir) with open(setup_script, 'w') as fd: fd.write(SETUP_SCRIPT) try: - self._docker_exec('python {}/openswitch_setup.py -d' - .format(self.shared_dir_mount)) + self._docker_exec( + 'python {}/openswitch_setup.py -d'.format( + self.shared_dir_mount + ) + ) except Exception as e: - check_call('touch {}/logs'.format(self.shared_dir), shell=True) - check_call('chmod 766 {}/logs'.format(self.shared_dir), - shell=True) - self._docker_exec('/bin/bash {}/process_log.sh' - .format(self.shared_dir_mount)) - check_call( - 'tail -n 2000 /var/log/syslog > {}/syslog'.format( - self.shared_dir - ), shell=True) - check_call( - 'docker ps -a >> {}/logs'.format(self.shared_dir), + global FAIL_LOG_PATH + lines_to_dump = 100 + + platforms_log_location = { + 'Ubuntu': 'cat /var/log/upstart/docker.log', + 'CentOS Linux': 'grep docker /var/log/daemon.log', + # FIXME: find the right values for the next dictionary keys: + # 'boot2docker': 'cat /var/log/docker.log', + # 'debian': 'cat /var/log/daemon.log', + # 'fedora': 'journalctl -u docker.service', + # 'red hat': 'grep docker /var/log/messages', + # 'opensuse': 'journalctl -u docker.service' + } + + # Here, we find the command to dump the last "lines_to_dump" lines + # of the docker log file in the logs. The location of the docker + # log file depends on the Linux distribution. These locations are + # defined the in "platforms_log_location" dictionary. + + operating_system = system() + + if operating_system != 'Linux': + LOG.warning( + 'Operating system is not Linux but {}.'.format( + operating_system + ) + ) + return + + linux_distro = linux_distribution()[0] + + if linux_distro not in platforms_log_location.keys(): + LOG.warning( + 'Unknown Linux distribution {}.'.format( + linux_distro + ) + ) + + docker_log_command = '{} | tail -n {}'.format( + platforms_log_location[linux_distro], lines_to_dump + ) + + container_commands = [ + 'ovs-vsctl list Daemon', + 'coredumpctl gdb', + 'ps -aef', + 'systemctl status', + 'systemctl --state=failed --all', + 'ovsdb-client dump', + 'systemctl status switchd -n 10000 -l', + 'cat /var/log/messages' + ] + + execution_machine_commands = [ + 'tail -n 2000 /var/log/syslog', + 'docker ps -a', + docker_log_command + ] + + log_commands( + container_commands, + '{}/container_logs'.format(self.shared_dir_mount), + self._docker_exec, + prefix=r'sh -c "', + suffix=r'"' + ) + log_commands( + execution_machine_commands, + '{}/execution_machine_logs'.format(self.shared_dir), + check_output, + escape=False, shell=True ) - check_call('cat {}/logs'.format(self.shared_dir), shell=True) - raise e + LOG_PATHS.append(self.shared_dir) + raise e # Read back port mapping port_mapping = '{}/port_mapping.json'.format(self.shared_dir) with open(port_mapping, 'r') as fd: mappings = loads(fd.read()) + LOG_PATHS.append(self.shared_dir) + if hasattr(self, 'ports'): self.ports.update(mappings) return diff --git a/lib/topology_docker_openswitch/plugin/plugin.py b/lib/topology_docker_openswitch/plugin/plugin.py index c0ed317..6508afb 100644 --- a/lib/topology_docker_openswitch/plugin/plugin.py +++ b/lib/topology_docker_openswitch/plugin/plugin.py @@ -15,58 +15,88 @@ # specific language governing permissions and limitations # under the License. -from os.path import exists, basename, splitext -from os import makedirs -from shutil import copytree, Error +from os.path import exists, basename, splitext, join +from shutil import copytree, Error, rmtree from logging import warning +from datetime import datetime + +from topology_docker_openswitch.openswitch import log_commands def pytest_runtest_teardown(item): """ - pytest hook to get the name of the test executed, it creates a folder with - the name, then copies the folders defined in the shared_dir_mount attribute - of each openswitch container, additionally the /var/log/messages of the - container is copied to the same folder. + Pytest hook to get node information after the test executed. + + This creates a folder with the name of the test case, copies the folders + defined in the shared_dir_mount attribute of each openswitch container + and the /var/log/messages file inside. + + FIXME: document the item argument """ - if 'topology' in item.funcargs: - topology = item.funcargs['topology'] - if topology.engine == 'docker': - logs_path = '/var/log/messages' - for node in topology.nodes: - node_obj = topology.get(node) - if node_obj.metadata.get('type', 'none') == 'openswitch': - shared_dir = node_obj.shared_dir - try: - node_obj.send_command( - 'cat {} > {}/var_messages.log'.format( - logs_path, node_obj.shared_dir_mount - ), - shell='bash', - silent=True - ) - except Error: - warning( - 'Unable to get {} from container'.format(logs_path) + test_suite = splitext(basename(item.parent.name))[0] + path_name = '/tmp/topology/docker/{}_{}_{}'.format( + test_suite, item.name, datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + ) + + # Being extra-prudent here + if exists(path_name): + rmtree(path_name) + + if 'topology' not in item.funcargs: + from topology_docker_openswitch.openswitch import LOG_PATHS + + for log_path in LOG_PATHS: + try: + copytree(log_path, join(path_name, basename(log_path))) + rmtree(path_name) + except Error as err: + errors = err.args[0] + for error in errors: + src, dest, msg = error + warning( + 'Unable to copy file {}, Error {}'.format( + src, msg ) - test_suite = splitext(basename(item.parent.name))[0] - path_name = '/tmp/{}_{}_{}'.format( - test_suite, item.name, str(id(item)) ) - if not exists(path_name): - makedirs(path_name) - try: - copytree( - shared_dir, '{}/{}'.format( - path_name, - basename(shared_dir) - ) - ) - except Error as err: - errors = err.args[0] - for error in errors: - src, dest, msg = error - warning( - 'Unable to copy file {}, Error {}'.format( - src, msg - ) - ) + return + + topology = item.funcargs['topology'] + + if topology.engine != 'docker': + return + + logs_path = '/var/log/messages' + + for node in topology.nodes: + node_obj = topology.get(node) + + if node_obj.metadata.get('type', None) != 'openswitch': + return + + shared_dir = node_obj.shared_dir + + try: + commands = ['cat {}'.format(logs_path)] + log_commands( + commands, join(node_obj.shared_dir_mount, 'container_logs'), + node_obj._docker_exec, prefix=r'sh -c "', suffix=r'"' + ) + except: + warning( + 'Unable to get {} from node {}.'.format( + logs_path, node_obj.identifier + ) + ) + + try: + copytree(shared_dir, join(path_name, basename(shared_dir))) + rmtree(shared_dir) + except Error as err: + errors = err.args[0] + for error in errors: + src, dest, msg = error + warning( + 'Unable to copy file {}, Error {}'.format( + src, msg + ) + ) From 1610e066cafdca5e91b6c7d94dfe112c2b4ec00f Mon Sep 17 00:00:00 2001 From: Vasanth Viswanathan Date: Tue, 26 Jul 2016 12:32:01 -0700 Subject: [PATCH 07/55] chg: dev: Modifying boot time checks * Modified existing check for cur_hw from cur_cfg_is_set to cur_hw_is_set * Added a new check for cur_cfg column of System table * Moved the check for cur_hw and cur_cfg ahead of ops-switchd is active check. This change is because during OPS boot, all daemons including ops-switchd will only daemonize and become active after cur_cfg is set to 1. So moved the checks to maintain logical flow of events during system boot up. Signed-off-by: Vasanth Viswanathan --- lib/topology_docker_openswitch/openswitch.py | 57 ++++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index ce5063a..02bf96c 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -57,7 +57,7 @@ hwdesc_dir = '/etc/openswitch/hwdesc' db_sock = '/var/run/openvswitch/db.sock' switchd_pid = '/var/run/openvswitch/ops-switchd.pid' -query = { +query_cur_hw = { 'method': 'transact', 'params': [ 'OpenSwitch', @@ -70,6 +70,19 @@ ], 'id': id(db_sock) } +query_cur_cfg = { + 'method': 'transact', + 'params': [ + 'OpenSwitch', + { + 'op': 'select', + 'table': 'System', + 'where': [], + 'columns': ['cur_cfg'] + } + ], + 'id': id(db_sock) +} sock = None @@ -149,18 +162,30 @@ def create_interfaces(): check_call(shsplit('touch /tmp/ops-virt-ports-ready')) logging.info(' - Ports readiness notified to the image') -def cur_cfg_is_set(): +def cur_hw_is_set(): global sock if sock is None: sock = socket(AF_UNIX, SOCK_STREAM) sock.connect(db_sock) - sock.send(dumps(query)) + sock.send(dumps(query_cur_hw)) response = loads(sock.recv(4096)) try: return response['result'][0]['rows'][0]['cur_hw'] == 1 except IndexError: return 0 +def cur_cfg_is_set(): + global sock + if sock is None: + sock = socket(AF_UNIX, SOCK_STREAM) + sock.connect(db_sock) + sock.send(dumps(query_cur_cfg)) + response = loads(sock.recv(4096)) + try: + return response['result'][0]['rows'][0]['cur_cfg'] == 1 + except IndexError: + return 0 + def ops_switchd_is_active(): is_active = call(["systemctl", "is-active", "switchd.service"]) return is_active == 0 @@ -200,6 +225,24 @@ def main(): else: raise Exception('Timed out while waiting for DB socket.') + logging.info('Waiting for cur_hw...') + for i in range(0, config_timeout): + if not cur_hw_is_set(): + sleep(0.1) + else: + break + else: + raise Exception('Timed out while waiting for cur_hw.') + + logging.info('Waiting for cur_cfg...') + for i in range(0, config_timeout): + if not cur_cfg_is_set(): + sleep(0.1) + else: + break + else: + raise Exception('Timed out while waiting for cur_cfg.') + logging.info('Waiting for switchd pid...') for i in range(0, config_timeout): if not exists(switchd_pid): @@ -228,14 +271,6 @@ def main(): else: raise Exception('Timed out while waiting for final hostname.') - logging.info('Waiting for cur_cfg...') - for i in range(0, config_timeout): - if not cur_cfg_is_set(): - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for cur_cfg.') if __name__ == '__main__': main() From dcd20c71fbaae5ae272530397bb3137762ab53ba Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 29 Jul 2016 16:09:06 -0600 Subject: [PATCH 08/55] fix: dev: Adding hanlder for existing files. --- lib/topology_docker_openswitch/plugin/plugin.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/plugin/plugin.py b/lib/topology_docker_openswitch/plugin/plugin.py index 6508afb..66f50ec 100644 --- a/lib/topology_docker_openswitch/plugin/plugin.py +++ b/lib/topology_docker_openswitch/plugin/plugin.py @@ -47,7 +47,12 @@ def pytest_runtest_teardown(item): for log_path in LOG_PATHS: try: - copytree(log_path, join(path_name, basename(log_path))) + destination = join(path_name, basename(log_path)) + try: + rmtree(destination) + except: + pass + copytree(log_path, destination) rmtree(path_name) except Error as err: errors = err.args[0] From ff8f4657989bde1233aeda0c627b8ce02806fd63 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 9 Aug 2016 12:34:53 -0600 Subject: [PATCH 09/55] new: dev: Adding starter for restd. --- lib/topology_docker_openswitch/openswitch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 02bf96c..0583d19 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -225,6 +225,11 @@ def main(): else: raise Exception('Timed out while waiting for DB socket.') + try: + check_call('systemctl start restd') + except: + raise Exception('Failed to start restd.') + logging.info('Waiting for cur_hw...') for i in range(0, config_timeout): if not cur_hw_is_set(): @@ -271,7 +276,6 @@ def main(): else: raise Exception('Timed out while waiting for final hostname.') - if __name__ == '__main__': main() """ From 8189f9799c62a915d63b4c36e900a66a704d5a65 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Wed, 10 Aug 2016 11:52:22 -0600 Subject: [PATCH 10/55] fix: dev: Improving error message for restd start. --- lib/topology_docker_openswitch/openswitch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 0583d19..07548fc 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -46,7 +46,7 @@ from os.path import exists, split from json import dumps, loads from shlex import split as shsplit -from subprocess import check_call, check_output, call +from subprocess import check_call, check_output, call, CalledProcessError from socket import AF_UNIX, SOCK_STREAM, socket, gethostname import yaml @@ -226,9 +226,9 @@ def main(): raise Exception('Timed out while waiting for DB socket.') try: - check_call('systemctl start restd') - except: - raise Exception('Failed to start restd.') + check_output('systemctl start restd') + except CalledProcessError as error: + raise Exception('Failed to start restd: {}'.format(error.output)) logging.info('Waiting for cur_hw...') for i in range(0, config_timeout): From de497fb4f7d545c3114ae87e310e7fe00dcda711 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Wed, 10 Aug 2016 14:48:49 -0600 Subject: [PATCH 11/55] fix: dev: Splitting command. --- lib/topology_docker_openswitch/openswitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 07548fc..0df4a78 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -226,7 +226,7 @@ def main(): raise Exception('Timed out while waiting for DB socket.') try: - check_output('systemctl start restd') + check_output(['systemctl', 'start', 'restd']) except CalledProcessError as error: raise Exception('Failed to start restd: {}'.format(error.output)) From ae60339d00fe2916ad96d99f08345b7dcedaaf15 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Wed, 10 Aug 2016 17:26:28 -0600 Subject: [PATCH 12/55] fix: dev: Checking for restd to be stopped. --- lib/topology_docker_openswitch/openswitch.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 0df4a78..3b230f5 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -226,9 +226,17 @@ def main(): raise Exception('Timed out while waiting for DB socket.') try: - check_output(['systemctl', 'start', 'restd']) - except CalledProcessError as error: - raise Exception('Failed to start restd: {}'.format(error.output)) + if 'Active: active' not in check_output( + ['systemctl', 'status', 'restd'] + ): + try: + check_output(['systemctl', 'start', 'restd']) + except CalledProcessError as error: + raise Exception( + 'Failed to start restd: {}'.format(error.output) + ) + except: + pass logging.info('Waiting for cur_hw...') for i in range(0, config_timeout): From 3fe6ea7f63c1559913c145580baf21485d6e814a Mon Sep 17 00:00:00 2001 From: fonsecamau Date: Thu, 11 Aug 2016 12:04:48 -0700 Subject: [PATCH 13/55] Fixing restd validation status and service start --- lib/topology_docker_openswitch/openswitch.py | 46 ++++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 3b230f5..3514989 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -225,19 +225,6 @@ def main(): else: raise Exception('Timed out while waiting for DB socket.') - try: - if 'Active: active' not in check_output( - ['systemctl', 'status', 'restd'] - ): - try: - check_output(['systemctl', 'start', 'restd']) - except CalledProcessError as error: - raise Exception( - 'Failed to start restd: {}'.format(error.output) - ) - except: - pass - logging.info('Waiting for cur_hw...') for i in range(0, config_timeout): if not cur_hw_is_set(): @@ -284,6 +271,39 @@ def main(): else: raise Exception('Timed out while waiting for final hostname.') + logging.info('Checking restd service status...') + output = '' + try: + output = check_output( + 'systemctl status restd', shell=True + ) + except CalledProcessError as e: + pass + if 'Active: active' not in output: + try: + logging.info('Starting restd daemon...') + check_output('systemctl start restd', shell=True) + logging.info('Checking restd service started...') + for i in range(0, config_timeout): + output = '' + output = check_output( + 'systemctl status restd', shell=True + ) + if 'Active: active' not in output: + sleep(0.1) + else: + break + else: + raise Exception("Failed to start restd service") + except CalledProcessError as e: + raise Exception( + 'Failed to start restd: {}'.format(e.output) + ) + except Exception as error: + raise Exception ( + error + ) + if __name__ == '__main__': main() """ From 3645c6f1e4480b39526b8e74760b7f92aa1d5586 Mon Sep 17 00:00:00 2001 From: fonsecamau Date: Tue, 23 Aug 2016 14:35:57 -0700 Subject: [PATCH 14/55] chg: dev: adding support for p4simulator images --- lib/topology_docker_openswitch/openswitch.py | 71 ++++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 3514989..4e298f6 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -54,6 +54,7 @@ config_timeout = 300 ops_switchd_active_timeout = 60 swns_netns = '/var/run/netns/swns' +emulns_netns = '/var/run/netns/emulns' hwdesc_dir = '/etc/openswitch/hwdesc' db_sock = '/var/run/openvswitch/db.sock' switchd_pid = '/var/run/openvswitch/ops-switchd.pid' @@ -92,33 +93,45 @@ def create_interfaces(): ports_hwdesc = yaml.load(fd) hwports = [str(p['name']) for p in ports_hwdesc['ports']] + netns = check_output("ls /var/run/netns", shell=True) # Get list of already created ports - not_in_swns = check_output(shsplit( + not_in_netns = check_output(shsplit( 'ls /sys/class/net/' )).split() - in_swns = check_output(shsplit( - 'ip netns exec swns ls /sys/class/net/' - )).split() + if "emulns" not in netns: + in_netns = check_output(shsplit( + 'ip netns exec swns ls /sys/class/net/' + )).split() + else: + in_netns = check_output(shsplit( + 'ip netns exec emulns ls /sys/class/net/' + )).split() logging.info( - ' - Not in swns {not_in_swns} '.format( + ' - Not in swns/emulns {not_in_netns} '.format( **locals() ) ) logging.info( - ' - In swns {in_swns} '.format( + ' - In swns/emulns {in_netns} '.format( **locals() ) ) create_cmd_tpl = 'ip tuntap add dev {hwport} mode tap' - netns_cmd_tpl = 'ip link set {hwport} netns swns' - rename_int = 'ip link set {portlbl} name {hwport}' + netns_cmd_tpl_swns = 'ip link set {hwport} netns swns' + netns_fp_cmd_tpl_swns = ('ip link set {hwport} netns swns') + netns_cmd_tpl_emulns = ('ip netns exec swns ip link set {hwport} ' + 'netns emulns') + + netns_fp_cmd_tpl_emulns = ('ip link set {hwport} netns emulns') + rename_int = ('ip link set {portlbl} name {hwport}') + ns_exec = 'ip netns exec emulns ' # Save port mapping information mapping_ports = {} # Map the port with the labels - for portlbl in not_in_swns: + for portlbl in not_in_netns: logging.info( ' - Port {portlbl} found'.format( **locals() @@ -129,13 +142,31 @@ def create_interfaces(): hwport = hwports.pop(0) mapping_ports[portlbl] = hwport logging.info( - ' - Port {portlbl} moved to swns netns as {hwport}.'.format( - **locals() - ) + ' - Port {portlbl} moved to swns/emulns netns as {hwport}.' + .format(**locals()) ) try: check_call(shsplit(rename_int.format(**locals()))) - check_call(shsplit(netns_cmd_tpl.format(hwport=hwport))) + if 'emulns' not in netns: + check_call(shsplit(netns_fp_cmd_tpl_swns + .format(hwport=hwport))) + else: + check_call(shsplit(netns_fp_cmd_tpl_emulns + .format(hwport=hwport))) + check_call('{ns_exec} ip link set dev {hwport} up' + .format(**locals()), shell=True) + out = check_output( + '{ns_exec} echo port_add {hwport} ' + ' {port} | {ns_exec} ' + '/usr/bin/bm_tools/runtime_CLI.py --json ' + '/usr/share/ovs_p4_plugin/switch_bmv2.json ' + '--thrift-port 10001'.format(ns_exec=ns_exec, + hwport=hwport, + port=str(int(hwport) - 1)), + shell=True + ) + logging.info('BM port creation...') + logging.info(out) except: raise Exception('Failed to map ports with port labels') @@ -145,20 +176,22 @@ def create_interfaces(): json_file.write(dumps(mapping_ports)) for hwport in hwports: - if hwport in in_swns: + if hwport in in_netns: logging.info(' - Port {} already present.'.format(hwport)) continue logging.info(' - Port {} created.'.format(hwport)) try: - check_call(shsplit(create_cmd_tpl.format(hwport=hwport))) + if "emulns" not in netns: + check_call(shsplit(create_cmd_tpl.format(hwport=hwport))) except: raise Exception('Failed to create tuntap') try: - check_call(shsplit(netns_cmd_tpl.format(hwport=hwport))) + if 'emulns' not in netns: + check_call(shsplit(netns_cmd_tpl_swns.format(hwport=hwport))) except: - raise Exception('Failed to move port to swns netns') + raise Exception('Failed to move port to swns/emulns netns') check_call(shsplit('touch /tmp/ops-virt-ports-ready')) logging.info(' - Ports readiness notified to the image') @@ -202,7 +235,7 @@ def main(): else: break else: - raise Exception('Timed out while waiting for swns.') + raise Exception('Timed out while waiting for swns/emulns.') logging.info('Waiting for hwdesc directory...') for i in range(0, config_timeout): @@ -356,10 +389,8 @@ def log_commands( class OpenSwitchNode(DockerNode): """ Custom OpenSwitch node for the Topology Docker platform engine. - This custom node loads an OpenSwitch image and has vtysh as default shell (in addition to bash). - See :class:`topology_docker.node.DockerNode`. """ From 4de919af1272372f9a5dc0cc5f192d3e5ba955c4 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Mon, 26 Sep 2016 17:11:11 -0600 Subject: [PATCH 15/55] fix: dev: Showing error message. --- lib/topology_docker_openswitch/openswitch.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 4e298f6..e3cf84f 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -167,8 +167,12 @@ def create_interfaces(): ) logging.info('BM port creation...') logging.info(out) - except: - raise Exception('Failed to map ports with port labels') + except Exception as error: + raise Exception( + 'Failed to map ports with port labels: {}'.format( + error.message + ) + ) # Writting mapping to file shared_dir_tmp = split(__file__)[0] From 866c96fac82b538e7eb27f66e0ab2f25f60de1bd Mon Sep 17 00:00:00 2001 From: fonsecamau Date: Tue, 27 Sep 2016 14:21:42 -0700 Subject: [PATCH 16/55] chg: dev: Collecting docker coredumps on teardown and adding checks for p4 simulator --- lib/topology_docker_openswitch/openswitch.py | 18 ++++++++++++++++++ .../plugin/plugin.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index e3cf84f..bb8c309 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -49,6 +49,7 @@ from subprocess import check_call, check_output, call, CalledProcessError from socket import AF_UNIX, SOCK_STREAM, socket, gethostname +import re import yaml config_timeout = 300 @@ -155,6 +156,17 @@ def create_interfaces(): .format(hwport=hwport))) check_call('{ns_exec} ip link set dev {hwport} up' .format(**locals()), shell=True) + for i in range(0, config_timeout): + link_state = check_output( + '{ns_exec} ip link show {hwport}' + .format(**locals()), shell=True) + if "UP" in link_state: + break; + else: + sleep(0.1) + else: + raise Exception('Emulns interface did not came up...') + out = check_output( '{ns_exec} echo port_add {hwport} ' ' {port} | {ns_exec} ' @@ -167,6 +179,12 @@ def create_interfaces(): ) logging.info('BM port creation...') logging.info(out) + re_str = r''''\s*Control utility for runtime P4 table \ +manipulation\s*\nRuntimeCmd:\s*\nRuntimeCmd:\s*$''' #noqa + + if re.findall(re_str, out, re.MULTILINE) is None: + raise Exception('Control utility for runtime P4 table ' + 'failed...') except Exception as error: raise Exception( 'Failed to map ports with port labels: {}'.format( diff --git a/lib/topology_docker_openswitch/plugin/plugin.py b/lib/topology_docker_openswitch/plugin/plugin.py index 66f50ec..4d3d59e 100644 --- a/lib/topology_docker_openswitch/plugin/plugin.py +++ b/lib/topology_docker_openswitch/plugin/plugin.py @@ -93,6 +93,24 @@ def pytest_runtest_teardown(item): ) ) + try: + core_files = join('/var/diagnostics/coredump', 'core.*') + files = node_obj.send_command('ls {}'.format(core_files), + shell='bash').splitlines() + + if len(files) > 0: + for file in files: + path = '/tmp' + node_obj.send_command('cp {file} {path}' + .format(**locals()), + shell='bash') + except: + warning( + 'Unable to get coredumps from node {}.'.format( + node_obj.identifier + ) + ) + try: copytree(shared_dir, join(path_name, basename(shared_dir))) rmtree(shared_dir) From 589d4569cb1ee08607cc028b848d14ceafe9ecff Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Wed, 28 Sep 2016 15:13:03 -0600 Subject: [PATCH 17/55] chg: dev: Using _register_shell. This is necessary to make shells register with the node and get information necessary to log properly in the pexpect file. --- lib/topology_docker_openswitch/openswitch.py | 37 +++++++++++++------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index bb8c309..2554c03 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -441,25 +441,38 @@ def __init__( # Add vtysh (default) shell # FIXME: Create a subclass to handle better the particularities of # vtysh, like prompt setup etc. - self._shells['vtysh'] = DockerShell( - self.container_id, 'vtysh', '(^|\n)switch(\([\-a-zA-Z0-9]*\))?#' + self._register_shell( + 'vtysh', + DockerShell( + self.container_id, 'vtysh', + '(^|\n)switch(\([\-a-zA-Z0-9]*\))?#' + ) ) # Add bash shells initial_prompt = '(^|\n).*[#$] ' - self._shells['bash'] = DockerBashShell( - self.container_id, 'bash', - initial_prompt=initial_prompt + self._register_shell( + 'bash', + DockerBashShell( + self.container_id, 'bash', + initial_prompt=initial_prompt + ) ) - self._shells['bash_swns'] = DockerBashShell( - self.container_id, 'ip netns exec swns bash', - initial_prompt=initial_prompt + self._register_shell( + 'bash_swns', + DockerBashShell( + self.container_id, 'ip netns exec swns bash', + initial_prompt=initial_prompt + ) ) - self._shells['vsctl'] = DockerBashShell( - self.container_id, 'bash', - initial_prompt=initial_prompt, - prefix='ovs-vsctl ', timeout=60 + self._register_shell( + 'vsctl', + DockerBashShell( + self.container_id, 'bash', + initial_prompt=initial_prompt, + prefix='ovs-vsctl ', timeout=60 + ) ) def notify_post_build(self): From ba60de6a59d25fabc0a885fff178471b83a450e7 Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Thu, 20 Oct 2016 15:08:53 -0700 Subject: [PATCH 18/55] fix: dev: Increasing config timeout. --- lib/topology_docker_openswitch/openswitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 2554c03..e2a15d4 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -52,7 +52,7 @@ import re import yaml -config_timeout = 300 +config_timeout = 1200 ops_switchd_active_timeout = 60 swns_netns = '/var/run/netns/swns' emulns_netns = '/var/run/netns/emulns' From 2b2dfddf988876194b245a6ca6cdce2496e05008 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 5 Jul 2016 13:32:57 -0600 Subject: [PATCH 19/55] chg: dev: Setting environment container to docker. --- lib/topology_docker_openswitch/openswitch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index e2a15d4..b23aff1 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -419,6 +419,7 @@ class OpenSwitchNode(DockerNode): def __init__( self, identifier, image='topology/ops:latest', binds=None, + environment={'container': 'docker'}, **kwargs): # Add binded directories From dcfc5f4afc97e03e81352e758f6594d2176dd937 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 11 Nov 2016 11:37:02 -0600 Subject: [PATCH 20/55] fix: dev: Removing read-only permissions. --- lib/topology_docker_openswitch/openswitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index b23aff1..8f4c024 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -425,7 +425,7 @@ def __init__( # Add binded directories container_binds = [ '/dev/log:/dev/log', - '/sys/fs/cgroup:/sys/fs/cgroup:ro' + '/sys/fs/cgroup:/sys/fs/cgroup' ] if binds is not None: container_binds.append(binds) From e7386dcb7617232004204016142546a633ae2b23 Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Tue, 22 Nov 2016 16:42:41 -0800 Subject: [PATCH 21/55] fix: dev: Passing 'container' env variable. --- lib/topology_docker_openswitch/openswitch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 8f4c024..966fad4 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -433,7 +433,8 @@ def __init__( super(OpenSwitchNode, self).__init__( identifier, image=image, command='/sbin/init', binds=';'.join(container_binds), hostname='switch', - network_mode='bridge', **kwargs + network_mode='bridge', environment={'container': 'docker'}, + **kwargs ) # FIXME: Remove this attribute to merge with version > 1.6.0 @@ -513,6 +514,7 @@ def _setup_system(self): platforms_log_location = { 'Ubuntu': 'cat /var/log/upstart/docker.log', 'CentOS Linux': 'grep docker /var/log/daemon.log', + 'debian': 'journalctl -u docker.service', # FIXME: find the right values for the next dictionary keys: # 'boot2docker': 'cat /var/log/docker.log', # 'debian': 'cat /var/log/daemon.log', From 513d1b52ccb6578a9a09dc1b97bcda632c246ee0 Mon Sep 17 00:00:00 2001 From: fonsecamau Date: Wed, 24 Aug 2016 11:29:47 -0700 Subject: [PATCH 22/55] fix: dev: Removing shell echo. --- doc/conf.py | 2 +- doc/developer.rst | 55 --- doc/index.rst | 15 +- doc/user.rst | 129 ++++++ lib/topology_docker_openswitch/openswitch.py | 367 ++---------------- .../openswitch_setup | 312 +++++++++++++++ .../plugin/plugin.py | 25 +- lib/topology_docker_openswitch/shell.py | 263 +++++++++++++ test/helpers.py | 52 --- test/test_ping.py | 102 ----- test/test_port_labels.py | 61 --- test/test_topology_docker_openswitch.py | 46 +++ test/test_vlan.py | 119 ------ 13 files changed, 800 insertions(+), 748 deletions(-) delete mode 100644 doc/developer.rst create mode 100644 doc/user.rst create mode 100644 lib/topology_docker_openswitch/openswitch_setup create mode 100644 lib/topology_docker_openswitch/shell.py delete mode 100644 test/helpers.py delete mode 100644 test/test_ping.py delete mode 100644 test/test_port_labels.py create mode 100644 test/test_topology_docker_openswitch.py delete mode 100644 test/test_vlan.py diff --git a/doc/conf.py b/doc/conf.py index dfa0bdd..e020aa1 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -53,7 +53,7 @@ master_doc = 'index' # General information about the project. -project = 'openswitch node for topology_docker' +project = 'OpenSwitch node for Topology Docker' copyright = '2016, Hewlett Packard Enterprise Development LP' author = 'Hewlett Packard Enterprise Development LP' diff --git a/doc/developer.rst b/doc/developer.rst deleted file mode 100644 index d1f544d..0000000 --- a/doc/developer.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. toctree:: - -.. highlight:: sh - -=============== -Developer Guide -=============== - - -Setup Development Environment -============================= - -#. Install ``pip`` and ``tox``: - - :: - - sudo apt-get install python-pip - sudo pip install tox - -#. Configure git pre-commit hook: - - :: - - sudo pip install flake8 pep8-naming - flake8 --install-hook - git config flake8.strict true - - -Building Documentation -====================== - -:: - - tox -e doc - -Output will be available at ``.tox/doc/tmp/html``. It is recommended to install -the ``webdev`` package: - -:: - - sudo pip install webdev - -So a development web server can serve any location like this: - -:: - - $ webdev .tox/doc/tmp/html - - -Running Test Suite -================== - -:: - - tox -e py27,py34 diff --git a/doc/index.rst b/doc/index.rst index 73f113f..16de4e9 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -1,29 +1,22 @@ .. toctree:: :hidden: - developer + user topology_docker_openswitch/topology_docker_openswitch =================================== -openswitch node for topology_docker +OpenSwitch node for Topology Docker =================================== -A Topology OpenSwitch Node for topology_docker. +An OpenSwitch Node for Topology Docker Documentation ============= -- :doc:`Developer Guide. ` +- :doc:`User Guide. ` - :doc:`Internal Documentation Reference. ` - -Development -=========== - -- `Project repository. `_ - - License ======= diff --git a/doc/user.rst b/doc/user.rst new file mode 100644 index 0000000..dc32aae --- /dev/null +++ b/doc/user.rst @@ -0,0 +1,129 @@ +.. toctree:: + +.. highlight:: sh + +========== +User Guide +========== + +This is an OpenSwitch node for Topology Docker. + +Shells +====== + +This node has several shells, their details are explained below. + +bash +.... + +This shell points to the ``bash`` shell of the node. Its prompt is set to this +value: + +:: + @~~==::BASH_PROMPT::==~~@ + +This shell has its echo disabled also, so that commands typed in it will not +show up in the console of the user. + + +bash_swns +......... + +This shell has all the attributes of the ``bash`` shell but all commands are +executed in the ``swns`` network namespace. + +vsctl +..... + +This shell has all the attributes of the ``bash`` shell but all commands are +prefixed with ``ovs-vsctl``. + +vtysh +..... + +This shell behaves differently depending on the availability of the ``vtysh`` +``set prompt`` command. + +If it is available, the shell will have its echo disabled and its prompt set to +this value: + +:: + + X@~~==::VTYSH_PROMPT::==~~@X + +Note that this is only the prompt that will be set, contexts will still be +appearing after it, for that reason you may want to use this regular +expression if you want to match with any context that has this forced prompt: + +:: + + r'(\r\n)?X@~~==::VTYSH_PROMPT::==~~@X(\([-\w\s]+\))?# ' + +If ``set prompt`` is not available, the echo will not be disabled and the +prompt will remain in its standard value. + +Be aware that in order for the node to detect the ``Segmentation fault`` error +message, the ``vytsh`` shell is started with ``stdbuf -oL vtysh``. + +The Booting Process +=================== + +The node copies a Python script in the container that performs the following +actions: + +#. Waits 30 seconds for ``/var/run/netns/swns``. +#. Waits 30 seconds for ``/etc/openswitch/hwdesc``. +#. Creates interfaces. +#. Waits 30 seconds for ``/var/run/openvswitch/db.sock``. +#. Waits 30 seconds for ``cur_hw``. +#. Waits 30 seconds for ``cur_cfg``. +#. Waits 30 seconds for ``/var/run/openvswitch/ops-switchd.pid``. +#. Waits 30 seconds for the hostname to be set to ``switch``. + +For the case of ``cur_hw`` and ``cur_cfg``, their value is taken from a query +sent to ``/var/run/openswitch/db.sock``. This query has this format: + +:: + + 'X': { + 'method': 'transact', + 'params': [ + 'OpenSwitch', + { + 'op': 'select', + 'table': 'System', + 'where': [], + 'columns': ['X'] + } + ], + 'id': id(db_sock) + } + +The value of ``1`` is looked for in: + +:: + + response['result'][0]['rows'][0][X] == 1 + +``X`` is a placeholder for ``cur_hw`` and ``cur_cfg``. + +If any of the previous waits times out, an exception of this kind will be +raised: + +:: + + The image did not boot correctly, ... + +These errors are caused by a faulty image, the framework is just reporting +them. This errors happen *before* the very first line of test case is executed. + +This node will create interfaces and will move them to ``swns`` or to +``emulns`` if the image is using the P4 simulator. Any failure in the process +of creating interfaces will be reported like this: + +:: + + Failed to map ports with port labels... + +Depending on the error, the failing command or other information will be +displayed after that message. diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 966fad4..d3ec692 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -29,340 +29,18 @@ from platform import system, linux_distribution from logging import StreamHandler, getLogger, INFO, Formatter from sys import stdout +from os.path import join, dirname, normpath, abspath from topology_docker.node import DockerNode -from topology_docker.shell import DockerShell, DockerBashShell +from topology_docker.shell import DockerBashShell + +from .shell import OpenSwitchVtyshShell # When a failure happens during boot time, logs and other information is # collected to help with the debugging. The path of this collection is to be # stored here at module level to be able to import it in the pytest teardown # hook later. Non-failing containers will append their log paths here also. LOG_PATHS = [] - -SETUP_SCRIPT = """\ -import logging -from sys import argv -from time import sleep -from os.path import exists, split -from json import dumps, loads -from shlex import split as shsplit -from subprocess import check_call, check_output, call, CalledProcessError -from socket import AF_UNIX, SOCK_STREAM, socket, gethostname - -import re -import yaml - -config_timeout = 1200 -ops_switchd_active_timeout = 60 -swns_netns = '/var/run/netns/swns' -emulns_netns = '/var/run/netns/emulns' -hwdesc_dir = '/etc/openswitch/hwdesc' -db_sock = '/var/run/openvswitch/db.sock' -switchd_pid = '/var/run/openvswitch/ops-switchd.pid' -query_cur_hw = { - 'method': 'transact', - 'params': [ - 'OpenSwitch', - { - 'op': 'select', - 'table': 'System', - 'where': [], - 'columns': ['cur_hw'] - } - ], - 'id': id(db_sock) -} -query_cur_cfg = { - 'method': 'transact', - 'params': [ - 'OpenSwitch', - { - 'op': 'select', - 'table': 'System', - 'where': [], - 'columns': ['cur_cfg'] - } - ], - 'id': id(db_sock) -} -sock = None - - -def create_interfaces(): - # Read ports from hardware description - with open('{}/ports.yaml'.format(hwdesc_dir), 'r') as fd: - ports_hwdesc = yaml.load(fd) - hwports = [str(p['name']) for p in ports_hwdesc['ports']] - - netns = check_output("ls /var/run/netns", shell=True) - # Get list of already created ports - not_in_netns = check_output(shsplit( - 'ls /sys/class/net/' - )).split() - if "emulns" not in netns: - in_netns = check_output(shsplit( - 'ip netns exec swns ls /sys/class/net/' - )).split() - else: - in_netns = check_output(shsplit( - 'ip netns exec emulns ls /sys/class/net/' - )).split() - logging.info( - ' - Not in swns/emulns {not_in_netns} '.format( - **locals() - ) - ) - logging.info( - ' - In swns/emulns {in_netns} '.format( - **locals() - ) - ) - - create_cmd_tpl = 'ip tuntap add dev {hwport} mode tap' - netns_cmd_tpl_swns = 'ip link set {hwport} netns swns' - netns_fp_cmd_tpl_swns = ('ip link set {hwport} netns swns') - netns_cmd_tpl_emulns = ('ip netns exec swns ip link set {hwport} ' - 'netns emulns') - - netns_fp_cmd_tpl_emulns = ('ip link set {hwport} netns emulns') - rename_int = ('ip link set {portlbl} name {hwport}') - ns_exec = 'ip netns exec emulns ' - - # Save port mapping information - mapping_ports = {} - - # Map the port with the labels - for portlbl in not_in_netns: - logging.info( - ' - Port {portlbl} found'.format( - **locals() - ) - ) - if portlbl in ['lo', 'oobm', 'eth0', 'bonding_masters']: - continue - hwport = hwports.pop(0) - mapping_ports[portlbl] = hwport - logging.info( - ' - Port {portlbl} moved to swns/emulns netns as {hwport}.' - .format(**locals()) - ) - try: - check_call(shsplit(rename_int.format(**locals()))) - if 'emulns' not in netns: - check_call(shsplit(netns_fp_cmd_tpl_swns - .format(hwport=hwport))) - else: - check_call(shsplit(netns_fp_cmd_tpl_emulns - .format(hwport=hwport))) - check_call('{ns_exec} ip link set dev {hwport} up' - .format(**locals()), shell=True) - for i in range(0, config_timeout): - link_state = check_output( - '{ns_exec} ip link show {hwport}' - .format(**locals()), shell=True) - if "UP" in link_state: - break; - else: - sleep(0.1) - else: - raise Exception('Emulns interface did not came up...') - - out = check_output( - '{ns_exec} echo port_add {hwport} ' - ' {port} | {ns_exec} ' - '/usr/bin/bm_tools/runtime_CLI.py --json ' - '/usr/share/ovs_p4_plugin/switch_bmv2.json ' - '--thrift-port 10001'.format(ns_exec=ns_exec, - hwport=hwport, - port=str(int(hwport) - 1)), - shell=True - ) - logging.info('BM port creation...') - logging.info(out) - re_str = r''''\s*Control utility for runtime P4 table \ -manipulation\s*\nRuntimeCmd:\s*\nRuntimeCmd:\s*$''' #noqa - - if re.findall(re_str, out, re.MULTILINE) is None: - raise Exception('Control utility for runtime P4 table ' - 'failed...') - except Exception as error: - raise Exception( - 'Failed to map ports with port labels: {}'.format( - error.message - ) - ) - - # Writting mapping to file - shared_dir_tmp = split(__file__)[0] - with open('{}/port_mapping.json'.format(shared_dir_tmp), 'w') as json_file: - json_file.write(dumps(mapping_ports)) - - for hwport in hwports: - if hwport in in_netns: - logging.info(' - Port {} already present.'.format(hwport)) - continue - - logging.info(' - Port {} created.'.format(hwport)) - try: - if "emulns" not in netns: - check_call(shsplit(create_cmd_tpl.format(hwport=hwport))) - except: - raise Exception('Failed to create tuntap') - - try: - if 'emulns' not in netns: - check_call(shsplit(netns_cmd_tpl_swns.format(hwport=hwport))) - except: - raise Exception('Failed to move port to swns/emulns netns') - check_call(shsplit('touch /tmp/ops-virt-ports-ready')) - logging.info(' - Ports readiness notified to the image') - -def cur_hw_is_set(): - global sock - if sock is None: - sock = socket(AF_UNIX, SOCK_STREAM) - sock.connect(db_sock) - sock.send(dumps(query_cur_hw)) - response = loads(sock.recv(4096)) - try: - return response['result'][0]['rows'][0]['cur_hw'] == 1 - except IndexError: - return 0 - -def cur_cfg_is_set(): - global sock - if sock is None: - sock = socket(AF_UNIX, SOCK_STREAM) - sock.connect(db_sock) - sock.send(dumps(query_cur_cfg)) - response = loads(sock.recv(4096)) - try: - return response['result'][0]['rows'][0]['cur_cfg'] == 1 - except IndexError: - return 0 - -def ops_switchd_is_active(): - is_active = call(["systemctl", "is-active", "switchd.service"]) - return is_active == 0 - -def main(): - - if '-d' in argv: - logging.basicConfig(level=logging.DEBUG) - - logging.info('Waiting for swns netns...') - for i in range(0, config_timeout): - if not exists(swns_netns): - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for swns/emulns.') - - logging.info('Waiting for hwdesc directory...') - for i in range(0, config_timeout): - if not exists(hwdesc_dir): - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for hwdesc directory.') - - logging.info('Creating interfaces...') - create_interfaces() - - logging.info('Waiting for DB socket...') - for i in range(0, config_timeout): - if not exists(db_sock): - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for DB socket.') - - logging.info('Waiting for cur_hw...') - for i in range(0, config_timeout): - if not cur_hw_is_set(): - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for cur_hw.') - - logging.info('Waiting for cur_cfg...') - for i in range(0, config_timeout): - if not cur_cfg_is_set(): - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for cur_cfg.') - - logging.info('Waiting for switchd pid...') - for i in range(0, config_timeout): - if not exists(switchd_pid): - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for switchd pid.') - - logging.info('Waiting for ops-switchd to become active...') - for i in range(0, ops_switchd_active_timeout): - if not ops_switchd_is_active(): - sleep(1) - else: - break - else: - raise Exception('Timed out while waiting for ops-switchd ' - 'to become active.') - - logging.info('Wait for final hostname...') - for i in range(0, config_timeout): - if gethostname() != 'switch': - sleep(0.1) - else: - break - else: - raise Exception('Timed out while waiting for final hostname.') - - logging.info('Checking restd service status...') - output = '' - try: - output = check_output( - 'systemctl status restd', shell=True - ) - except CalledProcessError as e: - pass - if 'Active: active' not in output: - try: - logging.info('Starting restd daemon...') - check_output('systemctl start restd', shell=True) - logging.info('Checking restd service started...') - for i in range(0, config_timeout): - output = '' - output = check_output( - 'systemctl status restd', shell=True - ) - if 'Active: active' not in output: - sleep(0.1) - else: - break - else: - raise Exception("Failed to start restd service") - except CalledProcessError as e: - raise Exception( - 'Failed to start restd: {}'.format(e.output) - ) - except Exception as error: - raise Exception ( - error - ) - -if __name__ == '__main__': - main() -""" - LOG = getLogger(__name__) LOG_HDLR = StreamHandler(stream=stdout) LOG_HDLR.setFormatter(Formatter('%(asctime)s %(message)s')) @@ -441,19 +119,16 @@ def __init__( self.shared_dir_mount = '/tmp' # Add vtysh (default) shell - # FIXME: Create a subclass to handle better the particularities of - # vtysh, like prompt setup etc. - self._register_shell( - 'vtysh', - DockerShell( - self.container_id, 'vtysh', - '(^|\n)switch(\([\-a-zA-Z0-9]*\))?#' - ) - ) + # This shell is started as a bash shell but it changes itself to a + # vtysh one afterwards. This is necessary because this shell must be + # started from a bash one that has echo disabled to avoid wrong + # matching with some command output and by setting an unique prompt + # with the set prompt vtysh command + self._register_shell('vtysh', OpenSwitchVtyshShell(self.container_id)) # Add bash shells - initial_prompt = '(^|\n).*[#$] ' + initial_prompt = '(^|\n).*[#$] ' self._register_shell( 'bash', DockerBashShell( @@ -497,9 +172,14 @@ def _setup_system(self): """ # Write and execute setup script + with open( + join(dirname(normpath(abspath(__file__))), 'openswitch_setup') + ) as openswitch_setup_file: + openswitch_setup = openswitch_setup_file.read() + setup_script = '{}/openswitch_setup.py'.format(self.shared_dir) with open(setup_script, 'w') as fd: - fd.write(SETUP_SCRIPT) + fd.write(openswitch_setup) try: self._docker_exec( @@ -612,5 +292,18 @@ def set_port_state(self, portlbl, state): command = '{prefix} ip link set dev {iface} {state}'.format(**locals()) self._docker_exec(command) + def stop(self): + """ + Exit all vtysh shells. + + See :meth:`DockerNode.stop` for more information. + """ + + for shell in self._shells.values(): + if isinstance(shell, OpenSwitchVtyshShell): + shell._exit() + + super(OpenSwitchNode, self).stop() + __all__ = ['OpenSwitchNode'] diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup new file mode 100644 index 0000000..820c593 --- /dev/null +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -0,0 +1,312 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This script prepares an OpenSwitch image to run as a Topology node. It is +copied as openswitch_setup.py in the docker container shared folder and then +executed with python /path/to/the/script/openswitch_setup.py -d. +""" + +from logging import info, DEBUG, basicConfig +from sys import argv +from time import sleep +from os.path import exists, split +from json import dumps, loads +from shlex import split as shsplit +from subprocess import check_call, check_output, call, CalledProcessError +from socket import AF_UNIX, SOCK_STREAM, socket, gethostname +from re import search, MULTILINE +from yaml import load + +config_timeout = 300 +ops_switchd_active_timeout = 60 +swns_netns = '/var/run/netns/swns' +emulns_netns = '/var/run/netns/emulns' +hwdesc_dir = '/etc/openswitch/hwdesc' +db_sock = '/var/run/openvswitch/db.sock' +switchd_pid = '/var/run/openvswitch/ops-switchd.pid' +sock = None + + +def create_interfaces(): + # Read ports from hardware description + with open('{}/ports.yaml'.format(hwdesc_dir), 'r') as fd: + ports_hwdesc = load(fd) + hwports = [str(p['name']) for p in ports_hwdesc['ports']] + + netns = check_output("ls /var/run/netns", shell=True) + + # Get list of already created ports + not_in_netns = check_output(shsplit( + 'ls /sys/class/net/' + )).split() + + if "emulns" not in netns: + in_netns = check_output(shsplit( + 'ip netns exec swns ls /sys/class/net/' + )).split() + else: + in_netns = check_output(shsplit( + 'ip netns exec emulns ls /sys/class/net/' + )).split() + + info('Not in swns/emulns: {not_in_netns} '.format(**locals())) + info('In swns/emulns {in_netns} '.format(**locals())) + + create_cmd_tpl = 'ip tuntap add dev {hwport} mode tap' + netns_cmd_tpl_swns = 'ip link set {hwport} netns swns' + netns_fp_cmd_tpl_swns = 'ip link set {hwport} netns swns' + netns_cmd_tpl_emulns = ( + 'ip netns exec swns ip link set {hwport} netns emulns' + ) + netns_fp_cmd_tpl_emulns = 'ip link set {hwport} netns emulns' + rename_int = 'ip link set {portlbl} name {hwport}' + ns_exec = 'ip netns exec emulns ' + + # Save port mapping information + mapping_ports = {} + + # Map the port with the labels + for portlbl in not_in_netns: + info('Port {portlbl} found'.format(**locals())) + + if portlbl in ['lo', 'oobm', 'eth0', 'bonding_masters']: + continue + + hwport = hwports.pop(0) + mapping_ports[portlbl] = hwport + + info( + 'Port {portlbl} moved to swns/emulns netns as {hwport}.' + .format(**locals()) + ) + + try: + check_call(shsplit(rename_int.format(**locals()))) + + if 'emulns' not in netns: + check_call( + shsplit(netns_fp_cmd_tpl_swns.format(hwport=hwport)) + ) + else: + check_call( + shsplit(netns_fp_cmd_tpl_emulns.format(hwport=hwport)) + ) + check_call( + '{ns_exec} ip link set dev {hwport} up'.format(**locals()), + shell=True + ) + + for i in range(0, config_timeout): + link_state = check_output( + '{ns_exec} ip link show {hwport}'.format(**locals()), + shell=True + ) + if "UP" in link_state: + break + else: + sleep(0.1) + else: + raise Exception('emulns interface did not came up.') + + out = check_output( + '{ns_exec} echo port_add {hwport} ' + ' {port} | {ns_exec} ' + '/usr/bin/bm_tools/runtime_CLI.py --json ' + '/usr/share/ovs_p4_plugin/switch_bmv2.json ' + '--thrift-port 10001'.format( + ns_exec=ns_exec, hwport=hwport, + port=str(int(hwport) - 1) + ), + shell=True + ) + + info('BM port creation: {}'.format(out)) + + regex = ( + r'\s*Control utility for runtime P4 table' + r' manipulation\s*\nRuntimeCmd:\s*\nRuntimeCmd:\s*$' + ) + + if search(regex, out, MULTILINE) is None: + raise Exception( + 'Control utility for runtime P4 table failed.' + ) + + except CalledProcessError as error: + raise Exception( + 'Failed to map ports with port labels, {} failed with this ' + 'error: {}'.format(error.cmd, error.output) + ) + + except Exception as error: + raise Exception( + 'Failed to map ports with port labels: {}'.format( + error.message + ) + ) + + # Writting mapping to file + shared_dir_tmp = split(__file__)[0] + + with open('{}/port_mapping.json'.format(shared_dir_tmp), 'w') as json_file: + json_file.write(dumps(mapping_ports)) + + for hwport in hwports: + if hwport in in_netns: + info('Port {} already present.'.format(hwport)) + continue + + info('Port {} created.'.format(hwport)) + try: + if 'emulns' not in netns: + check_call(shsplit(create_cmd_tpl.format(hwport=hwport))) + except: + raise Exception('Failed to create tuntap') + + try: + if 'emulns' not in netns: + check_call(shsplit(netns_cmd_tpl_swns.format(hwport=hwport))) + except: + raise Exception('Failed to move port to swns/emulns netns.') + + check_call(shsplit('touch /tmp/ops-virt-ports-ready')) + info('Port readiness notified to the image.') + + +def cur_is_set(cur_key): + queries = { + 'cur_hw': { + 'method': 'transact', + 'params': [ + 'OpenSwitch', + { + 'op': 'select', + 'table': 'System', + 'where': [], + 'columns': ['cur_hw'] + } + ], + 'id': id(db_sock) + }, + 'cur_cfg': { + 'method': 'transact', + 'params': [ + 'OpenSwitch', + { + 'op': 'select', + 'table': 'System', + 'where': [], + 'columns': ['cur_cfg'] + } + ], + 'id': id(db_sock) + } + } + + global sock + if sock is None: + sock = socket(AF_UNIX, SOCK_STREAM) + sock.connect(db_sock) + sock.send(dumps(queries[cur_key])) + response = loads(sock.recv(4096)) + + try: + return response['result'][0]['rows'][0][cur_key] == 1 + except IndexError: + return 0 + + +def ops_switchd_is_active(): + is_active = call(["systemctl", "is-active", "switchd.service"]) + return is_active == 0 + + +def main(): + + if '-d' in argv: + basicConfig(level=DEBUG) + + def wait_check(function, wait_name, wait_error, *args): + info('Waiting for {}'.format(wait_name)) + + for i in range(0, config_timeout): + if not function(*args): + sleep(0.1) + else: + break + else: + raise Exception( + 'The image did not boot correctly, ' + '{} after waiting {} seconds.'.format( + wait_error, int(0.1 * config_timeout) + ) + ) + + wait_check( + exists, swns_netns, '{} was not present'.format(swns_netns), swns_netns + ) + wait_check( + exists, hwdesc_dir, '{} was not present'.format(hwdesc_dir), hwdesc_dir + ) + + info('Creating interfaces') + create_interfaces() + + wait_check( + exists, db_sock, '{} was not present'.format(db_sock), db_sock + ) + wait_check( + cur_is_set, 'cur_hw to be set to 1', 'cur_hw is not set to 1', + 'cur_hw' + ) + wait_check( + cur_is_set, 'cur_cfg to be set to 1', 'cur_cfg is not set to 1', + 'cur_cfg' + ) + wait_check( + exists, switchd_pid, '{} was not present'.format(switchd_pid), + switchd_pid + ) + wait_check( + ops_switchd_is_active, 'ops-switchd to be active', + 'ops-switchd was not active' + ) + wait_check( + lambda: gethostname() == 'switch', 'final hostname', + 'hostname was not set' + ) + + info('Checking restd service status...') + output = '' + try: + output = check_output( + 'systemctl status restd', shell=True + ) + except CalledProcessError as e: + pass + if 'Active: active' not in output: + try: + info('Starting restd daemon.') + check_output('systemctl start restd', shell=True) + + info('Checking restd service started.') + for i in range(0, config_timeout): + output = '' + output = check_output( + 'systemctl status restd', shell=True + ) + if 'Active: active' not in output: + sleep(0.1) + else: + break + else: + raise Exception("Failed to start restd service") + + except CalledProcessError as e: + raise Exception( + 'Failed to start restd: {}'.format(e.output) + ) + +if __name__ == '__main__': + main() diff --git a/lib/topology_docker_openswitch/plugin/plugin.py b/lib/topology_docker_openswitch/plugin/plugin.py index 4d3d59e..0331fcd 100644 --- a/lib/topology_docker_openswitch/plugin/plugin.py +++ b/lib/topology_docker_openswitch/plugin/plugin.py @@ -93,17 +93,22 @@ def pytest_runtest_teardown(item): ) ) + bash_shell = node_obj.get_shell('bash') + try: - core_files = join('/var/diagnostics/coredump', 'core.*') - files = node_obj.send_command('ls {}'.format(core_files), - shell='bash').splitlines() - - if len(files) > 0: - for file in files: - path = '/tmp' - node_obj.send_command('cp {file} {path}' - .format(**locals()), - shell='bash') + core_path = '/var/diagnostics/coredump' + + bash_shell.send_command( + 'ls -1 {}/core.* 2>/dev/null'.format(core_path), silent=True + ) + + core_files = bash_shell.get_response(silent=True).splitlines() + + for core_file in core_files: + bash_shell.send_command( + 'cp {core_path}/{core_file} /tmp'.format(**locals()), + silent=True + ) except: warning( 'Unable to get coredumps from node {}.'.format( diff --git a/lib/topology_docker_openswitch/shell.py b/lib/topology_docker_openswitch/shell.py new file mode 100644 index 0000000..95cc892 --- /dev/null +++ b/lib/topology_docker_openswitch/shell.py @@ -0,0 +1,263 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +OpenSwitch shell module +""" + +from __future__ import unicode_literals, absolute_import +from __future__ import print_function, division + +from logging import warning +from re import match, search + +from pexpect import EOF + +from topology.platforms.shell import PExpectBashShell +from topology_docker.shell import DockerShell + + +_VTYSH_PROMPT_TPL = r'(\r\n)?{}(\([-\w\s]+\))?# ' +_VTYSH_FORCED = 'X@~~==::VTYSH_PROMPT::==~~@X' +# This is a regular expression that matches with values that may be found in +# unset vtysh prompts: +_VTYSH_STANDARD = '[-\w]+' + +VTYSH_FORCED_PROMPT = _VTYSH_PROMPT_TPL.format(_VTYSH_FORCED) +VTYSH_STANDARD_PROMPT = _VTYSH_PROMPT_TPL.format(_VTYSH_STANDARD) + +BASH_FORCED_PROMPT = PExpectBashShell.FORCED_PROMPT + + +class OpenSwitchVtyshShell(DockerShell): + """ + OpenSwitch ``vtysh`` shell + + This shell handles the particularities of the ``vtysh`` shell of an + OpenSwitch node. It is actually a shell that connects first to ``bash`` and + from the ``bash`` shell it then opens a ``vtysh`` shell. + + The actual process that this shell follows depends on the image of the + OpenSwitch node. Newer images support the ``vtysh`` ``set prompt`` command, + older images do not. This command allows the user to change the vtysh + prompt to any value without other side effects (like the hostname command + has). + + #. A connection to the ``bash`` shell of the node is done. + #. The ``bash`` prompt is set to ``@~~==::BASH_PROMPT::==~~@``. + #. A ``vtysh`` shell is opened with ``stdbuf -oL vtysh``. + #. The ``vtysh`` ``set prompt X@~~==::VTYSH_PROMPT::==~~@X`` command is + executed to set the ``vtysh`` forced prompt. + + If the next prompt received matches the ``vtysh`` forced prompt, this + process is followed: + + #. The ``vtysh`` shell is exited back to the ``bash`` shell by sending + ``exit``. + #. The echo of the ``bash`` shell is disabled with ``stty -echo``. This + will also disable the echo of the ``vtysh`` shell that will be started + from the ``bash`` shell. + #. A ``vtysh`` shell will be started with ``stdbuf -oL vtysh``. + #. The ``vtysh`` ``set prompt X@~~==::VTYSH_PROMPT::==~~@X`` command is + executed. + #. The shell prompt is set to the forced ``vtysh`` prompt. + #. In this case, the shell will not try to remove the echo of the ``vtysh`` + commands because they should not appear since the echo is disabled. + + If the next prompt received does not match the ``vtysh`` forced prompt, + this process is followed: + + #. The shell is configured to try to remove the echo of the ``vtysh`` + commands by looking for them in the command output. + #. The shell prompt is set to the standard ``vtysh`` prompt. + + Once the container is to be destroyed in the normal clean up of nodes, the + ``vtysh`` shell is exited to the ``bash`` one by sending the ``end`` + command followed by the ``exit`` command. + + :param str container: identifier of the container that holds this shell + """ + + def __init__(self, container): + # The parameter try_filter_echo is disabled by default here to handle + # images that support the vtysh "set prompt" command and will have its + # echo disabled since it extends from DockeBashShell. For other + # situations where this is not supported, the self._try_filter_echo + # attribute is disabled afterwards by altering it directly. + # The prompt value passed here is the one that will match with an + # OpenSwitch bash shell initial prompt. + super(OpenSwitchVtyshShell, self).__init__( + container, 'bash', '(^|\n).*[#$] ', try_filter_echo=False + ) + + def _setup_shell(self, connection=None): + """ + Get the shell ready to handle ``vtysh`` particularities. + + These particularities are the handling of segmentation fault errors + and forced or standard ``vtysh`` prompts. + + See :meth:`PExpectShell._setup_shell` for more information. + """ + + spawn = self._get_connection(connection) + # Since user, password or initial_command are not being used, this is + # the first expect done in the connection. The value of self._prompt at + # this moment is the initial prompt of an OpenSwitch bash shell prompt. + spawn.expect(self._prompt) + + # The bash prompt is set to a forced value for vtysh shells that + # support prompt setting and for the ones that do not. + spawn.sendline('export PS1={}'.format(BASH_FORCED_PROMPT)) + spawn.expect(BASH_FORCED_PROMPT) + + def determine_set_prompt(): + """ + This method determines if the vtysh command set prompt exists. + + This method starts wit a call to sendline and finishes with a call + to expect. + + :rtype: bool + :return: True if vtysh supports the ``set prompt`` command, False + otherwise. + """ + # When a segmentation fault error happens, the message + # "Segmentation fault" shows up in the terminal and then and EOF + # follows, making the vtysh shell to close ending up in the bash + # shell that opened it. + + # This starts the vtysh shell in a mode that forces the shell to + # always return the produced output, even if an EOF exception + # follows after it. This is done to handle the segmentation fault + # errors. + spawn.sendline('stdbuf -oL vtysh') + spawn.expect(VTYSH_STANDARD_PROMPT) + + # The newer images of OpenSwitch include this command that changes + # the prompt of the shell to an unique value. This is done to + # perform a safe matching that will match only with this value in + # each expect. + spawn.sendline('set prompt {}'.format(_VTYSH_FORCED)) + + # Since it is not possible to know beforehand if the image loaded + # in the node includes the "set prompt" command, an attempt to + # match any of the following prompts is done. If the command does + # not exist, the shell will return an standard prompt after showing + # an error message. + index = spawn.expect( + [VTYSH_STANDARD_PROMPT, VTYSH_FORCED_PROMPT] + ) + + return bool(index) + + def join_prompt(prompt): + return '{}|{}'.format(BASH_FORCED_PROMPT, prompt) + + if determine_set_prompt(): + # If this image supports "set prompt", then exit back to bash to + # set the bash shell without echo. + spawn.sendline('exit') + spawn.expect(BASH_FORCED_PROMPT) + + # This disables the echo in the bash and in the subsequent vtysh + # shell too. + spawn.sendline('stty -echo') + spawn.expect(BASH_FORCED_PROMPT) + + # Go into the vtysh shell again. Exiting vtysh after calling "set + # prompt" successfully disables the vtysh shell prompt to its + # standard value, so it is necessary to call it again. + determine_set_prompt() + + # From now on the shell _prompt attribute is set to the defined + # vtysh forced prompt. + self._prompt = join_prompt(VTYSH_FORCED_PROMPT) + + else: + # If the image does not support "set prompt", then enable the + # filtering of echo by setting the corresponding attribute to True. + # WARNING: Using a private attribute here. + self._try_filter_echo = True + + # From now on the shell _prompt attribute is set to the defined + # vtysh standard prompt. + self._prompt = join_prompt(VTYSH_STANDARD_PROMPT) + + # This sendline is used here just because a _setup_shell must end in an + # send/sendline call since it is followed by a call to expect in the + # connect method. + spawn.sendline('') + + def send_command( + self, command, matches=None, newline=True, timeout=None, + connection=None, silent=False + ): + # This parent method performs the connection to the shell and the set + # up of a bash prompt to an unique value. + match_index = super(OpenSwitchVtyshShell, self).send_command( + command, matches=matches, newline=newline, timeout=timeout, + connection=connection, silent=silent + ) + + spawn = self._get_connection(connection) + + # To find out if a segmentation fault error was produced, a search for + # the "Segmentation fault" string in the output of the command is done. + segmentation_fault = search( + r'Segmentation fault', self.get_response(silent=True) + ) + + # The other necessary condition to detect a segmentation fault error is + # to detect a forced bash prompt being matched. + forced_bash_prompt = match( + BASH_FORCED_PROMPT, spawn.after.decode( + encoding=self._encoding, errors=self._errors + ) + ) + + # This exception is raised to provide a meaningful error to the user. + if segmentation_fault is not None and forced_bash_prompt is not None: + raise Exception( + 'Segmentation fault received when executing "{}".'.format( + self._last_command + ) + ) + + return match_index + + def _exit(self): + """ + Attempt a clean exit from the shell. + + This is necessary to enable gathering of coverage information. + """ + try: + self.send_command('end', silent=True) + self.send_command( + 'exit', matches=[EOF, BASH_FORCED_PROMPT], + silent=True + ) + except Exception as error: + warning( + 'Exiting the shell failed with this error: {}'.format( + str(error) + ) + ) + + +__all__ = ['OpenSwitchVtyshShell'] diff --git a/test/helpers.py b/test/helpers.py deleted file mode 100644 index 5a14227..0000000 --- a/test/helpers.py +++ /dev/null @@ -1,52 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Testing helpers for OpenSwitch support node for the Topology framework. -""" - -from __future__ import unicode_literals, absolute_import -from __future__ import print_function, division - -from time import sleep - - -def wait_until_interface_up(switch, portlbl, timeout=30, polling_frequency=1): - """ - Wait until the interface, as mapped by the given portlbl, is marked as up. - - :param switch: The switch node. - :param str portlbl: Port label that is mapped to the interfaces. - :param int timeout: Number of seconds to wait. - :param int polling_frequency: Frequency of the polling. - :return: None if interface is brought-up. If not, an assertion is raised. - """ - for i in range(timeout): - status = switch.libs.vtysh.show_interface(portlbl) - if status['interface_state'] == 'up': - break - sleep(polling_frequency) - else: - assert False, ( - 'Interface {}:{} never brought-up after ' - 'waiting for {} seconds'.format( - switch.identifier, portlbl, timeout - ) - ) - - -__all__ = ['wait_until_interface_up'] diff --git a/test/test_ping.py b/test/test_ping.py deleted file mode 100644 index a0cb604..0000000 --- a/test/test_ping.py +++ /dev/null @@ -1,102 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -OpenSwitch Test for simple ping between nodes. -""" - -from __future__ import unicode_literals, absolute_import -from __future__ import print_function, division - -from time import sleep - -from .helpers import wait_until_interface_up - - -TOPOLOGY = """ -# +-------+ +-------+ -# | | +-------+ +-------+ | | -# | hs1 <-----> sw1 <-----> sw2 <-----> hs2 | -# | | +-------+ +-------+ | | -# +-------+ +-------+ - -# Nodes -[type=openswitch name="Switch 1"] sw1 -[type=openswitch name="Switch 2"] sw2 -[type=host name="Host 1"] hs1 -[type=host name="Host 2"] hs2 - -# Links -hs1:1 -- sw1:3 -sw1:4 -- sw2:3 -sw2:4 -- hs2:1 -""" - - -def test_ping(topology): - """ - Set network addresses and static routes between nodes and ping h2 from h1. - """ - sw1 = topology.get('sw1') - sw2 = topology.get('sw2') - hs1 = topology.get('hs1') - hs2 = topology.get('hs2') - - assert sw1 is not None - assert sw2 is not None - assert hs1 is not None - assert hs2 is not None - - # Configure IP and bring UP host 1 interfaces - hs1.libs.ip.interface('1', addr='10.0.10.1/24', up=True) - - # Configure IP and bring UP host 2 interfaces - hs2.libs.ip.interface('1', addr='10.0.30.1/24', up=True) - - # Configure IP and bring UP switch 1 interfaces - with sw1.libs.vtysh.ConfigInterface('3') as ctx: - ctx.ip_address('10.0.10.2/24') - ctx.no_shutdown() - - with sw1.libs.vtysh.ConfigInterface('4') as ctx: - ctx.ip_address('10.0.20.1/24') - ctx.no_shutdown() - - # Configure IP and bring UP switch 2 interfaces - with sw2.libs.vtysh.ConfigInterface('3') as ctx: - ctx.ip_address('10.0.20.2/24') - ctx.no_shutdown() - - with sw2.libs.vtysh.ConfigInterface('4') as ctx: - ctx.ip_address('10.0.30.2/24') - ctx.no_shutdown() - - # Wait until interfaces are up - for switch, portlbl in [(sw1, '3'), (sw1, '4'), (sw2, '3'), (sw2, '4')]: - wait_until_interface_up(switch, portlbl) - - # Set static routes in switches - sw1.libs.ip.add_route('10.0.30.0/24', '10.0.20.2', shell='bash_swns') - sw2.libs.ip.add_route('10.0.10.0/24', '10.0.20.1', shell='bash_swns') - - # Set gateway in hosts - hs1.libs.ip.add_route('default', '10.0.10.2') - hs2.libs.ip.add_route('default', '10.0.30.2') - - sleep(1) - ping = hs1.libs.ping.ping(1, '10.0.30.1') - assert ping['transmitted'] == ping['received'] == 1 diff --git a/test/test_port_labels.py b/test/test_port_labels.py deleted file mode 100644 index 3bfe301..0000000 --- a/test/test_port_labels.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -OpenSwitch Test for vlan related configurations. -""" - -from __future__ import unicode_literals, absolute_import -from __future__ import print_function, division - -from time import sleep - - -TOPOLOGY = """ -# +-------+ +-------+ -# | | +--------+ | | -# | hs1 <-----> ops1 <-----> hs2 | -# | | +--------+ | | -# +-------+ +-------+ - -# Nodes -[type=openswitch name="OpenSwitch 1"] ops1 -[type=host name="Host 1"] hs1 -[type=host name="Host 2"] hs2 - -# Links -hs1:if01 -- ops1:if01 -ops1:IF02 -- hs2:if01 -""" - - -def test_topology_nodes_openvswitch_port_labels(topology): - ops1 = topology.get('ops1') - hs1 = topology.get('hs1') - hs2 = topology.get('hs2') - - assert ops1 is not None - assert hs1 is not None - assert hs2 is not None - - ops1('configure terminal') - ops1('interface ' + str(ops1.ports['if01'])) - ops1('no shutdown') - ops1('end') - sleep(5) - result = ops1('show interface ' + str(ops1.ports['if01'])) - assert result != '% Unknown command.' diff --git a/test/test_topology_docker_openswitch.py b/test/test_topology_docker_openswitch.py new file mode 100644 index 0000000..b9d196d --- /dev/null +++ b/test/test_topology_docker_openswitch.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Test suite for module topology_docker_openswitch. +""" + +from __future__ import unicode_literals, absolute_import +from __future__ import print_function, division + +from topology import __version__ + + +def setup_module(module): + print('setup_module({})'.format(module.__name__)) + + +def teardown_module(module): + print('teardown_module({})'.format(module.__name__)) + + +def test_semantic_version(): + """ + Check that version follows the Semantic Versioning 2.0.0 specification. + + http://semver.org/ + """ + mayor, minor, rev = map(int, __version__.split('.')) + + assert mayor >= 0 + assert minor >= 0 + assert rev >= 0 diff --git a/test/test_vlan.py b/test/test_vlan.py deleted file mode 100644 index 3d7d149..0000000 --- a/test/test_vlan.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2015-2016 Hewlett Packard Enterprise Development LP -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -OpenSwitch Test for vlan related configurations. -""" - -from __future__ import unicode_literals, absolute_import -from __future__ import print_function, division - -from .helpers import wait_until_interface_up - - -TOPOLOGY = """ -# +-------+ +-------+ -# | | +--------+ | | -# | hs1 <-----> ops1 <-----> hs2 | -# | | +--------+ | | -# +-------+ +-------+ - -# Nodes -[type=openswitch name="OpenSwitch 1"] ops1 -[type=host name="Host 1"] hs1 -[type=host name="Host 2"] hs2 - -# Links -hs1:1 -- ops1:7 -ops1:8 -- hs2:1 -""" - - -def test_vlan(topology): - """ - Test that a vlan configuration is functional with a OpenSwitch switch. - - Build a topology of one switch and two hosts and connect the hosts to the - switch. Setup a VLAN for the ports connected to the hosts and ping from - host 1 to host 2. - """ - ops1 = topology.get('ops1') - hs1 = topology.get('hs1') - hs2 = topology.get('hs2') - - assert ops1 is not None - assert hs1 is not None - assert hs2 is not None - - p7 = ops1.ports['7'] - p8 = ops1.ports['8'] - - # Mark interfaces as enabled - # Note: It is possible that this test fails here with - # pexpect.exceptions.TIMEOUT. There not much we can do, OpenSwitch - # may have a race condition or something that makes this command to - # freeze or to take more than 60 seconds to complete. - iface_enabled = ops1( - 'set interface {p7} user_config:admin=up'.format(**locals()), - shell='vsctl' - ) - assert not iface_enabled - - iface_enabled = ops1( - 'set interface {p8} user_config:admin=up'.format(**locals()), - shell='vsctl' - ) - assert not iface_enabled - - # Configure interfaces - with ops1.libs.vtysh.ConfigInterface('7') as ctx: - ctx.no_routing() - ctx.no_shutdown() - - with ops1.libs.vtysh.ConfigInterface('8') as ctx: - ctx.no_routing() - ctx.no_shutdown() - - # Configure vlan and switch interfaces - with ops1.libs.vtysh.ConfigVlan('8') as ctx: - ctx.no_shutdown() - - with ops1.libs.vtysh.ConfigInterface('7') as ctx: - ctx.vlan_access('8') - - with ops1.libs.vtysh.ConfigInterface('8') as ctx: - ctx.vlan_access('8') - - # Wait until interfaces are up - for portlbl in ['7', '8']: - wait_until_interface_up(ops1, portlbl) - - # Assert vlan status - vlan_status = ops1.libs.vtysh.show_vlan('8').get('8') - assert vlan_status is not None - assert vlan_status['vlan_id'] == '8' - assert vlan_status['status'] == 'up' - assert vlan_status['reason'] == 'ok' - assert sorted(vlan_status['ports']) == [p7, p8] - - # Configure host interfaces - hs1.libs.ip.interface('1', addr='10.0.10.1/24', up=True) - hs2.libs.ip.interface('1', addr='10.0.10.2/24', up=True) - - # Test ping - ping = hs1.libs.ping.ping(1, '10.0.10.2') - assert ping['transmitted'] == ping['received'] == 1 From 0656c0256d4dcd2608c5e1fe8478f797aa7b2f15 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 22 Nov 2016 21:13:33 -0600 Subject: [PATCH 23/55] chg: dev: Bumping version to 1.0.0. --- .cookiecutter.json | 2 +- README.rst | 14 ++++++++++++++ lib/topology_docker_openswitch/__init__.py | 2 +- requirements.txt | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index 6c61e83..bd89895 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "0.1.0" + "version": "1.0.0" } diff --git a/README.rst b/README.rst index c49510a..7b8d150 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,20 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.0.0 +----- + +**Fixes** + +- Removing shell echo. + +0.1.0 +----- + +**New** + +- Initial release. + Documentation ============= diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index 796ad72..950c946 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '0.1.0' +__version__ = '1.0.0' diff --git a/requirements.txt b/requirements.txt index 2a437bf..011b309 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ six -topology_docker>=1.5.0 +topology_docker From af6cd9ea7053a0eab9dc691af49ac1e3709a477b Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 6 Dec 2016 13:25:01 -0600 Subject: [PATCH 24/55] fix: doc: Documenting exit from vtysh. --- doc/user.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/user.rst b/doc/user.rst index dc32aae..dbbb353 100644 --- a/doc/user.rst +++ b/doc/user.rst @@ -65,6 +65,9 @@ prompt will remain in its standard value. Be aware that in order for the node to detect the ``Segmentation fault`` error message, the ``vytsh`` shell is started with ``stdbuf -oL vtysh``. +Before the node is destroyed at the end of its life, this shell will be exited +by sending the ``end`` and ``exit`` commands. + The Booting Process =================== From cc2328f74c6a2208c8acb73c4c363ffd315742ac Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 6 Dec 2016 13:26:12 -0600 Subject: [PATCH 25/55] chg: dev: Bumping version to 1.0.1. --- .cookiecutter.json | 2 +- README.rst | 6 ++++++ lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index bd89895..7b053c4 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.0.0" + "version": "1.0.1" } diff --git a/README.rst b/README.rst index 7b8d150..b312a91 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,12 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.0.1 +----- + +**Fixes** + +- Documenting vtysh exit. 1.0.0 ----- diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index 950c946..63dbc68 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.0.0' +__version__ = '1.0.1' From 90c6bee287cf94072589b2732c2ea7952395de52 Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Fri, 9 Dec 2016 13:47:09 -0800 Subject: [PATCH 26/55] fix: dev: Replaced search for findall. --- doc/conf.py | 1 + lib/topology_docker_openswitch/openswitch_setup | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index e020aa1..26de76a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -310,6 +310,7 @@ def setup(app): app.add_stylesheet('styles/custom.css') + # autoapi configuration autoapi_modules = { 'topology_docker_openswitch': None diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index 820c593..49669d9 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -15,10 +15,10 @@ from json import dumps, loads from shlex import split as shsplit from subprocess import check_call, check_output, call, CalledProcessError from socket import AF_UNIX, SOCK_STREAM, socket, gethostname -from re import search, MULTILINE +from re import findall, MULTILINE from yaml import load -config_timeout = 300 +config_timeout = 1200 ops_switchd_active_timeout = 60 swns_netns = '/var/run/netns/swns' emulns_netns = '/var/run/netns/emulns' @@ -128,7 +128,7 @@ def create_interfaces(): r' manipulation\s*\nRuntimeCmd:\s*\nRuntimeCmd:\s*$' ) - if search(regex, out, MULTILINE) is None: + if findall(regex, out, MULTILINE) is None: raise Exception( 'Control utility for runtime P4 table failed.' ) From 6a2cb7bc2b4c3932f442757bff2f21b099b5a271 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 9 Dec 2016 17:47:32 -0600 Subject: [PATCH 27/55] chg: dev: Bumping version to 1.0.2. --- .cookiecutter.json | 2 +- README.rst | 7 +++++++ lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index 7b053c4..33db165 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.0.1" + "version": "1.0.2" } diff --git a/README.rst b/README.rst index b312a91..88eff82 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,13 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.0.2 +----- + +**Fixes** + +- Fixing wrong regular expression search. + 1.0.1 ----- diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index 63dbc68..81d1fae 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.0.1' +__version__ = '1.0.2' From fc8568c75112b18dacd687bb4fbe45495ba23aae Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 13 Dec 2016 13:11:41 -0600 Subject: [PATCH 28/55] chg: dev: Updating regex to match operator levels. --- doc/user.rst | 2 +- lib/topology_docker_openswitch/openswitch_setup | 1 + lib/topology_docker_openswitch/shell.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/user.rst b/doc/user.rst index dbbb353..f3ab46b 100644 --- a/doc/user.rst +++ b/doc/user.rst @@ -57,7 +57,7 @@ expression if you want to match with any context that has this forced prompt: :: - r'(\r\n)?X@~~==::VTYSH_PROMPT::==~~@X(\([-\w\s]+\))?# ' + r'(\r\n)?X@~~==::VTYSH_PROMPT::==~~@X(\([-\w\s]+\))?[#>] ' If ``set prompt`` is not available, the echo will not be disabled and the prompt will remain in its standard value. diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index 49669d9..694bc1d 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -308,5 +308,6 @@ def main(): 'Failed to start restd: {}'.format(e.output) ) + if __name__ == '__main__': main() diff --git a/lib/topology_docker_openswitch/shell.py b/lib/topology_docker_openswitch/shell.py index 95cc892..f69d3f0 100644 --- a/lib/topology_docker_openswitch/shell.py +++ b/lib/topology_docker_openswitch/shell.py @@ -31,7 +31,7 @@ from topology_docker.shell import DockerShell -_VTYSH_PROMPT_TPL = r'(\r\n)?{}(\([-\w\s]+\))?# ' +_VTYSH_PROMPT_TPL = r'(\r\n)?{}(\([-\w\s]+\))?[#>] ' _VTYSH_FORCED = 'X@~~==::VTYSH_PROMPT::==~~@X' # This is a regular expression that matches with values that may be found in # unset vtysh prompts: From 6aecc7a51b8b64cb9b50858dfdc3775d997ec3c0 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Mon, 19 Dec 2016 14:16:39 -0600 Subject: [PATCH 29/55] chg: dev: Making this work with topology_openswitch. --- lib/topology_docker_openswitch/shell.py | 94 ++++--------------------- requirements.dev.txt | 4 +- requirements.txt | 1 - 3 files changed, 15 insertions(+), 84 deletions(-) diff --git a/lib/topology_docker_openswitch/shell.py b/lib/topology_docker_openswitch/shell.py index f69d3f0..ce34ad4 100644 --- a/lib/topology_docker_openswitch/shell.py +++ b/lib/topology_docker_openswitch/shell.py @@ -23,27 +23,20 @@ from __future__ import print_function, division from logging import warning -from re import match, search from pexpect import EOF -from topology.platforms.shell import PExpectBashShell from topology_docker.shell import DockerShell +from topology_openswitch.vtysh import ( + BASH_FORCED_PROMPT, + VTYSH_FORCED_PROMPT, + VTYSH_STANDARD_PROMPT, + VtyshShellMixin +) -_VTYSH_PROMPT_TPL = r'(\r\n)?{}(\([-\w\s]+\))?[#>] ' -_VTYSH_FORCED = 'X@~~==::VTYSH_PROMPT::==~~@X' -# This is a regular expression that matches with values that may be found in -# unset vtysh prompts: -_VTYSH_STANDARD = '[-\w]+' -VTYSH_FORCED_PROMPT = _VTYSH_PROMPT_TPL.format(_VTYSH_FORCED) -VTYSH_STANDARD_PROMPT = _VTYSH_PROMPT_TPL.format(_VTYSH_STANDARD) - -BASH_FORCED_PROMPT = PExpectBashShell.FORCED_PROMPT - - -class OpenSwitchVtyshShell(DockerShell): +class OpenSwitchVtyshShell(DockerShell, VtyshShellMixin): """ OpenSwitch ``vtysh`` shell @@ -125,50 +118,10 @@ def _setup_shell(self, connection=None): spawn.sendline('export PS1={}'.format(BASH_FORCED_PROMPT)) spawn.expect(BASH_FORCED_PROMPT) - def determine_set_prompt(): - """ - This method determines if the vtysh command set prompt exists. - - This method starts wit a call to sendline and finishes with a call - to expect. - - :rtype: bool - :return: True if vtysh supports the ``set prompt`` command, False - otherwise. - """ - # When a segmentation fault error happens, the message - # "Segmentation fault" shows up in the terminal and then and EOF - # follows, making the vtysh shell to close ending up in the bash - # shell that opened it. - - # This starts the vtysh shell in a mode that forces the shell to - # always return the produced output, even if an EOF exception - # follows after it. This is done to handle the segmentation fault - # errors. - spawn.sendline('stdbuf -oL vtysh') - spawn.expect(VTYSH_STANDARD_PROMPT) - - # The newer images of OpenSwitch include this command that changes - # the prompt of the shell to an unique value. This is done to - # perform a safe matching that will match only with this value in - # each expect. - spawn.sendline('set prompt {}'.format(_VTYSH_FORCED)) - - # Since it is not possible to know beforehand if the image loaded - # in the node includes the "set prompt" command, an attempt to - # match any of the following prompts is done. If the command does - # not exist, the shell will return an standard prompt after showing - # an error message. - index = spawn.expect( - [VTYSH_STANDARD_PROMPT, VTYSH_FORCED_PROMPT] - ) - - return bool(index) - def join_prompt(prompt): return '{}|{}'.format(BASH_FORCED_PROMPT, prompt) - if determine_set_prompt(): + if self._determine_set_prompt(): # If this image supports "set prompt", then exit back to bash to # set the bash shell without echo. spawn.sendline('exit') @@ -182,11 +135,11 @@ def join_prompt(prompt): # Go into the vtysh shell again. Exiting vtysh after calling "set # prompt" successfully disables the vtysh shell prompt to its # standard value, so it is necessary to call it again. - determine_set_prompt() + self._determine_set_prompt() # From now on the shell _prompt attribute is set to the defined # vtysh forced prompt. - self._prompt = join_prompt(VTYSH_FORCED_PROMPT) + self._prompt = [BASH_FORCED_PROMPT, VTYSH_FORCED_PROMPT] else: # If the image does not support "set prompt", then enable the @@ -196,7 +149,7 @@ def join_prompt(prompt): # From now on the shell _prompt attribute is set to the defined # vtysh standard prompt. - self._prompt = join_prompt(VTYSH_STANDARD_PROMPT) + self._prompt = [BASH_FORCED_PROMPT, VTYSH_STANDARD_PROMPT] # This sendline is used here just because a _setup_shell must end in an # send/sendline call since it is followed by a call to expect in the @@ -214,29 +167,8 @@ def send_command( connection=connection, silent=silent ) - spawn = self._get_connection(connection) - - # To find out if a segmentation fault error was produced, a search for - # the "Segmentation fault" string in the output of the command is done. - segmentation_fault = search( - r'Segmentation fault', self.get_response(silent=True) - ) - - # The other necessary condition to detect a segmentation fault error is - # to detect a forced bash prompt being matched. - forced_bash_prompt = match( - BASH_FORCED_PROMPT, spawn.after.decode( - encoding=self._encoding, errors=self._errors - ) - ) - - # This exception is raised to provide a meaningful error to the user. - if segmentation_fault is not None and forced_bash_prompt is not None: - raise Exception( - 'Segmentation fault received when executing "{}".'.format( - self._last_command - ) - ) + # This will raise a proper exception if a crash has been found. + self._handle_crash(connection) return match_index diff --git a/requirements.dev.txt b/requirements.dev.txt index ec7568a..f7c4093 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -9,5 +9,5 @@ autoapi topology_lib_ping topology_lib_ip --e git+https://github.com/HPENetworking/topology_docker.git@master#egg=topology_docker --e git+https://git.openswitch.net/openswitch/ops-topology-lib-vtysh@8c6af11bf84813324a6518ccda101712297cd357#egg=topology_lib_vtysh +-e git+https://github.com/saenzpa/topology_docker.git@master#egg=topology_docker +-e git+file:///home/ocelotl/HPENetworking/topology_openswitch@master#egg=topology_openswitch diff --git a/requirements.txt b/requirements.txt index 011b309..ffe2fce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ six -topology_docker From f23a580aa39f916c9e93270be17d944ee18d65fb Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 20 Dec 2016 16:34:20 -0600 Subject: [PATCH 30/55] new: dev: Adding capabilities. --- lib/topology_docker_openswitch/openswitch.py | 75 ++++++++++++++++++++ lib/topology_docker_openswitch/shell.py | 29 ++------ 2 files changed, 79 insertions(+), 25 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index d3ec692..5ce86a4 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -30,6 +30,7 @@ from logging import StreamHandler, getLogger, INFO, Formatter from sys import stdout from os.path import join, dirname, normpath, abspath +from argparse import Namespace from topology_docker.node import DockerNode from topology_docker.shell import DockerBashShell @@ -49,6 +50,74 @@ LOG.setLevel(INFO) +class MissingCapacityError(Exception): + def __init__(self, capacity): + self._capacity = capacity + + def __str__(self): + return 'Missing capacity \'{}\''.format(self._capacity) + + +class TransactNamespace(Namespace): + """ + Provides a common call to ``ovsdb-client transact`` + """ + + def _transact(self, node, columns, op='select', table='System', where=''): + bash = node.get_shell('bash') + bash.send_command( + 'ovsdb-client transact \'["OpenSwitch", ' + '{{"op":"{}","table":"{}","where":[{}],' + '"columns":["{}"]}}]\''.format(op, table, where, columns), + silent=True + ) + return loads(bash.get_response(silent=True)) + + +class Capabilities(TransactNamespace): + """ + Represent the switch capabilities. + + This is a namespace that holds all defined capabilities. Each one of them + will be set to True, if an attribute outside the defined capabilities is + accessed, it will return False. This will allow the test engineer to try to + find a defined capability by querying against the requested attribute. + """ + + def __init__(self, node): + capabilities = self._transact( + node, 'capabilities' + )[0]['rows'][0]['capabilities'][1] + + super(Capabilities, self).__init__( + **{capability: True for capability in capabilities} + ) + + def __getattr__(self, name): + return False + + +class Capacities(TransactNamespace): + """ + Represent the switch capacities + + This class will raise a MissingCapacityError if an attempt to access a + non-existing capacity is made. + """ + + def __init__(self, node): + capacities = self._transact( + node, 'capacities' + )[0]['rows'][0]['capacities'][1] + + super(Capacities, self).__init__( + **{capacity: value for capacity, value in capacities} + ) + + def __getattr__(self, name): + raise MissingCapacityError(name) + + def log_commands( commands, location, function, escape=True, prefix=None, suffix=None, **kwargs @@ -265,6 +334,12 @@ def _setup_system(self): LOG_PATHS.append(self.shared_dir) raise e + + # Add capabilities + + self.capabilities = Capabilities(self) + self.capacities = Capacities(self) + # Read back port mapping port_mapping = '{}/port_mapping.json'.format(self.shared_dir) with open(port_mapping, 'r') as fd: diff --git a/lib/topology_docker_openswitch/shell.py b/lib/topology_docker_openswitch/shell.py index ce34ad4..ed68cb8 100644 --- a/lib/topology_docker_openswitch/shell.py +++ b/lib/topology_docker_openswitch/shell.py @@ -22,10 +22,6 @@ from __future__ import unicode_literals, absolute_import from __future__ import print_function, division -from logging import warning - -from pexpect import EOF - from topology_docker.shell import DockerShell from topology_openswitch.vtysh import ( @@ -139,7 +135,7 @@ def join_prompt(prompt): # From now on the shell _prompt attribute is set to the defined # vtysh forced prompt. - self._prompt = [BASH_FORCED_PROMPT, VTYSH_FORCED_PROMPT] + self._prompt = '|'.join([BASH_FORCED_PROMPT, VTYSH_FORCED_PROMPT]) else: # If the image does not support "set prompt", then enable the @@ -149,7 +145,9 @@ def join_prompt(prompt): # From now on the shell _prompt attribute is set to the defined # vtysh standard prompt. - self._prompt = [BASH_FORCED_PROMPT, VTYSH_STANDARD_PROMPT] + self._prompt = '|'.join( + [BASH_FORCED_PROMPT, VTYSH_STANDARD_PROMPT] + ) # This sendline is used here just because a _setup_shell must end in an # send/sendline call since it is followed by a call to expect in the @@ -172,24 +170,5 @@ def send_command( return match_index - def _exit(self): - """ - Attempt a clean exit from the shell. - - This is necessary to enable gathering of coverage information. - """ - try: - self.send_command('end', silent=True) - self.send_command( - 'exit', matches=[EOF, BASH_FORCED_PROMPT], - silent=True - ) - except Exception as error: - warning( - 'Exiting the shell failed with this error: {}'.format( - str(error) - ) - ) - __all__ = ['OpenSwitchVtyshShell'] From fba3df83f091cb6fbbfb85015fb747a69196f552 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Wed, 21 Dec 2016 18:39:06 -0600 Subject: [PATCH 31/55] new: dev: Adding detection of build type. --- lib/topology_docker_openswitch/openswitch.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 5ce86a4..07beb24 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -340,6 +340,16 @@ def _setup_system(self): self.capabilities = Capabilities(self) self.capacities = Capacities(self) + # Add virtual type + + vtysh = self.get_shell('vtysh') + + vtysh.send_command('show version', silent=True) + if 'genericx86-64' in vtysh.get_response(silent=True): + self.docker = Namespace(**{'type': 'genericx84-64'}) + else: + self.docker = Namespace(**{'type': 'p4'}) + # Read back port mapping port_mapping = '{}/port_mapping.json'.format(self.shared_dir) with open(port_mapping, 'r') as fd: From 82e0c26ab6373ed466d42396c622e1aae9b640ad Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Thu, 22 Dec 2016 18:53:20 -0600 Subject: [PATCH 32/55] fix: dev: Changing attribute to product_name. --- lib/topology_docker_openswitch/openswitch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 07beb24..69cce56 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -346,9 +346,9 @@ def _setup_system(self): vtysh.send_command('show version', silent=True) if 'genericx86-64' in vtysh.get_response(silent=True): - self.docker = Namespace(**{'type': 'genericx84-64'}) + self.product_name = 'genericx84-64' else: - self.docker = Namespace(**{'type': 'p4'}) + self.product_name = 'genericx86-p4' # Read back port mapping port_mapping = '{}/port_mapping.json'.format(self.shared_dir) From 81f47de098ce11d81386ed1072a3b5a4d8210f13 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 6 Jan 2017 07:13:20 -0600 Subject: [PATCH 33/55] fix: dev: Adapting files. --- lib/topology_docker_openswitch/openswitch.py | 102 ++++-------------- .../openswitch_setup | 2 +- 2 files changed, 23 insertions(+), 81 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 69cce56..e2d55f3 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -24,13 +24,17 @@ from __future__ import unicode_literals, absolute_import from __future__ import print_function, division +from abc import ABCMeta, abstractmethod from json import loads from subprocess import check_output, CalledProcessError from platform import system, linux_distribution from logging import StreamHandler, getLogger, INFO, Formatter from sys import stdout from os.path import join, dirname, normpath, abspath -from argparse import Namespace + +from six import add_metaclass + +from topology_openswitch.openswitch import OpenSwitchBase from topology_docker.node import DockerNode from topology_docker.shell import DockerBashShell @@ -50,74 +54,6 @@ LOG.setLevel(INFO) -class MissingCapacityError(Exception): - def __init__(self, capacity): - self._capacity = capacity - - def __str__(self): - return 'Missing capacity \'{}\''.format(self._capacity) - - -class TransactNamespace(Namespace): - """ - Provides a common call to ``ovsdb-client transact`` - """ - - def _transact(self, node, columns, op='select', table='System', where=''): - bash = node.get_shell('bash') - bash.send_command( - 'ovsdb-client transact \'["OpenSwitch", ' - '{{"op":"{}","table":"{}","where":[{}],' - '"columns":["{}"]}}]\''.format(op, table, where, columns), - silent=True - ) - return loads(bash.get_response(silent=True)) - - -class Capabilities(TransactNamespace): - """ - Represent the switch capabilities. - - This is a namespace that holds all defined capabilities. Each one of them - will be set to True, if an attribute outside the defined capabilities is - accessed, it will return False. This will allow the test engineer to try to - find a defined capability by querying against the requested attribute. - """ - - def __init__(self, node): - capabilities = self._transact( - node, 'capabilities' - )[0]['rows'][0]['capabilities'][1] - - super(Capabilities, self).__init__( - **{capability: True for capability in capabilities} - ) - - def __getattr__(self, name): - return False - - -class Capacities(TransactNamespace): - """ - Represent the switch capacities - - This class will raise a MissingCapacityError if an attempt to access a - non-existing capacity is made. - """ - - def __init__(self, node): - capacities = self._transact( - node, 'capacities' - )[0]['rows'][0]['capacities'][1] - - super(Capacities, self).__init__( - **{capacity: value for capacity, value in capacities} - ) - - def __getattr__(self, name): - raise MissingCapacityError(name) - - def log_commands( commands, location, function, escape=True, prefix=None, suffix=None, **kwargs @@ -155,7 +91,8 @@ def log_commands( ) -class OpenSwitchNode(DockerNode): +@add_metaclass(ABCMeta) +class DockerOpenSwitch(DockerNode, OpenSwitchBase): """ Custom OpenSwitch node for the Topology Docker platform engine. This custom node loads an OpenSwitch image and has vtysh as default @@ -163,6 +100,10 @@ class OpenSwitchNode(DockerNode): See :class:`topology_docker.node.DockerNode`. """ + # FIXME: document shared_dir_mount + _openswitch_attributes = {'shared_dir_mount': ''} + + @abstractmethod def __init__( self, identifier, image='topology/ops:latest', binds=None, @@ -177,7 +118,7 @@ def __init__( if binds is not None: container_binds.append(binds) - super(OpenSwitchNode, self).__init__( + super(DockerOpenSwitch, self).__init__( identifier, image=image, command='/sbin/init', binds=';'.join(container_binds), hostname='switch', network_mode='bridge', environment={'container': 'docker'}, @@ -185,7 +126,7 @@ def __init__( ) # FIXME: Remove this attribute to merge with version > 1.6.0 - self.shared_dir_mount = '/tmp' + self._shared_dir_mount = '/tmp' # Add vtysh (default) shell # This shell is started as a bash shell but it changes itself to a @@ -228,7 +169,7 @@ def notify_post_build(self): See :meth:`DockerNode.notify_post_build` for more information. """ - super(OpenSwitchNode, self).notify_post_build() + super(DockerOpenSwitch, self).notify_post_build() self._setup_system() def _setup_system(self): @@ -335,11 +276,6 @@ def _setup_system(self): raise e - # Add capabilities - - self.capabilities = Capabilities(self) - self.capacities = Capacities(self) - # Add virtual type vtysh = self.get_shell('vtysh') @@ -388,7 +324,13 @@ def stop(self): if isinstance(shell, OpenSwitchVtyshShell): shell._exit() - super(OpenSwitchNode, self).stop() + super(DockerOpenSwitch, self).stop() + + +class OpenSwitch(DockerOpenSwitch): + """ + FIXME: document this + """ -__all__ = ['OpenSwitchNode'] +__all__ = ['DockerOpenSwitch', 'OpenSwitch'] diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index 694bc1d..ce97fe2 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -32,7 +32,7 @@ def create_interfaces(): # Read ports from hardware description with open('{}/ports.yaml'.format(hwdesc_dir), 'r') as fd: ports_hwdesc = load(fd) - hwports = [str(p['name']) for p in ports_hwdesc['ports']] + hwports = ['m1s1p{}'.format(str(p['name'])) for p in ports_hwdesc['ports']] netns = check_output("ls /var/run/netns", shell=True) From 1409fa02234030068a690b853b8247ead596a03e Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 6 Jan 2017 08:09:57 -0600 Subject: [PATCH 34/55] fix: dev: Referring to correct class. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fa2a23e..0b46d07 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def find_requirements(filename): 'pytest11': ['topology_docker_openswitch ' '= topology_docker_openswitch.plugin.plugin'], 'topology_docker_node_10': [ - 'openswitch = topology_docker_openswitch.openswitch:OpenSwitchNode' + 'openswitch = topology_docker_openswitch.openswitch:OpenSwitch' ] } ) From bf9442ed245d2233bde0f1e30395042650764517 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 6 Jan 2017 15:59:15 -0600 Subject: [PATCH 35/55] fix: dev: Calling classes in the right order. --- lib/topology_docker_openswitch/openswitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index e2d55f3..378616c 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -92,7 +92,7 @@ def log_commands( @add_metaclass(ABCMeta) -class DockerOpenSwitch(DockerNode, OpenSwitchBase): +class DockerOpenSwitch(OpenSwitchBase, DockerNode): """ Custom OpenSwitch node for the Topology Docker platform engine. This custom node loads an OpenSwitch image and has vtysh as default From e84a06ae9f2c0b0156c0ebac60dd1ed5c8a6d620 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Mon, 9 Jan 2017 19:16:21 -0600 Subject: [PATCH 36/55] fix: dev: Updating attribute dictionary. --- lib/topology_docker_openswitch/openswitch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 378616c..852cb41 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -101,7 +101,9 @@ class DockerOpenSwitch(OpenSwitchBase, DockerNode): """ # FIXME: document shared_dir_mount - _openswitch_attributes = {'shared_dir_mount': ''} + _openswitch_attributes = OpenSwitchBase._openswitch_attributes.update( + {'shared_dir_mount': ''} + ) @abstractmethod def __init__( From 47697b2df3982fc60ffdbcbdbfb8926e2b782d39 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 10 Jan 2017 11:28:28 -0600 Subject: [PATCH 37/55] fix: dev: Setting right attributes. --- lib/topology_docker_openswitch/openswitch.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 852cb41..b16af40 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -101,9 +101,7 @@ class DockerOpenSwitch(OpenSwitchBase, DockerNode): """ # FIXME: document shared_dir_mount - _openswitch_attributes = OpenSwitchBase._openswitch_attributes.update( - {'shared_dir_mount': ''} - ) + _class_openswitch_attributes = {'shared_dir_mount': ''} @abstractmethod def __init__( From 3bba4086fc71d5edb49d8692f2bb1c1ea3af8698 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 13 Jan 2017 18:28:31 -0600 Subject: [PATCH 38/55] fix: dev: Adding missing __init__. --- lib/topology_docker_openswitch/openswitch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index b16af40..4e7ae5d 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -332,5 +332,8 @@ class OpenSwitch(DockerOpenSwitch): FIXME: document this """ + def __init__(self, *args, **kwargs): + super(OpenSwitch, self).__init__(*args, **kwargs) + __all__ = ['DockerOpenSwitch', 'OpenSwitch'] From 9196c26add6b6d3a80aa90fe7fec1090fbf9310c Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Fri, 13 Jan 2017 18:46:52 -0600 Subject: [PATCH 39/55] fix: dev: Removing port renaming. --- lib/topology_docker_openswitch/openswitch_setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index ce97fe2..694bc1d 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -32,7 +32,7 @@ def create_interfaces(): # Read ports from hardware description with open('{}/ports.yaml'.format(hwdesc_dir), 'r') as fd: ports_hwdesc = load(fd) - hwports = ['m1s1p{}'.format(str(p['name'])) for p in ports_hwdesc['ports']] + hwports = [str(p['name']) for p in ports_hwdesc['ports']] netns = check_output("ls /var/run/netns", shell=True) From 83dfa860b9f1f7f18022edd46d493afa7d5caf53 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Wed, 18 Jan 2017 18:11:43 -0600 Subject: [PATCH 40/55] fix: dev: Fixing requirements. --- requirements.dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index f7c4093..0edfcf8 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -10,4 +10,4 @@ topology_lib_ping topology_lib_ip -e git+https://github.com/saenzpa/topology_docker.git@master#egg=topology_docker --e git+file:///home/ocelotl/HPENetworking/topology_openswitch@master#egg=topology_openswitch +-e git+https://github.com/HPENetworking/topology_openswitch.git@master#egg=topology_openswitch From 1d58cc46346f34ca9149108cd10cadf2b541d6f2 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Wed, 18 Jan 2017 18:22:08 -0600 Subject: [PATCH 41/55] chg: dev: Bumping version to 1.1.0. --- .cookiecutter.json | 2 +- README.rst | 7 +++++++ lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index 33db165..1bd55f0 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.0.2" + "version": "1.1.0" } diff --git a/README.rst b/README.rst index 88eff82..710aebb 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,13 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.1.0 +----- + +**New** + +- Adapting for ``topology_openswitch``. + 1.0.2 ----- diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index 81d1fae..33f8ceb 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.0.2' +__version__ = '1.1.0' From f78c0f3e40a7189b374e909a1c8fc00dc035669e Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Tue, 31 Jan 2017 14:25:44 -0800 Subject: [PATCH 42/55] fix: dev: Parametrize setup script path. --- lib/topology_docker_openswitch/openswitch.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 4e7ae5d..736dfb5 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -162,29 +162,36 @@ def __init__( ) ) - def notify_post_build(self): + def notify_post_build(self, script_path=None): """ Get notified that the post build stage of the topology build was reached. + :param script_path: + string with the path of the setup script to be used + See :meth:`DockerNode.notify_post_build` for more information. """ super(DockerOpenSwitch, self).notify_post_build() - self._setup_system() + self._setup_system(script_path) - def _setup_system(self): + def _setup_system(self, script_path=None): """ Setup the OpenSwitch image for testing. #. Wait for daemons to converge. #. Assign an interface to each port label. #. Create remaining interfaces. + :param script_path: + string with the path of the setup script to be used """ # Write and execute setup script - with open( - join(dirname(normpath(abspath(__file__))), 'openswitch_setup') - ) as openswitch_setup_file: + openswitch_setup_path = script_path or join( + dirname(normpath(abspath(__file__))), 'openswitch_setup' + ) + + with open(openswitch_setup_path) as openswitch_setup_file: openswitch_setup = openswitch_setup_file.read() setup_script = '{}/openswitch_setup.py'.format(self.shared_dir) From 53bf52b8693adfb3a2a5b664a30186b6a881b63c Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Mon, 6 Feb 2017 21:12:16 -0600 Subject: [PATCH 43/55] fix: dev: Refactoring to remove race conditions. --- .../openswitch_setup | 252 ++++++++++-------- .../{plugin => pytest}/__init__.py | 0 .../{plugin => pytest}/plugin.py | 29 +- setup.py | 6 +- 4 files changed, 173 insertions(+), 114 deletions(-) rename lib/topology_docker_openswitch/{plugin => pytest}/__init__.py (100%) rename lib/topology_docker_openswitch/{plugin => pytest}/plugin.py (86%) diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index 694bc1d..0b14605 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -17,6 +17,7 @@ from subprocess import check_call, check_output, call, CalledProcessError from socket import AF_UNIX, SOCK_STREAM, socket, gethostname from re import findall, MULTILINE from yaml import load +from functools import partial config_timeout = 1200 ops_switchd_active_timeout = 60 @@ -28,27 +29,83 @@ switchd_pid = '/var/run/openvswitch/ops-switchd.pid' sock = None +def wait_check(function, wait_name, wait_error, *args): + info('Waiting for {}'.format(wait_name)) + + for i in range(0, config_timeout): + if not function(*args): + sleep(0.1) + else: + break + else: + raise Exception( + 'The image did not boot correctly, ' + '{} after waiting {} seconds.'.format( + wait_error, int(0.1 * config_timeout) + ) + ) + + +def start_restd(): + output = '' + try: + output = check_output( + 'systemctl status restd', shell=True + ) + except CalledProcessError as e: + pass + + if 'Active: active' not in output: + try: + def restd_started(): + """ + Attempt to start restd and report back if successful. + + :rtype: bool + :return: True if restd was started, False otherwise + """ + output = '' + try: + check_output( + 'systemctl start restd', shell=True + ) + output = check_output( + 'systemctl status restd', shell=True + ) + except CalledProcessError: + pass + + return 'Active: active' in output + + wait_check( + restd_started, 'restd service', 'restd did not start' + ) + + except CalledProcessError as e: + raise Exception( + 'Failed to start restd: {}'.format(e.output) + ) + + def create_interfaces(): # Read ports from hardware description with open('{}/ports.yaml'.format(hwdesc_dir), 'r') as fd: ports_hwdesc = load(fd) hwports = [str(p['name']) for p in ports_hwdesc['ports']] - netns = check_output("ls /var/run/netns", shell=True) + netns = check_output('ls /var/run/netns', shell=True) # Get list of already created ports - not_in_netns = check_output(shsplit( - 'ls /sys/class/net/' - )).split() - - if "emulns" not in netns: - in_netns = check_output(shsplit( - 'ip netns exec swns ls /sys/class/net/' - )).split() + not_in_netns = check_output(shsplit('ls /sys/class/net/')).split() + + if 'emulns' not in netns: + in_netns = check_output( + shsplit('ip netns exec swns ls /sys/class/net/') + ).split() else: - in_netns = check_output(shsplit( - 'ip netns exec emulns ls /sys/class/net/' - )).split() + in_netns = check_output( + shsplit('ip netns exec emulns ls /sys/class/net/') + ).split() info('Not in swns/emulns: {not_in_netns} '.format(**locals())) info('In swns/emulns {in_netns} '.format(**locals())) @@ -97,17 +154,14 @@ def create_interfaces(): shell=True ) - for i in range(0, config_timeout): - link_state = check_output( + wait_check( + lambda: 'UP' in check_output( '{ns_exec} ip link show {hwport}'.format(**locals()), shell=True - ) - if "UP" in link_state: - break - else: - sleep(0.1) - else: - raise Exception('emulns interface did not came up.') + ), + '{} to show up'.format(hwport), + '{} did not show up'.format(hwport) + ) out = check_output( '{ns_exec} echo port_add {hwport} ' @@ -130,7 +184,8 @@ def create_interfaces(): if findall(regex, out, MULTILINE) is None: raise Exception( - 'Control utility for runtime P4 table failed.' + 'Control utility for runtime' + ' P4 table failed: {}'.format(out) ) except CalledProcessError as error: @@ -205,9 +260,11 @@ def cur_is_set(cur_key): } global sock + if sock is None: sock = socket(AF_UNIX, SOCK_STREAM) sock.connect(db_sock) + sock.send(dumps(queries[cur_key])) response = loads(sock.recv(4096)) @@ -217,9 +274,67 @@ def cur_is_set(cur_key): return 0 -def ops_switchd_is_active(): - is_active = call(["systemctl", "is-active", "switchd.service"]) - return is_active == 0 +SWNS_CHECK = partial( + wait_check, + exists, + swns_netns, + '{} was not present'.format(swns_netns), + swns_netns +) + +HWDESC_CHECK = partial( + wait_check, + exists, + hwdesc_dir, + '{} was not present'.format(hwdesc_dir), + hwdesc_dir +) + +SWITCHD_CHECK = partial( + wait_check, + exists, + switchd_pid, + '{} was not present'.format(switchd_pid), + switchd_pid +) + +DB_SOCK_CHECK = partial( + wait_check, + exists, + db_sock, + '{} was not present'.format(db_sock), + db_sock +) + +CUR_HW_CHECK = partial( + wait_check, + cur_is_set, + 'cur_hw to be set to 1', + 'cur_hw is not set to 1', + 'cur_hw' +) + +CUR_CFG_CHECK = partial( + wait_check, + cur_is_set, + 'cur_hw to be set to 1', + 'cur_hw is not set to 1', + 'cur_hw' +) + +OPS_SWITCHD_CHECK = partial( + wait_check, + lambda: 0 == call(['systemctl', 'is-active', 'switchd.service']), + 'ops-switchd to be active', + 'ops-switchd was not active' +) + +HOSTNAME_CHECK = partial( + wait_check, + lambda: gethostname() == 'switch', + 'final hostname', + 'hostname was not set' +) def main(): @@ -227,86 +342,15 @@ def main(): if '-d' in argv: basicConfig(level=DEBUG) - def wait_check(function, wait_name, wait_error, *args): - info('Waiting for {}'.format(wait_name)) - - for i in range(0, config_timeout): - if not function(*args): - sleep(0.1) - else: - break - else: - raise Exception( - 'The image did not boot correctly, ' - '{} after waiting {} seconds.'.format( - wait_error, int(0.1 * config_timeout) - ) - ) - - wait_check( - exists, swns_netns, '{} was not present'.format(swns_netns), swns_netns - ) - wait_check( - exists, hwdesc_dir, '{} was not present'.format(hwdesc_dir), hwdesc_dir - ) - - info('Creating interfaces') + SWNS_CHECK() + HWDESC_CHECK() + SWITCHD_CHECK() create_interfaces() - - wait_check( - exists, db_sock, '{} was not present'.format(db_sock), db_sock - ) - wait_check( - cur_is_set, 'cur_hw to be set to 1', 'cur_hw is not set to 1', - 'cur_hw' - ) - wait_check( - cur_is_set, 'cur_cfg to be set to 1', 'cur_cfg is not set to 1', - 'cur_cfg' - ) - wait_check( - exists, switchd_pid, '{} was not present'.format(switchd_pid), - switchd_pid - ) - wait_check( - ops_switchd_is_active, 'ops-switchd to be active', - 'ops-switchd was not active' - ) - wait_check( - lambda: gethostname() == 'switch', 'final hostname', - 'hostname was not set' - ) - - info('Checking restd service status...') - output = '' - try: - output = check_output( - 'systemctl status restd', shell=True - ) - except CalledProcessError as e: - pass - if 'Active: active' not in output: - try: - info('Starting restd daemon.') - check_output('systemctl start restd', shell=True) - - info('Checking restd service started.') - for i in range(0, config_timeout): - output = '' - output = check_output( - 'systemctl status restd', shell=True - ) - if 'Active: active' not in output: - sleep(0.1) - else: - break - else: - raise Exception("Failed to start restd service") - - except CalledProcessError as e: - raise Exception( - 'Failed to start restd: {}'.format(e.output) - ) + DB_SOCK_CHECK() + CUR_HW_CHECK() + CUR_CFG_CHECK() + OPS_SWITCHD_CHECK() + HOSTNAME_CHECK() if __name__ == '__main__': diff --git a/lib/topology_docker_openswitch/plugin/__init__.py b/lib/topology_docker_openswitch/pytest/__init__.py similarity index 100% rename from lib/topology_docker_openswitch/plugin/__init__.py rename to lib/topology_docker_openswitch/pytest/__init__.py diff --git a/lib/topology_docker_openswitch/plugin/plugin.py b/lib/topology_docker_openswitch/pytest/plugin.py similarity index 86% rename from lib/topology_docker_openswitch/plugin/plugin.py rename to lib/topology_docker_openswitch/pytest/plugin.py index 0331fcd..c55354d 100644 --- a/lib/topology_docker_openswitch/plugin/plugin.py +++ b/lib/topology_docker_openswitch/pytest/plugin.py @@ -34,13 +34,26 @@ def pytest_runtest_teardown(item): FIXME: document the item argument """ test_suite = splitext(basename(item.parent.name))[0] - path_name = '/tmp/topology/docker/{}_{}_{}'.format( - test_suite, item.name, datetime.now().strftime('%Y_%m_%d_%H_%M_%S') - ) - # Being extra-prudent here - if exists(path_name): - rmtree(path_name) + from pytest import config + + topology_log_dir = config.getoption('--topology-log-dir') + + if not topology_log_dir: + return + else: + path_name = join( + topology_log_dir, + '{}_{}_{}'.format( + test_suite, + item.name, + datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + ) + ) + + # Being extra-prudent here + if exists(path_name): + rmtree(path_name) if 'topology' not in item.funcargs: from topology_docker_openswitch.openswitch import LOG_PATHS @@ -99,14 +112,14 @@ def pytest_runtest_teardown(item): core_path = '/var/diagnostics/coredump' bash_shell.send_command( - 'ls -1 {}/core.* 2>/dev/null'.format(core_path), silent=True + 'ls -1 {}/core* 2>/dev/null'.format(core_path), silent=True ) core_files = bash_shell.get_response(silent=True).splitlines() for core_file in core_files: bash_shell.send_command( - 'cp {core_path}/{core_file} /tmp'.format(**locals()), + 'cp {core_file} /tmp'.format(**locals()), silent=True ) except: diff --git a/setup.py b/setup.py index 0b46d07..a635950 100755 --- a/setup.py +++ b/setup.py @@ -93,8 +93,10 @@ def find_requirements(filename): # Entry points entry_points={ - 'pytest11': ['topology_docker_openswitch ' - '= topology_docker_openswitch.plugin.plugin'], + 'pytest11': [ + 'topology_docker_openswitch ' + '= topology_docker_openswitch.pytest.plugin' + ], 'topology_docker_node_10': [ 'openswitch = topology_docker_openswitch.openswitch:OpenSwitch' ] From d723b22fb0055cb5321fe29e8b1a13a738d766f3 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Sat, 11 Feb 2017 09:15:19 -0600 Subject: [PATCH 44/55] Revert "fix: dev: Refactoring to remove race conditions." This reverts commit 53bf52b8693adfb3a2a5b664a30186b6a881b63c. --- .../openswitch_setup | 252 ++++++++---------- .../{pytest => plugin}/__init__.py | 0 .../{pytest => plugin}/plugin.py | 29 +- setup.py | 6 +- 4 files changed, 114 insertions(+), 173 deletions(-) rename lib/topology_docker_openswitch/{pytest => plugin}/__init__.py (100%) rename lib/topology_docker_openswitch/{pytest => plugin}/plugin.py (86%) diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index 0b14605..694bc1d 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -17,7 +17,6 @@ from subprocess import check_call, check_output, call, CalledProcessError from socket import AF_UNIX, SOCK_STREAM, socket, gethostname from re import findall, MULTILINE from yaml import load -from functools import partial config_timeout = 1200 ops_switchd_active_timeout = 60 @@ -29,83 +28,27 @@ switchd_pid = '/var/run/openvswitch/ops-switchd.pid' sock = None -def wait_check(function, wait_name, wait_error, *args): - info('Waiting for {}'.format(wait_name)) - - for i in range(0, config_timeout): - if not function(*args): - sleep(0.1) - else: - break - else: - raise Exception( - 'The image did not boot correctly, ' - '{} after waiting {} seconds.'.format( - wait_error, int(0.1 * config_timeout) - ) - ) - - -def start_restd(): - output = '' - try: - output = check_output( - 'systemctl status restd', shell=True - ) - except CalledProcessError as e: - pass - - if 'Active: active' not in output: - try: - def restd_started(): - """ - Attempt to start restd and report back if successful. - - :rtype: bool - :return: True if restd was started, False otherwise - """ - output = '' - try: - check_output( - 'systemctl start restd', shell=True - ) - output = check_output( - 'systemctl status restd', shell=True - ) - except CalledProcessError: - pass - - return 'Active: active' in output - - wait_check( - restd_started, 'restd service', 'restd did not start' - ) - - except CalledProcessError as e: - raise Exception( - 'Failed to start restd: {}'.format(e.output) - ) - - def create_interfaces(): # Read ports from hardware description with open('{}/ports.yaml'.format(hwdesc_dir), 'r') as fd: ports_hwdesc = load(fd) hwports = [str(p['name']) for p in ports_hwdesc['ports']] - netns = check_output('ls /var/run/netns', shell=True) + netns = check_output("ls /var/run/netns", shell=True) # Get list of already created ports - not_in_netns = check_output(shsplit('ls /sys/class/net/')).split() - - if 'emulns' not in netns: - in_netns = check_output( - shsplit('ip netns exec swns ls /sys/class/net/') - ).split() + not_in_netns = check_output(shsplit( + 'ls /sys/class/net/' + )).split() + + if "emulns" not in netns: + in_netns = check_output(shsplit( + 'ip netns exec swns ls /sys/class/net/' + )).split() else: - in_netns = check_output( - shsplit('ip netns exec emulns ls /sys/class/net/') - ).split() + in_netns = check_output(shsplit( + 'ip netns exec emulns ls /sys/class/net/' + )).split() info('Not in swns/emulns: {not_in_netns} '.format(**locals())) info('In swns/emulns {in_netns} '.format(**locals())) @@ -154,14 +97,17 @@ def create_interfaces(): shell=True ) - wait_check( - lambda: 'UP' in check_output( + for i in range(0, config_timeout): + link_state = check_output( '{ns_exec} ip link show {hwport}'.format(**locals()), shell=True - ), - '{} to show up'.format(hwport), - '{} did not show up'.format(hwport) - ) + ) + if "UP" in link_state: + break + else: + sleep(0.1) + else: + raise Exception('emulns interface did not came up.') out = check_output( '{ns_exec} echo port_add {hwport} ' @@ -184,8 +130,7 @@ def create_interfaces(): if findall(regex, out, MULTILINE) is None: raise Exception( - 'Control utility for runtime' - ' P4 table failed: {}'.format(out) + 'Control utility for runtime P4 table failed.' ) except CalledProcessError as error: @@ -260,11 +205,9 @@ def cur_is_set(cur_key): } global sock - if sock is None: sock = socket(AF_UNIX, SOCK_STREAM) sock.connect(db_sock) - sock.send(dumps(queries[cur_key])) response = loads(sock.recv(4096)) @@ -274,67 +217,9 @@ def cur_is_set(cur_key): return 0 -SWNS_CHECK = partial( - wait_check, - exists, - swns_netns, - '{} was not present'.format(swns_netns), - swns_netns -) - -HWDESC_CHECK = partial( - wait_check, - exists, - hwdesc_dir, - '{} was not present'.format(hwdesc_dir), - hwdesc_dir -) - -SWITCHD_CHECK = partial( - wait_check, - exists, - switchd_pid, - '{} was not present'.format(switchd_pid), - switchd_pid -) - -DB_SOCK_CHECK = partial( - wait_check, - exists, - db_sock, - '{} was not present'.format(db_sock), - db_sock -) - -CUR_HW_CHECK = partial( - wait_check, - cur_is_set, - 'cur_hw to be set to 1', - 'cur_hw is not set to 1', - 'cur_hw' -) - -CUR_CFG_CHECK = partial( - wait_check, - cur_is_set, - 'cur_hw to be set to 1', - 'cur_hw is not set to 1', - 'cur_hw' -) - -OPS_SWITCHD_CHECK = partial( - wait_check, - lambda: 0 == call(['systemctl', 'is-active', 'switchd.service']), - 'ops-switchd to be active', - 'ops-switchd was not active' -) - -HOSTNAME_CHECK = partial( - wait_check, - lambda: gethostname() == 'switch', - 'final hostname', - 'hostname was not set' -) +def ops_switchd_is_active(): + is_active = call(["systemctl", "is-active", "switchd.service"]) + return is_active == 0 def main(): @@ -342,15 +227,86 @@ def main(): if '-d' in argv: basicConfig(level=DEBUG) - SWNS_CHECK() - HWDESC_CHECK() - SWITCHD_CHECK() + def wait_check(function, wait_name, wait_error, *args): + info('Waiting for {}'.format(wait_name)) + + for i in range(0, config_timeout): + if not function(*args): + sleep(0.1) + else: + break + else: + raise Exception( + 'The image did not boot correctly, ' + '{} after waiting {} seconds.'.format( + wait_error, int(0.1 * config_timeout) + ) + ) + + wait_check( + exists, swns_netns, '{} was not present'.format(swns_netns), swns_netns + ) + wait_check( + exists, hwdesc_dir, '{} was not present'.format(hwdesc_dir), hwdesc_dir + ) + + info('Creating interfaces') create_interfaces() - DB_SOCK_CHECK() - CUR_HW_CHECK() - CUR_CFG_CHECK() - OPS_SWITCHD_CHECK() - HOSTNAME_CHECK() + + wait_check( + exists, db_sock, '{} was not present'.format(db_sock), db_sock + ) + wait_check( + cur_is_set, 'cur_hw to be set to 1', 'cur_hw is not set to 1', + 'cur_hw' + ) + wait_check( + cur_is_set, 'cur_cfg to be set to 1', 'cur_cfg is not set to 1', + 'cur_cfg' + ) + wait_check( + exists, switchd_pid, '{} was not present'.format(switchd_pid), + switchd_pid + ) + wait_check( + ops_switchd_is_active, 'ops-switchd to be active', + 'ops-switchd was not active' + ) + wait_check( + lambda: gethostname() == 'switch', 'final hostname', + 'hostname was not set' + ) + + info('Checking restd service status...') + output = '' + try: + output = check_output( + 'systemctl status restd', shell=True + ) + except CalledProcessError as e: + pass + if 'Active: active' not in output: + try: + info('Starting restd daemon.') + check_output('systemctl start restd', shell=True) + + info('Checking restd service started.') + for i in range(0, config_timeout): + output = '' + output = check_output( + 'systemctl status restd', shell=True + ) + if 'Active: active' not in output: + sleep(0.1) + else: + break + else: + raise Exception("Failed to start restd service") + + except CalledProcessError as e: + raise Exception( + 'Failed to start restd: {}'.format(e.output) + ) if __name__ == '__main__': diff --git a/lib/topology_docker_openswitch/pytest/__init__.py b/lib/topology_docker_openswitch/plugin/__init__.py similarity index 100% rename from lib/topology_docker_openswitch/pytest/__init__.py rename to lib/topology_docker_openswitch/plugin/__init__.py diff --git a/lib/topology_docker_openswitch/pytest/plugin.py b/lib/topology_docker_openswitch/plugin/plugin.py similarity index 86% rename from lib/topology_docker_openswitch/pytest/plugin.py rename to lib/topology_docker_openswitch/plugin/plugin.py index c55354d..0331fcd 100644 --- a/lib/topology_docker_openswitch/pytest/plugin.py +++ b/lib/topology_docker_openswitch/plugin/plugin.py @@ -34,26 +34,13 @@ def pytest_runtest_teardown(item): FIXME: document the item argument """ test_suite = splitext(basename(item.parent.name))[0] + path_name = '/tmp/topology/docker/{}_{}_{}'.format( + test_suite, item.name, datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + ) - from pytest import config - - topology_log_dir = config.getoption('--topology-log-dir') - - if not topology_log_dir: - return - else: - path_name = join( - topology_log_dir, - '{}_{}_{}'.format( - test_suite, - item.name, - datetime.now().strftime('%Y_%m_%d_%H_%M_%S') - ) - ) - - # Being extra-prudent here - if exists(path_name): - rmtree(path_name) + # Being extra-prudent here + if exists(path_name): + rmtree(path_name) if 'topology' not in item.funcargs: from topology_docker_openswitch.openswitch import LOG_PATHS @@ -112,14 +99,14 @@ def pytest_runtest_teardown(item): core_path = '/var/diagnostics/coredump' bash_shell.send_command( - 'ls -1 {}/core* 2>/dev/null'.format(core_path), silent=True + 'ls -1 {}/core.* 2>/dev/null'.format(core_path), silent=True ) core_files = bash_shell.get_response(silent=True).splitlines() for core_file in core_files: bash_shell.send_command( - 'cp {core_file} /tmp'.format(**locals()), + 'cp {core_path}/{core_file} /tmp'.format(**locals()), silent=True ) except: diff --git a/setup.py b/setup.py index a635950..0b46d07 100755 --- a/setup.py +++ b/setup.py @@ -93,10 +93,8 @@ def find_requirements(filename): # Entry points entry_points={ - 'pytest11': [ - 'topology_docker_openswitch ' - '= topology_docker_openswitch.pytest.plugin' - ], + 'pytest11': ['topology_docker_openswitch ' + '= topology_docker_openswitch.plugin.plugin'], 'topology_docker_node_10': [ 'openswitch = topology_docker_openswitch.openswitch:OpenSwitch' ] From ae118299ecbf6cb2c49a32d170edbdf7a5b1d7b4 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Sat, 11 Feb 2017 09:24:49 -0600 Subject: [PATCH 45/55] fix: dev: Removing rest startup. --- .../openswitch_setup | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index 694bc1d..e97d24a 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -277,37 +277,6 @@ def main(): 'hostname was not set' ) - info('Checking restd service status...') - output = '' - try: - output = check_output( - 'systemctl status restd', shell=True - ) - except CalledProcessError as e: - pass - if 'Active: active' not in output: - try: - info('Starting restd daemon.') - check_output('systemctl start restd', shell=True) - - info('Checking restd service started.') - for i in range(0, config_timeout): - output = '' - output = check_output( - 'systemctl status restd', shell=True - ) - if 'Active: active' not in output: - sleep(0.1) - else: - break - else: - raise Exception("Failed to start restd service") - - except CalledProcessError as e: - raise Exception( - 'Failed to start restd: {}'.format(e.output) - ) - if __name__ == '__main__': main() From 2663d825367ed3524a704fe90e73c00ce6423ff0 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Sat, 11 Feb 2017 09:32:58 -0600 Subject: [PATCH 46/55] fix: dev: Fixing core dump getter. --- .../{plugin => pytest}/__init__.py | 0 .../{plugin => pytest}/plugin.py | 29 ++++++++++++++----- setup.py | 6 ++-- 3 files changed, 25 insertions(+), 10 deletions(-) rename lib/topology_docker_openswitch/{plugin => pytest}/__init__.py (100%) rename lib/topology_docker_openswitch/{plugin => pytest}/plugin.py (86%) diff --git a/lib/topology_docker_openswitch/plugin/__init__.py b/lib/topology_docker_openswitch/pytest/__init__.py similarity index 100% rename from lib/topology_docker_openswitch/plugin/__init__.py rename to lib/topology_docker_openswitch/pytest/__init__.py diff --git a/lib/topology_docker_openswitch/plugin/plugin.py b/lib/topology_docker_openswitch/pytest/plugin.py similarity index 86% rename from lib/topology_docker_openswitch/plugin/plugin.py rename to lib/topology_docker_openswitch/pytest/plugin.py index 0331fcd..c55354d 100644 --- a/lib/topology_docker_openswitch/plugin/plugin.py +++ b/lib/topology_docker_openswitch/pytest/plugin.py @@ -34,13 +34,26 @@ def pytest_runtest_teardown(item): FIXME: document the item argument """ test_suite = splitext(basename(item.parent.name))[0] - path_name = '/tmp/topology/docker/{}_{}_{}'.format( - test_suite, item.name, datetime.now().strftime('%Y_%m_%d_%H_%M_%S') - ) - # Being extra-prudent here - if exists(path_name): - rmtree(path_name) + from pytest import config + + topology_log_dir = config.getoption('--topology-log-dir') + + if not topology_log_dir: + return + else: + path_name = join( + topology_log_dir, + '{}_{}_{}'.format( + test_suite, + item.name, + datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + ) + ) + + # Being extra-prudent here + if exists(path_name): + rmtree(path_name) if 'topology' not in item.funcargs: from topology_docker_openswitch.openswitch import LOG_PATHS @@ -99,14 +112,14 @@ def pytest_runtest_teardown(item): core_path = '/var/diagnostics/coredump' bash_shell.send_command( - 'ls -1 {}/core.* 2>/dev/null'.format(core_path), silent=True + 'ls -1 {}/core* 2>/dev/null'.format(core_path), silent=True ) core_files = bash_shell.get_response(silent=True).splitlines() for core_file in core_files: bash_shell.send_command( - 'cp {core_path}/{core_file} /tmp'.format(**locals()), + 'cp {core_file} /tmp'.format(**locals()), silent=True ) except: diff --git a/setup.py b/setup.py index 0b46d07..a635950 100755 --- a/setup.py +++ b/setup.py @@ -93,8 +93,10 @@ def find_requirements(filename): # Entry points entry_points={ - 'pytest11': ['topology_docker_openswitch ' - '= topology_docker_openswitch.plugin.plugin'], + 'pytest11': [ + 'topology_docker_openswitch ' + '= topology_docker_openswitch.pytest.plugin' + ], 'topology_docker_node_10': [ 'openswitch = topology_docker_openswitch.openswitch:OpenSwitch' ] From d0c60a81e48f0e27848a7173efa5ff6be559f6d2 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Sat, 11 Feb 2017 18:10:35 -0600 Subject: [PATCH 47/55] chg: dev: Bumping version to 1.1.1. --- .cookiecutter.json | 2 +- README.rst | 8 ++++++++ lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index 1bd55f0..6d3b8de 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.1.0" + "version": "1.1.1" } diff --git a/README.rst b/README.rst index 710aebb..cea674a 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,14 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.1.1 +----- + +**Fix** + +- Removing startup of restd. +- Fixing coredump file getter. + 1.1.0 ----- diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index 33f8ceb..b376a21 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.1.0' +__version__ = '1.1.1' From 48a0bdabbc9789069886883af004999d3650dfc1 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 14 Feb 2017 17:26:43 -0600 Subject: [PATCH 48/55] fix: dev: Fixing typo. --- lib/topology_docker_openswitch/openswitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 736dfb5..1087c0e 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -289,7 +289,7 @@ def _setup_system(self, script_path=None): vtysh.send_command('show version', silent=True) if 'genericx86-64' in vtysh.get_response(silent=True): - self.product_name = 'genericx84-64' + self.product_name = 'genericx86-64' else: self.product_name = 'genericx86-p4' From c37232af2841e301a3e3a6f5a627be5581d472c2 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 14 Feb 2017 17:35:18 -0600 Subject: [PATCH 49/55] chg: dev: Bumping version to 1.1.2. --- .cookiecutter.json | 2 +- README.rst | 7 +++++++ lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index 6d3b8de..28cc8f5 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.1.1" + "version": "1.1.2" } diff --git a/README.rst b/README.rst index cea674a..3abfabe 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,13 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.1.2 +----- + +**Fix** + +- Fixing typo in ``product_name``. + 1.1.1 ----- diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index b376a21..d053765 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.1.1' +__version__ = '1.1.2' From ac61b7af09ffea0e000ba2581be382649f647552 Mon Sep 17 00:00:00 2001 From: Jose Hernandez Date: Mon, 20 Feb 2017 15:36:12 -0800 Subject: [PATCH 50/55] fix: dev: Change P4 boot to accept string labels Modified P4 boot logic to accept port names in any format when mapping them to interface labels --- lib/topology_docker_openswitch/openswitch_setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch_setup b/lib/topology_docker_openswitch/openswitch_setup index e97d24a..4e4579c 100644 --- a/lib/topology_docker_openswitch/openswitch_setup +++ b/lib/topology_docker_openswitch/openswitch_setup @@ -116,7 +116,7 @@ def create_interfaces(): '/usr/share/ovs_p4_plugin/switch_bmv2.json ' '--thrift-port 10001'.format( ns_exec=ns_exec, hwport=hwport, - port=str(int(hwport) - 1) + port=str(int(len(mapping_ports.keys())) - 1) ), shell=True ) From e152b6c7eec64aa1c666d2536180b0d40aa704e4 Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Tue, 21 Feb 2017 14:03:25 -0600 Subject: [PATCH 51/55] chg: dev: Bumping version to 1.1.3. --- .cookiecutter.json | 2 +- README.rst | 7 +++++++ lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index 28cc8f5..7086402 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.1.2" + "version": "1.1.3" } diff --git a/README.rst b/README.rst index 3abfabe..e7f301d 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,13 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.1.3 +----- + +**Fix** + +- Making P4 boot accept string labels. + 1.1.2 ----- diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index d053765..fdfdf60 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.1.2' +__version__ = '1.1.3' From 03f2bcdbf4fa15bf01e98eeb768a2cb7eaeae6f3 Mon Sep 17 00:00:00 2001 From: Sergio Barahona Date: Fri, 26 May 2017 14:41:54 -0700 Subject: [PATCH 52/55] Remove switch container volume mount dev/log This mount was used to gather logs from containers and log them in the execution machine, it was an old practice from first topology days and is not required anymore. The removal is mainly because there are some vtysh commands that extract data from local log files, so the logs need to be on the container instead of the execution machine. --- lib/topology_docker_openswitch/openswitch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/topology_docker_openswitch/openswitch.py b/lib/topology_docker_openswitch/openswitch.py index 1087c0e..7db30af 100644 --- a/lib/topology_docker_openswitch/openswitch.py +++ b/lib/topology_docker_openswitch/openswitch.py @@ -112,7 +112,6 @@ def __init__( # Add binded directories container_binds = [ - '/dev/log:/dev/log', '/sys/fs/cgroup:/sys/fs/cgroup' ] if binds is not None: From 9874c41f986c2ad4980d0de6777202ba2cb0f81e Mon Sep 17 00:00:00 2001 From: Diego Antonio Hurtado Pimentel Date: Thu, 1 Jun 2017 11:26:47 -0600 Subject: [PATCH 53/55] chg: dev: Bumping version to 1.1.4. --- .cookiecutter.json | 2 +- README.rst | 7 +++++++ lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index 7086402..f8ad5fa 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.1.3" + "version": "1.1.4" } diff --git a/README.rst b/README.rst index e7f301d..0306657 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,13 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. +1.1.4 +----- + +**Fix** + +- Removing log directory mount. + 1.1.3 ----- diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index fdfdf60..7050749 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.1.3' +__version__ = '1.1.4' From 6163686deed93b931a8c0f1c1bcbdfb67142ce56 Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Tue, 6 Mar 2018 10:59:41 -0800 Subject: [PATCH 54/55] fix: dev: Add openswitch_setup script to package. --- MANIFEST.in | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 6d25501..5e30912 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include README.rst LICENSE include requirements.txt +include lib/topology_docker_openswitch/openswitch_setup exclude test/* global-exclude __pycache__ *.py[co] diff --git a/setup.py b/setup.py index a635950..3e6afc1 100755 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ def find_requirements(filename): version=find_version('lib/topology_docker_openswitch/__init__.py'), package_dir={'': 'lib'}, packages=find_packages('lib'), + include_package_data=True, # Dependencies install_requires=find_requirements('requirements.txt'), From 60d73ecdc3cb9a4629af7a199dd0b7847a938cdf Mon Sep 17 00:00:00 2001 From: Pablo Saenz Date: Tue, 6 Mar 2018 11:04:10 -0800 Subject: [PATCH 55/55] chg: dev: Bumping version to 1.1.5. --- .cookiecutter.json | 2 +- README.rst | 227 ++++++++++++++++----- lib/topology_docker_openswitch/__init__.py | 2 +- 3 files changed, 175 insertions(+), 56 deletions(-) diff --git a/.cookiecutter.json b/.cookiecutter.json index f8ad5fa..6d42188 100644 --- a/.cookiecutter.json +++ b/.cookiecutter.json @@ -10,5 +10,5 @@ "project_name": "openswitch node for topology_docker", "short_description": "A Topology OpenSwitch Node for topology_docker.", "year": "2016", - "version": "1.1.4" + "version": "1.1.5" } diff --git a/README.rst b/README.rst index 0306657..0aa19eb 100644 --- a/README.rst +++ b/README.rst @@ -4,92 +4,211 @@ openswitch node for topology_docker A Topology OpenSwitch Node for topology_docker. -1.1.4 ------ +Changelog +========= -**Fix** -- Removing log directory mount. +1.1.5 (2018-03-06) +------------------ +- Merge pull request #31 from saenzpa/package_fix. [Pablo Saenz] -1.1.3 ------ + fix: dev: Add openswitch_setup script to package. -**Fix** -- Making P4 boot accept string labels. +1.1.4 (2017-06-01) +------------------ +- Merge pull request #30 from saenzpa/dev_log_removal. [Diego Hurtado] -1.1.2 ------ + Remove switch container volume mount dev/log +- Remove switch container volume mount dev/log. [Sergio Barahona] -**Fix** + This mount was used to gather logs from containers and log them in the + execution machine, it was an old practice from first topology days and + is not required anymore. -- Fixing typo in ``product_name``. + The removal is mainly because there are some vtysh commands that extract + data from local log files, so the logs need to be on the container instead + of the execution machine. -1.1.1 ------ -**Fix** +1.1.3 (2017-02-21) +------------------ +- Merge branch 'herjosep-p4_string_ifnames' [Diego Antonio Hurtado + Pimentel] -- Removing startup of restd. -- Fixing coredump file getter. -1.1.0 ------ +1.1.1 (2017-02-12) +------------------ +- Merge pull request #28 from saenzpa/no_rest. [Diego Hurtado] -**New** + No rest +- Revert "fix: dev: Refactoring to remove race conditions." [Diego + Antonio Hurtado Pimentel] -- Adapting for ``topology_openswitch``. + This reverts commit 53bf52b8693adfb3a2a5b664a30186b6a881b63c. +- Merge pull request #27 from saenzpa/P4refactor. [Pablo Saenz] -1.0.2 ------ + fix: dev: Refactoring to remove race conditions. +- Merge pull request #26 from saenzpa/remove_rest. [Diego Hurtado] -**Fixes** + fix: dev: Parametrize setup script path. -- Fixing wrong regular expression search. -1.0.1 ------ +1.1.0 (2017-01-19) +------------------ +- Merge pull request #25 from saenzpa/operator_prompt. [Diego Hurtado] -**Fixes** + Operator prompt -- Documenting vtysh exit. -1.0.0 ------ +1.0.1 (2016-12-06) +------------------ -**Fixes** +Fix +~~~ +- Documenting exit from vtysh. [Diego Antonio Hurtado Pimentel] -- Removing shell echo. -0.1.0 ------ +1.0.0 (2016-11-23) +------------------ +- Merge pull request #24 from saenzpa/no_echo. [Diego Hurtado] -**New** + fix: dev: Removing shell echo. +- Merge pull request #23 from saenzpa/environment_fix. [Pablo Saenz] -- Initial release. + fix: dev: Passing 'container' env variable. +- Merge pull request #22 from saenzpa/hpe_sync_with_logs. [Diego + Hurtado] -Documentation -============= + chg: dev: Setting environment container to docker. +- Merge pull request #21 from saenzpa/script_mod. [Pablo Saenz] - https://github.com/HPENetworking/topology_docker_openswitch/tree/master/doc + fix: dev: Increasing config timeout. +- Merge pull request #20 from saenzpa/register_shell. [Diego Hurtado] + chg: dev: Using _register_shell. +- Merge pull request #19 from saenzpa/coredumps. [Pablo Saenz] -License -======= + chg: dev: Collecting docker coredumps on teardown and adding checks for p4 simulator +- Merge pull request #17 from saenzpa/p4switch. [Pablo Saenz] -:: + chg: dev: adding support for p4simulator images - Copyright (C) 2016 Hewlett Packard Enterprise Development LP - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +0.1.0 (2016-08-12) +------------------ + +Changes +~~~~~~~ +- The binds attribute can now be injected and extended by users. [Carlos + Miguel Jenkins Perez] + +Fix +~~~ +- Fixed bug in initial prompt matching causing bash based shells to + timeout. [Carlos Miguel Jenkins Perez] + +Other +~~~~~ +- Merge pull request #16 from saenzpa/restd_start. [Diego Hurtado] + + Restd start +- Fixing restd validation status and service start. [fonsecamau] +- Merge pull request #15 from saenzpa/file_exist_fix. [Pablo Saenz] + + fix: dev: Adding handler for existing files. +- Merge pull request #14 from saenzpa/bringup_checks. [Pablo Saenz] + + chg: dev: Modifying boot time checks +- Merge pull request #13 from saenzpa/improved_logging. [Diego Hurtado] + + chg: dev: Improving logging. +- Merge pull request #11 from saenzpa/ops_switchd_active_timeout. [Pablo + Saenz] + + chg: dev: Increase ops-switchd active timeout +- Merge pull request #9 from saenzpa/switchd_active. [Pablo Saenz] + + chg: dev: Add check for ops-switchd to be active +- Merge pull request #8 from baraserg/master. [Pablo Saenz] + + Use safe method of querying dictionary +- Merge pull request #11 from baraserg/master. [Diego Hurtado] + + fix: dev: Change static path to shared_dir attribute +- Merge pull request #7 from saenzpa/log_messages. [Pablo Saenz] + + Log messages +- Merge pull request #6 from baraserg/master. [Pablo Saenz] + + Merge log plugin +- Merge pull request #5 from saenzpa/master_sync. [Pablo Saenz] + + Master sync +- Merge pull request #4 from HPENetworking/master. [Pablo Saenz] + + pulling from master +- Merge pull request #9 from fonsecamau/master. [Carlos Jenkins] + + chg: dev: Adding/modifying logging feature on process bring-up +- Merge pull request #8 from fonsecamau/master. [Carlos Jenkins] + + new: dev: Adding more logging and exception handling +- Merge pull request #24 from HPENetworking/new_shell_api_migration. + [David Diaz Barquero] + + chg: dev: Migrated all nodes shells to new Topology shell API. +- Merge pull request #23 from HPENetworking/new_binds_attribute. [Carlos + Jenkins] + + chg: usr: The binds attribute can now be injected and extended by users. +- Merge pull request #20 from HPENetworking/ddompe-patch-1. [Diego + Hurtado] + + Improvements during initialization +- Fix bugs during initialization. [Diego Dompe] + + - Handle support for sync the port readiness with the newer openswitch images + - Delay waiting for the cur_cfg, and handle the case where the cfg is not ready yet better. +- Merge pull request #19 from agustin-meneses-fuentes/master. [Carlos + Jenkins] + + fix: dev: Add bonding_masters to ip link set exceptions +- Merge pull request #11 from walintonc/master. [Carlos Jenkins] + + new: usr: Add support to specifying the hostname for a node. +- Add support to specifying hostname for create_container. [Walinton + Cambronero] + + - This allows that nodes can specify the hostname of choice + - In the openswitch node, the default hostname is 'switch' + - Clarify that tag must be specified in image param +- Merge pull request #2 from fonsecamau/fix_cut_output. [Carlos Jenkins] + + fix: dev: Make vtysh shell regular expression for prompt more specific. +- Merge pull request #19 from hpe-networking/fix_cut_output. [Carlos + Miguel Jenkins Perez] + + fix: dev: Output gets confused with switch prompt +- Merge pull request #17 from hpe-networking/ops_oobm. [Carlos Miguel + Jenkins Perez] + + chg: dev: Avoid moving new oobm interface to swns namespace +- Merge pull request #15 from hpe-networking/after_autopull. [David Diaz + Barquero] + + Refactored code, fixed minor issues and code quality. +- Merge pull request #8 from hpe-networking/docker_tmp. [David Diaz + Barquero] + + Mapping port to port labels for openswitch in topology +- Merge pull request #4 from hpe-networking/send_command_to_docker_exec. + [David Diaz Barquero] + + chg: dev: Refactored all send_commands to docker_exec to avoid using pexpect. +- Merge pull request #3 from hpe-networking/dockerfiles. [Carlos Miguel + Jenkins Perez] + + new: dev: Add docker file for toxin node - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. diff --git a/lib/topology_docker_openswitch/__init__.py b/lib/topology_docker_openswitch/__init__.py index 7050749..864965e 100644 --- a/lib/topology_docker_openswitch/__init__.py +++ b/lib/topology_docker_openswitch/__init__.py @@ -24,4 +24,4 @@ __author__ = 'Hewlett Packard Enterprise Development LP' __email__ = 'hpe-networking@lists.hp.com' -__version__ = '1.1.4' +__version__ = '1.1.5'