From 5a9e215d53507495790cbccb7b90111f0cf3eea0 Mon Sep 17 00:00:00 2001 From: Pablo Chacin Date: Wed, 20 Nov 2019 12:34:09 +0100 Subject: [PATCH] Add delay between joining masters Add deleay to allow etcd to become ready. Signed-off-by: Pablo Chacin --- ci/infra/testrunner/README.md | 14 +++++++++++++- ci/infra/testrunner/skuba/skuba.py | 15 +++++++++------ ci/infra/testrunner/testrunner.py | 12 ++++++------ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/ci/infra/testrunner/README.md b/ci/infra/testrunner/README.md index b0fbf66bf7..9d4deb74ce 100644 --- a/ci/infra/testrunner/README.md +++ b/ci/infra/testrunner/README.md @@ -380,7 +380,7 @@ usage: ... positional arguments: - {info,get_logs,cleanup,provision,bootstrap,status,cluster-upgrade-plan,join-node,remove-node,node-upgrade,ssh,test} + {info,get_logs,cleanup,provision,bootstrap,status,cluster-upgrade-plan,join-node,remove-node,node-upgrade,join-nodes,ssh,test} command info ip info get_logs gather logs from nodes @@ -394,6 +394,7 @@ positional arguments: join-node add node in k8s cluster with the given role. remove-node remove node from k8s cluster. node-upgrade plan or apply kubernetes version upgrade in node + join-nodes add multiple provisioned nodes k8s. ssh Execute command in node via ssh. test execute tests @@ -440,7 +441,18 @@ optional arguments: -n NODE, --node NODE node to be added or deleted. eg: -n 0 ``` +### Join nodes +``` + -h, --help show this help message and exit + -m MASTERS, --masters MASTERS + Specify how many masters to join. Default is all + -w WORKERS, --workers WORKERS + Specify how many workers to join. Default is all + -d DELAY, --delay DELAY + Delay between joining masters to allow etcd to + stabilize +``` #### Node Upgrade command ``` diff --git a/ci/infra/testrunner/skuba/skuba.py b/ci/infra/testrunner/skuba/skuba.py index 39652c4512..2089f54b44 100644 --- a/ci/infra/testrunner/skuba/skuba.py +++ b/ci/infra/testrunner/skuba/skuba.py @@ -99,17 +99,20 @@ def node_join(self, role="worker", nr=0): except Exception as ex: raise Exception("Error executing cmd {}") from ex - def join_nodes(self, masters=None, workers=None): + def join_nodes(self, masters=None, workers=None, delay=60): if masters is None: masters = self.platform.get_num_nodes("master") if workers is None: workers = self.platform.get_num_nodes("worker") - nodes = [("master", n) for n in range(1, masters)] + \ - [("worker", n) for n in range(0, workers)] - for role, node in nodes: - self.node_join(role, node) - self._wait_node_joined(role, node, timeout=180, backoff=20) + for node in range(1, masters): + self.node_join("master", node) + self._wait_node_joined("master", node, timeout=180, backoff=20) + # wait for etcd to become ready + time.sleep(delay) + + for node in range(0, workers): + self.node_join("worker", node) def _wait_node_joined(self, role, node, timeout=60, backoff=10): diff --git a/ci/infra/testrunner/testrunner.py b/ci/infra/testrunner/testrunner.py index 6913b6c13f..37ec5bfaa9 100644 --- a/ci/infra/testrunner/testrunner.py +++ b/ci/infra/testrunner/testrunner.py @@ -63,7 +63,7 @@ def join_node(options): def join_nodes(options): skuba = Skuba(options.conf, options.platform) - skuba.join_nodes(masters=options.masters, workers=options.workers) + skuba.join_nodes(masters=options.masters, workers=options.workers, delay=options.delay) def remove_node(options): @@ -166,13 +166,13 @@ def main(): # Start Join Nodes cmd_join_nodes = commands.add_parser("join-nodes", - help="add node in k8s cluster with the given role.") - cmd_join_nodes.add_argument("-m", "--masters", - type=int, + help="add multiple provisioned nodes k8s.") + cmd_join_nodes.add_argument("-m", "--masters", type=int, help="Specify how many masters to join. Default is all") - cmd_join_nodes.add_argument("-w", "--workers", - type=int, + cmd_join_nodes.add_argument("-w", "--workers", type=int, help="Specify how many workers to join. Default is all") + cmd_join_nodes.add_argument("-d", "--delay", type=int, default=120, + help="Delay between joining masters to allow etcd to stabilize") cmd_join_nodes.set_defaults(func=join_nodes) # End Join Nodes