Skip to content
This repository has been archived by the owner on Dec 16, 2024. It is now read-only.

CI: Add delay between joining masters #844

Merged
merged 1 commit into from
Nov 21, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion ci/infra/testrunner/README.md
Original file line number Diff line number Diff line change
@@ -380,7 +380,7 @@ usage:
...

positional arguments:
{info,get_logs,cleanup,provision,bootstrap,status,cluster-upgrade-plan,join-node,remove-node,node-upgrade,ssh,test}
{info,get_logs,cleanup,provision,bootstrap,status,cluster-upgrade-plan,join-node,remove-node,node-upgrade,join-nodes,ssh,test}
command
info ip info
get_logs gather logs from nodes
@@ -394,6 +394,7 @@ positional arguments:
join-node add node in k8s cluster with the given role.
remove-node remove node from k8s cluster.
node-upgrade plan or apply kubernetes version upgrade in node
join-nodes add multiple provisioned nodes k8s.
ssh Execute command in node via ssh.
test execute tests

@@ -440,7 +441,18 @@ optional arguments:
-n NODE, --node NODE node to be added or deleted. eg: -n 0

```
### Join nodes

```
-h, --help show this help message and exit
-m MASTERS, --masters MASTERS
Specify how many masters to join. Default is all
-w WORKERS, --workers WORKERS
Specify how many workers to join. Default is all
-d DELAY, --delay DELAY
Delay between joining masters to allow etcd to
stabilize
```
#### Node Upgrade command

```
15 changes: 9 additions & 6 deletions ci/infra/testrunner/skuba/skuba.py
Original file line number Diff line number Diff line change
@@ -99,17 +99,20 @@ def node_join(self, role="worker", nr=0):
except Exception as ex:
raise Exception("Error executing cmd {}") from ex

def join_nodes(self, masters=None, workers=None):
def join_nodes(self, masters=None, workers=None, delay=60):
if masters is None:
masters = self.platform.get_num_nodes("master")
if workers is None:
workers = self.platform.get_num_nodes("worker")

nodes = [("master", n) for n in range(1, masters)] + \
[("worker", n) for n in range(0, workers)]
for role, node in nodes:
self.node_join(role, node)
self._wait_node_joined(role, node, timeout=180, backoff=20)
for node in range(1, masters):
self.node_join("master", node)
self._wait_node_joined("master", node, timeout=180, backoff=20)
# wait for etcd to become ready
time.sleep(delay)

for node in range(0, workers):
self.node_join("worker", node)


def _wait_node_joined(self, role, node, timeout=60, backoff=10):
12 changes: 6 additions & 6 deletions ci/infra/testrunner/testrunner.py
Original file line number Diff line number Diff line change
@@ -63,7 +63,7 @@ def join_node(options):

def join_nodes(options):
skuba = Skuba(options.conf, options.platform)
skuba.join_nodes(masters=options.masters, workers=options.workers)
skuba.join_nodes(masters=options.masters, workers=options.workers, delay=options.delay)


def remove_node(options):
@@ -166,13 +166,13 @@ def main():

# Start Join Nodes
cmd_join_nodes = commands.add_parser("join-nodes",
help="add node in k8s cluster with the given role.")
cmd_join_nodes.add_argument("-m", "--masters",
type=int,
help="add multiple provisioned nodes k8s.")
cmd_join_nodes.add_argument("-m", "--masters", type=int,
help="Specify how many masters to join. Default is all")
cmd_join_nodes.add_argument("-w", "--workers",
type=int,
cmd_join_nodes.add_argument("-w", "--workers", type=int,
help="Specify how many workers to join. Default is all")
cmd_join_nodes.add_argument("-d", "--delay", type=int, default=120,
help="Delay between joining masters to allow etcd to stabilize")
cmd_join_nodes.set_defaults(func=join_nodes)
# End Join Nodes