From 6033ca80bceca1a9f3306d1db7e1eb9015a5bac6 Mon Sep 17 00:00:00 2001 From: Konstantinos Date: Mon, 15 Jun 2020 16:02:39 +0300 Subject: [PATCH] Handle dqlite and cluster agent ports on HA cluster --- microk8s-resources/actions/common/utils.sh | 4 ++- scripts/cluster/agent.py | 15 ++++++-- scripts/cluster/common/utils.py | 41 ++++++++++++++++++++++ scripts/cluster/join.py | 32 ++++++++++------- 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/microk8s-resources/actions/common/utils.sh b/microk8s-resources/actions/common/utils.sh index 4dc3c4c808f..cfa5ef24b5a 100644 --- a/microk8s-resources/actions/common/utils.sh +++ b/microk8s-resources/actions/common/utils.sh @@ -505,7 +505,9 @@ function valid_ip() { init_cluster() { mkdir -p ${SNAP_DATA}/var/kubernetes/backend IP="127.0.0.1" - # TODO: make the port configurable + # To configure dqlite do: + # echo "Address: 1.2.3.4:6364" > $STORAGE_DIR/update.yaml + # after the initialisation but before connecting other nodes echo "Address: $IP:19001" > ${SNAP_DATA}/var/kubernetes/backend/init.yaml DNS=$($SNAP/bin/hostname) mkdir -p $SNAP_DATA/var/tmp/ diff --git a/scripts/cluster/agent.py b/scripts/cluster/agent.py index 6c847b1812d..b68469a1cc9 100644 --- a/scripts/cluster/agent.py +++ b/scripts/cluster/agent.py @@ -19,6 +19,8 @@ get_callback_token, remove_token_from_file, is_token_expired, + get_dqlite_port, + get_cluster_agent_port, ) from flask import Flask, jsonify, request, abort, Response @@ -29,10 +31,11 @@ snapdata_path = os.environ.get('SNAP_DATA') snap_path = os.environ.get('SNAP') cluster_tokens_file = "{}/credentials/cluster-tokens.txt".format(snapdata_path) -callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path) callback_token_file = "{}/credentials/callback-token.txt".format(snapdata_path) +callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path) certs_request_tokens_file = "{}/credentials/certs-request-tokens.txt".format(snapdata_path) default_port = 25000 +dqlite_default_port = 19001 default_listen_interface = "0.0.0.0" @@ -496,12 +499,13 @@ def update_dqlite_ip(host): :param : the host others see for this node """ + dqlite_port = get_dqlite_port() subprocess.check_call("snapctl stop microk8s.daemon-apiserver".split()) time.sleep(10) cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path) # TODO make the port configurable - update_data = {'Address': "{}:19001".format(host)} + update_data = {'Address': "{}:{}".format(host, dqlite_port)} with open("{}/update.yaml".format(cluster_dir), 'w') as f: yaml.dump(update_data, f) subprocess.check_call("snapctl start microk8s.daemon-apiserver".split()) @@ -568,6 +572,13 @@ def join_node_dqlite(): error_msg = {"error": "Not possible to join. This is not an HA dqlite cluster."} return Response(json.dumps(error_msg), mimetype='application/json', status=501) + agent_port = get_cluster_agent_port() + if port != agent_port: + error_msg = { + "error": "The port of the cluster agent has to be set to {}.".format(agent_port) + } + return Response(json.dumps(error_msg), mimetype='application/json', status=502) + voters = get_dqlite_voters() # type: List[str] # Check if we need to set dqlite with external IP if len(voters) == 1 and voters[0].startswith("127.0.0.1"): diff --git a/scripts/cluster/common/utils.py b/scripts/cluster/common/utils.py index 174073f4834..a34bba53558 100644 --- a/scripts/cluster/common/utils.py +++ b/scripts/cluster/common/utils.py @@ -3,6 +3,7 @@ import time import string import random +import yaml def try_set_file_permissions(file): @@ -105,3 +106,43 @@ def is_node_running_dqlite(): """ ha_lock = os.path.expandvars("${SNAP_DATA}/var/lock/ha-cluster") return os.path.isfile(ha_lock) + + +def get_dqlite_port(): + """ + What is the port dqlite listens on + + :return: the dqlite port + """ + # We get the dqlite port from the already existing deployment + snapdata_path = os.environ.get('SNAP_DATA') + cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path) + dqlite_info = "{}/info.yaml".format(cluster_dir) + port = 19001 + if os.path.exists(dqlite_info): + with open(dqlite_info) as f: + data = yaml.load(f, Loader=yaml.FullLoader) + if 'Address' in data: + port = data['Address'].split(':')[1] + + return port + + +def get_cluster_agent_port(): + """ + What is the cluster agent port + + :return: the port + """ + cluster_agent_port = "25000" + snapdata_path = os.environ.get('SNAP_DATA') + filename = "{}/args/cluster-agent".format(snapdata_path) + with open(filename) as fp: + for _, line in enumerate(fp): + if line.startswith("--bind"): + port_parse = line.split(' ') + port_parse = port_parse[-1].split('=') + port_parse = port_parse[-1].split(':') + if len(port_parse) > 1: + cluster_agent_port = port_parse[1].rstrip() + return cluster_agent_port diff --git a/scripts/cluster/join.py b/scripts/cluster/join.py index 884f4163be1..01f2d100e28 100755 --- a/scripts/cluster/join.py +++ b/scripts/cluster/join.py @@ -16,7 +16,12 @@ import yaml import json -from common.utils import try_set_file_permissions, is_node_running_dqlite +from common.utils import ( + try_set_file_permissions, + is_node_running_dqlite, + get_dqlite_port, + get_cluster_agent_port, +) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) CLUSTER_API = "cluster/api/v1.0" @@ -47,17 +52,14 @@ def get_connection_info(master_ip, master_port, token, callback_token=None, clus :return: the json response of the master """ - cluster_agent_port = "25000" - filename = "{}/args/cluster-agent".format(snapdata_path) - with open(filename) as fp: - for _, line in enumerate(fp): - if line.startswith("--port"): - port_parse = line.split(' ') - port_parse = port_parse[-1].split('=') - cluster_agent_port = port_parse[0].rstrip() + cluster_agent_port = get_cluster_agent_port() if cluster_type == "dqlite": - req_data = {"token": token, "hostname": socket.gethostname(), "port": cluster_agent_port} + req_data = { + "token": token, + "hostname": socket.gethostname(), + "port": cluster_agent_port, + } # TODO: enable ssl verification connection_info = requests.post( @@ -391,7 +393,7 @@ def reset_current_dqlite_installation(): stderr=subprocess.DEVNULL, ) - # TODO make this port configurable + # We reset to the default port and address init_data = {'Address': '127.0.0.1:19001'} # type: Dict[str, str] with open("{}/init.yaml".format(cluster_dir), 'w') as f: yaml.dump(init_data, f) @@ -739,11 +741,15 @@ def update_dqlite(cluster_cert, cluster_key, voters, host): shutil.move(cluster_dir, cluster_backup_dir) os.mkdir(cluster_dir) store_cluster_certs(cluster_cert, cluster_key) + + # We get the dqlite port from the already existing deployment + port = 19001 with open("{}/info.yaml".format(cluster_backup_dir)) as f: data = yaml.load(f, Loader=yaml.FullLoader) + if 'Address' in data: + port = data['Address'].split(':')[1] - # TODO make port configurable - init_data = {'Cluster': voters, 'Address': "{}:19001".format(host)} + init_data = {'Cluster': voters, 'Address': "{}:{}".format(host, port)} with open("{}/init.yaml".format(cluster_dir), 'w') as f: yaml.dump(init_data, f)