Handle dqlite and cluster agent ports on HA cluster

microk8s-resources/actions/common/utils.sh

@@ -505,7 +505,9 @@ function valid_ip() {
 init_cluster() {
   mkdir -p ${SNAP_DATA}/var/kubernetes/backend
   IP="127.0.0.1"
-  # TODO: make the port configurable
+  # To configure dqlite do:
+  # echo "Address: 1.2.3.4:6364" > $STORAGE_DIR/update.yaml
+  # after the initialisation but before connecting other nodes
   echo "Address: $IP:19001" > ${SNAP_DATA}/var/kubernetes/backend/init.yaml
   DNS=$($SNAP/bin/hostname)
   mkdir -p $SNAP_DATA/var/tmp/

scripts/cluster/agent.py

@@ -19,6 +19,8 @@
     get_callback_token,
     remove_token_from_file,
     is_token_expired,
+    get_dqlite_port,
+    get_cluster_agent_port,
 )
 
 from flask import Flask, jsonify, request, abort, Response
@@ -29,10 +31,11 @@
 snapdata_path = os.environ.get('SNAP_DATA')
 snap_path = os.environ.get('SNAP')
 cluster_tokens_file = "{}/credentials/cluster-tokens.txt".format(snapdata_path)
-callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path)
 callback_token_file = "{}/credentials/callback-token.txt".format(snapdata_path)
+callback_tokens_file = "{}/credentials/callback-tokens.txt".format(snapdata_path)
 certs_request_tokens_file = "{}/credentials/certs-request-tokens.txt".format(snapdata_path)
 default_port = 25000
+dqlite_default_port = 19001
 default_listen_interface = "0.0.0.0"
 
 
@@ -496,12 +499,13 @@ def update_dqlite_ip(host):
 
     :param : the host others see for this node
     """
+    dqlite_port = get_dqlite_port()
     subprocess.check_call("snapctl stop microk8s.daemon-apiserver".split())
     time.sleep(10)
 
     cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path)
     # TODO make the port configurable
-    update_data = {'Address': "{}:19001".format(host)}
+    update_data = {'Address': "{}:{}".format(host, dqlite_port)}
     with open("{}/update.yaml".format(cluster_dir), 'w') as f:
         yaml.dump(update_data, f)
     subprocess.check_call("snapctl start microk8s.daemon-apiserver".split())
@@ -568,6 +572,13 @@ def join_node_dqlite():
         error_msg = {"error": "Not possible to join. This is not an HA dqlite cluster."}
         return Response(json.dumps(error_msg), mimetype='application/json', status=501)
 
+    agent_port = get_cluster_agent_port()
+    if port != agent_port:
+        error_msg = {
+            "error": "The port of the cluster agent has to be set to {}.".format(agent_port)
+        }
+        return Response(json.dumps(error_msg), mimetype='application/json', status=502)
+
     voters = get_dqlite_voters()  # type: List[str]
     # Check if we need to set dqlite with external IP
     if len(voters) == 1 and voters[0].startswith("127.0.0.1"):

scripts/cluster/common/utils.py

@@ -3,6 +3,7 @@
 import time
 import string
 import random
+import yaml
 
 
 def try_set_file_permissions(file):
@@ -105,3 +106,43 @@ def is_node_running_dqlite():
     """
     ha_lock = os.path.expandvars("${SNAP_DATA}/var/lock/ha-cluster")
     return os.path.isfile(ha_lock)
+
+
+def get_dqlite_port():
+    """
+    What is the port dqlite listens on
+
+    :return: the dqlite port
+    """
+    # We get the dqlite port from the already existing deployment
+    snapdata_path = os.environ.get('SNAP_DATA')
+    cluster_dir = "{}/var/kubernetes/backend".format(snapdata_path)
+    dqlite_info = "{}/info.yaml".format(cluster_dir)
+    port = 19001
+    if os.path.exists(dqlite_info):
+        with open(dqlite_info) as f:
+            data = yaml.load(f, Loader=yaml.FullLoader)
+        if 'Address' in data:
+            port = data['Address'].split(':')[1]
+
+    return port
+
+
+def get_cluster_agent_port():
+    """
+    What is the cluster agent port
+
+    :return: the port
+    """
+    cluster_agent_port = "25000"
+    snapdata_path = os.environ.get('SNAP_DATA')
+    filename = "{}/args/cluster-agent".format(snapdata_path)
+    with open(filename) as fp:
+        for _, line in enumerate(fp):
+            if line.startswith("--bind"):
+                port_parse = line.split(' ')
+                port_parse = port_parse[-1].split('=')
+                port_parse = port_parse[-1].split(':')
+                if len(port_parse) > 1:
+                    cluster_agent_port = port_parse[1].rstrip()
+    return cluster_agent_port

scripts/cluster/join.py

@@ -16,7 +16,12 @@
 import yaml
 import json
 
-from common.utils import try_set_file_permissions, is_node_running_dqlite
+from common.utils import (
+    try_set_file_permissions,
+    is_node_running_dqlite,
+    get_dqlite_port,
+    get_cluster_agent_port,
+)
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 CLUSTER_API = "cluster/api/v1.0"
@@ -47,17 +52,14 @@ def get_connection_info(master_ip, master_port, token, callback_token=None, clus
 
     :return: the json response of the master
     """
-    cluster_agent_port = "25000"
-    filename = "{}/args/cluster-agent".format(snapdata_path)
-    with open(filename) as fp:
-        for _, line in enumerate(fp):
-            if line.startswith("--port"):
-                port_parse = line.split(' ')
-                port_parse = port_parse[-1].split('=')
-                cluster_agent_port = port_parse[0].rstrip()
+    cluster_agent_port = get_cluster_agent_port()
 
     if cluster_type == "dqlite":
-        req_data = {"token": token, "hostname": socket.gethostname(), "port": cluster_agent_port}
+        req_data = {
+            "token": token,
+            "hostname": socket.gethostname(),
+            "port": cluster_agent_port,
+        }
 
         # TODO: enable ssl verification
         connection_info = requests.post(
@@ -391,7 +393,7 @@ def reset_current_dqlite_installation():
             stderr=subprocess.DEVNULL,
         )
 
-    # TODO make this port configurable
+    # We reset to the default port and address
     init_data = {'Address': '127.0.0.1:19001'}  # type: Dict[str, str]
     with open("{}/init.yaml".format(cluster_dir), 'w') as f:
         yaml.dump(init_data, f)
@@ -739,11 +741,15 @@ def update_dqlite(cluster_cert, cluster_key, voters, host):
     shutil.move(cluster_dir, cluster_backup_dir)
     os.mkdir(cluster_dir)
     store_cluster_certs(cluster_cert, cluster_key)
+
+    # We get the dqlite port from the already existing deployment
+    port = 19001
     with open("{}/info.yaml".format(cluster_backup_dir)) as f:
         data = yaml.load(f, Loader=yaml.FullLoader)
+    if 'Address' in data:
+        port = data['Address'].split(':')[1]
 
-    # TODO make port configurable
-    init_data = {'Cluster': voters, 'Address': "{}:19001".format(host)}
+    init_data = {'Cluster': voters, 'Address': "{}:{}".format(host, port)}
     with open("{}/init.yaml".format(cluster_dir), 'w') as f:
         yaml.dump(init_data, f)