dbctl for dqlite backup and restore (#1435)

microk8s-resources/wrappers/microk8s-dbctl.wrapper

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -eu
+
+export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH"
+ARCH="$($SNAP/bin/uname -m)"
+export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu"
+export PYTHONNOUSERSITE=false
+
+source $SNAP/actions/common/utils.sh
+
+exit_if_no_permissions
+
+LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dbctl.py $@

scripts/wrappers/dbctl.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python3
+import os
+import argparse
+
+import tempfile
+import datetime
+import subprocess
+import tarfile
+import os.path
+
+from common.utils import (
+    exit_if_no_permission,
+    is_cluster_locked,
+    is_ha_enabled,
+)
+
+
+def kine_exists():
+    """
+    Check the existence of the kine socket
+    :return: True if the kine socket exists
+    """
+    kine_socket = "/var/snap/microk8s/current/var/kubernetes/backend/kine.sock"
+    return os.path.exists(kine_socket)
+
+
+def generate_backup_name():
+    """
+    Generate a filename based on the current time and date
+    :return: a generated filename
+    """
+    now = datetime.datetime.now()
+    return "backup-{}".format(now.strftime("%Y-%m-%d-%H-%M-%S"))
+
+
+def run_command(command):
+    """
+    Run a command while printing the output
+    :param command: the command to run
+    :return: the return code of the command
+    """
+    process = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
+    while True:
+        output = process.stdout.readline()
+        if (not output or output == '') and process.poll() is not None:
+            break
+        if output:
+            print(output.decode().strip())
+    rc = process.poll()
+    return rc
+
+
+def backup(fname=None, debug=False):
+    """
+    Backup the database to a provided file
+    :param fname_tar: the tar file
+    :param debug: show debug output
+    """
+    snap_path = os.environ.get('SNAP')
+    snapdata_path = os.environ.get('SNAP_DATA')
+    # snap_path = '/snap/microk8s/current'
+    # snapdata_path = '/var/snap/microk8s/current'
+
+    if not fname:
+        fname = generate_backup_name()
+    if fname.endswith('.tar.gz'):
+        fname = fname[:-7]
+    fname_tar = '{}.tar.gz'.format(fname)
+
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        backup_cmd = '{}/bin/migrator --mode backup-dqlite --db-dir {}'.format(
+            snap_path, "{}/{}".format(tmpdirname, fname)
+        )
+        if debug:
+            backup_cmd = "{} {}".format(backup_cmd, "--debug")
+        try:
+            rc = run_command(backup_cmd)
+            if rc > 0:
+                print("Backup process failed. {}".format(rc))
+                exit(1)
+            with tarfile.open(fname_tar, "w:gz") as tar:
+                tar.add(
+                    "{}/{}".format(tmpdirname, fname),
+                    arcname=os.path.basename("{}/{}".format(tmpdirname, fname)),
+                )
+
+            target_file = '{}/var/tmp/{}'.format(snapdata_path, fname_tar)
+            print("The backup is: {}".format(fname_tar))
+        except subprocess.CalledProcessError as e:
+            print("Backup process failed. {}".format(e))
+            exit(2)
+
+
+def restore(fname_tar, debug=False):
+    """
+    Restore the database from the provided file
+    :param fname_tar: the tar file
+    :param debug: show debug output
+    """
+    snap_path = os.environ.get('SNAP')
+    # snap_path = '/snap/microk8s/current'
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        with tarfile.open(fname_tar, "r:gz") as tar:
+            tar.extractall(path=tmpdirname)
+        if fname_tar.endswith('.tar.gz'):
+            fname = fname_tar[:-7]
+        else:
+            fname = fname_tar
+        fname = os.path.basename(fname)
+        restore_cmd = '{}/bin/migrator --mode restore-to-dqlite --db-dir {}'.format(
+            snap_path, "{}/{}".format(tmpdirname, fname)
+        )
+        if debug:
+            restore_cmd = "{} {}".format(restore_cmd, "--debug")
+        try:
+            rc = run_command(restore_cmd)
+            if rc > 0:
+                print("Restore process failed. {}".format(rc))
+                exit(3)
+        except subprocess.CalledProcessError as e:
+            print("Restore process failed. {}".format(e))
+            exit(4)
+
+
+if __name__ == '__main__':
+    exit_if_no_permission()
+    is_cluster_locked()
+
+    if not kine_exists() or not is_ha_enabled():
+        print("Please ensure the kubernetes apiserver is running and HA is enabled.")
+        exit(10)
+
+    # initiate the parser with a description
+    parser = argparse.ArgumentParser(
+        description="backup and restore the Kubernetes datastore.", prog='microk8s dbctl'
+    )
+    parser.add_argument('--debug', action='store_true', help='print debug output')
+    commands = parser.add_subparsers(title='commands', help='backup and restore operations')
+    restore_parser = commands.add_parser("restore")
+    restore_parser.add_argument('backup-file', help='name of file with the backup')
+    backup_parser = commands.add_parser("backup")
+    backup_parser.add_argument('-o', metavar='backup-file', help='output filename')
+    args = parser.parse_args()
+
+    if 'backup-file' in args:
+        fname = vars(args)['backup-file']
+        print("Restoring from {}".format(fname))
+        restore(fname, args.debug)
+    elif 'o' in args:
+        print("Backing up the datastore")
+        backup(vars(args)['o'], args.debug)
+    else:
+        parser.print_help()

snap/snapcraft.yaml

@@ -96,6 +96,8 @@ apps:
     command: microk8s-cilium.wrapper
   juju:
     command: microk8s-juju.wrapper
+  dbctl:
+    command: microk8s-dbctl.wrapper
 
 parts:
   libco:

tests/test-addons.py

@@ -30,7 +30,7 @@
     microk8s_disable,
     microk8s_reset,
 )
-from subprocess import Popen, PIPE, STDOUT, CalledProcessError
+from subprocess import Popen, PIPE, STDOUT, CalledProcessError, check_call
 
 
 class TestAddons(object):
@@ -303,3 +303,13 @@ def test_ambassador(self):
         validate_ambassador()
         print("Disabling Ambassador")
         microk8s_disable("ambassador")
+
+    def test_backup_restore(self):
+        """
+        Test backup and restore commands.
+        """
+        print('Checking dbctl backup and restore')
+        if os.path.exists('backupfile.tar.gz'):
+            os.remove('backupfile.tar.gz')
+        check_call("/snap/bin/microk8s.dbctl --debug backup -o backupfile".split())
+        check_call("/snap/bin/microk8s.dbctl --debug restore backupfile.tar.gz".split())

tests/test-cluster.py

@@ -139,11 +139,6 @@ def setup_cluster(self):
                 for vm_name in reuse_vms:
                     self.VM.append(VM(vm_name))
 
-            # enable HA
-            for vm in self.VM:
-                print('Enabling ha-cluster on machine {}'.format(vm.vm_name))
-                vm.run('/snap/bin/microk8s.enable ha-cluster')
-
             # Form cluster
             vm_master = self.VM[0]
             connected_nodes = vm_master.run('/snap/bin/microk8s.kubectl get no')
@@ -213,8 +208,8 @@ def test_nodes_in_ha(self):
         while True:
             assert attempt > 0
             for vm in self.VM:
-                status = vm.run('/snap/bin/microk8s.status ha-cluster')
-                if "The cluster is highly available" not in status.decode():
+                status = vm.run('/snap/bin/microk8s.status')
+                if "high-availability: yes" not in status.decode():
                     attempt += 1
                     continue
             break
@@ -246,8 +241,8 @@ def test_nodes_in_ha(self):
         while True:
             assert attempt > 0
             for vm in leftVMs:
-                status = vm.run('/snap/bin/microk8s.status ha-cluster')
-                if "HA cluster has not formed yet" not in status.decode():
+                status = vm.run('/snap/bin/microk8s.status')
+                if "high-availability: no" not in status.decode():
                     attempt += 1
                     time.sleep(2)
                     continue
@@ -295,8 +290,8 @@ def test_nodes_in_ha(self):
         while True:
             assert attempt > 0
             for vm in self.VM:
-                status = vm.run('/snap/bin/microk8s.status ha-cluster')
-                if "The cluster is highly available" not in status.decode():
+                status = vm.run('/snap/bin/microk8s.status')
+                if "high-availability: yes" not in status.decode():
                     attempt += 1
                     time.sleep(2)
                     continue