From 4cb316d0ee36d87434f227b54ded2d989679015b Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Mon, 10 Aug 2020 14:48:56 +0300 Subject: [PATCH] dbctl for dqlite backup and restore (#1435) --- .../wrappers/microk8s-dbctl.wrapper | 14 ++ scripts/wrappers/dbctl.py | 153 ++++++++++++++++++ snap/snapcraft.yaml | 2 + tests/test-addons.py | 12 +- tests/test-cluster.py | 17 +- 5 files changed, 186 insertions(+), 12 deletions(-) create mode 100755 microk8s-resources/wrappers/microk8s-dbctl.wrapper create mode 100755 scripts/wrappers/dbctl.py diff --git a/microk8s-resources/wrappers/microk8s-dbctl.wrapper b/microk8s-resources/wrappers/microk8s-dbctl.wrapper new file mode 100755 index 0000000000..6de4e3abf4 --- /dev/null +++ b/microk8s-resources/wrappers/microk8s-dbctl.wrapper @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -eu + +export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" +ARCH="$($SNAP/bin/uname -m)" +export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" +export PYTHONNOUSERSITE=false + +source $SNAP/actions/common/utils.sh + +exit_if_no_permissions + +LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dbctl.py $@ diff --git a/scripts/wrappers/dbctl.py b/scripts/wrappers/dbctl.py new file mode 100755 index 0000000000..19816bf890 --- /dev/null +++ b/scripts/wrappers/dbctl.py @@ -0,0 +1,153 @@ +#!/usr/bin/python3 +import os +import argparse + +import tempfile +import datetime +import subprocess +import tarfile +import os.path + +from common.utils import ( + exit_if_no_permission, + is_cluster_locked, + is_ha_enabled, +) + + +def kine_exists(): + """ + Check the existence of the kine socket + :return: True if the kine socket exists + """ + kine_socket = "/var/snap/microk8s/current/var/kubernetes/backend/kine.sock" + return os.path.exists(kine_socket) + + +def generate_backup_name(): + """ + Generate a filename based on the current time and date + :return: a generated filename + """ + now = datetime.datetime.now() + return "backup-{}".format(now.strftime("%Y-%m-%d-%H-%M-%S")) + + +def run_command(command): + """ + Run a command while printing the output + :param command: the command to run + :return: the return code of the command + """ + process = subprocess.Popen(command.split(), stdout=subprocess.PIPE) + while True: + output = process.stdout.readline() + if (not output or output == '') and process.poll() is not None: + break + if output: + print(output.decode().strip()) + rc = process.poll() + return rc + + +def backup(fname=None, debug=False): + """ + Backup the database to a provided file + :param fname_tar: the tar file + :param debug: show debug output + """ + snap_path = os.environ.get('SNAP') + snapdata_path = os.environ.get('SNAP_DATA') + # snap_path = '/snap/microk8s/current' + # snapdata_path = '/var/snap/microk8s/current' + + if not fname: + fname = generate_backup_name() + if fname.endswith('.tar.gz'): + fname = fname[:-7] + fname_tar = '{}.tar.gz'.format(fname) + + with tempfile.TemporaryDirectory() as tmpdirname: + backup_cmd = '{}/bin/migrator --mode backup-dqlite --db-dir {}'.format( + snap_path, "{}/{}".format(tmpdirname, fname) + ) + if debug: + backup_cmd = "{} {}".format(backup_cmd, "--debug") + try: + rc = run_command(backup_cmd) + if rc > 0: + print("Backup process failed. {}".format(rc)) + exit(1) + with tarfile.open(fname_tar, "w:gz") as tar: + tar.add( + "{}/{}".format(tmpdirname, fname), + arcname=os.path.basename("{}/{}".format(tmpdirname, fname)), + ) + + target_file = '{}/var/tmp/{}'.format(snapdata_path, fname_tar) + print("The backup is: {}".format(fname_tar)) + except subprocess.CalledProcessError as e: + print("Backup process failed. {}".format(e)) + exit(2) + + +def restore(fname_tar, debug=False): + """ + Restore the database from the provided file + :param fname_tar: the tar file + :param debug: show debug output + """ + snap_path = os.environ.get('SNAP') + # snap_path = '/snap/microk8s/current' + with tempfile.TemporaryDirectory() as tmpdirname: + with tarfile.open(fname_tar, "r:gz") as tar: + tar.extractall(path=tmpdirname) + if fname_tar.endswith('.tar.gz'): + fname = fname_tar[:-7] + else: + fname = fname_tar + fname = os.path.basename(fname) + restore_cmd = '{}/bin/migrator --mode restore-to-dqlite --db-dir {}'.format( + snap_path, "{}/{}".format(tmpdirname, fname) + ) + if debug: + restore_cmd = "{} {}".format(restore_cmd, "--debug") + try: + rc = run_command(restore_cmd) + if rc > 0: + print("Restore process failed. {}".format(rc)) + exit(3) + except subprocess.CalledProcessError as e: + print("Restore process failed. {}".format(e)) + exit(4) + + +if __name__ == '__main__': + exit_if_no_permission() + is_cluster_locked() + + if not kine_exists() or not is_ha_enabled(): + print("Please ensure the kubernetes apiserver is running and HA is enabled.") + exit(10) + + # initiate the parser with a description + parser = argparse.ArgumentParser( + description="backup and restore the Kubernetes datastore.", prog='microk8s dbctl' + ) + parser.add_argument('--debug', action='store_true', help='print debug output') + commands = parser.add_subparsers(title='commands', help='backup and restore operations') + restore_parser = commands.add_parser("restore") + restore_parser.add_argument('backup-file', help='name of file with the backup') + backup_parser = commands.add_parser("backup") + backup_parser.add_argument('-o', metavar='backup-file', help='output filename') + args = parser.parse_args() + + if 'backup-file' in args: + fname = vars(args)['backup-file'] + print("Restoring from {}".format(fname)) + restore(fname, args.debug) + elif 'o' in args: + print("Backing up the datastore") + backup(vars(args)['o'], args.debug) + else: + parser.print_help() diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 894b831bd8..cf70636718 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -96,6 +96,8 @@ apps: command: microk8s-cilium.wrapper juju: command: microk8s-juju.wrapper + dbctl: + command: microk8s-dbctl.wrapper parts: libco: diff --git a/tests/test-addons.py b/tests/test-addons.py index e139baf1d1..0ad36b0cda 100644 --- a/tests/test-addons.py +++ b/tests/test-addons.py @@ -30,7 +30,7 @@ microk8s_disable, microk8s_reset, ) -from subprocess import Popen, PIPE, STDOUT, CalledProcessError +from subprocess import Popen, PIPE, STDOUT, CalledProcessError, check_call class TestAddons(object): @@ -303,3 +303,13 @@ def test_ambassador(self): validate_ambassador() print("Disabling Ambassador") microk8s_disable("ambassador") + + def test_backup_restore(self): + """ + Test backup and restore commands. + """ + print('Checking dbctl backup and restore') + if os.path.exists('backupfile.tar.gz'): + os.remove('backupfile.tar.gz') + check_call("/snap/bin/microk8s.dbctl --debug backup -o backupfile".split()) + check_call("/snap/bin/microk8s.dbctl --debug restore backupfile.tar.gz".split()) diff --git a/tests/test-cluster.py b/tests/test-cluster.py index 619a0a8890..77ce7e998e 100644 --- a/tests/test-cluster.py +++ b/tests/test-cluster.py @@ -139,11 +139,6 @@ def setup_cluster(self): for vm_name in reuse_vms: self.VM.append(VM(vm_name)) - # enable HA - for vm in self.VM: - print('Enabling ha-cluster on machine {}'.format(vm.vm_name)) - vm.run('/snap/bin/microk8s.enable ha-cluster') - # Form cluster vm_master = self.VM[0] connected_nodes = vm_master.run('/snap/bin/microk8s.kubectl get no') @@ -213,8 +208,8 @@ def test_nodes_in_ha(self): while True: assert attempt > 0 for vm in self.VM: - status = vm.run('/snap/bin/microk8s.status ha-cluster') - if "The cluster is highly available" not in status.decode(): + status = vm.run('/snap/bin/microk8s.status') + if "high-availability: yes" not in status.decode(): attempt += 1 continue break @@ -246,8 +241,8 @@ def test_nodes_in_ha(self): while True: assert attempt > 0 for vm in leftVMs: - status = vm.run('/snap/bin/microk8s.status ha-cluster') - if "HA cluster has not formed yet" not in status.decode(): + status = vm.run('/snap/bin/microk8s.status') + if "high-availability: no" not in status.decode(): attempt += 1 time.sleep(2) continue @@ -295,8 +290,8 @@ def test_nodes_in_ha(self): while True: assert attempt > 0 for vm in self.VM: - status = vm.run('/snap/bin/microk8s.status ha-cluster') - if "The cluster is highly available" not in status.decode(): + status = vm.run('/snap/bin/microk8s.status') + if "high-availability: yes" not in status.decode(): attempt += 1 time.sleep(2) continue