Merge pull request #177 from Aiven-Open/fix_xtrabackup_params

Read xtrabackup settings from config
Aiven-Open · Oct 30, 2023 · ae06c2e · ae06c2e
2 parents 9fb9bcc + 8686395
commit ae06c2e
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 7 deletions.
diff --git a/myhoard/backup_stream.py b/myhoard/backup_stream.py
@@ -8,6 +8,7 @@
     add_gtid_ranges_to_executed_set,
     are_gtids_in_executed_set,
     DEFAULT_MYSQL_TIMEOUT,
+    DEFAULT_XTRABACKUP_SETTINGS,
     ERR_TIMEOUT,
     first_contains_gtids_not_in_second,
     GtidExecuted,
@@ -173,6 +174,7 @@ def __init__(
         stats: "StatsClient",
         stream_id: Optional[str] = None,
         temp_dir: str,
+        xtrabackup_settings: Optional[Dict[str, int]] = None,
     ) -> None:
         super().__init__()
         stream_id = stream_id or self.new_stream_id()
@@ -252,6 +254,7 @@ def __init__(
         self.stats = stats
         self.temp_dir = temp_dir
         self.wakeup_event = threading.Event()
+        self.xtrabackup_settings = DEFAULT_XTRABACKUP_SETTINGS | (xtrabackup_settings or {})
 
     @contextlib.contextmanager
     def running(self):
@@ -951,6 +954,9 @@ def _take_basebackup(self) -> None:
         start_time = time.time()
         encryption_key = os.urandom(24)
         self.basebackup_operation = BasebackupOperation(
+            copy_threads=self.xtrabackup_settings["copy_threads"],
+            compress_threads=self.xtrabackup_settings["compress_threads"],
+            encrypt_threads=self.xtrabackup_settings["encrypt_threads"],
             encryption_algorithm="AES256",
             encryption_key=encryption_key,
             mysql_client_params=self.mysql_client_params,

diff --git a/myhoard/controller.py b/myhoard/controller.py
@@ -157,6 +157,7 @@ def __init__(
         stats,
         temp_dir,
         restore_free_memory_percentage=None,
+        xtrabackup_settings: Dict[str, int],
     ):
         super().__init__()
         self.log = logging.getLogger(self.__class__.__name__)
@@ -228,6 +229,7 @@ def __init__(
         self.stats = stats
         self.temp_dir = temp_dir
         self.wakeup_event = threading.Event()
+        self.xtrabackup_settings = xtrabackup_settings
         self._get_upload_backup_site()
         self._update_mode_tag()
 
@@ -756,6 +758,7 @@ def _build_backup_stream(self, backup):
             stats=self.stats,
             stream_id=stream_id,
             temp_dir=self.temp_dir,
+            xtrabackup_settings=self.xtrabackup_settings,
         )
 
     def _delete_backup_stream_state(self, stream_id):
@@ -1728,6 +1731,7 @@ def _start_new_backup(self, *, backup_reason: BackupStream.BackupReason, normali
             stats=self.stats,
             stream_id=stream_id,
             temp_dir=self.temp_dir,
+            xtrabackup_settings=self.xtrabackup_settings,
         )
         self.backup_streams.append(stream)
         self.state_manager.update_state(

diff --git a/myhoard/myhoard.py b/myhoard/myhoard.py
@@ -2,7 +2,7 @@
 from myhoard import version
 from myhoard.controller import Controller
 from myhoard.statsd import StatsClient
-from myhoard.util import detect_running_process_id, wait_for_port
+from myhoard.util import DEFAULT_XTRABACKUP_SETTINGS, detect_running_process_id, wait_for_port
 from myhoard.web_server import WebServer
 
 import argparse
@@ -209,6 +209,7 @@ async def _start(self):
             state_dir=self.config["state_directory"],
             stats=statsd,
             temp_dir=self.config["temporary_directory"],
+            xtrabackup_settings=self.config.get("xtrabackup", DEFAULT_XTRABACKUP_SETTINGS),
         )
         self.controller.start()
         self.web_server = WebServer(

diff --git a/myhoard/util.py b/myhoard/util.py
@@ -32,6 +32,11 @@
 XTRABACKUP_VERSION_REGEX = re.compile(r"^xtrabackup version ([\d\.\-]).+")
 VERSION_SPLITTING_REGEX = re.compile(r"[\.-]")
 
+DEFAULT_XTRABACKUP_SETTINGS = {
+    "copy_threads": 1,
+    "compress_threads": 1,
+    "encrypt_threads": 1,
+}
 
 GtidRangeTuple = tuple[int, int, str, int, int]
 

diff --git a/test/__init__.py b/test/__init__.py
@@ -102,6 +102,7 @@ def build_controller(
         state_dir=state_dir,
         stats=build_statsd_client(),
         temp_dir=temp_dir,
+        xtrabackup_settings=myhoard_util.DEFAULT_XTRABACKUP_SETTINGS,
     )
     return controller
 

diff --git a/test/conftest.py b/test/conftest.py
@@ -9,7 +9,7 @@
     random_basic_string,
 )
 from myhoard.controller import BackupSiteInfo, Controller
-from myhoard.util import atomic_create_file, change_master_to, mysql_cursor, wait_for_port
+from myhoard.util import atomic_create_file, change_master_to, DEFAULT_XTRABACKUP_SETTINGS, mysql_cursor, wait_for_port
 from myhoard.web_server import WebServer
 from py.path import local as LocalPath
 from typing import Callable, Iterator, Optional
@@ -382,11 +382,7 @@ def fixture_myhoard_config(default_backup_site, mysql_master, session_tmpdir):
         ],
         "systemd_service": None,
         "temporary_directory": temp_dir,
-        "xtrabackup": {
-            "copy_threads": 1,
-            "compress_threads": 1,
-            "encrypt_threads": 1,
-        },
+        "xtrabackup": DEFAULT_XTRABACKUP_SETTINGS,
     }