forked from ShahriyarR/MySQL-AutoXtraBackup
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathautoxtrabackup.py
342 lines (305 loc) · 13.2 KB
/
autoxtrabackup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
import click
import humanfriendly
import logging
import logging.handlers
import os
import pid
import re
import time
import sys
from logging.handlers import RotatingFileHandler
from sys import platform as _platform
from sys import exit
from backup_prepare.prepare import Prepare
from general_conf.generalops import GeneralClass
from general_conf import path_config
from master_backup_script.backuper import Backup
from partial_recovery.partial import PartialRecovery
from prepare_env_test_mode.runner_test_mode import RunnerTestMode
from process_runner.process_runner import ProcessRunner
logger = logging.getLogger('')
destinations_hash = {'linux': '/dev/log', 'linux2': '/dev/log', 'darwin': '/var/run/syslog'}
def address_matcher(plt):
return destinations_hash.get(plt, ('localhost', 514))
handler = logging.handlers.SysLogHandler(address=address_matcher(_platform))
# Set syslog for the root logger
logger.addHandler(handler)
def print_help(ctx, param, value):
if value is False:
return
click.echo(ctx.get_help())
ctx.exit()
def print_version(ctx, param, value):
if not value or ctx.resilient_parsing:
return
click.echo(
"Developed by Shahriyar Rzayev from Azerbaijan MUG(http://mysql.az)")
click.echo("Link : https://github.com/ShahriyarR/MySQL-AutoXtraBackup")
click.echo("Email: [email protected]")
click.echo(
"Based on Percona XtraBackup: https://github.com/percona/percona-xtrabackup/")
click.echo('MySQL-AutoXtraBackup Version: 1.5.5')
ctx.exit()
def check_file_content(file):
"""Check if all mandatory headers and keys exist in file"""
with open(file, 'r') as config_file:
file_content = config_file.read()
config_headers = ["MySQL", "Backup", "Encrypt", "Compress", "Commands"]
config_keys = [
"mysql",
"mycnf",
"mysqladmin",
"mysql_user",
"mysql_password",
"mysql_host",
"datadir",
"tmp_dir",
"backup_dir",
"backup_tool",
"xtra_prepare",
"start_mysql_command",
"stop_mysql_command",
"chown_command"]
for header in config_headers:
if header not in file_content:
raise KeyError(
"Mandatory header [%s] doesn't exist in %s" %
(header, file))
for key in config_keys:
if key not in file_content:
raise KeyError(
"Mandatory key \'%s\' doesn't exists in %s." %
(key, file))
return True
def validate_file(file):
"""
Check for validity of the file given in file path. If file doesn't exist or invalid
configuration file, throw error.
"""
if os.path.isfile(file):
# filename extension should be .cnf
pattern = re.compile(r'.*\.cnf')
if pattern.match(file):
# Lastly the file should have all 5 required headers
if check_file_content(file):
return
else:
raise ValueError("Invalid file extension. Expecting .cnf")
else:
raise FileNotFoundError("Specified file does not exist.")
@click.command()
@click.option('--dry-run', is_flag=True, help="Enable the dry run.")
@click.option('--prepare', is_flag=True, help="Prepare/recover backups.")
@click.option('--backup',
is_flag=True,
help="Take full and incremental backups.")
@click.option('--partial',
is_flag=True,
help="Recover specified table (partial recovery).")
@click.option('--version',
is_flag=True,
callback=print_version,
expose_value=False,
is_eager=True,
help="Version information.")
@click.option('--defaults-file',
default=path_config.config_path_file,
show_default=True,
help="Read options from the given file")
@click.option('--tag',
help="Pass the tag string for each backup")
@click.option('--show-tags',
is_flag=True,
help="Show backup tags and exit")
@click.option('-v', '--verbose', is_flag=True,
help="Be verbose (print to console)")
@click.option('-lf',
'--log-file',
default=path_config.log_file_path,
show_default=True,
help="Set log file")
@click.option('-l',
'--log',
'--log-level',
default='INFO',
show_default=True,
type=click.Choice(['DEBUG',
'INFO',
'WARNING',
'ERROR',
'CRITICAL']),
help="Set log level")
@click.option('--log-file-max-bytes',
default=1073741824,
show_default=True,
nargs=1,
type=int,
help="Set log file max size in bytes")
@click.option('--log-file-backup-count',
default=7,
show_default=True,
nargs=1,
type=int,
help="Set log file backup count")
@click.option('--keyring-vault',
default=0,
show_default=True,
nargs=1,
type=int,
help="Enable this when you pass keyring_vault options in default mysqld options in config"
"[Only for using with --test-mode]")
@click.option('--test-mode',
is_flag=True,
help="Enable test mode. Must be used with --defaults-file and only for TESTs for XtraBackup")
@click.option('--help',
is_flag=True,
callback=print_help,
expose_value=False,
is_eager=False,
help="Print help message and exit.")
@click.pass_context
def all_procedure(ctx, prepare, backup, partial, tag, show_tags,
verbose, log_file, log, defaults_file,
dry_run, test_mode, log_file_max_bytes,
log_file_backup_count, keyring_vault):
config = GeneralClass(defaults_file)
formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s [%(module)s:%(lineno)d] %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
if verbose:
ch = logging.StreamHandler()
# control console output log level
ch.setLevel(logging.INFO)
ch.setFormatter(formatter)
logger.addHandler(ch)
if log_file:
try:
if config.log_file_max_bytes and config.log_file_backup_count:
file_handler = RotatingFileHandler(log_file, mode='a',
maxBytes=int(config.log_file_max_bytes),
backupCount=int(config.log_file_backup_count))
else:
file_handler = RotatingFileHandler(log_file, mode='a',
maxBytes=log_file_max_bytes, backupCount=log_file_backup_count)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
except PermissionError as err:
exit("{} Please consider to run as root or sudo".format(err))
# set log level in order: 1. user argument 2. config file 3. @click default
if log is not None:
logger.setLevel(log)
elif 'log_level' in config.__dict__:
logger.setLevel(config.log_level)
else:
# this is the fallback default log-level.
logger.setLevel('INFO')
validate_file(defaults_file)
pid_file = pid.PidFile(piddir=config.pid_dir)
try:
with pid_file: # User PidFile for locking to single instance
if (prepare is False and
backup is False and
partial is False and
verbose is False and
dry_run is False and
test_mode is False and
show_tags is False):
print_help(ctx, None, value=True)
elif show_tags and defaults_file:
b = Backup(config=defaults_file)
b.show_tags(backup_dir=b.backupdir)
elif test_mode and defaults_file:
logger.warning("Enabled Test Mode!!!")
logger.info("Starting Test Mode")
test_obj = RunnerTestMode(config=defaults_file)
for basedir in test_obj.basedirs:
if ('5.7' in basedir) and ('2_4_ps_5_7' in defaults_file):
if keyring_vault == 1:
test_obj.wipe_backup_prepare_copyback(basedir=basedir, keyring_vault=1)
else:
test_obj.wipe_backup_prepare_copyback(basedir=basedir)
elif ('8.0' in basedir) and ('8_0_ps_8_0' in defaults_file):
if keyring_vault == 1:
test_obj.wipe_backup_prepare_copyback(basedir=basedir, keyring_vault=1)
else:
test_obj.wipe_backup_prepare_copyback(basedir=basedir)
elif ('5.6' in basedir) and ('2_4_ps_5_6' in defaults_file):
test_obj.wipe_backup_prepare_copyback(basedir=basedir)
elif ('5.6' in basedir) and ('2_3_ps_5_6' in defaults_file):
test_obj.wipe_backup_prepare_copyback(basedir=basedir)
elif ('5.5' in basedir) and ('2_3_ps_5_5' in defaults_file):
test_obj.wipe_backup_prepare_copyback(basedir=basedir)
elif ('5.5' in basedir) and ('2_4_ps_5_5' in defaults_file):
test_obj.wipe_backup_prepare_copyback(basedir=basedir)
else:
logger.error("Please pass proper already generated config file!")
logger.error("Please check also if you have run prepare_env.bats file")
elif prepare and not test_mode:
if not dry_run:
if tag:
a = Prepare(config=defaults_file, tag=tag)
a.prepare_backup_and_copy_back()
else:
a = Prepare(config=defaults_file)
a.prepare_backup_and_copy_back()
else:
logger.warning("Dry run enabled!")
logger.warning("Do not recover/copy-back in this mode!")
if tag:
a = Prepare(config=defaults_file, dry_run=1, tag=tag)
a.prepare_backup_and_copy_back()
else:
a = Prepare(config=defaults_file, dry_run=1)
a.prepare_backup_and_copy_back()
elif backup and not test_mode:
if not dry_run:
if tag:
b = Backup(config=defaults_file, tag=tag)
b.all_backup()
else:
b = Backup(config=defaults_file)
b.all_backup()
else:
logger.warning("Dry run enabled!")
if tag:
b = Backup(config=defaults_file, dry_run=1, tag=tag)
b.all_backup()
else:
b = Backup(config=defaults_file, dry_run=1)
b.all_backup()
elif partial:
if not dry_run:
c = PartialRecovery(config=defaults_file)
c.final_actions()
else:
logger.critical("Dry run is not implemented for partial recovery!")
except pid.PidFileAlreadyLockedError as error:
if hasattr(config, 'pid_runtime_warning'):
if time.time() - os.stat(pid_file.filename).st_ctime > config.pid_runtime_warning:
pid.fh.seek(0)
pid_str = pid.fh.read(16).split("\n", 1)[0].strip()
logger.critical(
"Backup (pid: " + pid_str + ") has been running for logger than: " + str(
humanfriendly.format_timespan(
config.pid_runtime_warning)))
# logger.warn("Pid file already exists: " + str(error))
except pid.PidFileAlreadyRunningError as error:
if hasattr(config, 'pid_runtime_warning'):
if time.time() - os.stat(pid_file.filename).st_ctime > config.pid_runtime_warning:
pid.fh.seek(0)
pid_str = pid.fh.read(16).split("\n", 1)[0].strip()
logger.critical(
"Backup (pid: " + pid_str + ") has been running for logger than: " + str(
humanfriendly.format_timespan(
config.pid_runtime_warning)))
# logger.warn("Pid already running: " + str(error))
except pid.PidFileUnreadableError as error:
logger.warning("Pid file can not be read: " + str(error))
except pid.PidFileError as error:
logger.warning("Generic error with pid file: " + str(error))
logger.info("Xtrabackup command history:")
for i in ProcessRunner.xtrabackup_history_log:
logger.info(str(i))
logger.info("Autoxtrabackup completed successfully!")
return True
if __name__ == "__main__":
all_procedure()