diff --git a/config.yml b/config.yml index 025bcf38..de7ff554 100644 --- a/config.yml +++ b/config.yml @@ -50,7 +50,7 @@ config: # cron tab configuration for base backups base_backups: # number of days base backups are kept/stored (0 means forever) - keep_days: 30 + keep_days: 15 # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc ) weekday: '*' # Day of the month the job should run ( 1-31, *, */2, etc ) diff --git a/doc/backups.md b/doc/backups.md index 242b738b..e4e50a47 100644 --- a/doc/backups.md +++ b/doc/backups.md @@ -14,7 +14,7 @@ The ansible deployment configuration enables automatically making backups of the Backups are stored on /var/postgres_backups or where the config.yml variable config.postgres_backups.folder points to. SQL dump files are on /var/postgres_backups/dump, base backups are on /var/postgres_backups/base and WAL files are on /var/postgres_backups/wal -In the backup system that deployment-tool installs, a SQL dump and a base backup are done daily (or with the frequency that you configure with config.postgres_backups.base_backups). For that, a cron task is configured to call /usr/bin/create_backup_postgres.sh. That script calls pg_dumpall to create the SQL dump file and pg_basebackup to create the continuous archiving base backup, so two different types of backup are created. It also calls /usr/bin/clean_old_postgres_backups.sh script, which will clean base and dump backups (not wal files) that are older than the parameter configured in config.postgres_backups.base_backups.keep_days. Also, at least every ten minutes (configured in config.postgres_backups.archive_timeout) a new WAL file is created. All backups are compressed in order to minimize the space used. +In the backup system that deployment-tool installs, a SQL dump and a base backup are done daily (or with the frequency that you configure with config.postgres_backups.base_backups). For that, a cron task is configured to call /usr/bin/create_backup_postgres.sh. That script calls pg_dumpall to create the SQL dump file and pg_basebackup to create the continuous archiving base backup, so two different types of backup are created. It also calls /usr/bin/clean_old_postgres_backups.sh script, which will clean base, dump and wal backups that are older than the parameter configured in config.postgres_backups.base_backups.keep_days. Also, at least every ten minutes (configured in config.postgres_backups.archive_timeout) a new WAL file is created. All backups are compressed in order to minimize the space used. The name of the base and dump backups is the date up to the second when the backup was created. For example, base backup files use this format: /var/postgres_backups/day_month_year_hour_min_sec/base.tar.gz and dump backups use this format: /var/postgres_backups/dump_day_month_year_hour_min_sec.gz diff --git a/doc/devel/auth1.config.yml b/doc/devel/auth1.config.yml index 2739c7cd..857861ad 100644 --- a/doc/devel/auth1.config.yml +++ b/doc/devel/auth1.config.yml @@ -51,7 +51,7 @@ config: # cron tab configuration for base backups base_backups: # number of days base backups are kept/stored (0 means forever) - keep_days: 30 + keep_days: 15 # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc ) weekday: '*' # Day of the month the job should run ( 1-31, *, */2, etc ) diff --git a/doc/devel/auth2.config.yml b/doc/devel/auth2.config.yml index ff0b2db5..33d41b92 100644 --- a/doc/devel/auth2.config.yml +++ b/doc/devel/auth2.config.yml @@ -51,7 +51,7 @@ config: # cron tab configuration for base backups base_backups: # number of days base backups are kept/stored (0 means forever) - keep_days: 30 + keep_days: 15 # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc ) weekday: '*' # Day of the month the job should run ( 1-31, *, */2, etc ) diff --git a/doc/devel/sequent.config.yml b/doc/devel/sequent.config.yml index 0d6bef8b..83bc6a1e 100644 --- a/doc/devel/sequent.config.yml +++ b/doc/devel/sequent.config.yml @@ -51,7 +51,7 @@ config: # cron tab configuration for base backups base_backups: # number of days base backups are kept/stored (0 means forever) - keep_days: 30 + keep_days: 15 # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc ) weekday: '*' # Day of the month the job should run ( 1-31, *, */2, etc ) diff --git a/doc/production/config.auth.yml b/doc/production/config.auth.yml index 235ab43e..90b748a0 100644 --- a/doc/production/config.auth.yml +++ b/doc/production/config.auth.yml @@ -51,7 +51,7 @@ config: # cron tab configuration for base backups base_backups: # number of days base backups are kept/stored (0 means forever) - keep_days: 30 + keep_days: 15 # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc ) weekday: '*' # Day of the month the job should run ( 1-31, *, */2, etc ) diff --git a/doc/production/config.master.yml b/doc/production/config.master.yml index 1160fe38..9f453fec 100644 --- a/doc/production/config.master.yml +++ b/doc/production/config.master.yml @@ -51,7 +51,7 @@ config: # cron tab configuration for base backups base_backups: # number of days base backups are kept/stored (0 means forever) - keep_days: 30 + keep_days: 15 # day of the week that the job should run ( 0-6 for Sunday-Saturday, *, etc ) weekday: '*' # Day of the month the job should run ( 1-31, *, */2, etc ) diff --git a/templates/clean_old_postgres_backups.sh b/templates/clean_old_postgres_backups.sh index f5032817..98adcc60 100644 --- a/templates/clean_old_postgres_backups.sh +++ b/templates/clean_old_postgres_backups.sh @@ -1,65 +1,57 @@ #!/bin/bash # This file is part of deployment-tool. -# Copyright (C) 2017 Sequent Tech Inc +# Copyright (C) 2017 Sequent Tech Inc # deployment-tool is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the License. -# deployment-tool is distributed in the hope that it will be useful, +# deployment-tool is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public License -# along with deployment-tool. If not, see . +# along with deployment-tool. If not, see . # Automatically exit bash on any error inside the exit set -e BACKUP_DIR={{ config.postgres_backups.folder }} KEEP_DAYS={{ config.postgres_backups.base_backups.keep_days }} -DATE_NOW=`date +%s` +DATE_NOW=$(date +%s) if [ "$KEEP_DAYS" -eq "0" ]; then echo "keeping backups indefinitely" exit 0 fi -DATE_OLDEST=`echo "$DATE_NOW - (24*3600*$KEEP_DAYS)" | bc` +DATE_OLDEST=$(echo "$DATE_NOW - (24*3600*$KEEP_DAYS)" | bc) -LIST_BASE=`ls $BACKUP_DIR/base` -LIST_DUMP=`ls $BACKUP_DIR/dump` +delete_old_files() { + local directory=$1 + local list=$2 -for FNAME in $LIST_BASE -do - RAW_DATE=`echo $FNAME | sed 's/\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)/20\3-\2-\1 \4:\5:\6/'` - if [ "$RAW_DATE" == "$FNAME" ]; then - continue - fi - FDATE=`date --date="$RAW_DATE" +%s` + for FNAME in $list; do + # Get file's last modification time in seconds since the epoch + local FDATE=$(date -r "$directory/$FNAME" +%s) - # check date and remove if it's too old - if [ "$DATE_OLDEST" -gt "$FDATE" ]; then - echo "deleting $BACKUP_DIR/base/$FNAME" - rm -Rf $BACKUP_DIR/base/$FNAME - fi -done + # check date and remove if it's too old + if [ "$DATE_OLDEST" -gt "$FDATE" ]; then + local full_path="$directory/$FNAME" + echo "deleting $full_path" + rm -rf "$full_path" + fi + done +} -for FNAME in $LIST_DUMP -do - RAW_DATE=`echo $FNAME | sed 's/dump_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)_\([[:digit:]]*\)\.gz/20\3-\2-\1 \4:\5:\6/'` - if [ "$RAW_DATE" == "$FNAME" ]; then - continue - fi - FDATE=`date --date="$RAW_DATE" +%s` +LIST_BASE=$(ls "$BACKUP_DIR/base") +LIST_DUMP=$(ls "$BACKUP_DIR/dump") +LIST_WAL=$(ls "$BACKUP_DIR/wal") - # check date and remove if it's too old - if [ "$DATE_OLDEST" -gt "$FDATE" ]; then - echo "deleting $BACKUP_DIR/base/$FNAME" - rm -Rf $BACKUP_DIR/base/$FNAME - fi -done +delete_old_files "$BACKUP_DIR/base" "$LIST_BASE" +delete_old_files "$BACKUP_DIR/dump" "$LIST_DUMP" +delete_old_files "$BACKUP_DIR/wal" "$LIST_WAL" -exit 0 \ No newline at end of file +exit 0