diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 79c5fda..61855fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,6 +52,15 @@ jobs: INSERT INTO books (name, author) VALUES ($$Fittstim$$, $$Linda Skugge$$), ($$DSM-5$$, $$American Psychiatric Association$$); + + CREATE TABLE movies ( + id serial PRIMARY KEY, + name VARCHAR ( 128 ) UNIQUE NOT NULL, + director VARCHAR (128 ) NOT NULL + ); + INSERT INTO movies (name, director) VALUES + ($$Beau Travail$$, $$Claire Denis$$), + ($$Reservoir Dogs$$, $$Quentin Tarantino$$); ' options: > -e PGPASSWORD=test @@ -110,6 +119,25 @@ jobs: -e AWS_DEFAULT_REGION=us-east-1 -e FILENAME=test_${{ matrix.version }}_env + - name: Take Backup (using PG_DUMP_EXTRA_OPTIONS) + uses: addnab/docker-run-action@main + with: + image: heyman/postgresql-backup:latest + run: python3 -u /backup/backup.py + options: > + -e S3_EXTRA_OPTIONS='--endpoint-url=http://s3:8000' + -e DB_USE_ENV=True + -e PGHOST=postgres + -e PGPASSWORD=test + -e PGUSER=test + -e PGDATABASE=test_${{ matrix.version }} + -e S3_PATH=s3://test-postgresql-backup/backups + -e AWS_ACCESS_KEY_ID=access_key + -e AWS_SECRET_ACCESS_KEY=secret + -e AWS_DEFAULT_REGION=us-east-1 + -e FILENAME=test_${{ matrix.version }}_exclude + -e PG_DUMP_EXTRA_OPTIONS='--exclude-table=movies' + - name: Check equality uses: addnab/docker-run-action@main with: @@ -132,6 +160,7 @@ jobs: run: > psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -c ' DROP TABLE books; + DROP TABLE movies; ' options: > -e PGPASSWORD=test @@ -186,6 +215,7 @@ jobs: run: > psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -c ' DROP TABLE books; + DROP TABLE movies; ' && [[ "0" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c ' SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$books$$; '` ]] @@ -196,7 +226,7 @@ jobs: uses: addnab/docker-run-action@main with: image: heyman/postgresql-backup:latest - run: python3 -u /backup/restore.py test_${{ matrix.version }} + run: python3 -u /backup/restore.py test_${{ matrix.version }}_env options: > -e S3_EXTRA_OPTIONS='--endpoint-url=http://s3:8000' -e DB_USE_ENV=True @@ -221,4 +251,51 @@ jobs: SELECT name FROM books WHERE author=$$Linda Skugge$$; '` ]] options: > - -e PGPASSWORD=test \ No newline at end of file + -e PGPASSWORD=test + + - name: Clear DB table + uses: addnab/docker-run-action@main + with: + image: postgres:${{ matrix.version }} + shell: bash + run: > + psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -c ' + DROP TABLE books; + DROP TABLE movies; + ' && [[ "0" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c ' + SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$books$$; + '` ]] + options: > + -e PGPASSWORD=test + + - name: Restore Backup (PG_DUMP_EXTRA_OPTIONS) + uses: addnab/docker-run-action@main + with: + image: heyman/postgresql-backup:latest + run: python3 -u /backup/restore.py test_${{ matrix.version }}_exclude + options: > + -e S3_EXTRA_OPTIONS='--endpoint-url=http://s3:8000' + -e DB_HOST=postgres + -e DB_PASS=test + -e DB_USER=test + -e DB_NAME=test_${{ matrix.version }} + -e S3_PATH=s3://test-postgresql-backup/backups + -e AWS_ACCESS_KEY_ID=access_key + -e AWS_SECRET_ACCESS_KEY=secret + -e AWS_DEFAULT_REGION=us-east-1 + + - name: Check that table got imported (PG_DUMP_EXTRA_OPTIONS) + uses: addnab/docker-run-action@main + with: + image: postgres:${{ matrix.version }} + shell: bash + run: > + [[ "1" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c ' + SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$books$$; + '` ]] && [[ "Fittstim" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c ' + SELECT name FROM books WHERE author=$$Linda Skugge$$; + '` ]] && [[ "0" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c ' + SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$movies$$; + '` ]] + options: > + -e PGPASSWORD=test diff --git a/10/backup.py b/10/backup.py index 598b7bd..c4bf67c 100644 --- a/10/backup.py +++ b/10/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/10/restore.py b/10/restore.py index ed5eadb..4ce1d26 100644 --- a/10/restore.py +++ b/10/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ])) diff --git a/11/backup.py b/11/backup.py index 598b7bd..c4bf67c 100644 --- a/11/backup.py +++ b/11/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/11/restore.py b/11/restore.py index ed5eadb..4ce1d26 100644 --- a/11/restore.py +++ b/11/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ])) diff --git a/12/backup.py b/12/backup.py index 598b7bd..c4bf67c 100644 --- a/12/backup.py +++ b/12/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/12/restore.py b/12/restore.py index ed5eadb..4ce1d26 100644 --- a/12/restore.py +++ b/12/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ])) diff --git a/13/backup.py b/13/backup.py index 598b7bd..c4bf67c 100644 --- a/13/backup.py +++ b/13/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/13/restore.py b/13/restore.py index ed5eadb..4ce1d26 100644 --- a/13/restore.py +++ b/13/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ])) diff --git a/14/backup.py b/14/backup.py index 598b7bd..c4bf67c 100644 --- a/14/backup.py +++ b/14/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/14/restore.py b/14/restore.py index ed5eadb..4ce1d26 100644 --- a/14/restore.py +++ b/14/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ])) diff --git a/15/backup.py b/15/backup.py index 598b7bd..c4bf67c 100644 --- a/15/backup.py +++ b/15/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/15/restore.py b/15/restore.py index ed5eadb..4ce1d26 100644 --- a/15/restore.py +++ b/15/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ])) diff --git a/16/backup.py b/16/backup.py index 598b7bd..c4bf67c 100644 --- a/16/backup.py +++ b/16/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/16/restore.py b/16/restore.py index ed5eadb..4ce1d26 100644 --- a/16/restore.py +++ b/16/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ])) diff --git a/README.md b/README.md index 3563f5b..249b891 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ docker run -it --rm --name=pgbackup \ * `WEBHOOK_DATA`: Add a body to the webhook being called, unless changed it implies that `POST` method is used. E.g. `{"text":"Backup completed at %(date)s %(time)s!"}` * `KEEP_BACKUP_DAYS`: The number of days to keep backups for when pruning old backups. Defaults to `7`. * `FILENAME`: String that is passed into `strftime()` and used as the backup dump's filename. Defaults to `$DB_NAME_%Y-%m-%d`. +* `PG_DUMP_EXTRA_OPTIONS`: Specify additional options for `pg_dump`, e.g. `--exclude-table-data=table_name` to exclude table data from the dump. ### Interpolation diff --git a/template/backup.py b/template/backup.py index 598b7bd..c4bf67c 100644 --- a/template/backup.py +++ b/template/backup.py @@ -33,6 +33,7 @@ WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or "" KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7)) FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d") +PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or "" file_name = dt.strftime(FILENAME) backup_file = os.path.join(BACKUP_DIR, file_name) @@ -69,7 +70,14 @@ def take_backup(): env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT}) # trigger postgres-backup - cmd("pg_dump -Fc > %s" % backup_file, env=env) + command = [ + "pg_dump", + "-Fc", + ] + if PG_DUMP_EXTRA_OPTIONS: + command.append(PG_DUMP_EXTRA_OPTIONS) + command.append("> %s" % backup_file) + cmd(" ".join(command), env=env) def upload_backup(): opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS) diff --git a/template/restore.py b/template/restore.py index ed5eadb..4ce1d26 100644 --- a/template/restore.py +++ b/template/restore.py @@ -35,7 +35,7 @@ def cmd(command, **kwargs): sys.stderr.write("\n".join([ "Command execution failed. Output:", "-"*80, - e.output, + e.output.decode(), "-"*80, "" ]))