Add PG_DUMP_EXTRA_OPTIONS argument (#19)

Can be used to specify additional parameters for pg_dump
heyman · Jan 31, 2024 · ba6885c · ba6885c
1 parent b16d286
commit ba6885c
Show file tree

Hide file tree

Showing 18 changed files with 160 additions and 18 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -52,6 +52,15 @@ jobs:
               INSERT INTO books (name, author) VALUES
                 ($$Fittstim$$, $$Linda Skugge$$),
                 ($$DSM-5$$, $$American Psychiatric Association$$);
+              
+              CREATE TABLE movies (
+                id serial PRIMARY KEY,
+                name VARCHAR ( 128 ) UNIQUE NOT NULL,
+                director VARCHAR (128 ) NOT NULL
+              );
+              INSERT INTO movies (name, director) VALUES
+                ($$Beau Travail$$, $$Claire Denis$$),
+                ($$Reservoir Dogs$$, $$Quentin Tarantino$$);
             '
           options: >
             -e PGPASSWORD=test
@@ -110,6 +119,25 @@ jobs:
             -e AWS_DEFAULT_REGION=us-east-1
             -e FILENAME=test_${{ matrix.version }}_env
       
+      - name: Take Backup (using PG_DUMP_EXTRA_OPTIONS)
+        uses: addnab/docker-run-action@main
+        with:
+          image: heyman/postgresql-backup:latest
+          run: python3 -u /backup/backup.py
+          options: >
+            -e S3_EXTRA_OPTIONS='--endpoint-url=http://s3:8000' 
+            -e DB_USE_ENV=True
+            -e PGHOST=postgres
+            -e PGPASSWORD=test
+            -e PGUSER=test
+            -e PGDATABASE=test_${{ matrix.version }}
+            -e S3_PATH=s3://test-postgresql-backup/backups
+            -e AWS_ACCESS_KEY_ID=access_key
+            -e AWS_SECRET_ACCESS_KEY=secret
+            -e AWS_DEFAULT_REGION=us-east-1
+            -e FILENAME=test_${{ matrix.version }}_exclude
+            -e PG_DUMP_EXTRA_OPTIONS='--exclude-table=movies'
+      
       - name: Check equality
         uses: addnab/docker-run-action@main
         with:
@@ -132,6 +160,7 @@ jobs:
           run: >
             psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -c '
               DROP TABLE books;
+              DROP TABLE movies;
             '
           options: >
             -e PGPASSWORD=test
@@ -186,6 +215,7 @@ jobs:
           run: >
             psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -c '
               DROP TABLE books;
+              DROP TABLE movies;
             ' && [[ "0" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c '
               SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$books$$;
             '` ]]
@@ -196,7 +226,7 @@ jobs:
         uses: addnab/docker-run-action@main
         with:
           image: heyman/postgresql-backup:latest
-          run: python3 -u /backup/restore.py test_${{ matrix.version }}
+          run: python3 -u /backup/restore.py test_${{ matrix.version }}_env
           options: >
             -e S3_EXTRA_OPTIONS='--endpoint-url=http://s3:8000' 
             -e DB_USE_ENV=True
@@ -221,4 +251,51 @@ jobs:
                 SELECT name FROM books WHERE author=$$Linda Skugge$$;
               '` ]]
           options: >
-            -e PGPASSWORD=test
+            -e PGPASSWORD=test
+      
+      - name: Clear DB table
+        uses: addnab/docker-run-action@main
+        with:
+          image: postgres:${{ matrix.version }}
+          shell: bash
+          run: >
+            psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -c '
+              DROP TABLE books;
+              DROP TABLE movies;
+            ' && [[ "0" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c '
+              SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$books$$;
+            '` ]]
+          options: >
+            -e PGPASSWORD=test
+      
+      - name: Restore Backup (PG_DUMP_EXTRA_OPTIONS)
+        uses: addnab/docker-run-action@main
+        with:
+          image: heyman/postgresql-backup:latest
+          run: python3 -u /backup/restore.py test_${{ matrix.version }}_exclude
+          options: >
+            -e S3_EXTRA_OPTIONS='--endpoint-url=http://s3:8000' 
+            -e DB_HOST=postgres
+            -e DB_PASS=test
+            -e DB_USER=test
+            -e DB_NAME=test_${{ matrix.version }}
+            -e S3_PATH=s3://test-postgresql-backup/backups
+            -e AWS_ACCESS_KEY_ID=access_key
+            -e AWS_SECRET_ACCESS_KEY=secret
+            -e AWS_DEFAULT_REGION=us-east-1
+
+      - name: Check that table got imported (PG_DUMP_EXTRA_OPTIONS)
+        uses: addnab/docker-run-action@main
+        with:
+          image: postgres:${{ matrix.version }}
+          shell: bash
+          run: >
+            [[ "1" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c '
+                SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$books$$;
+              '` ]] && [[ "Fittstim" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c '
+                SELECT name FROM books WHERE author=$$Linda Skugge$$;
+              '` ]] && [[ "0" == `psql -d test_${{ matrix.version }} -U test -h postgres -p ${{ job.services.postgres.ports[5432] }} -A -t -c '
+                SELECT count(*) FROM pg_catalog.pg_tables WHERE tablename=$$movies$$;
+              '` ]]
+          options: >
+            -e PGPASSWORD=test
diff --git a/10/backup.py b/10/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/10/restore.py b/10/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))

diff --git a/11/backup.py b/11/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/11/restore.py b/11/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))

diff --git a/12/backup.py b/12/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/12/restore.py b/12/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))

diff --git a/13/backup.py b/13/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/13/restore.py b/13/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))

diff --git a/14/backup.py b/14/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/14/restore.py b/14/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))

diff --git a/15/backup.py b/15/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/15/restore.py b/15/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))

diff --git a/16/backup.py b/16/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/16/restore.py b/16/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))

diff --git a/README.md b/README.md
@@ -58,6 +58,7 @@ docker run -it --rm --name=pgbackup \
 * `WEBHOOK_DATA`: Add a body to the webhook being called, unless changed it implies that `POST` method is used. E.g. `{"text":"Backup completed at %(date)s %(time)s!"}`
 * `KEEP_BACKUP_DAYS`: The number of days to keep backups for when pruning old backups. Defaults to `7`.
 * `FILENAME`: String that is passed into `strftime()` and used as the backup dump's filename. Defaults to `$DB_NAME_%Y-%m-%d`.
+* `PG_DUMP_EXTRA_OPTIONS`: Specify additional options for `pg_dump`, e.g. `--exclude-table-data=table_name` to exclude table data from the dump.
 
 ### Interpolation
 

diff --git a/template/backup.py b/template/backup.py
@@ -33,6 +33,7 @@
 WEBHOOK_CURL_OPTIONS = os.environ.get("WEBHOOK_CURL_OPTIONS") or ""
 KEEP_BACKUP_DAYS = int(os.environ.get("KEEP_BACKUP_DAYS", 7))
 FILENAME = os.environ.get("FILENAME", DB_NAME + "_%Y-%m-%d")
+PG_DUMP_EXTRA_OPTIONS = os.environ.get("PG_DUMP_EXTRA_OPTIONS") or ""
 
 file_name = dt.strftime(FILENAME)
 backup_file = os.path.join(BACKUP_DIR, file_name)
@@ -69,7 +70,14 @@ def take_backup():
         env.update({'PGPASSWORD': DB_PASS, 'PGHOST': DB_HOST, 'PGUSER': DB_USER, 'PGDATABASE': DB_NAME, 'PGPORT': DB_PORT})
 
     # trigger postgres-backup
-    cmd("pg_dump -Fc > %s" % backup_file, env=env)
+    command = [
+        "pg_dump",
+        "-Fc",
+    ]
+    if PG_DUMP_EXTRA_OPTIONS:
+        command.append(PG_DUMP_EXTRA_OPTIONS)
+    command.append("> %s" % backup_file)
+    cmd(" ".join(command), env=env)
 
 def upload_backup():
     opts = "--storage-class=%s %s" % (S3_STORAGE_CLASS, S3_EXTRA_OPTIONS)

diff --git a/template/restore.py b/template/restore.py
@@ -35,7 +35,7 @@ def cmd(command, **kwargs):
         sys.stderr.write("\n".join([
             "Command execution failed. Output:",
             "-"*80,
-            e.output,
+            e.output.decode(),
             "-"*80,
             ""
         ]))