AWS: Set backup variables

vitabaks · Oct 20, 2023 · 446f5aa · 446f5aa
1 parent a7db1e3
commit 446f5aa
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 12 deletions.
diff --git a/roles/cloud-resources/tasks/aws.yml b/roles/cloud-resources/tasks/aws.yml
@@ -218,7 +218,7 @@
             ([] if not firewall | bool else [patroni_cluster_name + '-private-security-group'])
           }}
 
-    # S3 bucket
+    # Backups
     - name: "AWS: Create S3 bucket '{{ patroni_cluster_name }}'"
       amazon.aws.s3_bucket:
         access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}"
@@ -229,6 +229,52 @@
         delete_public_access: true
         object_lock_enabled: "{{ aws_s3_bucket_object_lock_enabled | default(false) }}"
       when: (pgbackrest_install | bool) or (wal_g_install | bool)
+
+    - name: "Set variable: pgbackrest_conf"
+      ansible.builtin.set_fact:
+        pgbackrest_conf:
+          global:
+            - { option: "log-level-file", value: "detail" }
+            - { option: "log-path", value: "/var/log/pgbackrest" }
+            - { option: "repo1-type", value: "s3" }
+            - { option: "repo1-path", value: "/pgbackrest" }
+            - { option: "repo1-s3-key", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" }
+            - { option: "repo1-s3-key-secret", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" }
+            - { option: "repo1-s3-bucket", value: "{{ AWS_S3_BUCKET | default(patroni_cluster_name) }}" }
+            - { option: "repo1-s3-endpoint", value: "{{ AWS_S3_ENDPOINT | default('s3.' ~ (server_location | default('us-east-1')) ~ '.amazonaws.com') }}" }
+            - { option: "repo1-s3-region", value: "{{ AWS_S3_REGION | default(server_location | default('us-east-1')) }}" }
+            - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" }
+            - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" }
+            - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" }
+            - { option: "repo1-bundle", value: "y" }
+            - { option: "repo1-block", value: "y" }
+            - { option: "start-fast", value: "y" }
+            - { option: "stop-auto", value: "y" }
+            - { option: "link-all", value: "y" }
+            - { option: "resume", value: "n" }
+            - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+          stanza:
+            - { option: "log-level-console", value: "info" }
+            - { option: "recovery-option", value: "recovery_target_action=promote" }
+            - { option: "pg1-path", value: "{{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}" }
+      when: pgbackrest_install | bool
+
+    - name: "Set variable: wal_g_json"
+      ansible.builtin.set_fact:
+        wal_g_json:
+          - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" }
+          - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" }
+          - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://' + patroni_cluster_name) }}" }
+          - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" }
+          - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" }
+          - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+          - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+          - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+          - { option: "PGDATA", value: "{{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}" }
+          - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" }
+          - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" }
+          - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" }
+      when: wal_g_install | bool
   when: state == 'present'
 
 - name: Wait for EC2 instance to be available via SSH

diff --git a/vars/Debian.yml b/vars/Debian.yml
@@ -1,10 +1,11 @@
 ---
 
 # PostgreSQL variables
-postgresql_cluster_name: "main"
+postgresql_cluster_name: "{{ patroni_cluster_name }}"
+
 # When deploying to cloud providers, we create a disk and mount the data directory
 # along the path defined in the 'pg_data_mount_path' variable (or use '/pgdata' by default).
-# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/main"
+# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/{{ postgresql_cluster_name }}"
 postgresql_data_dir: "\
   {% if provision is defined and provision | lower != 'none' %}\
     {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ postgresql_cluster_name }}\

diff --git a/vars/RedHat.yml b/vars/RedHat.yml
@@ -4,12 +4,12 @@
 #
 # When deploying to cloud providers, we create a disk and mount the data directory
 # along the path defined in the 'pg_data_mount_path' variable (or use '/pgdata' by default).
-# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/data"
+# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/{{ patroni_cluster_name }}"
 postgresql_data_dir: "\
   {% if provision is defined and provision | lower != 'none' %}\
-    {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/data\
+    {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}\
   {% else %}\
-    /var/lib/pgsql/{{ postgresql_version }}/data\
+    /var/lib/pgsql/{{ postgresql_version }}/{{ patroni_cluster_name }}\
   {% endif %}"
 
 # You can specify custom WAL dir path. Example: "/pgwal/{{ postgresql_version }}/pg_wal"

diff --git a/vars/main.yml b/vars/main.yml
@@ -450,7 +450,8 @@ wal_g_backup_command:
   - " && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1"
 wal_g_delete_command:
   - "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ]"
-  - " && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1"
+  - " && wal-g delete retain FULL {{ WAL_G_RETENTION_FULL | default('4') }} --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1"
+# retain 4 full backups (adjust according to your company's backup retention policy)
 
 wal_g_cron_jobs:
   - name: "WAL-G: Create daily backup"
@@ -462,7 +463,7 @@ wal_g_cron_jobs:
     month: "*"
     weekday: "*"
     job: "{{ wal_g_backup_command | join('') }}"
-  - name: "WAL-G: Delete old backups" # retain 4 full backups (adjust according to your company's backup retention policy)
+  - name: "WAL-G: Delete old backups"
     user: "postgres"
     file: /etc/cron.d/walg
     minute: "30"
@@ -543,8 +544,8 @@ pgbackrest_cron_jobs:
     day: "*"
     month: "*"
     weekday: "0"
-    job: "pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup"
-    # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup; fi"
+    job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=full backup"
+    # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --stanza={{ pgbackrest_stanza }} --type=full backup; fi"
   - name: "pgBackRest: Diff Backup"
     file: "/etc/cron.d/pgbackrest-{{ patroni_cluster_name }}"
     user: "postgres"
@@ -553,8 +554,8 @@ pgbackrest_cron_jobs:
     day: "*"
     month: "*"
     weekday: "1-6"
-    job: "pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup"
-    # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup; fi"
+    job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=diff backup"
+    # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --stanza={{ pgbackrest_stanza }} --type=diff backup; fi"
 
 
 # PITR mode (if patroni_cluster_bootstrap_method: "pgbackrest" or "wal-g"):