From 446f5aa3e0c190daf1cb9317e5578f59814e60f5 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Fri, 20 Oct 2023 18:22:19 +0300 Subject: [PATCH] AWS: Set backup variables --- roles/cloud-resources/tasks/aws.yml | 48 ++++++++++++++++++++++++++++- vars/Debian.yml | 5 +-- vars/RedHat.yml | 6 ++-- vars/main.yml | 13 ++++---- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/roles/cloud-resources/tasks/aws.yml b/roles/cloud-resources/tasks/aws.yml index 84b26c0d3..8d4f9550c 100644 --- a/roles/cloud-resources/tasks/aws.yml +++ b/roles/cloud-resources/tasks/aws.yml @@ -218,7 +218,7 @@ ([] if not firewall | bool else [patroni_cluster_name + '-private-security-group']) }} - # S3 bucket + # Backups - name: "AWS: Create S3 bucket '{{ patroni_cluster_name }}'" amazon.aws.s3_bucket: access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" @@ -229,6 +229,52 @@ delete_public_access: true object_lock_enabled: "{{ aws_s3_bucket_object_lock_enabled | default(false) }}" when: (pgbackrest_install | bool) or (wal_g_install | bool) + + - name: "Set variable: pgbackrest_conf" + ansible.builtin.set_fact: + pgbackrest_conf: + global: + - { option: "log-level-file", value: "detail" } + - { option: "log-path", value: "/var/log/pgbackrest" } + - { option: "repo1-type", value: "s3" } + - { option: "repo1-path", value: "/pgbackrest" } + - { option: "repo1-s3-key", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } + - { option: "repo1-s3-key-secret", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" } + - { option: "repo1-s3-bucket", value: "{{ AWS_S3_BUCKET | default(patroni_cluster_name) }}" } + - { option: "repo1-s3-endpoint", value: "{{ AWS_S3_ENDPOINT | default('s3.' ~ (server_location | default('us-east-1')) ~ '.amazonaws.com') }}" } + - { option: "repo1-s3-region", value: "{{ AWS_S3_REGION | default(server_location | default('us-east-1')) }}" } + - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" } + - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" } + - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" } + - { option: "repo1-bundle", value: "y" } + - { option: "repo1-block", value: "y" } + - { option: "start-fast", value: "y" } + - { option: "stop-auto", value: "y" } + - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } + - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + stanza: + - { option: "log-level-console", value: "info" } + - { option: "recovery-option", value: "recovery_target_action=promote" } + - { option: "pg1-path", value: "{{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}" } + when: pgbackrest_install | bool + + - name: "Set variable: wal_g_json" + ansible.builtin.set_fact: + wal_g_json: + - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } + - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" } + - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://' + patroni_cluster_name) }}" } + - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } + - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } + - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "PGDATA", value: "{{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}" } + - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" } + - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" } + - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" } + when: wal_g_install | bool when: state == 'present' - name: Wait for EC2 instance to be available via SSH diff --git a/vars/Debian.yml b/vars/Debian.yml index 79b4c9e49..678e0a3d4 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -1,10 +1,11 @@ --- # PostgreSQL variables -postgresql_cluster_name: "main" +postgresql_cluster_name: "{{ patroni_cluster_name }}" + # When deploying to cloud providers, we create a disk and mount the data directory # along the path defined in the 'pg_data_mount_path' variable (or use '/pgdata' by default). -# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/main" +# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/{{ postgresql_cluster_name }}" postgresql_data_dir: "\ {% if provision is defined and provision | lower != 'none' %}\ {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ postgresql_cluster_name }}\ diff --git a/vars/RedHat.yml b/vars/RedHat.yml index a8639e555..0767f2e33 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -4,12 +4,12 @@ # # When deploying to cloud providers, we create a disk and mount the data directory # along the path defined in the 'pg_data_mount_path' variable (or use '/pgdata' by default). -# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/data" +# You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/{{ patroni_cluster_name }}" postgresql_data_dir: "\ {% if provision is defined and provision | lower != 'none' %}\ - {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/data\ + {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}\ {% else %}\ - /var/lib/pgsql/{{ postgresql_version }}/data\ + /var/lib/pgsql/{{ postgresql_version }}/{{ patroni_cluster_name }}\ {% endif %}" # You can specify custom WAL dir path. Example: "/pgwal/{{ postgresql_version }}/pg_wal" diff --git a/vars/main.yml b/vars/main.yml index 6878f13f7..a73f7c35a 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -450,7 +450,8 @@ wal_g_backup_command: - " && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" wal_g_delete_command: - "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ]" - - " && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" + - " && wal-g delete retain FULL {{ WAL_G_RETENTION_FULL | default('4') }} --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" +# retain 4 full backups (adjust according to your company's backup retention policy) wal_g_cron_jobs: - name: "WAL-G: Create daily backup" @@ -462,7 +463,7 @@ wal_g_cron_jobs: month: "*" weekday: "*" job: "{{ wal_g_backup_command | join('') }}" - - name: "WAL-G: Delete old backups" # retain 4 full backups (adjust according to your company's backup retention policy) + - name: "WAL-G: Delete old backups" user: "postgres" file: /etc/cron.d/walg minute: "30" @@ -543,8 +544,8 @@ pgbackrest_cron_jobs: day: "*" month: "*" weekday: "0" - job: "pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup" - # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup; fi" + job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=full backup" + # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --stanza={{ pgbackrest_stanza }} --type=full backup; fi" - name: "pgBackRest: Diff Backup" file: "/etc/cron.d/pgbackrest-{{ patroni_cluster_name }}" user: "postgres" @@ -553,8 +554,8 @@ pgbackrest_cron_jobs: day: "*" month: "*" weekday: "1-6" - job: "pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup" - # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup; fi" + job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=diff backup" + # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --stanza={{ pgbackrest_stanza }} --type=diff backup; fi" # PITR mode (if patroni_cluster_bootstrap_method: "pgbackrest" or "wal-g"):