diff --git a/roles/cloud-resources/defaults/main.yml b/roles/cloud-resources/defaults/main.yml index 2f15e48ab..9dfed82dc 100644 --- a/roles/cloud-resources/defaults/main.yml +++ b/roles/cloud-resources/defaults/main.yml @@ -31,11 +31,17 @@ database_public_access: false # Allow access to the database from the public ne database_public_allowed_ips: "" # (comma-separated list of IP addresses in CIDR format) If empty, then public access is allowed for any IP address. # Backups (if 'pgbackrest_install' or 'wal_g_install' is 'true') -aws_s3_bucket_create: true +aws_s3_bucket_create: true # if 'provision=aws' aws_s3_bucket_name: "backups-{{ patroni_cluster_name }}" # Name of the S3 bucket. Bucket naming rules: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html aws_s3_bucket_region: "{{ server_location }}" # The AWS region to use. aws_s3_bucket_object_lock_enabled: false # Whether S3 Object Lock to be enabled. aws_s3_bucket_encryption: "AES256" # Describes the default server-side encryption to apply to new objects in the bucket. Choices: "AES256", "aws:kms" aws_s3_bucket_absent: true # Allow to delete S3 bucket when deleting a cluster servers using the 'state=absent' variable. +gcp_bucket_create: true # if 'provision=gcp' +gcp_bucket_name: "backups-{{ patroni_cluster_name }}" # Name of the GCS bucket. +gcp_bucket_storage_class: "MULTI_REGIONAL" # The bucket’s default storage class. Values include: MULTI_REGIONAL, REGIONAL, STANDARD, NEARLINE, COLDLINE, ARCHIVE, DURABLE_REDUCED_AVAILABILITY. +gcp_bucket_default_object_acl: "projectPrivate" # Apply a predefined set of default object access controls to this bucket. +gcp_bucket_absent: true # Allow to delete GCS bucket when deleting a cluster servers using the 'state=absent' variable. + ... diff --git a/roles/cloud-resources/tasks/gcp.yml b/roles/cloud-resources/tasks/gcp.yml index 42e5a2504..68f763318 100644 --- a/roles/cloud-resources/tasks/gcp.yml +++ b/roles/cloud-resources/tasks/gcp.yml @@ -170,6 +170,21 @@ }} when: firewall | bool + # GCS Bucket + - name: "GCP: Create bucket '{{ gcp_bucket_name }}'" + google.cloud.gcp_storage_bucket: + auth_kind: "serviceaccount" + service_account_contents: "{{ lookup('ansible.builtin.env', 'GCP_SERVICE_ACCOUNT_CONTENTS') }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ gcp_bucket_name }}" + state: present + storage_class: "{{ gcp_bucket_storage_class }}" + predefined_default_object_acl: "{{ gcp_bucket_default_object_acl }}" + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - gcp_bucket_create | bool + + # Server and volume - name: "GCP: Create or modify VM instance" google.cloud.gcp_compute_instance: auth_kind: "serviceaccount" @@ -313,6 +328,17 @@ project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" name: "{{ patroni_cluster_name }}-firewall" state: absent + + - name: "GCP: Delete bucket '{{ gcp_bucket_name }}'" + google.cloud.gcp_storage_bucket: + auth_kind: "serviceaccount" + service_account_contents: "{{ lookup('ansible.builtin.env', 'GCP_SERVICE_ACCOUNT_CONTENTS') }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ gcp_bucket_name }}" + state: absent + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - gcp_bucket_absent | bool when: state == 'absent' ... diff --git a/roles/pgbackrest/tasks/auto_conf.yml b/roles/pgbackrest/tasks/auto_conf.yml index e0ab291b0..3b6ff7ad4 100644 --- a/roles/pgbackrest/tasks/auto_conf.yml +++ b/roles/pgbackrest/tasks/auto_conf.yml @@ -1,6 +1,6 @@ --- -# AWS S3 bucket +# AWS S3 bucket (if 'provision=aws') - name: "Set variable 'pgbackrest_conf' for backup in the AWS S3 bucket" ansible.builtin.set_fact: pgbackrest_conf: @@ -31,6 +31,69 @@ - { option: "log-level-console", value: "info" } - { option: "recovery-option", value: "recovery_target_action=promote" } - { option: "pg1-path", value: "{{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}" } + no_log: true # do not output contents to the ansible log when: provision | default('') | lower == 'aws' +# GCS Bucket (if 'provision=gcp') +- block: + - name: "Set variable 'pgbackrest_conf' for backup in the GCS Bucket" + ansible.builtin.set_fact: + pgbackrest_conf: + global: + - { option: "log-level-file", value: "detail" } + - { option: "log-path", value: "/var/log/pgbackrest" } + - { option: "repo1-type", value: "gcs" } + - { option: "repo1-path", value: "{{ PGBACKREST_REPO_PATH | default('/pgbackrest') }}" } + - { option: "repo1-gcs-key", value: "{{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" } + - { option: "repo1-gcs-bucket", value: "{{ PGBACKREST_GCS_BUCKET | default(gcp_bucket_name | default('backups-' + patroni_cluster_name)) }}" } + - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" } + - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" } + - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" } + - { option: "repo1-bundle", value: "y" } + - { option: "repo1-block", value: "y" } + - { option: "start-fast", value: "y" } + - { option: "stop-auto", value: "y" } + - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } + - { option: "archive-async", value: "y" } + - { option: "archive-get-queue-max", value: "1GiB" } + - { option: "spool-path", value: "/var/spool/pgbackrest" } + - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + stanza: + - { option: "log-level-console", value: "info" } + - { option: "recovery-option", value: "recovery_target_action=promote" } + - { option: "pg1-path", value: "{{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}" } + no_log: true # do not output contents to the ansible log + + # if 'gcs_key_file' is not defined, copy GCS key file from GCP_SERVICE_ACCOUNT_CONTENTS environment variable. + - block: + - name: "Get GCP service account contents from localhost" + ansible.builtin.set_fact: + gcp_service_account_contents: "{{ lookup('ansible.builtin.env', 'GCP_SERVICE_ACCOUNT_CONTENTS') }}" + delegate_to: localhost + run_once: true # noqa run-once + no_log: true # do not output GCP service account contents to the ansible log + + - name: "Copy GCP service account contents to {{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + content: "{{ gcp_service_account_contents }}" + dest: "{{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is not defined + + # if 'gcs_key_file' is defined, copy this GCS key file. + - name: "Copy GCS key file to {{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + src: "{{ gcs_key_file }}" + dest: "{{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is defined and gcs_key_file | length > 0 + when: provision | default('') | lower == 'gcp' + ... diff --git a/roles/wal-g/tasks/auto_conf.yml b/roles/wal-g/tasks/auto_conf.yml index 515c95384..37a6ab0e2 100644 --- a/roles/wal-g/tasks/auto_conf.yml +++ b/roles/wal-g/tasks/auto_conf.yml @@ -1,12 +1,12 @@ --- -# AWS S3 bucket +# AWS S3 bucket (if 'provision=aws') - name: "Set variable 'wal_g_json' for backup in the AWS S3 bucket" ansible.builtin.set_fact: wal_g_json: - { option: "AWS_ACCESS_KEY_ID", value: "{{ WALG_AWS_ACCESS_KEY_ID | default('') }}" } - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ WALG_AWS_SECRET_ACCESS_KEY | default('') }}" } - - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://backups-' + patroni_cluster_name) }}" } + - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://' + (aws_s3_bucket_name | default('backups-' + patroni_cluster_name))) }}" } - { option: "AWS_REGION", value: "{{ WALG_AWS_REGION | default(aws_s3_bucket_region | default(server_location)) }}" } - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } @@ -17,6 +17,56 @@ - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" } - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" } - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" } + no_log: true # do not output contents to the ansible log when: provision | default('') | lower == 'aws' +# GCS Bucket (if 'provision=gcp') +- block: + - name: "Set variable 'wal_g_json' for backup in the AWS S3 bucket" + ansible.builtin.set_fact: + wal_g_json: + - { option: "GOOGLE_APPLICATION_CREDENTIALS", value: "{{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" } + - { option: "WALG_GS_PREFIX", value: "{{ WALG_GS_PREFIX | default('gs://' + (gcp_bucket_name | default('backups-' + patroni_cluster_name))) }}" } + - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } + - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } + - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "PGDATA", value: "{{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ patroni_cluster_name }}" } + - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" } + - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" } + - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" } + no_log: true # do not output contents to the ansible log + + # if 'gcs_key_file' is not defined, copy GCS key file from GCP_SERVICE_ACCOUNT_CONTENTS environment variable. + - block: + - name: "Get GCP service account contents from localhost" + ansible.builtin.set_fact: + gcp_service_account_contents: "{{ lookup('ansible.builtin.env', 'GCP_SERVICE_ACCOUNT_CONTENTS') }}" + delegate_to: localhost + run_once: true # noqa run-once + no_log: true # do not output GCP service account contents to the ansible log + + - name: "Copy GCP service account contents to {{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + content: "{{ gcp_service_account_contents }}" + dest: "{{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is not defined + + # if 'gcs_key_file' is defined, copy this GCS key file. + - name: "Copy GCS key file to {{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + src: "{{ gcs_key_file }}" + dest: "{{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is defined and gcs_key_file | length > 0 + when: provision | default('') | lower == 'gcp' + ...