diff --git a/.config/make/docker.mak b/.config/make/docker.mak index 82c6c6d67..1d66aea45 100644 --- a/.config/make/docker.mak +++ b/.config/make/docker.mak @@ -1,11 +1,23 @@ ## —— Docker ————————————————————————————————————————————————————————————————————————————————————— +TAG ?= local +DOCKER_REGISTRY ?= vitabaks + .PHONY: docker-build -docker-build: ## Run docker build image in local - docker build --tag postgresql_cluster:local --file .config/gitpod/Dockerfile . +docker-build: ## Run docker build image (example: make docker-build TAG=my_tag) + @echo "Building container image with tag $(TAG)"; + docker build --no-cache --tag postgresql_cluster:$(TAG) --file Dockerfile . + +.PHONY: docker-push +docker-push: ## Push image to Dockerhub (example: make docker-push TAG=my_tag DOCKER_REGISTRY=my_repo DOCKER_REGISTRY_USER="my_username" DOCKER_REGISTRY_PASSWORD="my_password") + @echo "Pushing container image with tag $(TAG)"; + echo "$(DOCKER_REGISTRY_PASSWORD)" | docker login --username "$(DOCKER_REGISTRY_USER)" --password-stdin + docker tag postgresql_cluster:$(TAG) $(DOCKER_REGISTRY)/postgresql_cluster:$(TAG) + docker push $(DOCKER_REGISTRY)/postgresql_cluster:$(TAG) .PHONY: docker-lint docker-lint: ## Run hadolint command to lint Dokerfile - docker run --rm -i hadolint/hadolint < .config/gitpod/Dockerfile + docker run --rm -i -v ./Dockerfile:/Dockerfile \ + hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3059 /Dockerfile .PHONY: docker-tests docker-tests: ## Run tests for docker diff --git a/.config/make/molecule.mak b/.config/make/molecule.mak index 1dde4d49f..11915f133 100644 --- a/.config/make/molecule.mak +++ b/.config/make/molecule.mak @@ -1,66 +1,56 @@ +# Activate virtual environment +ACTIVATE_VENV = . .venv/bin/activate + ## —— Molecule ——————————————————————————————————————————————————————————————————————————————————— + .PHONY: molecule-test molecule-test: ## Run test sequence for default scenario - source .venv/bin/activate - molecule test + $(ACTIVATE_VENV) && molecule test .PHONY: molecule-destroy molecule-destroy: ## Run destroy sequence for default scenario - source .venv/bin/activate - molecule destroy + $(ACTIVATE_VENV) && molecule destroy .PHONY: molecule-converge molecule-converge: ## Run converge sequence for default scenario - source .venv/bin/activate - molecule converge + $(ACTIVATE_VENV) && molecule converge .PHONY: molecule-reconverge molecule-reconverge: ## Run destroy and converge sequence for default scenario - source .venv/bin/activate - molecule destroy - molecule converge + $(ACTIVATE_VENV) && molecule destroy && molecule converge .PHONY: molecule-test-all molecule-test-all: ## Run test sequence for all scenarios - source .venv/bin/activate - molecule test --all + $(ACTIVATE_VENV) && molecule test --all .PHONY: molecule-destroy-all molecule-destroy-all: ## Run destroy sequence for all scenarios - source .venv/bin/activate - molecule destroy --all + $(ACTIVATE_VENV) && molecule destroy --all .PHONY: molecule-test-scenario -molecule-test-scenario: ## Run molecule test with specific scenario (example: make molecule-test-scenario MOLECULE_SCENARIO="postgrespro") - source .venv/bin/activate - molecule test --scenario-name $(MOLECULE_SCENARIO) +molecule-test-scenario: ## Run molecule test with specific scenario (example: make molecule-test-scenario MOLECULE_SCENARIO="scenario_name") + $(ACTIVATE_VENV) && molecule test --scenario-name $(MOLECULE_SCENARIO) .PHONY: molecule-destroy-scenario -molecule-destroy-scenario: ## Run molecule destroy with specific scenario (example: make molecule-destroy-scenario MOLECULE_SCENARIO="postgrespro") - source .venv/bin/activate - molecule destroy --scenario-name $(MOLECULE_SCENARIO) +molecule-destroy-scenario: ## Run molecule destroy with specific scenario (example: make molecule-destroy-scenario MOLECULE_SCENARIO="scenario_name") + $(ACTIVATE_VENV) && molecule destroy --scenario-name $(MOLECULE_SCENARIO) .PHONY: molecule-converge-scenario -molecule-converge-scenario: ## Run molecule converge with specific scenario (example: make molecule-converge-scenario MOLECULE_SCENARIO="postgrespro") - source .venv/bin/activate - molecule converge --scenario-name $(MOLECULE_SCENARIO) +molecule-converge-scenario: ## Run molecule converge with specific scenario (example: make molecule-converge-scenario MOLECULE_SCENARIO="scenario_name") + $(ACTIVATE_VENV) && molecule converge --scenario-name $(MOLECULE_SCENARIO) .PHONY: molecule-dependency molecule-dependency: ## Run dependency sequence - source .venv/bin/activate - molecule dependency + $(ACTIVATE_VENV) && molecule dependency .PHONY: molecule-verify molecule-verify: ## Run verify sequence - source .venv/bin/activate - molecule verify + $(ACTIVATE_VENV) && molecule verify .PHONY: molecule-login molecule-login: ## Log in to one instance using custom host IP (example: make molecule-login MOLECULE_HOST="10.172.0.20") - source .venv/bin/activate - molecule login --host $(MOLECULE_HOST) + $(ACTIVATE_VENV) && molecule login --host $(MOLECULE_HOST) .PHONY: molecule-login-scenario -molecule-login-scenario: ## Log in to one instance using custom host IP and scenario name (example: make molecule-login-scenario MOLECULE_HOST="10.172.1.20" MOLECULE_SCENARIO="postgrespro") - source .venv/bin/activate - molecule login --host $(MOLECULE_HOST) --scenario-name $(MOLECULE_SCENARIO) +molecule-login-scenario: ## Log in to one instance using custom host IP and scenario name (example: make molecule-login-scenario MOLECULE_HOST="10.172.1.20" MOLECULE_SCENARIO="scenario_name") + $(ACTIVATE_VENV) && molecule login --host $(MOLECULE_HOST) --scenario-name $(MOLECULE_SCENARIO) diff --git a/.config/make/python.mak b/.config/make/python.mak index 2c638a9ad..f975d325d 100644 --- a/.config/make/python.mak +++ b/.config/make/python.mak @@ -3,6 +3,9 @@ python_launcher ?= python3.10 python_requirements_file ?= requirements.txt python_requirements_dev_file ?= .config/python/dev/requirements.txt +# Activate virtual environment +ACTIVATE_VENV = . .venv/bin/activate + ## —— Python ————————————————————————————————————————————————————————————————————————————————————— .PHONY: python-bootstrap python-bootstrap: ## Bootstrap python @@ -20,56 +23,50 @@ python-bootstrap-dev: ## Bootstrap python for dev env # =============================================================================================== .PHONY: python-venv-init python-venv-init: ## Create venv ".venv/" if not exist - if [ ! -d .venv ] ; then - $(python_launcher) -m venv .venv - fi + @echo "Checking if .venv directory exists..."; \ + if [ ! -d .venv ]; then echo "Creating virtual environment using $(python_launcher)..."; $(python_launcher) -m venv .venv; else echo ".venv directory already exists. Skipping creation."; fi .PHONY: python-venv-upgrade python-venv-upgrade: ## Upgrade venv with pip, setuptools and wheel - source .venv/bin/activate - pip install --upgrade pip setuptools wheel + @echo "Upgrading virtual environment..." + $(ACTIVATE_VENV) && pip install --upgrade pip setuptools wheel .PHONY: python-venv-requirements python-venv-requirements: ## Install or upgrade from $(python_requirements_file) - source .venv/bin/activate - pip install --upgrade --requirement $(python_requirements_file) + @echo "Installing or upgrading requirements from $(python_requirements_file)..." + $(ACTIVATE_VENV) && pip install --upgrade --requirement $(python_requirements_file) .PHONY: python-venv-requirements-dev python-venv-requirements-dev: ## Install or upgrade from $(python_requirements_dev_file) - source .venv/bin/activate - pip install --upgrade --requirement $(python_requirements_dev_file) + @echo "Installing or upgrading dev requirements from $(python_requirements_dev_file)..." + $(ACTIVATE_VENV) && pip install --upgrade --requirement $(python_requirements_dev_file) .PHONY: python-venv-linters-install python-venv-linters-install: ## Install or upgrade linters - source .venv/bin/activate - pip install --upgrade flake8 + @echo "Installing or upgrading linters..." + $(ACTIVATE_VENV) && pip install --upgrade flake8 .PHONY: python-venv-purge python-venv-purge: ## Remove venv ".venv/" folder - rm -rf .venv + @echo "Removing .venv directory..." + @rm -rf .venv # =============================================================================================== # Utils # =============================================================================================== .PHONY: python-purge-cache python-purge-cache: ## Purge cache to avoid used cached files - if [ -d .venv ] ; then - source .venv/bin/activate - pip cache purge - fi + @echo "Purging pip cache..." + @if [ -d .venv ] ; then $(ACTIVATE_VENV) && pip cache purge; fi .PHONY: python-version python-version: ## Displays the python version used for the .venv - source .venv/bin/activate - $(python_launcher) --version + $(ACTIVATE_VENV) && $(python_launcher) --version .PHONY: python-flake8 python-flake8: ## Run flake8 linter for python - source .venv/bin/activate - flake8 --config .config/.flake8 + $(ACTIVATE_VENV) && flake8 --config .config/.flake8 .PHONY: python-pytest python-pytest: ## Run pytest to test python scripts - source .venv/bin/activate - cd scripts/ - $(python_launcher) -m pytest + $(ACTIVATE_VENV) && cd scripts/ && $(python_launcher) -m pytest diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5078b5586..6de6d967c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,6 +5,8 @@ on: push: branches: - master + tags: + - '*' pull_request: branches: - master @@ -17,6 +19,25 @@ jobs: - name: Set TERM environment variable run: echo "TERM=xterm" >> $GITHUB_ENV + - name: Extract branch or tag name + shell: bash + run: | + REF_NAME="" + if [[ -n "${GITHUB_HEAD_REF}" ]]; then + # This is a PR, use the source branch name + REF_NAME="${GITHUB_HEAD_REF}" + else + # This is a push, use the branch or tag name from GITHUB_REF + REF_NAME="${GITHUB_REF##*/}" + fi + + # If this is the master branch, use 'latest' as the tag, otherwise use the REF_NAME + if [[ "$REF_NAME" == "master" ]]; then + echo "TAG=latest" >> $GITHUB_ENV + else + echo "TAG=$REF_NAME" >> $GITHUB_ENV + fi + - name: Checkout uses: actions/checkout@v3 @@ -28,5 +49,17 @@ jobs: - name: Install dependencies run: make bootstrap-dev - - name: Run Docker tests - run: make docker-tests + - name: Run Docker lint + run: make docker-lint + + - name: Run Docker build + run: make docker-build + env: + TAG: ${{ env.TAG }} + + - name: Run Docker push + run: make docker-push + env: + TAG: ${{ env.TAG }} + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_USERNAME }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..19a699223 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +FROM debian:bookworm-slim +LABEL maintainer="Vitaliy Kukharik vitabaks@gmail.com" + +USER root + +# Set SHELL to Bash to ensure pipefail is supported +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Copy postgresql_cluster repository +COPY . /postgresql_cluster + +# Install required packages, Python dependencies, Ansible requirements, and perform cleanup +RUN apt-get clean && rm -rf /var/lib/apt/lists/partial \ + && apt-get update -o Acquire::CompressionTypes::Order::=gz \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates gnupg git python3 python3-dev python3-pip keychain ssh-client sshpass\ + gcc g++ cmake make libssl-dev curl apt-transport-https lsb-release gnupg \ + && pip3 install --break-system-packages --no-cache-dir -r \ + /postgresql_cluster/requirements.txt \ + && ansible-galaxy install --force -r \ + /postgresql_cluster/requirements.yml \ + && ansible-galaxy install --force -r \ + /postgresql_cluster/roles/consul/requirements.yml \ + && ansible-galaxy collection list \ + && pip3 install --break-system-packages --no-cache-dir -r \ + /root/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt \ + && curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.asc.gpg > /dev/null \ + && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list \ + && apt-get update && apt-get install --no-install-recommends -y azure-cli \ + && apt-get autoremove -y --purge gnupg git python3-dev gcc g++ cmake make libssl-dev \ + && apt-get clean -y autoclean \ + && rm -rf /var/lib/apt/lists/* /tmp/* \ + && chmod +x /postgresql_cluster/entrypoint.sh + +# Set environment variable for Ansible collections paths +ENV ANSIBLE_COLLECTIONS_PATH=/root/.ansible/collections/ansible_collections:/usr/local/lib/python3.11/dist-packages/ansible_collections +ENV USER=root + +WORKDIR /postgresql_cluster + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/Makefile b/Makefile index 0dc39acf9..c7adc10d4 100644 --- a/Makefile +++ b/Makefile @@ -38,19 +38,7 @@ python_launcher := python$(shell cat .config/python_version.config | cut -d '=' -include $(addsuffix /*.mak, $(shell find .config/make -type d)) -## —— Tests collection ——————————————————————————————————————————————————————————————————————— -.PHONY: tests -tests: ## tests Ansible collection - $(MAKE) docker-tests - $(MAKE) lint - $(MAKE) molecule-test-all - -.PHONY: tests-fast -tests-fast: ## tests Ansible collection quickly - $(MAKE) lint - $(MAKE) molecule-converge - -## —— Bootstrap collection ——————————————————————————————————————————————————————————————————————— +## —— Bootstrap —————————————————————————————————————————————————————————————————————————————————— .PHONY: bootstrap bootstrap: ## Bootstrap Ansible collection $(MAKE) python-bootstrap @@ -60,19 +48,33 @@ bootstrap-dev: ## Bootstrap Ansible collection for development $(MAKE) bootstrap $(MAKE) python-bootstrap-dev -## —— Virtualenv ———————————————————————————————————————————————————————————————————————————————— +## —— Virtualenv ————————————————————————————————————————————————————————————————————————————————— .PHONY: reinitialization -reinitialization: ## Return to an initial state of Bootstrap Ansible collection +reinitialization: ## Return to initial state of Bootstrap Ansible collection $(MAKE) clean $(MAKE) bootstrap .PHONY: reinitialization-dev -reinitialization-dev: ## Return to an initial state of Bootstrap Ansible collection for development +reinitialization-dev: ## Return to initial state of Bootstrap Ansible collection for development $(MAKE) reinitialization $(MAKE) bootstrap-dev +## —— Tests —————————————————————————————————————————————————————————————————————————————————————— +.PHONY: tests +tests: ## tests Ansible + $(MAKE) docker-tests + $(MAKE) lint + $(MAKE) molecule-test-all + +.PHONY: tests-fast +tests-fast: ## tests Ansible quickly + $(MAKE) lint + $(MAKE) molecule-converge + +## —— Clean —————————————————————————————————————————————————————————————————————————————————————— .PHONY: clean -clean: ## Clean collection + $(MAKE) clean +clean: ## Clean rm -rf .venv/ rm -rf vendor/ rm -f *.mak @@ -81,4 +83,4 @@ clean: ## Clean collection rm -rf scripts/tests/__pycache__/ rm -rf scripts/modules/__pycache__/ rm -rf scripts/modules/services/__pycache__/ - rm -rf scripts/modules/utils/__pycache__/ \ No newline at end of file + rm -rf scripts/modules/utils/__pycache__/ diff --git a/README.md b/README.md index 78c3029c1..4e164e5ae 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Minimum supported Ansible version: 8.0.0 (ansible-core 2.15.0) ## Requirements This playbook requires root privileges or sudo. -Ansible ([What is Ansible](https://www.ansible.com/resources/videos/quick-start-video)?) +Ansible ([What is Ansible](https://www.ansible.com/how-ansible-works/)?) if dcs_type: "consul", please install consul role requirements on the control node: diff --git a/ansible.cfg b/ansible.cfg index d110bba5b..334116498 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,13 +1,22 @@ [defaults] +forks = 10 inventory = ./inventory +use_persistent_connections = True remote_tmp = /tmp/${USER}/ansible allow_world_readable_tmpfiles = false # or "true" if the temporary directory on the remote host is mounted with POSIX acls disabled or the remote machines use ZFS. host_key_checking = False timeout = 60 deprecation_warnings = False display_skipped_hosts = False +localhost_warning = False +stdout_callback = default +# Define the directory for custom callback plugins +callback_plugins = ./plugins/callback +# Enable JSON logging if 'ANSIBLE_JSON_LOG_FILE' environment variable is defined (example: ANSIBLE_JSON_LOG_FILE=./ansible_log.json) +callbacks_enabled = json_log [ssh_connection] +ssh_args = -o Ciphers=aes128-ctr -o MACs=hmac-sha2-256 -o ControlMaster=auto -o ControlPersist=30m -o ConnectionAttempts=3 -o ServerAliveInterval=5 -o ServerAliveCountMax=10 pipelining = True [persistent_connection] @@ -15,4 +24,5 @@ retries = 3 connect_timeout = 60 command_timeout = 30 -#https://github.com/ansible/ansible/blob/stable-2.9/examples/ansible.cfg +# https://github.com/ansible/ansible/blob/stable-2.11/examples/ansible.cfg +# https://docs.ansible.com/ansible/latest/reference_appendices/config.html diff --git a/config_pgcluster.yml b/config_pgcluster.yml index fd02663bf..fe1e4e562 100644 --- a/config_pgcluster.yml +++ b/config_pgcluster.yml @@ -1,6 +1,35 @@ --- - name: config_pgcluster.yml | Configuration PostgreSQL HA Cluster (based on "Patroni") + hosts: localhost + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Include main variables + ansible.builtin.include_vars: "vars/main.yml" + tags: always + # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI) + # Note: Applicable only for "aws", "gcp", "azure", because: + # "digitalocean" - requires the Spaces access keys "AWS_ACCESS_KEY_ID" and "AWS_SECRET_ACCESS_KEY" + # "hetzner" - currently, Hetzner Cloud does not provide S3 storage + - name: "Set variable: 'pgbackrest_install' to configure Postgres backups" + ansible.builtin.set_fact: + pgbackrest_install: true + when: + - not (pgbackrest_install | bool or wal_g_install | bool) + - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure'] + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: always + roles: + - role: cloud-resources + when: cloud_provider | default('') | length > 0 + vars: + postgresql_cluster_maintenance: true + tags: always + +- name: config_pgcluster.yml | Check the PostgreSQL cluster state and perform pre-checks hosts: postgres_cluster + become: true + become_method: sudo gather_facts: true pre_tasks: - name: Include main variables @@ -63,6 +92,17 @@ - "Cluster Leader: {{ ansible_hostname }}" when: inventory_hostname in groups['primary'] + # if 'cloud_provider' is 'aws', 'gcp', or 'azure' + # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI) + - name: "Set variable: 'pgbackrest_install' to configure Postgres backups" + ansible.builtin.set_fact: + pgbackrest_install: true + when: + - not (pgbackrest_install | bool or wal_g_install | bool) + - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure'] + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: always + roles: - role: pre-checks vars: @@ -71,7 +111,7 @@ tags: - always -- name: config_pgcluster.yml | Configure Postgres Cluster +- name: config_pgcluster.yml | Configure PostgreSQL Cluster hosts: 'primary:secondary' become: true become_method: sudo @@ -179,6 +219,17 @@ - name: Include OS-specific variables ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tags: always + + # if 'cloud_provider' is 'aws', 'gcp', or 'azure' + # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI) + - name: "Set variable: 'pgbackrest_install' to configure Postgres backups" + ansible.builtin.set_fact: + pgbackrest_install: true + when: + - not (pgbackrest_install | bool or wal_g_install | bool) + - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure'] + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: always roles: - role: pgbackrest when: pgbackrest_install | bool diff --git a/consul.yml b/consul.yml index 5e94229d3..645a9baef 100644 --- a/consul.yml +++ b/consul.yml @@ -36,6 +36,7 @@ delegate_to: localhost run_once: true # noqa run-once when: + - ansible_utils_result.stderr is defined - ansible_utils_result.stderr is search("unable to find") - name: Make sure the unzip package are present on the control host @@ -48,6 +49,7 @@ retries: 3 delegate_to: localhost run_once: true # noqa run-once + when: ansible_distribution != "MacOSX" - name: Make sure the python3-pip package are present on the control host ansible.builtin.package: @@ -59,6 +61,7 @@ retries: 3 delegate_to: localhost run_once: true # noqa run-once + when: ansible_distribution != "MacOSX" - name: Install netaddr dependency on the control host ansible.builtin.pip: @@ -136,7 +139,6 @@ - role: hostname - role: resolv_conf - role: etc_hosts - - role: sysctl - role: timezone - role: ntp diff --git a/deploy_pgcluster.yml b/deploy_pgcluster.yml index f2be17a01..c7efcb40c 100644 --- a/deploy_pgcluster.yml +++ b/deploy_pgcluster.yml @@ -1,11 +1,34 @@ --- - - name: Deploy PostgreSQL HA Cluster (based on "Patroni") + hosts: localhost + gather_facts: true + any_errors_fatal: true + pre_tasks: + - name: Include main variables + ansible.builtin.include_vars: "vars/main.yml" + tags: always + # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI) + # Note: Applicable only for "aws", "gcp", "azure", because: + # "digitalocean" - requires the Spaces access keys "AWS_ACCESS_KEY_ID" and "AWS_SECRET_ACCESS_KEY" + # "hetzner" - currently, Hetzner Cloud does not provide S3 storage + - name: "Set variable: 'pgbackrest_install' to configure Postgres backups" + ansible.builtin.set_fact: + pgbackrest_install: true + when: + - not (pgbackrest_install | bool or wal_g_install | bool) + - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure'] + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: always + roles: + - role: cloud-resources + when: cloud_provider | default('') | length > 0 + tags: always + +- name: deploy_pgcluster.yml | Perform pre-checks hosts: all become: true become_method: sudo gather_facts: true - tags: always any_errors_fatal: true environment: "{{ proxy_env | default({}) }}" @@ -22,19 +45,20 @@ ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tags: always - - name: System information + - name: System info ansible.builtin.debug: - var: system_info - vars: - system_info: - OS: "{{ ansible_distribution | default('N/A') }} {{ ansible_distribution_version | default('N/A') }}" - Kernel: "{{ ansible_kernel | default('N/A') }}" - CPU model: >- - {{ ansible_processor[2] | default('N/A') }}, - count: {{ ansible_processor_count | default('N/A') }}, - cores: {{ ansible_processor_cores | default('N/A') }} - Memory: "{{ (ansible_memtotal_mb / 1024) | round(2) if ansible_memtotal_mb is defined else 'N/A' }} GB" - Disk space total: >- + msg: + server_name: "{{ hostname | default(ansible_hostname) }}" + server_location: "{{ server_location | default(omit) }}" + ip_address: "{{ inventory_hostname | default('N/A') }}" + os: "{{ ansible_distribution | default('N/A') }} {{ ansible_distribution_version | default('N/A') }}" + kernel: "{{ ansible_kernel | default('N/A') }}" + cpu: + model: "{{ ansible_processor[2] | default('N/A') }}" + count: "{{ ansible_processor_count | default('N/A') }}" + cores: "{{ ansible_processor_cores | default('N/A') }}" + memory: "{{ (ansible_memtotal_mb / 1024) | round(2) if ansible_memtotal_mb is defined else 'N/A' }} GB" + disk_space_total: >- {{ (ansible_mounts | map(attribute='size_total') @@ -43,12 +67,27 @@ ) | round(2) if ansible_mounts is defined else 'N/A' }} GB - Architecture: "{{ ansible_architecture | default('N/A') }}" - Virtualization type: "{{ ansible_virtualization_type | default('N/A') }}" - Product name: "{{ ansible_product_name | default('N/A') }}" + architecture: "{{ ansible_architecture | default('N/A') }}" + virtualization_type: "{{ ansible_virtualization_type | default('N/A') }}" + product_name: "{{ ansible_product_name | default('N/A') }}" + tags: always + + # if 'cloud_provider' is 'aws', 'gcp', or 'azure' + # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI) + - name: "Set variable: 'pgbackrest_install' to configure Postgres backups" + ansible.builtin.set_fact: + pgbackrest_install: true + when: + - not (pgbackrest_install | bool or wal_g_install | bool) + - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure'] + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings tags: always roles: + # (optional) if 'ssh_public_keys' is defined + - role: authorized-keys + tags: ssh_public_keys + - role: pre-checks vars: minimal_ansible_version: 2.15.0 @@ -104,6 +143,65 @@ retries: 3 when: ansible_os_family == "Debian" + # (optional) Command or script to be executed before the Postgres cluster deployment. + - block: + - name: Print pre-deploy command + ansible.builtin.debug: + var: pre_deploy_command + when: pre_deploy_command_print | default(false) | bool + + - name: Run pre-deploy command + ansible.builtin.shell: "{{ pre_deploy_command }} > /tmp/pre_deploy_command.log 2>&1" + args: + executable: /bin/bash + register: pre_deploy_result + delegate_to: "{{ item }}" + loop: "{{ pre_deploy_command_hosts.split(',') | map('extract', groups) | list | flatten }}" + async: "{{ pre_deploy_command_timeout }}" # run the command asynchronously + poll: 0 + + - name: Wait for the pre-deploy command to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + register: pre_deploy_job_result + delegate_to: "{{ item.item }}" + loop: "{{ pre_deploy_result.results }}" + loop_control: + label: "{{ item.item }}" + until: pre_deploy_job_result.finished + retries: "{{ (pre_deploy_command_timeout | int) // 10 }}" + delay: 10 + ignore_errors: true # allows to collect logs before stopping execution (in case of failure) + when: + - pre_deploy_result.results is defined + - item.ansible_job_id is defined + + - name: Get pre-deploy command log + ansible.builtin.command: cat /tmp/pre_deploy_command.log + register: pre_deploy_command_log + delegate_to: "{{ item }}" + loop: "{{ pre_deploy_command_hosts.split(',') | map('extract', groups) | list | flatten }}" + changed_when: false + when: pre_deploy_command_print_result | default(false) | bool + + - name: Print pre-deploy command result + ansible.builtin.debug: + msg: "{{ item.stdout_lines }}" + loop: "{{ pre_deploy_command_log.results }}" + loop_control: + label: "{{ item.item }}" + when: + - pre_deploy_command_log.results is defined + - item.stdout_lines is defined + + - name: Stop if pre-deploy command failed + ansible.builtin.fail: + msg: "Pre-deploy command failed. See log for details." + when: pre_deploy_job_result.results | json_query('[?failed]') | length > 0 + run_once: true # noqa run-once + when: pre_deploy_command | default('') | length > 0 + tags: pre_deploy, pre_deploy_command + - name: deploy_pgcluster.yml | Deploy etcd cluster ansible.builtin.import_playbook: etcd_cluster.yml when: not dcs_exists|bool and dcs_type == "etcd" @@ -120,6 +218,7 @@ become_method: sudo gather_facts: true any_errors_fatal: true + environment: "{{ proxy_env | default({}) }}" pre_tasks: - name: Include main variables @@ -156,7 +255,6 @@ roles: - role: ansible-role-firewall - environment: "{{ proxy_env | default({}) }}" vars: firewall_allowed_tcp_ports: "{{ firewall_ports_dynamic_var | default([]) | unique }}" firewall_additional_rules: "{{ firewall_rules_dynamic_var | default([]) | unique }}" @@ -169,6 +267,7 @@ - role: add-repository - role: packages - role: sudo + - role: mount - role: swap - role: sysctl - role: transparent_huge_pages @@ -201,9 +300,19 @@ ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tags: always + # if 'cloud_provider' is 'aws', 'gcp', or 'azure' + # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI) + - name: "Set variable: 'pgbackrest_install' to configure Postgres backups" + ansible.builtin.set_fact: + pgbackrest_install: true + when: + - not (pgbackrest_install | bool or wal_g_install | bool) + - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure'] + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: always roles: - role: pgbackrest - when: pgbackrest_install|bool + when: pgbackrest_install | bool - name: deploy_pgcluster.yml | PostgreSQL Cluster Deployment hosts: postgres_cluster @@ -227,6 +336,17 @@ ansible.builtin.include_vars: "vars/{{ ansible_os_family }}.yml" tags: always + # if 'cloud_provider' is 'aws', 'gcp', or 'azure' + # set_fact: 'pgbackrest_install' to configure Postgres backups (TODO: Add the ability to configure backups in the UI) + - name: "Set variable: 'pgbackrest_install' to configure Postgres backups" + ansible.builtin.set_fact: + pgbackrest_install: true + when: + - not (pgbackrest_install | bool or wal_g_install | bool) + - cloud_provider | default('') | lower in ['aws', 'gcp', 'azure'] + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: always + roles: - role: wal-g when: wal_g_install|bool @@ -244,7 +364,7 @@ - role: patroni - role: pgbackrest/stanza-create - when: pgbackrest_install|bool + when: pgbackrest_install | bool - role: vip-manager when: not with_haproxy_load_balancing|bool and @@ -271,3 +391,65 @@ # finish (info) - role: deploy-finish + + tasks: + # (optional) Command or script to be executed after the Postgres cluster deployment. + - block: + - name: Print post-deploy command + ansible.builtin.debug: + var: post_deploy_command + when: post_deploy_command_print | default(false) | bool + + - name: Run post-deploy command + ansible.builtin.shell: "{{ post_deploy_command }} > /tmp/post_deploy_command.log 2>&1" + args: + executable: /bin/bash + register: post_deploy_result + delegate_to: "{{ item }}" + loop: "{{ post_deploy_command_hosts.split(',') | map('extract', groups) | list | flatten }}" + async: "{{ post_deploy_command_timeout }}" # run the command asynchronously + poll: 0 + + - name: Wait for the post-deploy command to complete + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + register: post_deploy_job_result + delegate_to: "{{ item.item }}" + loop: "{{ post_deploy_result.results }}" + loop_control: + label: "{{ item.item }}" + until: post_deploy_job_result.finished + retries: "{{ (post_deploy_command_timeout | int) // 10 }}" + delay: 10 + ignore_errors: true # allows to collect logs before stopping execution (in case of failure) + when: + - post_deploy_result.results is defined + - item.ansible_job_id is defined + + - name: Get post-deploy command log + ansible.builtin.command: cat /tmp/post_deploy_command.log + register: post_deploy_command_log + delegate_to: "{{ item }}" + loop: "{{ post_deploy_command_hosts.split(',') | map('extract', groups) | list | flatten }}" + changed_when: false + when: post_deploy_command_print_result | default(false) | bool + + - name: Print post-deploy command result + ansible.builtin.debug: + msg: "{{ item.stdout_lines }}" + loop: "{{ post_deploy_command_log.results }}" + loop_control: + label: "{{ item.item }}" + when: + - post_deploy_command_log.results is defined + - item.stdout_lines is defined + + - name: Stop if post-deploy command failed + ansible.builtin.fail: + msg: "Post-deploy command failed. See log for details." + when: post_deploy_job_result.results | json_query('[?failed]') | length > 0 + run_once: true # noqa run-once + when: post_deploy_command | default('') | length > 0 + tags: post_deploy, post_deploy_command + +... diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 000000000..76a32780a --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +is_base64() { + # Check if input is base64 encoded + if [[ "$1" =~ ^[A-Za-z0-9+/=]+$ ]]; then + return 0 + else + return 1 + fi +} + +# Check if ANSIBLE_INVENTORY_JSON is set and create inventory.json if it is +if [[ -n "${ANSIBLE_INVENTORY_JSON}" ]]; then + if is_base64 "${ANSIBLE_INVENTORY_JSON}"; then + echo "Creating inventory.json with the (base64 decoded) content of ANSIBLE_INVENTORY_JSON" + echo "${ANSIBLE_INVENTORY_JSON}" | base64 -d > /postgresql_cluster/inventory.json + else + echo "Creating inventory.json with the content of ANSIBLE_INVENTORY_JSON" + echo "${ANSIBLE_INVENTORY_JSON}" > /postgresql_cluster/inventory.json + fi + # Set ANSIBLE_INVENTORY environment variable + export ANSIBLE_INVENTORY=/postgresql_cluster/inventory.json + # Set ANSIBLE_SSH_ARGS environment variable + export ANSIBLE_SSH_ARGS="-o StrictHostKeyChecking=no" +fi + +# Check if SSH_PRIVATE_KEY_CONTENT is set and create the SSH private key file if it is +if [[ -n "${SSH_PRIVATE_KEY_CONTENT}" ]]; then + mkdir -p /root/.ssh + if is_base64 "${SSH_PRIVATE_KEY_CONTENT}"; then + echo "Creating SSH private key file with the (base64 decoded) content of SSH_PRIVATE_KEY_CONTENT" + echo "${SSH_PRIVATE_KEY_CONTENT}" | base64 -d > /root/.ssh/id_rsa + else + echo "Creating SSH private key file with the content of SSH_PRIVATE_KEY_CONTENT" + echo "${SSH_PRIVATE_KEY_CONTENT}" > /root/.ssh/id_rsa + fi + + chmod 600 /root/.ssh/id_rsa + + # Ensure the key file ends with a newline + sed -i -e '$a\' /root/.ssh/id_rsa + + echo "Checking SSH private key with ssh-keygen" + ssh-keygen -y -f /root/.ssh/id_rsa > /dev/null + + # Set ANSIBLE_PRIVATE_KEY_FILE environment variable + export ANSIBLE_PRIVATE_KEY_FILE=/root/.ssh/id_rsa +fi + +# Execute the passed command +exec "$@" diff --git a/etcd_cluster.yml b/etcd_cluster.yml index 6ef34d5ec..152e56c2b 100644 --- a/etcd_cluster.yml +++ b/etcd_cluster.yml @@ -69,7 +69,6 @@ - role: hostname - role: resolv_conf - role: etc_hosts - - role: sysctl - role: timezone - role: ntp diff --git a/group_vars/all b/group_vars/all index 55de5ec90..90f82a9a8 100644 --- a/group_vars/all +++ b/group_vars/all @@ -24,4 +24,4 @@ os_minimum_versions: # See: # - https://github.com/ansible/ansible/issues/83476 # - https://github.com/ansible/ansible/issues/83603 -ansible_python_interpreter: "{{ ansible_version['full'] is version('2.17', '>=') | ternary('/usr/bin/python3', '/usr/bin/env python3') }}" \ No newline at end of file +ansible_python_interpreter: "{{ ansible_version['full'] is version('2.17', '>=') | ternary('/usr/bin/python3', '/usr/bin/env python3') }}" diff --git a/inventory b/inventory index 97eb4ef5f..9f7628657 100644 --- a/inventory +++ b/inventory @@ -11,33 +11,33 @@ # if dcs_exists: false and dcs_type: "etcd" [etcd_cluster] # recommendation: 3, or 5-7 nodes -10.128.64.140 -10.128.64.142 -10.128.64.143 +#10.128.64.140 +#10.128.64.142 +#10.128.64.143 # if dcs_exists: false and dcs_type: "consul" [consul_instances] # recommendation: 3 or 5-7 nodes -10.128.64.140 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1 -10.128.64.142 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1 -10.128.64.143 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1 +#10.128.64.140 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1 +#10.128.64.142 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1 +#10.128.64.143 consul_node_role=server consul_bootstrap_expect=true consul_datacenter=dc1 #10.128.64.144 consul_node_role=client consul_datacenter=dc2 #10.128.64.145 consul_node_role=client consul_datacenter=dc2 # if with_haproxy_load_balancing: true [balancers] -10.128.64.140 # balancer_tags="datacenter=dc1" -10.128.64.142 # balancer_tags="datacenter=dc1" -10.128.64.143 # balancer_tags="datacenter=dc1" +#10.128.64.140 # balancer_tags="datacenter=dc1" +#10.128.64.142 # balancer_tags="datacenter=dc1" +#10.128.64.143 # balancer_tags="datacenter=dc1" #10.128.64.144 balancer_tags="datacenter=dc2" #10.128.64.145 balancer_tags="datacenter=dc2" new_node=true # PostgreSQL nodes [master] -10.128.64.140 hostname=pgnode01 postgresql_exists=false # patroni_tags="datacenter=dc1" +#10.128.64.140 hostname=pgnode01 postgresql_exists=false # patroni_tags="datacenter=dc1" [replica] -10.128.64.142 hostname=pgnode02 postgresql_exists=false # patroni_tags="datacenter=dc1" -10.128.64.143 hostname=pgnode03 postgresql_exists=false # patroni_tags="datacenter=dc1" +#10.128.64.142 hostname=pgnode02 postgresql_exists=false # patroni_tags="datacenter=dc1" +#10.128.64.143 hostname=pgnode03 postgresql_exists=false # patroni_tags="datacenter=dc1" #10.128.64.144 hostname=pgnode04 postgresql_exists=false patroni_tags="datacenter=dc2" #10.128.64.145 hostname=pgnode04 postgresql_exists=false patroni_tags="datacenter=dc2" new_node=true @@ -47,18 +47,21 @@ replica # if pgbackrest_install: true and "repo_host" is set [pgbackrest] # optional (Dedicated Repository Host) +#10.128.64.110 +[pgbackrest:vars] +#ansible_user='postgres' +#ansible_ssh_pass='secretpassword' # Connection settings [all:vars] ansible_connection='ssh' ansible_ssh_port='22' -ansible_user='root' -ansible_ssh_pass='secretpassword' # "sshpass" package is required for use "ansible_ssh_pass" +#ansible_user='root' +#ansible_ssh_pass='secretpassword' # "sshpass" package is required for use "ansible_ssh_pass" #ansible_ssh_private_key_file= -#ansible_python_interpreter='/usr/bin/env python3' +#ansible_python_interpreter='/usr/bin/python3' [pgbackrest:vars] #ansible_user='postgres' #ansible_ssh_pass='secretpassword' - diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index a2de0bdbe..98d8ea6ac 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -48,9 +48,19 @@ postgresql_data_dir: "/pgdata/{{ postgresql_version }}/main" postgresql_wal_dir: "/pgwal/{{ postgresql_version }}/pg_wal" - - name: Set variables for TimescaleDB cluster deployment test + - name: Set variables for Extensions test ansible.builtin.set_fact: enable_timescale: true + enable_pg_repack: true + enable_pg_cron: true + enable_pgaudit: true + enable_pgvector: true + enable_postgis: true + enable_pgrouting: true + enable_pg_stat_kcache: true + enable_pg_wait_sampling: true + enable_pg_partman: true + enable_citus: "{{ 'false' if ansible_distribution_version == '24.04' else 'true' }}" # TODO Ubuntu 24.04 - name: Set variables for PostgreSQL Cluster update test ansible.builtin.set_fact: diff --git a/molecule/tests/postgres/postgres.yml b/molecule/tests/postgres/postgres.yml index 7f46a410e..d728d9e85 100644 --- a/molecule/tests/postgres/postgres.yml +++ b/molecule/tests/postgres/postgres.yml @@ -13,8 +13,7 @@ - name: Try to connect to PostgreSQL postgresql_ping: - login_host: "127.0.0.1" + login_unix_socket: "{{ postgresql_unix_socket_dir }}" login_port: "{{ postgresql_port }}" login_user: "{{ patroni_superuser_username }}" - login_password: "{{ patroni_superuser_password }}" login_db: template1 diff --git a/molecule/tests/roles/patroni/variables/custom_wal_dir.yml b/molecule/tests/roles/patroni/variables/custom_wal_dir.yml index 71b069314..24fae8e12 100644 --- a/molecule/tests/roles/patroni/variables/custom_wal_dir.yml +++ b/molecule/tests/roles/patroni/variables/custom_wal_dir.yml @@ -16,7 +16,7 @@ - name: Molecule.tests.roles.patroni.variables.custom_wal_dir | Set pg_wal_dir based on postgresql_version run_once: true ansible.builtin.set_fact: - pg_wal_dir: "{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}" + pg_wal_dir: "{{ 'pg_wal' if postgresql_version | int >= 10 else 'pg_xlog' }}" # 🔄 Determine the name based on postgresql_version - name: Molecule.tests.roles.patroni.variables.custom_wal_dir | Determine name for scenario 1 @@ -74,7 +74,7 @@ - name: Molecule.tests.roles.patroni.variables.custom_wal_dir | Set pg_wal_dir based on postgresql_version run_once: true ansible.builtin.set_fact: - pg_wal_dir: "{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}" + pg_wal_dir: "{{ 'pg_wal' if postgresql_version | int >= 10 else 'pg_xlog' }}" # 🔄 Determine the name based on postgresql_version - name: Molecule.tests.roles.patroni.variables.custom_wal_dir | Determine name for scenario 2 diff --git a/molecule/tests/roles/pre-checks/variables/pgbouncer.yml b/molecule/tests/roles/pre-checks/variables/pgbouncer.yml index 20423f952..cfcd5a7f2 100644 --- a/molecule/tests/roles/pre-checks/variables/pgbouncer.yml +++ b/molecule/tests/roles/pre-checks/variables/pgbouncer.yml @@ -100,18 +100,18 @@ var: pgbouncer_total_pool_size # ✅ Verifying the correctness of the calculated overall pool size -# The expected overall pool size is 50 +# The expected overall pool size is 130 # 10 from db1 # 20 from db2 -# 20 from db3 as db3 is not defined in pgbouncer_pools and hence, its pool size is pgbouncer_default_pool_size which is 20 -# If the calculated overall pool size is not 50, the test fails and an error message is displayed +# 100 from db3 as db3 is not defined in pgbouncer_pools and hence, its pool size is pgbouncer_default_pool_size which is 100 +# If the calculated overall pool size is not 130, the test fails and an error message is displayed - name: Molecule.tests.roles.pre-checks.variables.pgbouncer | Verify Total Pool Size Calculation run_once: true ansible.builtin.assert: that: - - pgbouncer_total_pool_size | int == 50 - fail_msg: "Test failed: pgbouncer_total_pool_size is not equal to 50." - success_msg: "Test passed: pgbouncer_total_pool_size is equal to 50." + - pgbouncer_total_pool_size | int == 130 + fail_msg: "Test failed: pgbouncer_total_pool_size is not equal to 130." + success_msg: "Test passed: pgbouncer_total_pool_size is equal to 130." # ====================================================================== # 📊 Start pgbouncer_total_pool_size (postgresql_databases not defined) diff --git a/plugins/callback/json_log.py b/plugins/callback/json_log.py new file mode 100644 index 000000000..a8e1b8d04 --- /dev/null +++ b/plugins/callback/json_log.py @@ -0,0 +1,125 @@ +import json +import os +from datetime import datetime +from ansible.plugins.callback import CallbackBase + + +# This Ansible callback plugin logs playbook results in JSON format. +# The log file path can be specified using the environment variable ANSIBLE_JSON_LOG_FILE. +# The log level can be controlled via the environment variable ANSIBLE_JSON_LOG_LEVEL. +# Available log levels: INFO (default), DETAIL, and DEBUG. + +class CallbackModule(CallbackBase): + CALLBACK_VERSION = 1.0 + CALLBACK_TYPE = 'notification' + CALLBACK_NAME = 'json_log' + CALLBACK_NEEDS_WHITELIST = True + + def __init__(self): + super(CallbackModule, self).__init__() + self.log_file_path = os.getenv('ANSIBLE_JSON_LOG_FILE') + self.log_level = os.getenv('ANSIBLE_JSON_LOG_LEVEL', 'INFO').upper() + self.results_started = False + + if self.log_file_path: + self._display.display(f"JSON Log callback plugin initialized. Log file: {self.log_file_path}") + # Initialize the log file + with open(self.log_file_path, 'w') as log_file: + log_file.write('[\n') + + def _record_task_result(self, result): + if not self.log_file_path: + return + + # Build the basic result structure with task, host, and timestamp + base_result = { + 'time': datetime.now().isoformat(), + 'task': result._task.get_name(), + 'host': result._host.get_name() + } + + # Add item information if available + if '_ansible_item_label' in result._result: + base_result['item'] = result._result['_ansible_item_label'] + elif 'item' in result._result: + base_result['item'] = result._result['item'] + + # Extend the result based on the log level + if self.log_level == 'DEBUG': + full_result = {**base_result, **result._result} + self._write_result_to_file(full_result) + elif self.log_level == 'DETAIL': + detailed_result = { + 'changed': result._result.get('changed', False), + 'failed': result._result.get('failed', False), + 'msg': result._result.get('msg', ''), + 'stdout': result._result.get('stdout', ''), + 'stderr': result._result.get('stderr', '') + } + self._write_result_to_file({**base_result, **detailed_result}) + else: + basic_result = { + 'changed': result._result.get('changed', False), + 'failed': result._result.get('failed', False), + 'msg': result._result.get('msg', '') + } + self._write_result_to_file({**base_result, **basic_result}) + + def _write_result_to_file(self, result): + try: + with open(self.log_file_path, 'a') as log_file: + if self.results_started: + log_file.write(',\n') + self.results_started = True + json.dump(result, log_file, indent=4) + except IOError as e: + self._display.warning(f"Failed to write to log file {self.log_file_path}: {e}") + + def v2_runner_item_on_ok(self, result): + # Records the result of a successfully executed task item. + self._record_task_result(result) + + def v2_runner_item_on_failed(self, result): + # Records the result of a failed task item. + self._record_task_result(result) + + def v2_runner_item_on_skipped(self, result): + # Do not record the result of a skipped task item. + pass + + def v2_runner_on_ok(self, result): + # Records the result of a successfully executed task. + self._record_task_result(result) + + def v2_runner_on_failed(self, result, ignore_errors=False): + # Records the result of a failed task. + self._record_task_result(result) + + def v2_runner_on_unreachable(self, result): + # Records the result of a task that failed because the host was unreachable. + self._record_task_result(result) + + def v2_playbook_on_stats(self, stats): + # Closes the JSON array in the log file when the playbook execution is complete. + if not self.log_file_path: + return + + summary = { + 'time': datetime.now().isoformat(), + 'summary': {}, + 'status': 'success' + } + + for host in stats.processed.keys(): + host_summary = stats.summarize(host) + summary['summary'][host] = host_summary + if host_summary['failures'] > 0 or host_summary['unreachable'] > 0: + summary['status'] = 'failed' + + try: + with open(self.log_file_path, 'a') as log_file: + log_file.write(',\n') + json.dump(summary, log_file, indent=4) + log_file.write('\n]\n') + except IOError as e: + self._display.warning(f"Failed to write to log file {self.log_file_path}: {e}") diff --git a/requirements.txt b/requirements.txt index 4b38e6738..a5d641bae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,5 @@ -ansible==9.2.0 -ansible-core==2.16.3 -Jinja2==3.1.4 -PyYAML==6.0.1 -cryptography==42.0.4 -packaging==23.2 -resolvelib==1.0.1 -MarkupSafe==2.1.5 -cffi==1.16.0 -pycparser==2.21 +ansible==9.8.0 +boto3==1.34.158 +dopy==0.3.7 +google-auth==2.33.0 +hcloud==2.2.0 diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 000000000..18f8c6e74 --- /dev/null +++ b/requirements.yml @@ -0,0 +1,12 @@ +--- +collections: + - name: amazon.aws + version: "==8.1.0" + - name: google.cloud + version: "==1.3.0" + - name: azure.azcollection + version: "==2.6.0" + - name: community.digitalocean + version: "==1.26.0" + - name: hetzner.hcloud + version: "==4.1.0" diff --git a/roles/add-repository/tasks/extensions.yml b/roles/add-repository/tasks/extensions.yml new file mode 100644 index 000000000..e20b6c0b5 --- /dev/null +++ b/roles/add-repository/tasks/extensions.yml @@ -0,0 +1,63 @@ +--- +# Extension Auto-Setup: repository + +# TimescaleDB (if 'enable_timescale' is 'true') +- block: + # Debian based + - name: Add TimescaleDB repository + ansible.builtin.deb822_repository: + name: "timescaledb" + types: [deb] + uris: "https://packagecloud.io/timescale/timescaledb/{{ ansible_distribution | lower }}" + signed_by: "https://packagecloud.io/timescale/timescaledb/gpgkey" + suites: "{{ ansible_distribution_release }}" + components: [main] + state: present + enabled: true + when: ansible_os_family == "Debian" + + # RedHat based + - name: Add TimescaleDB repository + ansible.builtin.yum_repository: + name: "timescaledb" + description: "TimescaleDB Repository" + baseurl: "https://packagecloud.io/timescale/timescaledb/el/{{ ansible_distribution_major_version }}/x86_64" + gpgkey: "https://packagecloud.io/timescale/timescaledb/gpgkey" + gpgcheck: "no" + when: ansible_os_family == "RedHat" + environment: "{{ proxy_env | default({}) }}" + when: (enable_timescale | default(false) | bool) or (enable_timescaledb | default(false) | bool) + tags: add_repo, timescaledb, timescale + +# Citus (if 'enable_citus' is 'true') +- block: + # Debian based + - name: Add Citus repository + ansible.builtin.deb822_repository: + name: "citusdata" + types: [deb] + uris: "https://repos.citusdata.com/community/{{ ansible_distribution | lower }}/" + suites: "{{ ansible_distribution_release }}" + components: [main] + signed_by: "https://repos.citusdata.com/community/gpgkey" + state: present + enabled: true + when: ansible_os_family == "Debian" + + # RedHat based + # TODO: Tests have shown that distributions such as Rocky Linux, AlmaLinux, Oracle Linux, and CentOS Stream are not yet supported. +# - name: Add Citus repository +# ansible.builtin.yum_repository: +# name: "citusdata" +# description: "Citus Repository" +# baseurl: "https://repos.citusdata.com/community/yum/{{ ansible_distribution_major_version }}/x86_64" +# gpgkey: "https://repos.citusdata.com/community/gpgkey" +# gpgcheck: "no" +# when: ansible_os_family == "RedHat" + environment: "{{ proxy_env | default({}) }}" + when: + - enable_citus | default(false) | bool + - postgresql_version | int >= 11 + tags: add_repo, citus + +... diff --git a/roles/add-repository/tasks/main.yml b/roles/add-repository/tasks/main.yml index 1dbedb498..2500fc540 100644 --- a/roles/add-repository/tasks/main.yml +++ b/roles/add-repository/tasks/main.yml @@ -45,8 +45,9 @@ name: "{{ item.name }}" description: "{{ item.description }}" baseurl: "{{ item.baseurl }}" - gpgkey: "{{ item.gpgkey }}" - gpgcheck: "{{ item.gpgcheck }}" + gpgkey: "{{ item.gpgkey | default(omit) }}" + gpgcheck: "{{ item.gpgcheck | default(true) }}" + enabled: "{{ item.enabled | default(true) }}" loop: "{{ yum_repository | flatten(1) }}" when: yum_repository | length > 0 @@ -121,48 +122,48 @@ retries: 3 when: install_postgresql_repo|bool tags: install_postgresql_repo + + # Enable Debuginfo repository + - block: + - name: Enable PostgreSQL debuginfo repository + ansible.builtin.shell: | + set -o pipefail; + sed -i '/\[pgdg[0-9]*-debuginfo\]/,/^$/ s/enabled=0/enabled=1/' {{ pgdg_redhat_repo_path }} + sed -i '/\[pgdg[0-9]*-debuginfo\]/,/^$/ s/gpgcheck=1/gpgcheck=0/' {{ pgdg_redhat_repo_path }} + vars: + pgdg_redhat_repo_path: "/etc/yum.repos.d/pgdg-redhat-all.repo" + + # Check if the repository entry exists in the file + - name: Check if pgdg{{ postgresql_version }}-debuginfo exists in repo file + ansible.builtin.lineinfile: + path: "{{ pgdg_redhat_repo_path }}" + regexp: '^\[pgdg{{ postgresql_version }}-debuginfo\]' + state: absent + check_mode: true + changed_when: false + register: repo_check + vars: + pgdg_redhat_repo_path: "/etc/yum.repos.d/pgdg-redhat-all.repo" + + # If the entry does not exist, try to add the repository + - name: Add pgdg{{ postgresql_version }}-debuginfo repo if not present + ansible.builtin.yum_repository: + name: "pgdg{{ postgresql_version }}-debuginfo" + description: "PostgreSQL {{ postgresql_version }} for RHEL {{ ansible_distribution_major_version }} - x86_64 - Debuginfo" + baseurl: "https://download.postgresql.org/pub/repos/yum/debug/{{ postgresql_version }}/redhat/rhel-{{ ansible_distribution_major_version }}-x86_64/" + gpgcheck: false + enabled: true + when: repo_check.found == 0 + when: debuginfo_package in postgresql_packages + vars: + debuginfo_package: "postgresql{{ postgresql_version }}-debuginfo" + tags: install_postgresql_repo, debuginfo environment: "{{ proxy_env | default({}) }}" when: installation_method == "repo" and ansible_os_family == "RedHat" tags: add_repo -# timescaledb (if enable_timescale is defined) -- block: - # Debian based - - name: Add TimescaleDB repository - ansible.builtin.deb822_repository: - name: "timescaledb" - types: [deb] - uris: "https://packagecloud.io/timescale/timescaledb/{{ ansible_distribution | lower }}" - signed_by: "https://packagecloud.io/timescale/timescaledb/gpgkey" - suites: "{{ ansible_distribution_release }}" - components: [main] - state: present - enabled: true - when: ansible_os_family == "Debian" - - - name: Update apt cache - ansible.builtin.apt: - update_cache: true - register: apt_status - until: apt_status is success - delay: 5 - retries: 3 - when: ansible_os_family == "Debian" - - # RedHat based - - name: Add TimescaleDB repository - ansible.builtin.yum_repository: - name: "timescale_timescaledb" - description: "timescaledb repo" - baseurl: "https://packagecloud.io/timescale/timescaledb/el/{{ ansible_distribution_major_version }}/x86_64" - gpgkey: "https://packagecloud.io/timescale/timescaledb/gpgkey" - gpgcheck: "no" - when: ansible_os_family == "RedHat" - environment: "{{ proxy_env | default({}) }}" - when: - - installation_method == "repo" - - enable_timescale is defined - - enable_timescale | bool - tags: add_repo +- name: Extensions repository + ansible.builtin.import_tasks: extensions.yml + when: installation_method == "repo" ... diff --git a/roles/authorized-keys/defaults/main.yml b/roles/authorized-keys/defaults/main.yml new file mode 100644 index 000000000..d42baf42c --- /dev/null +++ b/roles/authorized-keys/defaults/main.yml @@ -0,0 +1,5 @@ +--- + +ssh_public_keys: [] + +... diff --git a/roles/authorized-keys/tasks/main.yml b/roles/authorized-keys/tasks/main.yml new file mode 100644 index 000000000..0edc2ffc0 --- /dev/null +++ b/roles/authorized-keys/tasks/main.yml @@ -0,0 +1,31 @@ +--- + +- block: + - name: Get system username + become: false + ansible.builtin.command: whoami + register: system_user + changed_when: false + + - name: "Add public keys to ~{{ system_user.stdout }}/.ssh/authorized_keys" + ansible.posix.authorized_key: + user: "{{ system_user.stdout }}" + key: "{{ item }}" + state: present + loop: "{{ ssh_public_keys_list | map('replace', '\"', '') | map('replace', \"'\", \"\") | list }}" + vars: + ssh_public_keys_list: >- + {{ + (ssh_public_keys + | replace('\n', ',') + | split(',') + | map('trim') + | list) + if ssh_public_keys is string else ssh_public_keys + }} + when: + - ssh_public_keys is defined + - ssh_public_keys | length > 0 + tags: ssh_public_keys + +... diff --git a/roles/cloud-resources/defaults/main.yml b/roles/cloud-resources/defaults/main.yml new file mode 100644 index 000000000..c7ce6c4b3 --- /dev/null +++ b/roles/cloud-resources/defaults/main.yml @@ -0,0 +1,69 @@ +# yamllint disable rule:line-length +--- + +cloud_provider: "{{ provision | default('') }}" # Specifies the Cloud provider for server creation. Available options: 'aws', 'gcp', 'azure', 'digitalocean', 'hetzner'. +state: present # Set to 'present' to create a server, 'absent' to delete. + +server_count: "{{ servers_count | default(3) }}" # Number of servers in the cluster. +server_name: "{{ patroni_cluster_name }}-pgnode" # (optional) If not provided, a name will be auto-generated. Servers will be automatically named with suffixes 01, 02, 03, etc. +server_type: "" # (required) Server type. +server_image: "" # (required) OS image for the server. For Azure, use variables 'azure_vm_image_offer', 'azure_vm_image_publisher', 'azure_vm_image_sku', 'azure_vm_image_version' instead of variable 'server_image' +server_location: "" # (required) Server location or region. +server_network: "" # (optional) If provided, the server will be added to this network (needs to be created beforehand). +server_spot: false # Spot instance. Applicable for AWS, GCP, Azure. + +volume_type: "" # Volume type. Defaults: 'gp3' for AWS, 'pd-ssd' for GCP, 'StandardSSD_LRS' for Azure. +volume_size: 100 # Storage size for the data directory (in gigabytes). +system_volume_type: "" # System volume type. Defaults: 'gp3' for AWS, 'pd-ssd' for GCP, 'StandardSSD_LRS' for Azure. +system_volume_size: 100 # System volume size (in gigabytes). Applicable for AWS, GCP, Azure. + +ssh_key_name: "" # Name of the SSH key to be added to the server. +# Note: If not provided, all cloud available SSH keys will be added (applicable to DigitalOcean, Hetzner). +ssh_key_content: "" # (optional) If provided, the public key content will be added to the cloud (directly to the server for GCP). + +# Firewall / Security Group +cloud_firewall: true # Specify 'false' if you don't want to configure Firewall rules, or want to manage them yourself. + +ssh_public_access: true # Allow public ssh access (required for deployment from the public network). +ssh_public_allowed_ips: "" # (comma-separated list of IP addresses in CIDR format) If empty, then public access is allowed for any IP address. +netdata_public_access: true # Allow access to the Netdata monitoring from the public network (if 'netdata_install' is 'true'). +netdata_public_allowed_ips: "" # (comma-separated list of IP addresses in CIDR format) If empty, then public access is allowed for any IP address. +database_public_access: false # Allow access to the database from the public network. +database_public_allowed_ips: "" # (comma-separated list of IP addresses in CIDR format) If empty, then public access is allowed for any IP address. + +# Load balancer +cloud_load_balancer: true # Create a Load Balancer in the Cloud. + +# Backups (if 'pgbackrest_install' or 'wal_g_install' is 'true') +aws_s3_bucket_create: true # if 'cloud_provider=aws' +aws_s3_bucket_name: "{{ patroni_cluster_name }}}-backup" # Name of the S3 bucket. Bucket naming rules: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html +aws_s3_bucket_region: "{{ server_location }}" # The AWS region to use. +aws_s3_bucket_object_lock_enabled: false # Whether S3 Object Lock to be enabled. +aws_s3_bucket_encryption: "AES256" # Describes the default server-side encryption to apply to new objects in the bucket. Choices: "AES256", "aws:kms" +aws_s3_bucket_block_public_acls: true # Sets BlockPublicAcls value. +aws_s3_bucket_ignore_public_acls: true # Sets IgnorePublicAcls value. +aws_s3_bucket_absent: false # Allow to delete S3 bucket when deleting a cluster servers using the 'state=absent' variable. + +gcp_bucket_create: true # if 'cloud_provider=gcp' +gcp_bucket_name: "{{ patroni_cluster_name }}-backup" # Name of the GCS bucket. +gcp_bucket_storage_class: "MULTI_REGIONAL" # The bucket’s default storage class. Values include: MULTI_REGIONAL, REGIONAL, STANDARD, NEARLINE, COLDLINE, ARCHIVE, DURABLE_REDUCED_AVAILABILITY. +gcp_bucket_default_object_acl: "projectPrivate" # Apply a predefined set of default object access controls to this bucket. +gcp_bucket_absent: false # Allow to delete GCS bucket when deleting a cluster servers using the 'state=absent' variable. + +azure_blob_storage_create: true # if 'cloud_provider=azure' +azure_blob_storage_name: "{{ patroni_cluster_name }}-backup" # Name of a blob container within the storage account. +azure_blob_storage_blob_type: "block" # Type of blob object. Values include: block, page. +azure_blob_storage_account_name: "{{ patroni_cluster_name | lower | replace('-', '') | truncate(24, true, '') }}" # Storage account name must be between 3 and 24 characters in length and use numbers and lower-case letters only. +azure_blob_storage_account_type: "Standard_RAGRS" # Type of storage account. Values include: Standard_LRS, Standard_GRS, Standard_RAGRS, Standard_ZRS, Standard_RAGZRS, Standard_GZRS, Premium_LRS, Premium_ZRS. +azure_blob_storage_account_kind: "BlobStorage" # The kind of storage. Values include: Storage, StorageV2, BlobStorage, BlockBlobStorage, FileStorage. +azure_blob_storage_account_access_tier: "Hot" # The access tier for this storage account. Required when kind=BlobStorage. +azure_blob_storage_account_public_network_access: "Enabled" # Allow public network access to Storage Account to create Blob Storage container. +azure_blob_storage_account_allow_blob_public_access: false # Disallow public anonymous access. +azure_blob_storage_absent: false # Allow to delete Azure Blob Storage when deleting a cluster servers using the 'state=absent' variable. + +digital_ocean_spaces_create: true # if 'cloud_provider=digitalocean' +digital_ocean_spaces_name: "{{ patroni_cluster_name }}-backup" # Name of the Spaces Object Storage (S3 bucket). +digital_ocean_spaces_region: "nyc3" # The region to create the Space in. +digital_ocean_spaces_absent: false # Allow to delete Spaces Object Storage when deleting a cluster servers using the 'state=absent' variable. + +... diff --git a/roles/cloud-resources/tasks/aws.yml b/roles/cloud-resources/tasks/aws.yml new file mode 100644 index 000000000..a71810094 --- /dev/null +++ b/roles/cloud-resources/tasks/aws.yml @@ -0,0 +1,506 @@ +--- +# Dependencies +- name: Install Python dependencies + block: + - name: Ensure that 'python3-pip' package is present on controlling host + ansible.builtin.package: + name: python3-pip + state: present + register: package_status + until: package_status is success + delay: 10 + retries: 3 + delegate_to: 127.0.0.1 + run_once: true + when: ansible_distribution != "MacOSX" + + - name: Ensure that 'boto3' dependency is present on controlling host + ansible.builtin.pip: + name: boto3 + extra_args: --user + delegate_to: 127.0.0.1 + become: false + vars: + ansible_become: false + run_once: true + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + PIP_BREAK_SYSTEM_PACKAGES: "1" + +# SSH key +- block: + # Delete the temporary ssh key from the cloud (if exists) + - name: "AWS: Remove temporary SSH key '{{ ssh_key_name }}' from cloud (if any)" + amazon.aws.ec2_key: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ ssh_key_name }}" + region: "{{ server_location }}" + state: absent + when: + - ssh_key_name is defined + - tmp_ssh_key_name is defined + - ssh_key_name == tmp_ssh_key_name + + # if ssh_key_name and ssh_key_content is specified, add this ssh key to the cloud + - name: "AWS: Add SSH key '{{ ssh_key_name }}' to cloud" + amazon.aws.ec2_key: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ ssh_key_name }}" + key_material: "{{ ssh_key_content }}" + region: "{{ server_location }}" + state: present + register: ssh_key_result + when: + - ssh_key_name | length > 0 + - ssh_key_content | length > 0 + when: state == 'present' + +# Create (if state is present) +- block: + # if server_network is specified, get vpc id for this subnet + - name: "AWS: Gather information about VPC for '{{ server_network }}'" + amazon.aws.ec2_vpc_subnet_info: + region: "{{ server_location }}" + subnet_ids: "{{ server_network }}" + register: vpc_subnet_info + when: server_network | length > 0 + + - name: "Set variable: vpc_id" + ansible.builtin.set_fact: + vpc_id: "{{ vpc_subnet_info.subnets[0].vpc_id }}" + when: + - server_network | length > 0 + - vpc_subnet_info.subnets[0].vpc_id is defined + + # if server_network is not specified, use default vpc subnet + - name: "AWS: Gather information about default VPC" + amazon.aws.ec2_vpc_net_info: + region: "{{ server_location }}" + filters: + "is-default": true + register: vpc_info + when: server_network | length < 1 + + - name: "AWS: Gather information about VPC subnet for default VPC" + amazon.aws.ec2_vpc_subnet_info: + region: "{{ server_location }}" + filters: + vpc-id: "{{ vpc_info.vpcs[0].id }}" + register: vpc_subnet_info + when: + - server_network | length < 1 + - vpc_info.vpcs[0].id is defined + + - name: "Set variable: vpc_id" + ansible.builtin.set_fact: + vpc_id: "{{ vpc_info.vpcs[0].id }}" + when: + - server_network | length < 1 + - vpc_info.vpcs[0].id is defined + + - name: "Set variable: server_network" + ansible.builtin.set_fact: + server_network: "{{ vpc_subnet_info.subnets[0].id }}" + when: + - server_network | length < 1 + - vpc_subnet_info.subnets[0].id is defined + + # Security Group (Firewall) + - name: "AWS: Create or modify Security Group" + amazon.aws.ec2_security_group: + name: "{{ patroni_cluster_name }}-security-group" + state: present + description: "Security Group for Postgres cluster" + vpc_id: "{{ vpc_id }}" + region: "{{ server_location }}" + rules: "{{ rules }}" + vars: + rules: >- + {{ + ([ + { + 'rule_desc': 'SSH public access', + 'proto': 'tcp', + 'ports': [ansible_ssh_port | default(22)], + 'cidr_ip': ssh_public_allowed_ips | default('0.0.0.0/0', true) | split(',') + } + ] if ssh_public_access | bool else []) + + ([ + { + 'rule_desc': 'Netdata public access', + 'proto': 'tcp', + 'ports': [netdata_port | default('19999')], + 'cidr_ip': netdata_public_allowed_ips | default('0.0.0.0/0', true) | split(',') + } + ] if netdata_install | bool and netdata_public_access | bool else []) + + ([ + { + 'rule_desc': 'Database public access', + 'proto': 'tcp', + 'ports': + ([ + haproxy_listen_port.master | default('5000'), + haproxy_listen_port.replicas | default('5001'), + haproxy_listen_port.replicas_sync | default('5002'), + haproxy_listen_port.replicas_async | default('5003') + ] if with_haproxy_load_balancing | bool else []) + + ([ + pgbouncer_listen_port | default('6432') + ] if not with_haproxy_load_balancing | bool and pgbouncer_install | bool else []) + + ([ + postgresql_port | default('5432') + ] if not with_haproxy_load_balancing | bool and not pgbouncer_install | bool else []), + 'cidr_ip': database_public_allowed_ips | default('0.0.0.0/0', true) | split(',') + } + ] if database_public_access | bool else []) + + [{ + 'rule_desc': 'Postgres cluster ports', + 'proto': 'tcp', + 'ports': + [ansible_ssh_port | default(22)] + + ([ + netdata_port | default('19999') + ] if netdata_install | bool else []) + + ([ + pgbouncer_listen_port | default('6432') + ] if pgbouncer_install | bool else []) + + [ + postgresql_port | default('5432'), + patroni_restapi_port | default('8008') + ] + + ([ + haproxy_listen_port.master | default('5000'), + haproxy_listen_port.replicas | default('5001'), + haproxy_listen_port.replicas_sync | default('5002'), + haproxy_listen_port.replicas_async | default('5003'), + haproxy_listen_port.stats | default('7000') + ] if with_haproxy_load_balancing | bool else []) + + ([ + etcd_client_port | default('2379'), + etcd_peer_port | default('2380') + ] if dcs_type == 'etcd' else []) + + ([ + consul_ports_dns | default('8600'), + consul_ports_http | default('8500'), + consul_ports_rpc | default('8400'), + consul_ports_serf_lan | default('8301'), + consul_ports_serf_wan | default('8302'), + consul_ports_server | default('8300') + ] if dcs_type == 'consul' else []), + 'cidr_ip': vpc_subnet_info.subnets[0].cidr_block + }] + }} + register: ec2_security_group_result + when: cloud_firewall | bool + + # Server and volume + - name: "AWS: Create or modify EC2 instance" + amazon.aws.ec2_instance: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + state: present + instance_type: "{{ server_type }}" + image_id: "{{ server_image }}" + key_name: "{{ ssh_key_name }}" + region: "{{ server_location }}" + security_groups: "{{ ([] if not cloud_firewall | bool else [patroni_cluster_name + '-security-group']) }}" + vpc_subnet_id: "{{ server_network }}" + network: + assign_public_ip: true + delete_on_termination: true + volumes: + - device_name: /dev/sda1 + ebs: + volume_type: "{{ system_volume_type | default('gp3', true) }}" + volume_size: "{{ system_volume_size | default(80) | int }}" + delete_on_termination: true + - device_name: /dev/sdb + ebs: + volume_type: "{{ volume_type | default('gp3', true) }}" + volume_size: "{{ volume_size | int }}" + delete_on_termination: true + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: server_result + until: + - server_result.instances[0].public_ip_address is defined + - server_result.instances[0].public_ip_address | length > 0 + retries: 3 + delay: 10 + when: not server_spot | default(aws_ec2_spot_instance | default(false)) | bool + + # Spot instance (if 'server_spot' is 'true') + - block: + - name: "AWS: Gather information about EC2 Spot instances" + amazon.aws.ec2_instance_info: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + region: "{{ server_location }}" + filters: + instance-lifecycle: "spot" + instance-type: "{{ server_type }}" + image-id: "{{ server_image }}" + instance-state-name: ["pending", "running", "shutting-down", "stopping", "stopped"] + "tag:Name": "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: ec2_spot_instance_info + + # if spot instances are still created, create them + - name: "AWS: Create a request for EC2 Spot instance" + amazon.aws.ec2_spot_instance: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + region: "{{ server_location }}" + state: present + launch_specification: + instance_type: "{{ server_type }}" + image_id: "{{ server_image }}" + key_name: "{{ ssh_key_name }}" + network_interfaces: + - subnet_id: "{{ server_network }}" + groups: "{{ ec2_security_group_result.group_id }}" + associate_public_ip_address: true + delete_on_termination: true + device_index: 0 + block_device_mappings: + - device_name: /dev/sda1 + ebs: + volume_type: "{{ volume_type | default('gp3', true) }}" + volume_size: 100 # TODO: use 'system_volume_size' variable (https://github.com/ansible-collections/amazon.aws/issues/1949) + delete_on_termination: true + - device_name: /dev/sdb + ebs: + volume_type: "{{ volume_type | default('gp3', true) }}" + volume_size: 100 # TODO: use 'volume_size' variable (https://github.com/ansible-collections/amazon.aws/issues/1949) + delete_on_termination: true + tags: + Name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + loop: "{{ ec2_spot_instance_info.results }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: ec2_spot_request_result + when: item.instances[0] | default('') | length < 1 + + - name: "AWS: Rename the EC2 Spot instance" + amazon.aws.ec2_instance: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + region: "{{ server_location }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + filters: + spot-instance-request-id: "{{ item.spot_request.spot_instance_request_id }}" + loop: "{{ ec2_spot_request_result.results }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: ec2_spot_instance_result + until: + - ec2_spot_instance_result.instances[0].public_ip_address is defined + - ec2_spot_instance_result.instances[0].public_ip_address | length > 0 + retries: 3 + delay: 10 + when: item.spot_request.spot_instance_request_id is defined + + # if spot instances are created now + - name: "Set variable: server_result" + ansible.builtin.set_fact: + server_result: "{{ ec2_spot_instance_result }}" + when: ec2_spot_instance_result.changed | default(false) + + # if spot instances have already been created + - name: "Set variable: server_result" + ansible.builtin.set_fact: + server_result: "{{ ec2_spot_instance_info }}" + when: not ec2_spot_instance_result.changed | default(false) + when: server_spot | default(aws_ec2_spot_instance | default(false)) | bool + + # Load Balancer (ELB) + - name: "AWS: Create or modify Elastic Load Balancer (ELB)" + amazon.aws.elb_classic_lb: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + region: "{{ server_location }}" + security_group_ids: + - "{{ ec2_security_group_result.group_id }}" + subnets: + - "{{ server_network }}" + instance_ids: "{{ server_result.results | map(attribute='instances') | map('first') | map(attribute='instance_id') }}" + purge_instance_ids: true + listeners: + - protocol: tcp + load_balancer_port: "{{ pgbouncer_listen_port | default('6432') if pgbouncer_install | bool else postgresql_port | default('5432') }}" + instance_port: "{{ pgbouncer_listen_port | default('6432') if pgbouncer_install | bool else postgresql_port | default('5432') }}" + health_check: + ping_protocol: "http" + ping_port: "{{ patroni_restapi_port }}" + ping_path: "/{{ item }}" + interval: 5 + timeout: 2 + unhealthy_threshold: 2 + healthy_threshold: 3 + idle_timeout: 600 + scheme: "{{ 'internet-facing' if database_public_access | bool else 'internal' }}" + state: present + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + register: aws_elb_classic_lb + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + # S3 bucket (Backups) + - name: "AWS: Create S3 bucket '{{ aws_s3_bucket_name }}'" + amazon.aws.s3_bucket: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ aws_s3_bucket_name }}" + region: "{{ aws_s3_bucket_region }}" + object_lock_enabled: "{{ aws_s3_bucket_object_lock_enabled }}" + encryption: "{{ aws_s3_bucket_encryption }}" + public_access: + block_public_acls: "{{ aws_s3_bucket_block_public_acls }}" + ignore_public_acls: "{{ aws_s3_bucket_ignore_public_acls }}" + state: present + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - aws_s3_bucket_create | bool + when: state == 'present' + +- name: Wait for host to be available via SSH + ansible.builtin.wait_for: + host: "{{ item.instances[0].public_ip_address }}" + port: 22 + delay: 5 + timeout: 300 + loop: "{{ server_result.results }}" + loop_control: + label: "{{ item.instances[0].public_ip_address | default('N/A') }}" + when: + - server_result.results is defined + - item.instances is defined + +# Info +- name: Server info + ansible.builtin.debug: + msg: + id: "{{ item.instances[0].instance_id }}" + name: "{{ item.instances[0].tags.Name }}" + image: "{{ item.instances[0].image_id }}" + type: "{{ item.instances[0].instance_type }}" + volume_size: "{{ volume_size }} GB" + public_ip: "{{ item.instances[0].public_ip_address }}" + private_ip: "{{ item.instances[0].private_ip_address }}" + loop: "{{ server_result.results }}" + loop_control: + label: "{{ item.instances[0].public_ip_address | default('N/A') }}" + when: + - server_result.results is defined + - item.instances is defined + +# Inventory +- block: + - name: "Inventory | Initialize ip_addresses variable" + ansible.builtin.set_fact: + ip_addresses: [] + + - name: "Inventory | Extract IP addresses" + ansible.builtin.set_fact: + ip_addresses: >- + {{ ip_addresses + + [{'public_ip': item.instances[0].public_ip_address, + 'private_ip': item.instances[0].private_ip_address}] + }} + loop: "{{ server_result.results | selectattr('instances', 'defined') }}" + loop_control: + label: "public_ip: {{ item.instances[0].public_ip_address }}, private_ip: {{ item.instances[0].private_ip_address }}" + + - name: "Inventory | Generate in-memory inventory" + ansible.builtin.import_tasks: inventory.yml + when: + - server_result.results is defined + - server_result.results | selectattr('instances', 'defined') + +# Delete the temporary ssh key from the cloud after creating the EC2 instance +- name: "AWS: Remove temporary SSH key '{{ ssh_key_name }}' from cloud" + amazon.aws.ec2_key: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ ssh_key_name }}" + region: "{{ server_location }}" + state: absent + register: ssh_key_result + when: + - ssh_key_name is defined + - tmp_ssh_key_name is defined + - ssh_key_name == tmp_ssh_key_name + +# Delete (if state is absent) +- block: + - name: "AWS: Delete EC2 instance" + amazon.aws.ec2_instance: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + region: "{{ server_location }}" + state: absent + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + + - name: "AWS: Delete Elastic Load Balancer (ELB)" + amazon.aws.elb_classic_lb: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + region: "{{ server_location }}" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "AWS: Delete Security Group" + amazon.aws.ec2_security_group: + name: "{{ patroni_cluster_name }}-security-group" + region: "{{ server_location }}" + state: absent + register: ec2_security_group_delete + until: ec2_security_group_delete is success + delay: 10 + retries: 3 + + - name: "AWS: Delete S3 bucket '{{ aws_s3_bucket_name }}'" + amazon.aws.s3_bucket: + access_key: "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID') }}" + secret_key: "{{ lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY') }}" + name: "{{ aws_s3_bucket_name }}" + region: "{{ aws_s3_bucket_region }}" + state: absent + force: true + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - aws_s3_bucket_absent | default(false) | bool + when: state == 'absent' + +... diff --git a/roles/cloud-resources/tasks/azure.yml b/roles/cloud-resources/tasks/azure.yml new file mode 100644 index 000000000..949be5eb6 --- /dev/null +++ b/roles/cloud-resources/tasks/azure.yml @@ -0,0 +1,557 @@ +--- +# Dependencies +- name: Install Python dependencies + block: + - name: Ensure that 'python3-pip' package is present on controlling host + ansible.builtin.package: + name: python3-pip + state: present + register: package_status + until: package_status is success + delay: 10 + retries: 3 + when: ansible_distribution != "MacOSX" + + - name: Ensure that Azure collection is installed on controlling host + ansible.builtin.command: ansible-galaxy collection list azure.azcollection + changed_when: false + failed_when: false + register: azcollection_result + + - name: Azure collection not installed + ansible.builtin.fail: + msg: + - "Please install Azure collection" + - "ansible-galaxy collection install azure.azcollection" + when: + - azcollection_result.stderr is search("unable to find") + + - name: Get ansible_collections path + ansible.builtin.shell: > + set -o pipefail; + ansible-galaxy collection list | grep ansible_collections | head -n 1 | awk '{print $2}' + args: + executable: /bin/bash + register: collections_path + changed_when: false + when: ansible_collections_path is not defined + + - name: Ensure that Azure collection requirements is present on controlling host + ansible.builtin.pip: + requirements: "{{ ansible_collections_path | default(collections_path.stdout) }}/azure/azcollection/requirements.txt" + executable: pip3 + extra_args: "--trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org" + umask: "0022" + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + PIP_BREAK_SYSTEM_PACKAGES: "1" + + # CLI required for task "Add virtual machine IP addresses to Load Balancer backend pool" + - name: Check if Azure CLI is installed + ansible.builtin.command: az --version + register: az_version_result + changed_when: false + failed_when: false + + # try to install CLI (if not installed) + - name: Install Azure CLI + community.general.homebrew: + name: azure-cli + state: present + ignore_errors: true + when: + - az_version_result.rc != 0 + - ansible_distribution == "MacOSX" + + - name: Install Azure CLI + ansible.builtin.shell: > + set -o pipefail; + curl -sL https://aka.ms/InstallAzureCli | bash + args: + executable: /bin/bash + ignore_errors: true + when: + - az_version_result.rc != 0 + - ansible_distribution != "MacOSX" + delegate_to: 127.0.0.1 + become: false + run_once: true + +# Create (if state is present) +- block: + # if ssh_key_content is not defined, get the user public key from the system (if exists) + - name: "Set variable: ssh_key_content" + ansible.builtin.set_fact: + ssh_key_content: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" + no_log: true # do not display the public key + when: ssh_key_content is not defined or ssh_key_content | length < 1 + + - name: "Azure: Create resource group" + azure.azcollection.azure_rm_resourcegroup: + name: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + location: "{{ server_location }}" + + # if server_network is not specified, create a network and subnet + - block: + - name: "Azure: Create virtual network" + azure.azcollection.azure_rm_virtualnetwork: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ azure_virtual_network | default('postgres-cluster-network') }}" + address_prefixes_cidr: ["{{ azure_virtual_network_prefix | default('10.0.0.0/16') }}"] + + - name: "Azure: Create subnet" + azure.azcollection.azure_rm_subnet: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ azure_subnet | default('postgres-cluster-subnet') }}" + address_prefix_cidr: "{{ azure_subnet_prefix | default('10.0.1.0/24') }}" + virtual_network: "{{ azure_virtual_network | default('postgres-cluster-network') }}" + when: server_network | length < 1 + + - name: "Azure: Gather information about network" + azure.azcollection.azure_rm_virtualnetwork_info: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ server_network | default(azure_virtual_network | default('postgres-cluster-network'), true) }}" + register: network_info + + - name: "Azure: Create public IP address" + azure.azcollection.azure_rm_publicipaddress: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-public-ip" + allocation_method: "Static" + sku: "Standard" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-public-ip" + register: public_ip_address + + # Security Group (Firewall) + - name: "Azure: Create or modify Security Group" + azure.azcollection.azure_rm_securitygroup: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ patroni_cluster_name }}-security-group" + rules: "{{ rules }}" + vars: + rules: >- + {{ + ([{ + 'name': 'public-ssh-rule', + 'description': 'SSH public access', + 'protocol': 'Tcp', + 'destination_port_range': [ansible_ssh_port | default(22)], + 'source_address_prefix': ssh_public_allowed_ips | default('0.0.0.0/0', true) | split(','), + 'access': 'Allow', + 'priority': 1200, + 'direction': 'Inbound' + }] if ssh_public_access | bool else []) + + ([{ + 'name': 'public-netdata-rule', + 'description': 'Netdata public access', + 'protocol': 'Tcp', + 'destination_port_range': [netdata_port | default('19999')], + 'source_address_prefix': netdata_public_allowed_ips | default('0.0.0.0/0', true) | split(','), + 'access': 'Allow', + 'priority': 1400, + 'direction': 'Inbound' + }] if netdata_install | bool and netdata_public_access | bool else []) + + ([{ + 'name': 'public-database-rule', + 'description': 'Database public access', + 'protocol': 'Tcp', + 'destination_port_range': ([ + haproxy_listen_port.master | default('5000'), + haproxy_listen_port.replicas | default('5001'), + haproxy_listen_port.replicas_sync | default('5002'), + haproxy_listen_port.replicas_async | default('5003') + ] if with_haproxy_load_balancing | bool else []) + + ([pgbouncer_listen_port | default('6432')] if not with_haproxy_load_balancing | bool and pgbouncer_install | bool else []) + + ([postgresql_port | default('5432')] if not with_haproxy_load_balancing | bool and not pgbouncer_install | bool else []), + 'source_address_prefix': '0.0.0.0/0', + 'access': 'Allow', + 'priority': 1300, + 'direction': 'Inbound' + }] if database_public_access | bool else []) + + [{ + 'name': 'private-postgres-cluster-rule', + 'description': 'Postgres cluster ports', + 'protocol': 'Tcp', + 'destination_port_range': [ansible_ssh_port | default(22)] + + ([ + haproxy_listen_port.master | default('5000'), + haproxy_listen_port.replicas | default('5001'), + haproxy_listen_port.replicas_sync | default('5002'), + haproxy_listen_port.replicas_async | default('5003'), + haproxy_listen_port.stats | default('7000') + ] if with_haproxy_load_balancing | bool else []) + + ([pgbouncer_listen_port | default('6432')] if pgbouncer_install | bool else []) + + [ + postgresql_port | default('5432'), + patroni_restapi_port | default('8008'), + ] + + ([ + etcd_client_port | default('2379'), + etcd_peer_port | default('2380'), + ] if dcs_type == 'etcd' else []) + + ([ + consul_ports_dns | default('8600'), + consul_ports_http | default('8500'), + consul_ports_rpc | default('8400'), + consul_ports_serf_lan | default('8301'), + consul_ports_serf_wan | default('8302'), + consul_ports_server | default('8300') + ] if dcs_type == 'consul' else []) + + ([netdata_port | default('19999')] if netdata_install | bool else []), + 'source_address_prefix': network_info.virtualnetworks[0].address_prefixes, + 'access': 'Allow', + 'priority': 1000, + 'direction': 'Inbound' + }] + }} + when: cloud_firewall | bool + + # Network interface + - name: "Azure: Create network interface" + azure.azcollection.azure_rm_networkinterface: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-network-interface" + virtual_network: "{{ server_network | default(azure_virtual_network | default('postgres-cluster-network'), true) }}" + subnet_name: "{{ azure_subnet | default('postgres-cluster-subnet') }}" + security_group: "{{ patroni_cluster_name }}-security-group" + ip_configurations: + - name: ipconfig1 + primary: true + public_ip_address_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-public-ip" + dns_servers: + - 8.8.8.8 + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-network-interface" + + # Server and volume + - name: "Azure: Create virtual machine" + azure.azcollection.azure_rm_virtualmachine: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + state: present + started: true + location: "{{ server_location }}" + vm_size: "{{ server_type }}" + priority: "{{ 'Spot' if server_spot | default(false) | bool else 'None' }}" + eviction_policy: "{{ 'Deallocate' if server_spot | default(false) | bool else omit }}" + admin_username: "{{ azure_admin_username | default('azureadmin') }}" + ssh_public_keys: + - path: /home/azureadmin/.ssh/authorized_keys + key_data: "{{ ssh_key_content }}" + ssh_password_enabled: false + image: + offer: "{{ azure_vm_image_offer | default('0001-com-ubuntu-server-jammy') }}" + publisher: "{{ azure_vm_image_publisher | default('Canonical') }}" + sku: "{{ azure_vm_image_sku | default('22_04-lts-gen2') }}" + version: "{{ azure_vm_image_version | default('latest') }}" + os_type: Linux + os_disk_size_gb: "{{ system_volume_size | default('80') }}" # system disk size + managed_disk_type: "{{ system_volume_type | default('StandardSSD_LRS', true) }}" + data_disks: + - lun: 0 + disk_size_gb: "{{ volume_size | int }}" + managed_disk_type: "{{ volume_type | default('StandardSSD_LRS', true) }}" + network_interface_names: + - "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-network-interface" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: server_result + + # Load Balancer + - name: "Azure: Create public IP address for Load Balancer" + azure.azcollection.azure_rm_publicipaddress: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-public-ip" + allocation_method: "Static" + sku: "Standard" + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-public-ip" + register: azure_load_balancer_public_ip + when: database_public_access | bool and cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Azure: Create or modify Load Balancer" + azure.azcollection.azure_rm_loadbalancer: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + location: "{{ server_location }}" + frontend_ip_configurations: + - name: "{{ patroni_cluster_name }}-{{ item }}-frontend" + public_ip_address: "{{ database_public_access | bool | ternary(patroni_cluster_name ~ '-' ~ item ~ '-public-ip', omit) }}" + subnet: "{{ database_public_access | bool | ternary(omit, network_info.virtualnetworks[0].subnets[0].id) }}" + backend_address_pools: + - name: "{{ patroni_cluster_name }}-{{ item }}-backend" + probes: + - name: "{{ patroni_cluster_name }}-{{ item }}-health-probe" + protocol: "Http" + port: "{{ patroni_restapi_port }}" + request_path: "/{{ item }}" + interval: 5 + fail_count: 2 + load_balancing_rules: + - name: "{{ patroni_cluster_name }}-{{ item }}-rule" + frontend_ip_configuration: "{{ patroni_cluster_name }}-{{ item }}-frontend" + frontend_port: "{{ pgbouncer_listen_port | default('6432') if pgbouncer_install | bool else postgresql_port | default('5432') }}" + backend_address_pool: "{{ patroni_cluster_name }}-{{ item }}-backend" + backend_port: "{{ pgbouncer_listen_port | default('6432') if pgbouncer_install | bool else postgresql_port | default('5432') }}" + probe: "{{ patroni_cluster_name }}-{{ item }}-health-probe" + protocol: "Tcp" + idle_timeout: 10 # in minutes + enable_floating_ip: false + disable_outbound_snat: true + sku: "Standard" + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + register: azure_load_balancer + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: Extract virtual machine private IPs + ansible.builtin.set_fact: + private_ips: >- + {{ + private_ips | default([]) + + [item.ansible_facts.azure_vm.network_profile.network_interfaces[0].properties.ip_configurations[0].private_ip_address] + }} + loop: "{{ server_result.results | selectattr('ansible_facts.azure_vm', 'defined') }}" + loop_control: + label: "{{ item.ansible_facts.azure_vm.network_profile.network_interfaces[0].properties.ip_configurations[0].private_ip_address }}" + + # Note: We use Azure CLI here because there is no ansible module available to manage the list of IP addresses within a backend pool. + - name: "Azure: Add virtual machine IP addresses to Load Balancer backend pool" + ansible.builtin.shell: | + {% for ip in private_ips %} + az network lb address-pool address add \ + --resource-group {{ azure_resource_group | default('postgres-cluster-resource-group-' ~ server_location) }} \ + --lb-name {{ patroni_cluster_name }}-{{ item }} \ + --pool-name {{ patroni_cluster_name }}-{{ item }}-backend \ + --vnet {{ azure_virtual_network | default('postgres-cluster-network') }} \ + --name address-{{ ip }} \ + --ip-address {{ ip }} + {% endfor %} + args: + executable: /bin/bash + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-backend" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + # Azure Blob Storage (Backups) + - block: + - name: "Azure: Create Storage Account '{{ azure_blob_storage_account_name }}'" + azure.azcollection.azure_rm_storageaccount: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ azure_blob_storage_account_name }}" + account_type: "{{ azure_blob_storage_account_type }}" + kind: "{{ azure_blob_storage_account_kind }}" + access_tier: "{{ azure_blob_storage_account_access_tier }}" + public_network_access: "{{ azure_blob_storage_account_public_network_access }}" + allow_blob_public_access: "{{ azure_blob_storage_account_allow_blob_public_access }}" + state: present + + - name: "Azure: Get Storage Account info" + azure.azcollection.azure_rm_storageaccount_info: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ azure_blob_storage_account_name }}" + show_connection_string: true + no_log: true # do not output storage account contents to the ansible log + register: azure_storage_account_info + + - name: "Set variable: azure_storage_account_key" + ansible.builtin.set_fact: + azure_storage_account_key: "{{ azure_storage_account_info.storageaccounts[0].primary_endpoints.key }}" + no_log: true # do not output storage account contents to the ansible log + + - name: "Azure: Create Blob Storage container '{{ azure_blob_storage_name }}'" + azure.azcollection.azure_rm_storageblob: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + account_name: "{{ azure_blob_storage_account_name }}" + container: "{{ azure_blob_storage_name }}" + blob_type: "{{ azure_blob_storage_blob_type }}" + state: present + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - azure_blob_storage_create | bool + when: state == 'present' + +- name: "Wait for host to be available via SSH" + ansible.builtin.wait_for: + host: "{{ item.state.ip_address }}" + port: 22 + delay: 5 + timeout: 300 + loop: "{{ public_ip_address.results }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + when: + - public_ip_address.results is defined + - item.state.ip_address is defined + +# Info +- name: Server info + ansible.builtin.debug: + msg: + id: "{{ item.ansible_facts.azure_vm.id | default('N/A') }}" + name: "{{ item.ansible_facts.azure_vm.name | default('N/A') }}" + image: "{{ item.ansible_facts.azure_vm.storage_profile.image_reference | default('N/A') }}" + type: "{{ item.ansible_facts.azure_vm.hardware_profile.vm_size | default('N/A') }}" + volume_size: "{{ item.ansible_facts.azure_vm.storage_profile.data_disks[0].disk_size_gb | default('N/A') }} GB" + volume_type: "{{ item.ansible_facts.azure_vm.storage_profile.data_disks[0].managed_disk.storage_account_type | default('N/A') }}" + public_ip: >- + {{ + public_ip_address.results | selectattr('idx', 'equalto', item.idx) | map(attribute='state.ip_address') | first | default('N/A') + }} + private_ip: >- + {{ + item.ansible_facts.azure_vm.network_profile.network_interfaces[0].properties.ip_configurations[0].private_ip_address | default('N/A') + }} + loop: "{{ server_result.results }}" + loop_control: + label: "{{ item.ansible_facts.azure_vm.name | default('N/A') }}" + when: + - server_result.results is defined + - item.ansible_facts is defined + +# Inventory +- block: + - name: "Inventory | Initialize ip_addresses variable" + ansible.builtin.set_fact: + ip_addresses: [] + + - name: "Inventory | Extract IP addresses" + ansible.builtin.set_fact: + ip_addresses: >- + {{ ip_addresses + + [{ + 'public_ip': public_ip_address.results | selectattr('idx', 'equalto', item.idx) | map(attribute='state.ip_address') | first, + 'private_ip': item.ansible_facts.azure_vm.network_profile.network_interfaces[0].properties.ip_configurations[0].private_ip_address + }] + }} + loop: "{{ server_result.results | selectattr('ansible_facts.azure_vm', 'defined') }}" + loop_control: + label: >- + public_ip: {{ public_ip_address.results | selectattr('idx', 'equalto', item.idx) | map(attribute='state.ip_address') | first }}, + private_ip: {{ item.ansible_facts.azure_vm.network_profile.network_interfaces[0].properties.ip_configurations[0].private_ip_address }} + + - name: "Inventory | Generate in-memory inventory" + ansible.builtin.import_tasks: inventory.yml + when: + - server_result.results is defined + - server_result.results | selectattr('ansible_facts.azure_vm', 'defined') + +# Delete (if state is absent) +- block: + - name: "Azure: Delete virtual machine" + azure.azcollection.azure_rm_virtualmachine: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + state: absent + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + + - name: "Azure: Delete network interface" + azure.azcollection.azure_rm_networkinterface: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-network-interface" + state: absent + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-network-interface" + + - name: "Azure: Delete public IP address" + azure.azcollection.azure_rm_publicipaddress: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-public-ip" + state: absent + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-public-ip" + + - name: "Azure: Delete Load Balancer" + azure.azcollection.azure_rm_loadbalancer: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + location: "{{ server_location }}" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Azure: Delete Load Balancer public IP address" + azure.azcollection.azure_rm_publicipaddress: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-public-ip" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-public-ip" + when: database_public_access | bool and cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Azure: Delete Security Group" + azure.azcollection.azure_rm_securitygroup: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ patroni_cluster_name }}-security-group" + state: absent + + - block: + - name: "Azure: Delete Blob Storage '{{ azure_blob_storage_name }}'" + azure.azcollection.azure_rm_storageblob: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + storage_account_name: "{{ azure_blob_storage_account_name }}" + container: "{{ azure_blob_storage_name }}" + state: absent + + - name: "Azure: Delete Storage Account '{{ azure_blob_storage_account_name }}'" + azure.azcollection.azure_rm_storageaccount: + resource_group: "{{ azure_resource_group | default('postgres-cluster-resource-group' ~ '-' ~ server_location) }}" + name: "{{ azure_blob_storage_account_name }}" + force_delete_nonempty: true + state: absent + ignore_errors: true + when: (pgbackrest_install | bool or wal_g_install | bool) and azure_blob_storage_absent | bool + when: state == 'absent' + +... diff --git a/roles/cloud-resources/tasks/digitalocean.yml b/roles/cloud-resources/tasks/digitalocean.yml new file mode 100644 index 000000000..cb8a3e74e --- /dev/null +++ b/roles/cloud-resources/tasks/digitalocean.yml @@ -0,0 +1,740 @@ +--- +# Dependencies +- name: Install Python dependencies + block: + - name: Ensure that 'python3-pip' package is present on controlling host + ansible.builtin.package: + name: python3-pip + state: present + register: package_status + until: package_status is success + delay: 10 + retries: 3 + delegate_to: 127.0.0.1 + run_once: true + when: ansible_distribution != "MacOSX" + + - name: Ensure that 'dopy' dependency is present on controlling host + ansible.builtin.pip: + name: dopy + extra_args: --user + delegate_to: 127.0.0.1 + become: false + vars: + ansible_become: false + run_once: true + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + PIP_BREAK_SYSTEM_PACKAGES: "1" + + - name: Ensure that 'boto3' dependency is present on controlling host + ansible.builtin.pip: + name: boto3 + extra_args: --user + delegate_to: 127.0.0.1 + become: false + vars: + ansible_become: false + run_once: true + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - digital_ocean_spaces_create | bool + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + PIP_BREAK_SYSTEM_PACKAGES: "1" + +# SSH key +- block: + # Delete the temporary ssh key from the cloud (if exists) + - name: "DigitalOcean: Remove temporary SSH key '{{ ssh_key_name }}' from cloud (if any)" + community.digitalocean.digital_ocean_sshkey: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ ssh_key_name }}" + ssh_pub_key: "{{ ssh_key_content }}" + state: absent + when: + - ssh_key_name is defined + - tmp_ssh_key_name is defined + - ssh_key_name == tmp_ssh_key_name + + # if ssh_key_name and ssh_key_content is specified, add this ssh key to the cloud + - name: "DigitalOcean: Add SSH key '{{ ssh_key_name }}' to cloud" + community.digitalocean.digital_ocean_sshkey: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ ssh_key_name }}" + ssh_pub_key: "{{ ssh_key_content }}" + state: present + when: + - ssh_key_name | length > 0 + - ssh_key_content | length > 0 + + - name: "DigitalOcean: Gather information about SSH keys" + community.digitalocean.digital_ocean_sshkey_info: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + register: ssh_keys + + # if ssh_key_name is specified, get the fingerprint of one ssh key + # or if tmp_ssh_key_name is used and ssh_public_keys is difined + - name: "DigitalOcean: Get fingerprint for SSH key '{{ ssh_key_name }}'" + ansible.builtin.set_fact: + ssh_key_fingerprint: "{{ [item.fingerprint] }}" + loop: "{{ ssh_keys.data | lower }}" + loop_control: # do not display the public key + label: "{{ item.name }}" + when: + - ((ssh_key_name | length > 0 and ssh_key_name != (tmp_ssh_key_name | default(''))) or + (ssh_key_name == (tmp_ssh_key_name | default('')) and ssh_public_keys | default('') | length > 0)) + - item.name == ssh_key_name | lower + + # Stop, if the ssh key is not found + - name: "DigitalOcean: Fail if SSH key '{{ ssh_key_name }}' is not found" + ansible.builtin.fail: + msg: "SSH key {{ ssh_key_name }} not found. Ensure that key has been added to DigitalOcean." + when: + - (ssh_key_name | length > 0 and ssh_key_name != (tmp_ssh_key_name | default(''))) + - ssh_key_fingerprint is not defined + + # if ssh_key_name is not specified, and ssh_public_keys is not defined + # get the fingerprint of all ssh keys + - name: "DigitalOcean: Get fingerprint for all SSH keys" + ansible.builtin.set_fact: + ssh_key_fingerprint: "{{ ssh_key_fingerprint | default([]) + [item.fingerprint] }}" + loop: "{{ ssh_keys.data | lower }}" + loop_control: # do not display the public key + label: "{{ item.name }}" + when: + - (ssh_key_name | length < 1 or ssh_key_name == (tmp_ssh_key_name | default(''))) + - (ssh_public_keys is not defined or ssh_public_keys | length < 1) + when: state == 'present' + +# Create (if state is present) +- block: + - name: "DigitalOcean: Gather information about VPC" + community.digitalocean.digital_ocean_vpc_info: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + register: vpc_info + + # if server_network is not specified and the default VPC is present + - name: Extract ip_range from default VPC + ansible.builtin.set_fact: + default_ip_range: >- + {{ + vpc_info.data + | selectattr('region', 'equalto', server_location) + | selectattr('default', 'equalto', true) + | map(attribute='ip_range') + | first + }} + when: + - server_network | length < 1 + - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length > 0 + + # if server_network is not specified and there is no default VPC, create a network + - name: "DigitalOcean: Create a VPC '{{ digital_ocean_vpc_name | default('network-' + server_location) }}'" + community.digitalocean.digital_ocean_vpc: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ digital_ocean_vpc_name | default('network-' + server_location) }}" + region: "{{ server_location }}" + state: present + register: digital_ocean_vpc + when: + - server_network | length < 1 + - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('default', 'equalto', true) | list | length == 0 + + - name: "Set variable: server_network" + ansible.builtin.set_fact: + server_network: "{{ digital_ocean_vpc_name | default('network-' + server_location) }}" + when: digital_ocean_vpc is changed + + - name: "DigitalOcean: Gather information about VPC" + community.digitalocean.digital_ocean_vpc_info: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + register: vpc_info + when: digital_ocean_vpc is changed + + # if server_network is specified + - name: "Fail if no VPC found in the specified region" + ansible.builtin.fail: + msg: "No VPC found with name '{{ server_network }}' in region '{{ server_location }}'" + when: + - server_network | length > 0 + - vpc_info.data | selectattr('region', 'equalto', server_location) | selectattr('name', 'equalto', server_network) | list | length == 0 + + - name: Extract ip_range from VPC "{{ server_network }}" + ansible.builtin.set_fact: + vpc_ip_range: >- + {{ + vpc_info.data + | selectattr('region', 'equalto', server_location) + | selectattr('name', 'equalto', server_network) + | map(attribute='ip_range') + | first + }} + when: server_network | length > 0 + + - name: Extract id from VPC "{{ server_network }}" + ansible.builtin.set_fact: + vpc_id: >- + {{ + vpc_info.data + | selectattr('region', 'equalto', server_location) + | selectattr('name', 'equalto', server_network) + | map(attribute='id') + | first + }} + when: server_network | length > 0 + + - name: "DigitalOcean: Create a tag '{{ patroni_cluster_name }}'" + community.digitalocean.digital_ocean_tag: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}" + state: present + + # Firewall + - name: "DigitalOcean: Create or modify public firewall" + community.digitalocean.digital_ocean_firewall: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}-public-firewall" + state: "present" + inbound_rules: "{{ inbound_rules }}" + outbound_rules: + - protocol: "tcp" + ports: "1-65535" + destinations: + addresses: ["0.0.0.0/0", "::/0"] + - protocol: "udp" + ports: "1-65535" + destinations: + addresses: ["0.0.0.0/0", "::/0"] + - protocol: "icmp" + ports: "1-65535" + destinations: + addresses: ["0.0.0.0/0", "::/0"] + tags: + - "{{ patroni_cluster_name }}" # Only VMs with this tag will be affected + vars: + inbound_rules: >- + {{ + ([ + { + 'protocol': 'tcp', + 'ports': ansible_ssh_port | default('22'), + 'sources': { + 'addresses': ssh_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + } + ] if ssh_public_access | bool else []) + + ([ + { + 'protocol': 'tcp', + 'ports': netdata_port | default('19999'), + 'sources': { + 'addresses': netdata_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + } + ] if netdata_install | bool and netdata_public_access | bool else []) + + ([ + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.master | default('5000'), + 'sources': { + 'addresses': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + }, + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.replicas | default('5001'), + 'sources': { + 'addresses': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + }, + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.replicas_sync | default('5002'), + 'sources': { + 'addresses': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + }, + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.replicas_async | default('5003'), + 'sources': { + 'addresses': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + } + ] if database_public_access | bool and with_haproxy_load_balancing | bool else []) + + ([ + { + 'protocol': 'tcp', + 'ports': pgbouncer_listen_port | default('6432'), + 'sources': { + 'addresses': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + } + ] if database_public_access | bool and (not with_haproxy_load_balancing | bool and pgbouncer_install | bool) else []) + + ([ + { + 'protocol': 'tcp', + 'ports': postgresql_port | default('5432'), + 'sources': { + 'addresses': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + } + ] if database_public_access | bool and (not with_haproxy_load_balancing | bool and not pgbouncer_install | bool) else []) + }} + when: + - cloud_firewall | bool + - (ssh_public_access | bool or netdata_public_access | bool or database_public_access | bool) + + - name: "DigitalOcean: Create or modify Postgres cluster firewall" + community.digitalocean.digital_ocean_firewall: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}-private-firewall" + state: "present" + inbound_rules: "{{ inbound_rules }}" + outbound_rules: + - protocol: "tcp" + ports: "1-65535" + destinations: + addresses: ["0.0.0.0/0", "::/0"] + - protocol: "udp" + ports: "1-65535" + destinations: + addresses: ["0.0.0.0/0", "::/0"] + - protocol: "icmp" + ports: "1-65535" + destinations: + addresses: ["0.0.0.0/0", "::/0"] + tags: + - "{{ patroni_cluster_name }}" # Only VMs with this tag will be affected + vars: + sources_addresses: "{{ (server_network | length > 0) | ternary(vpc_ip_range, default_ip_range) }}" + inbound_rules: >- + {{ + ([ + { + 'protocol': 'tcp', + 'ports': ansible_ssh_port | default('22'), + 'sources': { + 'addresses': [sources_addresses] + } + } + ]) + + ([ + { + 'protocol': 'tcp', + 'ports': netdata_port | default('19999'), + 'sources': { + 'addresses': [sources_addresses] + } + } + ] if netdata_install | bool else []) + + ([ + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.master | default('5000'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.replicas | default('5001'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.replicas_sync | default('5002'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.replicas_async | default('5003'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': haproxy_listen_port.stats | default('7000'), + 'sources': { + 'addresses': [sources_addresses] + } + } + ] if with_haproxy_load_balancing | bool else []) + + ([ + { + 'protocol': 'tcp', + 'ports': pgbouncer_listen_port | default('6432'), + 'sources': { + 'addresses': [sources_addresses] + } + } + ] if pgbouncer_install | bool else []) + + ([ + { + 'protocol': 'tcp', + 'ports': postgresql_port | default('5432'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': patroni_restapi_port | default('8008'), + 'sources': { + 'addresses': [sources_addresses] + } + } + ]) + + ([ + { + 'protocol': 'tcp', + 'ports': etcd_client_port | default('2379'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': etcd_peer_port | default('2380'), + 'sources': { + 'addresses': [sources_addresses] + } + } + ] if dcs_type == 'etcd' else []) + + ([ + { + 'protocol': 'tcp', + 'ports': consul_ports.dns | default('8600'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': consul_ports.http | default('8500'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': consul_ports.rpc | default('8400'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': consul_ports.serf_lan | default('8301'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': consul_ports.serf_wan | default('8302'), + 'sources': { + 'addresses': [sources_addresses] + } + }, + { + 'protocol': 'tcp', + 'ports': consul_ports.server | default('8300'), + 'sources': { + 'addresses': [sources_addresses] + } + } + ] if dcs_type == 'consul' else []) + }} + when: cloud_firewall | bool + + # Server and volume + - name: "DigitalOcean: Create or modify Droplet" + community.digitalocean.digital_ocean_droplet: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: present + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + unique_name: true + size: "{{ server_type }}" + region: "{{ server_location }}" + image: "{{ server_image }}" + ssh_keys: "{{ ssh_key_fingerprint }}" + vpc_uuid: "{{ vpc_id | default(omit) }}" + wait_timeout: 500 + tags: + - "{{ patroni_cluster_name }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: droplet_result + + - name: "DigitalOcean: Create or modify Block Storage" + community.digitalocean.digital_ocean_block_storage: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: present + command: create + volume_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + region: "{{ server_location }}" + block_size: "{{ volume_size | int }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + register: block_storage_result + + - name: "DigitalOcean: Attach Block Storage to Droplet" + community.digitalocean.digital_ocean_block_storage: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: present + command: attach + volume_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + region: "{{ server_location }}" + droplet_id: "{{ item.data.droplet.id }}" + loop: "{{ droplet_result.results }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + when: droplet_result.results is defined + + # Load Balancer + - name: "Set variable: digital_ocean_load_balancer_port" + ansible.builtin.set_fact: + digital_ocean_load_balancer_port: "{{ pgbouncer_listen_port }}" + when: + - cloud_load_balancer | bool + - pgbouncer_install | bool + - digital_ocean_load_balancer_port | default('') | length < 1 + + - name: "Set variable: digital_ocean_load_balancer_target_port" + ansible.builtin.set_fact: + digital_ocean_load_balancer_target_port: "{{ pgbouncer_listen_port }}" + when: + - cloud_load_balancer | bool + - pgbouncer_install | bool + - digital_ocean_load_balancer_target_port | default('') | length < 1 + + # if 'pgbouncer_install' is 'false' + - name: "Set variable: digital_ocean_load_balancer_port" + ansible.builtin.set_fact: + digital_ocean_load_balancer_port: "{{ postgresql_port }}" + when: + - cloud_load_balancer | bool + - not pgbouncer_install | bool + - digital_ocean_load_balancer_port | default('') | length < 1 + + - name: "Set variable: digital_ocean_load_balancer_target_port" + ansible.builtin.set_fact: + digital_ocean_load_balancer_target_port: "{{ postgresql_port }}" + when: + - cloud_load_balancer | bool + - not pgbouncer_install | bool + - digital_ocean_load_balancer_target_port | default('') | length < 1 + + - name: "DigitalOcean: Create or modify Load Balancer" + community.digitalocean.digital_ocean_load_balancer: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: present + name: "{{ patroni_cluster_name }}-{{ item }}" + region: "{{ server_location }}" + forwarding_rules: + - entry_protocol: tcp + entry_port: "{{ digital_ocean_load_balancer_port }}" + target_protocol: tcp + target_port: "{{ digital_ocean_load_balancer_target_port }}" + health_check: + protocol: http + port: "{{ patroni_restapi_port }}" + path: "/{{ item }}" + check_interval_seconds: 5 + response_timeout_seconds: 3 + unhealthy_threshold: 2 + healthy_threshold: 3 + size: "{{ (digital_ocean_load_balancer_size | default('lb-medium')) if server_location in ['ams2', 'nyc2', 'sfo1'] else omit }}" + size_unit: "{{ (digital_ocean_load_balancer_size_unit | default(3)) if server_location not in ['ams2', 'nyc2', 'sfo1'] else omit }}" + vpc_uuid: "{{ vpc_id | default(omit) }}" + tag: "{{ patroni_cluster_name }}" # a tag associated with droplets for load balancing. + loop: + - primary + - replica + - sync + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "DigitalOcean: Gather information about load balancers" + community.digitalocean.digital_ocean_load_balancer_info: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + register: digitalocean_load_balancer + when: cloud_load_balancer | bool + + # Spaces Object Storage (Backups) + - name: "DigitalOcean: Create Spaces Bucket '{{ digital_ocean_spaces_name }}'" + community.digitalocean.digital_ocean_spaces: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ digital_ocean_spaces_name }}" + region: "{{ digital_ocean_spaces_region }}" + aws_access_key_id: "{{ AWS_ACCESS_KEY_ID }}" + aws_secret_access_key: "{{ AWS_SECRET_ACCESS_KEY }}" + state: present + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - digital_ocean_spaces_create | bool + when: state == 'present' + +- name: Wait for host to be available via SSH + ansible.builtin.wait_for: + host: "{{ (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public')).0.ip_address }}" + port: 22 + delay: 5 + timeout: 300 + loop: "{{ droplet_result.results }}" + loop_control: + label: "{{ (item.data.droplet.networks.v4 | default('') | selectattr('type', 'equalto', 'public')).0.ip_address | default('N/A') }}" + when: + - droplet_result.results is defined + - item.data is defined + +# Info +- name: Server info + ansible.builtin.debug: + msg: + id: "{{ item.data.droplet.id }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + image: "{{ item.data.droplet.image.description }}" + type: "{{ server_type }}" + volume_size: "{{ volume_size }} GB" + public_ip: "{{ (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public')).0.ip_address }}" + private_ip: "{{ (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'private')).0.ip_address }}" + loop: "{{ droplet_result.results }}" + loop_control: + index_var: idx + label: "{{ (item.data.droplet.networks.v4 | default('') | selectattr('type', 'equalto', 'public')).0.ip_address | default('N/A') }}" + when: + - droplet_result.results is defined + - item.data is defined + +# Inventory +- block: + - name: "Inventory | Initialize ip_addresses variable" + ansible.builtin.set_fact: + ip_addresses: [] + + - name: "Inventory | Extract IP addresses" + ansible.builtin.set_fact: + ip_addresses: >- + {{ ip_addresses + + [{'public_ip': (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public')).0.ip_address, + 'private_ip': (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'private')).0.ip_address}] + }} + loop: "{{ droplet_result.results | selectattr('data', 'defined') }}" + loop_control: + label: >- + public_ip: {{ (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'public')).0.ip_address }}, + private_ip: {{ (item.data.droplet.networks.v4 | selectattr('type', 'equalto', 'private')).0.ip_address }} + + - name: "Inventory | Generate in-memory inventory" + ansible.builtin.import_tasks: inventory.yml + when: + - droplet_result.results is defined + - droplet_result.results | selectattr('data', 'defined') + +# Delete the temporary SSH key from the cloud after creating the droplet +- name: "DigitalOcean: Remove temporary SSH key '{{ ssh_key_name }}' from cloud" + community.digitalocean.digital_ocean_sshkey: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ ssh_key_name }}" + ssh_pub_key: "{{ ssh_key_content }}" + state: absent + when: + - ssh_key_name is defined + - tmp_ssh_key_name is defined + - ssh_key_name == tmp_ssh_key_name + +# Delete (if state is absent) +- block: + - name: "DigitalOcean: Delete Droplet" + community.digitalocean.digital_ocean_droplet: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: absent + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + unique_name: true + region: "{{ server_location }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: droplet_absent + until: not droplet_absent.failed + retries: 3 + delay: 5 + + - name: "DigitalOcean: Delete Block Storage" + community.digitalocean.digital_ocean_block_storage: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: absent + command: create + volume_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + region: "{{ server_location }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + register: block_storage_absent + until: not block_storage_absent.failed + retries: 3 + delay: 5 + + - name: "DigitalOcean: Delete Load Balancer" + community.digitalocean.digital_ocean_load_balancer: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: absent + name: "{{ patroni_cluster_name }}-{{ item }}" + region: "{{ server_location }}" + loop: + - primary + - replica + - sync + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "DigitalOcean: Delete public firewall" + community.digitalocean.digital_ocean_firewall: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: "absent" + name: "{{ patroni_cluster_name }}-public-firewall" + + - name: "DigitalOcean: Delete Postgres cluster firewall" + community.digitalocean.digital_ocean_firewall: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + state: "absent" + name: "{{ patroni_cluster_name }}-private-firewall" + + - name: "DigitalOcean: Delete Spaces Bucket '{{ digital_ocean_spaces_name }}'" + community.digitalocean.digital_ocean_spaces: + oauth_token: "{{ lookup('ansible.builtin.env', 'DO_API_TOKEN') }}" + name: "{{ digital_ocean_spaces_name }}" + region: "{{ digital_ocean_spaces_region }}" + aws_access_key_id: "{{ AWS_ACCESS_KEY_ID }}" + aws_secret_access_key: "{{ AWS_SECRET_ACCESS_KEY }}" + state: absent + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - digital_ocean_spaces_absent | bool + ignore_errors: true + when: state == 'absent' + +... diff --git a/roles/cloud-resources/tasks/gcp.yml b/roles/cloud-resources/tasks/gcp.yml new file mode 100644 index 000000000..29376bb03 --- /dev/null +++ b/roles/cloud-resources/tasks/gcp.yml @@ -0,0 +1,688 @@ +--- +# Dependencies +- name: Install Python dependencies + block: + - name: Ensure that 'python3-pip' package is present on controlling host + ansible.builtin.package: + name: python3-pip + state: present + register: package_status + until: package_status is success + delay: 10 + retries: 3 + delegate_to: 127.0.0.1 + run_once: true + when: ansible_distribution != "MacOSX" + + - name: Ensure that 'google-auth' dependency is present on controlling host + ansible.builtin.pip: + name: google-auth + extra_args: --user + delegate_to: 127.0.0.1 + become: false + vars: + ansible_become: false + run_once: true + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + PIP_BREAK_SYSTEM_PACKAGES: "1" + +# Check if GCP_SERVICE_ACCOUNT_CONTENTS is defined +- name: Lookup the GCP_SERVICE_ACCOUNT_CONTENTS environmental variable + ansible.builtin.set_fact: + gcp_service_account_contents_raw: "{{ lookup('ansible.builtin.env', 'GCP_SERVICE_ACCOUNT_CONTENTS') | default('') }}" + no_log: true + +- name: "Fail if no GCP service account information is provided" + ansible.builtin.fail: + msg: "GCP_SERVICE_ACCOUNT_CONTENTS is not defined or empty. Please provide GCP service account credentials." + when: gcp_service_account_contents_raw | length < 1 + +# Decode GCP Service Account if base64 encoded +- name: "Set variable: gcp_service_account_contents (b64decode)" + ansible.builtin.set_fact: + gcp_service_account_contents: "{{ gcp_service_account_contents_raw | b64decode }}" + no_log: true + when: gcp_service_account_contents_raw is match('^[a-zA-Z0-9+/]+={0,2}$') + +# Set GCP Service Account Contents to raw value if not base64 encoded +- name: "Set variable: gcp_service_account_contents" + ansible.builtin.set_fact: + gcp_service_account_contents: "{{ gcp_service_account_contents_raw }}" + no_log: true + when: gcp_service_account_contents is not defined + +# Project info +- name: "GCP: Gather information about project" + google.cloud.gcp_resourcemanager_project_info: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + register: project_info + when: gcp_project is not defined or gcp_project | length < 1 + +# Create (if state is present) +- block: + # if ssh_key_content is not defined, get the user public key from the system (if exists) + - name: "Set variable: ssh_key_content" + ansible.builtin.set_fact: + ssh_key_content: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" + no_log: true # do not display the public key + when: ssh_key_content is not defined or + ssh_key_content | length < 1 + + # if server_network is not specified, use default network + - name: "Set variable: gcp_network_name" + ansible.builtin.set_fact: + gcp_network_name: "{{ server_network if server_network is defined and server_network | length > 0 else 'default' }}" + + - name: "GCP: Gather information about network" + google.cloud.gcp_compute_subnetwork_info: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + region: "{{ server_location[:-2] if server_location[-2:] | regex_search('-[a-z]$') else server_location }}" + filters: + - name = "{{ gcp_network_name }}" + register: subnetwork_info + + - name: "GCP: Extract ip_range for network '{{ gcp_network_name }}'" + ansible.builtin.set_fact: + gcp_network_ip_range: "{{ subnetwork_info.resources[0].ipCidrRange }}" + + # Firewall + - name: "GCP: Create or modify SSH public firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-ssh-public" + description: "Firewall rule for public SSH access to Postgres cluster servers" + allowed: + - ip_protocol: tcp + ports: + - "{{ ansible_ssh_port | default(22) }}" + source_ranges: "{{ ssh_public_allowed_ips | default('0.0.0.0/0', true) | split(',') }}" + target_tags: + - "{{ patroni_cluster_name }}" # Only VMs with this tag will be affected + network: + selfLink: "global/networks/{{ gcp_network_name }}" + state: present + when: + - ssh_public_access | bool + - cloud_firewall | bool + + - name: "GCP: Create or modify Netdata public firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-netdata-public" + description: "Firewall rule for public Netdata monitoring access" + allowed: + - ip_protocol: tcp + ports: + - "{{ netdata_port | default('19999') }}" + source_ranges: "{{ netdata_public_allowed_ips | default('0.0.0.0/0', true) | split(',') }}" + target_tags: + - "{{ patroni_cluster_name }}" # Only VMs with this tag will be affected + network: + selfLink: "global/networks/{{ gcp_network_name }}" + state: present + when: + - netdata_install | bool + - netdata_public_access | bool + - cloud_firewall | bool + + - name: "GCP: Create or modify Database public firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-database-public" + description: "Firewall rule for public database access" + allowed: + - ip_protocol: tcp + ports: "{{ allowed_ports }}" + source_ranges: "{{ database_public_allowed_ips | default('0.0.0.0/0', true) | split(',') }}" + target_tags: + - "{{ patroni_cluster_name }}" # Only VMs with this tag will be affected + network: + selfLink: "global/networks/{{ gcp_network_name }}" + state: present + vars: + allowed_ports: >- + {{ + ([postgresql_port | default('5432')] if not with_haproxy_load_balancing | bool and not pgbouncer_install | bool else []) + + ([pgbouncer_listen_port | default('6432')] if not with_haproxy_load_balancing | bool and pgbouncer_install | bool else []) + + ([haproxy_listen_port.master | default('5000'), + haproxy_listen_port.replicas | default('5001'), + haproxy_listen_port.replicas_sync | default('5002'), + haproxy_listen_port.replicas_async | default('5003')] if with_haproxy_load_balancing | bool else []) + }} + when: + - database_public_access | bool + - cloud_firewall | bool + + - name: "GCP: Create or modify Postgres cluster firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-firewall-rule" + description: "Firewall rule for Postgres cluster" + allowed: + - ip_protocol: tcp + ports: "{{ allowed_ports }}" + source_ranges: + - "{{ gcp_network_ip_range }}" + target_tags: + - "{{ patroni_cluster_name }}" # Only VMs with this tag will be affected + network: + selfLink: "global/networks/{{ gcp_network_name }}" + state: present + vars: + allowed_ports: >- + {{ + [ansible_ssh_port | default('22')] + + ([netdata_port | default('19999')] if netdata_install | bool else []) + + ([pgbouncer_listen_port | default('6432')] if pgbouncer_install | bool else []) + + [postgresql_port | default('5432')] + + [patroni_restapi_port | default('8008')] + + ([haproxy_listen_port.master | default('5000'), + haproxy_listen_port.replicas | default('5001'), + haproxy_listen_port.replicas_sync | default('5002'), + haproxy_listen_port.replicas_async | default('5003'), + haproxy_listen_port.stats | default('7000')] if with_haproxy_load_balancing | bool else []) + + ([etcd_client_port | default('2379'), etcd_peer_port | default('2380')] if dcs_type == 'etcd' else []) + + ([consul_ports_dns | default('8600'), + consul_ports_http | default('8500'), + consul_ports_rpc | default('8400'), + consul_ports_serf_lan | default('8301'), + consul_ports_serf_wan | default('8302'), + consul_ports_server | default('8300')] if dcs_type == 'consul' else []) + }} + when: cloud_firewall | bool + + # if 'cloud_load_balancer' is 'true' + # https://cloud.google.com/load-balancing/docs/tcp#firewall-rules + - name: "GCP: Create health checks and LB firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-lb-firewall-rule" + description: "Firewall rule for Health Checks and Load Balancer access to the database" + priority: 900 + allowed: + - ip_protocol: tcp + ports: + - "{{ patroni_restapi_port | default('8008') }}" + - "{{ pgbouncer_listen_port | default('6432') if pgbouncer_install | bool else postgresql_port | default('5432') }}" + source_ranges: + - "35.191.0.0/16" + - "130.211.0.0/22" + target_tags: + - "{{ patroni_cluster_name }}" # Only VMs with this tag will be affected + network: + selfLink: "global/networks/{{ gcp_network_name }}" + state: present + when: cloud_load_balancer | bool + + # Server and volume + - name: "GCP: Create or modify VM instance" + google.cloud.gcp_compute_instance: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + zone: "{{ server_location + '-b' if not server_location is match('.*-[a-z]$') else server_location }}" # add "-b" if the zone is not defined + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + machine_type: "{{ server_type }}" + disks: + - device_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-system" + auto_delete: true + boot: true + initialize_params: + disk_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-system" + source_image: "{{ server_image }}" + disk_size_gb: "{{ system_volume_size | default('80') }}" # system disk size + disk_type: "{{ system_volume_type | default('pd-ssd', true) }}" + - device_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + auto_delete: true + initialize_params: + disk_name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + disk_size_gb: "{{ volume_size | int }}" + disk_type: "{{ volume_type | default('pd-ssd', true) }}" + network_interfaces: + - network: + selfLink: "global/networks/{{ gcp_network_name }}" + access_configs: + - name: External NAT + type: ONE_TO_ONE_NAT + metadata: + ssh-keys: "root:{{ ssh_key_content }}" + scheduling: + preemptible: "{{ server_spot | default(gcp_compute_instance_preemptible | default(false)) | bool }}" + tags: + items: + - "{{ patroni_cluster_name }}" + labels: + cluster: "{{ patroni_cluster_name }}" + status: "{{ gcp_instance_status | default('RUNNING') }}" + state: present + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: server_result + until: server_result is success + delay: 10 + retries: 3 + + # Load Balancer + # This block creates Global External classic proxy Network Load Balancer. + # Global objects are required because the gcp_compute_target_tcp_proxy module can only be global, as it requires the use of a global forwarding rule. + # Using global objects instead of regional ones allows us to utilize a TCP proxy for correct traffic load balancing. + # Note: Regional internal load balancers are passthrough and do not work correctly with the health checks we use through the Patroni REST API. + - block: + - name: "GCP: [Load Balancer] Create instance group" + google.cloud.gcp_compute_instance_group: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}" + description: "{{ patroni_cluster_name }} instance group" + region: "{{ region }}" + zone: "{{ zone }}" + named_ports: + - name: postgres + port: "{{ postgresql_port | default('5432') }}" + - name: pgbouncer + port: "{{ pgbouncer_listen_port | default('6432') }}" + network: + selfLink: "global/networks/{{ gcp_network_name }}" + instances: "{{ instances_selflink }}" + state: present + vars: + region: "{{ server_location[:-2] if server_location[-2:] | regex_search('-[a-z]$') else server_location }}" + zone: "{{ server_location + '-b' if not server_location is match('.*-[a-z]$') else server_location }}" # add "-b" if the zone is not defined + # The module only works if selfLink is set manually, issue: https://github.com/ansible-collections/google.cloud/issues/614 + instances_selflink: >- # TODO: use "{{ server_result.results | map(attribute='selfLink') | map('community.general.dict_kv', 'selfLink') | list }}" + [ + {% for i in range(1, (server_count | int) + 1) %} + { + "selfLink": "zones/{{ zone }}/instances/{{ server_name }}{{ '%02d' % i }}" + }{% if not loop.last %},{% endif %} + {% endfor %} + ] + register: instance_group + # Ignore error if resource already exists on re-run + failed_when: instance_group is failed and 'memberAlreadyExists' not in (instance_group.msg | default('')) + + - name: "GCP: [Load Balancer] Create health check" + google.cloud.gcp_compute_health_check: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-hc" + description: "{{ patroni_cluster_name }} {{ item }} health check" + type: "HTTP" + http_health_check: + port: "{{ patroni_restapi_port }}" + request_path: "/{{ item }}" + check_interval_sec: "{{ gcp_compute_health_check_interval_sec | default(3) }}" + timeout_sec: "{{ gcp_compute_health_check_check_timeout_sec | default(2) }}" + unhealthy_threshold: "{{ gcp_compute_health_check_unhealthy_threshold | default(2) }}" + healthy_threshold: "{{ gcp_compute_health_check_healthy_threshold | default(3) }}" + state: present + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-hc" + register: health_check + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Create backend service" + google.cloud.gcp_compute_backend_service: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + description: "{{ patroni_cluster_name }} {{ item }} backend" + protocol: "TCP" + port_name: "{{ 'pgbouncer' if pgbouncer_install | bool else 'postgres' }}" + load_balancing_scheme: "EXTERNAL" + backends: + - group: "zones/{{ zone }}/instanceGroups/{{ patroni_cluster_name }}" + balancing_mode: "CONNECTION" + max_connections_per_instance: "{{ gcp_lb_max_connections | default(10000) }}" + health_checks: + - "/global/healthChecks/{{ patroni_cluster_name }}-{{ item }}-hc" + timeout_sec: "{{ gcp_compute_backend_service_timeout_sec | default(5) }}" + log_config: + enable: "{{ gcp_compute_backend_service_log_enable | default(false) }}" + state: present + vars: + zone: "{{ server_location + '-b' if not server_location is match('.*-[a-z]$') else server_location }}" # add "-b" if the zone is not defined + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + register: backend_service + # Ignore error if resource already exists on re-run + failed_when: backend_service is failed and 'resource.fingerprint' not in (backend_service.msg | default('')) + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Create target TCP proxy" + google.cloud.gcp_compute_target_tcp_proxy: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-proxy" + description: "{{ patroni_cluster_name }} {{ item }} TCP Proxy" + service: + selfLink: "/global/backendServices/{{ patroni_cluster_name }}-{{ item }}" + proxy_header: "NONE" + state: present + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-proxy" + register: target_tcp_proxy + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Reserve static IP address" + google.cloud.gcp_compute_global_address: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-ip" + description: "{{ patroni_cluster_name }} {{ item }} load balancer IP address" + address_type: "EXTERNAL" + ip_version: "IPV4" + state: present + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-ip" + register: load_balancer_ip + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Create forwarding rule" + google.cloud.gcp_compute_global_forwarding_rule: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-fr" + description: "{{ patroni_cluster_name }} {{ item }} forwarding rule" + load_balancing_scheme: "EXTERNAL" + ip_address: "{{ (load_balancer_ip.results | selectattr('item', 'equalto', item) | map(attribute='address') | first) }}" + ip_protocol: "TCP" + port_range: "{{ pgbouncer_listen_port | default('6432') if pgbouncer_install | bool else postgresql_port | default('5432') }}" + target: "/global/targetTcpProxies/{{ patroni_cluster_name }}-{{ item }}-proxy" + state: present + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-fr" + register: gcp_load_balancer + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + when: cloud_load_balancer | bool + + # GCS Bucket (Backups) + - name: "GCP: Create bucket '{{ gcp_bucket_name }}'" + google.cloud.gcp_storage_bucket: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ gcp_bucket_name }}" + storage_class: "{{ gcp_bucket_storage_class }}" + predefined_default_object_acl: "{{ gcp_bucket_default_object_acl }}" + state: present + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - gcp_bucket_create | bool + when: state == 'present' + +- name: Wait for host to be available via SSH + ansible.builtin.wait_for: + host: "{{ item.networkInterfaces[0].accessConfigs[0].natIP }}" + port: 22 + delay: 5 + timeout: 300 + loop: "{{ server_result.results }}" + loop_control: + label: "{{ item.networkInterfaces[0].accessConfigs[0].natIP | default('N/A') }}" + when: + - server_result.results is defined + - item.networkInterfaces is defined + +# Info +- name: Server info + ansible.builtin.debug: + msg: + id: "{{ item.id }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + image: "{{ item.disks[0].licenses[0] | basename }}" + type: "{{ item.machineType | basename }}" + volume_size: "{{ volume_size }} GB" + public_ip: "{{ item.networkInterfaces[0].accessConfigs[0].natIP }}" + private_ip: "{{ item.networkInterfaces[0].networkIP }}" + loop: "{{ server_result.results }}" + loop_control: + index_var: idx + label: "{{ item.networkInterfaces[0].accessConfigs[0].natIP | default('N/A') }}" + when: + - server_result.results is defined + - item.networkInterfaces is defined + +# Inventory +- block: + - name: "Inventory | Initialize ip_addresses variable" + ansible.builtin.set_fact: + ip_addresses: [] + + - name: "Inventory | Extract IP addresses" + ansible.builtin.set_fact: + ip_addresses: >- + {{ ip_addresses + + [{'public_ip': item.networkInterfaces[0].accessConfigs[0].natIP, + 'private_ip': item.networkInterfaces[0].networkIP}] + }} + loop: "{{ server_result.results | selectattr('networkInterfaces', 'defined') }}" + loop_control: + label: "public_ip: {{ item.networkInterfaces[0].accessConfigs[0].natIP }}, private_ip: {{ item.networkInterfaces[0].networkIP }}" + + - name: "Inventory | Generate in-memory inventory" + ansible.builtin.import_tasks: inventory.yml + when: + - server_result.results is defined + - server_result.results | selectattr('networkInterfaces', 'defined') + +# Delete (if state is absent) +- block: + - name: "GCP: Delete VM instance" + google.cloud.gcp_compute_instance: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + zone: "{{ server_location + '-b' if not server_location is match('.*-[a-z]$') else server_location }}" # add "-b" if the zone is not defined + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + state: absent + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + + - name: "GCP: [Load Balancer] Delete forwarding rule" + google.cloud.gcp_compute_global_forwarding_rule: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-fr" + target: "/global/targetTcpProxies/{{ patroni_cluster_name }}-{{ item }}-proxy" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-fr" + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Delete static IP address" + google.cloud.gcp_compute_global_address: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-ip" + address_type: "EXTERNAL" + ip_version: "IPV4" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-ip" + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Delete target TCP proxy" + google.cloud.gcp_compute_target_tcp_proxy: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-proxy" + service: + selfLink: "/global/backendServices/{{ patroni_cluster_name }}-{{ item }}" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-proxy" + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Delete backend service" + google.cloud.gcp_compute_backend_service: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Delete health check" + google.cloud.gcp_compute_health_check: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-{{ item }}-hc" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}-hc" + when: item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool) + + - name: "GCP: [Load Balancer] Delete instance group" + google.cloud.gcp_compute_instance_group: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}" + region: "{{ server_location[:-2] if server_location[-2:] | regex_search('-[a-z]$') else server_location }}" + zone: "{{ server_location + '-b' if not server_location is match('.*-[a-z]$') else server_location }}" + state: absent + + - name: "GCP: Delete SSH public firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-ssh-public" + state: absent + + - name: "GCP: Delete Netdata public firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-netdata-public" + state: absent + + - name: "GCP: Delete Database public firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-database-public" + state: absent + + - name: "GCP: Delete Postgres cluster firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-firewall-rule" + state: absent + + - name: "GCP: Delete health checks and LB firewall rule" + google.cloud.gcp_compute_firewall: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ patroni_cluster_name }}-lb-firewall-rule" + state: absent + + - name: "GCP: Delete bucket '{{ gcp_bucket_name }}'" + google.cloud.gcp_storage_bucket: + auth_kind: "serviceaccount" + service_account_contents: "{{ gcp_service_account_contents }}" + project: "{{ gcp_project | default(project_info.resources[0].projectNumber) }}" + name: "{{ gcp_bucket_name }}" + state: absent + when: + - (pgbackrest_install | bool or wal_g_install | bool) + - gcp_bucket_absent | bool + when: state == 'absent' + +... diff --git a/roles/cloud-resources/tasks/hetzner.yml b/roles/cloud-resources/tasks/hetzner.yml new file mode 100644 index 000000000..eb173f6f0 --- /dev/null +++ b/roles/cloud-resources/tasks/hetzner.yml @@ -0,0 +1,737 @@ +--- +# Dependencies +- name: Install python dependencies + block: + - name: Ensure that 'python3-pip' package is present on controlling host + ansible.builtin.package: + name: python3-pip + state: present + register: package_status + until: package_status is success + delay: 10 + retries: 3 + delegate_to: 127.0.0.1 + run_once: true + when: ansible_distribution != "MacOSX" + + - name: Ensure that 'hcloud' dependency is present on controlling host + ansible.builtin.pip: + name: hcloud + extra_args: --user + delegate_to: 127.0.0.1 + become: false + vars: + ansible_become: false + run_once: true + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin" + PIP_BREAK_SYSTEM_PACKAGES: "1" + +# SSH key +- block: + # Delete the temporary ssh key from the cloud (if exists) + - name: "Hetzner Cloud: Remove temporary SSH key '{{ ssh_key_name }}' from cloud (if any)" + hetzner.hcloud.ssh_key: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ ssh_key_name }}" + state: absent + when: + - ssh_key_name is defined + - tmp_ssh_key_name is defined + - ssh_key_name == tmp_ssh_key_name + + # if ssh_key_name and ssh_key_content is specified, add this ssh key to the cloud + - name: "Hetzner Cloud: Add SSH key '{{ ssh_key_name }}' to cloud" + hetzner.hcloud.ssh_key: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ ssh_key_name }}" + public_key: "{{ ssh_key_content }}" + state: present + when: + - ssh_key_name | length > 0 + - ssh_key_content | length > 0 + + # if ssh_key_name is specified + - name: "Hetzner Cloud: Gather information about SSH key '{{ ssh_key_name }}'" + hetzner.hcloud.ssh_key_info: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ ssh_key_name }}" + register: ssh_keys + when: ssh_key_name | length > 0 + + # Stop, if the ssh key is not found + - name: "Hetzner Cloud: Fail if SSH key is not found" + ansible.builtin.fail: + msg: "SSH key {{ ssh_key_name }} not found. Ensure that key has been added to Hetzner Cloud." + when: + - ssh_key_name | length > 0 + - ssh_keys.hcloud_ssh_key_info is defined + - ssh_keys.hcloud_ssh_key_info | length < 1 + + - name: "Set variable: ssh_key_names" + ansible.builtin.set_fact: + ssh_key_names: "{{ ssh_key_names | default([]) + [item.name] }}" + loop: "{{ ssh_keys.hcloud_ssh_key_info }}" + no_log: true # do not display the public key + when: + - ssh_key_name | length > 0 + - ssh_keys.hcloud_ssh_key_info is defined + - ssh_keys.hcloud_ssh_key_info | length > 0 + + # if ssh_key_name is not specified, and ssh_public_keys is not defined + # get the names of all ssh keys + - name: "Hetzner Cloud: Gather information about SSH keys" + hetzner.hcloud.ssh_key_info: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + register: ssh_keys + when: + - (ssh_key_name | length < 1 or ssh_key_name == (tmp_ssh_key_name | default(''))) + - (ssh_public_keys is not defined or ssh_public_keys | length < 1) + + - name: "Hetzner Cloud: Get names of all SSH keys" + ansible.builtin.set_fact: + ssh_key_names: "{{ ssh_key_names | default([]) + [item.name] }}" + loop: "{{ ssh_keys.hcloud_ssh_key_info }}" + loop_control: # do not display the public key + label: "{{ item.name }}" + when: + - (ssh_key_name | length < 1 or ssh_key_name == (tmp_ssh_key_name | default(''))) + - (ssh_public_keys is not defined or ssh_public_keys | length < 1) + when: state == 'present' + +# Create (if state is present) +- block: + - name: "Hetzner Cloud: Gather information about network zones" + ansible.builtin.uri: + url: "https://api.hetzner.cloud/v1/locations" + method: GET + headers: + Authorization: "Bearer {{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + return_content: true + register: hetzner_locations_response + failed_when: hetzner_locations_response.status != 200 + + - name: "Hetzner Cloud: Extract network zone for server_location" + ansible.builtin.set_fact: + target_network_zone: "{{ item.network_zone }}" + loop: "{{ hetzner_locations_response.json.locations }}" + loop_control: + label: "network_zone: {{ item.network_zone }}" + when: item.name == server_location + + - name: "Hetzner Cloud: Gather information about networks" + hetzner.hcloud.network_info: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + register: network_info + until: network_info is success + delay: 5 + retries: 3 + + # if server_network is specified + - name: "Hetzner Cloud: Check if network '{{ server_network }}' exists for given location" + ansible.builtin.fail: + msg: "No network with name '{{ server_network }}' in location '{{ target_network_zone }}'" + when: + - server_network | length > 0 + - not (network_info.hcloud_network_info + | selectattr("name", "equalto", server_network) + | selectattr("subnetworks", "defined") + | map(attribute='subnetworks') + | flatten + | selectattr("network_zone", "equalto", target_network_zone) + | list | length > 0) + + - name: "Hetzner Cloud: Extract ip_range for network '{{ server_network }}'" + ansible.builtin.set_fact: + server_network_ip_range: "{{ item.ip_range }}" + loop: "{{ network_info.hcloud_network_info }}" + when: + - server_network | length > 0 + - item.name == server_network + loop_control: + label: "{{ item.ip_range }}" + + # if server_network is not specified, create a network and subnet + - block: + - name: "Hetzner Cloud: Create a network '{{ hcloud_network_name | default('postgres-cluster-network-' + target_network_zone) }}'" + hetzner.hcloud.network: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ hcloud_network_name | default('postgres-cluster-network-' + target_network_zone) }}" + ip_range: "{{ hcloud_network_ip_range | default('10.0.0.0/16') }}" + state: present + + - name: "Hetzner Cloud: Create a subnetwork in network '{{ hcloud_network_name | default('postgres-cluster-network-' + target_network_zone) }}'" + hetzner.hcloud.subnetwork: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + network: "{{ hcloud_network_name | default('postgres-cluster-network-' + target_network_zone) }}" + ip_range: "{{ hcloud_subnetwork_ip_range | default('10.0.1.0/24') }}" + network_zone: "{{ target_network_zone }}" + type: server + state: present + + - name: "Set variable: server_network" + ansible.builtin.set_fact: + server_network: "{{ hcloud_network_name | default('postgres-cluster-network-' + target_network_zone) }}" + server_network_ip_range: "{{ hcloud_network_ip_range | default('10.0.0.0/16') }}" + when: server_network | length < 1 + + # Firewall + - name: "Hetzner Cloud: Create or modify public firewall" + hetzner.hcloud.firewall: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + state: "present" + name: "{{ patroni_cluster_name }}-public-firewall" + rules: "{{ rules }}" + vars: + rules: >- + {{ + ([ + { + 'description': 'SSH', + 'direction': 'in', + 'protocol': 'tcp', + 'port': ansible_ssh_port | default('22'), + 'source_ips': ssh_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + ] if ssh_public_access | bool else []) + + ([ + { + 'description': 'Netdata', + 'direction': 'in', + 'protocol': 'tcp', + 'port': netdata_port | default('19999'), + 'source_ips': netdata_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + ] if netdata_install | bool and netdata_public_access | bool else []) + + ([ + { + 'description': 'HAProxy - master', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.master, + 'source_ips': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + }, + { + 'description': 'HAProxy - replicas', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.replicas, + 'source_ips': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + }, + { + 'description': 'HAProxy - replicas_sync', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.replicas_sync, + 'source_ips': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + }, + { + 'description': 'HAProxy - replicas_async', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.replicas_async, + 'source_ips': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + ] if database_public_access | bool and with_haproxy_load_balancing | bool else []) + + ([ + { + 'description': 'PgBouncer', + 'direction': 'in', + 'protocol': 'tcp', + 'port': pgbouncer_listen_port | default('6432'), + 'source_ips': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + ] if database_public_access | bool and (not with_haproxy_load_balancing | bool and pgbouncer_install | bool) else []) + + ([ + { + 'description': 'PostgreSQL', + 'direction': 'in', + 'protocol': 'tcp', + 'port': postgresql_port | default('5432'), + 'source_ips': database_public_allowed_ips | default('0.0.0.0/0,::/0', true) | split(',') + } + ] if database_public_access | bool and (not with_haproxy_load_balancing | bool and not pgbouncer_install | bool) else []) + }} + when: + - cloud_firewall | bool + - (ssh_public_access | bool or netdata_public_access | bool or database_public_access | bool) + + - name: "Hetzner Cloud: Create or modify Postgres cluster firewall" + hetzner.hcloud.firewall: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + state: "present" + name: "{{ patroni_cluster_name }}-firewall" + rules: "{{ rules }}" + vars: + rules: >- + {{ + ([ + { + 'description': 'SSH', + 'direction': 'in', + 'protocol': 'tcp', + 'port': ansible_ssh_port | default('22'), + 'source_ips': [server_network_ip_range] + } + ]) + + ([ + { + 'description': 'Netdata', + 'direction': 'in', + 'protocol': 'tcp', + 'port': netdata_port | default('19999'), + 'source_ips': [server_network_ip_range] + } + ] if netdata_install | bool else []) + + ([ + { + 'description': 'HAProxy - master', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.master, + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'HAProxy - replicas', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.replicas, + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'HAProxy - replicas_sync', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.replicas_sync, + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'HAProxy - replicas_async', + 'direction': 'in', + 'protocol': 'tcp', + 'port': haproxy_listen_port.replicas_async, + 'source_ips': [server_network_ip_range] + } + ] if with_haproxy_load_balancing | bool else []) + + ([ + { + 'description': 'PgBouncer', + 'direction': 'in', + 'protocol': 'tcp', + 'port': pgbouncer_listen_port | default('6432'), + 'source_ips': [server_network_ip_range] + } + ] if pgbouncer_install | bool else []) + + ([ + { + 'description': 'PostgreSQL', + 'direction': 'in', + 'protocol': 'tcp', + 'port': postgresql_port | default('5432'), + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'Patroni', + 'direction': 'in', + 'protocol': 'tcp', + 'port': patroni_restapi_port | default('8008'), + 'source_ips': [server_network_ip_range] + } + ]) + + ([ + { + 'description': 'ETCD', + 'direction': 'in', + 'protocol': 'tcp', + 'port': etcd_client_port | default('2379'), + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'ETCD', + 'direction': 'in', + 'protocol': 'tcp', + 'port': etcd_peer_port | default('2380'), + 'source_ips': [server_network_ip_range] + } + ] if dcs_type == 'etcd' else []) + + ([ + { + 'description': 'Consul', + 'direction': 'in', + 'protocol': 'tcp', + 'port': consul_ports.dns | default('8600'), + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'Consul', + 'direction': 'in', + 'protocol': 'tcp', + 'port': consul_ports.http | default('8500'), + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'Consul', + 'direction': 'in', + 'protocol': 'tcp', + 'port': consul_ports.rpc | default('8400'), + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'Consul', + 'direction': 'in', + 'protocol': 'tcp', + 'port': consul_ports.serf_lan | default('8301'), + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'Consul', + 'direction': 'in', + 'protocol': 'tcp', + 'port': consul_ports.serf_wan | default('8302'), + 'source_ips': [server_network_ip_range] + }, + { + 'description': 'Consul', + 'direction': 'in', + 'protocol': 'tcp', + 'port': consul_ports.server | default('8300'), + 'source_ips': [server_network_ip_range] + } + ] if dcs_type == 'consul' else []) + }} + when: + - cloud_firewall | bool + + # Server and volume + - name: "Hetzner Cloud: Create or modify server" + hetzner.hcloud.server: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + state: present + server_type: "{{ server_type | lower }}" + image: "{{ server_image | lower }}" + ssh_keys: "{{ ssh_key_names }}" + location: "{{ server_location }}" + enable_ipv4: true + enable_ipv6: false + private_networks: + - "{{ server_network }}" + firewalls: "{{ firewalls_list }}" + labels: + cluster: "{{ patroni_cluster_name }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + register: server_result + vars: + firewalls_list: >- + {{ + ([patroni_cluster_name + '-public-firewall'] if cloud_firewall | bool and + (ssh_public_access | bool or netdata_public_access | bool or database_public_access | bool) else []) + + ([patroni_cluster_name + '-firewall'] if cloud_firewall | bool else []) + }} + + - name: "Hetzner Cloud: Add server to network '{{ server_network }}'" + hetzner.hcloud.server_network: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + network: "{{ server_network }}" + server: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + state: present + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + when: server_network | length > 0 + + - name: "Hetzner Cloud: Create or modify volume" + hetzner.hcloud.volume: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + state: present + size: "{{ volume_size | int }}" + server: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + + # Load Balancer + - name: "Hetzner Cloud: Create or modify Load Balancer" + hetzner.hcloud.load_balancer: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + location: "{{ server_location }}" + load_balancer_type: "{{ hetzner_load_balancer_type | default('lb21') }}" + algorithm: round_robin + delete_protection: true + state: present + labels: + cluster: "{{ patroni_cluster_name }}" + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Configure Load Balancer service" + hetzner.hcloud.load_balancer_service: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + load_balancer: "{{ patroni_cluster_name }}-{{ item }}" + listen_port: "{{ hetzner_load_balancer_port | default(database_port) }}" + destination_port: "{{ pgbouncer_listen_port | default(6432) if pgbouncer_install | bool else postgresql_port | default(5432) }}" + protocol: tcp + health_check: + protocol: http + port: "{{ patroni_restapi_port }}" + interval: 5 + timeout: 2 + retries: 3 + http: + path: "/{{ item }}" + status_codes: + - "200" + state: present + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + vars: + database_port: "{{ pgbouncer_listen_port | default(6432) if pgbouncer_install | bool else postgresql_port | default(5432) }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Add Load Balancer to network '{{ server_network }}'" + hetzner.hcloud.load_balancer_network: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + load_balancer: "{{ patroni_cluster_name }}-{{ item }}" + network: "{{ server_network }}" + state: present + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Disable public interface for Load Balancer" + hetzner.hcloud.load_balancer: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + location: "{{ server_location }}" + disable_public_interface: true + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + register: hetzner_load_balancer_disable_public + until: hetzner_load_balancer_disable_public is success + delay: 5 + retries: 3 + when: (cloud_load_balancer | bool and not database_public_access | bool) and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Enable public interface for Load Balancer" + hetzner.hcloud.load_balancer: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + location: "{{ server_location }}" + disable_public_interface: false + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + register: hetzner_load_balancer_enable_public + until: hetzner_load_balancer_enable_public is success + delay: 5 + retries: 3 + when: (cloud_load_balancer | bool and database_public_access | bool) and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Add servers to Load Balancer (use label_selector 'cluster={{ patroni_cluster_name }}')" + hetzner.hcloud.load_balancer_target: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + load_balancer: "{{ patroni_cluster_name }}-{{ item }}" + type: label_selector + label_selector: "cluster={{ patroni_cluster_name }}" + use_private_ip: true + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Gather information about Load Balancers" + hetzner.hcloud.load_balancer_info: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + register: hetzner_load_balancer + until: hetzner_load_balancer is success + delay: 5 + retries: 3 + when: state == 'present' + +- name: Wait for host to be available via SSH + ansible.builtin.wait_for: + host: "{{ item.hcloud_server.ipv4_address }}" + port: 22 + delay: 5 + timeout: 300 + loop: "{{ server_result.results }}" + loop_control: + label: "{{ item.hcloud_server.ipv4_address | default('N/A') }}" + when: + - server_result.results is defined + - item.hcloud_server is defined + +# Info +- name: Server info + ansible.builtin.debug: + msg: + id: "{{ item.hcloud_server.id }}" + name: "{{ item.hcloud_server.name }}" + image: "{{ item.hcloud_server.image }}" + type: "{{ item.hcloud_server.server_type }}" + volume_size: "{{ volume_size }} GB" + public_ip: "{{ item.hcloud_server.ipv4_address }}" + private_ip: "{{ item.hcloud_server.private_networks_info[0].ip }}" + loop: "{{ server_result.results }}" + loop_control: + index_var: idx + label: "{{ item.hcloud_server.ipv4_address | default('N/A') }}" + when: + - server_result.results is defined + - item.hcloud_server is defined + +# Inventory +- block: + - name: "Inventory | Initialize ip_addresses variable" + ansible.builtin.set_fact: + ip_addresses: [] + + - name: "Inventory | Extract IP addresses" + ansible.builtin.set_fact: + ip_addresses: >- + {{ ip_addresses + + [{'public_ip': item.hcloud_server.ipv4_address, + 'private_ip': item.hcloud_server.private_networks_info[0].ip}] + }} + loop: "{{ server_result.results | selectattr('hcloud_server', 'defined') }}" + loop_control: + label: "public_ip: {{ item.hcloud_server.ipv4_address }}, private_ip: {{ item.hcloud_server.private_networks_info[0].ip }}" + + - name: "Inventory | Generate in-memory inventory" + ansible.builtin.import_tasks: inventory.yml + when: + - server_result.results is defined + - server_result.results | selectattr('hcloud_server', 'defined') + +# Delete the temporary ssh key from the cloud after creating the server +- name: "Hetzner Cloud: Remove temporary SSH key {{ ssh_key_name }} from cloud" + hetzner.hcloud.ssh_key: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ ssh_key_name }}" + state: absent + when: + - ssh_key_name is defined + - tmp_ssh_key_name is defined + - ssh_key_name == tmp_ssh_key_name + +# Delete (if state is absent) +- block: + - name: "Hetzner Cloud: Delete server" + hetzner.hcloud.server: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + state: absent + location: "{{ server_location }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}" + + - name: "Hetzner Cloud: Delete volume" + hetzner.hcloud.volume: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + state: absent + location: "{{ server_location }}" + loop: "{{ range(0, server_count | int) | list }}" + loop_control: + index_var: idx + label: "{{ server_name | lower }}{{ '%02d' % (idx + 1) }}-storage" + + - name: "Hetzner Cloud: Disable protection for Load Balancer (if exists)" + hetzner.hcloud.load_balancer: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + location: "{{ server_location }}" + delete_protection: false + failed_when: false + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Delete Load Balancer" + hetzner.hcloud.load_balancer: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + name: "{{ patroni_cluster_name }}-{{ item }}" + location: "{{ server_location }}" + state: absent + loop: + - "primary" + - "replica" + - "sync" + loop_control: + label: "{{ patroni_cluster_name }}-{{ item }}" + when: cloud_load_balancer | bool and + (item == 'primary' or + (item == 'replica' and server_count | int > 1) or + (item in ['sync', 'async'] and server_count | int > 1 and synchronous_mode | bool)) + + - name: "Hetzner Cloud: Delete public firewall" + hetzner.hcloud.firewall: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + state: "absent" + name: "{{ patroni_cluster_name }}-public-firewall" + + - name: "Hetzner Cloud: Delete Postgres cluster firewall" + hetzner.hcloud.firewall: + api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}" + state: "absent" + name: "{{ patroni_cluster_name }}-firewall" + when: state == 'absent' + +... diff --git a/roles/cloud-resources/tasks/inventory.yml b/roles/cloud-resources/tasks/inventory.yml new file mode 100644 index 000000000..25a5e6080 --- /dev/null +++ b/roles/cloud-resources/tasks/inventory.yml @@ -0,0 +1,76 @@ +--- +- name: "Set variable: ssh_private_key_file" + ansible.builtin.set_fact: + ssh_private_key_file: "~{{ lookup('env', 'USER') }}/.ssh/{{ tmp_ssh_key_name }}" + when: + - ssh_key_name is defined + - tmp_ssh_key_name is defined + - ssh_key_name == tmp_ssh_key_name + +- name: "Inventory | Add host to 'postgres_cluster', 'master' groups" + ansible.builtin.add_host: + name: "{{ item.private_ip }}" + groups: + - postgres_cluster + - master + ansible_ssh_host: "{{ item.public_ip }}" + ansible_ssh_private_key_file: "{{ ssh_private_key_file | default(None) }}" + loop: "{{ [ip_addresses[0]] }}" # add the first item in the list + loop_control: + label: "{{ item.public_ip }}" + changed_when: false + +- name: "Inventory | Add host to 'postgres_cluster', 'replica' groups" + ansible.builtin.add_host: + name: "{{ item.private_ip }}" + groups: + - postgres_cluster + - replica + ansible_ssh_host: "{{ item.public_ip }}" + ansible_ssh_private_key_file: "{{ ssh_private_key_file | default(None) }}" + loop: "{{ ip_addresses[1:] }}" # start with the 2nd item of the list + loop_control: + label: "{{ item.public_ip }}" + when: ip_addresses | length > 1 # only if there is more than one item + changed_when: false + +- name: "Inventory | Add host to 'balancers' group" + ansible.builtin.add_host: + name: "{{ item.private_ip }}" + group: balancers + ansible_ssh_host: "{{ item.public_ip }}" + ansible_ssh_private_key_file: "{{ ssh_private_key_file | default(None) }}" + loop: "{{ ip_addresses }}" + loop_control: + label: "{{ item.public_ip }}" + changed_when: false + when: with_haproxy_load_balancing | bool + +- name: "Inventory | Add host to 'etcd_cluster' group" + ansible.builtin.add_host: + name: "{{ item.private_ip }}" + group: etcd_cluster + ansible_ssh_host: "{{ item.public_ip }}" + ansible_ssh_private_key_file: "{{ ssh_private_key_file | default(None) }}" + loop: "{{ ip_addresses[:7] }}" # no more than 7 servers for the etcd cluster + loop_control: + label: "{{ item.public_ip }}" + changed_when: false + when: not dcs_exists | bool and dcs_type == "etcd" + +- name: "Inventory | Add host to 'consul_instances' group" + ansible.builtin.add_host: + name: "{{ item.private_ip }}" + group: consul_instances + ansible_ssh_host: "{{ item.public_ip }}" + ansible_ssh_private_key_file: "{{ ssh_private_key_file | default(None) }}" + consul_node_role: "{{ not dcs_exists | ternary('server', 'client') }}" + consul_bootstrap_expect: true + consul_datacenter: "{{ server_location }}" + loop: "{{ ip_addresses }}" + loop_control: + label: "{{ item.public_ip }}" + changed_when: false + when: dcs_type == "consul" + +... diff --git a/roles/cloud-resources/tasks/main.yml b/roles/cloud-resources/tasks/main.yml new file mode 100644 index 000000000..0c62f66d6 --- /dev/null +++ b/roles/cloud-resources/tasks/main.yml @@ -0,0 +1,64 @@ +--- + +- name: Ensure that required variables are specified + ansible.builtin.fail: + msg: + - "One or more required variables have empty values." + - "Please specify value for variables: 'server_type', 'server_image', 'server_location'." + when: state == 'present' and + (server_type | length < 1 or + (server_image | length < 1 and cloud_provider != 'azure') or + server_location | length < 1) + +# if ssh_key_name is not specified +# with each new execution of the playbook, a new temporary ssh key is created +- block: + - name: Generate a unique temporary SSH key name + ansible.builtin.set_fact: + tmp_ssh_key_name: "ssh_key_tmp_{{ lookup('password', '/dev/null chars=ascii_lowercase length=7') }}" + + - name: Generate a new temporary SSH key to access the server for deployment + ansible.builtin.user: + name: "{{ lookup('env', 'USER') }}" + generate_ssh_key: true + ssh_key_bits: 2048 + ssh_key_file: ".ssh/{{ tmp_ssh_key_name }}" + ssh_key_comment: "{{ tmp_ssh_key_name }}" + force: true + register: tmp_ssh_key_result + when: + - state == 'present' + - ssh_key_name | length < 1 + - ssh_key_content | length < 1 + - not (postgresql_cluster_maintenance|default(false)|bool) # exclude for config_pgcluster.yml + +# set_fact: ssh_key_name and ssh_key_content +- name: "Set variable: ssh_key_name and ssh_key_content" + ansible.builtin.set_fact: + ssh_key_name: "{{ tmp_ssh_key_name }}" + ssh_key_content: "{{ tmp_ssh_key_result.ssh_public_key }}" + when: + - tmp_ssh_key_result.ssh_public_key is defined + - tmp_ssh_key_result.ssh_public_key | length > 0 + +- name: Import tasks for AWS + ansible.builtin.import_tasks: aws.yml + when: cloud_provider | lower == 'aws' + +- name: Import tasks for GCP + ansible.builtin.import_tasks: gcp.yml + when: cloud_provider | lower == 'gcp' + +- name: Import tasks for Azure + ansible.builtin.import_tasks: azure.yml + when: cloud_provider | lower == 'azure' + +- name: Import tasks for DigitalOcean + ansible.builtin.import_tasks: digitalocean.yml + when: cloud_provider | lower in ['digitalocean', 'do'] + +- name: Import tasks for Hetzner Cloud + ansible.builtin.import_tasks: hetzner.yml + when: cloud_provider | lower == 'hetzner' + +... diff --git a/roles/consul/files/consul_1.19.1_SHA256SUMS b/roles/consul/files/consul_1.19.1_SHA256SUMS deleted file mode 100644 index d6e1606fd..000000000 --- a/roles/consul/files/consul_1.19.1_SHA256SUMS +++ /dev/null @@ -1,11 +0,0 @@ -0b3b78d11d31a66938c9a90f5a9361e8363a43688f7f25fe300e37a95373d209 consul_1.19.1_darwin_amd64.zip -f2fc99fa8fb5e193f3ceefc7594f11200fa539ddfa8800c5925c59f62facee48 consul_1.19.1_darwin_arm64.zip -e0294fdcaa198fd15b058b5500c4847ee4d0b002b2b4665b8322e0668e879f94 consul_1.19.1_freebsd_386.zip -2229d8ace4066f3cf51031dae00c4fc05d3025b98bc300def6e717134aafa9c5 consul_1.19.1_freebsd_amd64.zip -87431003ee2c0caf86d827a0dfb43fc285ed9d4864f35d505a116f4744d496ca consul_1.19.1_linux_386.zip -aa48085aaa6f4130d0f1ee98c416dcd51b1b0f980d34f5b91834fd5b3387891c consul_1.19.1_linux_amd64.zip -a4a54fd0ca6991d48d617311dfb1623d6030140a10c005ad33809dad864da239 consul_1.19.1_linux_arm.zip -9699e5a2b85b4447a81b01138c3e0ef42dbcdd9df4f04e9318af9017aae73cc4 consul_1.19.1_linux_arm64.zip -ed81780dd374a00292f864ac457e28feffb637964acc397c9cd2676ca565041b consul_1.19.1_solaris_amd64.zip -5e6cc24d3219c1c331f9b39ade2961b9948c86e254a214751b921b4027f168a5 consul_1.19.1_windows_386.zip -a33bed52d6004c956b5b9a1fa6659477a32db14a07d37425f9ed96a6b1eaeae2 consul_1.19.1_windows_amd64.zip diff --git a/roles/consul/files/consul_1.19.1_linux_amd64.zip b/roles/consul/files/consul_1.19.1_linux_amd64.zip deleted file mode 100644 index 6648b4338..000000000 Binary files a/roles/consul/files/consul_1.19.1_linux_amd64.zip and /dev/null differ diff --git a/roles/consul/tasks/install_linux_repo.yml b/roles/consul/tasks/install_linux_repo.yml index 1b8798867..a2a2aa15a 100644 --- a/roles/consul/tasks/install_linux_repo.yml +++ b/roles/consul/tasks/install_linux_repo.yml @@ -72,7 +72,7 @@ retries: 3 when: ansible_os_family == "Debian" - - name: Add hashicorp repository and an apt signing key, uses whichever key is at the URL + - name: Add hashicorp repository ansible.builtin.deb822_repository: name: "{{ consul_repo_url.split('//')[1] | replace('.', '-') }}" types: "deb" diff --git a/roles/deploy-finish/tasks/main.yml b/roles/deploy-finish/tasks/main.yml index c2d916d10..9c7051efe 100644 --- a/roles/deploy-finish/tasks/main.yml +++ b/roles/deploy-finish/tasks/main.yml @@ -1,241 +1,308 @@ +# yamllint disable rule:line-length --- - name: Make sure handlers are flushed immediately ansible.builtin.meta: flush_handlers -# users info -- block: - - name: Get postgresql users list - run_once: true - become: true - become_user: postgres - ansible.builtin.command: - "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -Xc\"\\du\"" - register: users_result - delegate_to: "{{ groups.master[0] }}" - changed_when: false - - - name: PostgreSQL list of users - run_once: true - ansible.builtin.debug: - var: users_result.stdout_lines +# Get info +- name: Get Postgres users + run_once: true + become: true + become_user: postgres + ansible.builtin.command: + "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -Xc '\\du'" + register: users_result + delegate_to: "{{ groups.master[0] }}" + changed_when: false ignore_errors: true tags: users, users_list, cluster_info, cluster_status, point_in_time_recovery -# databases info -- block: - - name: Get postgresql database list - run_once: true - become: true - become_user: postgres - ansible.builtin.command: - "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -Xc - \" - SELECT - d.datname as name, - pg_get_userbyid(d.datdba) as owner, - pg_encoding_to_char(d.encoding) as encoding, - d.datcollate as collate, - d.datctype as ctype, - CASE - WHEN has_database_privilege(d.datname, 'CONNECT') - THEN pg_size_pretty(pg_database_size(d.datname)) - ELSE 'No Access' - END - size, - t.spcname as tablespace - FROM pg_catalog.pg_database d - JOIN pg_catalog.pg_tablespace t - ON d.dattablespace = t.oid - WHERE NOT datistemplate - ORDER BY 1 - \"" - register: dbs_result - delegate_to: "{{ groups.master[0] }}" - changed_when: false - - - name: PostgreSQL list of databases - run_once: true - ansible.builtin.debug: - var: dbs_result.stdout_lines +- name: Get Postgres databases + run_once: true + become: true + become_user: postgres + ansible.builtin.command: + "{{ postgresql_bin_dir }}/psql -p {{ postgresql_port }} -U {{ patroni_superuser_username }} -d postgres -Xc '\\l'" + register: dbs_result + delegate_to: "{{ groups.master[0] }}" + changed_when: false ignore_errors: true tags: databases, db_list, cluster_info, cluster_status, point_in_time_recovery -# cluster info -- block: - - name: Check postgresql cluster health - run_once: true - become: true - become_user: postgres - ansible.builtin.command: patronictl -c /etc/patroni/patroni.yml list - register: patronictl_result - environment: - PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" - changed_when: false - - - name: PostgreSQL Cluster health - run_once: true - ansible.builtin.debug: - var: patronictl_result.stdout_lines +- name: Get Postgres cluster info + run_once: true + become: true + become_user: postgres + ansible.builtin.command: patronictl -c /etc/patroni/patroni.yml list + register: patronictl_result + environment: + PATH: "{{ ansible_env.PATH }}:/usr/bin:/usr/local/bin" + changed_when: false ignore_errors: true tags: patroni_status, cluster_info, cluster_status, point_in_time_recovery -# connection info -- block: # if cluster_vip is defined - - name: PostgreSQL Cluster connection info - run_once: true - ansible.builtin.debug: - msg: - - +------------------------------------------------+ - - address (VIP) {{ cluster_vip }} - - port {{ haproxy_listen_port.master }} (read/write) master - - port {{ haproxy_listen_port.replicas }} (read only) all replicas - - port {{ haproxy_listen_port.replicas_sync }} (read only) synchronous replica only - - port {{ haproxy_listen_port.replicas_async }} (read only) asynchronous replicas only - - +------------------------------------------------+ - when: - - with_haproxy_load_balancing | bool - - synchronous_mode | bool - - - name: PostgreSQL Cluster connection info +# Print info +- name: Postgres list of users + run_once: true + ansible.builtin.debug: + msg: "{{ users_result.stdout_lines }}" + when: users_result.stdout_lines is defined + tags: users, users_list, cluster_info, cluster_status, point_in_time_recovery + +- name: Postgres list of databases + run_once: true + ansible.builtin.debug: + msg: "{{ dbs_result.stdout_lines }}" + when: dbs_result.stdout_lines is defined + tags: databases, db_list, cluster_info, cluster_status, point_in_time_recovery + +- name: Postgres Cluster info + run_once: true + ansible.builtin.debug: + msg: "{{ patronictl_result.stdout_lines }}" + when: patronictl_result.stdout_lines is defined + tags: patroni_status, cluster_info, cluster_status, point_in_time_recovery + +# Connection info +# Note: if the variable 'mask_password' is 'true', do not print the superuser password in connection info. + +# if 'cluster_vip' is defined +- block: + # if 'with_haproxy_load_balancing' is 'true' + - name: Connection info run_once: true ansible.builtin.debug: msg: - - +------------------------------------------------+ - - address (VIP) {{ cluster_vip }} - - port {{ haproxy_listen_port.master }} (read/write) master - - port {{ haproxy_listen_port.replicas }} (read only) all replicas - - +------------------------------------------------+ - when: - - with_haproxy_load_balancing | bool - - not synchronous_mode | bool - - - name: PostgreSQL Cluster connection info + address: "{{ cluster_vip }}" + port: + primary: "{{ haproxy_listen_port.master }}" + replica: "{{ haproxy_listen_port.replicas }}" + replica_sync: "{{ haproxy_listen_port.replicas_sync if synchronous_mode | bool else omit }}" + replica_async: "{{ haproxy_listen_port.replicas_async if synchronous_mode | bool else omit }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + when: with_haproxy_load_balancing | bool + + # if 'with_haproxy_load_balancing' is 'false' + - name: Connection info run_once: true ansible.builtin.debug: msg: - - +------------------------------------------------+ - - address (VIP) {{ cluster_vip }} - - port {% if pgbouncer_install %}{{ pgbouncer_listen_port }} (pgbouncer){% else %}{{ postgresql_port }}{% endif %} - - +------------------------------------------------+ - when: - - not with_haproxy_load_balancing | bool + address: "{{ cluster_vip }}" + port: "{{ pgbouncer_listen_port if pgbouncer_install | bool else postgresql_port }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + when: not with_haproxy_load_balancing | bool + ignore_errors: true + vars: + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" when: - - (cluster_vip is defined and cluster_vip | length > 0) + - cluster_vip is defined and cluster_vip | length > 0 - dcs_type == "etcd" - ignore_errors: true + - (cloud_provider | default('') | length < 1 or not cloud_load_balancer | default(true) | bool) tags: conn_info, cluster_info, cluster_status +# if 'cluster_vip' is not defined - block: - - name: Get vip info - ansible.builtin.set_fact: - man_ip: "{{ item }}" - loop: "{{ ansible_all_ipv4_addresses }}" - when: item == cluster_vip - - - name: Virtual IP Address (VIP) info - ansible.builtin.debug: - msg: - "Cluster ip address (VIP) {{ cluster_vip }} - is running on server {{ ansible_hostname }}" - when: man_ip is defined and man_ip == cluster_vip - when: - - (cluster_vip is defined and cluster_vip | length > 0) - - dcs_type == "etcd" - ignore_errors: true - tags: vip_owner, vip_status, cluster_info, cluster_status - - -- block: # if cluster_vip is not defined - - name: Create list of nodes - run_once: true - ansible.builtin.set_fact: - haproxy_nodes: >- - {{ - groups['balancers'] - | default([]) - | map('extract', hostvars, 'inventory_hostname') - | join(',') - }} - postgres_cluster_nodes: >- - {{ - groups['postgres_cluster'] - | default([]) - | map('extract', hostvars, 'inventory_hostname') - | join(',') - }} - - - name: PostgreSQL Cluster connection info + # if 'with_haproxy_load_balancing' is 'true' + - name: Connection info run_once: true ansible.builtin.debug: msg: - - +------------------------------------------------+ - - address {{ haproxy_nodes }} - - port {{ haproxy_listen_port.master }} (read/write) master - - port {{ haproxy_listen_port.replicas }} (read only) all replicas - - port {{ haproxy_listen_port.replicas_sync }} (read only) synchronous replica only - - port {{ haproxy_listen_port.replicas_async }} (read only) asynchronous replicas only - - +------------------------------------------------+ - when: - - with_haproxy_load_balancing | bool - - synchronous_mode | bool - - - name: PostgreSQL Cluster connection info + public_address: "{{ public_haproxy_ip_addresses if database_public_access | default(false) | bool else omit }}" + address: "{{ haproxy_ip_addresses }}" + port: + primary: "{{ haproxy_listen_port.master }}" + replica: "{{ haproxy_listen_port.replicas }}" + replica_sync: "{{ haproxy_listen_port.replicas_sync if synchronous_mode | bool else omit }}" + replica_async: "{{ haproxy_listen_port.replicas_async if synchronous_mode | bool else omit }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + when: with_haproxy_load_balancing | bool + + # if 'with_haproxy_load_balancing' is 'false' and 'pgbouncer_install' is 'true' + - name: Connection info run_once: true ansible.builtin.debug: msg: - - +------------------------------------------------+ - - address {{ haproxy_nodes }} - - port {{ haproxy_listen_port.master }} (read/write) master - - port {{ haproxy_listen_port.replicas }} (read only) all replicas - - +------------------------------------------------+ - when: - - with_haproxy_load_balancing | bool - - not synchronous_mode | bool - - - name: PostgreSQL Cluster connection info + public_address: "{{ public_postgres_ip_addresses if database_public_access | default(false) | bool else omit }}" + address: "{{ postgres_ip_addresses }}" + port: "{{ pgbouncer_listen_port }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + when: not with_haproxy_load_balancing | bool + + # if 'with_haproxy_load_balancing' is 'false' and 'pgbouncer_install' is 'false' + - name: Connection info run_once: true ansible.builtin.debug: msg: - - +------------------------------------------------+ - - address {{ postgres_cluster_nodes }} - - port {% if pgbouncer_install %}{{ pgbouncer_listen_port }} (pgbouncer){% else %}{{ postgresql_port }}{% endif %} - - +------------------------------------------------+ - when: - - not with_haproxy_load_balancing | bool + public_address: "{{ public_postgres_ip_addresses if database_public_access | default(false) | bool else omit }}" + address: "{{ postgres_ip_addresses }}" + port: "{{ postgresql_port }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + connection_string: + read_write: "postgresql://{{ superuser_username }}:{{ superuser_password }}@{{ libpq_postgres_host_port }}/postgres?target_session_attrs=read-write" + read_only: "postgresql://{{ superuser_username }}:{{ superuser_password }}@{{ libpq_postgres_host_port }}/postgres?target_session_attrs=read-only{{ libpq_load_balance }}" + when: not with_haproxy_load_balancing | bool and not pgbouncer_install | bool ignore_errors: true + vars: + public_haproxy_ip_addresses: "{{ groups['balancers'] | default([]) | map('extract', hostvars, 'ansible_ssh_host') | join(',') }}" + public_postgres_ip_addresses: "{{ groups['postgres_cluster'] | default([]) | map('extract', hostvars, 'ansible_ssh_host') | join(',') }}" + haproxy_ip_addresses: "{{ groups['balancers'] | default([]) | map('extract', hostvars, 'inventory_hostname') | join(',') }}" + postgres_ip_addresses: "{{ groups['postgres_cluster'] | default([]) | map('extract', hostvars, 'inventory_hostname') | join(',') }}" + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" + libpq_postgres_host_port: "{{ postgres_ip_addresses.split(',') | map('regex_replace', '$', ':' + postgresql_port | string) | join(',') }}" + libpq_load_balance: "{{ '&load_balance_hosts=random' if postgresql_version | int >= 16 else '' }}" when: - (cluster_vip is not defined or cluster_vip | length < 1) - dcs_type == "etcd" + - (cloud_provider | default('') | length < 1 or not cloud_load_balancer | default(true) | bool) tags: conn_info, cluster_info, cluster_status +# if dcs_type: "consul" +- name: Connection info + run_once: true + ansible.builtin.debug: + msg: + address: + primary: "master.{{ patroni_cluster_name }}.service.consul" + replica: "replica.{{ patroni_cluster_name }}.service.consul" + replica_sync: "{{ 'sync-replica.' ~ patroni_cluster_name ~ '.service.consul' if synchronous_mode | bool else omit }}" + replica_async: "{{ 'async-replica.' ~ patroni_cluster_name ~ '.service.consul' if synchronous_mode | bool else omit }}" + port: "{{ pgbouncer_listen_port if pgbouncer_install | bool else postgresql_port }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + ignore_errors: true + vars: + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" + when: + - dcs_type == "consul" + - (cloud_provider | default('') | length < 1 or not cloud_load_balancer | default(true) | bool) + tags: conn_info, cluster_info, cluster_status -- block: # if dcs_type: "consul" - - name: PostgreSQL Cluster connection info - run_once: true - ansible.builtin.debug: - msg: - - +------------------------------------------------+ - - "Client access point (DNS):" - - " master.{{ patroni_cluster_name }}.service.consul " - - " replica.{{ patroni_cluster_name }}.service.consul " - - port {% if pgbouncer_install %}{{ pgbouncer_listen_port }} (pgbouncer){% else %}{{ postgresql_port }}{% endif %} - - +------------------------------------------------+ - when: not synchronous_mode | bool - - - name: PostgreSQL Cluster connection info - run_once: true - ansible.builtin.debug: - msg: - - +------------------------------------------------+ - - "Client access point (DNS):" - - " master.{{ patroni_cluster_name }}.service.consul " - - " replica.{{ patroni_cluster_name }}.service.consul " - - " sync-replica.{{ patroni_cluster_name }}.service.consul " - - " async-replica.{{ patroni_cluster_name }}.service.consul " - - port {% if pgbouncer_install %}{{ pgbouncer_listen_port }} (pgbouncer){% else %}{{ postgresql_port }}{% endif %} - - +------------------------------------------------+ - when: synchronous_mode | bool - when: dcs_type == "consul" +# if 'cloud_provider' and `cloud_load_balancer` is defined + +# AWS +- name: Connection info + run_once: true + ansible.builtin.debug: + msg: + address: + primary: "{{ load_balancer_primary }}" + replica: "{{ load_balancer_replica if load_balancer_replica != 'N/A' else omit }}" + replica_sync: "{{ load_balancer_replica_sync if synchronous_mode | bool else omit }}" + port: "{{ pgbouncer_listen_port if pgbouncer_install | bool else postgresql_port }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + ignore_errors: true + vars: + load_balancer_primary: "{{ (hostvars['localhost']['aws_elb_classic_lb']['results'] | selectattr('item', 'equalto', 'primary') | first).elb.dns_name | default('N/A') }}" + load_balancer_replica: "{{ (hostvars['localhost']['aws_elb_classic_lb']['results'] | selectattr('item', 'equalto', 'replica') | first).elb.dns_name | default('N/A') }}" + load_balancer_replica_sync: "{{ (hostvars['localhost']['aws_elb_classic_lb']['results'] | selectattr('item', 'equalto', 'sync') | first).elb.dns_name | default('N/A') }}" + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" + when: cloud_provider | default('') | lower == 'aws' and cloud_load_balancer | default(true) | bool + tags: conn_info, cluster_info, cluster_status + +# GCP +- name: Connection info + run_once: true + ansible.builtin.debug: + msg: + address: + primary: "{{ load_balancer_primary }}" + replica: "{{ load_balancer_replica if load_balancer_replica != 'N/A' else omit }}" + replica_sync: "{{ load_balancer_replica_sync if synchronous_mode | bool else omit }}" + port: "{{ pgbouncer_listen_port if pgbouncer_install | bool else postgresql_port }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + ignore_errors: true + vars: + load_balancer_primary: "{{ (hostvars['localhost']['gcp_load_balancer']['results'] | selectattr('item', 'equalto', 'primary') | first).IPAddress | default('N/A') }}" + load_balancer_replica: "{{ (hostvars['localhost']['gcp_load_balancer']['results'] | selectattr('item', 'equalto', 'replica') | first).IPAddress | default('N/A') }}" + load_balancer_replica_sync: "{{ (hostvars['localhost']['gcp_load_balancer']['results'] | selectattr('item', 'equalto', 'sync') | first).IPAddress | default('N/A') }}" + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" + when: cloud_provider | default('') | lower == 'gcp' and cloud_load_balancer | default(true) | bool + tags: conn_info, cluster_info, cluster_status + +# Azure +- name: Connection info + run_once: true + ansible.builtin.debug: + msg: + address: + primary: "{{ lb_primary_public if database_public_access | default(false) | bool else lb_primary_private }}" + replica: "{{ (lb_replica_public if lb_replica_public != 'N/A' else omit) if database_public_access | default(false) | bool else (lb_replica_private if lb_replica_private != 'N/A' else omit) }}" + replica_sync: "{{ (lb_sync_public if database_public_access | default(false) | bool else lb_sync_private) if synchronous_mode | bool else omit }}" + port: "{{ pgbouncer_listen_port if pgbouncer_install | bool else postgresql_port }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + ignore_errors: true + vars: + lb_primary_public: "{{ (hostvars['localhost']['azure_load_balancer_public_ip']['results'] | selectattr('item', 'equalto', 'primary') | first).state.ip_address | default('N/A') }}" + lb_replica_public: "{{ (hostvars['localhost']['azure_load_balancer_public_ip']['results'] | selectattr('item', 'equalto', 'replica') | first).state.ip_address | default('N/A') }}" + lb_sync_public: "{{ (hostvars['localhost']['azure_load_balancer_public_ip']['results'] | selectattr('item', 'equalto', 'sync') | first).state.ip_address | default('N/A') }}" + lb_primary_private: "{{ (hostvars['localhost']['azure_load_balancer']['results'] | selectattr('item', 'equalto', 'primary') | first).state.frontend_ip_configurations[0].private_ip_address | default('N/A') }}" + lb_replica_private: "{{ (hostvars['localhost']['azure_load_balancer']['results'] | selectattr('item', 'equalto', 'replica') | first).state.frontend_ip_configurations[0].private_ip_address | default('N/A') }}" + lb_sync_private: "{{ (hostvars['localhost']['azure_load_balancer']['results'] | selectattr('item', 'equalto', 'sync') | first).state.frontend_ip_configurations[0].private_ip_address | default('N/A') }}" + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" + when: cloud_provider | default('') | lower == 'azure' and cloud_load_balancer | default(true) | bool + tags: conn_info, cluster_info, cluster_status + +# DigitalOcean +- name: Connection info + run_once: true + ansible.builtin.debug: + msg: + address: + primary: "{{ load_balancer_primary }}" + replica: "{{ load_balancer_replica if load_balancer_replica != 'N/A' else omit }}" + replica_sync: "{{ load_balancer_replica_sync if synchronous_mode | bool else omit }}" + port: "{{ digital_ocean_load_balancer_port | default(database_port) }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + ignore_errors: true + vars: + load_balancer_primary: "{{ (hostvars['localhost']['digitalocean_load_balancer']['data'] | selectattr('name', 'equalto', patroni_cluster_name + '-primary') | first).ip | default('N/A') }}" + load_balancer_replica: "{{ (hostvars['localhost']['digitalocean_load_balancer']['data'] | selectattr('name', 'equalto', patroni_cluster_name + '-replica') | first).ip | default('N/A') }}" + load_balancer_replica_sync: "{{ (hostvars['localhost']['digitalocean_load_balancer']['data'] | selectattr('name', 'equalto', patroni_cluster_name + '-sync') | first).ip | default('N/A') }}" + database_port: "{{ pgbouncer_listen_port if pgbouncer_install | bool else postgresql_port }}" + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" + when: cloud_provider | default('') | lower == 'digitalocean' and cloud_load_balancer | default(true) | bool + tags: conn_info, cluster_info, cluster_status + +# Hetzner Cloud +- name: Connection info + run_once: true + ansible.builtin.debug: + msg: + address: + primary: "{{ lb_primary_public if database_public_access | default(false) | bool else lb_primary_private }}" + replica: "{{ (lb_replica_public if lb_replica_public != 'N/A' else omit) if database_public_access | default(false) | bool else (lb_replica_private if lb_replica_private != 'N/A' else omit) }}" + replica_sync: "{{ (lb_replica_sync_public if database_public_access | default(false) | bool else lb_replica_sync_private) if synchronous_mode | bool else omit }}" + port: "{{ hetzner_load_balancer_port | default(database_port) }}" + superuser: "{{ superuser_username }}" + password: "{{ superuser_password }}" + ignore_errors: true + vars: + lb_primary_public: "{{ (hostvars['localhost']['hetzner_load_balancer']['hcloud_load_balancer_info'] | selectattr('name', 'equalto', patroni_cluster_name + '-primary') | first).ipv4_address | default('N/A') }}" + lb_primary_private: "{{ (hostvars['localhost']['hetzner_load_balancer']['hcloud_load_balancer_info'] | selectattr('name', 'equalto', patroni_cluster_name + '-primary') | first).private_ipv4_address | default('N/A') }}" + lb_replica_public: "{{ (hostvars['localhost']['hetzner_load_balancer']['hcloud_load_balancer_info'] | selectattr('name', 'equalto', patroni_cluster_name + '-replica') | first).ipv4_address | default('N/A') }}" + lb_replica_private: "{{ (hostvars['localhost']['hetzner_load_balancer']['hcloud_load_balancer_info'] | selectattr('name', 'equalto', patroni_cluster_name + '-replica') | first).private_ipv4_address | default('N/A') }}" + lb_replica_sync_public: "{{ (hostvars['localhost']['hetzner_load_balancer']['hcloud_load_balancer_info'] | selectattr('name', 'equalto', patroni_cluster_name + '-sync') | first).ipv4_address | default('N/A') }}" + lb_replica_sync_private: "{{ (hostvars['localhost']['hetzner_load_balancer']['hcloud_load_balancer_info'] | selectattr('name', 'equalto', patroni_cluster_name + '-sync') | first).private_ipv4_address | default('N/A') }}" + database_port: "{{ pgbouncer_listen_port if pgbouncer_install | bool else postgresql_port }}" + superuser_username: "{{ patroni_superuser_username }}" + superuser_password: "{{ '********' if mask_password | default(false) | bool else patroni_superuser_password }}" + when: cloud_provider | default('') | lower == 'hetzner' and cloud_load_balancer | default(true) | bool + tags: conn_info, cluster_info, cluster_status ... diff --git a/roles/mount/defaults/main.yml b/roles/mount/defaults/main.yml new file mode 100644 index 000000000..8ae8b03ad --- /dev/null +++ b/roles/mount/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# Example for --extra-vars +# '{"mount": [{"path": "/pgdata", "src": "UUID=83304ebb-d942-4093-975b-8253be2aabe1", "fstype": "ext4", "opts": "defaults,noatime", "state": "mounted"}]}' + +mount: + - path: "" + src: "" + fstype: "" + opts: "" + state: "" + +... diff --git a/roles/mount/tasks/main.yml b/roles/mount/tasks/main.yml new file mode 100644 index 000000000..fdc63479a --- /dev/null +++ b/roles/mount/tasks/main.yml @@ -0,0 +1,149 @@ +--- + +- block: + # Try to detect an empty disk (if 'cloud_provider' is defined) + - name: Detect empty volume + ansible.builtin.shell: | + set -o pipefail; + lsblk -e7 --output NAME,FSTYPE,TYPE --json \ + | jq -r '.blockdevices[] | select(.children == null and .fstype == null and .type == "disk") | .name' + args: + executable: /bin/bash + register: lsblk_disk + changed_when: false + when: (cloud_provider | default('') | length > 0) and mount[0].src | length < 1 + + # Show the error message, if empty volume is not detected + - name: Empty volume is not detected + ansible.builtin.fail: + msg: "Whoops! The empty volume is not detected. Skip mounting." + ignore_errors: true + when: lsblk_disk.stdout is defined and lsblk_disk.stdout | length < 1 + + # Filesystem + - name: Create "{{ pg_data_mount_fstype | default('ext4') }}" filesystem on the disk "/dev/{{ lsblk_disk.stdout }}" + community.general.filesystem: + dev: "/dev/{{ lsblk_disk.stdout }}" + fstype: "{{ pg_data_mount_fstype | default('ext4') }}" + when: + - (lsblk_disk.stdout is defined and lsblk_disk.stdout | length > 0) + - ((pg_data_mount_fstype is defined and pg_data_mount_fstype != 'zfs') or + (pg_data_mount_fstype is not defined and mount[0].fstype != 'zfs')) + + # UUID + - name: Get UUID of the disk "/dev/{{ lsblk_disk.stdout }}" + ansible.builtin.shell: | + set -o pipefail; + lsblk -no UUID /dev/{{ lsblk_disk.stdout }} | tr -d '\n' + args: + executable: /bin/bash + register: lsblk_uuid + changed_when: false + when: + - (lsblk_disk.stdout is defined and lsblk_disk.stdout | length > 0) + - ((pg_data_mount_fstype is defined and pg_data_mount_fstype != 'zfs') or + (pg_data_mount_fstype is not defined and mount[0].fstype != 'zfs')) + + - name: "Set mount variables" + ansible.builtin.set_fact: + mount: + - src: "UUID={{ lsblk_uuid.stdout }}" + path: "{{ pg_data_mount_path | default('/pgdata', true) }}" + fstype: "{{ pg_data_mount_fstype | default('ext4', true) }}" + when: lsblk_uuid.stdout is defined + + # Mount + - name: Mount the filesystem + ansible.posix.mount: + path: "{{ item.path }}" + src: "{{ item.src }}" + fstype: "{{ item.fstype | default(pg_data_mount_fstype | default('ext4', true), true) }}" + opts: "{{ item.opts | default('defaults,noatime') }}" + state: "{{ item.state | default('mounted') }}" + loop: "{{ mount }}" + when: + - (item.src | length > 0 and item.path | length > 0) + - ((pg_data_mount_fstype is defined and pg_data_mount_fstype != 'zfs') or + (pg_data_mount_fstype is not defined and item.fstype != 'zfs')) + + # ZFS Pool (if fstype is 'zfs') + - block: + - name: Install zfs + ansible.builtin.package: + name: zfsutils-linux + state: present + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: ansible_distribution == 'Ubuntu' + + - name: Install zfs + ansible.builtin.package: + name: + - "linux-headers-{{ ansible_kernel }}" + - dpkg-dev + - zfs-dkms + - zfsutils-linux + state: present + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: ansible_distribution == 'Debian' + + - block: # RedHat based + - name: Download zfs-release + ansible.builtin.get_url: + url: "https://zfsonlinux.org/epel/zfs-release-2-3.el{{ ansible_distribution_major_version }}.noarch.rpm" + dest: /tmp/zfs-release.rpm + + - name: Install zfs-release + ansible.builtin.package: + name: /tmp/zfs-release.rpm + state: present + disable_gpg_check: true + register: package_status + until: package_status is success + delay: 5 + retries: 3 + + - name: Install zfs + ansible.builtin.package: + name: + - kernel-devel + - zfs + state: present + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: ansible_os_family == 'RedHat' + + - name: Load the ZFS module + community.general.modprobe: + name: zfs + state: present + + - name: Ensure zfs is loaded at boot + ansible.builtin.lineinfile: + path: /etc/modules-load.d/zfs.conf + line: zfs + create: true + + - name: Create zpool (use {{ mount[0].src | default("/dev/" + lsblk_disk.stdout, true) }}) + ansible.builtin.command: >- + zpool create -f + -O compression=on + -O atime=off + -O recordsize=128k + -O logbias=throughput + -m {{ pg_data_mount_path | default(mount[0].path | default('/pgdata', true), true) }} + pgdata {{ mount[0].src | default("/dev/" + lsblk_disk.stdout, true) }} + when: + - (mount[0].src | length > 0 or lsblk_disk.stdout | default('') | length > 0) + - ((pg_data_mount_fstype is defined and pg_data_mount_fstype == 'zfs') or + (pg_data_mount_fstype is not defined and mount[0].fstype == 'zfs')) + tags: mount, zpool + +... diff --git a/roles/netdata/tasks/main.yml b/roles/netdata/tasks/main.yml index 6711086fb..8e319c806 100644 --- a/roles/netdata/tasks/main.yml +++ b/roles/netdata/tasks/main.yml @@ -1,15 +1,22 @@ --- - block: - - name: Download the installation script "kickstart.sh" ansible.builtin.get_url: url: https://my-netdata.io/kickstart.sh dest: /tmp/kickstart.sh mode: +x + register: get_url_status + until: get_url_status is success + delay: 10 + retries: 3 - name: Install Netdata ansible.builtin.command: /tmp/kickstart.sh {{ netdata_install_options | default('--dont-wait') }} + register: install_status + until: install_status is success + delay: 10 + retries: 3 - name: Configure Netdata ansible.builtin.template: @@ -23,7 +30,6 @@ ansible.builtin.service: name: netdata state: restarted - environment: "{{ proxy_env | default({}) }}" tags: netdata diff --git a/roles/packages/tasks/extensions.yml b/roles/packages/tasks/extensions.yml new file mode 100644 index 000000000..b87d8e576 --- /dev/null +++ b/roles/packages/tasks/extensions.yml @@ -0,0 +1,240 @@ +--- +# Extension Auto-Setup: packages + +# TimescaleDB (if 'enable_timescale' is 'true') +- name: Install TimescaleDB package + ansible.builtin.package: + name: "{{ item }}" + state: present + loop: "{{ timescaledb_package }}" + vars: + timescaledb_package: >- + [{% if postgresql_version | int >= 11 %} + "timescaledb-2-postgresql-{{ postgresql_version }}" + {% else %} + "timescaledb-postgresql-{{ postgresql_version }}" + {% endif %}] + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: (enable_timescale | default(false) | bool) or (enable_timescaledb | default(false) | bool) + tags: timescaledb, timescale + +# Citus (if 'enable_citus' is 'true') +- name: Install Citus package + ansible.builtin.package: + name: "{{ item }}" + state: present + loop: "{{ citus_package }}" + vars: + citus_package: >- + [{% if ansible_os_family == 'Debian' and postgresql_version | int >= 14 %} + "postgresql-{{ postgresql_version }}-citus-{{ citus_version | default('12.1') }}" + {% elif ansible_os_family == 'Debian' and postgresql_version | int == 13 %} + "postgresql-{{ postgresql_version }}-citus-11.3" + {% elif ansible_os_family == 'Debian' and postgresql_version | int == 12 %} + "postgresql-{{ postgresql_version }}-citus-10.2" + {% elif ansible_os_family == 'Debian' and postgresql_version | int == 11 %} + "postgresql-{{ postgresql_version }}-citus-10.0" + {% else %} + "citus_{{ postgresql_version }}" + {% endif %}] + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: + - enable_citus | default(false) | bool + - (ansible_os_family == 'Debian' and postgresql_version | int >= 11) or + (ansible_os_family == 'RedHat' and postgresql_version | int >= 12) + tags: citus + +# pg_repack (if 'enable_pg_repack' is 'true') +- name: Install pg_repack package + ansible.builtin.package: + name: "{{ pg_repack_package }}" + state: present + vars: + pg_repack_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-repack + {% else %} + pg_repack_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_repack | default(false) | bool + tags: pg_repack + +# pg_cron (if 'enable_pg_cron' is 'true') +- name: Install pg_cron package + ansible.builtin.package: + name: "{{ pg_cron_package }}" + state: present + vars: + pg_cron_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-cron + {% else %} + pg_cron_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_cron | default(false) | bool + tags: pg_cron + +# pgaudit (if 'enable_pgaudit' is 'true') +- name: Install pgaudit package + ansible.builtin.package: + name: "{{ pgaudit_package }}" + state: present + vars: + pgaudit_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-pgaudit + {% elif ansible_os_family == 'RedHat' and postgresql_version | int >= 16 %} + pgaudit_{{ postgresql_version }} + {% elif ansible_os_family == 'RedHat' and postgresql_version | int == 15 %} + pgaudit17_{{ postgresql_version }} + {% elif ansible_os_family == 'RedHat' and postgresql_version | int == 14 %} + pgaudit16_{{ postgresql_version }} + {% elif ansible_os_family == 'RedHat' and postgresql_version | int == 13 %} + pgaudit15_{{ postgresql_version }} + {% elif ansible_os_family == 'RedHat' and postgresql_version | int == 12 %} + pgaudit14_{{ postgresql_version }} + {% elif ansible_os_family == 'RedHat' and postgresql_version | int == 11 %} + pgaudit13_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pgaudit | default(false) | bool + tags: pgaudit + +# pgvector (if 'enable_pgvector' is 'true') +- name: Install pgvector package + ansible.builtin.package: + name: "{{ pgvector_package }}" + state: present + vars: + pgvector_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-pgvector + {% else %} + pgvector_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: + - enable_pgvector | default(false)| bool + - (ansible_os_family == 'Debian' and postgresql_version | int >= 11) or + (ansible_os_family == 'RedHat' and postgresql_version | int >= 12) + tags: pgvector + +# postgis (if 'enable_postgis' is 'true') +- name: Install postgis package + ansible.builtin.package: + name: "{{ postgis_package }}" + state: present + vars: + postgis_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-postgis-3 + {% elif ansible_os_family == 'RedHat' and postgresql_version | int == 16 %} + postgis34_{{ postgresql_version }} + {% else %} + postgis33_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_postgis | default(false) | bool + tags: postgis + +# pgrouting (if 'enable_pgrouting' is 'true') +- name: Install pgrouting package + ansible.builtin.package: + name: "{{ pgrouting_package }}" + state: present + vars: + pgrouting_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-pgrouting + {% else %} + pgrouting_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pgrouting | default(false) | bool and + not (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('20.04', '<=')) + tags: pgrouting + +# pg_stat_kcache (if 'enable_pg_stat_kcache' is 'true') +- name: Install pg_stat_kcache package + ansible.builtin.package: + name: "{{ pg_stat_kcache_package }}" + state: present + vars: + pg_stat_kcache_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-pg-stat-kcache + {% else %} + pg_stat_kcache_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_stat_kcache | default(false) | bool + tags: pg_stat_kcache + +# pg_wait_sampling (if 'enable_pg_wait_sampling' is 'true') +- name: Install pg_wait_sampling package + ansible.builtin.package: + name: "{{ pg_wait_sampling_package }}" + state: present + vars: + pg_wait_sampling_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-pg-wait-sampling + {% else %} + pg_wait_sampling_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_wait_sampling | default(false) | bool + tags: pg_wait_sampling + +# pg_partman (if 'enable_pg_partman' is 'true') +- name: Install pg_partman package + ansible.builtin.package: + name: "{{ pg_partman_package }}" + state: present + vars: + pg_partman_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ postgresql_version }}-partman + {% else %} + pg_partman_{{ postgresql_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_partman | default(false) | bool + tags: pg_partman + +... diff --git a/roles/packages/tasks/main.yml b/roles/packages/tasks/main.yml index dc1973675..a51b375d9 100644 --- a/roles/packages/tasks/main.yml +++ b/roles/packages/tasks/main.yml @@ -34,23 +34,29 @@ when: packages_from_file is defined and packages_from_file | length > 0 tags: install_packages_from_file -- block: # RedHat (update cache) - - name: Update dnf cache - ansible.builtin.shell: dnf clean all && dnf -y makecache - args: - executable: /bin/bash +# Install packages from repository + +# RedHat +- name: Update dnf cache + ansible.builtin.shell: dnf clean all && dnf -y makecache + args: + executable: /bin/bash + register: dnf_status + until: dnf_status is success + delay: 5 + retries: 3 + when: + - installation_method == "repo" + - ansible_os_family == "RedHat" environment: "{{ proxy_env | default({}) }}" - when: ansible_os_family == "RedHat" and installation_method == "repo" tags: install_packages, install_postgres -# Install packages from repository -# RedHat - name: Install system packages ansible.builtin.dnf: name: "{{ item }}" state: present disablerepo: 'pgdg*' - loop: "{{ system_packages }}" + loop: "{{ system_packages | list }}" register: package_status until: package_status is success delay: 5 @@ -59,6 +65,7 @@ when: - installation_method == "repo" - ansible_os_family == "RedHat" + - system_packages | default('') | length > 0 tags: install_packages - name: Set Python alternative @@ -91,21 +98,71 @@ tags: install_packages # Debian +- name: Update apt cache + ansible.builtin.apt: + update_cache: true + register: apt_status + until: apt_status is success + delay: 5 + retries: 3 + environment: "{{ proxy_env | default({}) }}" + when: + - installation_method == "repo" + - ansible_os_family == "Debian" + - name: Install system packages ansible.builtin.apt: name: "{{ item }}" state: present - loop: "{{ system_packages }}" - environment: "{{ proxy_env | default({}) }}" + loop: "{{ system_packages | list }}" register: apt_status until: apt_status is success delay: 5 retries: 3 - when: ansible_os_family == "Debian" and installation_method == "repo" + environment: "{{ proxy_env | default({}) }}" + when: + - installation_method == "repo" + - ansible_os_family == "Debian" + - system_packages | default('') | length > 0 tags: install_packages -# PostgreSQL prepare for install (for Debian based only) -- block: +# Install PostgreSQL from repository + +# RedHat +- block: # Preparing to install PostgreSQL + - name: PostgreSQL | check if appstream module is enabled + ansible.builtin.command: 'dnf -y -C module list postgresql' + register: postgresql_module_result + changed_when: false + + - name: PostgreSQL | disable appstream module + ansible.builtin.command: 'dnf -y -C module disable postgresql' + when: "'[x] ' not in postgresql_module_result.stdout" + when: + - installation_method == "repo" + - ansible_os_family == "RedHat" + - ansible_distribution_major_version >= '8' + ignore_errors: true + tags: install_postgres + +- name: Install PostgreSQL packages + ansible.builtin.package: + name: "{{ item }}" + state: present + loop: "{{ postgresql_packages | list }}" + register: package_status + until: package_status is success + delay: 5 + retries: 3 + environment: "{{ proxy_env | default({}) }}" + when: + - installation_method == "repo" + - ansible_os_family == "RedHat" + - postgresql_packages | default('') | length > 0 + tags: install_packages, install_postgres + +# Debian +- block: # Preparing to install PostgreSQL - name: PostgreSQL | ensure postgresql database-cluster manager package ansible.builtin.package: name: postgresql-common @@ -126,7 +183,9 @@ ansible.builtin.file: dest: /etc/logrotate.d/postgresql-common state: absent - when: installation_method == "repo" and ansible_os_family == "Debian" + when: + - installation_method == "repo" + - ansible_os_family == "Debian" tags: install_postgres # PostgreSQL prepare for install (for RHEL) @@ -155,7 +214,10 @@ delay: 5 retries: 3 environment: "{{ proxy_env | default({}) }}" - when: ansible_os_family == "RedHat" and installation_method == "repo" + when: + - installation_method == "repo" + - ansible_os_family == "RedHat" + - postgresql_packages | default('') | length > 0 tags: install_packages, install_postgres # Debian @@ -163,37 +225,28 @@ ansible.builtin.apt: name: "{{ item }}" state: present - loop: "{{ postgresql_packages }}" + loop: "{{ postgresql_packages | list }}" environment: "{{ proxy_env | default({}) }}" register: apt_status until: apt_status is success delay: 5 retries: 3 - when: ansible_os_family == "Debian" and installation_method == "repo" - tags: install_packages, install_postgres - -# timescaledb (if enable_timescale is defined) -- name: Install TimescaleDB package - ansible.builtin.package: - name: "{{ item }}" - state: present - loop: "{{ timescaledb_package }}" - vars: - timescaledb_package: >- - [{% if postgresql_version | int >= 11 %} - "timescaledb-2-postgresql-{{ postgresql_version }}" - {% else %} - "timescaledb-postgresql-{{ postgresql_version }}" - {% endif %}] - register: package_status - until: package_status is success - delay: 5 - retries: 3 - environment: "{{ proxy_env | default({}) }}" when: - installation_method == "repo" - - enable_timescale is defined - - enable_timescale | bool - tags: install_packages, install_postgres, install_timescaledb + - ansible_os_family == "Debian" + - postgresql_packages | default('') | length > 0 + tags: install_packages, install_postgres + +# Extensions +- name: Install PostgreSQL Extensions + ansible.builtin.import_tasks: extensions.yml + when: installation_method == "repo" + tags: install_packages, install_postgres, install_extensions + +# Install perf (if 'install_perf' is 'true') +- name: Install perf + ansible.builtin.import_tasks: perf.yml + when: install_perf | bool + tags: install_packages, install_perf, perf ... diff --git a/roles/packages/tasks/perf.yml b/roles/packages/tasks/perf.yml new file mode 100644 index 000000000..7fb10e5fc --- /dev/null +++ b/roles/packages/tasks/perf.yml @@ -0,0 +1,193 @@ +--- +# Install "perf" (Linux profiling with performance counters) and "FlameGraph". + +# RedHat +- name: Install perf + ansible.builtin.yum: + name: perf + state: present + disablerepo: 'pgdg*' + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: + - ansible_os_family == "RedHat" + - ansible_distribution_major_version == '7' + +- name: Install perf + ansible.builtin.dnf: + name: perf + state: present + disablerepo: 'pgdg*' + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: + - ansible_os_family == "RedHat" + - ansible_distribution_major_version >= '8' + +# Debian +- name: Install perf + ansible.builtin.apt: + name: "{{ 'linux-tools-common' if ansible_distribution == 'Ubuntu' else 'linux-perf' }}" + state: present + register: apt_status + until: apt_status is success + delay: 5 + retries: 3 + when: + - ansible_os_family == "Debian" + +# Check if perf is installed correctly, or build a perf from the source code +- name: Check if perf is installed + ansible.builtin.command: perf --version + register: perf_result + failed_when: false + changed_when: false + tags: perf + +# Build perf from source (if perf is not installed) +- block: + - name: Extract kernel version + ansible.builtin.set_fact: + kernel_version: >- + {{ ansible_kernel.split('-')[0] + if not ansible_kernel.split('-')[0].endswith('.0') + else ansible_kernel.split('-')[0][:-2] }} + kernel_major_version: "{{ ansible_kernel.split('.')[0] }}" + + - name: Download kernel source + ansible.builtin.get_url: + url: "https://mirrors.edge.kernel.org/pub/linux/kernel/v{{ kernel_major_version }}.x/linux-{{ kernel_version }}.tar.gz" + dest: "/tmp/linux-source-{{ kernel_version }}.tar.gz" + register: get_url_result + + - name: Extract kernel source + ansible.builtin.unarchive: + src: "/tmp/linux-source-{{ kernel_version }}.tar.gz" + dest: /tmp/ + remote_src: true + when: + - get_url_result is defined + - get_url_result is success + + - name: Install basic build tools + ansible.builtin.package: + name: + - make + - gcc + - flex + - bison + state: present + register: build_tools_result + when: + - get_url_result is defined + - get_url_result is success + + - name: Install required libraries + ansible.builtin.package: + name: + - pkg-config + - libzstd1 + - libdwarf-dev + - libdw-dev + - binutils-dev + - libcap-dev + - libelf-dev + - libnuma-dev + - python3-dev + - libssl-dev + - libunwind-dev + - libdwarf-dev + - zlib1g-dev + - liblzma-dev + - libaio-dev + - libtraceevent-dev + - debuginfod + - libpfm4-dev + - libslang2-dev + - systemtap-sdt-dev + - libperl-dev + - binutils-dev + - libbabeltrace-dev + - libiberty-dev + - libzstd-dev + state: present + when: + - ansible_os_family == "Debian" + - build_tools_result is defined + - build_tools_result is success + + - name: Install required libraries + ansible.builtin.package: + name: + - pkgconf + - libzstd + - libdwarf-devel + - elfutils-libelf-devel + - binutils-devel + - libcap-devel + - numactl-devel + - python3-devel + - openssl-devel + - libunwind-devel + - zlib-devel + - xz-devel + - libaio-devel + - libtraceevent-devel + - slang-devel + - systemtap-sdt-devel + - perl-devel + - libbabeltrace-devel + - libzstd-devel + state: present + when: + - ansible_os_family == "RedHat" + - build_tools_result is defined + - build_tools_result is success + + - name: Build perf from source + become: true + become_user: root + community.general.make: + chdir: "/tmp/linux-{{ kernel_version }}/tools/perf" + jobs: "{{ ansible_processor_vcpus }}" # use all CPU cores + register: build_perf_result + when: + - build_tools_result is defined + - build_tools_result is success + + - name: Copy perf to /usr/local/bin + ansible.builtin.copy: + src: "/tmp/linux-{{ kernel_version }}/tools/perf/perf" + dest: "/usr/local/bin/perf" + mode: '0755' + remote_src: true + when: + - build_perf_result is defined + - build_perf_result is success + ignore_errors: true # do not stop the playbook if perf could not be installed + when: + - perf_result.rc is defined + - perf_result.rc != 0 + tags: perf + +# FlameGraph +- block: + - name: Make sure the git are present + ansible.builtin.package: + name: git + state: present + + - name: "Download 'FlameGraph' to /var/opt/FlameGraph" + ansible.builtin.git: + repo: https://github.com/brendangregg/FlameGraph.git + dest: "/var/opt/FlameGraph" + single_branch: true + version: master + update: false + tags: flamegraph + +... diff --git a/roles/patroni/tasks/custom_wal_dir.yml b/roles/patroni/tasks/custom_wal_dir.yml index 8b63bf6cc..811f54e49 100644 --- a/roles/patroni/tasks/custom_wal_dir.yml +++ b/roles/patroni/tasks/custom_wal_dir.yml @@ -3,7 +3,7 @@ # 🔄 Determine base pg_wal_dir name - name: Roles.patroni.custom_wal_dir | Set pg_wal_dir based on postgresql_version ansible.builtin.set_fact: - pg_wal_dir: "{{ 'pg_wal' if postgresql_version is version('10', '>=') else 'pg_xlog' }}" + pg_wal_dir: "{{ 'pg_wal' if postgresql_version | int >= 10 else 'pg_xlog' }}" - name: "Make sure {{ postgresql_data_dir }}/{{ pg_wal_dir }} is symlink" ansible.builtin.stat: diff --git a/roles/patroni/tasks/main.yml b/roles/patroni/tasks/main.yml index cf54e94a4..123945fea 100644 --- a/roles/patroni/tasks/main.yml +++ b/roles/patroni/tasks/main.yml @@ -383,7 +383,7 @@ when: - postgresql_stats_temp_directory_path is defined - postgresql_stats_temp_directory_path != 'none' - - postgresql_version is version('14', '<=') + - postgresql_version | int <= 14 tags: patroni, pgsql_stats_tmp - name: Prepare PostgreSQL | mount the statistics directory in memory (tmpfs) @@ -396,7 +396,7 @@ when: - postgresql_stats_temp_directory_path is defined - postgresql_stats_temp_directory_path != 'none' - - postgresql_version is version('14', '<=') + - postgresql_version | int <= 14 tags: patroni, pgsql_stats_tmp - name: Prepare PostgreSQL | make sure the postgresql log directory "{{ postgresql_log_dir }}" exists @@ -427,16 +427,26 @@ state: directory mode: "0700" - # for Debian based distros only - # patroni bootstrap failure is possible if the postgresql config files are missing - - name: Prepare PostgreSQL | make sure the postgresql config files exists + # for Debian based distros only + # patroni bootstrap failure is possible if the PostgreSQL config files are missing + - name: Prepare PostgreSQL | make sure PostgreSQL config directory exists + ansible.builtin.file: + path: /etc/postgresql + state: directory + owner: postgres + group: postgres + recurse: true + when: ansible_os_family == "Debian" and + postgresql_packages|join(" ") is not search("postgrespro") + + - name: Prepare PostgreSQL | make sure PostgreSQL config files exists ansible.builtin.stat: path: "{{ postgresql_conf_dir }}/postgresql.conf" register: postgresql_conf_file when: ansible_os_family == "Debian" and postgresql_packages|join(" ") is not search("postgrespro") - - name: Prepare PostgreSQL | generate default postgresql config files + - name: Prepare PostgreSQL | generate default PostgreSQL config files become: true become_user: postgres ansible.builtin.command: > diff --git a/roles/patroni/templates/patroni.yml.j2 b/roles/patroni/templates/patroni.yml.j2 index e9d445b03..d23cd670c 100644 --- a/roles/patroni/templates/patroni.yml.j2 +++ b/roles/patroni/templates/patroni.yml.j2 @@ -120,19 +120,12 @@ bootstrap: {% endfor %} {% endif %} -{% if postgresql_exists|bool %} -# initdb: # List options to be passed on to initdb -# - encoding: UTF8 -# - data-checksums -{% endif %} -{% if not postgresql_exists|bool %} initdb: # List options to be passed on to initdb - encoding: {{ postgresql_encoding }} - locale: {{ postgresql_locale }} {% if postgresql_data_checksums|bool %} - data-checksums {% endif %} -{% endif %} pg_hba: # Add following lines to pg_hba.conf after running 'initdb' - host replication {{ patroni_replication_username }} 127.0.0.1/32 {{ postgresql_password_encryption_algorithm }} @@ -161,7 +154,7 @@ postgresql: # password: rewind_password parameters: unix_socket_directories: {{ postgresql_unix_socket_dir }} -{% if postgresql_stats_temp_directory_path is defined and postgresql_stats_temp_directory_path != 'none' and postgresql_version is version('14', '<=') %} +{% if postgresql_stats_temp_directory_path is defined and postgresql_stats_temp_directory_path != 'none' and postgresql_version | int <= 14 %} stats_temp_directory: {{ postgresql_stats_temp_directory_path }} {% endif %} diff --git a/roles/pgbackrest/stanza-create/tasks/main.yml b/roles/pgbackrest/stanza-create/tasks/main.yml index 334393eda..c1947e0d0 100644 --- a/roles/pgbackrest/stanza-create/tasks/main.yml +++ b/roles/pgbackrest/stanza-create/tasks/main.yml @@ -5,6 +5,7 @@ - name: Get repo1-path value ansible.builtin.set_fact: repo1_path: "{{ pgbackrest_conf['global'] | selectattr('option', 'equalto', 'repo1-path') | map(attribute='value') | list | first }}" + when: pgbackrest_repo_type | lower == 'posix' - name: "Make sure the {{ repo1_path }} directory exists" ansible.builtin.file: @@ -13,7 +14,7 @@ owner: postgres group: postgres mode: "0750" - when: repo1_path | length > 0 + when: repo1_path | default('') | length > 0 - name: Create stanza "{{ pgbackrest_stanza }}" become: true @@ -38,6 +39,7 @@ run_once: true ansible.builtin.set_fact: repo1_path: "{{ pgbackrest_server_conf['global'] | selectattr('option', 'equalto', 'repo1-path') | map(attribute='value') | list | first }}" + when: pgbackrest_repo_type | lower == 'posix' - name: "Make sure the {{ repo1_path }} directory exists" delegate_to: "{{ groups['pgbackrest'][0] }}" @@ -48,7 +50,7 @@ owner: "{{ pgbackrest_repo_user }}" group: "{{ pgbackrest_repo_user }}" mode: "0750" - when: repo1_path | length > 0 + when: repo1_path | default('') | length > 0 - name: Create stanza "{{ pgbackrest_stanza }}" become: true diff --git a/roles/pgbackrest/tasks/auto_conf.yml b/roles/pgbackrest/tasks/auto_conf.yml new file mode 100644 index 000000000..96013b988 --- /dev/null +++ b/roles/pgbackrest/tasks/auto_conf.yml @@ -0,0 +1,175 @@ +# yamllint disable rule:line-length +--- + +# AWS S3 bucket (if 'cloud_provider=aws') +- name: "Set variable 'pgbackrest_conf' for backup in AWS S3 bucket" + ansible.builtin.set_fact: + pgbackrest_conf: + global: + - { option: "log-level-file", value: "detail" } + - { option: "log-path", value: "/var/log/pgbackrest" } + - { option: "repo1-type", value: "s3" } + - { option: "repo1-path", value: "{{ PGBACKREST_REPO_PATH | default('/pgbackrest') }}" } + - { option: "repo1-s3-key", value: "{{ PGBACKREST_S3_KEY | default(lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID')) }}" } + - { option: "repo1-s3-key-secret", value: "{{ PGBACKREST_S3_KEY_SECRET | default(lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY')) }}" } + - { option: "repo1-s3-bucket", value: "{{ PGBACKREST_S3_BUCKET | default(aws_s3_bucket_name | default(patroni_cluster_name + '-backup')) }}" } + - { option: "repo1-s3-endpoint", value: "{{ PGBACKREST_S3_ENDPOINT | default('s3.' + (aws_s3_bucket_region | default(server_location)) + '.amazonaws.com') }}" } + - { option: "repo1-s3-region", value: "{{ PGBACKREST_S3_REGION | default(aws_s3_bucket_region | default(server_location)) }}" } + - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" } + - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" } + - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" } + - { option: "repo1-bundle", value: "y" } + - { option: "repo1-block", value: "y" } + - { option: "start-fast", value: "y" } + - { option: "stop-auto", value: "y" } + - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } + - { option: "archive-async", value: "y" } + - { option: "archive-get-queue-max", value: "1GiB" } + - { option: "spool-path", value: "/var/spool/pgbackrest" } + - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "backup-standby", value: "{{ 'y' if groups['postgres_cluster'] | length > 1 else 'n' }}" } + stanza: + - { option: "log-level-console", value: "info" } + - { option: "recovery-option", value: "recovery_target_action=promote" } + - { option: "pg1-path", value: "{{ postgresql_data_dir }}" } + delegate_to: localhost + run_once: true # noqa run-once + no_log: true # do not output contents to the ansible log + when: cloud_provider | default('') | lower == 'aws' + +# GCS Bucket (if 'cloud_provider=gcp') +- block: + - name: "Set variable 'pgbackrest_conf' for backup in GCS Bucket" + ansible.builtin.set_fact: + pgbackrest_conf: + global: + - { option: "log-level-file", value: "detail" } + - { option: "log-path", value: "/var/log/pgbackrest" } + - { option: "repo1-type", value: "gcs" } + - { option: "repo1-path", value: "{{ PGBACKREST_REPO_PATH | default('/pgbackrest') }}" } + - { option: "repo1-gcs-key", value: "{{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" } + - { option: "repo1-gcs-bucket", value: "{{ PGBACKREST_GCS_BUCKET | default(gcp_bucket_name | default(patroni_cluster_name + '-backup')) }}" } + - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" } + - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" } + - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" } + - { option: "repo1-bundle", value: "y" } + - { option: "repo1-block", value: "y" } + - { option: "start-fast", value: "y" } + - { option: "stop-auto", value: "y" } + - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } + - { option: "archive-async", value: "y" } + - { option: "archive-get-queue-max", value: "1GiB" } + - { option: "spool-path", value: "/var/spool/pgbackrest" } + - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "backup-standby", value: "{{ 'y' if groups['postgres_cluster'] | length > 1 else 'n' }}" } + stanza: + - { option: "log-level-console", value: "info" } + - { option: "recovery-option", value: "recovery_target_action=promote" } + - { option: "pg1-path", value: "{{ postgresql_data_dir }}" } + no_log: true # do not output contents to the ansible log + + # if 'gcs_key_file' is not defined, copy GCS key file from GCP_SERVICE_ACCOUNT_CONTENTS environment variable. + - block: + - name: "Get GCP service account contents from localhost" + ansible.builtin.set_fact: + gcp_service_account_contents: "{{ lookup('ansible.builtin.env', 'GCP_SERVICE_ACCOUNT_CONTENTS') }}" + delegate_to: localhost + run_once: true # noqa run-once + no_log: true # do not output GCP service account contents to the ansible log + + - name: "Copy GCP service account contents to {{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + content: "{{ gcp_service_account_contents }}" + dest: "{{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is not defined + + # if 'gcs_key_file' is defined, copy this GCS key file. + - name: "Copy GCS key file to {{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + src: "{{ gcs_key_file }}" + dest: "{{ PGBACKREST_GCS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is defined and gcs_key_file | length > 0 + when: cloud_provider | default('') | lower == 'gcp' + +# Azure Blob Storage (if 'cloud_provider=azure') +- name: "Set variable 'pgbackrest_conf' for backup in Azure Blob Storage" + ansible.builtin.set_fact: + pgbackrest_conf: + global: + - { option: "log-level-file", value: "detail" } + - { option: "log-path", value: "/var/log/pgbackrest" } + - { option: "repo1-type", value: "azure" } + - { option: "repo1-path", value: "{{ PGBACKREST_REPO_PATH | default('/pgbackrest') }}" } + - { option: "repo1-azure-key", value: "{{ PGBACKREST_AZURE_KEY | default(hostvars['localhost']['azure_storage_account_key'] | default('')) }}" } + - { option: "repo1-azure-key-type", value: "{{ PGBACKREST_AZURE_KEY_TYPE | default('shared') }}" } + - { option: "repo1-azure-account", value: "{{ PGBACKREST_AZURE_ACCOUNT | default(azure_blob_storage_account_name | default(patroni_cluster_name | lower | replace('-', '') | truncate(24, true, ''))) }}" } + - { option: "repo1-azure-container", value: "{{ PGBACKREST_AZURE_CONTAINER | default(azure_blob_storage_name | default(patroni_cluster_name + '-backup')) }}" } + - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" } + - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" } + - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" } + - { option: "repo1-bundle", value: "y" } + - { option: "repo1-block", value: "y" } + - { option: "start-fast", value: "y" } + - { option: "stop-auto", value: "y" } + - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } + - { option: "archive-async", value: "y" } + - { option: "archive-get-queue-max", value: "1GiB" } + - { option: "spool-path", value: "/var/spool/pgbackrest" } + - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "backup-standby", value: "{{ 'y' if groups['postgres_cluster'] | length > 1 else 'n' }}" } + stanza: + - { option: "log-level-console", value: "info" } + - { option: "recovery-option", value: "recovery_target_action=promote" } + - { option: "pg1-path", value: "{{ postgresql_data_dir }}" } + no_log: true # do not output contents to the ansible log + when: cloud_provider | default('') | lower == 'azure' + +# DigitalOcean Spaces Object Storage (if 'cloud_provider=digitalocean') +# Note: requires the Spaces access keys "AWS_ACCESS_KEY_ID" and "AWS_SECRET_ACCESS_KEY" (https://cloud.digitalocean.com/account/api/spaces) +- name: "Set variable 'pgbackrest_conf' for backup in DigitalOcean Spaces Object Storage" + ansible.builtin.set_fact: + pgbackrest_conf: + global: + - { option: "log-level-file", value: "detail" } + - { option: "log-path", value: "/var/log/pgbackrest" } + - { option: "repo1-type", value: "s3" } + - { option: "repo1-path", value: "{{ PGBACKREST_REPO_PATH | default('/pgbackrest') }}" } + - { option: "repo1-s3-key", value: "{{ PGBACKREST_S3_KEY | default(AWS_ACCESS_KEY_ID | default('')) }}" } + - { option: "repo1-s3-key-secret", value: "{{ PGBACKREST_S3_KEY_SECRET | default(AWS_SECRET_ACCESS_KEY | default('')) }}" } + - { option: "repo1-s3-bucket", value: "{{ PGBACKREST_S3_BUCKET | default(digital_ocean_spaces_name | default(patroni_cluster_name + '-backup')) }}" } + - { option: "repo1-s3-endpoint", value: "{{ PGBACKREST_S3_ENDPOINT | default('https://' + (digital_ocean_spaces_region | default(server_location)) + '.digitaloceanspaces.com') }}" } + - { option: "repo1-s3-region", value: "{{ PGBACKREST_S3_REGION | default(digital_ocean_spaces_region | default(server_location)) }}" } + - { option: "repo1-s3-uri-style", value: "{{ PGBACKREST_S3_URI_STYLE | default('path') }}" } + - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" } + - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" } + - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" } + - { option: "repo1-bundle", value: "y" } + - { option: "repo1-block", value: "y" } + - { option: "start-fast", value: "y" } + - { option: "stop-auto", value: "y" } + - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } + - { option: "archive-async", value: "y" } + - { option: "archive-get-queue-max", value: "1GiB" } + - { option: "spool-path", value: "/var/spool/pgbackrest" } + - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "backup-standby", value: "{{ 'y' if groups['postgres_cluster'] | length > 1 else 'n' }}" } + stanza: + - { option: "log-level-console", value: "info" } + - { option: "recovery-option", value: "recovery_target_action=promote" } + - { option: "pg1-path", value: "{{ postgresql_data_dir }}" } + no_log: true # do not output contents to the ansible log + when: cloud_provider | default('') | lower == 'digitalocean' + +... diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml index ce781b9cb..73c083cf0 100644 --- a/roles/pgbackrest/tasks/main.yml +++ b/roles/pgbackrest/tasks/main.yml @@ -1,5 +1,13 @@ --- +# Automatic setup of the backup configuration based on the selected cloud provider. +# if 'cloud_provider' is 'aws', 'gcp', 'azure', 'digitalocean'. +- ansible.builtin.import_tasks: auto_conf.yml + when: + - cloud_provider | default('') | length > 0 + - pgbackrest_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: pgbackrest, pgbackrest_conf + - block: # Debian pgdg repo - name: Make sure the gnupg, apt-transport-https and python3-debian packages are present ansible.builtin.apt: diff --git a/roles/pre-checks/tasks/extensions.yml b/roles/pre-checks/tasks/extensions.yml new file mode 100644 index 000000000..14a1a497f --- /dev/null +++ b/roles/pre-checks/tasks/extensions.yml @@ -0,0 +1,63 @@ +# yamllint disable rule:line-length +--- + +# TimescaleDB pre-check (if 'enable_timescale' is 'true') +- name: TimescaleDB | Checking PostgreSQL version + run_once: true + ansible.builtin.fail: + msg: + - "The current PostgreSQL version ({{ postgresql_version }}) is not supported by the TimescaleDB." + - "PostgreSQL version must be {{ timescale_minimal_pg_version }} or higher." + when: postgresql_version|string is version(timescale_minimal_pg_version|string, '<') + +# Extension Auto-Setup: shared_preload_libraries +- name: Create a list of extensions + run_once: true + ansible.builtin.set_fact: + extensions: >- + {{ (extensions | default([])) + + (['timescaledb'] if ((enable_timescale | default(false) | bool) or (enable_timescaledb | default(false) | bool)) and (postgresql_version is version('15', '<=')) else []) + + (['citus'] if (enable_citus | default(false) | bool) and postgresql_version | int >= 11 else []) + + (['pg_cron'] if (enable_pg_cron | default(false) | bool) else []) + + (['pgaudit'] if (enable_pgaudit | default(false) | bool) else []) + + (['pg_stat_statements'] if (enable_pg_stat_kcache | default(false) | bool) else []) + + (['pg_stat_kcache'] if (enable_pg_stat_kcache | default(false) | bool) else []) + + (['pg_wait_sampling'] if (enable_pg_wait_sampling | default(false) | bool) else []) + + (['pg_partman_bgw'] if (enable_pg_partman | default(false) | bool) else []) + }} + +- name: Add required extensions to 'shared_preload_libraries' (if missing) + run_once: true + ansible.builtin.set_fact: + # This complex line does several things: + # 1. It takes the current list of PostgreSQL parameters, + # 2. Removes any item where the option is 'shared_preload_libraries', + # 3. Then appends a new 'shared_preload_libraries' item at the end. + # The new value of this item is based on whether extension is already present in the old value. + # If it is not present, it appends ',' to the old value. Otherwise, it leaves the value unchanged. + postgresql_parameters: >- + {{ postgresql_parameters | rejectattr('option', 'equalto', 'shared_preload_libraries') | list + + [{'option': 'shared_preload_libraries', 'value': new_value}] }} + vars: + # Find the last item in postgresql_parameters where the option is 'shared_preload_libraries' + shared_preload_libraries_item: >- + {{ + postgresql_parameters + | selectattr('option', 'equalto', 'shared_preload_libraries') + | list | last | default({'value': ''}) + }} + # Ensure that all required extensions are added to the 'shared_preload_libraries' parameter. + # 1. If the 'citus' extension is not yet added to 'shared_preload_libraries', it's added to the beginning of the list. + # This is necessary as 'citus' needs to be first in the list as per Citus documentation. + # 2. For all other extensions: if they are not yet added, they are appended to the end of the list. + new_value: >- + {{ + (item ~ ',' ~ shared_preload_libraries_item.value if item == 'citus' and item not in shared_preload_libraries_item.value.split(',') else + (shared_preload_libraries_item.value ~ (',' if shared_preload_libraries_item.value else '') + if item not in shared_preload_libraries_item.value.split(',') else shared_preload_libraries_item.value)) + ~ (item if item not in shared_preload_libraries_item.value.split(',') and item != 'citus' else '') + }} + loop: "{{ extensions | default([]) | unique }}" + when: extensions | default([]) | length > 0 + +... diff --git a/roles/pre-checks/tasks/main.yml b/roles/pre-checks/tasks/main.yml index cf8079049..b95235850 100644 --- a/roles/pre-checks/tasks/main.yml +++ b/roles/pre-checks/tasks/main.yml @@ -17,13 +17,6 @@ msg: "{{ ansible_distribution_version }} of {{ ansible_distribution }} is not supported" when: ansible_distribution_version is version_compare(os_minimum_versions[ansible_distribution], '<') -- name: Perform pre-checks for timescaledb - ansible.builtin.import_tasks: timescaledb.yml - when: - - enable_timescale is defined - - enable_timescale | bool - - inventory_hostname in groups['postgres_cluster'] - - name: Perform pre-checks for pgbouncer ansible.builtin.import_tasks: pgbouncer.yml when: @@ -54,3 +47,11 @@ - wal_g_install is defined - wal_g_install | bool - inventory_hostname in groups['postgres_cluster'] + +- name: Perform pre-checks for extensions + ansible.builtin.import_tasks: extensions.yml + when: inventory_hostname in groups['postgres_cluster'] + +- name: Generate passwords + ansible.builtin.import_tasks: passwords.yml + when: inventory_hostname in groups['postgres_cluster'] diff --git a/roles/pre-checks/tasks/passwords.yml b/roles/pre-checks/tasks/passwords.yml new file mode 100644 index 000000000..072c75c40 --- /dev/null +++ b/roles/pre-checks/tasks/passwords.yml @@ -0,0 +1,95 @@ +--- +# Generate passwords (if not defined) +- block: + - name: Generate a password for patroni superuser + ansible.builtin.set_fact: + patroni_superuser_password: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}" + run_once: true + delegate_to: "{{ groups['master'][0] }}" + when: patroni_superuser_password | default('') | length < 1 + + - name: Generate a password for patroni replication user + ansible.builtin.set_fact: + patroni_replication_password: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}" + run_once: true + delegate_to: "{{ groups['master'][0] }}" + when: patroni_replication_password | default('') | length < 1 + + - name: Generate a password for patroni restapi + ansible.builtin.set_fact: + patroni_restapi_password: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}" + run_once: true + delegate_to: "{{ groups['master'][0] }}" + when: patroni_restapi_password | default('') | length < 1 + + - name: Generate a password for pgbouncer auth user + ansible.builtin.set_fact: + pgbouncer_auth_password: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}" + run_once: true + delegate_to: "{{ groups['master'][0] }}" + when: + - pgbouncer_install | bool + - pgbouncer_auth_user | bool + - pgbouncer_auth_password | default('') | length < 1 + when: + - not (postgresql_cluster_maintenance | default(false) | bool) # exclude for config_pgcluster.yml + - not (new_node | default(false) | bool) # exclude for add_pgnode.yml + +# Get current passwords (if not defined) - for config_pgcluster.yml +- block: + - name: Get patroni superuser password + ansible.builtin.shell: | + set -o pipefail; + grep -A10 "authentication:" /etc/patroni/patroni.yml | \ + grep -A3 "superuser" | grep "password:" | awk '{ print $2 }' | tail -n 1 + args: + executable: /bin/bash + run_once: true + delegate_to: "{{ groups['master'][0] }}" + register: superuser_password_result + changed_when: false # This tells Ansible that this task doesn't change anything + when: patroni_superuser_password | default('') | length < 1 + + - name: Get patroni replication user password + ansible.builtin.shell: | + set -o pipefail; + grep -A10 "authentication:" /etc/patroni/patroni.yml | \ + grep -A3 "replication" | grep "password:" | awk '{ print $2 }' | tail -n 1 + args: + executable: /bin/bash + run_once: true + delegate_to: "{{ groups['master'][0] }}" + register: replication_password_result + changed_when: false # This tells Ansible that this task doesn't change anything + when: patroni_replication_password | default('') | length < 1 + + - name: Get patroni restapi password + ansible.builtin.shell: | + set -o pipefail; + grep -A10 "restapi:" /etc/patroni/patroni.yml | \ + grep -A3 "authentication" | grep "password:" | awk '{ print $2 }' | tail -n 1 + args: + executable: /bin/bash + run_once: true + delegate_to: "{{ groups['master'][0] }}" + register: patroni_restapi_password_result + changed_when: false # This tells Ansible that this task doesn't change anything + when: patroni_restapi_password | default('') | length < 1 + + - name: "Set variable: patroni_superuser_password" + ansible.builtin.set_fact: + patroni_superuser_password: "{{ superuser_password_result.stdout }}" + when: superuser_password_result.stdout is defined + + - name: "Set variable: patroni_replication_password" + ansible.builtin.set_fact: + patroni_replication_password: "{{ replication_password_result.stdout }}" + when: replication_password_result.stdout is defined + + - name: "Set variable: patroni_restapi_password" + ansible.builtin.set_fact: + patroni_restapi_password: "{{ patroni_restapi_password_result.stdout }}" + when: patroni_restapi_password_result.stdout is defined + when: + - (postgresql_cluster_maintenance | default(false) | bool) + or (new_node | default(false) | bool) # include for add_pgnode.yml diff --git a/roles/pre-checks/tasks/timescaledb.yml b/roles/pre-checks/tasks/timescaledb.yml deleted file mode 100644 index 39a344398..000000000 --- a/roles/pre-checks/tasks/timescaledb.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- - -- name: TimescaleDB | Checking PostgreSQL version - run_once: true - ansible.builtin.fail: - msg: - - "The current PostgreSQL version ({{ postgresql_version }}) is not supported by the TimescaleDB." - - "PostgreSQL version must be {{ timescale_minimal_pg_version }} or higher." - when: - - postgresql_version|string is version(timescale_minimal_pg_version|string, '<') - -- block: - - name: TimescaleDB | Ensure 'timescaledb' is in 'shared_preload_libraries' - ansible.builtin.set_fact: - # This complex line does several things: - # 1. It takes the current list of PostgreSQL parameters, - # 2. Removes any item where the option is 'shared_preload_libraries', - # 3. Then appends a new 'shared_preload_libraries' item at the end. - # The new value of this item is based on whether 'timescaledb' is already present in the old value. - # If it is not present, it appends ',timescaledb' to the old value. Otherwise, it leaves the value unchanged. - postgresql_parameters: >- - {{ postgresql_parameters | rejectattr('option', 'equalto', 'shared_preload_libraries') | list - + [{'option': 'shared_preload_libraries', 'value': new_value}] }} - vars: - # Find the last item in postgresql_parameters where the option is 'shared_preload_libraries' - shared_preload_libraries_item: >- - {{ - postgresql_parameters - | selectattr('option', 'equalto', 'shared_preload_libraries') - | list | last | default({'value': ''}) - }} - # Determine the new value based on whether 'timescaledb' is already present - new_value: >- - {{ - (shared_preload_libraries_item.value ~ (',' if shared_preload_libraries_item.value else '') - if 'timescaledb' not in shared_preload_libraries_item.value.split(',') else shared_preload_libraries_item.value) - ~ ('timescaledb' if 'timescaledb' not in shared_preload_libraries_item.value.split(',') else '') - }} - when: - - enable_timescale is defined - - enable_timescale | bool diff --git a/roles/upgrade/tasks/maintenance_disable.yml b/roles/upgrade/tasks/maintenance_disable.yml index 65936259b..23a763f0d 100644 --- a/roles/upgrade/tasks/maintenance_disable.yml +++ b/roles/upgrade/tasks/maintenance_disable.yml @@ -27,6 +27,8 @@ delegate_to: "{{ item }}" loop: "{{ groups.balancers | default([]) | list }}" run_once: true + when: dcs_type == "etcd" + become: true become_user: root ignore_errors: true # show the error and continue the playbook execution diff --git a/roles/upgrade/tasks/packages.yml b/roles/upgrade/tasks/packages.yml index 72b8bdc08..b541210cb 100644 --- a/roles/upgrade/tasks/packages.yml +++ b/roles/upgrade/tasks/packages.yml @@ -1,13 +1,17 @@ --- -- name: Clean dnf cache - ansible.builtin.command: dnf clean all +# Update dnf cache +- name: Update dnf cache + ansible.builtin.shell: dnf clean all && dnf -y makecache + args: + executable: /bin/bash register: dnf_status until: dnf_status is success delay: 5 retries: 3 when: ansible_os_family == "RedHat" +# Update apt cache - name: Update apt cache ansible.builtin.apt: update_cache: true @@ -18,6 +22,7 @@ retries: 3 when: ansible_os_family == "Debian" +# Install PostgreSQL packages - name: "Install PostgreSQL {{ pg_new_version }} packages" ansible.builtin.package: name: "{{ item }}" @@ -28,8 +33,11 @@ delay: 5 retries: 3 -# timescaledb (if enable_timescale is defined) -- name: Install TimescaleDB package for PostgreSQL {{ pg_new_version }} +# Extension Auto-Setup: new packages +#################################### + +# TimescaleDB (if 'enable_timescale' is 'true') +- name: "Install TimescaleDB package for PostgreSQL {{ pg_new_version }}" ansible.builtin.package: name: "{{ item }}" state: latest @@ -45,8 +53,212 @@ until: package_status is success delay: 5 retries: 3 + when: (enable_timescale | default(false) | bool) or (enable_timescaledb | default(false) | bool) + +# Citus (if 'enable_citus' is 'true') +- name: "Install Citus package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ item }}" + state: latest + loop: "{{ citus_package }}" + vars: + citus_package: >- + [{% if ansible_os_family == 'Debian' and pg_new_version | int >= 14 %} + "postgresql-{{ pg_new_version }}-citus-{{ citus_version | default('12.1') }}" + {% elif ansible_os_family == 'Debian' and pg_new_version | int == 13 %} + "postgresql-{{ pg_new_version }}-citus-11.3" + {% elif ansible_os_family == 'Debian' and pg_new_version | int == 12 %} + "postgresql-{{ pg_new_version }}-citus-10.2" + {% elif ansible_os_family == 'Debian' and pg_new_version | int == 11 %} + "postgresql-{{ pg_new_version }}-citus-10.0" + {% else %} + "citus_{{ pg_new_version }}" + {% endif %}] + register: package_status + until: package_status is success + delay: 5 + retries: 3 when: - - enable_timescale is defined - - enable_timescale | bool + - enable_citus | default(false) | bool + - (ansible_os_family == 'Debian' and pg_new_version | int >= 11) or + (ansible_os_family == 'RedHat' and pg_new_version | int >= 12) + +# pg_repack (if 'enable_pg_repack' is 'true') +- name: "Install pg_repack package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pg_repack_package }}" + state: latest + vars: + pg_repack_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-repack + {% else %} + pg_repack_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_repack | default(false) | bool + +# pg_cron (if 'enable_pg_cron' is 'true') +- name: "Install pg_cron package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pg_cron_package }}" + state: latest + vars: + pg_cron_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-cron + {% else %} + pg_cron_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_cron | default(false) | bool + +# pgaudit (if 'enable_pgaudit' is 'true') +- name: "Install pgaudit package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pgaudit_package }}" + state: latest + vars: + pgaudit_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-pgaudit + {% elif ansible_os_family == 'RedHat' and pg_new_version | int >= 16 %} + pgaudit_{{ pg_new_version }} + {% elif ansible_os_family == 'RedHat' and pg_new_version | int == 15 %} + pgaudit17_{{ pg_new_version }} + {% elif ansible_os_family == 'RedHat' and pg_new_version | int == 14 %} + pgaudit16_{{ pg_new_version }} + {% elif ansible_os_family == 'RedHat' and pg_new_version | int == 13 %} + pgaudit15_{{ pg_new_version }} + {% elif ansible_os_family == 'RedHat' and pg_new_version | int == 12 %} + pgaudit14_{{ pg_new_version }} + {% elif ansible_os_family == 'RedHat' and pg_new_version | int == 11 %} + pgaudit13_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pgaudit | default(false) | bool + +# pgvector (if 'enable_pgvector' is 'true') +- name: "Install pgvector package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pgvector_package }}" + state: latest + vars: + pgvector_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-pgvector + {% else %} + pgvector_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: + - enable_pgvector | default(false)| bool + - (ansible_os_family == 'Debian' and pg_new_version | int >= 11) or + (ansible_os_family == 'RedHat' and pg_new_version | int >= 12) + +# postgis (if 'enable_postgis' is 'true') +- name: "Install postgis package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ postgis_package }}" + state: latest + vars: + postgis_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-postgis-3 + {% elif ansible_os_family == 'RedHat' and pg_new_version | int == 16 %} + postgis34_{{ pg_new_version }} + {% else %} + postgis33_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_postgis | default(false) | bool + +# pgrouting (if 'enable_pgrouting' is 'true') +- name: "Install pgrouting package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pgrouting_package }}" + state: latest + vars: + pgrouting_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-pgrouting + {% else %} + pgrouting_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pgrouting | default(false) | bool and + not (ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('20.04', '<=')) + +# pg_stat_kcache (if 'enable_pg_stat_kcache' is 'true') +- name: "Install pg_stat_kcache package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pg_stat_kcache_package }}" + state: latest + vars: + pg_stat_kcache_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-pg-stat-kcache + {% else %} + pg_stat_kcache_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_stat_kcache | default(false) | bool + +# pg_wait_sampling (if 'enable_pg_wait_sampling' is 'true') +- name: "Install pg_wait_sampling package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pg_wait_sampling_package }}" + state: latest + vars: + pg_wait_sampling_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-pg-wait-sampling + {% else %} + pg_wait_sampling_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_wait_sampling | default(false) | bool + +# pg_partman (if 'enable_pg_partman' is 'true') +- name: "Install pg_partman package for PostgreSQL {{ pg_new_version }}" + ansible.builtin.package: + name: "{{ pg_partman_package }}" + state: latest + vars: + pg_partman_package: >- + {% if ansible_os_family == 'Debian' %} + postgresql-{{ pg_new_version }}-partman + {% else %} + pg_partman_{{ pg_new_version }} + {% endif %} + register: package_status + until: package_status is success + delay: 5 + retries: 3 + when: enable_pg_partman | default(false) | bool ... diff --git a/roles/upgrade/tasks/upgrade_check.yml b/roles/upgrade/tasks/upgrade_check.yml index 6fc42b605..ea16e0a49 100644 --- a/roles/upgrade/tasks/upgrade_check.yml +++ b/roles/upgrade/tasks/upgrade_check.yml @@ -31,7 +31,7 @@ --link --check args: - chdir: "{{ pg_upper_datadir }}" + chdir: "/tmp" vars: shared_preload_libraries: "-c shared_preload_libraries='{{ pg_shared_preload_libraries_value }}'" timescaledb_restoring: "{{ \"-c timescaledb.restoring='on'\" if 'timescaledb' in pg_shared_preload_libraries_value else '' }}" diff --git a/roles/upgrade/tasks/upgrade_primary.yml b/roles/upgrade/tasks/upgrade_primary.yml index 0c912ff2f..bbb3b332a 100644 --- a/roles/upgrade/tasks/upgrade_primary.yml +++ b/roles/upgrade/tasks/upgrade_primary.yml @@ -16,7 +16,7 @@ --jobs={{ ansible_processor_vcpus }} --link args: - chdir: "{{ pg_upper_datadir }}" + chdir: "/tmp" vars: shared_preload_libraries: "-c shared_preload_libraries='{{ pg_shared_preload_libraries_value }}'" timescaledb_restoring: "{{ \"-c timescaledb.restoring='on'\" if 'timescaledb' in pg_shared_preload_libraries_value else '' }}" diff --git a/roles/wal-g/tasks/auto_conf.yml b/roles/wal-g/tasks/auto_conf.yml new file mode 100644 index 000000000..54de39ff7 --- /dev/null +++ b/roles/wal-g/tasks/auto_conf.yml @@ -0,0 +1,117 @@ +# yamllint disable rule:line-length +--- + +# AWS S3 bucket (if 'cloud_provider=aws') +- name: "Set variable 'wal_g_json' for backup in AWS S3 bucket" + ansible.builtin.set_fact: + wal_g_json: + - { option: "AWS_ACCESS_KEY_ID", value: "{{ WALG_AWS_ACCESS_KEY_ID | default(lookup('ansible.builtin.env', 'AWS_ACCESS_KEY_ID')) }}" } + - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ WALG_AWS_SECRET_ACCESS_KEY | default(lookup('ansible.builtin.env', 'AWS_SECRET_ACCESS_KEY')) }}" } + - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://' + (aws_s3_bucket_name | default(patroni_cluster_name + '-backup'))) }}" } + - { option: "AWS_REGION", value: "{{ WALG_AWS_REGION | default(aws_s3_bucket_region | default(server_location)) }}" } + - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } + - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } + - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } + - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" } + - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" } + - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" } + delegate_to: localhost + run_once: true # noqa run-once + no_log: true # do not output contents to the ansible log + when: cloud_provider | default('') | lower == 'aws' + +# GCS Bucket (if 'cloud_provider=gcp') +- block: + - name: "Set variable 'wal_g_json' for backup in GCS Bucket" + ansible.builtin.set_fact: + wal_g_json: + - { option: "GOOGLE_APPLICATION_CREDENTIALS", value: "{{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" } + - { option: "WALG_GS_PREFIX", value: "{{ WALG_GS_PREFIX | default('gs://' + (gcp_bucket_name | default(patroni_cluster_name + '-backup'))) }}" } + - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } + - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } + - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } + - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" } + - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" } + - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" } + no_log: true # do not output contents to the ansible log + + # if 'gcs_key_file' is not defined, copy GCS key file from GCP_SERVICE_ACCOUNT_CONTENTS environment variable. + - block: + - name: "Get GCP service account contents from localhost" + ansible.builtin.set_fact: + gcp_service_account_contents: "{{ lookup('ansible.builtin.env', 'GCP_SERVICE_ACCOUNT_CONTENTS') }}" + delegate_to: localhost + run_once: true # noqa run-once + no_log: true # do not output GCP service account contents to the ansible log + + - name: "Copy GCP service account contents to {{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + content: "{{ gcp_service_account_contents }}" + dest: "{{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is not defined + + # if 'gcs_key_file' is defined, copy this GCS key file. + - name: "Copy GCS key file to {{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + ansible.builtin.copy: + src: "{{ gcs_key_file }}" + dest: "{{ WALG_GS_KEY | default(postgresql_home_dir + '/gcs-key.json') }}" + mode: '0600' + owner: "postgres" + group: "postgres" + no_log: true # do not output GCP service account contents to the ansible log + when: gcs_key_file is defined and gcs_key_file | length > 0 + when: cloud_provider | default('') | lower == 'gcp' + +# Azure Blob Storage (if 'cloud_provider=azure') +- name: "Set variable 'wal_g_json' for backup in Azure Blob Storage" + ansible.builtin.set_fact: + wal_g_json: + - { option: "AZURE_STORAGE_ACCOUNT", value: "{{ WALG_AZURE_STORAGE_ACCOUNT | default(azure_blob_storage_account_name | default(patroni_cluster_name | lower | replace('-', '') | truncate(24, true, ''))) }}" } + - { option: "AZURE_STORAGE_ACCESS_KEY", value: "{{ WALG_AZURE_STORAGE_ACCESS_KEY | default(hostvars['localhost']['azure_storage_account_key'] | default('')) }}" } + - { option: "WALG_AZ_PREFIX", value: "{{ WALG_AZ_PREFIX | default('azure://' + (azure_blob_storage_name | default(patroni_cluster_name + '-backup'))) }}" } + - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } + - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } + - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } + - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" } + - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" } + - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" } + no_log: true # do not output contents to the ansible log + when: cloud_provider | default('') | lower == 'azure' + +# DigitalOcean Spaces Object Storage (if 'cloud_provider=digitalocean') +# Note: requires the Spaces access keys "AWS_ACCESS_KEY_ID" and "AWS_SECRET_ACCESS_KEY" (https://cloud.digitalocean.com/account/api/spaces) +- name: "Set variable 'wal_g_json' for backup in DigitalOcean Spaces Object Storage" + ansible.builtin.set_fact: + wal_g_json: + - { option: "AWS_ACCESS_KEY_ID", value: "{{ WALG_AWS_ACCESS_KEY_ID | default(AWS_ACCESS_KEY_ID | default('')) }}" } + - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ WALG_AWS_SECRET_ACCESS_KEY | default(AWS_SECRET_ACCESS_KEY | default('')) }}" } + - { option: "AWS_ENDPOINT", value: "{{ WALG_S3_ENDPOINT | default('https://' + (digital_ocean_spaces_region | default(server_location)) + '.digitaloceanspaces.com') }}" } + - { option: "AWS_REGION", value: "{{ WALG_S3_REGION | default(digital_ocean_spaces_region | default(server_location)) }}" } + - { option: "AWS_S3_FORCE_PATH_STYLE", value: "{{ AWS_S3_FORCE_PATH_STYLE | default(true) }}" } + - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://' + (digital_ocean_spaces_name | default(patroni_cluster_name + '-backup'))) }}" } + - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } + - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } + - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" } + - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } + - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" } + - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" } + - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" } + no_log: true # do not output contents to the ansible log + when: cloud_provider | default('') | lower == 'digitalocean' + +... diff --git a/roles/wal-g/tasks/main.yml b/roles/wal-g/tasks/main.yml index 2de011c17..1326ec97b 100644 --- a/roles/wal-g/tasks/main.yml +++ b/roles/wal-g/tasks/main.yml @@ -1,5 +1,13 @@ --- +# Automatic setup of the backup configuration based on the selected cloud provider. +# if 'cloud_provider' is 'aws', 'gcp', 'azure', 'digitalocean'. +- ansible.builtin.import_tasks: auto_conf.yml + when: + - cloud_provider | default('') | length > 0 + - wal_g_auto_conf | default(true) | bool # to be able to disable auto backup settings + tags: wal-g, wal_g, wal_g_conf + - name: Check if WAL-G is already installed ansible.builtin.shell: | set -o pipefail; diff --git a/tags.md b/tags.md index b42721af3..010fcf3df 100644 --- a/tags.md +++ b/tags.md @@ -104,3 +104,6 @@ - - pg_probackup_install - cron - netdata +- ssh_public_keys +- mount, zpool +- perf, flamegraph diff --git a/vars/Debian.yml b/vars/Debian.yml index 5cf67a790..daead7640 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -3,7 +3,15 @@ # PostgreSQL variables postgresql_cluster_name: "main" # You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/main" -postgresql_data_dir: "/var/lib/postgresql/{{ postgresql_version }}/{{ postgresql_cluster_name }}" +postgresql_data_dir: "\ + {% if cloud_provider | default('') | length > 0 %}\ + {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/{{ postgresql_cluster_name }}\ + {% else %}\ + /var/lib/postgresql/{{ postgresql_version }}/{{ postgresql_cluster_name }}\ + {% endif %}" +# Note: When deploying to cloud providers, we create a disk and mount the data directory, +# along the path defined in the 'pg_data_mount_path' variable (or use '/pgdata' by default). + # You can specify custom WAL dir path. Example: "/pgwal/{{ postgresql_version }}/pg_wal" postgresql_wal_dir: "" # if defined, symlink will be created [optional] postgresql_conf_dir: "/etc/postgresql/{{ postgresql_version }}/{{ postgresql_cluster_name }}" @@ -41,13 +49,16 @@ system_packages: - iptables - acl - dnsutils + - moreutils + +install_perf: false # or 'true' to install "perf" (Linux profiling with performance counters) and "FlameGraph". postgresql_packages: - postgresql-{{ postgresql_version }} - postgresql-client-{{ postgresql_version }} - postgresql-contrib-{{ postgresql_version }} - postgresql-server-dev-{{ postgresql_version }} -# - postgresql-{{ postgresql_version }}-dbgsym + - postgresql-{{ postgresql_version }}-dbgsym # - postgresql-{{ postgresql_version }}-repack # - postgresql-{{ postgresql_version }}-cron # - postgresql-{{ postgresql_version }}-pg-stat-kcache @@ -56,6 +67,7 @@ postgresql_packages: # - postgresql-{{ postgresql_version }}-pgrouting # - postgresql-{{ postgresql_version }}-pgvector # - postgresql-{{ postgresql_version }}-pgaudit +# - postgresql-{{ postgresql_version }}-partman # Extra packages etcd_package_repo: "https://github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz" diff --git a/vars/RedHat.yml b/vars/RedHat.yml index 87eb7e7c7..816594dfe 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -1,8 +1,17 @@ --- # PostgreSQL variables +# # You can specify custom data dir path. Example: "/pgdata/{{ postgresql_version }}/data" -postgresql_data_dir: "/var/lib/pgsql/{{ postgresql_version }}/data" +postgresql_data_dir: "\ + {% if cloud_provider | default('') | length > 0 %}\ + {{ pg_data_mount_path | default('/pgdata') }}/{{ postgresql_version }}/data\ + {% else %}\ + /var/lib/pgsql/{{ postgresql_version }}/data\ + {% endif %}" +# Note: When deploying to cloud providers, we create a disk and mount the data directory, +# along the path defined in the 'pg_data_mount_path' variable (or use '/pgdata' by default). + # You can specify custom WAL dir path. Example: "/pgwal/{{ postgresql_version }}/pg_wal" postgresql_wal_dir: "" # if defined, symlink will be created [optional] postgresql_conf_dir: "{{ postgresql_data_dir }}" @@ -24,8 +33,8 @@ yum_repository: [] # gpgkey: "https://my-repo-key.url" # gpgcheck: "yes" -install_epel_repo: true # or 'false' (installed from the package "epel-release-latest.noarch.rpm") install_postgresql_repo: true # or 'false' (installed from the package "pgdg-redhat-repo-latest.noarch.rpm") +install_epel_repo: true # or 'false' (installed from the package "epel-release-latest.noarch.rpm") install_scl_repo: true # or 'false' (Redhat 7 family only) # Packages (for yum repo) @@ -59,6 +68,9 @@ system_packages: - iptables - acl - bind-utils + - moreutils + +install_perf: false # or 'true' to install "perf" (Linux profiling with performance counters) and "FlameGraph". # The glibc-langpack package includes the basic information required to support the language in your applications. # for RHEL version 8 (only) @@ -71,7 +83,8 @@ postgresql_packages: - postgresql{{ postgresql_version }} - postgresql{{ postgresql_version }}-server - postgresql{{ postgresql_version }}-contrib -# - postgresql{{ postgresql_version }}-devel + - postgresql{{ postgresql_version }}-devel + - postgresql{{ postgresql_version }}-debuginfo # - pg_repack_{{ postgresql_version }} # - pg_cron_{{ postgresql_version }} # - pg_stat_kcache_{{ postgresql_version }} @@ -80,6 +93,7 @@ postgresql_packages: # - pgrouting_{{ postgresql_version }} # - pgvector_{{ postgresql_version }} # - pgaudit17_{{ postgresql_version }} +# - pg_partman_{{ postgresql_version }} # Extra packages etcd_package_repo: "https://github.com/etcd-io/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz" diff --git a/vars/main.yml b/vars/main.yml index 537a301ec..a8fe6a13e 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -9,14 +9,14 @@ proxy_env: {} # yamllint disable rule:braces # Cluster variables cluster_vip: "" # IP address for client access to the databases in the cluster (optional). vip_interface: "{{ ansible_default_ipv4.interface }}" # interface name (e.g., "ens32"). -# Note: VIP-based solutions such as keepalived or vip-manager may not function correctly in cloud environments like AWS. +# Note: VIP-based solutions such as keepalived or vip-manager may not function correctly in cloud environments. patroni_cluster_name: "postgres-cluster" # the cluster name (must be unique for each cluster) patroni_superuser_username: "postgres" -patroni_superuser_password: "postgres-pass" # please change password +patroni_superuser_password: "" # Please specify a password. If not defined, will be generated automatically during deployment. patroni_replication_username: "replicator" -patroni_replication_password: "replicator-pass" # please change password +patroni_replication_password: "" # Please specify a password. If not defined, will be generated automatically during deployment. synchronous_mode: false # or 'true' for enable synchronous database replication synchronous_mode_strict: false # if 'true' then block all client writes to the master, when a synchronous replica is not available @@ -173,10 +173,10 @@ consul_services: # PostgreSQL variables -postgresql_version: "16" +postgresql_version: 16 # postgresql_data_dir: see vars/Debian.yml or vars/RedHat.yml postgresql_listen_addr: "0.0.0.0" # Listen on all interfaces. Or use "{{ inventory_hostname }},127.0.0.1" to listen on a specific IP address. -postgresql_port: "5432" +postgresql_port: 5432 postgresql_encoding: "UTF8" # for bootstrap only (initdb) postgresql_locale: "en_US.UTF-8" # for bootstrap only (initdb) postgresql_data_checksums: true # for bootstrap only (initdb) @@ -217,7 +217,7 @@ postgresql_extensions: [] # postgresql parameters to bootstrap dcs (are parameters for example) postgresql_parameters: - - { option: "max_connections", value: "500" } + - { option: "max_connections", value: "1000" } - { option: "superuser_reserved_connections", value: "5" } - { option: "password_encryption", value: "{{ postgresql_password_encryption_algorithm }}" } - { option: "max_locks_per_transaction", value: "512" } @@ -314,6 +314,7 @@ postgresql_pg_hba: - { type: "local", database: "all", user: "all", address: "", method: "{{ postgresql_password_encryption_algorithm }}" } - { type: "host", database: "all", user: "all", address: "127.0.0.1/32", method: "{{ postgresql_password_encryption_algorithm }}" } - { type: "host", database: "all", user: "all", address: "::1/128", method: "{{ postgresql_password_encryption_algorithm }}" } + - { type: "host", database: "all", user: "all", address: "0.0.0.0/0", method: "{{ postgresql_password_encryption_algorithm }}" } # - { type: "host", database: "mydatabase", user: "mydb-user", address: "192.168.0.0/24", method: "{{ postgresql_password_encryption_algorithm }}" } # - { type: "host", database: "all", user: "all", address: "192.168.0.0/24", method: "ident", options: "map=main" } # use pg_ident @@ -337,11 +338,11 @@ pgbouncer_conf_dir: "/etc/pgbouncer" pgbouncer_log_dir: "/var/log/pgbouncer" pgbouncer_listen_addr: "0.0.0.0" # Listen on all interfaces. Or use "{{ inventory_hostname }}" to listen on a specific IP address. pgbouncer_listen_port: 6432 -pgbouncer_max_client_conn: 10000 -pgbouncer_max_db_connections: 1000 +pgbouncer_max_client_conn: 100000 +pgbouncer_max_db_connections: 10000 pgbouncer_max_prepared_statements: 1024 -pgbouncer_default_pool_size: 20 pgbouncer_query_wait_timeout: 120 +pgbouncer_default_pool_size: 100 pgbouncer_default_pool_mode: "session" pgbouncer_admin_users: "{{ patroni_superuser_username }}" # comma-separated list of users, who are allowed to change settings pgbouncer_stats_users: "{{ patroni_superuser_username }}" # comma-separated list of users who are just allowed to use SHOW command @@ -349,7 +350,7 @@ pgbouncer_ignore_startup_parameters: "extra_float_digits,geqo,search_path" pgbouncer_auth_type: "{{ postgresql_password_encryption_algorithm }}" pgbouncer_auth_user: true # or 'false' if you want to manage the list of users for authentication in the database via userlist.txt pgbouncer_auth_username: pgbouncer # user who can query the database via the user_search function -pgbouncer_auth_password: "pgbouncer-pass" # please change password +pgbouncer_auth_password: "" # If not defined, a password will be generated automatically during deployment pgbouncer_auth_dbname: "postgres" pgbouncer_client_tls_sslmode: "disable" pgbouncer_client_tls_key_file: "" @@ -369,7 +370,7 @@ pgbouncer_pools: patroni_restapi_listen_addr: "0.0.0.0" # Listen on all interfaces. Or use "{{ inventory_hostname }}" to listen on a specific IP address. patroni_restapi_port: 8008 patroni_restapi_username: "patroni" -patroni_restapi_password: "restapi-pass" # please change password +patroni_restapi_password: "" # If not defined, a password will be generated automatically during deployment. patroni_ttl: 30 patroni_loop_wait: 10 patroni_retry_timeout: 10 @@ -454,7 +455,7 @@ wal_g: - { option: "command", value: "{{ wal_g_path }} backup-fetch {{ postgresql_data_dir }} LATEST" } - { option: "no_params", value: "True" } basebackup: - - { option: "max-rate", value: "100M" } + - { option: "max-rate", value: "1000M" } - { option: "checkpoint", value: "fast" } # - { option: "waldir", value: "{{ postgresql_wal_dir }}" } pg_probackup: @@ -495,9 +496,9 @@ wal_g_path: "/usr/local/bin/wal-g --config {{ postgresql_home_dir }}/.walg.json" wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } # define values or pass via --extra-vars - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" } # define values or pass via --extra-vars - - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('') }}" } # define values or pass via --extra-vars - - { option: "WALG_COMPRESSION_METHOD", value: "brotli" } # or "lz4", "lzma", "zstd" - - { option: "WALG_DELTA_MAX_STEPS", value: "6" } # determines how many delta backups can be between full backups + - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://' + patroni_cluster_name) }}" } # define values or pass via --extra-vars + - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" } # or "lz4", "lzma", "zstd" + - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" } # determines how many delta backups can be between full backups - { option: "PGDATA", value: "{{ postgresql_data_dir }}" } - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir }}" } - { option: "PGPORT", value: "{{ postgresql_port }}" } @@ -521,13 +522,13 @@ wal_g_cron_jobs: - name: "WAL-G: Create daily backup" user: "postgres" file: /etc/cron.d/walg - minute: "30" - hour: "3" + minute: "00" + hour: "{{ WALG_BACKUP_HOUR | default('3') }}" day: "*" month: "*" weekday: "*" job: "{{ wal_g_backup_command | join('') }}" - - name: "WAL-G: Delete old backups" # retain 4 full backups (adjust according to your company's backup retention policy) + - name: "WAL-G: Delete old backups" user: "postgres" file: /etc/cron.d/walg minute: "30" @@ -538,28 +539,30 @@ wal_g_cron_jobs: job: "{{ wal_g_delete_command | join('') }}" # pgBackRest -pgbackrest_install: false # or 'true' +pgbackrest_install: false # or 'true' to install and configure backups using pgBackRest pgbackrest_install_from_pgdg_repo: true # or 'false' pgbackrest_stanza: "{{ patroni_cluster_name }}" # specify your --stanza pgbackrest_repo_type: "posix" # or "s3", "gcs", "azure" pgbackrest_repo_host: "" # dedicated repository host (optional) -pgbackrest_repo_user: "postgres" +pgbackrest_repo_user: "postgres" # if "repo_host" is set (optional) pgbackrest_conf_file: "/etc/pgbackrest/pgbackrest.conf" # config https://pgbackrest.org/configuration.html pgbackrest_conf: global: # [global] section - { option: "log-level-file", value: "detail" } - { option: "log-path", value: "/var/log/pgbackrest" } + - { option: "repo1-type", value: "{{ pgbackrest_repo_type | lower }}" } # - { option: "repo1-host", value: "{{ pgbackrest_repo_host }}" } # - { option: "repo1-host-user", value: "{{ pgbackrest_repo_user }}" } - - { option: "repo1-type", value: "{{ pgbackrest_repo_type | lower }}" } - { option: "repo1-path", value: "/var/lib/pgbackrest" } - { option: "repo1-retention-full", value: "4" } - { option: "repo1-retention-archive", value: "4" } + - { option: "repo1-bundle", value: "y" } + - { option: "repo1-block", value: "y" } - { option: "start-fast", value: "y" } - { option: "stop-auto", value: "y" } - - { option: "resume", value: "n" } - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } - { option: "spool-path", value: "/var/spool/pgbackrest" } - { option: "archive-async", value: "y" } # Enables asynchronous WAL archiving (details: https://pgbackrest.org/user-guide.html#async-archiving) - { option: "archive-get-queue-max", value: "1GiB" } @@ -585,13 +588,13 @@ pgbackrest_server_conf: - { option: "repo1-retention-archive", value: "4" } - { option: "repo1-bundle", value: "y" } - { option: "repo1-block", value: "y" } - - { option: "start-fast", value: "y" } - - { option: "stop-auto", value: "y" } - - { option: "resume", value: "n" } - - { option: "link-all", value: "y" } - { option: "archive-check", value: "y" } - { option: "archive-copy", value: "n" } - { option: "backup-standby", value: "y" } + - { option: "start-fast", value: "y" } + - { option: "stop-auto", value: "y" } + - { option: "link-all", value: "y" } + - { option: "resume", value: "n" } # - { option: "", value: "" } # the stanza section will be generated automatically @@ -607,23 +610,23 @@ pgbackrest_cron_jobs: - name: "pgBackRest: Full Backup" file: "/etc/cron.d/pgbackrest-{{ patroni_cluster_name }}" user: "postgres" - minute: "30" - hour: "6" + minute: "00" + hour: "{{ PGBACKREST_BACKUP_HOUR | default('3') }}" day: "*" month: "*" weekday: "0" job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=full backup" - # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup; fi" + # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --stanza={{ pgbackrest_stanza }} --type=full backup; fi" - name: "pgBackRest: Diff Backup" file: "/etc/cron.d/pgbackrest-{{ patroni_cluster_name }}" user: "postgres" - minute: "30" - hour: "6" + minute: "00" + hour: "3" day: "*" month: "*" weekday: "1-6" job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=diff backup" - # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup; fi" + # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --stanza={{ pgbackrest_stanza }} --type=diff backup; fi" # PITR mode (if patroni_cluster_bootstrap_method: "pgbackrest" or "wal-g"): diff --git a/vars/system.yml b/vars/system.yml index 6dae04e93..376c7f942 100644 --- a/vars/system.yml +++ b/vars/system.yml @@ -35,7 +35,7 @@ locale: "en_US.utf-8" # Configure swap space (if not already exists) swap_file_create: true # or 'false' swap_file_path: /swapfile -swap_file_size_mb: '2048' # change this value for your system +swap_file_size_mb: '4096' # change this value for your system # Kernel parameters sysctl_set: true # or 'false' @@ -119,6 +119,8 @@ ssh_key_user: "postgres" ssh_key_state: "present" ssh_known_hosts: "{{ groups['postgres_cluster'] }}" +# List of public SSH keys. These keys will be added to the database server's  ~/.ssh/authorized_keys  file. +ssh_public_keys: [] # sudo sudo_users: @@ -214,4 +216,31 @@ cron_jobs: [] # weekday: "*" # job: "echo 'example job two command'" +# (optional) Configure mount points in /etc/fstab and mount the file system (if 'mount.src' is defined) +mount: + - path: "/pgdata" + src: "" # device UUID or path. + fstype: ext4 # if 'zfs' is specified a ZFS pool will be created + opts: defaults,noatime # not applicable to 'zfs' + state: mounted +# - path: "/pgwal" +# src: "" +# fstype: ext4 +# opts: defaults,noatime +# state: mounted + +# (optional) Execute custom commands or scripts +# This can be a direct command, a bash script content, or a path to a script on the host +pre_deploy_command: "" # Command or script to be executed before the Postgres cluster deployment +pre_deploy_command_timeout: 3600 # Timeout in seconds +pre_deploy_command_hosts: "postgres_cluster" # host groups where the pre_deploy_command should be executed +pre_deploy_command_print: true # Print the command in the ansible log +pre_deploy_command_print_result: true # Print the result of the command execution to the ansible log + +post_deploy_command: "" # Command or script to be executed after the Postgres cluster deployment +post_deploy_command_timeout: 3600 # Timeout in seconds +post_deploy_command_hosts: "postgres_cluster" # host groups where the post_deploy_command should be executed +post_deploy_command_print: true # Print the command in the ansible log +post_deploy_command_print_result: true # Print the result of the command execution to the ansible log + ...