Provision of resources in a popular cloud providers for PostgreSQL cl…

…uster deployment (#464)

.config/make/docker.mak

@@ -1,11 +1,23 @@
 ## —— Docker —————————————————————————————————————————————————————————————————————————————————————
+TAG ?= local
+DOCKER_REGISTRY ?= vitabaks
+
 .PHONY: docker-build
-docker-build: ## Run docker build image in local
-	docker build --tag postgresql_cluster:local --file .config/gitpod/Dockerfile .
+docker-build: ## Run docker build image (example: make docker-build TAG=my_tag)
+	@echo "Building container image with tag $(TAG)";
+	docker build --no-cache --tag postgresql_cluster:$(TAG) --file Dockerfile .
+
+.PHONY: docker-push
+docker-push: ## Push image to Dockerhub (example: make docker-push TAG=my_tag DOCKER_REGISTRY=my_repo DOCKER_REGISTRY_USER="my_username" DOCKER_REGISTRY_PASSWORD="my_password")
+	@echo "Pushing container image with tag $(TAG)";
+	echo "$(DOCKER_REGISTRY_PASSWORD)" | docker login --username "$(DOCKER_REGISTRY_USER)" --password-stdin
+	docker tag postgresql_cluster:$(TAG) $(DOCKER_REGISTRY)/postgresql_cluster:$(TAG)
+	docker push $(DOCKER_REGISTRY)/postgresql_cluster:$(TAG)
 
 .PHONY: docker-lint
 docker-lint: ## Run hadolint command to lint Dokerfile
-	docker run --rm -i hadolint/hadolint < .config/gitpod/Dockerfile
+	docker run --rm -i -v ./Dockerfile:/Dockerfile \
+	hadolint/hadolint hadolint --ignore DL3002 --ignore DL3008 --ignore DL3059 /Dockerfile
 
 .PHONY: docker-tests
 docker-tests: ## Run tests for docker

.config/make/molecule.mak

@@ -1,66 +1,56 @@
+# Activate virtual environment
+ACTIVATE_VENV = . .venv/bin/activate
+
 ## —— Molecule ———————————————————————————————————————————————————————————————————————————————————
+
 .PHONY: molecule-test
 molecule-test: ## Run test sequence for default scenario
-	source .venv/bin/activate
-	molecule test
+	$(ACTIVATE_VENV) && molecule test
 
 .PHONY: molecule-destroy
 molecule-destroy: ## Run destroy sequence for default scenario
-	source .venv/bin/activate
-	molecule destroy
+	$(ACTIVATE_VENV) && molecule destroy
 
 .PHONY: molecule-converge
 molecule-converge: ## Run converge sequence for default scenario
-	source .venv/bin/activate
-	molecule converge
+	$(ACTIVATE_VENV) && molecule converge
 
 .PHONY: molecule-reconverge
 molecule-reconverge: ## Run destroy and converge sequence for default scenario
-	source .venv/bin/activate
-	molecule destroy
-	molecule converge
+	$(ACTIVATE_VENV) && molecule destroy && molecule converge
 
 .PHONY: molecule-test-all
 molecule-test-all: ## Run test sequence for all scenarios
-	source .venv/bin/activate
-	molecule test --all
+	$(ACTIVATE_VENV) && molecule test --all
 
 .PHONY: molecule-destroy-all
 molecule-destroy-all: ## Run destroy sequence for all scenarios
-	source .venv/bin/activate
-	molecule destroy --all
+	$(ACTIVATE_VENV) && molecule destroy --all
 
 .PHONY: molecule-test-scenario
-molecule-test-scenario: ## Run molecule test with specific scenario (example: make molecule-test-scenario MOLECULE_SCENARIO="postgrespro")
-	source .venv/bin/activate
-	molecule test --scenario-name $(MOLECULE_SCENARIO)
+molecule-test-scenario: ## Run molecule test with specific scenario (example: make molecule-test-scenario MOLECULE_SCENARIO="scenario_name")
+	$(ACTIVATE_VENV) && molecule test --scenario-name $(MOLECULE_SCENARIO)
 
 .PHONY: molecule-destroy-scenario
-molecule-destroy-scenario: ## Run molecule destroy with specific scenario (example: make molecule-destroy-scenario MOLECULE_SCENARIO="postgrespro")
-	source .venv/bin/activate
-	molecule destroy --scenario-name $(MOLECULE_SCENARIO)
+molecule-destroy-scenario: ## Run molecule destroy with specific scenario (example: make molecule-destroy-scenario MOLECULE_SCENARIO="scenario_name")
+	$(ACTIVATE_VENV) && molecule destroy --scenario-name $(MOLECULE_SCENARIO)
 
 .PHONY: molecule-converge-scenario
-molecule-converge-scenario: ## Run molecule converge with specific scenario (example: make molecule-converge-scenario MOLECULE_SCENARIO="postgrespro")
-	source .venv/bin/activate
-	molecule converge --scenario-name $(MOLECULE_SCENARIO)
+molecule-converge-scenario: ## Run molecule converge with specific scenario (example: make molecule-converge-scenario MOLECULE_SCENARIO="scenario_name")
+	$(ACTIVATE_VENV) && molecule converge --scenario-name $(MOLECULE_SCENARIO)
 
 .PHONY: molecule-dependency
 molecule-dependency: ## Run dependency sequence
-	source .venv/bin/activate
-	molecule dependency
+	$(ACTIVATE_VENV) && molecule dependency
 
 .PHONY: molecule-verify
 molecule-verify: ## Run verify sequence
-	source .venv/bin/activate
-	molecule verify
+	$(ACTIVATE_VENV) && molecule verify
 
 .PHONY: molecule-login
 molecule-login: ## Log in to one instance using custom host IP (example: make molecule-login MOLECULE_HOST="10.172.0.20")
-	source .venv/bin/activate
-	molecule login --host $(MOLECULE_HOST)
+	$(ACTIVATE_VENV) && molecule login --host $(MOLECULE_HOST)
 
 .PHONY: molecule-login-scenario
-molecule-login-scenario: ## Log in to one instance using custom host IP and scenario name (example: make molecule-login-scenario MOLECULE_HOST="10.172.1.20" MOLECULE_SCENARIO="postgrespro")
-	source .venv/bin/activate
-	molecule login --host $(MOLECULE_HOST) --scenario-name $(MOLECULE_SCENARIO)
+molecule-login-scenario: ## Log in to one instance using custom host IP and scenario name (example: make molecule-login-scenario MOLECULE_HOST="10.172.1.20" MOLECULE_SCENARIO="scenario_name")
+	$(ACTIVATE_VENV) && molecule login --host $(MOLECULE_HOST) --scenario-name $(MOLECULE_SCENARIO)

.config/make/python.mak

@@ -3,6 +3,9 @@ python_launcher ?= python3.10
 python_requirements_file ?= requirements.txt
 python_requirements_dev_file ?= .config/python/dev/requirements.txt
 
+# Activate virtual environment
+ACTIVATE_VENV = . .venv/bin/activate
+
 ## —— Python —————————————————————————————————————————————————————————————————————————————————————
 .PHONY: python-bootstrap
 python-bootstrap: ## Bootstrap python
@@ -20,56 +23,50 @@ python-bootstrap-dev: ## Bootstrap python for dev env
 # ===============================================================================================
 .PHONY: python-venv-init
 python-venv-init: ## Create venv ".venv/" if not exist
-	if [ ! -d .venv ] ; then
-		$(python_launcher) -m venv .venv
-	fi
+	@echo "Checking if .venv directory exists..."; \
+	if [ ! -d .venv ]; then echo "Creating virtual environment using $(python_launcher)..."; $(python_launcher) -m venv .venv; else echo ".venv directory already exists. Skipping creation."; fi
 
 .PHONY: python-venv-upgrade
 python-venv-upgrade: ## Upgrade venv with pip, setuptools and wheel
-	source .venv/bin/activate
-	pip install --upgrade pip setuptools wheel
+	@echo "Upgrading virtual environment..."
+	$(ACTIVATE_VENV) && pip install --upgrade pip setuptools wheel
 
 .PHONY: python-venv-requirements
 python-venv-requirements: ## Install or upgrade from $(python_requirements_file)
-	source .venv/bin/activate
-	pip install --upgrade --requirement $(python_requirements_file)
+	@echo "Installing or upgrading requirements from $(python_requirements_file)..."
+	$(ACTIVATE_VENV) && pip install --upgrade --requirement $(python_requirements_file)
 
 .PHONY: python-venv-requirements-dev
 python-venv-requirements-dev: ## Install or upgrade from $(python_requirements_dev_file)
-	source .venv/bin/activate
-	pip install --upgrade --requirement $(python_requirements_dev_file)
+	@echo "Installing or upgrading dev requirements from $(python_requirements_dev_file)..."
+	$(ACTIVATE_VENV) && pip install --upgrade --requirement $(python_requirements_dev_file)
 
 .PHONY: python-venv-linters-install
 python-venv-linters-install: ## Install or upgrade linters
-	source .venv/bin/activate
-	pip install --upgrade flake8
+	@echo "Installing or upgrading linters..."
+	$(ACTIVATE_VENV) && pip install --upgrade flake8
 
 .PHONY: python-venv-purge
 python-venv-purge: ## Remove venv ".venv/" folder
-	rm -rf .venv
+	@echo "Removing .venv directory..."
+	@rm -rf .venv
 
 # ===============================================================================================
 # Utils
 # ===============================================================================================
 .PHONY: python-purge-cache
 python-purge-cache: ## Purge cache to avoid used cached files
-	if [ -d .venv ] ; then
-		source .venv/bin/activate
-		pip cache purge
-	fi
+	@echo "Purging pip cache..."
+	@if [ -d .venv ] ; then $(ACTIVATE_VENV) && pip cache purge; fi
 
 .PHONY: python-version
 python-version: ## Displays the python version used for the .venv
-	source .venv/bin/activate
-	$(python_launcher) --version
+	$(ACTIVATE_VENV) && $(python_launcher) --version
 
 .PHONY: python-flake8
 python-flake8: ## Run flake8 linter for python
-	source .venv/bin/activate
-	flake8 --config .config/.flake8
+	$(ACTIVATE_VENV) && flake8 --config .config/.flake8
 
 .PHONY: python-pytest
 python-pytest: ## Run pytest to test python scripts
-	source .venv/bin/activate
-	cd scripts/
-	$(python_launcher) -m pytest
+	$(ACTIVATE_VENV) && cd scripts/ && $(python_launcher) -m pytest

.github/workflows/docker.yml

@@ -5,6 +5,8 @@ on:
   push:
     branches:
       - master
+    tags:
+      - '*'
   pull_request:
     branches:
       - master
@@ -17,6 +19,25 @@ jobs:
       - name: Set TERM environment variable
         run: echo "TERM=xterm" >> $GITHUB_ENV
 
+      - name: Extract branch or tag name
+        shell: bash
+        run: |
+          REF_NAME=""
+          if [[ -n "${GITHUB_HEAD_REF}" ]]; then
+            # This is a PR, use the source branch name
+            REF_NAME="${GITHUB_HEAD_REF}"
+          else
+            # This is a push, use the branch or tag name from GITHUB_REF
+            REF_NAME="${GITHUB_REF##*/}"
+          fi
+
+          # If this is the master branch, use 'latest' as the tag, otherwise use the REF_NAME
+          if [[ "$REF_NAME" == "master" ]]; then
+            echo "TAG=latest" >> $GITHUB_ENV
+          else
+            echo "TAG=$REF_NAME" >> $GITHUB_ENV
+          fi
+
       - name: Checkout
         uses: actions/checkout@v3
 
@@ -28,5 +49,17 @@ jobs:
       - name: Install dependencies
         run: make bootstrap-dev
 
-      - name: Run Docker tests
-        run: make docker-tests
+      - name: Run Docker lint
+        run: make docker-lint
+
+      - name: Run Docker build
+        run: make docker-build
+        env:
+          TAG: ${{ env.TAG }}
+
+      - name: Run Docker push
+        run: make docker-push
+        env:
+          TAG: ${{ env.TAG }}
+          DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

Dockerfile

@@ -0,0 +1,41 @@
+FROM debian:bookworm-slim
+LABEL maintainer="Vitaliy Kukharik [email protected]"
+
+USER root
+
+# Set SHELL to Bash to ensure pipefail is supported
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Copy postgresql_cluster repository
+COPY . /postgresql_cluster
+
+# Install required packages, Python dependencies, Ansible requirements, and perform cleanup
+RUN apt-get clean && rm -rf /var/lib/apt/lists/partial \
+    && apt-get update -o Acquire::CompressionTypes::Order::=gz \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+       ca-certificates gnupg git python3 python3-dev python3-pip keychain ssh-client sshpass\
+       gcc g++ cmake make libssl-dev curl apt-transport-https lsb-release gnupg \
+    && pip3 install --break-system-packages --no-cache-dir -r \
+      /postgresql_cluster/requirements.txt \
+    && ansible-galaxy install --force -r \
+       /postgresql_cluster/requirements.yml \
+    && ansible-galaxy install --force -r \
+       /postgresql_cluster/roles/consul/requirements.yml \
+    && ansible-galaxy collection list \
+    && pip3 install --break-system-packages --no-cache-dir -r \
+       /root/.ansible/collections/ansible_collections/azure/azcollection/requirements.txt \
+    && curl -sLS https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/microsoft.asc.gpg > /dev/null \
+    && echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/azure-cli.list \
+    && apt-get update && apt-get install --no-install-recommends -y azure-cli \
+    && apt-get autoremove -y --purge gnupg git python3-dev gcc g++ cmake make libssl-dev \
+    && apt-get clean -y autoclean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* \
+    && chmod +x /postgresql_cluster/entrypoint.sh
+
+# Set environment variable for Ansible collections paths
+ENV ANSIBLE_COLLECTIONS_PATH=/root/.ansible/collections/ansible_collections:/usr/local/lib/python3.11/dist-packages/ansible_collections
+ENV USER=root
+
+WORKDIR /postgresql_cluster
+
+ENTRYPOINT ["./entrypoint.sh"]

Makefile

@@ -38,19 +38,7 @@ python_launcher := python$(shell cat .config/python_version.config | cut -d '='
 
 -include $(addsuffix /*.mak, $(shell find .config/make -type d))
 
-## —— Tests collection ———————————————————————————————————————————————————————————————————————
-.PHONY: tests
-tests: ## tests Ansible collection
-	$(MAKE) docker-tests
-	$(MAKE) lint
-	$(MAKE) molecule-test-all
-
-.PHONY: tests-fast
-tests-fast: ## tests Ansible collection quickly
-	$(MAKE) lint
-	$(MAKE) molecule-converge
-
-## —— Bootstrap collection ———————————————————————————————————————————————————————————————————————
+## —— Bootstrap ——————————————————————————————————————————————————————————————————————————————————
 .PHONY: bootstrap
 bootstrap: ## Bootstrap Ansible collection
 	$(MAKE) python-bootstrap
@@ -60,19 +48,33 @@ bootstrap-dev: ## Bootstrap Ansible collection for development
 	$(MAKE) bootstrap
 	$(MAKE) python-bootstrap-dev
 
-## —— Virtualenv ————————————————————————————————————————————————————————————————————————————————
+## —— Virtualenv —————————————————————————————————————————————————————————————————————————————————
 .PHONY: reinitialization
-reinitialization: ## Return to an initial state of Bootstrap Ansible collection
+reinitialization: ## Return to initial state of Bootstrap Ansible collection
 	$(MAKE) clean
 	$(MAKE) bootstrap
 
 .PHONY: reinitialization-dev
-reinitialization-dev: ## Return to an initial state of Bootstrap Ansible collection for development
+reinitialization-dev: ## Return to initial state of Bootstrap Ansible collection for development
 	$(MAKE) reinitialization
 	$(MAKE) bootstrap-dev
 
+## —— Tests ——————————————————————————————————————————————————————————————————————————————————————
+.PHONY: tests
+tests: ## tests Ansible
+	$(MAKE) docker-tests
+	$(MAKE) lint
+	$(MAKE) molecule-test-all
+
+.PHONY: tests-fast
+tests-fast: ## tests Ansible quickly
+	$(MAKE) lint
+	$(MAKE) molecule-converge
+
+## —— Clean ——————————————————————————————————————————————————————————————————————————————————————
 .PHONY: clean
-clean: ## Clean collection
+	$(MAKE) clean
+clean: ## Clean
 	rm -rf .venv/
 	rm -rf vendor/
 	rm -f *.mak
@@ -81,4 +83,4 @@ clean: ## Clean collection
 	rm -rf scripts/tests/__pycache__/
 	rm -rf scripts/modules/__pycache__/
 	rm -rf scripts/modules/services/__pycache__/
-	rm -rf scripts/modules/utils/__pycache__/
+	rm -rf scripts/modules/utils/__pycache__/

README.md

@@ -122,7 +122,7 @@ Minimum supported Ansible version: 8.0.0 (ansible-core 2.15.0)
 ## Requirements
 This playbook requires root privileges or sudo.
 
-Ansible ([What is Ansible](https://www.ansible.com/resources/videos/quick-start-video)?)
+Ansible ([What is Ansible](https://www.ansible.com/how-ansible-works/)?)
 
 if dcs_type: "consul", please install consul role requirements on the control node: