feat: Update Compose Builder to Support PostgreSQL as Database (#464)

* feat: Update Compose Builder to Support PostgreSQL as Database

closes #463

Signed-off-by: Jack Chen <[email protected]>

README.md

@@ -128,16 +128,16 @@ The compose files under the `taf` subfolder are used for the automated TAF tests
     Use `make portainer`and `make portainer-down` to start and stop Portainer.
 
 ### Use PostgreSQL as the persistence layer in EdgeX
-- **docker-compose-postgres-no-secty.yml** Contains just the services needed to run in non-secure configuration. Includes Postgres, Redis, Device Virtual and MQTT Broker services using a mix of Postgres and Redis as the databases and MQTT as the message bus.
-- **docker-compose-postgres-no-secty-arm64.yml** Contains just the services needed to run in non-secure configuration on `ARM64` system. Includes Postgres, Redis, Device Virtual and MQTT Broker services using a mix of Postgres and Redis as the databases and MQTT as the message bus.
+EdgeX services can be configured to use PostgreSQL as the persistence layer. The compose builder now supports generating compose files that use PostgreSQL.
 
-> **Note:** Only the services as listed below support Postgres as the database in EdgeX. More EdgeX services will support Postgres once the development work is done.
-> - **Core Data**
-> - **Core Keeper**
-> - The Store and Forward capability of **app-mqtt-export** App Service
+**To use PostgreSQL as the persistence layer, follow these steps**
 
-  **Start the EdgeX Services using Postgres and Redis as the databases**
+- Go to `/compose-builder` folder
+- `make run no-secty keeper mqtt-bus postgres`
 
-  - Use `docker compose -f docker-compose-postgres-no-secty.yml up -d` to start the services using this compose file.
-  - Use `docker compose -f docker-compose-postgres-no-secty.yml down` to stop the services.
-  - Replace **docker-compose-postgres-no-secty.yml** with **docker-compose-postgres-no-secty-arm64.yml** in the above commands on `ARM64` system.
+    Runs the services with PostgreSQL as the persistence layer in non-secure mode.
+- `make run keeper mqtt-bus postgres`
+
+    Runs the services with PostgreSQL as the persistence layer in secure mode.
+
+> **Note:** `keeper` and `mqtt-bus` are required services for EdgeX to run with PostgreSQL as the persistence layer.

compose-builder/.env

@@ -1,5 +1,6 @@
 # /*******************************************************************************
 #  * Copyright 2022 Intel
+#  * Copyright 2024 IOTech Ltd
 #  *
 #  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 #  * in compliance with the License. You may obtain a copy of the License at
@@ -45,13 +46,27 @@ CP_FLAGS='-cp=consul.http://edgex-core-consul:8500'
 REGISTRY_HOST=edgex-core-consul
 REGISTRY_PORT=8500
 REGISTRY_TYPE=consul
+DATABASE_HOST=edgex-redis
+DATABASE_TYPE=redisdb
+DATABASE_PORT=6379
+MESSAGEBUS_HOST=edgex-redis
+MESSAGEBUS_SECRETNAME=redisdb
+MESSAGEBUS_AUTHMODE=none
+MESSAGEBUS_PORT=6379
+MESSAGEBUS_PROTOCOL=redis
+MESSAGEBUS_TYPE=redis
+WRITABLE_INSECURESECRETS_DB_SECRETNAME=redisdb
+WRITABLE_INSECURESECRETS_DB_SECRETDATA_USERNAME=
+WRITABLE_INSECURESECRETS_DB_SECRETDATA_PASSWORD=
+COMMON_SEC_STAGE_GATE_FILE_NAME=common-sec-stage-gate.env
 
 # Lock on Vault 1.14 (last MPL-2.0 version)
 VAULT_VERSION=1.14
 # Lock on Consul 1.16 (last MPL-2.0 version)
 CONSUL_VERSION=1.16
 # Lock on Redis 7.0 until EdgeX 4.0
 REDIS_VERSION=7.0-alpine
+POSTGRES_VERSION=16.3-alpine3.20
 KUIPER_VERSION=v1.14.0-alpha.2
 MOSQUITTO_VERSION=2.0
 NANOMQ_VERSION=0.18

compose-builder/Makefile

@@ -1,5 +1,6 @@
 # /*******************************************************************************
 #  * Copyright 2023 Intel
+#  * Copyright 2024 IOTech Ltd
 #  *
 #  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 #  * in compliance with the License. You may obtain a copy of the License at
@@ -43,7 +44,8 @@ export USERID:=$(shell id -u)
 export DOCKER_SOCKET_PATH=/var/run/docker.sock
 
 # Get total system memory in megabytes for vault config
-export TOTAL_SYSTEM_MEMORY:=$(shell grep MemTotal /proc/meminfo | awk '{print $$2}')m
+#export TOTAL_SYSTEM_MEMORY:=$(shell grep MemTotal /proc/meminfo | awk '{print $$2}')m
+export TOTAL_SYSTEM_MEMORY:=327352832m
 
 BROKER_YAML=add-mqtt-broker-mosquitto.yml
 TAF_BROKER_YAML=add-taf-mqtt-broker-mosquitto.yml
@@ -60,6 +62,7 @@ define OPTIONS
  - asc-http asc-mqtt asc-sample asc-metrics as-llrp as-record-replay asc-ex-mqtt -
  - modbus-sim zero-trust no-cleanup -
  - consul keeper -
+ - postgres -
 endef
 export OPTIONS
 
@@ -139,6 +142,35 @@ ifeq (keeper, $(filter keeper,$(ARGS)))
 		export STAGEGATE_REGISTRY_HOST=edgex-core-keeper
 		export STAGEGATE_REGISTRY_PORT=59890
 	endif
+
+	ifeq (postgres, $(filter postgres,$(ARGS)))
+		export DATABASE_HOST=edgex-postgres
+		export DATABASE_TYPE=postgres
+		export DATABASE_PORT=5432
+		export MESSAGEBUS_HOST=edgex-mqtt-broker
+		export MESSAGEBUS_PORT=1883
+		export MESSAGEBUS_PROTOCOL=tcp
+		export MESSAGEBUS_AUTHMODE=usernamepassword
+		export MESSAGEBUS_SECRETNAME=message-bus
+		export MESSAGEBUS_TYPE=mqtt
+		export WRITABLE_INSECURESECRETS_DB_SECRETNAME=
+		ifeq (no-secty, $(filter no-secty,$(ARGS)))
+			export MESSAGEBUS_AUTHMODE=none
+            export MESSAGEBUS_SECRETNAME=
+			export WRITABLE_INSECURESECRETS_DB_SECRETNAME=postgres
+			export WRITABLE_INSECURESECRETS_DB_SECRETDATA_USERNAME=postgres
+			export WRITABLE_INSECURESECRETS_DB_SECRETDATA_PASSWORD=postgres
+		endif
+	endif
+endif
+
+ifeq (postgres, $(filter postgres,$(ARGS)))
+	# When using postgres, change the default compose file to use docker-compose-base-postgres.yml
+	COMPOSE_FILES:= -f docker-compose-base-postgres.yml
+	COMPOSE_FILES += -f add-cron-scheduler.yml
+	ifneq (no-secty, $(filter no-secty,$(ARGS)))
+		export COMMON_SEC_STAGE_GATE_FILE_NAME=common-sec-stage-gate-postgres.env
+	endif
 endif
 
 # When in delay-start mode, ensure support services are delay-start-compliant by adding runtime-token configuration
@@ -828,7 +860,12 @@ endif
 ifeq (no-secty, $(filter no-secty,$(ARGS)))
 	NO_SECURITY:=-no-secty
 else
-	COMPOSE_FILES:=$(COMPOSE_FILES) -f add-security.yml
+	ifeq (postgres, $(filter postgres,$(ARGS)))
+		COMPOSE_FILES:=$(COMPOSE_FILES) -f add-secure-postgres.yml
+		COMPOSE_FILES:=$(COMPOSE_FILES) -f add-security-postgres.yml
+	else
+		COMPOSE_FILES:=$(COMPOSE_FILES) -f add-security.yml
+	endif
 	ifeq ($(BUS),) # if BUS not set, then we are using secure redis messagebus by default
 		COMPOSE_FILES:=$(COMPOSE_FILES) -f add-secure-redis-messagebus.yml
 	endif
@@ -1059,6 +1096,8 @@ define COMPOSE_DOWN
 		-f add-secure-consul.yml \
 		-f add-keeper.yml \
 		-f add-secure-keeper.yml \
+		-f add-secure-postgres.yml \
+		-f add-cron-scheduler.yml \
 		down $1
 endef
 

compose-builder/README.md

@@ -52,7 +52,9 @@ The `Extending using multiple Compose files` approach along with environment fil
 This folder contains the following compose files:
 
 - **docker-compose-base.yml**<br/>
-  Base non-secure mode compose file. Contains all the services that run in the non-secure configuration, including the UI.  
+  Base non-secure mode compose file. Contains all the services that run in the non-secure configuration, including the UI.
+- **docker-compose-base-postgres.yml**<br/>
+  Base non-secure mode compose file with PostgreSQL as database. Contains all the services that run in the non-secure configuration, including the UI.
 - **add-security.yml**<br/>
     Security **extending** compose file. Adds the additional security services and configuration of services so that all the services are running in the secure configuration.
 - **add-secure-redis-messagebus.yml**<br/>
@@ -256,6 +258,7 @@ Options:
                       The NATS Server service is also included.
     no-cleanup:       Leaves generated files behind for debugging purposes.
     keeper:           Runs to registry service to core-keeper
+    postgres:         Runs with PostgreSQL as database, 'mqtt-bus' is required when using this option
 Services:
     <names...>: Runs only services listed (and their dependent services) where 'name' matches a service name in one of the compose files used
 ```
@@ -315,6 +318,7 @@ Options:
     nats-bus:         Pull includes additional services for NATS Message Bus
     no-cleanup:       Leaves generated files behind for debugging purposes
     keeper:           Pull includes core-keeper
+    postgres:         Pull includes PostgreSQL
 
 Services:
     <names...>: Pulls only images for the service(s) listed
@@ -373,6 +377,7 @@ Options:
     no-cleanup:       Leaves generated files behind for debugging purposes.
     keeper:           Generates compose file with services registry to core-keeper
                       The core-keeper service is also included
+    postgres:         Generates compose file with PostgreSQL as database, 'mqtt-bus' is required when using this option
 ```
 #### Clean
 
@@ -486,6 +491,7 @@ Options:
                       The NATS Server service is also included.
     no-cleanup:       Leaves generated files behind for debugging purposes.
     keeper:           Generates compose file to registry service to core-keeper
+    postgres:         Generates compose file with PostgreSQL as database, 'mqtt-bus' is required when using this option
 ```
 
 #### TAF Compose

compose-builder/add-cron-scheduler.yml

@@ -0,0 +1,44 @@
+# /*******************************************************************************
+#  * Copyright 2024 IOTech Ltd
+#  *
+#  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+#  * in compliance with the License. You may obtain a copy of the License at
+#  *
+#  * http://www.apache.org/licenses/LICENSE-2.0
+#  *
+#  * Unless required by applicable law or agreed to in writing, software distributed under the License
+#  * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  * or implied. See the License for the specific language governing permissions and limitations under
+#  * the License.
+#  *
+#  *******************************************************************************/
+
+# This file is used to add the support-cron-scheduler service to the base docker-compose.yml file.
+# TODO: Remove this file and add this service into docker-compose-base.yml while deprecating the support-scheduler service.
+services:
+  support-cron-scheduler:
+    image: ${CORE_EDGEX_REPOSITORY}/support-cron-scheduler${ARCH}:${CORE_EDGEX_VERSION}
+    command: --registry ${CP_FLAGS}
+    user: "${EDGEX_USER}:${EDGEX_GROUP}"
+    ports:
+      - "127.0.0.1:59863:59863"
+    container_name: edgex-support-cron-scheduler
+    hostname: edgex-support-cron-scheduler
+    read_only: true
+    restart: always
+    networks:
+      - edgex-network
+    env_file:
+      - common-non-security.env
+    environment:
+      SERVICE_HOST: edgex-support-cron-scheduler
+      DATABASE_HOST: edgex-postgres
+    depends_on:
+      - ${REGISTRY_TYPE}
+      - database
+      - core-common-config-bootstrapper
+    security_opt:
+      - no-new-privileges:true
+    volumes:
+      # use host timezone
+      - /etc/localtime:/etc/localtime:ro

compose-builder/add-keeper.yml

@@ -28,8 +28,18 @@ services:
     env_file:
       - common-non-security.env
     environment:
-      DATABASE_HOST: edgex-redis
-      MESSAGEBUS_HOST: edgex-redis
+      DATABASE_HOST: ${DATABASE_HOST}
+      DATABASE_TYPE: ${DATABASE_TYPE}
+      DATABASE_PORT: ${DATABASE_PORT}
+      MESSAGEBUS_HOST: ${MESSAGEBUS_HOST}
+      MESSAGEBUS_AUTHMODE: ${MESSAGEBUS_AUTHMODE}
+      MESSAGEBUS_SECRETNAME: ${MESSAGEBUS_SECRETNAME}
+      MESSAGEBUS_PORT: ${MESSAGEBUS_PORT}
+      MESSAGEBUS_PROTOCOL: ${MESSAGEBUS_PROTOCOL}
+      MESSAGEBUS_TYPE: ${MESSAGEBUS_TYPE}
+      WRITABLE_INSECURESECRETS_DB_SECRETNAME: ${WRITABLE_INSECURESECRETS_DB_SECRETNAME}
+      WRITABLE_INSECURESECRETS_DB_SECRETDATA_USERNAME: ${WRITABLE_INSECURESECRETS_DB_SECRETDATA_USERNAME}
+      WRITABLE_INSECURESECRETS_DB_SECRETDATA_PASSWORD: ${WRITABLE_INSECURESECRETS_DB_SECRETDATA_PASSWORD}
       SERVICE_HOST: edgex-core-keeper
     security_opt:
       - no-new-privileges:true

compose-builder/add-secure-keeper.yml

@@ -18,12 +18,11 @@ services:
     command: "/core-keeper"
     env_file:
       - common-security.env
-      - common-sec-stage-gate.env
+      - ${COMMON_SEC_STAGE_GATE_FILE_NAME}
     volumes:
       - edgex-init:/edgex-init:ro
       - /tmp/edgex/secrets/core-keeper:/tmp/edgex/secrets/core-keeper:ro,z
     depends_on:
       - security-bootstrapper
       - security-secretstore-setup
       - database
-

compose-builder/add-secure-mqtt-broker.yml

@@ -1,5 +1,6 @@
 # /*******************************************************************************
 #  * Copyright 2022 Intel Corporation.
+#  * Copyright 2024 IOTech Ltd
 #  *
 #  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 #  * in compliance with the License. You may obtain a copy of the License at
@@ -22,7 +23,7 @@ services:
     entrypoint: ["/edgex-init/messagebus_wait_install.sh"]
     env_file:
       - common-security.env
-      - common-sec-stage-gate.env
+      - ${COMMON_SEC_STAGE_GATE_FILE_NAME}
     environment:
       BROKER_TYPE: mosquitto
       CONF_DIR: /edgex-init/bootstrap-mosquitto/res

compose-builder/add-secure-mqtt-messagebus.yml

@@ -1,5 +1,6 @@
 # /*******************************************************************************
 #  * Copyright 2022 Intel Corporation.
+#  * Copyright 2024 IOTech Ltd
 #  *
 #  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
 #  * in compliance with the License. You may obtain a copy of the License at
@@ -33,7 +34,7 @@ services:
   rules-engine:
     entrypoint: [ "/edgex-init/kuiper_wait_install.sh" ]
     env_file:
-      - common-sec-stage-gate.env
+      - ${COMMON_SEC_STAGE_GATE_FILE_NAME}
     volumes:
       - kuiper-sources:/kuiper/etc/sources
       - kuiper-connections:/kuiper/etc/connections

compose-builder/add-secure-postgres.yml

@@ -0,0 +1,39 @@
+# /*******************************************************************************
+#  * Copyright 2024 IOTech Ltd
+#  *
+#  * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+#  * in compliance with the License. You may obtain a copy of the License at
+#  *
+#  * http://www.apache.org/licenses/LICENSE-2.0
+#  *
+#  * Unless required by applicable law or agreed to in writing, software distributed under the License
+#  * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+#  * or implied. See the License for the specific language governing permissions and limitations under
+#  * the License.
+#  *
+#  *******************************************************************************/
+
+# TODO: Move the content of this Docker Compose file to add-security.yml when the redis database support is removed
+
+volumes:
+  postgres-data:
+
+services:
+  database:
+    entrypoint: ["/edgex-init/postgres_wait_install.sh"]
+    env_file:
+      - common-security.env
+      - common-sec-stage-gate-postgres.env
+    environment:
+      DATABASECONFIG_PATH: /tmp/postgres-init-scripts
+      DATABASECONFIG_NAME: create-users.sh
+    tmpfs:
+      - /run
+      - /tmp
+    volumes:
+      - edgex-init:/edgex-init:ro
+      - postgres-data:/var/lib/postgresql/data
+      - /tmp/edgex/secrets/security-bootstrapper-postgres:/tmp/edgex/secrets/security-bootstrapper-postgres:ro,z
+    depends_on:
+      - security-bootstrapper
+      - security-secretstore-setup