feat: Add core-keeper service

Signed-off-by: Cherry Wang <[email protected]>
edgexfoundry · Jul 24, 2024 · bd09114 · bd09114
1 parent 8445870
commit bd09114
Show file tree

Hide file tree

Showing 69 changed files with 21,911 additions and 6,920 deletions.
diff --git a/compose-builder/.env b/compose-builder/.env
@@ -41,6 +41,10 @@ DEVICE_UART_VERSION=latest
 DEVICE_ONVIFCAM_VERSION=latest
 DEVICE_USBCAM_VERSION=latest
 DEVICE_S7_VERSION=latest
+CP_FLAGS='-cp=consul.http://edgex-core-consul:8500'
+REGISTRY_HOST=edgex-core-consul
+REGISTRY_PORT=8500
+REGISTRY_TYPE=consul
 
 # Lock on Vault 1.14 (last MPL-2.0 version)
 VAULT_VERSION=1.14
@@ -55,5 +59,3 @@ NATS_VERSION=2.9-alpine
 NGINX_VERSION=1.25-alpine-slim
 EDGEX_USER=2002
 EDGEX_GROUP=2001
-
-DEFAULT_EDGEX_RUN_CMD_PARMS="-cp=consul.http://edgex-core-consul:8500 --registry"
diff --git a/compose-builder/Makefile b/compose-builder/Makefile
@@ -34,6 +34,7 @@ EXTRA_PROXY_ROUTE_LIST=
 GEN_EXT_DIR=gen_ext_compose
 BUS=
 NANOMQ=
+REGISTRY=
 
 BROKER_YAML=add-mqtt-broker-mosquitto.yml
 TAF_BROKER_YAML=add-taf-mqtt-broker-mosquitto.yml
@@ -49,6 +50,7 @@ define OPTIONS
  - ds-coap ds-gpio ds-uart ds-s7 -
  - asc-http asc-mqtt asc-sample asc-metrics as-llrp as-record-replay asc-ex-mqtt -
  - modbus-sim zero-trust no-cleanup -
+ - consul keeper -
 endef
 export OPTIONS
 
@@ -119,6 +121,17 @@ else
     export MQTT_VERBOSE=
 endif
 
+ifeq (keeper, $(filter keeper,$(ARGS)))
+	export CP_FLAGS='-cp=keeper.http://edgex-core-keeper:59890'
+	export REGISTRY_HOST=edgex-core-keeper
+	export REGISTRY_PORT=59890
+	export REGISTRY_TYPE=keeper
+	ifneq (no-secty, $(filter no-secty,$(ARGS)))
+		export STAGEGATE_REGISTRY_HOST=edgex-core-keeper
+		export STAGEGATE_REGISTRY_PORT=59890
+	endif
+endif
+
 # When in delay-start mode, we have to make sure support serivces be delayed-start-compliant: i.e. the runtime-token configuration be added etc..
 ifeq (delayed-start, $(filter delayed-start,$(ARGS)))
 	ext_file_sup_notif:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_runtime_token_config_compose_ext.sh support-notifications)
@@ -202,7 +215,7 @@ ifeq (ds-bacnet-ip, $(filter ds-bacnet-ip,$(ARGS)))
 		else
 			EXTRA_PROXY_ROUTE_LIST:=$(EXTRA_PROXY_ROUTE_LIST),$(PROXY_ROUTE)
 		endif
-		extension_file:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-bacnet-ip device-bacnet-ip device-bacnet-ip/device-bacnet-c " -cp=consul://edgex-core-consul:8500 --registry")
+		extension_file:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-bacnet-ip device-bacnet-ip device-bacnet-ip/device-bacnet-c " --registry ${CP_FLAGS}")
 
 		COMPOSE_FILES:=$(COMPOSE_FILES) -f $(extension_file)
 		# add runtime token config for delayed-start if specified
@@ -231,7 +244,7 @@ ifeq (ds-bacnet-mstp, $(filter ds-bacnet-mstp,$(ARGS)))
 		else
 			EXTRA_PROXY_ROUTE_LIST:=$(EXTRA_PROXY_ROUTE_LIST),$(PROXY_ROUTE)
 		endif
-		extension_file:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-bacnet-mstp device-bacnet-mstp device-bacnet-mstp/device-bacnet-c " -cp=consul://edgex-core-consul:8500 --registry")
+		extension_file:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-bacnet-mstp device-bacnet-mstp device-bacnet-mstp/device-bacnet-c " --registry ${CP_FLAGS}")
 
 		COMPOSE_FILES:=$(COMPOSE_FILES) -f $(extension_file)
 		# add runtime token config for delayed-start if specified
@@ -808,6 +821,20 @@ else
 	endif
 endif
 
+# Add switch to set registry
+ifeq (keeper, $(filter keeper,$(ARGS)))
+	COMPOSE_FILES:=$(COMPOSE_FILES) -f add-keeper.yml
+	REGISTRY:=-keeper
+	ifneq (no-secty, $(filter no-secty,$(ARGS)))
+                COMPOSE_FILES:=$(COMPOSE_FILES) -f add-secure-keeper.yml
+	endif
+else
+	COMPOSE_FILES:=$(COMPOSE_FILES) -f add-consul.yml
+	ifneq (no-secty, $(filter no-secty,$(ARGS)))
+		COMPOSE_FILES:=$(COMPOSE_FILES) -f add-secure-consul.yml
+	endif
+endif
+
 # Build compose for TAF secure testing (ignore all other compose file options)
 ifeq (taf-secty, $(filter taf-secty,$(ARGS)))
 	TOKEN_LIST:=app-http-export,app-mqtt-export,app-functional-tests,app-scalability-test-mqtt-export,app-sample,device-modbus,app-external-mqtt-trigger,device-onvif-camera
@@ -846,6 +873,13 @@ ifeq (taf-secty, $(filter taf-secty,$(ARGS)))
 		IS_MQTT_BUS:=0
 	endif
 
+	ifeq (keeper, $(filter keeper,$(ARGS)))
+		COMPOSE_FILES:=$(COMPOSE_FILES) -f add-keeper.yml -f add-secure-keeper.yml
+		REGISTRY:=-keeper
+	else
+		COMPOSE_FILES:=$(COMPOSE_FILES) -f add-consul.yml -f add-secure-consul.yml
+	endif
+
 	asc_http_export_ext:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)"  ./gen_secure_compose_ext.sh app-http-export \
 		app-http-export app-service-configurable)
 	asc_mqtt_export_ext:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" IS_MQTT_BUS="$(IS_MQTT_BUS)" ./gen_secure_compose_ext.sh app-mqtt-export \
@@ -860,9 +894,9 @@ ifeq (taf-secty, $(filter taf-secty,$(ARGS)))
 	# taf has its special place holder from taf-device-services-mods and thus we need to keep it
 	# and extend security related things on top of it
 	ds_virtual_ext:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-virtual \
-		device-virtual device-virtual ' -cp=consul.http:\/\/edgex-core-consul:8500 --registry --configDir=CONFIG_DIR_PLACE_HOLDER')
+		device-virtual device-virtual " ${CP_FLAGS} --configDir=CONFIG_DIR_PLACE_HOLDER")
 	ds_modbus_ext:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-modbus \
-		device-modbus device-modbus ' -cp=consul.http:\/\/edgex-core-consul:8500 --registry --configDir=CONFIG_DIR_PLACE_HOLDER')
+		device-modbus device-modbus " ${CP_FLAGS} --configDir=CONFIG_DIR_PLACE_HOLDER")
 	ds_camera_ext:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-onvif-camera)
 	COMPOSE_FILES:=$(COMPOSE_FILES) -f $(asc_http_export_ext) -f $(asc_mqtt_export_ext) -f $(asc_external_mqtt_trigger_ext) -f $(scalability_mqtt_export_ext) -f $(asc_sample_ext)
 	COMPOSE_FILES:=$(COMPOSE_FILES) -f $(ds_virtual_ext) -f $(ds_rest_ext) -f $(ds_modbus_ext) -f $(ds_camera_ext)
@@ -897,6 +931,13 @@ else
             IS_SECURE_MODE:=0
             COMPOSE_FILES:=$(COMPOSE_FILES) -f add-mqtt-messagebus.yml
         endif
+
+		ifeq (keeper, $(filter keeper,$(ARGS)))
+			COMPOSE_FILES:=$(COMPOSE_FILES) -f add-keeper.yml
+			REGISTRY:=-keeper
+		else
+			COMPOSE_FILES:=$(COMPOSE_FILES) -f add-consul.yml
+		endif
     else
     	# Build compose for TAF secure performance testing (ignore all other compose file options)
     	ifeq (taf-perf, $(filter taf-perf,$(ARGS)))
@@ -907,16 +948,24 @@ else
 			EXTRA_PROXY_ROUTE_LIST:=device-modbus.http://edgex-device-modbus:59901
 
     		COMPOSE_FILES:= \
-    			-f docker-compose-base.yml \
-    			-f add-security.yml \
-    			-f add-security-proxy.yml \
-    			-f add-secure-redis-messagebus.yml \
-    			-f add-asc-mqtt-export.yml \
-    			-f add-device-virtual.yml \
-    			-f add-device-rest.yml \
-    			-f add-mqtt-broker-mosquitto.yml \
+				-f docker-compose-base.yml \
+				-f add-security.yml \
+				-f add-security-proxy.yml \
+				-f add-secure-redis-messagebus.yml \
+				-f add-asc-mqtt-export.yml \
+				-f add-device-virtual.yml \
+				-f add-device-rest.yml \
+				-f add-mqtt-broker-mosquitto.yml \
 				-f add-taf-mqtt-broker-mosquitto.yml \
-			-f add-delayed-start-services.yml
+				-f add-delayed-start-services.yml
+
+			ifeq (keeper, $(filter keeper,$(ARGS)))
+				COMPOSE_FILES:=$(COMPOSE_FILES) -f add-keeper.yml -f add-secure-keeper.yml
+				REGISTRY:=-keeper
+			else
+				COMPOSE_FILES:=$(COMPOSE_FILES) -f add-consul.yml -f add-secure-consul.yml
+			endif
+
 			asc_mqtt_export_ext:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)"  IS_MQTT_BUS="0" ./gen_secure_compose_ext.sh app-mqtt-export \
 				app-mqtt-export app-service-configurable)
 			ds_virtual_ext:= $(shell ZERO_TRUST="$(MAKE_ZERO_TRUST)" GEN_EXT_DIR="$(GEN_EXT_DIR)" ./gen_secure_compose_ext.sh device-virtual)
@@ -933,6 +982,12 @@ else
 					-f ${BROKER_YAML} \
 					-f ${TAF_BROKER_YAML}
 
+				ifeq (keeper, $(filter keeper,$(ARGS)))
+					COMPOSE_FILES:=$(COMPOSE_FILES) -f add-keeper.yml
+					REGISTRY:=-keeper
+				else
+					COMPOSE_FILES:=$(COMPOSE_FILES) -f add-consul.yml
+				endif
 				NO_SECURITY:=-no-secty
 			endif
 		endif
@@ -983,6 +1038,10 @@ define COMPOSE_DOWN
 		-f add-security-proxy.yml \
 		-f add-secure-redis-messagebus.yml \
 		-f add-delayed-start-services.yml \
+		-f add-consul.yml \
+		-f add-secure-consul.yml \
+		-f add-keeper.yml \
+		-f add-secure-keeper.yml \
 		down $1
 endef
 
@@ -1029,20 +1088,30 @@ build-taf:
 	make taf-compose taf-no-secty mqtt-bus mqtt-verbose
 	make taf-compose taf-secty mqtt-bus mqtt-verbose arm64
 	make taf-compose taf-no-secty mqtt-bus mqtt-verbose arm64
+	make taf-compose taf-secty keeper
+	make taf-compose taf-no-secty keeper
+	make taf-compose taf-secty keeper arm64
+	make taf-compose taf-no-secty keeper arm64
+	make taf-compose taf-secty keeper mqtt-bus mqtt-verbose
+	make taf-compose taf-no-secty keeper mqtt-bus mqtt-verbose
+	make taf-compose taf-secty keeper mqtt-bus mqtt-verbose arm64
+	make taf-compose taf-no-secty keeper mqtt-bus mqtt-verbose arm64
 	make taf-compose-perf taf-perf
 	make taf-compose-perf taf-perf-no-secty
 	make taf-compose-perf taf-perf arm64
 	make taf-compose-perf taf-perf-no-secty arm64
 
 build-taf-nanomq:
 	make taf-compose taf-no-secty mqtt-bus nanomq no-secty
+	make taf-compose taf-no-secty keeper mqtt-bus nanomq no-secty
 	make taf-compose-perf taf-perf-no-secty mqtt-bus nanomq no-secty
+	make taf-compose-perf taf-perf-no-secty consul mqtt-bus nanomq no-secty
 
 compose: gen
 	cat gen-header docker-compose.yml > $(RELEASE_FOLDER)/docker-compose$(NO_SECURITY)$(APP_SAMPLE)$(BUS)$(NANOMQ)$(ZERO_TRUST_COMPOSE)$(ARCH).yml
 
 taf-compose: gen
-	cat gen-header docker-compose.yml > $(RELEASE_FOLDER)/taf/docker-compose-taf$(NO_SECURITY)$(BUS)$(NANOMQ)$(ARCH).yml
+	cat gen-header docker-compose.yml > $(RELEASE_FOLDER)/taf/docker-compose-taf$(NO_SECURITY)$(BUS)$(NANOMQ)$(REGISTRY)$(ARCH).yml
 
 taf-compose-perf: gen
 	cat gen-header docker-compose.yml > $(RELEASE_FOLDER)taf/docker-compose-taf-perf$(NO_SECURITY)$(BUS)$(NANOMQ)$(ARCH).yml

diff --git a/compose-builder/README.md b/compose-builder/README.md
@@ -128,6 +128,10 @@ This folder contains the following compose files:
     TAF App Services **extending** `add-taf-app-services` compose file, and services are enabled with secret store by default.
 - **add-taf-device-services-mods.yml**<br/>
     TAF Device Services **extending** compose file. Modifies setting of Device Virtual and Device Modbus for the TAF testing compose files. **Must be used in conjunction with add-device-modbus.yml and add-device-virtual.yml**
+- **add-keeper.yml**<br/>
+    Registry Service **extending** compose file. Adds the **Core Keeper** service.
+- **add-consul.yml**<br/>
+    Registry Service **extending** compose file. Adds the **Consul** service.
 
 ### Environment Files
 
@@ -182,11 +186,23 @@ Standard compose variations are:
    full secure general testing (docker-compose-taf.yml)
    full secure general testing for arm64 (docker-compose-taf-arm64.yml)
    non-secure general testing (docker-compose-taf-no-secty.yml)
-   nonsecure general testing for arm64 (docker-compose-taf-no-secty-arm64.yml)
+   non-secure general testing for arm64 (docker-compose-taf-no-secty-arm64.yml)
    full secure perf testing (docker-compose-taf-perf.yml)
    full secure perf testing for arm64 (docker-compose-taf-perf-arm64.yml)
    non-secure perf testing (docker-compose-taf-perf-no-secty.yml)
-   nonsecure perf testing for arm64 (docker-compose-taf-perf-no-secty-arm64.yml)
+   non-secure perf testing for arm64 (docker-compose-taf-perf-no-secty-arm64.yml)
+   full secure general testing with mqtt-bus (docker-compose-taf-mqtt-bus.yml)
+   full secure general testing with mqtt-bus for arm64 (docker-compose-taf-mqtt-bus-arm64.yml)
+   non-secure general testing with mqtt-bus (docker-compose-taf-no-secty-mqtt-bus.yml)
+   non-secure general testing with mqtt-bus for arm64 (docker-compose-taf-no-secty-mqtt-bus-arm64.yml)
+   full secure general testing with core-keeper (docker-compose-taf-keeper.yml)
+   full secure general testing with core-keeper for arm64 (docker-compose-taf-keeper-arm64.yml)
+   non-secure general testing with core-keeper (docker-compose-taf-no-secty-keeper.yml)
+   non-secure general testing with core-keeper for arm64 (docker-compose-taf-no-secty-keeper-arm64.yml)
+   full secure general testing with mqtt-bus with core-keeper (docker-compose-taf-mqtt-bus-keeper.yml)
+   full secure general testing with mqtt-bus with core-keeper for arm64 (docker-compose-taf-mqtt-bus-keeper-arm64.yml)
+   non-secure general testing with mqtt-bus with core-keeper (docker-compose-taf-no-secty-mqtt-bus-keeper.yml)
+   non-secure general testing with mqtt-bus with core-keeper for arm64 (docker-compose-taf-no-secty-mqtt-bus-keeper-arm64.yml)
 ```
 #### Run
 
@@ -239,6 +255,7 @@ Options:
     nats-bus:         Runs with services configure for NATS Message Bus
                       The NATS Server service is also included.
     no-cleanup:       Leaves generated files behind for debugging purposes.
+    keeper:           Runs to registry service to core-keeper
 Services:
     <names...>: Runs only services listed (and their dependent services) where 'name' matches a service name in one of the compose files used
 ```
@@ -296,7 +313,8 @@ Options:
                       Pull includes NonoMQ MQTT broker when mqtt-broker or mqtt-bus are specified
                       Not valid in secure mode when uses with mqtt-bus                    
     nats-bus:         Pull includes additional services for NATS Message Bus
-    no-cleanup:       Leaves generated files behind for debugging purposes.
+    no-cleanup:       Leaves generated files behind for debugging purposes
+    keeper:           Pull includes core-keeper
 
 Services:
     <names...>: Pulls only images for the service(s) listed
@@ -353,6 +371,8 @@ Options:
     nats-bus:         Generates compose file with services configured for NAT Message Bus
                       The NATS Server service is also included.
     no-cleanup:       Leaves generated files behind for debugging purposes.
+    keeper:           Generates compose file with services registry to core-keeper
+                      The core-keeper service is also included
 ```
 #### Clean
 
@@ -465,6 +485,7 @@ Options:
     nats-bus:         Generates compose file with services configure for NATS Message Bus
                       The NATS Server service is also included.
     no-cleanup:       Leaves generated files behind for debugging purposes.
+    keeper:           Generates compose file to registry service to core-keeper
 ```
 
 #### TAF Compose
@@ -477,6 +498,7 @@ Options:
     taf-secty:	  Generates general TAF testing compose file with security services
     taf-no-secty: Generates general TAF testing compose file without security services
     arm64:        Generates TAF compose file using ARM64 images
+    keeper:       Generates compose file to registry service to core-keeper
 ```
 
 #### Taf Perf Compose

diff --git a/compose-builder/add-app-record-replay.yml b/compose-builder/add-app-record-replay.yml
@@ -18,6 +18,7 @@ version: '3.7'
 services:
   app-record-replay:
     image: ${APP_SVC_REPOSITORY}/app-record-replay${ARCH}:${APP_RECORD_REPLAY_VERSION}
+    command: "--registry ${CP_FLAGS}"
     ports:
       - 127.0.0.1:59712:59712/tcp
     container_name: edgex-app-record-replay
@@ -27,7 +28,7 @@ services:
     environment:
       SERVICE_HOST: edgex-app-record-replay
     depends_on:
-      - consul
+      - ${REGISTRY_TYPE}
       - database
       - core-metadata
       - core-common-config-bootstrapper

diff --git a/compose-builder/add-app-rfid-llrp-inventory.yml b/compose-builder/add-app-rfid-llrp-inventory.yml
@@ -21,6 +21,7 @@ volumes:
 services:
   app-rfid-llrp-inventory:
     image: ${APP_SVC_REPOSITORY}/app-rfid-llrp-inventory${ARCH}:${APP_LLRP_VERSION}
+    command: "--registry ${CP_FLAGS}"
     ports:
       - 127.0.0.1:59711:59711/tcp
     container_name: edgex-app-rfid-llrp-inventory
@@ -30,7 +31,7 @@ services:
     environment:
       SERVICE_HOST: edgex-app-rfid-llrp-inventory
     depends_on:
-      - consul
+      - ${REGISTRY_TYPE}
       - core-data
       - core-metadata
       - core-common-config-bootstrapper

diff --git a/compose-builder/add-asc-external-mqtt-trigger.yml b/compose-builder/add-asc-external-mqtt-trigger.yml
@@ -18,6 +18,7 @@ version: '3.7'
 services:
   app-external-mqtt-trigger:
     image: ${APP_SVC_REPOSITORY}/app-service-configurable${ARCH}:${APP_SERVICE_CONFIG_VERSION}
+    command: "--registry ${CP_FLAGS}"
     ports:
       - 127.0.0.1:59706:59706/tcp
     container_name: edgex-app-external-mqtt-trigger
@@ -32,7 +33,7 @@ services:
       WRITABLE_PIPELINE_FUNCTIONS_MQTTEXPORT_PARAMETERS_TOPIC: edgex-export
       WRITABLE_LOGLEVEL: INFO
     depends_on:
-      - consul
+      - ${REGISTRY_TYPE}
       - core-data
       - core-metadata
       - core-common-config-bootstrapper

diff --git a/compose-builder/add-asc-http-export.yml b/compose-builder/add-asc-http-export.yml
@@ -18,6 +18,7 @@ version: '3.7'
 services:
   app-http-export:
     image: ${APP_SVC_REPOSITORY}/app-service-configurable${ARCH}:${APP_SERVICE_CONFIG_VERSION}
+    command: "--registry ${CP_FLAGS}"
     ports:
       - 127.0.0.1:59704:59704/tcp
     container_name: edgex-app-http-export
@@ -30,7 +31,7 @@ services:
       WRITABLE_PIPELINE_FUNCTIONS_HTTPEXPORT_PARAMETERS_URL: "http://EXPORT_HOST_PLACE_HOLDER:7770"
       WRITABLE_LOGLEVEL: INFO  # allows scripts to find and change with sed
     depends_on:
-      - consul
+      - ${REGISTRY_TYPE}
       - core-data
       - core-metadata
       - core-common-config-bootstrapper

diff --git a/compose-builder/add-asc-metrics-influxdb.yml b/compose-builder/add-asc-metrics-influxdb.yml
@@ -18,6 +18,7 @@ version: '3.7'
 services:
   app-metrics-influxdb:
     image: ${APP_SVC_REPOSITORY}/app-service-configurable${ARCH}:${APP_SERVICE_CONFIG_VERSION}
+    command: "--registry ${CP_FLAGS}"
     ports:
       - 127.0.0.1:59707:59707/tcp
     container_name: edgex-app-metrics-influxdb
@@ -28,7 +29,7 @@ services:
       SERVICE_HOST: edgex-app-metrics-influxdb
       EDGEX_PROFILE: metrics-influxdb
     depends_on:
-      - consul
+      - ${REGISTRY_TYPE}
       - core-metadata
       - core-common-config-bootstrapper
     read_only: true