diff --git a/docs_src/about.md b/docs_src/about.md index 828134966b..36fb196bc5 100644 --- a/docs_src/about.md +++ b/docs_src/about.md @@ -168,7 +168,7 @@ SDKs are language specific; meaning an SDK is written to create services in a pa ### Sensor Data Collection EdgeX’s primary job is to collect data from sensors and devices and make that data available to north side applications and systems. Data is collected from a sensor by a device service that speaks the protocol of that device. Example: a Modbus device service would communicate in Modbus to get a pressure reading from a Modbus pump. The device service translates the sensor data into an EdgeX event object. The device service can then either: -1. put the event object on a message bus (which may be implemented via Redis Streams or MQTT). Subscribers to the event message on the message bus can be application services or core data or both (see step 1.1 below). +1. put the event object on a message bus (which may be implemented via MQTT). Subscribers to the event message on the message bus can be application services or core data or both (see step 1.1 below). ![image](./general/EdgeX_step1-1.png) @@ -176,12 +176,12 @@ EdgeX’s primary job is to collect data from sensors and devices and make that ![image](./general/EdgeX_step1-2.png) -When core data receives the event (either via message bus or REST), it persists the sensor data in the local edge database. EdgeX uses Redis as our persistence store. There is an abstraction in place to allow you to use another database (which has allowed other databases to be used in the past). Persistence is not required and can be turned off. Data is persisted in EdgeX at the edge for two basics reasons: +When core data receives the event (either via message bus or REST), it persists the sensor data in the local edge database. EdgeX uses PostgreSQL as our persistence store. There is an abstraction in place to allow you to use another database (which has allowed other databases to be used in the past). Persistence is not required and can be turned off. Data is persisted in EdgeX at the edge for two basics reasons: - Edge nodes are not always connected. During periods of disconnected operations, the sensor data must be saved so that it can be transmitted northbound when connectivity is restored. This is referred to as store and forward capability. - In some cases, analytics of sensor data needs to look back in history in order to understand the trend and to make the right decision based on that history. If a sensor reports that it is 72° F right now, you might want to know what the temperature was ten minutes ago before you make a decision to adjust a heating or cooling system. If the temperature was 85° F, you may decide that adjustments to lower the room temperature you made ten minutes ago were sufficient to cool the room. It is the context of historical data that are important to local analytic decisions. -When core data receives event objects from the device service via REST, it will put sensor data events on a message topic destined for application services. Redis Pub/Sub is used as the messaging infrastructure by default (step 2). MQTT or NATS (opt-in during build) can also be used as the messaging infrastructure between core data and the application services. +When core data receives event objects from the device service via REST, it will put sensor data events on a message topic destined for application services. MQTT is used as the messaging infrastructure by default (step 2). NATS (opt-in during build) can also be used as the messaging infrastructure between core data and the application services. ![image](./general/EdgeX_step2.png) diff --git a/docs_src/design/ucr/Common Configuration.md b/docs_src/design/ucr/Common Configuration.md index afaadc9fcb..ae378b5e1d 100644 --- a/docs_src/design/ucr/Common Configuration.md +++ b/docs_src/design/ucr/Common Configuration.md @@ -65,12 +65,12 @@ In the Levski release the additional common security metrics require all service # Gateway="my-iot-gateway" # Tag must be added here or via Consul Env Override can only chnage existing value, not added new ones. ... [MessageQueue] - Protocol = "redis" + Protocol = "mqtt" Host = "localhost" Port = 6379 - Type = "redis" - AuthMode = "usernamepassword" # required for redis messagebus (secure or insecure). - SecretName = "redisdb" + Type = "mqtt" + AuthMode = "usernamepassword" # required for mqtt messagebus (secure or insecure). + SecretName = "mqtt-bus" PublishTopicPrefix = SubscribeEnabled = SubscribeTopic = diff --git a/docs_src/general/ContainerNames.md b/docs_src/general/ContainerNames.md index bac7533d73..fcded76132 100644 --- a/docs_src/general/ContainerNames.md +++ b/docs_src/general/ContainerNames.md @@ -40,7 +40,7 @@ The following table provides the list of the default EdgeX Docker image names to === "Security" |Docker image name|Docker container name |Docker network hostname|Docker Compose service name| |---|---|---|---| - |vault|edgex-vault|edgex-vault|vault| + |openbao|edgex-secret-store|edgex-secret-store|secret-store| |nginx|edgex-nginx|edgex-nginx|nginx| |edgexfoundry/security-proxy-auth|edgex-proxy-auth|edgex-proxy-auth|security-proxy-auth| |edgexfoundry/security-proxy-setup|edgex-security-proxy-setup|edgex-security-proxy-setup|security-proxy-setup| @@ -49,5 +49,4 @@ The following table provides the list of the default EdgeX Docker image names to === "Miscellaneous" |Docker image name|Docker container name |Docker network hostname|Docker Compose service name| |---|---|---|---| - |consul|edgex-core-consul|edgex-core-consul|consul| - |redis|edgex-redis|edgex-redis|database| \ No newline at end of file + |postgres|edgex-postgres|edgex-postgres|database| \ No newline at end of file diff --git a/docs_src/general/EdgeX_CN.md b/docs_src/general/EdgeX_CN.md index 868208168d..b9007330ba 100644 --- a/docs_src/general/EdgeX_CN.md +++ b/docs_src/general/EdgeX_CN.md @@ -97,7 +97,7 @@ EdgeX has not adopted the [Agile Manifesto](https://agilemanifesto.org/), but th **Distributed** -EdgeX is a micro service architecture. Services communicate with each other via REST or message bus and that communication can occur across nodes (aka machines, hosts, etc.). Services have even been built to wait and continue to attempt to communicate with a dependent service - allowing for some resiliency. As such, EdgeX is, at its core, distributable. It was designed such that the services could operate largely independently and on top of whatever limited resources are available at the edge. As an example deployment, an EdgeX device service could run on a Raspberry Pi or smaller compute platform that is directly connected by GPIO to a physical sensor, while the core services are run on an edge gateway, and the application and analytic services (rules engine) run on an edge service. This would allow each service to maximize the available resources available to the solution. Having said that, there are some complexities around real world distributed solutions that adopters would still need to solve depending on their use case and environment. For example, while services can communicate across a distributed set of nodes, the communications between EdgeX services are not secure by default (as would be provided via something like a cloud native service mesh). Adopters would need to provide for their own means to secure all traffic between services in most production environments. Service discovery is not fully implemented. EdgeX services do register with a service registry (Consul) but the services do not use that registry to locate other services. If a service changed location, other services would need to have their configuration changed in order to know and use the service at its new location. Finally, latency is a real concern in edge systems. In addition to service to service communications, most services use stores of information (Redis for data, Vault for secrets, Consul for configuration) which could also be distributed. These are referred to as backing services in cloud native terminology. Even if the communications were secure, if these stores or other services are all distributed, then the additional latency to constantly communicate with services and stores may not be conducive to the edge use case it supports. Each "hop" on a network of distributed services costs and that cost adds up when building solutions that operate and manage physical edge capability. +EdgeX is a micro service architecture. Services communicate with each other via REST or message bus and that communication can occur across nodes (aka machines, hosts, etc.). Services have even been built to wait and continue to attempt to communicate with a dependent service - allowing for some resiliency. As such, EdgeX is, at its core, distributable. It was designed such that the services could operate largely independently and on top of whatever limited resources are available at the edge. As an example deployment, an EdgeX device service could run on a Raspberry Pi or smaller compute platform that is directly connected by GPIO to a physical sensor, while the core services are run on an edge gateway, and the application and analytic services (rules engine) run on an edge service. This would allow each service to maximize the available resources available to the solution. Having said that, there are some complexities around real world distributed solutions that adopters would still need to solve depending on their use case and environment. For example, while services can communicate across a distributed set of nodes, the communications between EdgeX services are not secure by default (as would be provided via something like a cloud native service mesh). Adopters would need to provide for their own means to secure all traffic between services in most production environments. Service discovery is not fully implemented. EdgeX services do register with a service registry (Core-Keeper) but the services do not use that registry to locate other services. If a service changed location, other services would need to have their configuration changed in order to know and use the service at its new location. Finally, latency is a real concern in edge systems. In addition to service to service communications, most services use stores of information (PostgreSQL for data, OpenBao for secrets, Core-Keeper for configuration) which could also be distributed. These are referred to as backing services in cloud native terminology. Even if the communications were secure, if these stores or other services are all distributed, then the additional latency to constantly communicate with services and stores may not be conducive to the edge use case it supports. Each "hop" on a network of distributed services costs and that cost adds up when building solutions that operate and manage physical edge capability. ### Cloud Native Ingredients Not In EdgeX (and why) diff --git a/docs_src/general/PlatformRequirements.md b/docs_src/general/PlatformRequirements.md index a88de38382..ca172b511e 100644 --- a/docs_src/general/PlatformRequirements.md +++ b/docs_src/general/PlatformRequirements.md @@ -4,11 +4,12 @@ EdgeX Foundry is an operating system (OS)-agnostic and hardware (HW)-agnostic Io === "Memory" Memory: minimum of 1 GB - When considering memory for your EdgeX platform consider your use of database - Redis is the current default. Redis is an open source (BSD licensed), in-memory data structure store, used as a database and message broker in EdgeX. Redis is durable and uses persistence only for recovering state; the only data Redis operates on is in-memory. Redis uses a number of techniques to optimize memory utilization. Antirez and Redis Labs have written a number of articles on the underlying details (see list below). Those strategies has continued to [evolve](http://antirez.com/news/128). When thinking about your system architecture, consider how long data will be living at the edge and consuming memory (physical or physical + virtual). + When considering memory for your EdgeX platform, take into account your database choice - PostgreSQL is the default in EdgeX 4.0. PostgreSQL is a powerful, open-source relational database management system that offers robust data storage, query capabilities, and performance optimization. PostgreSQL stores data on disk and leverages advanced caching strategies to enhance performance. - - [Antirez](http://antirez.com/news/92) - - [Redis RAM Ramifications](https://redislabs.com/blog/redis-ram-ramifications-part-i/) - - [Redis IO Memory Optimization](https://redis.io/topics/memory-optimization) + For more information about PostgreSQL's memory utilization and optimization techniques, refer to the following resources: + + - [PostgreSQL Documentation: Resource Consumption](https://www.postgresql.org/docs/current/runtime-config-resource.html) + - [Tuning PostgreSQL Memory Parameters](https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server) === "Storage" Hard drive space: minimum of 3 GB of space to run the EdgeX Foundry containers, but you may want more depending on how long sensor and device data is to be retained. Approximately 32GB of storage is minimally recommended to start. diff --git a/docs_src/general/ServicePorts.md b/docs_src/general/ServicePorts.md index a5ef66a7d5..dc355494a6 100644 --- a/docs_src/general/ServicePorts.md +++ b/docs_src/general/ServicePorts.md @@ -7,7 +7,7 @@ The following tables (organized by type of service) capture the default service |core-data| 59880| |core-metadata |59881| |core-command |59882| - |redis|6379| + |postgres|5432| |consul|8500| === "Supporting" |Services Name| Port Definition| diff --git a/docs_src/getting-started/Ch-GettingStartedDockerUsers.md b/docs_src/getting-started/Ch-GettingStartedDockerUsers.md index 71959c0d76..8b15311fc9 100644 --- a/docs_src/getting-started/Ch-GettingStartedDockerUsers.md +++ b/docs_src/getting-started/Ch-GettingStartedDockerUsers.md @@ -48,10 +48,10 @@ Find the Docker Compose file that matches: |filename|Docker Compose contents| |---|---| -|docker-compose-arm64.yml|Specifies x86 containers, uses Redis database for persistence, and includes security services| -|docker-compose-no-secty-arm64.yml|Specifies ARM 64 containers, uses Redis database for persistence, but does not include security services| -|docker-compose-no-secty.yml|Specifies x86 containers, uses Redis database for persistence, but does not include security services| -|docker-compose.yml|Specifies x86 containers, uses Redis database for persistence, and includes security services| +|docker-compose-arm64.yml|Specifies x86 containers, uses PostgreSQL database for persistence, and includes security services| +|docker-compose-no-secty-arm64.yml|Specifies ARM 64 containers, uses PostgreSQL database for persistence, but does not include security services| +|docker-compose-no-secty.yml|Specifies x86 containers, uses PostgreSQL database for persistence, but does not include security services| +|docker-compose.yml|Specifies x86 containers, uses PostgreSQL database for persistence, and includes security services| |docker-compose-no-secty-with-ui-arm64.|Same as docker-compose-no-secty-arm64.yml but also includes EdgeX user interface| |docker-compose-no-secty-with-ui.yml|Same as docker-compose-no-secty.yml but also includes EdgeX user interface| |docker-compose-portainer.yml|Specifies the Portainer user interface extension (to be used with the x86 or ARM EdgeX platform)| diff --git a/docs_src/getting-started/Ch-GettingStartedGoDevelopers.md b/docs_src/getting-started/Ch-GettingStartedGoDevelopers.md index b3677a07b7..ef25c43105 100644 --- a/docs_src/getting-started/Ch-GettingStartedGoDevelopers.md +++ b/docs_src/getting-started/Ch-GettingStartedGoDevelopers.md @@ -108,8 +108,7 @@ make build Several of the EdgeX Foundry micro services use a database. This includes core-data, core-metadata, support-scheduler, among others. Therefore, when working with EdgeX Foundry its a good idea to have the database up and -running as a general rule. See the [Redis Quick Start Guide](https://redis.io/topics/quickstart) -for how to run Redis in a Linux environment (or find similar documentation for other environments). +running as a general rule. See the [PostgreSQL Quick Start Guide](https://www.postgresql.org/docs/current/tutorial-start.html)for how to run PostgreSQL in a Linux environment (or find similar documentation for other environments). ### Run EdgeX Services diff --git a/docs_src/microservices/application/details/Triggers.md b/docs_src/microservices/application/details/Triggers.md index 1b344f3672..1c1be29986 100644 --- a/docs_src/microservices/application/details/Triggers.md +++ b/docs_src/microservices/application/details/Triggers.md @@ -19,7 +19,7 @@ There are 4 types of `Triggers` supported in the App Functions SDK which are dis An EdgeX MessageBus trigger will execute the pipeline every time data is received from the configured Edgex MessageBus `SubscribeTopics`. The EdgeX MessageBus is the central message bus internal to EdgeX and has a specific message envelope that wraps all data published to this message bus. -There currently are four implementations of the EdgeX MessageBus available to be used. Two of these are available out of the box: `Redis Pub/Sub`(default) and `MQTT`. Additionally NATS (both core and JetStream) options can be made available with the build flag mentioned above. The implementation type is selected via the `[Trigger.EdgexMessageBus]` configuration described below. +There currently are four implementations of the EdgeX MessageBus available to be used. Two of these are available out of the box: `MQTT`(default) . Additionally NATS (both core and JetStream) options can be made available with the build flag mentioned above. The implementation type is selected via the `[Trigger.EdgexMessageBus]` configuration described below. ### Type Configuration diff --git a/docs_src/microservices/application/sdk/details/StoreAndForward.md b/docs_src/microservices/application/sdk/details/StoreAndForward.md index 36cbdb2beb..132f468cfe 100644 --- a/docs_src/microservices/application/sdk/details/StoreAndForward.md +++ b/docs_src/microservices/application/sdk/details/StoreAndForward.md @@ -30,9 +30,9 @@ Database configuration section describes which database type to use and the info !!! example "Example - Database configuration" ```yaml Database: - Type: "redisdb" + Type: "postgres" Host: "localhost" - Port: 6379 + Port: 5432 Timeout: "5s" ``` @@ -66,7 +66,7 @@ One of three outcomes can occur after the export retried has completed. Changing Writable.Pipeline.ExecutionOrder will invalidate all currently stored data and result in it all being removed from the database on the next retry. This is because the position of the *export* function can no longer be guaranteed and no way to ensure it is properly executed on the retry. ## Custom Storage -The default backing store is redis. Custom implementations of the `StoreClient` interface can be provided if redis does not meet your requirements. +The default backing store is PostgreSQL. Custom implementations of the `StoreClient` interface can be provided if PostgreSQL does not meet your requirements. ```go type StoreClient interface { diff --git a/docs_src/microservices/application/services/AppServiceConfigurable/details/Profiles.md b/docs_src/microservices/application/services/AppServiceConfigurable/details/Profiles.md index c520292ef0..5950780aec 100644 --- a/docs_src/microservices/application/services/AppServiceConfigurable/details/Profiles.md +++ b/docs_src/microservices/application/services/AppServiceConfigurable/details/Profiles.md @@ -24,7 +24,7 @@ The following profiles and their purposes are provided with App Service Configur ### rules-engine -Profile used to push Event messages to the Rules Engine via the **Redis Pub/Sub** Message Bus. This is used in the default docker compose files for the `app-rules-engine` service +Profile used to push Event messages to the Rules Engine via the **MQTT** Message Bus. This is used in the default docker compose files for the `app-rules-engine` service One can optionally add Filter function via environment overrides diff --git a/docs_src/microservices/configuration/CommonConfiguration.md b/docs_src/microservices/configuration/CommonConfiguration.md index ec15759673..503ab49e6f 100644 --- a/docs_src/microservices/configuration/CommonConfiguration.md +++ b/docs_src/microservices/configuration/CommonConfiguration.md @@ -43,7 +43,7 @@ The tables in each of the tabs below document configuration properties that are |---|---|---| |||entries in the Writable section of the configuration can be changed on the fly while the service is running if the service is running with the `-cp/--configProvider` flag| |LogLevel|---|log entry [severity level](https://en.wikipedia.org/wiki/Syslog#Severity_level). (specific for each service) | - |**InsecureSecrets**|---|This section a map of secrets which simulates the SecretStore for accessing secrets when running in non-secure mode. All services have a default entry for Redis DB credentials called `redisdb`| + |**InsecureSecrets**|---|This section a map of secrets which simulates the SecretStore for accessing secrets when running in non-secure mode. All services have a default entry for PostgreSQL DB credentials called `postgres`| !!! note @@ -95,25 +95,25 @@ The tables in each of the tabs below document configuration properties that are |---|---|---| |||configuration that govern database connectivity and the type of database to use. While not all services require DB connectivity, most do and so this has been included in the common configuration docs.| |Host |localhost |DB host name| - |Port |6379 |DB port number| + |Port |5432 |DB port number| |Name |---- |Database or document store name (Specific to the service) | |Timeout |5s |DB connection timeout | - |Type |redisdb |DB type. Redis is the only supported DB| + |Type |postgres |DB type. | === "MessageBus" |Property|Default Value|Description| |---|---|---| ||Entries in the MessageBus section of the configuration allow for connecting to the internal MessageBus and define a common base topic prefix| - |Protocol | redis| Indicates the connectivity protocol to use when connecting to the bus.| + |Protocol | mqtt| Indicates the connectivity protocol to use when connecting to the bus.| |Host | localhost | Indicates the host of the messaging broker, if applicable.| - |Port | 6379| Indicates the port to use when publishing a message.| - |Type | redis| Indicates the type of messaging library to use. Currently this is Redis by default. Refer to the [go-mod-messaging](https://github.com/edgexfoundry/go-mod-messaging) module for more information. | + |Port | 1883| Indicates the port to use when publishing a message.| + |Type | mqtt| Indicates the type of messaging library to use. Currently this is MQTT by default. Refer to the [go-mod-messaging](https://github.com/edgexfoundry/go-mod-messaging) module for more information. | |AuthMode | usernamepassword| Auth Mode to connect to EdgeX MessageBus.| - |SecretName | redisdb | Name of the secret in the Secret Store to find the MessageBus credentials.| + |SecretName | mqtt-bus | Name of the secret in the Secret Store to find the MessageBus credentials.| |BaseTopicPrefix | edgex| Indicates the base topic prefix which is prepended to all internal MessageBus topics. | === "MessageBus.Optional" |Property|Default Value|Description| |---|---|---| - ||Configuration and connection parameters for use with MQTT or NATS message bus - in place of Redis| + ||Configuration and connection parameters for use with MQTT or NATS message bus| |ClientId| ---|Client ID used to put messages on the bus (specific for each service)| |Qos|'0'| Quality of Service values are 0 (At most once), 1 (At least once) or 2 (Exactly once)| |KeepAlive |'10'| Period of time in seconds to keep the connection alive when there are no messages flowing (must be 2 or greater)| diff --git a/docs_src/microservices/core/command/GettingStarted.md b/docs_src/microservices/core/command/GettingStarted.md index de6a51fa4b..4d9a388fa8 100644 --- a/docs_src/microservices/core/command/GettingStarted.md +++ b/docs_src/microservices/core/command/GettingStarted.md @@ -12,7 +12,7 @@ The 3rd party system makes a REST call of the command service which then relays There was no built-in means to make a message-based request of EdgeX or the devices/sensors it manages. From Levski release, core command service adds support for an external MQTT connection (in the same manner that app services provide an external MQTT connection), -which will allow it to act as a bridge between the internal message bus (implemented via either MQTT or Redis Pub/Sub) and external MQTT message bus. +which will allow it to act as a bridge between the internal message bus (implemented via MQTT) and external MQTT message bus. #### Core Command as Message Bus Bridge diff --git a/docs_src/microservices/core/data/GettingStarted.md b/docs_src/microservices/core/data/GettingStarted.md index b159c20c10..10566450ea 100644 --- a/docs_src/microservices/core/data/GettingStarted.md +++ b/docs_src/microservices/core/data/GettingStarted.md @@ -22,7 +22,7 @@ The simplest way to run all the required services is to use the [Compose Builder make run no-secty ds-virtual ``` This runs, in non-secure mode, all the standard EdgeX services, including core-data, along with the Device Virtual. -Core Data will use the Redis database as its datastore. +Core Data will use the PostgreSQL database as its datastore. ## Running Services without Core Data diff --git a/docs_src/microservices/core/data/Purpose.md b/docs_src/microservices/core/data/Purpose.md index 7d7cdc2aec..0d7e729574 100644 --- a/docs_src/microservices/core/data/Purpose.md +++ b/docs_src/microservices/core/data/Purpose.md @@ -8,7 +8,7 @@ The Core Data microservice provides a centralized persistence for data collected Device services that collect sensor data call on the core data service to store the sensor data on the edge system (such as in a [gateway](../../../general/Definitions.md#gateway)) until the data gets moved "north" and then exported to Enterprise and cloud systems. Core data persists the data in a local database. -[Redis](https://redis.io/) is used by default, but a database abstraction layer allows for other database implementations to be added. +[PosgreSQL](https://www.postgresql.org/) is used by default, but a database abstraction layer allows for other database implementations to be added. ![image](EdgeX_CoreData.png) @@ -26,7 +26,7 @@ Sensor data can be sent to core data via two different means: This is the default means of getting data to core data. Any service (like an application service or rules engine service) or 3rd system could also subscribe to the same topic. If the sensor data does not need to persisted locally, core data does not have to subscribe to the message bus topic - making core data completely optional. - By default, the message bus is implemented using Redis Pub/Sub. + By default, the message bus is implemented using MQTT. MQTT can be used as an alternate message bus implementation. ![image](EdgeX_CoreDataSubscriber.png) @@ -36,7 +36,7 @@ Sensor data can be sent to core data via two different means: ![image](EdgeX_CoreDataRESTEndpoint.png) -Core data moves data to the application service (and [edge analytcs](../../../general/Definitions.md#edge-analytics)) via Redis Pub/Sub by default. MQTT or NATS (opt-in at build time) can alternately be used. +Core data moves data to the application service (and [edge analytcs](../../../general/Definitions.md#edge-analytics)) via MQTT by default. MQTT or NATS (opt-in at build time) can alternately be used. Use of MQTT requires the use of an MQTT broker like mosquitto. Use of NATS requires all service to be built with NATS enabled and the installation of NATS Server. A messaging infrastructure abstraction is in place that allows for other message bus (e.g., AMQP) implementations to be created and used. \ No newline at end of file diff --git a/docs_src/microservices/core/metadata/GettingStarted.md b/docs_src/microservices/core/metadata/GettingStarted.md index c460bfe7da..d3b8a59f2a 100644 --- a/docs_src/microservices/core/metadata/GettingStarted.md +++ b/docs_src/microservices/core/metadata/GettingStarted.md @@ -6,7 +6,7 @@ title: Core Metadata - Getting Started ## Data Models -To understand metadata, its important to understand the EdgeX data objects it manages. Metadata stores its knowledge in a local persistence database. [Redis](https://redis.io/) is used by default, but a database abstraction layer allows for other databases to be used. +To understand metadata, its important to understand the EdgeX data objects it manages. Metadata stores its knowledge in a local persistence database. [PostgreSQL](https://www.postgresql.org/) is used by default, but a database abstraction layer allows for other databases to be used. ### Device Profile diff --git a/docs_src/microservices/device/Configuration.md b/docs_src/microservices/device/Configuration.md index dbdbc8c6bf..2478a38fd9 100644 --- a/docs_src/microservices/device/Configuration.md +++ b/docs_src/microservices/device/Configuration.md @@ -133,7 +133,7 @@ Device services can have custom configuration in one of two ways. See the table !!! edgey "Edgex 3.0" For EdgeX 3.0 the **SecretStore** configuration has been removed from each service's configuration files. It has default values which can be overridden with environment variables. See the [SecretStore Overrides](../configuration/CommonEnvironmentVariables.md#secretstore-configuration-overrides) section for more details. -All instances of Device Services running in secure mode require a `SecretStore` to be created for the service by the Security Services. See [Configuring Add-on Service](../../../security/Ch-Configuring-Add-On-Services) for details on configuring a `SecretStore` to be created for the Device Service. With the use of `Redis Pub/Sub` as the default EdgeX MessageBus all Device Services need the `redisdb` known secret added to their `SecretStore` so they can connect to the Secure EdgeX MessageBus. See the [Secure MessageBus](../../../security/Ch-Secure-MessageBus) documentation for more details. +All instances of Device Services running in secure mode require a `SecretStore` to be created for the service by the Security Services. See [Configuring Add-on Service](../../../security/Ch-Configuring-Add-On-Services) for details on configuring a `SecretStore` to be created for the Device Service. With the use of `MQTT` as the default EdgeX MessageBus all Device Services need the `postgres` known secret added to their `SecretStore` so they can connect to the Secure EdgeX MessageBus. See the [Secure MessageBus](../../../security/Ch-Secure-MessageBus) documentation for more details. Each Device Service also has detailed configuration to enable connection to it's exclusive `SecretStore` @@ -152,10 +152,10 @@ When running in insecure mode, the secrets are stored and retrieved from the *Wr Writable: InsecureSecrets: DB: - SecretName: "redisdb" + SecretName: "postgres" SecretData: - username: "" - password: "" + username: "postgres" + password: "postgres" MQTT: SecretName: "credentials" SecretData: diff --git a/docs_src/microservices/device/sdk/devicesdk-getting-started/configuration.yaml b/docs_src/microservices/device/sdk/devicesdk-getting-started/configuration.yaml index 32a2e5fbd8..f503e42978 100644 --- a/docs_src/microservices/device/sdk/devicesdk-getting-started/configuration.yaml +++ b/docs_src/microservices/device/sdk/devicesdk-getting-started/configuration.yaml @@ -17,12 +17,12 @@ Clients: Port: 59881 MessageBus: - Protocol: redis + Protocol: mqtt Host: localhost Port: 6379 - Type: redis - AuthMode: "usernamepassword" # required for redis messagebus (secure or insecure) - SecretName: redisdb + Type: mqtt + AuthMode: "usernamepassword" # required for mqtt messagebus (secure or insecure) + SecretName: mqtt-bus Optional: ClientId: device-example diff --git a/docs_src/microservices/general/database/Ch-Postgres.md b/docs_src/microservices/general/database/Ch-Postgres.md index 6aef1d8249..f9201476a5 100644 --- a/docs_src/microservices/general/database/Ch-Postgres.md +++ b/docs_src/microservices/general/database/Ch-Postgres.md @@ -1,6 +1,6 @@ # PostgreSQL -EdgeX Foundry also supports PostgreSQL as the persistence database. +EdgeX Foundry uses PostgreSQL as the persistence database by default. PostgreSQL is an open-source object-relational database system (licensed under the PostgreSQL License) that offers exceptional scalability and performance, with support for both relational and document (JSON) data models. diff --git a/docs_src/microservices/general/database/Ch-Redis.md b/docs_src/microservices/general/database/Ch-Redis.md index eaf057556b..f321851a2d 100644 --- a/docs_src/microservices/general/database/Ch-Redis.md +++ b/docs_src/microservices/general/database/Ch-Redis.md @@ -15,7 +15,7 @@ Redis uses a number of techniques to optimize memory utilization. Antirez and [R ## On-disk Persistence -Redis supports a number of different levels of on-disk persistence. By default, the configuration includes multiple save intervals: +Redis supports a number of different levels of on-disk persistence. By default, the configuration includes multiple save intervals: - After 1 hour, if at least 1 key has changed. - After 5 minutes, if at least 100 keys have changed. diff --git a/docs_src/microservices/general/messagebus.md b/docs_src/microservices/general/messagebus.md index d141956394..86a34769df 100644 --- a/docs_src/microservices/general/messagebus.md +++ b/docs_src/microservices/general/messagebus.md @@ -59,7 +59,7 @@ The common MessageBus configuration elements for each implementation are: - `tcp` for **NATS JetStream** !!! note - In general all EdgeX Services running in a deployment must be configured to use the same EdgeX MessageBus implementation. By default all services that use the EdgeX MessageBus are configured to use the Redis Pub/Sub implementation. NATS does support a compatibility mode with MQTT. See the [NATS MQTT Mode](#nats-mqtt-mode) section below for details. + In general all EdgeX Services running in a deployment must be configured to use the same EdgeX MessageBus implementation. By default all services that use the EdgeX MessageBus are configured to use the MQTT implementation. NATS does support a compatibility mode with MQTT. See the [NATS MQTT Mode](#nats-mqtt-mode) section below for details. ### MQTT 3.1 (default) @@ -151,8 +151,6 @@ A JetStream enabled server can support MQTT connections on the same set of under The EdgeX MessageBus uses multi-level topics and wildcards to allow filtering of data via subscriptions and has standardized on a MQTT like scheme. See [MQTT multi-level topics and wildcards](https://www.hivemq.com/blog/mqtt-essentials-part-5-mqtt-topics-best-practices) for more information. -The Redis implementation converts the Redis Pub/Sub multi-level topic scheme to match that of MQTT. In Redis Pub/Sub the "**.**" is used as a level separator, "\*" followed by a level separator is used as the single level wildcard and "*" at the end is used as the multiple level wildcard. These are converted to "/" and "+" and "#" respectively, which are used by MQTT. - The NATS implementations convert the NATS multi-level topic scheme to match that of MQTT. In NATS "**.**" is used as a level separator, "\*" is used as the single level wildcard and ">" is used for the multi-level wild card. These are converted to "/", "+" and "#" respectively, which are compliant with the MQTT scheme. diff --git a/docs_src/microservices/support/eKuiper/Ch-eKuiper.md b/docs_src/microservices/support/eKuiper/Ch-eKuiper.md index 9e784e6dbf..33178aea47 100644 --- a/docs_src/microservices/support/eKuiper/Ch-eKuiper.md +++ b/docs_src/microservices/support/eKuiper/Ch-eKuiper.md @@ -10,7 +10,7 @@ LF Edge eKuiper is a lightweight open source software (Apache 2.0 open source license agreement) package for IoT edge analytics and stream processing implemented in Go lang, which can run on various resource constrained edge devices. Users can realize fast data processing on the edge and write rules in SQL. The eKuiper rules engine is based on three components `Source`, `SQL` and `Sink`. -- Source: Source of stream data, such as data from an MQTT server. For EdgeX, the data source is an EdgeX message bus, which can be implemented by Redis Pub/Sub, MQTT or NATS; +- Source: Source of stream data, such as data from an MQTT server. For EdgeX, the data source is an EdgeX message bus, which can be implemented by MQTT or NATS; - SQL: SQL is where the specified business logic is processed. eKuiper provides SQL statements to extract, filter, and transform data; - Sink: Used to send the analysis result to a specific target, such as sending the analysis results to EdgeX's Command service, or an MQTT broker in the cloud; diff --git a/docs_src/security/Ch-RemoteDeviceServices.md b/docs_src/security/Ch-RemoteDeviceServices.md index 1bd9773217..560b3a157d 100644 --- a/docs_src/security/Ch-RemoteDeviceServices.md +++ b/docs_src/security/Ch-RemoteDeviceServices.md @@ -101,9 +101,9 @@ level=INFO ts=2022-05-05T14:28:30.005673094Z app=device-virtual source=config.go level=INFO ts=2022-05-05T14:28:30.006211643Z app=device-virtual source=variables.go:352 msg="Variables override of 'SecretStore.RuntimeTokenProvider.Port' by environment variable: SECRETSTORE_RUNTIMETOKENPROVIDER_PORT=59841" level=INFO ts=2022-05-05T14:28:30.006286584Z app=device-virtual source=variables.go:352 msg="Variables override of 'SecretStore.RuntimeTokenProvider.Protocol' by environment variable: SECRETSTORE_RUNTIMETOKENPROVIDER_PROTOCOL=https" level=INFO ts=2022-05-05T14:28:30.006341968Z app=device-virtual source=variables.go:352 msg="Variables override of 'Clients.core-metadata.Host' by environment variable: CLIENTS_CORE_METADATA_HOST=edgex-core-metadata" -level=INFO ts=2022-05-05T14:28:30.006382102Z app=device-virtual source=variables.go:352 msg="Variables override of 'MessageBus.Host' by environment variable: MESSAGEBUS_HOST=edgex-redis" +level=INFO ts=2022-05-05T14:28:30.006382102Z app=device-virtual source=variables.go:352 msg="Variables override of 'MessageBus.Host' by environment variable: MESSAGEBUS_HOST=edgex-postgres" level=INFO ts=2022-05-05T14:28:30.006416098Z app=device-virtual source=variables.go:352 msg="Variables override of 'SecretStore.RuntimeTokenProvider.EndpointSocket' by environment variable: SECRETSTORE_RUNTIMETOKENPROVIDER_ENDPOINTSOCKET=/tmp/edgex/secrets/spiffe/public/api.sock" -level=INFO ts=2022-05-05T14:28:30.006457406Z app=device-virtual source=variables.go:352 msg="Variables override of 'SecretStore.RuntimeTokenProvider.RequiredSecrets' by environment variable: SECRETSTORE_RUNTIMETOKENPROVIDER_REQUIREDSECRETS=redisdb" +level=INFO ts=2022-05-05T14:28:30.006457406Z app=device-virtual source=variables.go:352 msg="Variables override of 'SecretStore.RuntimeTokenProvider.RequiredSecrets' by environment variable: SECRETSTORE_RUNTIMETOKENPROVIDER_REQUIREDSECRETS=postgres" level=INFO ts=2022-05-05T14:28:30.006495791Z app=device-virtual source=variables.go:352 msg="Variables override of 'SecretStore.RuntimeTokenProvider.Enabled' by environment variable: SECRETSTORE_RUNTIMETOKENPROVIDER_ENABLED=true" level=INFO ts=2022-05-05T14:28:30.006529808Z app=device-virtual source=variables.go:352 msg="Variables override of 'SecretStore.RuntimeTokenProvider.Host' by environment variable: SECRETSTORE_RUNTIMETOKENPROVIDER_HOST=edgex-security-spiffe-token-provider" level=INFO ts=2022-05-05T14:28:30.006575741Z app=device-virtual source=variables.go:352 msg="Variables override of 'Clients.core-data.Host' by environment variable: CLIENTS_CORE_DATA_HOST=edgex-core-data" @@ -137,8 +137,8 @@ level=INFO ts=2022-05-05T14:29:25.172359472Z app=device-virtual source=secret.go level=INFO ts=2022-05-05T14:29:25.172539631Z app=device-virtual source=secrets.go:276 msg="kick off token renewal with interval: 30m0s" level=INFO ts=2022-05-05T14:29:25.172433598Z app=device-virtual source=config.go:551 msg="Using local configuration from file (14 envVars overrides applied)" level=INFO ts=2022-05-05T14:29:25.172916142Z app=device-virtual source=httpserver.go:131 msg="Web server starting (edgex-device-virtual:59900)" -level=INFO ts=2022-05-05T14:29:25.172948285Z app=device-virtual source=messaging.go:69 msg="Setting options for secure MessageBus with AuthMode='usernamepassword' and SecretName='redisdb" -level=INFO ts=2022-05-05T14:29:25.174321296Z app=device-virtual source=messaging.go:97 msg="Connected to redis Message Bus @ redis://edgex-redis:6379 publishing on 'edgex/events/device' prefix topic with AuthMode='usernamepassword'" +level=INFO ts=2022-05-05T14:29:25.172948285Z app=device-virtual source=messaging.go:69 msg="Setting options for secure MessageBus with AuthMode='usernamepassword' and SecretName='postgres" +level=INFO ts=2022-05-05T14:29:25.174321296Z app=device-virtual source=messaging.go:97 msg="Connected to mqtt Message Bus @ mqtt://edgex-mqtt:1883 publishing on 'edgex/events/device' prefix topic with AuthMode='usernamepassword'" level=INFO ts=2022-05-05T14:29:25.174585076Z app=device-virtual source=init.go:135 msg="Check core-metadata service's status by ping..." level=INFO ts=2022-05-05T14:29:25.176202842Z app=device-virtual source=init.go:54 msg="Service clients initialize successful." level=INFO ts=2022-05-05T14:29:25.176377929Z app=device-virtual source=clients.go:124 msg="Using configuration for URL for 'core-metadata': http://edgex-core-metadata:59881" @@ -247,7 +247,7 @@ The port-forwarding is transparent to the EdgeX services running on the local ma This step is to show the reverse direction of SSH tunneling: from the remote back to the local machine. -The reverse SSH tunneling is also needed because the device services depends on the core services like `core-data`, `core-metadata`, Redis (for message queuing), Vault (for the secret store), and Consul (for registry and configuration). +The reverse SSH tunneling is also needed because the device services depends on the core services like `core-data`, `core-metadata`, MQTT (for message queuing), OpenBao (for the secret store), and Core-Keeper (for registry and configuration). These core services are running on the local machine and should be **reverse** tunneled back from the remote machine. Essentially, the `sshd` container will impersonate these services on the remote side. @@ -284,7 +284,7 @@ device services: - edgex-core-consul - edgex-core-data - edgex-core-metadata - - edgex-redis + - edgex-postgres - edgex-secret-store - edgex-security-spire-server - edgex-security-spiffe-token-provider @@ -392,7 +392,7 @@ $ curl -s http://127.0.0.1:59900/api/{{api_version}}/config | jq "LogLevel": "INFO", "InsecureSecrets": { "DB": { - "Path": "redisdb", + "Path": "postgres", "Secrets": { "password": "", "username": "" @@ -478,18 +478,18 @@ $ curl -s http://127.0.0.1:59900/api/{{api_version}}/config | jq "Port": 59841, "TrustDomain": "edgexfoundry.org", "EndpointSocket": "/tmp/edgex/secrets/spiffe/public/api.sock", - "RequiredSecrets": "redisdb" + "RequiredSecrets": "posgres" } }, "MessageBus": { - "Type": "redis", - "Protocol": "redis", - "Host": "edgex-redis", + "Type": "mqtt", + "Protocol": "mqtt", + "Host": "edgex-mqtt", "Port": 6379, "PublishTopicPrefix": "edgex/events/device", "SubscribeTopic": "", "AuthMode": "usernamepassword", - "SecretName": "redisdb", + "SecretName": "mqtt-bus", "Optional": { "AutoReconnect": "true", "ClientId": "device-virtual", @@ -498,8 +498,7 @@ $ curl -s http://127.0.0.1:59900/api/{{api_version}}/config | jq "Password": "(redacted)", "Qos": "0", "Retained": "false", - "SkipCertVerify": "false", - "Username": "redis5" + "SkipCertVerify": "false" }, "SubscribeEnabled": false }, diff --git a/mkdocs.yml b/mkdocs.yml index 00d2846194..9e288be398 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -323,7 +323,7 @@ nav: - microservices/general/index.md - microservices/general/messagebus.md - Database: - - microservices/general/database/Ch-Redis.md + # - microservices/general/database/Ch-Redis.md - microservices/general/database/Ch-Postgres.md - Security: - Security Services: security/Ch-Security.md