From 45bffc682f71a49d0366f72b3420b1baff749789 Mon Sep 17 00:00:00 2001 From: jerbob92 Date: Mon, 1 Nov 2021 14:58:29 +0100 Subject: [PATCH] Improve Redis support, add username and Sentinel (#549) Signed-off-by: Jeroen Bobbeldijk --- .../docs/2.5/scalers/redis-cluster-lists.md | 13 +- .../docs/2.5/scalers/redis-cluster-streams.md | 15 +- content/docs/2.5/scalers/redis-lists.md | 15 +- .../docs/2.5/scalers/redis-sentinel-lists.md | 124 ++++++++++++++ .../2.5/scalers/redis-sentinel-streams.md | 158 ++++++++++++++++++ content/docs/2.5/scalers/redis-streams.md | 15 +- 6 files changed, 325 insertions(+), 15 deletions(-) create mode 100644 content/docs/2.5/scalers/redis-sentinel-lists.md create mode 100644 content/docs/2.5/scalers/redis-sentinel-streams.md diff --git a/content/docs/2.5/scalers/redis-cluster-lists.md b/content/docs/2.5/scalers/redis-cluster-lists.md index cfdb64c90..4f515ff09 100644 --- a/content/docs/2.5/scalers/redis-cluster-lists.md +++ b/content/docs/2.5/scalers/redis-cluster-lists.md @@ -16,6 +16,7 @@ triggers: - type: redis-cluster metadata: addresses: localhost:6379 # Comma separated list of the format host:port + usernameFromEnv: REDIS_USERNAME # optional passwordFromEnv: REDIS_PASSWORD listName: mylist # Required listLength: "5" # Required @@ -30,8 +31,9 @@ triggers: - `addresses` - Comma separated list of hosts and ports of the Redis Cluster nodes. - `hosts` - Comma separated list of hosts of the Redis Cluster nodes. Alternative to `addresses` and requires `ports` to be configured as well. - `ports` - Comma separated list of corresponding ports for the hosts of the Redis Cluster nodes. Alternative to `addresses` and requires `hosts` to be configured as well. +- `usernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis server. - `passwordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis server. - - Both the hostname and password fields need to be set to the names of the environment variables in the target deployment that contain the host name and password respectively. + - Both the hostname, username and password fields need to be set to the names of the environment variables in the target deployment that contain the host name, username and password respectively. - `listName` - Name of the Redis List that you want to monitor. - `listLength` - Average target value to trigger scaling actions. - `enableTLS` - Allow a connection to a redis queue using tls. (Values: `true`, `false`, Default: `false`, Optional) @@ -52,15 +54,16 @@ You can authenticate by using a password. - `hosts` - Comma separated list of hostname of the Redis Cluster nodes. If specified, the `ports` should also be specified. - `ports` - Comma separated list of ports of the Redis Cluster nodes. If specified, the `hosts` should also be specified. -**Password Authentication:** +**Authentication:** +- `username` - Redis username to authenticate with. - `password` - Redis password to authenticate with. ### Example Here is an example of how to deploy a scaled object with the `redis-cluster` scale trigger which uses `TriggerAuthentication`. -You can also provide the `passwordFromEnv` on the `ScaledObject` directly. +You can also provide the `usernameFromEnv` and `passwordFromEnv` on the `ScaledObject` directly. ```yaml apiVersion: v1 @@ -70,6 +73,7 @@ metadata: namespace: my-project type: Opaque data: + redis_username: YWRtaW4= redis_password: YWRtaW4= --- apiVersion: keda.sh/v1alpha1 @@ -79,6 +83,9 @@ metadata: namespace: my-project spec: secretTargetRef: + - parameter: username + name: votes-db-secret + key: redis_username - parameter: password name: votes-db-secret key: redis_password diff --git a/content/docs/2.5/scalers/redis-cluster-streams.md b/content/docs/2.5/scalers/redis-cluster-streams.md index 2875d575e..8ebe30b65 100644 --- a/content/docs/2.5/scalers/redis-cluster-streams.md +++ b/content/docs/2.5/scalers/redis-cluster-streams.md @@ -23,6 +23,7 @@ triggers: addresses: localhost:6379 # Required if hosts and ports are not provided. Format - comma separated list of host:port hosts: localhost # Comma separated lists of hosts. Required if address is not provided ports: "6379" # Comma separated lists of ports. Required if addresses are not provided and hosts has been provided. + usernameFromEnv: REDIS_USERNAME # optional (can also use authenticationRef) passwordFromEnv: REDIS_PASSWORD # optional (can also use authenticationRef) stream: my-stream # Required - name of the Redis Stream consumerGroup: my-consumer-group # Required - name of consumer group associated with Redis Stream @@ -49,6 +50,7 @@ triggers: > It is only to be used along with the `hosts`/`hostsFromEnv` attribute and not required if `addresses` has been provided. +- `usernameFromEnv` - Name of the environment variable your deployment uses to get the Redis username. (Optional) - `passwordFromEnv` - Name of the environment variable your deployment uses to get the Redis password. (Optional) - `stream` - Name of the Redis Stream. @@ -66,9 +68,9 @@ Some parameters could be provided using environmental variables, instead of sett The scaler supports two modes of authentication: -#### Using password authentication +#### Using username/password authentication -Use the `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis password. +Use the `username` and `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis username/password. This is usually resolved from a `Secret V1` or a `ConfigMap V1` collections. `env` and `envFrom` are both supported. @@ -91,6 +93,7 @@ spec: - type: redis-cluster-streams metadata: addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment stream: my-stream consumerGroup: consumer-group-1 @@ -105,9 +108,10 @@ You can use `TriggerAuthentication` CRD to configure the authentication. For exa apiVersion: v1 kind: Secret metadata: - name: redis-streams-password + name: redis-streams-auth type: Opaque data: + redis_username: redis_password: --- apiVersion: keda.sh/v1alpha1 @@ -116,8 +120,11 @@ metadata: name: keda-redis-stream-triggerauth spec: secretTargetRef: + - parameter: username + name: redis-streams-auth # name of the Secret + key: redis_username # name of the key in the Secret - parameter: password - name: redis-streams-password # name of the Secret + name: redis-streams-auth # name of the Secret key: redis_password # name of the key in the Secret --- apiVersion: keda.sh/v1alpha1 diff --git a/content/docs/2.5/scalers/redis-lists.md b/content/docs/2.5/scalers/redis-lists.md index 28b0317f3..8642e8902 100644 --- a/content/docs/2.5/scalers/redis-lists.md +++ b/content/docs/2.5/scalers/redis-lists.md @@ -16,6 +16,7 @@ triggers: - type: redis metadata: address: localhost:6379 # Format must be host:port + usernameFromEnv: REDIS_USERNAME # optional passwordFromEnv: REDIS_PASSWORD listName: mylist # Required listLength: "5" # Required @@ -31,8 +32,9 @@ triggers: - `address` - The host and port of the Redis server. - `host` - The host of the Redis server. Alternative to `address` and requires `port` to be configured as well. - `port` - The port of the Redis server. Alternative to `address` and requires `host` to be configured as well. +- `usernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis server. - `passwordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis server. - - Both the hostname and password fields need to be set to the names of the environment variables in the target deployment that contain the host name and password respectively. + - Both the hostname, username and password fields need to be set to the names of the environment variables in the target deployment that contain the host name, username and password respectively. - `listName` - Name of the Redis List that you want to monitor. - `listLength` - Average target value to trigger scaling actions. - `enableTLS` - Allow a connection to a redis queue using tls. (Values: `true`, `false`, Default: `false`, Optional) @@ -46,7 +48,7 @@ Some parameters could be provided using environmental variables, instead of sett ### Authentication Parameters -You can authenticate by using a password. +You can authenticate by using a username (optional) and password. **Connection Authentication:** @@ -54,15 +56,16 @@ You can authenticate by using a password. - `host` - The hostname of the Redis server. If specified, the `port` should also be specified. - `port` - The port of the Redis server. If specified, the `host` should also be specified. -**Password Authentication:** +**Authentication:** +- `username` - Redis username to authenticate with. - `password` - Redis password to authenticate with. ### Example Here is an example of how to deploy a scaled object with the `redis` scale trigger which uses `TriggerAuthentication`. -You can also provide the `passwordFromEnv` on the `ScaledObject` directly. +You can also provide the `usernameFromEnv` and `passwordFromEnv` on the `ScaledObject` directly. ```yaml apiVersion: v1 @@ -72,6 +75,7 @@ metadata: namespace: my-project type: Opaque data: + redis_username: YWRtaW4= redis_password: YWRtaW4= --- apiVersion: keda.sh/v1alpha1 @@ -81,6 +85,9 @@ metadata: namespace: my-project spec: secretTargetRef: + - parameter: username + name: votes-db-secret + key: redis_username - parameter: password name: votes-db-secret key: redis_password diff --git a/content/docs/2.5/scalers/redis-sentinel-lists.md b/content/docs/2.5/scalers/redis-sentinel-lists.md new file mode 100644 index 000000000..cf1b29712 --- /dev/null +++ b/content/docs/2.5/scalers/redis-sentinel-lists.md @@ -0,0 +1,124 @@ ++++ +title = "Redis Lists (supports Redis Sentinel)" +layout = "scaler" +availability = "v2.5+" +maintainer = "Community" +description = "Redis Lists scaler with support for Redis Sentinel topology" +go_file = "redis_scaler" ++++ + +### Trigger Specification + +This specification describes the `redis-sentinel` trigger that scales based on the length of a list in a Redis Sentinel setup. + +```yaml +triggers: +- type: redis-sentinel + metadata: + addresses: localhost:26379 # Comma separated list of the format host:port + usernameFromEnv: REDIS_USERNAME # optional + passwordFromEnv: REDIS_PASSWORD # optional + sentinelUsernameFromEnv: REDIS_SENTINEL_USERNAME # optional + sentinelPasswordFromEnv: REDIS_SENTINEL_PASSWORD # optional + sentinelMasterFromEnv: REDIS_SENTINEL_MASTER # optional + listName: mylist # Required + listLength: "5" # Required + enableTLS: "false" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressesFromEnv: REDIS_ADDRESSES # Optional. You can use this instead of `addresses` parameter +``` + +**Parameter list:** + +- `addresses` - Comma separated list of hosts and ports of the Redis Sentinel nodes. +- `hosts` - Comma separated list of hosts of the Redis Sentinel nodes. Alternative to `addresses` and requires `ports` to be configured as well. +- `ports` - Comma separated list of corresponding ports for the hosts of the Redis Sentinel nodes. Alternative to `addresses` and requires `hosts` to be configured as well. +- `usernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis server. +- `passwordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis server. + - Both the hostname, username and password fields need to be set to the names of the environment variables in the target deployment that contain the host name, username and password respectively. +- `sentinelUsernameFromEnv` - Environment variable to read the authentication username from to authenticate with the Redis Sentinel server. +- `sentinelPasswordFromEnv` - Environment variable to read the authentication password from to authenticate with the Redis Sentinel server. +- `sentinelMaster` - The name of the master in Sentinel to get the Redis server address for. +- `listName` - Name of the Redis List that you want to monitor. +- `listLength` - Average target value to trigger scaling actions. +- `enableTLS` - Allow a connection to a redis queue using tls. (Values: `true`, `false`, Default: `false`, Optional) + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressesFromEnv` - The hosts and their respective ports of the Redis Sentinel nodes, similar to `addresses`, but reads it from an environment variable on the scale target. +- `hostsFromEnv` - The hosts of the Redis Sentinel nodes, similar to `hosts`, but reads it from an environment variable on the scale target. +- `portsFromEnv` - The corresponding ports for the hosts of the Redis Sentinel nodes, similar to `ports`, but reads it from an environment variable on the scale target. +- `sentinelMasterFromEnv` - The name of the master in Sentinel to get the Redis server address for, similar to `sentinelMaster`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +You can authenticate by using a password. + +**Connection Authentication:** + +- `addresses` - Comma separated list of host:port format. +- `hosts` - Comma separated list of hostname of the Redis Sentinel nodes. If specified, the `ports` should also be specified. +- `ports` - Comma separated list of ports of the Redis Sentinel nodes. If specified, the `hosts` should also be specified. +- `sentinelMaster` - The name of the master in Sentinel to get the Redis server address for. + +**Authentication:** + +- `username` - Redis username to authenticate with. +- `password` - Redis password to authenticate with. + +**Sentinel Authentication:** + +- `sentinelUsername` - Redis Sentinel username to authenticate with. +- `sentinelPassword` - Redis Sentinel password to authenticate with. + + +### Example + +Here is an example of how to deploy a scaled object with the `redis-sentinel` scale trigger which uses `TriggerAuthentication`. + +You can also provide the `usernameFromEnv` and `passwordFromEnv` on the `ScaledObject` directly. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: votes-db-secret + namespace: my-project +type: Opaque +data: + redis_username: YWRtaW4= + redis_password: YWRtaW4= +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-redis-secret + namespace: my-project +spec: + secretTargetRef: + - parameter: username + name: votes-db-secret + key: redis_username + - parameter: password + name: votes-db-secret + key: redis_password +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-scaledobject + namespace: my-project +spec: + scaleTargetRef: + name: votes + triggers: + - type: redis-sentinel + metadata: + addresses: node1:26379, node2:26379, node3:26379 + listName: mylist + listLength: "10" + sentinelMaster: "mymaster" + authenticationRef: + name: keda-trigger-auth-redis-secret +``` diff --git a/content/docs/2.5/scalers/redis-sentinel-streams.md b/content/docs/2.5/scalers/redis-sentinel-streams.md new file mode 100644 index 000000000..514bd728d --- /dev/null +++ b/content/docs/2.5/scalers/redis-sentinel-streams.md @@ -0,0 +1,158 @@ ++++ +title = "Redis Streams (supports Redis Sentinel)" +layout = "scaler" +availability = "v2.5+" +maintainer = "Community" +description = "Redis Streams scaler with support for Redis Sentinel topology" +go_file = "redis_streams_scaler" ++++ + +### Trigger Specification + +Redis 5.0 introduced [Redis Streams](https://redis.io/topics/streams-intro) which is an append-only log data structure. + +One of its features includes [`Consumer Groups`](https://redis.io/topics/streams-intro#consumer-groups), that allows a group of clients to co-operate consuming a different portion of the same stream of messages. + +This specification describes the `redis-sentinel-streams` trigger that scales based on the *Pending Entries List* (see [`XPENDING`](https://redis.io/commands/xpending)) for a specific Consumer Group of a Redis Stream and supports a Redis Sentinel setup. + + +```yaml +triggers: +- type: redis-sentinel-streams + metadata: + addresses: localhost:26379 # Required if hosts and ports are not provided. Format - comma separated list of host:port + hosts: localhost # Comma separated lists of hosts. Required if address is not provided + ports: "26379" # Comma separated lists of ports. Required if addresses are not provided and hosts has been provided. + usernameFromEnv: REDIS_USERNAME # optional (can also use authenticationRef) + passwordFromEnv: REDIS_PASSWORD # optional (can also use authenticationRef) + stream: my-stream # Required - name of the Redis Stream + consumerGroup: my-consumer-group # Required - name of consumer group associated with Redis Stream + pendingEntriesCount: "10" # Required - number of entries in the Pending Entries List for the specified consumer group in the Redis Stream + enableTLS: "false" # optional + # Alternatively, you can use existing environment variables to read configuration from: + # See details in "Parameter list" section + addressesFromEnv: REDIS_ADDRESSES # Optional. You can use this instead of `addresses` parameter + hostsFromEnv: REDIS_HOSTS # Optional. You can use this instead of `hosts` parameter + portsFromEnv: REDIS_PORTS # Optional. You can use this instead of `ports` parameter +``` + +**Parameter list:** + +- `addresses` - Comma separated list of hosts and ports of Redis Sentinel nodes in the format `host:port` for example `node1:26379, node2:26379, node3:26379`. + +> As an alternative to the `addresses` field, the user can specify `hosts` and `ports` parameters. + +- `hosts` - Comma separated list of hosts of Redis Sentinel nodes. + +> It is not required if `addresses` has been provided. + +- `ports`: Comma separated list of ports for corresponding hosts of Redis Sentinel nodes. + +> It is only to be used along with the `hosts`/`hostsFromEnv` attribute and not required if `addresses` has been provided. + +- `usernameFromEnv` - Name of the environment variable your deployment uses to get the Redis username. (Optional) +- `passwordFromEnv` - Name of the environment variable your deployment uses to get the Redis password. (Optional) + +- `sentinelUsernameFromEnv` - Name of the environment variable your deployment uses to get the Redis Sentinel username. (Optional) +- `sentinelPasswordFromEnv` - Name of the environment variable your deployment uses to get the Redis Sentinel password. (Optional) + +- `sentinelMaster` - The name of the master in Sentinel to get the Redis server address for. +- `stream` - Name of the Redis Stream. +- `consumerGroup` - Name of the Consumer group associated with Redis Stream. +- `pendingEntriesCount` - Threshold for the number of `Pending Entries List`. This is the average target value to scale the workload. (Default: `5`, Optional) +- `enableTLS` - Allow a connection to Redis using tls. (Values: `true`, `false`, Default: `false`, Optional) + +Some parameters could be provided using environmental variables, instead of setting them directly in metadata. Here is a list of parameters you can use to retrieve values from environment variables: + +- `addressesFromEnv` - The hosts and corresponding ports of Redis Sentinel nodes, similar to `addresses`, but reads it from an environment variable on the scale target. Name of the environment variable your deployment uses to get the URLs of Redis Sentinel nodes. The resolved hosts should follow a format like `node1:26379, node2:26379, node3:26379 ...`. +- `hostsFromEnv` - The hosts of the Redis Sentinel nodes, similar to `hosts`, but reads it from an environment variable on the scale target. +- `portsFromEnv` - The corresponding ports for the hosts of Redis Sentinel nodes, similar to `ports`, but reads it from an environment variable on the scale target. +- `sentinelMasterFromEnv` - The name of the master in Sentinel to get the Redis server address for, similar to `sentinelMaster`, but reads it from an environment variable on the scale target. + +### Authentication Parameters + +The scaler supports two modes of authentication: + +#### Using username/password authentication + +Use the `username` and `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis username/password. + +This is usually resolved from a `Secret V1` or a `ConfigMap V1` collections. `env` and `envFrom` are both supported. + +Here is an example: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-sentinel-streams + metadata: + addressesFromEnv: REDIS_ADDRESSES + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment + passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" + sentinelMaster: "mymaster" +``` + +#### Using `TriggerAuthentication` + +You can use `TriggerAuthentication` CRD to configure the authentication. For example: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: redis-streams-auth +type: Opaque +data: + redis_username: + redis_password: +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-redis-stream-triggerauth +spec: + secretTargetRef: + - parameter: username + name: redis-streams-auth # name of the Secret + key: redis_username # name of the key in the Secret + - parameter: password + name: redis-streams-auth # name of the Secret + key: redis_password # name of the key in the Secret +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: redis-streams-scaledobject + namespace: default +spec: + scaleTargetRef: + name: redis-streams-consumer + pollingInterval: 15 + cooldownPeriod: 200 + maxReplicaCount: 25 + minReplicaCount: 1 + triggers: + - type: redis-sentinel-streams + metadata: + address: node1:26379, node2:26379, node3:26379 + stream: my-stream + consumerGroup: consumer-group-1 + pendingEntriesCount: "10" + sentinelMaster: "mymaster" + authenticationRef: + name: keda-redis-stream-triggerauth # name of the TriggerAuthentication resource +``` diff --git a/content/docs/2.5/scalers/redis-streams.md b/content/docs/2.5/scalers/redis-streams.md index 7f7e9f0c4..db20552cf 100644 --- a/content/docs/2.5/scalers/redis-streams.md +++ b/content/docs/2.5/scalers/redis-streams.md @@ -23,6 +23,7 @@ triggers: address: localhost:6379 # Required if host and port are not provided. Format - host:port host: localhost # Required if address is not provided port: "6379" # Required if address is not provided and host has been provided. + usernameFromEnv: REDIS_USERNAME # optional (can also use authenticationRef) passwordFromEnv: REDIS_PASSWORD # optional (can also use authenticationRef) stream: my-stream # Required - name of the Redis Stream consumerGroup: my-consumer-group # Required - name of consumer group associated with Redis Stream @@ -50,6 +51,7 @@ triggers: > It is only to be used along with the `host`/`hostFromEnv` attribute and not required if `address` has been provided. +- `usernameFromEnv` - Name of the environment variable your deployment uses to get the Redis username. (Optional) - `passwordFromEnv` - Name of the environment variable your deployment uses to get the Redis password. (Optional) - `stream` - Name of the Redis Stream. @@ -68,9 +70,9 @@ Some parameters could be provided using environmental variables, instead of sett The scaler supports two modes of authentication: -#### Using password authentication +#### Using username/password authentication -Use the `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis password. +Use the `username` and `password` field in the `metadata` to specify the name of an environment variable that your deployment uses to get the Redis username/password. This is usually resolved from a `Secret V1` or a `ConfigMap V1` collections. `env` and `envFrom` are both supported. @@ -93,6 +95,7 @@ spec: - type: redis-streams metadata: addressFromEnv: REDIS_HOST + usernameFromEnv: REDIS_USERNAME # name of the environment variable in the Deployment passwordFromEnv: REDIS_PASSWORD # name of the environment variable in the Deployment stream: my-stream consumerGroup: consumer-group-1 @@ -107,9 +110,10 @@ You can use `TriggerAuthentication` CRD to configure the authentication. For exa apiVersion: v1 kind: Secret metadata: - name: redis-streams-password + name: redis-streams-auth type: Opaque data: + redis_username: redis_password: --- apiVersion: keda.sh/v1alpha1 @@ -118,8 +122,11 @@ metadata: name: keda-redis-stream-triggerauth spec: secretTargetRef: + - parameter: username + name: redis-streams-auth # name of the Secret + key: redis_username # name of the key in the Secret - parameter: password - name: redis-streams-password # name of the Secret + name: redis-streams-auth # name of the Secret key: redis_password # name of the key in the Secret --- apiVersion: keda.sh/v1alpha1