Refactor Redis connections to use Redis URL - closes #7421

packages/twenty-docker/.env.example

@@ -3,10 +3,9 @@ TAG=latest
 # POSTGRES_ADMIN_PASSWORD=replace_me_with_a_strong_password
 
 PG_DATABASE_HOST=db:5432
+REDIS_URL=redis://redis:6379
 
 SERVER_URL=http://localhost:3000
-# REDIS_HOST=redis
-# REDIS_PORT=6379
 
 # Use openssl rand -base64 32 for each secret
 # ACCESS_TOKEN_SECRET=replace_me_with_a_random_string_access

packages/twenty-docker/docker-compose.yml

@@ -25,8 +25,7 @@ services:
       PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default
       SERVER_URL: ${SERVER_URL}
       FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_URL: ${REDIS_URL:-redis://localhost:6379}
 
       ENABLE_DB_MIGRATIONS: "true"
 
@@ -59,8 +58,7 @@ services:
       PG_DATABASE_URL: postgres://twenty:twenty@${PG_DATABASE_HOST}/default
       SERVER_URL: ${SERVER_URL}
       FRONT_BASE_URL: ${FRONT_BASE_URL:-$SERVER_URL}
-      REDIS_PORT: ${REDIS_PORT:-6379}
-      REDIS_HOST: ${REDIS_HOST:-redis}
+      REDIS_URL: ${REDIS_URL:-redis://localhost:6379}
 
       ENABLE_DB_MIGRATIONS: "false" # it already runs on the server
 

packages/twenty-docker/k8s/manifests/deployment-server.yaml

@@ -41,10 +41,8 @@ spec:
               value: "https://crm.example.com:443"
             - name: "PG_DATABASE_URL"
               value: "postgres://twenty:[email protected]/default"
-            - name: "REDIS_HOST"
-              value: "twentycrm-redis.twentycrm.svc.cluster.local"
-            - name: "REDIS_PORT"
-              value: 6379
+            - name: "REDIS_URL"
+              value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379"
             - name: ENABLE_DB_MIGRATIONS
               value: "true"
             - name: SIGN_IN_PREFILLED

packages/twenty-docker/k8s/manifests/deployment-worker.yaml

@@ -40,10 +40,8 @@ spec:
               value: "bull-mq"
             - name: "CACHE_STORAGE_TYPE"
               value: "redis"
-            - name: "REDIS_HOST"
-              value: "twentycrm-redis.twentycrm.svc.cluster.local"
-            - name: "REDIS_PORT"
-              value: 6379
+            - name: "REDIS_URL"
+              value: "redis://twentycrm-redis.twentycrm.svc.cluster.local:6379"
             - name: ACCESS_TOKEN_SECRET
               valueFrom:
                 secretKeyRef:

packages/twenty-docker/k8s/terraform/deployment-server.tf

@@ -61,12 +61,8 @@ resource "kubernetes_deployment" "twentycrm_server" {
             value = "postgres://twenty:${var.twentycrm_pgdb_admin_password}@${kubernetes_service.twentycrm_db.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local/default"
           }
           env {
-            name  = "REDIS_HOST"
-            value = "${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local"
-          }
-          env {
-            name  = "REDIS_PORT"
-            value = 6379
+            name  = "REDIS_URL"
+            value = "redis://${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local:6379"
           }
           env {
             name  = "ENABLE_DB_MIGRATIONS"

packages/twenty-docker/k8s/terraform/deployment-worker.tf

@@ -59,13 +59,8 @@ resource "kubernetes_deployment" "twentycrm_worker" {
           }
 
           env {
-            name  = "REDIS_HOST"
-            value = "${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local"
-          }
-
-          env {
-            name  = "REDIS_PORT"
-            value = 6379
+            name  = "REDIS_URL"
+            value = "redis://${kubernetes_service.twentycrm_redis.metadata.0.name}.${kubernetes_namespace.twentycrm.metadata.0.name}.svc.cluster.local:6379"
           }
 
           env {

packages/twenty-server/.env.example

@@ -50,10 +50,7 @@ SIGN_IN_PREFILLED=true
 # SENTRY_FRONT_DSN=https://[email protected]/xxx
 # LOG_LEVELS=error,warn
 # MESSAGE_QUEUE_TYPE=pg-boss
-# REDIS_HOST=127.0.0.1
-# REDIS_PORT=6379
-# REDIS_USERNAME=
-# REDIS_PASSWORD=
+# REDIS_URL=redis://localhost:6379
 # DEMO_WORKSPACE_IDS=REPLACE_ME_WITH_A_RANDOM_UUID
 # SERVER_URL=http://localhost:3000
 # WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION=30

packages/twenty-server/.env.test

@@ -13,10 +13,7 @@ DEMO_WORKSPACE_IDS=63db4589-590f-42b3-bdf1-85268b3da02f,8de58f3f-7e86-4a0b-998d-
 MUTATION_MAXIMUM_RECORD_AFFECTED=100
 MESSAGE_QUEUE_TYPE=pg-boss
 CACHE_STORAGE_TYPE=redis
-REDIS_HOST=127.0.0.1
-REDIS_PORT=6379
-REDIS_USERNAME=default
-REDIS_PASSWORD=
+REDIS_URL=redis://localhost:6379
 
 AUTH_GOOGLE_ENABLED=false
 MESSAGING_PROVIDER_GMAIL_ENABLED=false

packages/twenty-server/src/engine/core-modules/cache-storage/cache-storage.module-factory.ts

@@ -20,27 +20,18 @@ export const cacheStorageModuleFactory = (
       return cacheModuleOptions;
     }
     case CacheStorageType.Redis: {
-      const host = environmentService.get('REDIS_HOST');
-      const port = environmentService.get('REDIS_PORT');
+      const connectionString = environmentService.get('REDIS_URL');
 
-      if (!(host && port)) {
+      if (!connectionString) {
         throw new Error(
-          `${cacheStorageType} cache storage requires host: ${host} and port: ${port} to be defined, check your .env file`,
+          `${cacheStorageType} cache storage requires REDIS_URL to be defined, check your .env file`,
         );
       }
 
-      const username = environmentService.get('REDIS_USERNAME');
-      const password = environmentService.get('REDIS_PASSWORD');
-
       return {
         ...cacheModuleOptions,
         store: redisStore,
-        socket: {
-          host,
-          port,
-          username,
-          password,
-        },
+        url: connectionString,
       };
     }
     default:

packages/twenty-server/src/engine/core-modules/environment/environment-variables.ts

@@ -375,14 +375,18 @@ export class EnvironmentVariables {
   @IsNumber()
   MUTATION_MAXIMUM_AFFECTED_RECORDS = 100;
 
-  REDIS_HOST = '127.0.0.1';
-
-  @CastToPositiveNumber()
-  REDIS_PORT = 6379;
-
-  REDIS_USERNAME: string;
-
-  REDIS_PASSWORD: string;
+  @IsOptional()
+  @ValidateIf(
+    (env) =>
+      env.CACHE_STORAGE_TYPE === CacheStorageType.Redis ||
+      env.MESSAGE_QUEUE_TYPE === MessageQueueDriverType.BullMQ,
+  )
+  @IsUrl({
+    protocols: ['redis'],
+    require_tld: false,
+    allow_underscores: true,
+  })
+  REDIS_URL: string;
 
   API_TOKEN_EXPIRES_IN = '100y';
 

packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts

@@ -1,15 +1,15 @@
 import { OnModuleDestroy } from '@nestjs/common';
 
-import omitBy from 'lodash.omitby';
 import { JobsOptions, Queue, QueueOptions, Worker } from 'bullmq';
+import omitBy from 'lodash.omitby';
 
 import {
   QueueCronJobOptions,
   QueueJobOptions,
 } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
+import { MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
 import { MessageQueueJob } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
 import { MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';
-import { MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
 
 import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
 

packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module-factory.ts

@@ -1,7 +1,12 @@
+import { ConnectionOptions } from 'tls';
+
 import { EnvironmentService } from 'src/engine/core-modules/environment/environment.service';
 import {
+  BullMQDriverFactoryOptions,
   MessageQueueDriverType,
   MessageQueueModuleOptions,
+  PgBossDriverFactoryOptions,
+  SyncDriverFactoryOptions,
 } from 'src/engine/core-modules/message-queue/interfaces';
 
 /**
@@ -19,7 +24,7 @@ export const messageQueueModuleFactory = async (
       return {
         type: MessageQueueDriverType.Sync,
         options: {},
-      };
+      } satisfies SyncDriverFactoryOptions;
     }
     case MessageQueueDriverType.PgBoss: {
       const connectionString = environmentService.get('PG_DATABASE_URL');
@@ -29,25 +34,23 @@ export const messageQueueModuleFactory = async (
         options: {
           connectionString,
         },
-      };
+      } satisfies PgBossDriverFactoryOptions;
     }
     case MessageQueueDriverType.BullMQ: {
-      const host = environmentService.get('REDIS_HOST');
-      const port = environmentService.get('REDIS_PORT');
-      const username = environmentService.get('REDIS_USERNAME');
-      const password = environmentService.get('REDIS_PASSWORD');
+      const connectionString = environmentService.get('REDIS_URL');
+
+      if (!connectionString) {
+        throw new Error(
+          `${MessageQueueDriverType.BullMQ} message queue requires REDIS_URL to be defined, check your .env file`,
+        );
+      }
 
       return {
         type: MessageQueueDriverType.BullMQ,
         options: {
-          connection: {
-            host,
-            port,
-            username,
-            password,
-          },
+          connection: connectionString as ConnectionOptions,
         },
-      };
+      } satisfies BullMQDriverFactoryOptions;
     }
     default:
       throw new Error(

packages/twenty-website/src/content/developers/self-hosting/self-hosting-var.mdx

@@ -34,10 +34,7 @@ yarn command:prod cron:calendar:calendar-event-list-fetch
 <ArticleTable options={[
     ['PG_DATABASE_URL', 'postgres://user:pw@localhost:5432/default?connection_limit=1', 'Database connection'],
     ['PG_SSL_ALLOW_SELF_SIGNED', 'false', 'Allow self signed certificates'],
-    ['REDIS_HOST', '127.0.0.1', 'Redis connection host'],
-    ['REDIS_PORT', '6379', 'Redis connection port'],
-    ['REDIS_USERNAME', 'redis_username', 'Redis connection username'],
-    ['REDIS_PASSWORD', 'redis_pwd', 'Redis connection password'],
+    ['REDIS_URL', 'redis://localhost:6379', 'Redis connection url'],
     ['FRONT_BASE_URL', 'http://localhost:3001', 'Url to the hosted frontend'],
     ['SERVER_URL', 'http://localhost:3000', 'Url to the hosted server'],
     ['PORT', '3000', 'Port'],

packages/twenty-website/src/content/developers/self-hosting/upgrade-guide.mdx

@@ -99,4 +99,15 @@ yarn command:prod upgrade-0.31
 The `yarn database:migrate:prod` command will apply the migrations to the database structure (core and metadata schemas)
 The `yarn command:prod upgrade-31` takes care of the data migration of all workspaces.
 
+# v0.31.0 to v0.32.0
+
+### Environment Variables
+
+The following environment variables have been changed:
+
+- Removed: `REDIS_HOST`, `REDIS_PORT`, `REDIS_USERNAME`, `REDIS_PASSWORD`
+- Added: `REDIS_URL`
+
+Update your `.env` file to use the new `REDIS_URL` variable instead of the individual Redis connection parameters.
+
 <ArticleEditContent></ArticleEditContent>