diff --git a/docs/companion.md b/docs/companion.md index 15ca91cacd..c8dfd0fcb6 100644 --- a/docs/companion.md +++ b/docs/companion.md @@ -381,11 +381,12 @@ in the future, we plan and changing the default to `companion:` and possibly remove this option. This is a standalone-only option. See also `COMPANION_REDIS_PUBSUB_SCOPE`. -#### `redisOptions` +#### `redisOptions` `COMPANION_REDIS_OPTIONS` An object of -[options supported by redis client](https://www.npmjs.com/package/redis#options-object-properties). -This option can be used in place of `redisUrl`. +[options supported by the `ioredis` client](https://github.com/redis/ioredis). +See also +[`RedisOptions`](https://github.com/redis/ioredis/blob/af832752040e616daf51621681bcb40cab965a9b/lib/redis/RedisOptions.ts#L8). #### `redisPubSubScope` `COMPANION_REDIS_PUBSUB_SCOPE` diff --git a/e2e/mock-server.mjs b/e2e/mock-server.mjs index 6a62a7fe84..02d84b77da 100644 --- a/e2e/mock-server.mjs +++ b/e2e/mock-server.mjs @@ -23,7 +23,7 @@ const requestListener = (req, res) => { export default function startMockServer (host, port) { const server = http.createServer(requestListener) server.listen(port, host, () => { - console.log(`Server is running on http://${host}:${port}`) + console.log(`Mock server is running on http://${host}:${port}`) }) } diff --git a/packages/@uppy/companion/package.json b/packages/@uppy/companion/package.json index 901f2b1e6b..d2f3d4a16a 100644 --- a/packages/@uppy/companion/package.json +++ b/packages/@uppy/companion/package.json @@ -50,6 +50,7 @@ "got": "^13.0.0", "grant": "5.4.22", "helmet": "^4.6.0", + "ioredis": "^5.3.2", "ipaddr.js": "^2.0.1", "jsonwebtoken": "9.0.2", "lodash": "^4.17.21", @@ -60,7 +61,6 @@ "ms": "2.1.3", "node-schedule": "2.1.1", "prom-client": "14.0.1", - "redis": "4.6.13", "serialize-error": "^2.1.0", "serialize-javascript": "^6.0.0", "tus-js-client": "^3.1.3", diff --git a/packages/@uppy/companion/src/server/Uploader.js b/packages/@uppy/companion/src/server/Uploader.js index 63dc0226ba..eecd1de42f 100644 --- a/packages/@uppy/companion/src/server/Uploader.js +++ b/packages/@uppy/companion/src/server/Uploader.js @@ -158,6 +158,9 @@ class StreamableBlob { } class Uploader { + /** @type {import('ioredis').Redis} */ + storage + /** * Uploads file to destination based on the supplied protocol (tus, s3-multipart, multipart) * For tus uploads, the deferredLength option is enabled, because file size value can be unreliable @@ -446,9 +449,7 @@ class Uploader { // https://github.com/transloadit/uppy/issues/3748 const keyExpirySec = 60 * 60 * 24 const redisKey = `${Uploader.STORAGE_PREFIX}:${this.token}` - this.storage.set(redisKey, jsonStringify(state), { - EX: keyExpirySec, - }) + this.storage.set(redisKey, jsonStringify(state), 'EX', keyExpirySec) } throttledEmitProgress = throttle((dataToEmit) => { diff --git a/packages/@uppy/companion/src/server/emitter/redis-emitter.js b/packages/@uppy/companion/src/server/emitter/redis-emitter.js index 974a60b54e..ca2bce68ab 100644 --- a/packages/@uppy/companion/src/server/emitter/redis-emitter.js +++ b/packages/@uppy/companion/src/server/emitter/redis-emitter.js @@ -6,17 +6,22 @@ const logger = require('../logger') * This module simulates the builtin events.EventEmitter but with the use of redis. * This is useful for when companion is running on multiple instances and events need * to be distributed across. + * + * @param {import('ioredis').Redis} redisClient + * @param {string} redisPubSubScope + * @returns */ module.exports = (redisClient, redisPubSubScope) => { const prefix = redisPubSubScope ? `${redisPubSubScope}:` : '' const getPrefixedEventName = (eventName) => `${prefix}${eventName}` - const publisher = redisClient.duplicate() - publisher.on('error', err => logger.error('publisher redis error', err)) + const publisher = redisClient.duplicate({ lazyConnect: true }) + publisher.on('error', err => logger.error('publisher redis error', err.toString())) + /** @type {import('ioredis').Redis} */ let subscriber const connectedPromise = publisher.connect().then(() => { subscriber = publisher.duplicate() - subscriber.on('error', err => logger.error('subscriber redis error', err)) + subscriber.on('error', err => logger.error('subscriber redis error', err.toString())) return subscriber.connect() }) @@ -55,20 +60,32 @@ module.exports = (redisClient, redisPubSubScope) => { handlersByThisEventName.delete(handler) if (handlersByThisEventName.size === 0) handlersByEvent.delete(eventName) - return subscriber.pUnsubscribe(getPrefixedEventName(eventName), actualHandler) + subscriber.off('pmessage', actualHandler) + return subscriber.punsubscribe(getPrefixedEventName(eventName)) }) } + /** + * + * @param {string} eventName + * @param {*} handler + * @param {*} _once + */ function addListener (eventName, handler, _once = false) { - function actualHandler (message) { + function actualHandler (pattern, channel, message) { + if (pattern !== getPrefixedEventName(eventName)) { + return + } + if (_once) removeListener(eventName, handler) let args try { args = JSON.parse(message) } catch (ex) { - return handleError(new Error(`Invalid JSON received! Channel: ${eventName} Message: ${message}`)) + handleError(new Error(`Invalid JSON received! Channel: ${eventName} Message: ${message}`)) + return } - return handler(...args) + handler(...args) } let handlersByThisEventName = handlersByEvent.get(eventName) @@ -78,7 +95,10 @@ module.exports = (redisClient, redisPubSubScope) => { } handlersByThisEventName.set(handler, actualHandler) - runWhenConnected(() => subscriber.pSubscribe(getPrefixedEventName(eventName), actualHandler)) + runWhenConnected(() => { + subscriber.on('pmessage', actualHandler) + return subscriber.psubscribe(getPrefixedEventName(eventName)) + }) } /** @@ -134,7 +154,7 @@ module.exports = (redisClient, redisPubSubScope) => { return runWhenConnected(() => { handlersByEvent.delete(eventName) - return subscriber.pUnsubscribe(getPrefixedEventName(eventName)) + return subscriber.punsubscribe(getPrefixedEventName(eventName)) }) } diff --git a/packages/@uppy/companion/src/server/redis.js b/packages/@uppy/companion/src/server/redis.js index e7391e3234..bbbec1728d 100644 --- a/packages/@uppy/companion/src/server/redis.js +++ b/packages/@uppy/companion/src/server/redis.js @@ -1,43 +1,34 @@ -const redis = require('redis') +const Redis = require('ioredis').default const logger = require('./logger') +/** @type {import('ioredis').Redis} */ let redisClient /** * A Singleton module that provides a single redis client through out * the lifetime of the server * - * @param {{ redisUrl?: string, redisOptions?: Record }} [companionOptions] options + * @param {string} [redisUrl] ioredis url + * @param {Record} [redisOptions] ioredis client options */ -function createClient (companionOptions) { +function createClient (redisUrl, redisOptions) { if (!redisClient) { - const { redisUrl, redisOptions } = companionOptions - redisClient = redis.createClient({ - ...redisOptions, - ...(redisUrl && { url: redisUrl }), - }) - - redisClient.on('error', err => logger.error('redis error', err)) - - ;(async () => { - try { - // fire and forget. - // any requests made on the client before connection is established will be auto-queued by node-redis - await redisClient.connect() - } catch (err) { - logger.error(err.message, 'redis.error') - } - })() + if (redisUrl) { + redisClient = new Redis(redisUrl, redisOptions) + } else { + redisClient = new Redis(redisOptions) + } + redisClient.on('error', err => logger.error('redis error', err.toString())) } return redisClient } -module.exports.client = (companionOptions) => { - if (!companionOptions?.redisUrl && !companionOptions?.redisOptions) { +module.exports.client = ({ redisUrl, redisOptions } = { redisUrl: undefined, redisOptions: undefined }) => { + if (!redisUrl && !redisOptions) { return redisClient } - return createClient(companionOptions) + return createClient(redisUrl, redisOptions) } diff --git a/packages/@uppy/companion/src/standalone/helper.js b/packages/@uppy/companion/src/standalone/helper.js index 1a77119e16..426a317dbd 100644 --- a/packages/@uppy/companion/src/standalone/helper.js +++ b/packages/@uppy/companion/src/standalone/helper.js @@ -147,9 +147,9 @@ const getConfigFromEnv = () => { periodicPingCount: process.env.COMPANION_PERIODIC_PING_COUNT ? parseInt(process.env.COMPANION_PERIODIC_PING_COUNT, 10) : undefined, filePath: process.env.COMPANION_DATADIR, - redisUrl: process.env.COMPANION_REDIS_URL, redisPubSubScope: process.env.COMPANION_REDIS_PUBSUB_SCOPE, - // redisOptions refers to https://www.npmjs.com/package/redis#options-object-properties + redisUrl: process.env.COMPANION_REDIS_URL, + // redisOptions refers to https://redis.github.io/ioredis/index.html#RedisOptions redisOptions: (() => { try { if (!process.env.COMPANION_REDIS_OPTIONS) { diff --git a/yarn.lock b/yarn.lock index 9b78b7ffd9..f1d48416da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5106,6 +5106,13 @@ __metadata: languageName: node linkType: hard +"@ioredis/commands@npm:^1.1.1": + version: 1.2.0 + resolution: "@ioredis/commands@npm:1.2.0" + checksum: 10/a8253c9539b7e5463d4a98e6aa5b1b863fb4a4978191ba9dc42ec2c0fb5179d8d1fe4a29096d5954f91ba9600d1bdc6c1d18b044eab36f645f267fd37d7c0906 + languageName: node + linkType: hard + "@isaacs/cliui@npm:^8.0.2": version: 8.0.2 resolution: "@isaacs/cliui@npm:8.0.2" @@ -7036,62 +7043,6 @@ __metadata: languageName: node linkType: hard -"@redis/bloom@npm:1.2.0": - version: 1.2.0 - resolution: "@redis/bloom@npm:1.2.0" - peerDependencies: - "@redis/client": ^1.0.0 - checksum: 10/a16408f729ddd032a52c9d998661dfa7beabc0e92760d30619c3166c7a53a98c037956d93d230b787005fd8a599a7456461ca7429c1916893c2d13d59a41e0e6 - languageName: node - linkType: hard - -"@redis/client@npm:1.5.14": - version: 1.5.14 - resolution: "@redis/client@npm:1.5.14" - dependencies: - cluster-key-slot: "npm:1.1.2" - generic-pool: "npm:3.9.0" - yallist: "npm:4.0.0" - checksum: 10/aab53eff9456e0a5e0ef78ce16db3eca4b837274b8285c5d66ced549573dbacf75972935806911274d6dd906a53d982ef9b1a6f11a8efe4a18efa94ec9c2a4b3 - languageName: node - linkType: hard - -"@redis/graph@npm:1.1.1": - version: 1.1.1 - resolution: "@redis/graph@npm:1.1.1" - peerDependencies: - "@redis/client": ^1.0.0 - checksum: 10/96b8ee9bec124947465848b56a014805f9639e09704e03c75a92072a319599ac9dcd4f9ace22970a7f72131a241166ad31db4dc6931b34808d22a5ca94649ba5 - languageName: node - linkType: hard - -"@redis/json@npm:1.0.6": - version: 1.0.6 - resolution: "@redis/json@npm:1.0.6" - peerDependencies: - "@redis/client": ^1.0.0 - checksum: 10/bedd8b6fd152ed480f993c6372288f210a9c0e60bb39c02861d5ce2cb5452119229435572cd94886cdbde5fbae014471fc179dff1dbc86f045782e0358af1b0f - languageName: node - linkType: hard - -"@redis/search@npm:1.1.6": - version: 1.1.6 - resolution: "@redis/search@npm:1.1.6" - peerDependencies: - "@redis/client": ^1.0.0 - checksum: 10/7a2543012fc2c88ff4c6a6c9c1b537b472d5af340c2717f968562ef2ead713b02dd22cfadc5d5e16c0d32279a4c04bee974e0f20de416a3561a1221b3dccc790 - languageName: node - linkType: hard - -"@redis/time-series@npm:1.0.5": - version: 1.0.5 - resolution: "@redis/time-series@npm:1.0.5" - peerDependencies: - "@redis/client": ^1.0.0 - checksum: 10/be735fe7497b157ef8291fed157342a9a5017884488fa519b271745cfb9500a498d6f8e4bee6d34b58892d65f8ef7a3f4c458d083fb19892b4d3633d0d6c7db6 - languageName: node - linkType: hard - "@reduxjs/toolkit@npm:^1.9.3": version: 1.9.7 resolution: "@reduxjs/toolkit@npm:1.9.7" @@ -10130,6 +10081,7 @@ __metadata: got: "npm:^13.0.0" grant: "npm:5.4.22" helmet: "npm:^4.6.0" + ioredis: "npm:^5.3.2" ipaddr.js: "npm:^2.0.1" jest: "npm:^29.0.0" jsonwebtoken: "npm:9.0.2" @@ -10142,7 +10094,6 @@ __metadata: nock: "npm:^13.1.3" node-schedule: "npm:2.1.1" prom-client: "npm:14.0.1" - redis: "npm:4.6.13" serialize-error: "npm:^2.1.0" serialize-javascript: "npm:^6.0.0" supertest: "npm:6.2.4" @@ -13309,7 +13260,7 @@ __metadata: languageName: node linkType: hard -"cluster-key-slot@npm:1.1.2": +"cluster-key-slot@npm:^1.1.0": version: 1.1.2 resolution: "cluster-key-slot@npm:1.1.2" checksum: 10/516ed8b5e1a14d9c3a9c96c72ef6de2d70dfcdbaa0ec3a90bc7b9216c5457e39c09a5775750c272369070308542e671146120153062ab5f2f481bed5de2c925f @@ -14716,6 +14667,13 @@ __metadata: languageName: node linkType: hard +"denque@npm:^2.1.0": + version: 2.1.0 + resolution: "denque@npm:2.1.0" + checksum: 10/8ea05321576624b90acfc1ee9208b8d1d04b425cf7573b9b4fa40a2c3ed4d4b0af5190567858f532f677ed2003d4d2b73c8130b34e3c7b8d5e88cdcfbfaa1fe7 + languageName: node + linkType: hard + "depd@npm:2.0.0, depd@npm:~2.0.0": version: 2.0.0 resolution: "depd@npm:2.0.0" @@ -17736,13 +17694,6 @@ __metadata: languageName: node linkType: hard -"generic-pool@npm:3.9.0": - version: 3.9.0 - resolution: "generic-pool@npm:3.9.0" - checksum: 10/3c632d30a6a7d47412dc67ddc517992691e0fde819c0cb6b5871bc87d10f61a7c09f12a60dbd77c78ae3e6ca10db41e2eaee28985ce724d9620354a006205ce1 - languageName: node - linkType: hard - "gensync@npm:^1.0.0-beta.2": version: 1.0.0-beta.2 resolution: "gensync@npm:1.0.0-beta.2" @@ -19017,6 +18968,23 @@ __metadata: languageName: node linkType: hard +"ioredis@npm:^5.3.2": + version: 5.4.1 + resolution: "ioredis@npm:5.4.1" + dependencies: + "@ioredis/commands": "npm:^1.1.1" + cluster-key-slot: "npm:^1.1.0" + debug: "npm:^4.3.4" + denque: "npm:^2.1.0" + lodash.defaults: "npm:^4.2.0" + lodash.isarguments: "npm:^3.1.0" + redis-errors: "npm:^1.2.0" + redis-parser: "npm:^3.0.0" + standard-as-callback: "npm:^2.1.0" + checksum: 10/9043b812ac58065e80c759d130602cc64490fcaeaacf93723453fda04c7ba61dab0e2f50380eacb045592378ededf44f270c0d43e13e3e8b8d7c5a8d7fecb823 + languageName: node + linkType: hard + "ip-address@npm:^9.0.5": version: 9.0.5 resolution: "ip-address@npm:9.0.5" @@ -21672,6 +21640,13 @@ __metadata: languageName: node linkType: hard +"lodash.defaults@npm:^4.2.0": + version: 4.2.0 + resolution: "lodash.defaults@npm:4.2.0" + checksum: 10/6a2a9ea5ad7585aff8d76836c9e1db4528e5f5fa50fc4ad81183152ba8717d83aef8aec4fa88bf3417ed946fd4b4358f145ee08fbc77fb82736788714d3e12db + languageName: node + linkType: hard + "lodash.frompairs@npm:^4.0.1": version: 4.0.1 resolution: "lodash.frompairs@npm:4.0.1" @@ -21686,6 +21661,13 @@ __metadata: languageName: node linkType: hard +"lodash.isarguments@npm:^3.1.0": + version: 3.1.0 + resolution: "lodash.isarguments@npm:3.1.0" + checksum: 10/e5186d5fe0384dcb0652501d9d04ebb984863ebc9c9faa2d4b9d5dfd81baef9ffe8e2887b9dc471d62ed092bc0788e5f1d42e45c72457a2884bbb54ac132ed92 + languageName: node + linkType: hard + "lodash.isboolean@npm:^3.0.3": version: 3.0.3 resolution: "lodash.isboolean@npm:3.0.3" @@ -27473,17 +27455,19 @@ __metadata: languageName: node linkType: hard -"redis@npm:4.6.13": - version: 4.6.13 - resolution: "redis@npm:4.6.13" +"redis-errors@npm:^1.0.0, redis-errors@npm:^1.2.0": + version: 1.2.0 + resolution: "redis-errors@npm:1.2.0" + checksum: 10/001c11f63ddd52d7c80eb4f4ede3a9433d29a458a7eea06b9154cb37c9802a218d93b7988247aa8c958d4b5d274b18354e8853c148f1096fda87c6e675cfd3ee + languageName: node + linkType: hard + +"redis-parser@npm:^3.0.0": + version: 3.0.0 + resolution: "redis-parser@npm:3.0.0" dependencies: - "@redis/bloom": "npm:1.2.0" - "@redis/client": "npm:1.5.14" - "@redis/graph": "npm:1.1.1" - "@redis/json": "npm:1.0.6" - "@redis/search": "npm:1.1.6" - "@redis/time-series": "npm:1.0.5" - checksum: 10/cc66182b8fa78c2a63b5300b15fa6fbf8908773d78bc5ca3960018f465595b51dfecaebe8c848111a3b723530f17bdaa1c186f73875cd9ba351f32d2e5e14d5f + redis-errors: "npm:^1.0.0" + checksum: 10/b10846844b4267f19ce1a6529465819c3d78c3e89db7eb0c3bb4eb19f83784797ec411274d15a77dbe08038b48f95f76014b83ca366dc955a016a3a0a0234650 languageName: node linkType: hard @@ -29788,6 +29772,13 @@ __metadata: languageName: node linkType: hard +"standard-as-callback@npm:^2.1.0": + version: 2.1.0 + resolution: "standard-as-callback@npm:2.1.0" + checksum: 10/88bec83ee220687c72d94fd86a98d5272c91d37ec64b66d830dbc0d79b62bfa6e47f53b71646011835fc9ce7fae62739545d13124262b53be4fbb3e2ebad551c + languageName: node + linkType: hard + "start-server-and-test@npm:1.14.0": version: 1.14.0 resolution: "start-server-and-test@npm:1.14.0" @@ -33285,13 +33276,6 @@ __metadata: languageName: node linkType: hard -"yallist@npm:4.0.0, yallist@npm:^4.0.0": - version: 4.0.0 - resolution: "yallist@npm:4.0.0" - checksum: 10/4cb02b42b8a93b5cf50caf5d8e9beb409400a8a4d85e83bb0685c1457e9ac0b7a00819e9f5991ac25ffabb56a78e2f017c1acc010b3a1babfe6de690ba531abd - languageName: node - linkType: hard - "yallist@npm:^2.1.2": version: 2.1.2 resolution: "yallist@npm:2.1.2" @@ -33306,6 +33290,13 @@ __metadata: languageName: node linkType: hard +"yallist@npm:^4.0.0": + version: 4.0.0 + resolution: "yallist@npm:4.0.0" + checksum: 10/4cb02b42b8a93b5cf50caf5d8e9beb409400a8a4d85e83bb0685c1457e9ac0b7a00819e9f5991ac25ffabb56a78e2f017c1acc010b3a1babfe6de690ba531abd + languageName: node + linkType: hard + "yaml@npm:2.3.1": version: 2.3.1 resolution: "yaml@npm:2.3.1"