From be991f1d18006a4820c1e9ca6625bf2bd2bfedac Mon Sep 17 00:00:00 2001 From: Gabriel Simmer Date: Fri, 1 Sep 2023 16:44:28 +0100 Subject: [PATCH] Move to ioredis for streaming (#26581) Co-authored-by: Emelia Smith --- package.json | 2 +- streaming/index.js | 82 +++++++++++++++++------------------- yarn.lock | 101 +++++++++++++++++++++------------------------ 3 files changed, 85 insertions(+), 100 deletions(-) diff --git a/package.json b/package.json index 403883105..e92115610 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "immutable": "^4.3.0", "imports-loader": "^1.2.0", "intl-messageformat": "^10.3.5", + "ioredis": "^5.3.2", "js-yaml": "^4.1.0", "jsdom": "^22.1.0", "lodash": "^4.17.21", @@ -118,7 +119,6 @@ "react-swipeable-views": "^0.14.0", "react-textarea-autosize": "^8.4.1", "react-toggle": "^4.1.3", - "redis": "^4.6.5", "redux": "^4.2.1", "redux-immutable": "^4.0.0", "redux-thunk": "^2.4.2", diff --git a/streaming/index.js b/streaming/index.js index a241fa328..c9fac063d 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -6,12 +6,12 @@ const url = require('url'); const dotenv = require('dotenv'); const express = require('express'); +const Redis = require('ioredis'); const { JSDOM } = require('jsdom'); const log = require('npmlog'); const pg = require('pg'); const dbUrlToConfig = require('pg-connection-string').parse; const metrics = require('prom-client'); -const redis = require('redis'); const uuid = require('uuid'); const WebSocket = require('ws'); @@ -24,30 +24,12 @@ dotenv.config({ log.level = process.env.LOG_LEVEL || 'verbose'; /** - * @param {Object.} defaultConfig - * @param {string} redisUrl + * @param {Object.} config */ -const redisUrlToClient = async (defaultConfig, redisUrl) => { - const config = defaultConfig; - - let client; - - if (!redisUrl) { - client = redis.createClient(config); - } else if (redisUrl.startsWith('unix://')) { - client = redis.createClient(Object.assign(config, { - socket: { - path: redisUrl.slice(7), - }, - })); - } else { - client = redis.createClient(Object.assign(config, { - url: redisUrl, - })); - } - +const createRedisClient = async (config) => { + const { redisParams, redisUrl } = config; + const client = new Redis(redisUrl, redisParams); client.on('error', (err) => log.error('Redis Client Error!', err)); - await client.connect(); return client; }; @@ -147,23 +129,22 @@ const pgConfigFromEnv = (env) => { * @returns {Object.} configuration for the Redis connection */ const redisConfigFromEnv = (env) => { - const redisNamespace = env.REDIS_NAMESPACE || null; + // ioredis *can* transparently add prefixes for us, but it doesn't *in some cases*, + // which means we can't use it. But this is something that should be looked into. + const redisPrefix = env.REDIS_NAMESPACE ? `${env.REDIS_NAMESPACE}:` : ''; const redisParams = { - socket: { - host: env.REDIS_HOST || '127.0.0.1', - port: env.REDIS_PORT || 6379, - }, - database: env.REDIS_DB || 0, + host: env.REDIS_HOST || '127.0.0.1', + port: env.REDIS_PORT || 6379, + db: env.REDIS_DB || 0, password: env.REDIS_PASSWORD || undefined, }; - if (redisNamespace) { - redisParams.namespace = redisNamespace; + // redisParams.path takes precedence over host and port. + if (env.REDIS_URL && env.REDIS_URL.startsWith('unix://')) { + redisParams.path = env.REDIS_URL.slice(7); } - const redisPrefix = redisNamespace ? `${redisNamespace}:` : ''; - return { redisParams, redisPrefix, @@ -179,15 +160,15 @@ const startServer = async () => { const pgPool = new pg.Pool(pgConfigFromEnv(process.env)); const server = http.createServer(app); - const { redisParams, redisUrl, redisPrefix } = redisConfigFromEnv(process.env); - /** * @type {Object.): void>>} */ const subs = {}; - const redisSubscribeClient = await redisUrlToClient(redisParams, redisUrl); - const redisClient = await redisUrlToClient(redisParams, redisUrl); + const redisConfig = redisConfigFromEnv(process.env); + const redisSubscribeClient = await createRedisClient(redisConfig); + const redisClient = await createRedisClient(redisConfig); + const { redisPrefix } = redisConfig; // Collect metrics from Node.js metrics.collectDefaultMetrics(); @@ -277,13 +258,13 @@ const startServer = async () => { }; /** - * @param {string} message * @param {string} channel + * @param {string} message */ - const onRedisMessage = (message, channel) => { + const onRedisMessage = (channel, message) => { const callbacks = subs[channel]; - log.silly(`New message on channel ${channel}`); + log.silly(`New message on channel ${redisPrefix}${channel}`); if (!callbacks) { return; @@ -294,6 +275,7 @@ const startServer = async () => { callbacks.forEach(callback => callback(json)); }; + redisSubscribeClient.on("message", onRedisMessage); /** * @callback SubscriptionListener @@ -312,8 +294,14 @@ const startServer = async () => { if (subs[channel].length === 0) { log.verbose(`Subscribe ${channel}`); - redisSubscribeClient.subscribe(channel, onRedisMessage); - redisSubscriptions.inc(); + redisSubscribeClient.subscribe(channel, (err, count) => { + if (err) { + log.error(`Error subscribing to ${channel}`); + } + else { + redisSubscriptions.set(count); + } + }); } subs[channel].push(callback); @@ -334,8 +322,14 @@ const startServer = async () => { if (subs[channel].length === 0) { log.verbose(`Unsubscribe ${channel}`); - redisSubscribeClient.unsubscribe(channel); - redisSubscriptions.dec(); + redisSubscribeClient.unsubscribe(channel, (err, count) => { + if (err) { + log.error(`Error unsubscribing to ${channel}`); + } + else { + redisSubscriptions.set(count); + } + }); delete subs[channel]; } }; diff --git a/yarn.lock b/yarn.lock index 880ded977..74ed5214e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1452,6 +1452,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" @@ -1786,40 +1791,6 @@ resolved "https://registry.yarnpkg.com/@rails/ujs/-/ujs-7.0.7.tgz#54af8d66160a8a7bf7d8f184703d2bf4b3fab914" integrity sha512-J2v5Ca7HgejO7diGKiDylaVDQKmbQ5FJih6Oo3hXuBKEuXlcaccJu64lj8MNVLaPVyZx0g4gaOQZQz95QEb/hg== -"@redis/bloom@1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" - integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== - -"@redis/client@1.5.9": - version "1.5.9" - resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.9.tgz#c4ee81bbfedb4f1d9c7c5e9859661b9388fb4021" - integrity sha512-SffgN+P1zdWJWSXBvJeynvEnmnZrYmtKSRW00xl8pOPFOMJjxRR9u0frSxJpPR6Y4V+k54blJjGW7FgxbTI7bQ== - dependencies: - cluster-key-slot "1.1.2" - generic-pool "3.9.0" - yallist "4.0.0" - -"@redis/graph@1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" - integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== - -"@redis/json@1.0.4": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" - integrity sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw== - -"@redis/search@1.1.3": - version "1.1.3" - resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.3.tgz#b5a6837522ce9028267fe6f50762a8bcfd2e998b" - integrity sha512-4Dg1JjvCevdiCBTZqjhKkGoC5/BcB7k9j99kdMnaXFXg8x4eyOIVg9487CMv7/BUVkFLZCaIh8ead9mU15DNng== - -"@redis/time-series@1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad" - integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg== - "@reduxjs/toolkit@^1.9.5": version "1.9.5" resolved "https://registry.yarnpkg.com/@reduxjs/toolkit/-/toolkit-1.9.5.tgz#d3987849c24189ca483baa7aa59386c8e52077c4" @@ -4111,7 +4082,7 @@ clone-deep@^4.0.1: kind-of "^6.0.2" shallow-clone "^3.0.0" -cluster-key-slot@1.1.2: +cluster-key-slot@^1.1.0: version "1.1.2" resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== @@ -4857,6 +4828,11 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ== +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -6139,11 +6115,6 @@ gauge@^5.0.0: strip-ansi "^6.0.1" wide-align "^1.1.5" -generic-pool@3.9.0: - version "3.9.0" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" - integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== - gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -6823,6 +6794,21 @@ invariant@^2.2.2, invariant@^2.2.4: dependencies: loose-envify "^1.0.0" +ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" @@ -10283,17 +10269,17 @@ redent@^4.0.0: indent-string "^5.0.0" strip-indent "^4.0.0" -redis@^4.6.5: - version "4.6.8" - resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.8.tgz#54c5992e8a5ba512506fe9f53142cadc405547e7" - integrity sha512-S7qNkPUYrsofQ0ztWlTHSaK0Qqfl1y+WMIxrzeAGNG+9iUZB4HGeBgkHxE6uJJ6iXrkvLd1RVJ2nvu6H1sAzfQ== +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== dependencies: - "@redis/bloom" "1.2.0" - "@redis/client" "1.5.9" - "@redis/graph" "1.1.0" - "@redis/json" "1.0.4" - "@redis/search" "1.1.3" - "@redis/time-series" "1.0.5" + redis-errors "^1.0.0" redux-immutable@^4.0.0: version "4.0.0" @@ -11211,6 +11197,11 @@ stacktrace-js@^2.0.2: stack-generator "^2.0.5" stacktrace-gps "^3.0.4" +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + static-extend@^0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" @@ -12966,16 +12957,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yallist@4.0.0, yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yallist@^3.0.2: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yaml@^1.10.0: version "1.10.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"