From 62950becf93b475fbc97c006713820d05e84a522 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Fri, 17 May 2024 13:37:40 +0800 Subject: [PATCH 01/11] feat(monitoring): add scheduler functionality --- .aws/deploy/backend-task-definition.prod.json | 1 + .../backend-task-definition.staging.json | 1 + .aws/deploy/support-task-definition.prod.json | 1 + .../support-task-definition.staging.json | 1 + .env.test | 4 +- docker-compose.dev.yml | 9 + package-lock.json | 257 ++++++++++++++++++ package.json | 1 + src/config/config.ts | 9 +- src/monitoring/index.ts | 76 ++++++ src/server.ts | 1 + 11 files changed, 359 insertions(+), 2 deletions(-) diff --git a/.aws/deploy/backend-task-definition.prod.json b/.aws/deploy/backend-task-definition.prod.json index 4de452228..965be6072 100644 --- a/.aws/deploy/backend-task-definition.prod.json +++ b/.aws/deploy/backend-task-definition.prod.json @@ -132,6 +132,7 @@ "name": "REDIRECT_URI", "valueFrom": "PROD_REDIRECT_URI" }, + { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "PROD_SESSION_SECRET" diff --git a/.aws/deploy/backend-task-definition.staging.json b/.aws/deploy/backend-task-definition.staging.json index 2dcaa69a4..c6c1938d8 100644 --- a/.aws/deploy/backend-task-definition.staging.json +++ b/.aws/deploy/backend-task-definition.staging.json @@ -141,6 +141,7 @@ "name": "REDIRECT_URI", "valueFrom": "STAGING_REDIRECT_URI" }, + { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "STAGING_SESSION_SECRET" diff --git a/.aws/deploy/support-task-definition.prod.json b/.aws/deploy/support-task-definition.prod.json index 7db06bc19..0c62d5ff6 100644 --- a/.aws/deploy/support-task-definition.prod.json +++ b/.aws/deploy/support-task-definition.prod.json @@ -128,6 +128,7 @@ "name": "REDIRECT_URI", "valueFrom": "PROD_REDIRECT_URI" }, + { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "PROD_SESSION_SECRET" diff --git a/.aws/deploy/support-task-definition.staging.json b/.aws/deploy/support-task-definition.staging.json index c77efa8fc..dac20c2ef 100644 --- a/.aws/deploy/support-task-definition.staging.json +++ b/.aws/deploy/support-task-definition.staging.json @@ -137,6 +137,7 @@ "name": "REDIRECT_URI", "valueFrom": "STAGING_REDIRECT_URI" }, + { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "STAGING_SESSION_SECRET" diff --git a/.env.test b/.env.test index 5d4a4ec13..ad87b5418 100644 --- a/.env.test +++ b/.env.test @@ -86,4 +86,6 @@ export SGID_REDIRECT_URI="http://localhost:8081/v2/auth/sgid/auth-redirect" # GrowthBook export GROWTHBOOK_CLIENT_KEY="some random key" -export KEYCDN_API_KEY="secret" \ No newline at end of file +export KEYCDN_API_KEY="secret" + +export REDIS_HOST="redis" \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 046c0eae1..31896b87f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,10 +1,18 @@ version: "3" services: + redis: + image: redis + container_name: isomercms-redis + ports: + - 6379:6379 support: build: dockerfile: ./support/Dockerfile ports: - "8082:8082" + depends_on: + - postgres + - redis env_file: - .env volumes: @@ -18,6 +26,7 @@ services: dockerfile: Dockerfile depends_on: - postgres + - redis ports: - "8081:8081" env_file: diff --git a/package-lock.json b/package-lock.json index cce7c7ed4..f03f28a91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.19.2", + "bullmq": "^5.7.8", "cache-parser": "^1.2.4", "cloudmersive-virus-api-client": "^1.2.7", "connect-session-sequelize": "^7.1.5", @@ -3312,6 +3313,11 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==" + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -4457,6 +4463,78 @@ "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==" }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.2.tgz", + "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", + "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", + "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", + "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", + "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", + "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -7459,6 +7537,32 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/bullmq": { + "version": "5.7.8", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.8.tgz", + "integrity": "sha512-F/Haeu6AVHkFrfeaU/kLOjhfrH6x3CaKAZlQQ+76fa8l3kfI9oaUHeFMW+1mYVz0NtYPF7PNTWFq4ylAHYcCgA==", + "dependencies": { + "cron-parser": "^4.6.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.10.1", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, + "node_modules/bullmq/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -7804,6 +7908,14 @@ "superagent": "3.7.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -9130,6 +9242,17 @@ "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-fetch": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", @@ -9590,6 +9713,14 @@ "dev": true, "optional": true }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "engines": { + "node": ">=0.10" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -12191,6 +12322,50 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, + "node_modules/ioredis/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/ioredis/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/ip": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz", @@ -15675,6 +15850,11 @@ "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" + }, "node_modules/lodash.find": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.find/-/lodash.find-4.6.0.tgz", @@ -15685,6 +15865,11 @@ "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" + }, "node_modules/lodash.isboolean": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", @@ -15876,6 +16061,14 @@ "node": ">=12" } }, + "node_modules/luxon": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", + "integrity": "sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==", + "engines": { + "node": ">=12" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -16379,6 +16572,35 @@ "msgpack": "bin/msgpack" } }, + "node_modules/msgpackr": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.2.tgz", + "integrity": "sha512-L60rsPynBvNE+8BWipKKZ9jHcSGbtyJYIwjRq0VrIvQ08cRjntGXJYW/tmciZ2IHWIY8WEW32Qa2xbh5+SKBZA==", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.2.tgz", + "integrity": "sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.0.7" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.2", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.2" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -16531,6 +16753,17 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.7.tgz", + "integrity": "sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==", + "optional": true, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -18123,6 +18356,25 @@ "node": ">= 12.13.0" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -19181,6 +19433,11 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" + }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", diff --git a/package.json b/package.json index e90cc5838..a3215558d 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.19.2", + "bullmq": "^5.7.8", "cache-parser": "^1.2.4", "cloudmersive-virus-api-client": "^1.2.7", "connect-session-sequelize": "^7.1.5", diff --git a/src/config/config.ts b/src/config/config.ts index 4b4d60888..5e2cebb69 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -195,7 +195,14 @@ const config = convict({ }, }, }, - + bullmq: { + redisHostname: { + doc: "Redis host name for bullmq", + env: "REDIS_HOST", + format: "required-string", + default: "isomerpages", + }, + }, github: { orgName: { doc: "GitHub organization that owns all site repositories", diff --git a/src/monitoring/index.ts b/src/monitoring/index.ts index de6e43ec0..7378b9daf 100644 --- a/src/monitoring/index.ts +++ b/src/monitoring/index.ts @@ -4,6 +4,7 @@ import { retry } from "@octokit/plugin-retry" import { Octokit } from "@octokit/rest" import autoBind from "auto-bind" import axios from "axios" +import { Job, Queue, Worker } from "bullmq" import _ from "lodash" import { errAsync, okAsync, ResultAsync } from "neverthrow" @@ -12,6 +13,7 @@ import parentLogger from "@logger/logger" import config from "@root/config/config" import MonitoringError from "@root/errors/MonitoringError" import LaunchesService from "@root/services/identity/LaunchesService" +import convertNeverThrowToPromise from "@root/utils/neverthrow" import promisifyPapaParse from "@root/utils/papa-parse" interface MonitoringServiceProps { @@ -65,9 +67,74 @@ export default class MonitoringService { module: "monitoringService", }) + private readonly REDIS_CONNECTION = { + host: config.get("bullmq.redisHostname"), + port: 6379, + } + + private readonly queue = new Queue("MonitoringQueue", { + connection: { + ...this.REDIS_CONNECTION, + }, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + attempts: 3, + backoff: { + type: "exponential", + delay: 60000, // this operation is not critical, so we can wait a minute + }, + }, + }) + + private readonly worker: Worker | undefined + constructor({ launchesService }: MonitoringServiceProps) { autoBind(this) + const jobName = "dnsMonitoring" this.launchesService = launchesService + this.worker = new Worker( + this.queue.name, + async (job: Job) => { + this.monitoringServiceLogger.info(`Monitoring Worker ${job.id}`) + if (job.name === jobName) { + // The retry's work on a thrown error, so we need to convert the neverthrow to a promise + const res = await convertNeverThrowToPromise(this.driver()) + return res + } + throw new MonitoringError("Invalid job name") + }, + { + connection: { + ...this.REDIS_CONNECTION, + }, + lockDuration: 60000, // 1 minute, since this is a relatively expensive operation + } + ) + + const dailyCron = "0 0 9 * *" + + ResultAsync.fromPromise( + this.queue.add( + jobName, + {}, + { + repeat: { + pattern: dailyCron, + }, + } + ), + (e) => e + ) + .map((ok) => { + this.monitoringServiceLogger.info( + `Monitoring job scheduled at ${dailyCron}` + ) + return ok + }) + .mapErr((error) => { + this.monitoringServiceLogger.error(`Failed to schedule job: ${error}`) + }) } getKeyCdnDomains() { @@ -227,7 +294,9 @@ export default class MonitoringService { } driver() { + const start = Date.now() this.monitoringServiceLogger.info("Monitoring service started") + return this.getAllDomains() .andThen(this.generateReportCard) .andThen((reportCard) => { @@ -240,5 +309,12 @@ export default class MonitoringService { }) return okAsync(reportCard) }) + .orElse(() => okAsync([])) + .andThen(() => { + this.monitoringServiceLogger.info( + `Monitoring service completed in ${Date.now() - start}ms` + ) + return okAsync("Monitoring service completed") + }) } } diff --git a/src/server.ts b/src/server.ts index 4c1614333..58ff93c66 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3,6 +3,7 @@ import "./utils/tracer" import "module-alias/register" import { SgidClient } from "@opengovsg/sgid-client" +import { Queue } from "bullmq" import SequelizeStoreFactory from "connect-session-sequelize" import cors from "cors" import express from "express" From 475bf977de62689aa0842c42bdfa49a837864fd4 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:48:23 +0800 Subject: [PATCH 02/11] feat(monitoring): integration with siteup --- package-lock.json | 8 +- package.json | 2 +- src/constants/featureFlags.ts | 1 + src/middleware/featureFlag.ts | 2 +- src/monitoring/index.ts | 155 ++++------- src/types/featureFlags.ts | 1 + src/utils/dns-utils.ts | 259 +++++++++++++++++ src/utils/growthbook-utils.ts | 7 + support/routes/v2/isobot/ops/botService.ts | 310 +-------------------- 9 files changed, 343 insertions(+), 402 deletions(-) create mode 100644 src/utils/dns-utils.ts diff --git a/package-lock.json b/package-lock.json index f03f28a91..b17ae63e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,7 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.19.2", - "bullmq": "^5.7.8", + "bullmq": "^5.7.15", "cache-parser": "^1.2.4", "cloudmersive-virus-api-client": "^1.2.7", "connect-session-sequelize": "^7.1.5", @@ -7538,9 +7538,9 @@ "dev": true }, "node_modules/bullmq": { - "version": "5.7.8", - "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.8.tgz", - "integrity": "sha512-F/Haeu6AVHkFrfeaU/kLOjhfrH6x3CaKAZlQQ+76fa8l3kfI9oaUHeFMW+1mYVz0NtYPF7PNTWFq4ylAHYcCgA==", + "version": "5.7.15", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.7.15.tgz", + "integrity": "sha512-XR5sTA8BPUY67sS37sMKGCDvSLaVpMq7aaQG8FGSKOUnPoJMRf17n1TibVWP3+yK0xKLdK5Y7PY9D874Fpeqpg==", "dependencies": { "cron-parser": "^4.6.0", "ioredis": "^5.4.1", diff --git a/package.json b/package.json index a3215558d..c47a856f0 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "bcryptjs": "^2.4.3", "bluebird": "^3.7.2", "body-parser": "^1.19.2", - "bullmq": "^5.7.8", + "bullmq": "^5.7.15", "cache-parser": "^1.2.4", "cloudmersive-virus-api-client": "^1.2.7", "connect-session-sequelize": "^7.1.5", diff --git a/src/constants/featureFlags.ts b/src/constants/featureFlags.ts index f01f0c0af..4901c2cfb 100644 --- a/src/constants/featureFlags.ts +++ b/src/constants/featureFlags.ts @@ -4,4 +4,5 @@ export const FEATURE_FLAGS = { IS_SHOW_STAGING_BUILD_STATUS_ENABLED: "is_show_staging_build_status_enabled", IS_CLOUDMERSIVE_ENABLED: "is_cloudmersive_enabled", IS_LOCAL_DIFF_ENABLED: "is_local_diff_enabled", + IS_MONITORING_ENABLED: "is_monitoring_enabled", } as const diff --git a/src/middleware/featureFlag.ts b/src/middleware/featureFlag.ts index bc1223e99..2b9208ca2 100644 --- a/src/middleware/featureFlag.ts +++ b/src/middleware/featureFlag.ts @@ -6,7 +6,7 @@ import { getNewGrowthbookInstance } from "@root/utils/growthbook-utils" // Keep one GrowthBook instance at module level // The instance will handle internal cache refreshes via a SSE connection -const gb = getNewGrowthbookInstance({ +export const gb = getNewGrowthbookInstance({ clientKey: config.get("growthbook.clientKey"), subscribeToChanges: true, }) diff --git a/src/monitoring/index.ts b/src/monitoring/index.ts index 7378b9daf..f96e4167d 100644 --- a/src/monitoring/index.ts +++ b/src/monitoring/index.ts @@ -1,5 +1,3 @@ -import dns from "dns/promises" - import { retry } from "@octokit/plugin-retry" import { Octokit } from "@octokit/rest" import autoBind from "auto-bind" @@ -9,10 +7,14 @@ import _ from "lodash" import { errAsync, okAsync, ResultAsync } from "neverthrow" import parentLogger from "@logger/logger" +import logger from "@logger/logger" import config from "@root/config/config" import MonitoringError from "@root/errors/MonitoringError" +import { gb } from "@root/middleware/featureFlag" import LaunchesService from "@root/services/identity/LaunchesService" +import { dnsMonitor } from "@root/utils/dns-utils" +import { isMonitoringEnabled } from "@root/utils/growthbook-utils" import convertNeverThrowToPromise from "@root/utils/neverthrow" import promisifyPapaParse from "@root/utils/papa-parse" @@ -41,15 +43,6 @@ interface RedirectionDomain { target: string } -interface ReportCard { - domain: string - type: typeof IsomerHostedDomainType[keyof typeof IsomerHostedDomainType] - aRecord: string[] - quadArecord: string[] - cNameRecord: string[] - caaRecord: string[] -} - function isKeyCdnZoneAlias(object: unknown): object is KeyCdnZoneAlias { return "name" in (object as KeyCdnZoneAlias) } @@ -59,7 +52,7 @@ function isKeyCdnResponse(object: unknown): object is KeyCdnZoneAlias[] { if (Array.isArray(object)) return object.every(isKeyCdnZoneAlias) return false } - +const ONE_MINUTE = 60000 export default class MonitoringService { private readonly launchesService: MonitoringServiceProps["launchesService"] @@ -82,17 +75,42 @@ export default class MonitoringService { attempts: 3, backoff: { type: "exponential", - delay: 60000, // this operation is not critical, so we can wait a minute + delay: ONE_MINUTE, // this operation is not critical, so we can wait a minute }, }, }) - private readonly worker: Worker | undefined + private readonly worker: Worker constructor({ launchesService }: MonitoringServiceProps) { autoBind(this) const jobName = "dnsMonitoring" this.launchesService = launchesService + + const FIVE_MINUTE_CRON = "5 * * * *" + + const jobData = { + name: "monitoring sites", + } + + ResultAsync.fromPromise( + this.queue.add(jobName, jobData, { + repeat: { + pattern: FIVE_MINUTE_CRON, + }, + }), + (e) => e + ) + .map((okRes) => { + this.monitoringServiceLogger.info( + `Monitoring job scheduled at interval ${FIVE_MINUTE_CRON}` + ) + return okRes + }) + .mapErr((errRes) => { + this.monitoringServiceLogger.error(`Failed to schedule job: ${errRes}`) + }) + this.worker = new Worker( this.queue.name, async (job: Job) => { @@ -112,29 +130,15 @@ export default class MonitoringService { } ) - const dailyCron = "0 0 9 * *" - - ResultAsync.fromPromise( - this.queue.add( - jobName, - {}, - { - repeat: { - pattern: dailyCron, - }, - } - ), - (e) => e - ) - .map((ok) => { - this.monitoringServiceLogger.info( - `Monitoring job scheduled at ${dailyCron}` - ) - return ok - }) - .mapErr((error) => { - this.monitoringServiceLogger.error(`Failed to schedule job: ${error}`) + this.worker.on("failed", (job: Job | undefined, error: Error) => { + logger.error({ + message: "Monitoring service has failed", + error, + meta: { + ...job?.data, + }, }) + }) } getKeyCdnDomains() { @@ -192,8 +196,8 @@ export default class MonitoringService { octokitWithRetry.request( "GET /repos/opengovsg/isomer-redirection/contents/src/certbot-websites.csv" ), - (error) => - new MonitoringError(`Failed to fetch redirection domains: ${error}`) + (err) => + new MonitoringError(`Failed to fetch redirection domains: ${err}`) ) .andThen((response) => { const content = Buffer.from(response.data.content, "base64").toString( @@ -202,7 +206,7 @@ export default class MonitoringService { return ResultAsync.fromPromise( promisifyPapaParse(content), - (error) => new MonitoringError(`Failed to parse csv: ${error}`) + (err) => new MonitoringError(`Failed to parse csv: ${err}`) ) }) .map((redirectionDomains) => @@ -223,7 +227,7 @@ export default class MonitoringService { this.monitoringServiceLogger.info("Fetching all domains") return ResultAsync.combine([ this.getAmplifyDeployments().mapErr( - (error) => new MonitoringError(error.message) + (err) => new MonitoringError(err.message) ), this.getRedirectionDomains(), this.getKeyCdnDomains(), @@ -238,76 +242,35 @@ export default class MonitoringService { }) } - // todo: once /siteup logic is merged into dev, we can add that as to alert isomer team generateReportCard(domains: IsomerHostedDomain[]) { - const reportCard: ReportCard[] = [] + const dnsPromises: ResultAsync[] = [] - const domainResolvers = domains.map(({ domain, type }) => { - const aRecord = ResultAsync.fromPromise( - dns.resolve(domain, "A"), - (e) => e - ).orElse(() => okAsync([])) - const quadArecord = ResultAsync.fromPromise( - dns.resolve(domain, "AAAA"), - (e) => e - ).orElse(() => okAsync([])) - - const cNameRecord = ResultAsync.fromPromise( - dns.resolve(domain, "CNAME"), - (e) => e - ).orElse(() => okAsync([])) - - const caaRecord = ResultAsync.fromPromise( - dns.resolve(domain, "CAA"), - (e) => e - ) - .orElse(() => okAsync([])) - .map((records) => records.map((record) => record.toString())) - - return ResultAsync.combineWithAllErrors([ - aRecord, - quadArecord, - cNameRecord, - caaRecord, - ]) - .andThen((resolvedDns) => - okAsync({ - domain, - type, - aRecord: resolvedDns[0], - quadArecord: resolvedDns[1], - cNameRecord: resolvedDns[2], - caaRecord: resolvedDns[3], - }) - ) - .map((value) => - reportCard.push({ - ...value, - }) - ) - .andThen(() => okAsync(reportCard)) - }) - - return ResultAsync.combineWithAllErrors(domainResolvers).map( - () => reportCard - ) + domains.forEach((domain) => dnsPromises.push(dnsMonitor(domain.domain))) + return ResultAsync.combineWithAllErrors(dnsPromises) } driver() { + if (!isMonitoringEnabled(gb)) return okAsync("Monitoring Service disabled") const start = Date.now() this.monitoringServiceLogger.info("Monitoring service started") return this.getAllDomains() .andThen(this.generateReportCard) - .andThen((reportCard) => { - this.monitoringServiceLogger.info({ - message: "Report card generated", + .mapErr((reportCardErr: MonitoringError | string[]) => { + if (reportCardErr instanceof MonitoringError) { + this.monitoringServiceLogger.error({ + error: reportCardErr, + message: "Error running monitoring service", + }) + return + } + this.monitoringServiceLogger.error({ + message: "Error running monitoring service", meta: { - reportCard, + dnsCheckerResult: reportCardErr, date: new Date(), }, }) - return okAsync(reportCard) }) .orElse(() => okAsync([])) .andThen(() => { diff --git a/src/types/featureFlags.ts b/src/types/featureFlags.ts index 0fecdd8ca..69923b119 100644 --- a/src/types/featureFlags.ts +++ b/src/types/featureFlags.ts @@ -13,6 +13,7 @@ export interface FeatureFlags { is_show_staging_build_status_enabled: boolean is_cloudmersive_enabled: CloudmersiveConfigType is_local_diff_enabled: boolean + is_monitoring_enabled: boolean } // List of attributes we set in GrowthBook Instance in auth middleware diff --git a/src/utils/dns-utils.ts b/src/utils/dns-utils.ts new file mode 100644 index 000000000..8752663e4 --- /dev/null +++ b/src/utils/dns-utils.ts @@ -0,0 +1,259 @@ +import dns from "node:dns/promises" + +import { err, ok, okAsync, Result, ResultAsync } from "neverthrow" + +import { + DNS_CNAME_SUFFIXES, + DNS_INDIRECTION_DOMAIN, + DNS_KEYCDN_SUFFIX, + REDIRECTION_SERVER_IPS, +} from "@root/constants" +import logger from "@root/logger/logger" + +export function checkCname(domain: string) { + return ResultAsync.fromPromise(dns.resolveCname(domain), () => { + logger.error({ + message: "Error resolving CNAME", + meta: { domain, method: "checkCname" }, + }) + return new Error() + }) + .andThen((cname) => { + if (!cname || cname.length === 0) { + return okAsync(null) + } + + return okAsync(cname[0]) + }) + .orElse(() => okAsync(null)) +} + +export function checkA(domain: string) { + return ResultAsync.fromPromise(dns.resolve4(domain), () => { + logger.error({ + message: "Error resolving A record", + meta: { domain, method: "checkA" }, + }) + return new Error() + }) + .andThen((a) => { + if (!a || a.length === 0) { + return okAsync(null) + } + + return okAsync(a) + }) + .orElse(() => okAsync(null)) +} + +export default function getDnsCheckerMessage( + domain: string, + cnameDomain: string | null, + redirectionDomain: string, + cnameRecord: string | null, + indirectionDomain: string, + intermediateRecords: string[] | null, + redirectionRecords: string[] | null +): Result { + // Domain has a CNAME pointing to one of our known suffixes + const isDomainCnameCorrect = + !!cnameRecord && + (cnameRecord.endsWith(`.${DNS_INDIRECTION_DOMAIN}`) || + DNS_CNAME_SUFFIXES.some((suffix) => cnameRecord.endsWith(`.${suffix}`))) + + // Domain is directly pointing to our indirection layer + const isDomainOnIndirection = + cnameRecord && cnameRecord.endsWith(`.${DNS_INDIRECTION_DOMAIN}`) + + // The intermediate layer (indirection or CNAME) is resolving to valid IPs + // If on KeyCDN, then there should only be 1 IP, otherwise there should be 4 + const isIntermediateValid = + intermediateRecords && + ((cnameRecord && + cnameRecord.endsWith(`.${DNS_KEYCDN_SUFFIX}`) && + intermediateRecords.length === 1) || + (intermediateRecords.length === 4 && + (isDomainCnameCorrect || + indirectionDomain.endsWith(`.${DNS_INDIRECTION_DOMAIN}`)))) + + // The redirection domain has records that are all resolving to our known IPs + const isRedirectionValid = + redirectionRecords && + redirectionRecords.every((ip) => REDIRECTION_SERVER_IPS.includes(ip)) + + if (isDomainCnameCorrect && cnameDomain && !cnameDomain.startsWith("www.")) { + // Domain is a CNAME domain and does not have www in it -> No redirection domain to check + if (isDomainOnIndirection) { + if (isIntermediateValid) { + return ok(`The domain ${domain} is *all valid*!`) + } + return err( + `Jialat, the domain \`${domain}\` is correctly pointing to our indirection layer, but the indirection layer does not seem to be configured correctly.\nThis is *OUR* fault` + ) + } + return err( + `Hmm, the domain ${domain} is *valid*, but it points to ${cnameRecord} which is not something that I recognise. Probably not an Isomer site?\n- A records checker: https://dnschecker.org/#A/${domain}\n- CNAME records checker: https://dnschecker.org/#CNAME/${domain}` + ) + } + if (isDomainCnameCorrect && cnameDomain && cnameDomain.startsWith("www.")) { + // Domain is a CNAME domain and also starts with www -> Check the apex domain as well + if (isDomainOnIndirection) { + if (isIntermediateValid && isRedirectionValid) { + return ok( + `The domain ${domain} is *all valid*! The apex domain \`${redirectionDomain}\` is also correctly configured to our redirection service.` + ) + } + if (isIntermediateValid && !isRedirectionValid) { + return err( + `Weird eh, the domain \`${domain}\` is correctly pointing to our indirection layer but the apex domain is not configured correctly.\nThis is *AGENCY fault*!` + ) + } + if (!isIntermediateValid && isRedirectionValid) { + return err( + `Jialat, the domain \`${domain}\` is correctly pointing to our indirection layer and the apex domain is correctly configured, but the indirection layer does not seem to be configured correctly.\nThis is *OUR fault*!` + ) + } + return err( + `Wah, the domain \`${domain}\` is correctly pointing to our indirection layer, but both the indirection layer and the apex domain does not seem to be configured correctly.\nThis is *both OUR and AGENCY fault*!` + ) + } + if (isIntermediateValid && isRedirectionValid) { + return err( + `The domain ${domain} is *all valid*! Although it is directly pointing to our CDN hosting provider and not the indirection layer. The apex domain \`${redirectionDomain}\` is also correctly configured to our redirection service.` + ) + } + if (isIntermediateValid && !isRedirectionValid) { + return err( + `Weird eh, the domain \`${domain}\` is correctly pointing to our CDN hosting provider (not indirection layer) but the apex domain is not configured correctly.\nThis is *AGENCY fault*!` + ) + } + if (!isIntermediateValid && isRedirectionValid) { + return err( + `Jialat, the domain \`${domain}\` is correctly pointing to our CDN hosting provider (not indirection layer) and the apex domain is correctly configured, but the CDN hosting provider does not seem to be configured correctly.\nThis is *OUR fault*!` + ) + } + return err( + `Wah, the domain \`${domain}\` is correctly pointing to our CDN hosting provider (not indirection layer), but both the CDN hosting provider and the apex domain does not seem to be configured correctly.\nThis is *both OUR and AGENCY fault*!` + ) + } + if (isIntermediateValid) { + // Domain is likely gone or points directly to A records, but our indirection layer is still correct + return err( + `Oh no, the domain \`${domain}\` seems to be gone, but okay lah our indirection layer is correctly configured.\nThis is *AGENCY fault*!` + ) + } + if (!cnameRecord && !intermediateRecords && !redirectionRecords) { + // Everything is gone + return err( + `Jialat, the DNS for \`${domain}\` seems to be gone, and our indirection layer does not seem to be configured correctly.\nIf this site is supposed to be live, then this is *both OUR and AGENCY fault*!` + ) + } + // Some weird configuration + + let responseMeta = `Wah, I don't know how to handle \`${domain}\` sia, might need to manually check.\n- A records checker: https://dnschecker.org/#A/${domain}\n- CNAME records checker: https://dnschecker.org/#CNAME/${domain}\n- \`${domain}\` points to ${ + cnameRecord ? `\`${cnameRecord}\`` : "no CNAME records" + }` + + if (cnameRecord || indirectionDomain) { + responseMeta += `- \`${cnameRecord || indirectionDomain}\` points to ${ + intermediateRecords ? `\`${intermediateRecords.join(", ")}\`` : "nothing" + }` + } + + if (redirectionDomain !== cnameDomain) { + responseMeta += `- \`${redirectionDomain}\` points to ${ + redirectionRecords + ? `\`${redirectionRecords.join(", ")}\`` + : "no A records" + }` + } + return err(responseMeta) +} + +export function dnsMonitor(domain: string): ResultAsync { + return checkCname(domain) + .andThen((cname) => { + // Original domain does not have a CNAME record, check if the www + // version has a valid CNAME record + if (!cname && !domain.startsWith("www.")) { + const cnameDomain = `www.${domain}` + return ResultAsync.combine([ + okAsync(cnameDomain), + checkCname(cnameDomain), + ]) + } + + return ResultAsync.combine([okAsync(domain), okAsync(cname)]) + }) + .andThen(([cnameDomain, cnameRecord]) => { + // Original and www version of the domain do not have a CNAME record, + // check if our indirection domain is still correct + if (!cnameRecord) { + const indirectionDomain = domain.startsWith("www.") + ? `${domain.slice(4).replaceAll(".", "-")}.${DNS_INDIRECTION_DOMAIN}` + : `${domain.replaceAll(".", "-")}.${DNS_INDIRECTION_DOMAIN}` + + return ResultAsync.combine([ + okAsync({ + cnameDomain: null, + cnameRecord, + indirectionDomain, + }), + checkA(indirectionDomain), + ]) + } + + // Either the original or www version of the domain has a CNAME record, + // check if the CNAME record is valid + return ResultAsync.combine([ + okAsync({ + cnameDomain, + cnameRecord, + indirectionDomain: cnameRecord, + }), + checkA(cnameRecord), + ]) + }) + .andThen( + ([ + { cnameDomain, cnameRecord, indirectionDomain }, + indirectionRecords, + ]) => { + const redirectionDomain = domain.startsWith("www.") + ? domain.slice(4) + : domain + + return ResultAsync.combine([ + okAsync({ + cnameDomain, + cnameRecord, + indirectionDomain, + indirectionRecords, + redirectionDomain, + }), + checkA(redirectionDomain), + ]) + } + ) + .andThen( + ([ + { + cnameDomain, + cnameRecord, + indirectionDomain, + indirectionRecords, + redirectionDomain, + }, + redirection, + ]) => + getDnsCheckerMessage( + domain, + cnameDomain, + redirectionDomain, + cnameRecord, + indirectionDomain, + indirectionRecords, + redirection + ) + ) +} diff --git a/src/utils/growthbook-utils.ts b/src/utils/growthbook-utils.ts index a6cdee8c3..bf55490d4 100644 --- a/src/utils/growthbook-utils.ts +++ b/src/utils/growthbook-utils.ts @@ -67,3 +67,10 @@ export const isCloudmersiveEnabled = ( defaultConfig ) } + +export const isMonitoringEnabled = ( + growthbook: GrowthBook | undefined +): boolean => { + if (!growthbook) return true + return growthbook.getFeatureValue(FEATURE_FLAGS.IS_MONITORING_ENABLED, true) +} diff --git a/support/routes/v2/isobot/ops/botService.ts b/support/routes/v2/isobot/ops/botService.ts index ea5b79c6c..5ad3098e2 100644 --- a/support/routes/v2/isobot/ops/botService.ts +++ b/support/routes/v2/isobot/ops/botService.ts @@ -1,17 +1,10 @@ -import dns from "node:dns/promises" - import { SlashCommand } from "@slack/bolt" -import { ResultAsync, okAsync } from "neverthrow" +import { ok } from "neverthrow" -import { - DNS_CNAME_SUFFIXES, - DNS_INDIRECTION_DOMAIN, - DNS_KEYCDN_SUFFIX, - REDIRECTION_SERVER_IPS, -} from "@root/constants" import logger from "@root/logger/logger" import WhitelistService from "@root/services/identity/WhitelistService" import { DnsCheckerResponse } from "@root/types/dnsChecker" +import { dnsMonitor } from "@root/utils/dns-utils" class BotService { whitelistService: WhitelistService @@ -20,42 +13,6 @@ class BotService { this.whitelistService = whitelistService } - private checkCname(domain: string) { - return ResultAsync.fromPromise(dns.resolveCname(domain), () => { - logger.error({ - message: "Error resolving CNAME", - meta: { domain, method: "checkCname" }, - }) - return new Error() - }) - .andThen((cname) => { - if (!cname || cname.length === 0) { - return okAsync(null) - } - - return okAsync(cname[0]) - }) - .orElse(() => okAsync(null)) - } - - private checkA(domain: string) { - return ResultAsync.fromPromise(dns.resolve4(domain), () => { - logger.error({ - message: "Error resolving A record", - meta: { domain, method: "checkA" }, - }) - return new Error() - }) - .andThen((a) => { - if (!a || a.length === 0) { - return okAsync(null) - } - - return okAsync(a) - }) - .orElse(() => okAsync(null)) - } - private getSlackMessage(message: string | string[]): DnsCheckerResponse { return { response_type: "in_channel", @@ -110,172 +67,6 @@ class BotService { return domain } - getDnsCheckerMessage( - domain: string, - cnameDomain: string | null, - redirectionDomain: string, - cnameRecord: string | null, - indirectionDomain: string, - intermediateRecords: string[] | null, - redirectionRecords: string[] | null - ) { - // Domain has a CNAME pointing to one of our known suffixes - const isDomainCnameCorrect = - !!cnameRecord && - (cnameRecord.endsWith(`.${DNS_INDIRECTION_DOMAIN}`) || - DNS_CNAME_SUFFIXES.some((suffix) => cnameRecord.endsWith(`.${suffix}`))) - - // Domain is directly pointing to our indirection layer - const isDomainOnIndirection = - cnameRecord && cnameRecord.endsWith(`.${DNS_INDIRECTION_DOMAIN}`) - - // The intermediate layer (indirection or CNAME) is resolving to valid IPs - // If on KeyCDN, then there should only be 1 IP, otherwise there should be 4 - const isIntermediateValid = - intermediateRecords && - ((cnameRecord && - cnameRecord.endsWith(`.${DNS_KEYCDN_SUFFIX}`) && - intermediateRecords.length === 1) || - (intermediateRecords.length === 4 && - (isDomainCnameCorrect || - indirectionDomain.endsWith(`.${DNS_INDIRECTION_DOMAIN}`)))) - - // The redirection domain has records that are all resolving to our known IPs - const isRedirectionValid = - redirectionRecords && - redirectionRecords.every((ip) => REDIRECTION_SERVER_IPS.includes(ip)) - - const response = [] - - if ( - isDomainCnameCorrect && - cnameDomain && - !cnameDomain.startsWith("www.") - ) { - // Domain is a CNAME domain and does not have www in it -> No redirection domain to check - if (isDomainOnIndirection) { - if (isIntermediateValid) { - response.push(`The domain ${domain} is *all valid*!`) - } else { - response.push( - `Jialat, the domain \`${domain}\` is correctly pointing to our indirection layer, but the indirection layer does not seem to be configured correctly.` - ) - response.push("This is *OUR* fault!") - } - } else { - response.push( - `Hmm, the domain ${domain} is *valid*, but it points to ${cnameRecord} which is not something that I recognise. Probably not an Isomer site?` - ) - response.push( - `- A records checker: https://dnschecker.org/#A/${domain}` - ) - response.push( - `- CNAME records checker: https://dnschecker.org/#CNAME/${domain}` - ) - } - } else if ( - isDomainCnameCorrect && - cnameDomain && - cnameDomain.startsWith("www.") - ) { - // Domain is a CNAME domain and also starts with www -> Check the apex domain as well - if (isDomainOnIndirection) { - if (isIntermediateValid && isRedirectionValid) { - response.push( - `The domain ${domain} is *all valid*! The apex domain \`${redirectionDomain}\` is also correctly configured to our redirection service.` - ) - } else if (isIntermediateValid && !isRedirectionValid) { - response.push( - `Weird eh, the domain \`${domain}\` is correctly pointing to our indirection layer but the apex domain is not configured correctly.` - ) - response.push("This is *AGENCY fault*!") - } else if (!isIntermediateValid && isRedirectionValid) { - response.push( - `Jialat, the domain \`${domain}\` is correctly pointing to our indirection layer and the apex domain is correctly configured, but the indirection layer does not seem to be configured correctly.` - ) - response.push("This is *OUR fault*!") - } else { - response.push( - `Wah, the domain \`${domain}\` is correctly pointing to our indirection layer, but both the indirection layer and the apex domain does not seem to be configured correctly.` - ) - response.push("This is *both OUR and AGENCY fault*!") - } - } else if (isIntermediateValid && isRedirectionValid) { - response.push( - `The domain ${domain} is *all valid*! Although it is directly pointing to our CDN hosting provider and not the indirection layer. The apex domain \`${redirectionDomain}\` is also correctly configured to our redirection service.` - ) - } else if (isIntermediateValid && !isRedirectionValid) { - response.push( - `Weird eh, the domain \`${domain}\` is correctly pointing to our CDN hosting provider (not indirection layer) but the apex domain is not configured correctly.` - ) - response.push("This is *AGENCY fault*!") - } else if (!isIntermediateValid && isRedirectionValid) { - response.push( - `Jialat, the domain \`${domain}\` is correctly pointing to our CDN hosting provider (not indirection layer) and the apex domain is correctly configured, but the CDN hosting provider does not seem to be configured correctly.` - ) - response.push("This is *OUR fault*!") - } else { - response.push( - `Wah, the domain \`${domain}\` is correctly pointing to our CDN hosting provider (not indirection layer), but both the CDN hosting provider and the apex domain does not seem to be configured correctly.` - ) - response.push("This is *both OUR and AGENCY fault*!") - } - } else if (isIntermediateValid) { - // Domain is likely gone or points directly to A records, but our indirection layer is still correct - response.push( - `Oh no, the domain \`${domain}\` seems to be gone, but okay lah our indirection layer is correctly configured.` - ) - response.push("This is *AGENCY fault*!") - } else if (!cnameRecord && !intermediateRecords && !redirectionRecords) { - // Everything is gone - response.push( - `Jialat, the DNS for \`${domain}\` seems to be gone, and our indirection layer does not seem to be configured correctly.` - ) - response.push( - "If this site is supposed to be live, then this is *both OUR and AGENCY fault*!" - ) - } else { - // Some weird configuration - response.push( - `Wah, I don't know how to handle \`${domain}\` sia, might need to manually check.` - ) - response.push( - `- A records checker: https://dnschecker.org/#A/${domain}` - ) - response.push( - `- CNAME records checker: https://dnschecker.org/#CNAME/${domain}` - ) - } - - response.push( - `- \`${domain}\` points to ${ - cnameRecord ? `\`${cnameRecord}\`` : "no CNAME records" - }` - ) - - if (cnameRecord || indirectionDomain) { - response.push( - `- \`${cnameRecord || indirectionDomain}\` points to ${ - intermediateRecords - ? `\`${intermediateRecords.join(", ")}\`` - : "nothing" - }` - ) - } - - if (redirectionDomain !== cnameDomain) { - response.push( - `- \`${redirectionDomain}\` points to ${ - redirectionRecords - ? `\`${redirectionRecords.join(", ")}\`` - : "no A records" - }` - ) - } - - return this.getSlackMessage(response) - } - dnsChecker(payload: SlashCommand) { // Step 1: Get the domain name provided by the user const { user_name: user, channel_name: channel, text: domain } = payload @@ -289,95 +80,14 @@ class BotService { }, }) - return this.checkCname(domain) - .andThen((cname) => { - // Original domain does not have a CNAME record, check if the www - // version has a valid CNAME record - if (!cname && !domain.startsWith("www.")) { - const cnameDomain = `www.${domain}` - return ResultAsync.combine([ - okAsync(cnameDomain), - this.checkCname(cnameDomain), - ]) - } - - return ResultAsync.combine([okAsync(domain), okAsync(cname)]) - }) - .andThen(([cnameDomain, cnameRecord]) => { - // Original and www version of the domain do not have a CNAME record, - // check if our indirection domain is still correct - if (!cnameRecord) { - const indirectionDomain = domain.startsWith("www.") - ? `${domain - .slice(4) - .replaceAll(".", "-")}.${DNS_INDIRECTION_DOMAIN}` - : `${domain.replaceAll(".", "-")}.${DNS_INDIRECTION_DOMAIN}` - - return ResultAsync.combine([ - okAsync({ - cnameDomain: null, - cnameRecord, - indirectionDomain, - }), - this.checkA(indirectionDomain), - ]) - } - - // Either the original or www version of the domain has a CNAME record, - // check if the CNAME record is valid - return ResultAsync.combine([ - okAsync({ - cnameDomain, - cnameRecord, - indirectionDomain: cnameRecord, - }), - this.checkA(cnameRecord), - ]) - }) - .andThen( - ([ - { cnameDomain, cnameRecord, indirectionDomain }, - indirectionRecords, - ]) => { - const redirectionDomain = domain.startsWith("www.") - ? domain.slice(4) - : domain - - return ResultAsync.combine([ - okAsync({ - cnameDomain, - cnameRecord, - indirectionDomain, - indirectionRecords, - redirectionDomain, - }), - this.checkA(redirectionDomain), - ]) - } - ) - .andThen( - ([ - { - cnameDomain, - cnameRecord, - indirectionDomain, - indirectionRecords, - redirectionDomain, - }, - redirection, - ]) => - okAsync( - this.getDnsCheckerMessage( - domain, - cnameDomain, - redirectionDomain, - cnameRecord, - indirectionDomain, - indirectionRecords, - redirection - ) - ) - ) + return ( + dnsMonitor(domain) + // when running the slack bot command, the slack bot + // does not need to know what is the error state, it just needs the + // string to show isomer admins of what the state is + .orElse((res) => ok(res)) + .map((res) => this.getSlackMessage(res)) + ) } } From 0d722e0ca741e68dc554c6cfedaa42e9c0d577c9 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:01:58 +0800 Subject: [PATCH 03/11] chore(env.test): add new line --- .env.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.test b/.env.test index ad87b5418..1228238d9 100644 --- a/.env.test +++ b/.env.test @@ -88,4 +88,4 @@ export GROWTHBOOK_CLIENT_KEY="some random key" export KEYCDN_API_KEY="secret" -export REDIS_HOST="redis" \ No newline at end of file +export REDIS_HOST="redis" From 2c3484bbd6ca15b4265b6cb8d1c1d009d1706cbd Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:02:51 +0800 Subject: [PATCH 04/11] chore(env-var): remove redis host frm be --- .aws/deploy/backend-task-definition.prod.json | 1 - .aws/deploy/backend-task-definition.staging.json | 1 - 2 files changed, 2 deletions(-) diff --git a/.aws/deploy/backend-task-definition.prod.json b/.aws/deploy/backend-task-definition.prod.json index 965be6072..4de452228 100644 --- a/.aws/deploy/backend-task-definition.prod.json +++ b/.aws/deploy/backend-task-definition.prod.json @@ -132,7 +132,6 @@ "name": "REDIRECT_URI", "valueFrom": "PROD_REDIRECT_URI" }, - { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "PROD_SESSION_SECRET" diff --git a/.aws/deploy/backend-task-definition.staging.json b/.aws/deploy/backend-task-definition.staging.json index c6c1938d8..2dcaa69a4 100644 --- a/.aws/deploy/backend-task-definition.staging.json +++ b/.aws/deploy/backend-task-definition.staging.json @@ -141,7 +141,6 @@ "name": "REDIRECT_URI", "valueFrom": "STAGING_REDIRECT_URI" }, - { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "STAGING_SESSION_SECRET" From 93547166b828c703e7d7a43c6914d93451231952 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:04:26 +0800 Subject: [PATCH 05/11] chore(monitoring): refactor service & worker --- common/index.ts | 5 - src/monitoring/MonitoringService.ts | 109 +++++++++++++++++ .../{index.ts => MonitoringWorker.ts} | 111 +++--------------- support/index.ts | 19 +-- 4 files changed, 136 insertions(+), 108 deletions(-) create mode 100644 src/monitoring/MonitoringService.ts rename src/monitoring/{index.ts => MonitoringWorker.ts} (65%) diff --git a/common/index.ts b/common/index.ts index d0e7e976c..21f22bbc1 100644 --- a/common/index.ts +++ b/common/index.ts @@ -27,7 +27,6 @@ import { Reviewer, ReviewRequestView, } from "@database/models" -import MonitoringService from "@root/monitoring" import AuditLogsService from "@root/services/admin/AuditLogsService" import RepoManagementService from "@root/services/admin/RepoManagementService" import GitFileCommitService from "@root/services/db/GitFileCommitService" @@ -249,7 +248,3 @@ export const auditLogsService = new AuditLogsService({ sitesService, usersService, }) - -export const monitoringService = new MonitoringService({ - launchesService, -}) diff --git a/src/monitoring/MonitoringService.ts b/src/monitoring/MonitoringService.ts new file mode 100644 index 000000000..3a411a9f2 --- /dev/null +++ b/src/monitoring/MonitoringService.ts @@ -0,0 +1,109 @@ +import autoBind from "auto-bind" +import { Job, Queue, Worker } from "bullmq" +import _ from "lodash" +import { ResultAsync } from "neverthrow" + +import parentLogger from "@logger/logger" +import logger from "@logger/logger" + +import config from "@root/config/config" +import MonitoringError from "@root/errors/MonitoringError" +import convertNeverThrowToPromise from "@root/utils/neverthrow" + +import MonitoringWorker from "./MonitoringWorker" + +const ONE_MINUTE = 60000 +interface MonitoringServiceInterface { + monitoringWorker: MonitoringWorker +} + +export default class MonitoringService { + private readonly monitoringServiceLogger = parentLogger.child({ + module: "monitoringService", + }) + + private readonly REDIS_CONNECTION = { + host: config.get("bullmq.redisHostname"), + port: 6379, + } + + private readonly queue = new Queue("MonitoringQueue", { + connection: { + ...this.REDIS_CONNECTION, + }, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + attempts: 3, + backoff: { + type: "exponential", + delay: ONE_MINUTE, // this operation is not critical, so we can wait a minute + }, + }, + }) + + private readonly worker: Worker + + private readonly monitoringWorker: MonitoringServiceInterface["monitoringWorker"] + + constructor({ monitoringWorker }: MonitoringServiceInterface) { + this.monitoringWorker = monitoringWorker + autoBind(this) + const jobName = "dnsMonitoring" + + const FIVE_MINUTE_CRON = "5 * * * *" + + const jobData = { + name: "monitoring sites", + } + + ResultAsync.fromPromise( + this.queue.add(jobName, jobData, { + repeat: { + pattern: FIVE_MINUTE_CRON, + }, + }), + (e) => e + ) + .map((okRes) => { + this.monitoringServiceLogger.info( + `Monitoring job scheduled at interval ${FIVE_MINUTE_CRON}` + ) + return okRes + }) + .mapErr((errRes) => { + this.monitoringServiceLogger.error(`Failed to schedule job: ${errRes}`) + }) + + this.worker = new Worker( + this.queue.name, + async (job: Job) => { + this.monitoringServiceLogger.info(`Monitoring Worker ${job.id}`) + if (job.name === jobName) { + // The retry's work on a thrown error, so we need to convert the neverthrow to a promise + const res = await convertNeverThrowToPromise( + this.monitoringWorker.driver() + ) + return res + } + throw new MonitoringError("Invalid job name") + }, + { + connection: { + ...this.REDIS_CONNECTION, + }, + lockDuration: ONE_MINUTE, // since this is a relatively expensive operation + } + ) + + this.worker.on("failed", (job: Job | undefined, error: Error) => { + logger.error({ + message: "Monitoring service has failed", + error, + meta: { + ...job?.data, + }, + }) + }) + } +} diff --git a/src/monitoring/index.ts b/src/monitoring/MonitoringWorker.ts similarity index 65% rename from src/monitoring/index.ts rename to src/monitoring/MonitoringWorker.ts index f96e4167d..c228b8a43 100644 --- a/src/monitoring/index.ts +++ b/src/monitoring/MonitoringWorker.ts @@ -2,12 +2,10 @@ import { retry } from "@octokit/plugin-retry" import { Octokit } from "@octokit/rest" import autoBind from "auto-bind" import axios from "axios" -import { Job, Queue, Worker } from "bullmq" import _ from "lodash" -import { errAsync, okAsync, ResultAsync } from "neverthrow" +import { ResultAsync, errAsync, okAsync } from "neverthrow" import parentLogger from "@logger/logger" -import logger from "@logger/logger" import config from "@root/config/config" import MonitoringError from "@root/errors/MonitoringError" @@ -15,13 +13,8 @@ import { gb } from "@root/middleware/featureFlag" import LaunchesService from "@root/services/identity/LaunchesService" import { dnsMonitor } from "@root/utils/dns-utils" import { isMonitoringEnabled } from "@root/utils/growthbook-utils" -import convertNeverThrowToPromise from "@root/utils/neverthrow" import promisifyPapaParse from "@root/utils/papa-parse" -interface MonitoringServiceProps { - launchesService: LaunchesService -} - const IsomerHostedDomainType = { REDIRECTION: "redirection", INDIRECTION: "indirection", @@ -52,93 +45,21 @@ function isKeyCdnResponse(object: unknown): object is KeyCdnZoneAlias[] { if (Array.isArray(object)) return object.every(isKeyCdnZoneAlias) return false } -const ONE_MINUTE = 60000 -export default class MonitoringService { - private readonly launchesService: MonitoringServiceProps["launchesService"] - private readonly monitoringServiceLogger = parentLogger.child({ - module: "monitoringService", - }) - - private readonly REDIS_CONNECTION = { - host: config.get("bullmq.redisHostname"), - port: 6379, - } +interface MonitoringWorkerInterface { + launchesService: LaunchesService +} - private readonly queue = new Queue("MonitoringQueue", { - connection: { - ...this.REDIS_CONNECTION, - }, - defaultJobOptions: { - removeOnComplete: true, - removeOnFail: true, - attempts: 3, - backoff: { - type: "exponential", - delay: ONE_MINUTE, // this operation is not critical, so we can wait a minute - }, - }, +export default class MonitoringWorker { + private readonly monitoringWorkerLogger = parentLogger.child({ + module: "monitoringWorker", }) - private readonly worker: Worker + private readonly launchesService: MonitoringWorkerInterface["launchesService"] - constructor({ launchesService }: MonitoringServiceProps) { - autoBind(this) - const jobName = "dnsMonitoring" + constructor({ launchesService }: MonitoringWorkerInterface) { this.launchesService = launchesService - - const FIVE_MINUTE_CRON = "5 * * * *" - - const jobData = { - name: "monitoring sites", - } - - ResultAsync.fromPromise( - this.queue.add(jobName, jobData, { - repeat: { - pattern: FIVE_MINUTE_CRON, - }, - }), - (e) => e - ) - .map((okRes) => { - this.monitoringServiceLogger.info( - `Monitoring job scheduled at interval ${FIVE_MINUTE_CRON}` - ) - return okRes - }) - .mapErr((errRes) => { - this.monitoringServiceLogger.error(`Failed to schedule job: ${errRes}`) - }) - - this.worker = new Worker( - this.queue.name, - async (job: Job) => { - this.monitoringServiceLogger.info(`Monitoring Worker ${job.id}`) - if (job.name === jobName) { - // The retry's work on a thrown error, so we need to convert the neverthrow to a promise - const res = await convertNeverThrowToPromise(this.driver()) - return res - } - throw new MonitoringError("Invalid job name") - }, - { - connection: { - ...this.REDIS_CONNECTION, - }, - lockDuration: 60000, // 1 minute, since this is a relatively expensive operation - } - ) - - this.worker.on("failed", (job: Job | undefined, error: Error) => { - logger.error({ - message: "Monitoring service has failed", - error, - meta: { - ...job?.data, - }, - }) - }) + autoBind(this) } getKeyCdnDomains() { @@ -224,7 +145,7 @@ export default class MonitoringService { * of any subdomains and redirects. */ getAllDomains() { - this.monitoringServiceLogger.info("Fetching all domains") + this.monitoringWorkerLogger.info("Fetching all domains") return ResultAsync.combine([ this.getAmplifyDeployments().mapErr( (err) => new MonitoringError(err.message) @@ -232,7 +153,7 @@ export default class MonitoringService { this.getRedirectionDomains(), this.getKeyCdnDomains(), ]).andThen(([amplifyDeployments, redirectionDomains, keyCdnDomains]) => { - this.monitoringServiceLogger.info("Fetched all domains") + this.monitoringWorkerLogger.info("Fetched all domains") return okAsync( _.sortBy( [...amplifyDeployments, ...redirectionDomains, ...keyCdnDomains], @@ -252,19 +173,19 @@ export default class MonitoringService { driver() { if (!isMonitoringEnabled(gb)) return okAsync("Monitoring Service disabled") const start = Date.now() - this.monitoringServiceLogger.info("Monitoring service started") + this.monitoringWorkerLogger.info("Monitoring service started") return this.getAllDomains() .andThen(this.generateReportCard) .mapErr((reportCardErr: MonitoringError | string[]) => { if (reportCardErr instanceof MonitoringError) { - this.monitoringServiceLogger.error({ + this.monitoringWorkerLogger.error({ error: reportCardErr, message: "Error running monitoring service", }) return } - this.monitoringServiceLogger.error({ + this.monitoringWorkerLogger.error({ message: "Error running monitoring service", meta: { dnsCheckerResult: reportCardErr, @@ -274,7 +195,7 @@ export default class MonitoringService { }) .orElse(() => okAsync([])) .andThen(() => { - this.monitoringServiceLogger.info( + this.monitoringWorkerLogger.info( `Monitoring service completed in ${Date.now() - start}ms` ) return okAsync("Monitoring service completed") diff --git a/support/index.ts b/support/index.ts index 3c8e454d5..535aeab29 100644 --- a/support/index.ts +++ b/support/index.ts @@ -2,16 +2,12 @@ import "module-alias/register" import express from "express" -import { - infraService, - launchesService, - monitoringService, - sequelize, -} from "@common/index" +import { infraService, launchesService, sequelize } from "@common/index" import { useSharedMiddleware } from "@common/middleware" import { config } from "@root/config/config" import logger from "@root/logger/logger" -import MonitoringService from "@root/monitoring" +import MonitoringService from "@root/monitoring/MonitoringService" +import MonitoringWorker from "@root/monitoring/monitoringWorker" import { ROUTE_VERSION } from "./constants" import { v2Router } from "./routes" @@ -24,7 +20,14 @@ const app = express() // poller site launch updates infraService.pollMessages() -monitoringService.driver() +// only needed for support container +export const monitoringWorker = new MonitoringWorker({ + launchesService, +}) + +export const monitoringService = new MonitoringService({ + monitoringWorker, +}) const ROUTE_PREFIX_ISOBOT = `/${ROUTE_VERSION}/isobot` app.use(ROUTE_PREFIX_ISOBOT, isobotRouter) From 285741d7a1050e6802478f772f884be77a67bfca Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:52:39 +0800 Subject: [PATCH 06/11] chore(env): add back env --- .aws/deploy/backend-task-definition.prod.json | 6 ++ .../backend-task-definition.staging.json | 6 ++ .aws/deploy/support-task-definition.prod.json | 4 + .../support-task-definition.staging.json | 4 + src/config/config.ts | 7 ++ src/constants/constants.ts | 1 + src/monitoring/MonitoringWorker.ts | 23 ++++- src/server.ts | 1 - src/types/featureFlags.ts | 7 +- src/utils/dns-utils.ts | 88 ++++++++++++++++--- src/utils/growthbook-utils.ts | 21 +++-- support/index.ts | 5 +- 12 files changed, 151 insertions(+), 22 deletions(-) diff --git a/.aws/deploy/backend-task-definition.prod.json b/.aws/deploy/backend-task-definition.prod.json index 4de452228..475663f24 100644 --- a/.aws/deploy/backend-task-definition.prod.json +++ b/.aws/deploy/backend-task-definition.prod.json @@ -103,6 +103,7 @@ "valueFrom": "PROD_INCOMING_QUEUE_URL" }, { "name": "JWT_SECRET", "valueFrom": "PROD_JWT_SECRET" }, + { "name": "KEYCDN_API_KEY", "valueFrom": "PROD_KEYCDN_API_KEY" }, { "name": "MAX_NUM_OTP_ATTEMPTS", "valueFrom": "PROD_MAX_NUM_OTP_ATTEMPTS" @@ -132,6 +133,11 @@ "name": "REDIRECT_URI", "valueFrom": "PROD_REDIRECT_URI" }, + { + "name": "REDIRECTION_REPO_GITHUB_TOKEN", + "valueFrom": "PROD_REDIRECTION_REPO_GITHUB_TOKEN" + }, + { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "PROD_SESSION_SECRET" diff --git a/.aws/deploy/backend-task-definition.staging.json b/.aws/deploy/backend-task-definition.staging.json index 2dcaa69a4..5c55e8f4d 100644 --- a/.aws/deploy/backend-task-definition.staging.json +++ b/.aws/deploy/backend-task-definition.staging.json @@ -112,6 +112,7 @@ "valueFrom": "STAGING_INCOMING_QUEUE_URL" }, { "name": "JWT_SECRET", "valueFrom": "STAGING_JWT_SECRET" }, + { "name": "KEYCDN_API_KEY", "valueFrom": "STAGING_KEYCDN_API_KEY" }, { "name": "MAX_NUM_OTP_ATTEMPTS", "valueFrom": "STAGING_MAX_NUM_OTP_ATTEMPTS" @@ -141,6 +142,11 @@ "name": "REDIRECT_URI", "valueFrom": "STAGING_REDIRECT_URI" }, + { + "name": "REDIRECTION_REPO_GITHUB_TOKEN", + "valueFrom": "STAGING_REDIRECTION_REPO_GITHUB_TOKEN" + }, + { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "STAGING_SESSION_SECRET" diff --git a/.aws/deploy/support-task-definition.prod.json b/.aws/deploy/support-task-definition.prod.json index 0c62d5ff6..f5b5b5863 100644 --- a/.aws/deploy/support-task-definition.prod.json +++ b/.aws/deploy/support-task-definition.prod.json @@ -128,6 +128,10 @@ "name": "REDIRECT_URI", "valueFrom": "PROD_REDIRECT_URI" }, + { + "name": "REDIRECTION_REPO_GITHUB_TOKEN", + "valueFrom": "PROD_REDIRECTION_REPO_GITHUB_TOKEN" + }, { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" }, { "name": "SESSION_SECRET", diff --git a/.aws/deploy/support-task-definition.staging.json b/.aws/deploy/support-task-definition.staging.json index dac20c2ef..e8e21b0ab 100644 --- a/.aws/deploy/support-task-definition.staging.json +++ b/.aws/deploy/support-task-definition.staging.json @@ -137,6 +137,10 @@ "name": "REDIRECT_URI", "valueFrom": "STAGING_REDIRECT_URI" }, + { + "name": "REDIRECTION_REPO_GITHUB_TOKEN", + "valueFrom": "STAGING_REDIRECTION_REPO_GITHUB_TOKEN" + }, { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" }, { "name": "SESSION_SECRET", diff --git a/src/config/config.ts b/src/config/config.ts index 5e2cebb69..87438fb99 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -254,6 +254,13 @@ const config = convict({ format: "required-string", default: "", }, + redirectionRepoGithubToken: { + doc: "Github access to read opengovsg/isomer-redirection", + env: "REDIRECTION_REPO_GITHUB_TOKEN", + sensitive: true, + format: "required-string", + default: "", + }, }, dataDog: { env: { diff --git a/src/constants/constants.ts b/src/constants/constants.ts index c3fc8743a..24c184dbc 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -78,6 +78,7 @@ export const REDIRECTION_SERVER_IPS = [ ] export const ALLOWED_DNS_ERROR_CODES = ["ENOTFOUND", "ENODATA"] +export const BUILT_WITH_ISOMER_LOGO = `

Created with Isomer Logo` export const DNS_INDIRECTION_DOMAIN = "hostedon.isomer.gov.sg" export const DNS_KEYCDN_SUFFIX = "kxcdn.com" export const DNS_CNAME_SUFFIXES = ["cloudfront.net", DNS_KEYCDN_SUFFIX] diff --git a/src/monitoring/MonitoringWorker.ts b/src/monitoring/MonitoringWorker.ts index c228b8a43..a3835e046 100644 --- a/src/monitoring/MonitoringWorker.ts +++ b/src/monitoring/MonitoringWorker.ts @@ -12,7 +12,7 @@ import MonitoringError from "@root/errors/MonitoringError" import { gb } from "@root/middleware/featureFlag" import LaunchesService from "@root/services/identity/LaunchesService" import { dnsMonitor } from "@root/utils/dns-utils" -import { isMonitoringEnabled } from "@root/utils/growthbook-utils" +import { getMonitoringConfig } from "@root/utils/growthbook-utils" import promisifyPapaParse from "@root/utils/papa-parse" const IsomerHostedDomainType = { @@ -106,10 +106,12 @@ export default class MonitoringWorker { * @returns List of redirection domains that are listed in the isomer-redirection repository */ getRedirectionDomains() { - const SYSTEM_GITHUB_TOKEN = config.get("github.systemToken") const OctokitRetry = Octokit.plugin(retry) + const REDIRECTION_REPO_GITHUB_TOKEN = config.get( + "github.redirectionRepoGithubToken" + ) const octokitWithRetry: Octokit = new OctokitRetry({ - auth: SYSTEM_GITHUB_TOKEN, + auth: REDIRECTION_REPO_GITHUB_TOKEN, request: { retries: 5 }, }) @@ -171,7 +173,11 @@ export default class MonitoringWorker { } driver() { - if (!isMonitoringEnabled(gb)) return okAsync("Monitoring Service disabled") + const monitoringConfig = getMonitoringConfig(gb) + if (!monitoringConfig.isEnabled) { + this.monitoringWorkerLogger.info("Monitoring Service disabled") + return okAsync("Monitoring Service disabled") + } const start = Date.now() this.monitoringWorkerLogger.info("Monitoring service started") @@ -192,6 +198,15 @@ export default class MonitoringWorker { date: new Date(), }, }) + reportCardErr.map((err) => + this.monitoringWorkerLogger.error({ + message: "Error running monitoring service", + meta: { + dnsCheckerResult: err, + date: new Date(), + }, + }) + ) }) .orElse(() => okAsync([])) .andThen(() => { diff --git a/src/server.ts b/src/server.ts index 58ff93c66..0b17a9324 100644 --- a/src/server.ts +++ b/src/server.ts @@ -81,7 +81,6 @@ import { mailer } from "@services/utilServices/MailClient" import { apiLogger } from "./middleware/apiLogger" import { NotificationOnEditHandler } from "./middleware/notificationOnEditHandler" -import MonitoringService from "./monitoring" import getAuthenticatedSubrouter from "./routes/v2/authenticated" import { ReviewsRouter } from "./routes/v2/authenticated/review" import getAuthenticatedSitesSubrouter from "./routes/v2/authenticatedSites" diff --git a/src/types/featureFlags.ts b/src/types/featureFlags.ts index 69923b119..4130e65fe 100644 --- a/src/types/featureFlags.ts +++ b/src/types/featureFlags.ts @@ -7,13 +7,18 @@ export interface CloudmersiveConfigType { timeout: number } +export interface MonitoringConfig { + isEnabled: boolean + whitelistedRepos: string[] +} + export interface FeatureFlags { is_build_times_reduction_enabled: boolean is_ggs_enabled: boolean is_show_staging_build_status_enabled: boolean is_cloudmersive_enabled: CloudmersiveConfigType is_local_diff_enabled: boolean - is_monitoring_enabled: boolean + is_monitoring_enabled: MonitoringConfig } // List of attributes we set in GrowthBook Instance in auth middleware diff --git a/src/utils/dns-utils.ts b/src/utils/dns-utils.ts index 8752663e4..ebc90974c 100644 --- a/src/utils/dns-utils.ts +++ b/src/utils/dns-utils.ts @@ -1,14 +1,19 @@ import dns from "node:dns/promises" -import { err, ok, okAsync, Result, ResultAsync } from "neverthrow" +import { err, errAsync, ok, okAsync, Result, ResultAsync } from "neverthrow" import { + BUILT_WITH_ISOMER_LOGO, DNS_CNAME_SUFFIXES, DNS_INDIRECTION_DOMAIN, DNS_KEYCDN_SUFFIX, REDIRECTION_SERVER_IPS, } from "@root/constants" +import MonitoringError from "@root/errors/MonitoringError" import logger from "@root/logger/logger" +import { gb } from "@root/middleware/featureFlag" + +import { getMonitoringConfig } from "./growthbook-utils" export function checkCname(domain: string) { return ResultAsync.fromPromise(dns.resolveCname(domain), () => { @@ -23,6 +28,11 @@ export function checkCname(domain: string) { return okAsync(null) } + //! todo: remove, this is some colima weird logic + if (cname.length === 1 && cname[0] === domain) { + return okAsync(null) + } + return okAsync(cname[0]) }) .orElse(() => okAsync(null)) @@ -118,7 +128,7 @@ export default function getDnsCheckerMessage( ) } if (isIntermediateValid && isRedirectionValid) { - return err( + return ok( `The domain ${domain} is *all valid*! Although it is directly pointing to our CDN hosting provider and not the indirection layer. The apex domain \`${redirectionDomain}\` is also correctly configured to our redirection service.` ) } @@ -167,23 +177,36 @@ export default function getDnsCheckerMessage( : "no A records" }` } + return err(responseMeta) } export function dnsMonitor(domain: string): ResultAsync { + if (!domain) { + return okAsync("Empty domain provided") + } return checkCname(domain) .andThen((cname) => { + if (cname) { + return ResultAsync.combine([okAsync(domain), okAsync(cname)]) + } // Original domain does not have a CNAME record, check if the www // version has a valid CNAME record if (!cname && !domain.startsWith("www.")) { const cnameDomain = `www.${domain}` - return ResultAsync.combine([ - okAsync(cnameDomain), - checkCname(cnameDomain), - ]) + return checkCname(cnameDomain).andThen((cnameDomainRes) => { + if (cnameDomainRes) { + return ResultAsync.combine([ + okAsync(cnameDomain), + okAsync(cnameDomainRes), + ]) + } + // this might be a redirection only domain + return ResultAsync.combine([okAsync(domain), okAsync(null)]) + }) } - - return ResultAsync.combine([okAsync(domain), okAsync(cname)]) + // might be a redirection only domain + return ResultAsync.combine([okAsync(domain), okAsync(null)]) }) .andThen(([cnameDomain, cnameRecord]) => { // Original and www version of the domain do not have a CNAME record, @@ -245,8 +268,8 @@ export function dnsMonitor(domain: string): ResultAsync { redirectionDomain, }, redirection, - ]) => - getDnsCheckerMessage( + ]) => { + const dnsCheckerMessage = getDnsCheckerMessage( domain, cnameDomain, redirectionDomain, @@ -255,5 +278,50 @@ export function dnsMonitor(domain: string): ResultAsync { indirectionRecords, redirection ) + + const domainCheckerRes = dnsCheckerMessage.isOk() + ? dnsCheckerMessage.value + : dnsCheckerMessage.error + + return ResultAsync.fromPromise( + // using fetch as it has a convenient redirect option + + fetch(`https://${domain}`), + () => new MonitoringError(`Failed to fetch site: ${domain}`) + ) + .andThen((res) => + ResultAsync.fromPromise( + res.text(), + () => new MonitoringError(`Failed to get text from response`) + ) + ) + .andThen((res) => { + const isStillHostingIsomerSite = res.includes( + BUILT_WITH_ISOMER_LOGO + ) + + if (isStillHostingIsomerSite) { + return okAsync( + `${domainCheckerRes}\nThe site is still being hosted with Isomer logo, and is likely still viewable by MOP` + ) + } + + const monitoringConfig = getMonitoringConfig(gb) + + const isWeirdDomain = monitoringConfig.whitelistedRepos.includes( + domain + ) + if (isWeirdDomain) { + return okAsync( + `${domainCheckerRes}\nThe site is not being hosted with Isomer logo, and is known to be an edge case.` + ) + } + return errAsync(false) + }) + .mapErr( + () => + `${domainCheckerRes}\nThe site is **NOT** being hosted with Isomer logo, and is likely **NOT** viewable by MOP` + ) + } ) } diff --git a/src/utils/growthbook-utils.ts b/src/utils/growthbook-utils.ts index bf55490d4..4b6f3c484 100644 --- a/src/utils/growthbook-utils.ts +++ b/src/utils/growthbook-utils.ts @@ -2,7 +2,11 @@ import { GrowthBook, setPolyfills } from "@growthbook/growthbook" import { config } from "@root/config/config" import { FEATURE_FLAGS } from "@root/constants/featureFlags" -import { FeatureFlags, CloudmersiveConfigType } from "@root/types/featureFlags" +import { + FeatureFlags, + CloudmersiveConfigType, + MonitoringConfig, +} from "@root/types/featureFlags" const GROWTHBOOK_API_HOST = "https://cdn.growthbook.io" @@ -68,9 +72,16 @@ export const isCloudmersiveEnabled = ( ) } -export const isMonitoringEnabled = ( +export const getMonitoringConfig = ( growthbook: GrowthBook | undefined -): boolean => { - if (!growthbook) return true - return growthbook.getFeatureValue(FEATURE_FLAGS.IS_MONITORING_ENABLED, true) +): MonitoringConfig => { + const defaultConfig = { + isEnabled: true, + whitelistedRepos: [], + } + if (!growthbook) return defaultConfig + return growthbook.getFeatureValue( + FEATURE_FLAGS.IS_MONITORING_ENABLED, + defaultConfig + ) } diff --git a/support/index.ts b/support/index.ts index 535aeab29..6568d5fa7 100644 --- a/support/index.ts +++ b/support/index.ts @@ -7,7 +7,7 @@ import { useSharedMiddleware } from "@common/middleware" import { config } from "@root/config/config" import logger from "@root/logger/logger" import MonitoringService from "@root/monitoring/MonitoringService" -import MonitoringWorker from "@root/monitoring/monitoringWorker" +import MonitoringWorker from "@root/monitoring/MonitoringWorker" import { ROUTE_VERSION } from "./constants" import { v2Router } from "./routes" @@ -24,6 +24,9 @@ infraService.pollMessages() export const monitoringWorker = new MonitoringWorker({ launchesService, }) +// dnsMonitor("isomer.gov.sg").mapErr(console.log).map(console.log) +// todo: remove after testing +monitoringWorker.driver() export const monitoringService = new MonitoringService({ monitoringWorker, From f9a25e627b2c3d461a95059c688300c844960b91 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Fri, 21 Jun 2024 10:35:37 +0800 Subject: [PATCH 07/11] fix(bullmq): add error handler make env var optional, add scoped gb instance --- .aws/deploy/backend-task-definition.prod.json | 6 ------ .../deploy/backend-task-definition.staging.json | 6 ------ src/config/config.ts | 6 +++--- src/monitoring/MonitoringService.ts | 16 ++++++++++++++-- src/monitoring/MonitoringWorker.ts | 10 ++++++++-- src/utils/dns-utils.ts | 17 +++++++++-------- support/index.ts | 3 --- 7 files changed, 34 insertions(+), 30 deletions(-) diff --git a/.aws/deploy/backend-task-definition.prod.json b/.aws/deploy/backend-task-definition.prod.json index 475663f24..4de452228 100644 --- a/.aws/deploy/backend-task-definition.prod.json +++ b/.aws/deploy/backend-task-definition.prod.json @@ -103,7 +103,6 @@ "valueFrom": "PROD_INCOMING_QUEUE_URL" }, { "name": "JWT_SECRET", "valueFrom": "PROD_JWT_SECRET" }, - { "name": "KEYCDN_API_KEY", "valueFrom": "PROD_KEYCDN_API_KEY" }, { "name": "MAX_NUM_OTP_ATTEMPTS", "valueFrom": "PROD_MAX_NUM_OTP_ATTEMPTS" @@ -133,11 +132,6 @@ "name": "REDIRECT_URI", "valueFrom": "PROD_REDIRECT_URI" }, - { - "name": "REDIRECTION_REPO_GITHUB_TOKEN", - "valueFrom": "PROD_REDIRECTION_REPO_GITHUB_TOKEN" - }, - { "name": "REDIS_HOST", "valueFrom": "PROD_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "PROD_SESSION_SECRET" diff --git a/.aws/deploy/backend-task-definition.staging.json b/.aws/deploy/backend-task-definition.staging.json index 5c55e8f4d..2dcaa69a4 100644 --- a/.aws/deploy/backend-task-definition.staging.json +++ b/.aws/deploy/backend-task-definition.staging.json @@ -112,7 +112,6 @@ "valueFrom": "STAGING_INCOMING_QUEUE_URL" }, { "name": "JWT_SECRET", "valueFrom": "STAGING_JWT_SECRET" }, - { "name": "KEYCDN_API_KEY", "valueFrom": "STAGING_KEYCDN_API_KEY" }, { "name": "MAX_NUM_OTP_ATTEMPTS", "valueFrom": "STAGING_MAX_NUM_OTP_ATTEMPTS" @@ -142,11 +141,6 @@ "name": "REDIRECT_URI", "valueFrom": "STAGING_REDIRECT_URI" }, - { - "name": "REDIRECTION_REPO_GITHUB_TOKEN", - "valueFrom": "STAGING_REDIRECTION_REPO_GITHUB_TOKEN" - }, - { "name": "REDIS_HOST", "valueFrom": "STAGING_REDIS_HOST" }, { "name": "SESSION_SECRET", "valueFrom": "STAGING_SESSION_SECRET" diff --git a/src/config/config.ts b/src/config/config.ts index 87438fb99..ee6ace999 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -199,7 +199,7 @@ const config = convict({ redisHostname: { doc: "Redis host name for bullmq", env: "REDIS_HOST", - format: "required-string", + format: String, default: "isomerpages", }, }, @@ -258,7 +258,7 @@ const config = convict({ doc: "Github access to read opengovsg/isomer-redirection", env: "REDIRECTION_REPO_GITHUB_TOKEN", sensitive: true, - format: "required-string", + format: String, default: "", }, }, @@ -491,7 +491,7 @@ const config = convict({ apiKey: { doc: "KeyCDN API key", env: "KEYCDN_API_KEY", - format: "required-string", + format: String, default: "", }, }, diff --git a/src/monitoring/MonitoringService.ts b/src/monitoring/MonitoringService.ts index 3a411a9f2..8aa84fada 100644 --- a/src/monitoring/MonitoringService.ts +++ b/src/monitoring/MonitoringService.ts @@ -42,7 +42,13 @@ export default class MonitoringService { }, }) - private readonly worker: Worker + private readonly worker: Worker< + { + name: string + }, + string, + string + > private readonly monitoringWorker: MonitoringServiceInterface["monitoringWorker"] @@ -96,9 +102,15 @@ export default class MonitoringService { } ) + this.worker.on("error", (error: Error) => { + logger.error({ + message: "Monitoring service has errored", + error, + }) + }) this.worker.on("failed", (job: Job | undefined, error: Error) => { logger.error({ - message: "Monitoring service has failed", + message: "Monitoring service has errored", error, meta: { ...job?.data, diff --git a/src/monitoring/MonitoringWorker.ts b/src/monitoring/MonitoringWorker.ts index a3835e046..40b712b49 100644 --- a/src/monitoring/MonitoringWorker.ts +++ b/src/monitoring/MonitoringWorker.ts @@ -9,10 +9,12 @@ import parentLogger from "@logger/logger" import config from "@root/config/config" import MonitoringError from "@root/errors/MonitoringError" -import { gb } from "@root/middleware/featureFlag" import LaunchesService from "@root/services/identity/LaunchesService" import { dnsMonitor } from "@root/utils/dns-utils" -import { getMonitoringConfig } from "@root/utils/growthbook-utils" +import { + getMonitoringConfig, + getNewGrowthbookInstance, +} from "@root/utils/growthbook-utils" import promisifyPapaParse from "@root/utils/papa-parse" const IsomerHostedDomainType = { @@ -173,6 +175,10 @@ export default class MonitoringWorker { } driver() { + const gb = getNewGrowthbookInstance({ + clientKey: config.get("growthbook.clientKey"), + subscribeToChanges: true, + }) const monitoringConfig = getMonitoringConfig(gb) if (!monitoringConfig.isEnabled) { this.monitoringWorkerLogger.info("Monitoring Service disabled") diff --git a/src/utils/dns-utils.ts b/src/utils/dns-utils.ts index ebc90974c..941899286 100644 --- a/src/utils/dns-utils.ts +++ b/src/utils/dns-utils.ts @@ -2,6 +2,7 @@ import dns from "node:dns/promises" import { err, errAsync, ok, okAsync, Result, ResultAsync } from "neverthrow" +import config from "@root/config/config" import { BUILT_WITH_ISOMER_LOGO, DNS_CNAME_SUFFIXES, @@ -11,9 +12,11 @@ import { } from "@root/constants" import MonitoringError from "@root/errors/MonitoringError" import logger from "@root/logger/logger" -import { gb } from "@root/middleware/featureFlag" -import { getMonitoringConfig } from "./growthbook-utils" +import { + getMonitoringConfig, + getNewGrowthbookInstance, +} from "./growthbook-utils" export function checkCname(domain: string) { return ResultAsync.fromPromise(dns.resolveCname(domain), () => { @@ -28,11 +31,6 @@ export function checkCname(domain: string) { return okAsync(null) } - //! todo: remove, this is some colima weird logic - if (cname.length === 1 && cname[0] === domain) { - return okAsync(null) - } - return okAsync(cname[0]) }) .orElse(() => okAsync(null)) @@ -305,7 +303,10 @@ export function dnsMonitor(domain: string): ResultAsync { `${domainCheckerRes}\nThe site is still being hosted with Isomer logo, and is likely still viewable by MOP` ) } - + const gb = getNewGrowthbookInstance({ + clientKey: config.get("growthbook.clientKey"), + subscribeToChanges: true, + }) const monitoringConfig = getMonitoringConfig(gb) const isWeirdDomain = monitoringConfig.whitelistedRepos.includes( diff --git a/support/index.ts b/support/index.ts index 6568d5fa7..393bd9ccc 100644 --- a/support/index.ts +++ b/support/index.ts @@ -24,9 +24,6 @@ infraService.pollMessages() export const monitoringWorker = new MonitoringWorker({ launchesService, }) -// dnsMonitor("isomer.gov.sg").mapErr(console.log).map(console.log) -// todo: remove after testing -monitoringWorker.driver() export const monitoringService = new MonitoringService({ monitoringWorker, From f915b04a7927627d8a0e16c6f61d7fb2f964ad75 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Mon, 24 Jun 2024 23:03:12 +0800 Subject: [PATCH 08/11] fix(octokit): fix type mismatch --- src/monitoring/MonitoringWorker.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/monitoring/MonitoringWorker.ts b/src/monitoring/MonitoringWorker.ts index 40b712b49..998bec64b 100644 --- a/src/monitoring/MonitoringWorker.ts +++ b/src/monitoring/MonitoringWorker.ts @@ -108,10 +108,11 @@ export default class MonitoringWorker { * @returns List of redirection domains that are listed in the isomer-redirection repository */ getRedirectionDomains() { - const OctokitRetry = Octokit.plugin(retry) const REDIRECTION_REPO_GITHUB_TOKEN = config.get( "github.redirectionRepoGithubToken" ) + + const OctokitRetry = Octokit.plugin(retry) const octokitWithRetry: Octokit = new OctokitRetry({ auth: REDIRECTION_REPO_GITHUB_TOKEN, request: { retries: 5 }, @@ -177,7 +178,7 @@ export default class MonitoringWorker { driver() { const gb = getNewGrowthbookInstance({ clientKey: config.get("growthbook.clientKey"), - subscribeToChanges: true, + subscribeToChanges: false, }) const monitoringConfig = getMonitoringConfig(gb) if (!monitoringConfig.isEnabled) { From 5a1276f27e9de29c9c7ef40428f8a0362b975811 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Mon, 24 Jun 2024 23:09:55 +0800 Subject: [PATCH 09/11] chore(featureFlag): rm export --- src/middleware/featureFlag.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middleware/featureFlag.ts b/src/middleware/featureFlag.ts index 2b9208ca2..bc1223e99 100644 --- a/src/middleware/featureFlag.ts +++ b/src/middleware/featureFlag.ts @@ -6,7 +6,7 @@ import { getNewGrowthbookInstance } from "@root/utils/growthbook-utils" // Keep one GrowthBook instance at module level // The instance will handle internal cache refreshes via a SSE connection -export const gb = getNewGrowthbookInstance({ +const gb = getNewGrowthbookInstance({ clientKey: config.get("growthbook.clientKey"), subscribeToChanges: true, }) From fd00e27c59411198f26362b4996a1fc1e82b86b1 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Tue, 25 Jun 2024 13:57:44 +0800 Subject: [PATCH 10/11] chore(monitoring): remove on retry --- src/monitoring/MonitoringService.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/monitoring/MonitoringService.ts b/src/monitoring/MonitoringService.ts index 8aa84fada..83c5a4767 100644 --- a/src/monitoring/MonitoringService.ts +++ b/src/monitoring/MonitoringService.ts @@ -34,11 +34,6 @@ export default class MonitoringService { defaultJobOptions: { removeOnComplete: true, removeOnFail: true, - attempts: 3, - backoff: { - type: "exponential", - delay: ONE_MINUTE, // this operation is not critical, so we can wait a minute - }, }, }) From 32ffc4920413f93ff6baef74979ab96145499400 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:03:07 +0800 Subject: [PATCH 11/11] fix(cron): run monitoring every 5 mins --- src/constants/constants.ts | 2 +- src/monitoring/MonitoringService.ts | 2 +- src/utils/dns-utils.ts | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/constants/constants.ts b/src/constants/constants.ts index 24c184dbc..c3307fc42 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -78,7 +78,7 @@ export const REDIRECTION_SERVER_IPS = [ ] export const ALLOWED_DNS_ERROR_CODES = ["ENOTFOUND", "ENODATA"] -export const BUILT_WITH_ISOMER_LOGO = `

Created with Isomer Logo` +export const BUILT_WITH_ISOMER_LOGO = `

{ - logger.error({ + logger.info({ message: "Error resolving CNAME", meta: { domain, method: "checkCname" }, }) @@ -38,7 +38,7 @@ export function checkCname(domain: string) { export function checkA(domain: string) { return ResultAsync.fromPromise(dns.resolve4(domain), () => { - logger.error({ + logger.info({ message: "Error resolving A record", meta: { domain, method: "checkA" }, }) @@ -180,6 +180,7 @@ export default function getDnsCheckerMessage( } export function dnsMonitor(domain: string): ResultAsync { + console.log("dnsMonitor", domain) if (!domain) { return okAsync("Empty domain provided") }