From 1d5e0252fa2dbf5e6624bbad6283c479bfcc96de Mon Sep 17 00:00:00 2001 From: caando Date: Fri, 21 Jul 2023 03:59:19 +0800 Subject: [PATCH 1/7] feat(preview): added preview images --- package-lock.json | 75 +++++++++++++++++++++++++ package.json | 1 + src/errors/PreviewParsingError.ts | 10 ++++ src/integration/Sites.spec.ts | 4 ++ src/routes/v2/authenticated/sites.ts | 14 +++++ src/server.js | 4 +- src/services/identity/PreviewService.ts | 31 ++++++++++ src/services/identity/SitesService.ts | 32 ++++++++++- src/types/previewInfo.ts | 3 + 9 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 src/errors/PreviewParsingError.ts create mode 100644 src/services/identity/PreviewService.ts create mode 100644 src/types/previewInfo.ts diff --git a/package-lock.json b/package-lock.json index 27ae377c4..6c51f6bba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -87,6 +87,7 @@ "@types/express": "^4.17.13", "@types/express-session": "^1.17.5", "@types/jest": "^27.4.1", + "@types/jsdom": "^21.1.1", "@types/lodash": "^4.14.186", "@types/node": "^17.0.21", "@types/supertest": "^2.0.11", @@ -4523,6 +4524,29 @@ "pretty-format": "^27.0.0" } }, + "node_modules/@types/jsdom": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.1.tgz", + "integrity": "sha512-cZFuoVLtzKP3gmq9eNosUL1R50U+USkbLtUQ1bYVgl/lKp0FZM7Cq4aIHAL8oIvQ17uSHi7jXPtfDOdjPwBE7A==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^7.0.0" + } + }, + "node_modules/@types/jsdom/node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dev": true, + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/@types/json-schema": { "version": "7.0.12", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.12.tgz", @@ -4665,6 +4689,12 @@ "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", "integrity": "sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==" }, + "node_modules/@types/tough-cookie": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", + "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==", + "dev": true + }, "node_modules/@types/trusted-types": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.3.tgz", @@ -7790,6 +7820,17 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -10679,6 +10720,40 @@ "node": ">=12" } }, + "node_modules/isomorphic-dompurify/node_modules/dompurify": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.4.1.tgz", + "integrity": "sha512-ewwFzHzrrneRjxzmK6oVz/rZn9VWspGFRDb4/rRtIsM1n36t9AKma/ye8syCpcw+XJ25kOK/hOG7t1j2I2yBqA==" + }, + "node_modules/isomorphic-dompurify/node_modules/escodegen": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", + "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2", + "optionator": "^0.8.1" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/isomorphic-dompurify/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/isomorphic-dompurify/node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", diff --git a/package.json b/package.json index 5e7a347ff..8a8c4c2b0 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ "@types/express": "^4.17.13", "@types/express-session": "^1.17.5", "@types/jest": "^27.4.1", + "@types/jsdom": "^21.1.1", "@types/lodash": "^4.14.186", "@types/node": "^17.0.21", "@types/supertest": "^2.0.11", diff --git a/src/errors/PreviewParsingError.ts b/src/errors/PreviewParsingError.ts new file mode 100644 index 000000000..9f52c6d4f --- /dev/null +++ b/src/errors/PreviewParsingError.ts @@ -0,0 +1,10 @@ +import { BaseIsomerError } from "@errors/BaseError" + +export default class FaviconParsingError extends BaseIsomerError { + url: string + + constructor(url: string, message = "Unable parse favicon from document") { + super({ status: 422, code: "FaviconParsingError", message }) + this.url = url + } +} diff --git a/src/integration/Sites.spec.ts b/src/integration/Sites.spec.ts index a3a6b7ada..d0ac87540 100644 --- a/src/integration/Sites.spec.ts +++ b/src/integration/Sites.spec.ts @@ -42,6 +42,7 @@ import DeploymentsService from "@root/services/identity/DeploymentsService" import IsomerAdminsService from "@root/services/identity/IsomerAdminsService" import LaunchClient from "@root/services/identity/LaunchClient" import LaunchesService from "@root/services/identity/LaunchesService" +import PreviewService from "@root/services/identity/PreviewService" import QueueService from "@root/services/identity/QueueService" import ReposService from "@root/services/identity/ReposService" import { SitesCacheService } from "@root/services/identity/SitesCacheService" @@ -117,6 +118,8 @@ const reviewRequestService = new ReviewRequestService( const MockSitesCacheService = { getLastUpdated: jest.fn(), } + +const MockPreviewService = {} const sitesService = new SitesService({ siteRepository: Site, gitHubService, @@ -125,6 +128,7 @@ const sitesService = new SitesService({ isomerAdminsService, reviewRequestService, sitesCacheService: (MockSitesCacheService as unknown) as SitesCacheService, + previewService: (MockPreviewService as unknown) as PreviewService, }) const collaboratorsService = new CollaboratorsService({ siteRepository: Site, diff --git a/src/routes/v2/authenticated/sites.ts b/src/routes/v2/authenticated/sites.ts index f962f91f3..2164e9bd9 100644 --- a/src/routes/v2/authenticated/sites.ts +++ b/src/routes/v2/authenticated/sites.ts @@ -15,6 +15,7 @@ import InfraService from "@root/services/infra/InfraService" import type { RequestHandler } from "@root/types" import { ResponseErrorBody } from "@root/types/dto/error" import { ProdPermalink, StagingPermalink } from "@root/types/pages" +import { PreviewInfo } from "@root/types/previewInfo" import { RepositoryData } from "@root/types/repoInfo" import { SiteInfo, SiteLaunchDto } from "@root/types/siteInfo" import type SitesService from "@services/identity/SitesService" @@ -158,6 +159,17 @@ export class SitesRouter { .mapErr(({ message }) => res.status(400).json({ message })) } + getPreviewInfo: RequestHandler< + { siteName: string }, + PreviewInfo[] | ResponseErrorBody, + { sites: string[]; email: string }, + never, + { userSessionData: UserSessionData } + > = async (req, res) => + this.sitesService + .getSitesPreview(req.body.sites, res.locals.userSessionData) + .then((previews) => res.status(200).json(previews)) + getRouter() { const router = express.Router({ mergeParams: true }) @@ -199,6 +211,8 @@ export class SitesRouter { this.authorizationMiddleware.verifySiteMember, attachReadRouteHandlerWrapper(this.launchSite) ) + router.post("/preview", attachReadRouteHandlerWrapper(this.getPreviewInfo)) + return router } } diff --git a/src/server.js b/src/server.js index 46c72d92f..b151ea5f2 100644 --- a/src/server.js +++ b/src/server.js @@ -54,6 +54,7 @@ import { notificationsService, } from "@services/identity" import DeploymentsService from "@services/identity/DeploymentsService" +import PreviewService from "@services/identity/PreviewService" import QueueService from "@services/identity/QueueService" import ReposService from "@services/identity/ReposService" import { SitesCacheService } from "@services/identity/SitesCacheService" @@ -194,7 +195,7 @@ const reviewRequestService = new ReviewRequestService( ) const cacheRefreshInterval = 1000 * 60 * 5 // 5 minutes const sitesCacheService = new SitesCacheService(cacheRefreshInterval) - +const previewService = new PreviewService() const sitesService = new SitesService({ siteRepository: Site, gitHubService, @@ -203,6 +204,7 @@ const sitesService = new SitesService({ isomerAdminsService, reviewRequestService, sitesCacheService, + previewService, }) const reposService = new ReposService({ repository: Repo }) const deploymentsService = new DeploymentsService({ diff --git a/src/services/identity/PreviewService.ts b/src/services/identity/PreviewService.ts new file mode 100644 index 000000000..364ea6486 --- /dev/null +++ b/src/services/identity/PreviewService.ts @@ -0,0 +1,31 @@ +// import { getLinkPreview, getPreviewFromContent } from "link-preview-js"; +import axios from "axios" +import { JSDOM } from "jsdom" +import { ResultAsync, okAsync, errAsync } from "neverthrow" + +import PreviewParsingError from "@errors/PreviewParsingError" + +import { PreviewInfo } from "@root/types/previewInfo" + +export default class PreviewService { + getImageUrl = (siteUrl: string): ResultAsync => + ResultAsync.fromPromise( + axios.get(siteUrl), + () => new PreviewParsingError(siteUrl) + ).andThen((documentResponse) => { + const { window } = new JSDOM(documentResponse.data) + const faviconLink = window.document + .querySelector('[rel="shortcut icon"]') + ?.getAttribute("href") + if (faviconLink) { + return okAsync(siteUrl.concat(faviconLink)) + } + return errAsync(new PreviewParsingError(siteUrl)) + }) + + getPreviewInfo = (siteUrl: string): Promise => + this.getImageUrl(siteUrl).match( + (imageUrl) => ({ imageUrl }), + (_error) => ({ imageUrl: undefined }) + ) +} diff --git a/src/services/identity/SitesService.ts b/src/services/identity/SitesService.ts index 59fb87e07..65086b108 100644 --- a/src/services/identity/SitesService.ts +++ b/src/services/identity/SitesService.ts @@ -2,15 +2,18 @@ import _ from "lodash" import { errAsync, okAsync, ResultAsync } from "neverthrow" import { ModelStatic } from "sequelize" +import PreviewParsingError from "@errors/PreviewParsingError" + import { Deployment, Repo, Site } from "@database/models" import type UserSessionData from "@root/classes/UserSessionData" -import type UserWithSiteSessionData from "@root/classes/UserWithSiteSessionData" +import UserWithSiteSessionData from "@root/classes/UserWithSiteSessionData" import DatabaseError from "@root/errors/DatabaseError" import MissingSiteError from "@root/errors/MissingSiteError" import MissingUserEmailError from "@root/errors/MissingUserEmailError" import MissingUserError from "@root/errors/MissingUserError" import { NotFoundError } from "@root/errors/NotFoundError" import { UnprocessableError } from "@root/errors/UnprocessableError" +import PreviewService from "@root/services/identity/PreviewService" import { getAllRepoData, SitesCacheService, @@ -18,6 +21,7 @@ import { import { GitHubCommitData } from "@root/types/commitData" import { ConfigYmlData } from "@root/types/configYml" import { ProdPermalink, StagingPermalink } from "@root/types/pages" +import { PreviewInfo } from "@root/types/previewInfo" import type { RepositoryData, SiteUrls } from "@root/types/repoInfo" import { SiteInfo } from "@root/types/siteInfo" import { Brand } from "@root/types/util" @@ -36,6 +40,7 @@ interface SitesServiceProps { isomerAdminsService: IsomerAdminsService reviewRequestService: ReviewRequestService sitesCacheService: SitesCacheService + previewService: PreviewService } class SitesService { @@ -55,6 +60,8 @@ class SitesService { private readonly sitesCacheService: SitesServiceProps["sitesCacheService"] + private readonly previewService: SitesServiceProps["previewService"] + constructor({ siteRepository, gitHubService, @@ -63,6 +70,7 @@ class SitesService { isomerAdminsService, reviewRequestService, sitesCacheService, + previewService, }: SitesServiceProps) { this.siteRepository = siteRepository this.gitHubService = gitHubService @@ -71,6 +79,7 @@ class SitesService { this.isomerAdminsService = isomerAdminsService this.reviewRequestService = reviewRequestService this.sitesCacheService = sitesCacheService + this.previewService = previewService } isGitHubCommitData(commit: unknown): commit is GitHubCommitData { @@ -444,6 +453,27 @@ class SitesService { stagingUrl: Brand.fromString(stagingUrl), })) } + + async getSitesPreview( + siteNames: string[], + userSessionData: UserSessionData + ): Promise { + return Promise.all( + siteNames.map(async (siteName) => { + const urls = await this.getUrlsOfSite( + new UserWithSiteSessionData({ siteName, ...userSessionData }) + ) + if (urls.isOk()) { + if (urls.value.prod) { + return await this.previewService.getPreviewInfo(urls.value.prod) + } + } + return { + imageUrl: undefined, + } + }) + ) + } } export default SitesService diff --git a/src/types/previewInfo.ts b/src/types/previewInfo.ts new file mode 100644 index 000000000..3008e01aa --- /dev/null +++ b/src/types/previewInfo.ts @@ -0,0 +1,3 @@ +export type PreviewInfo = { + imageUrl?: string +} From 3c3002d0a986a1f654aa10ed123a73fc0b1431e6 Mon Sep 17 00:00:00 2001 From: caando Date: Fri, 21 Jul 2023 04:23:16 +0800 Subject: [PATCH 2/7] fix(deps): fix package-lock.json --- package-lock.json | 89 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6c51f6bba..90d48980e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4684,17 +4684,17 @@ "@types/superagent": "*" } }, - "node_modules/@types/triple-beam": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", - "integrity": "sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==" - }, "node_modules/@types/tough-cookie": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.2.tgz", "integrity": "sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==", "dev": true }, + "node_modules/@types/triple-beam": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", + "integrity": "sha512-txGIh+0eDFzKGC25zORnswy+br1Ha7hj5cMVwKIU7+s0U2AxxJru/jZSMU6OC9MJWP6+pc/hc6ZjyZShpsyY2g==" + }, "node_modules/@types/trusted-types": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.3.tgz", @@ -7443,8 +7443,7 @@ "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" }, "node_modules/deepmerge": { "version": "4.3.1", @@ -7672,11 +7671,6 @@ "node": ">=8" } }, - "node_modules/dompurify": { - "version": "2.4.7", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-2.4.7.tgz", - "integrity": "sha512-kxxKlPEDa6Nc5WJi+qRgPbOAbgTpSULL+vI3NUXsZMlkJxTqYI9wg5ZTay2sFrdZRWHPWNi+EdAhcJf81WtoMQ==" - }, "node_modules/dotenv": { "version": "16.3.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", @@ -7820,17 +7814,6 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -8924,8 +8907,7 @@ "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, "node_modules/fast-safe-stringify": { "version": "2.1.1", @@ -10846,11 +10828,39 @@ } } }, + "node_modules/isomorphic-dompurify/node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/isomorphic-dompurify/node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/isomorphic-dompurify/node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/isomorphic-dompurify/node_modules/parse5": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", @@ -10862,6 +10872,14 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, + "node_modules/isomorphic-dompurify/node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/isomorphic-dompurify/node_modules/saxes": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", @@ -10873,6 +10891,15 @@ "node": ">=v12.22.7" } }, + "node_modules/isomorphic-dompurify/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/isomorphic-dompurify/node_modules/tr46": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", @@ -10884,6 +10911,17 @@ "node": ">=12" } }, + "node_modules/isomorphic-dompurify/node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/isomorphic-dompurify/node_modules/w3c-xmlserializer": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", @@ -18646,7 +18684,6 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", - "dev": true, "engines": { "node": ">=0.10.0" } From 77bcc52e7faf09914708b764aef33702298aada7 Mon Sep 17 00:00:00 2001 From: caando Date: Fri, 21 Jul 2023 09:28:54 +0800 Subject: [PATCH 3/7] fix(sanitizer): build error --- src/services/utilServices/Sanitizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/utilServices/Sanitizer.ts b/src/services/utilServices/Sanitizer.ts index 741306547..b83d7e64f 100644 --- a/src/services/utilServices/Sanitizer.ts +++ b/src/services/utilServices/Sanitizer.ts @@ -25,7 +25,7 @@ DOMPurify.addHook("uponSanitizeElement", (node, data) => { // Adapted from https://github.com/cure53/DOMPurify/blob/e0970d88053c1c564b6ccd633b4af7e7d9a10375/src/purify.js#L719-L736 DOMPurify.removed.push({ element: node }) try { - node.parentNode.removeChild(node) + node.parentNode?.removeChild(node) } catch (e) { try { // eslint-disable-next-line no-param-reassign From ab3075d04fd0797182e8ed9a8930acaf5f95611d Mon Sep 17 00:00:00 2001 From: caando Date: Fri, 21 Jul 2023 10:02:18 +0800 Subject: [PATCH 4/7] fix(test): fix tests with PreviewService --- src/integration/NotificationOnEditHandler.spec.ts | 3 +++ src/integration/Notifications.spec.ts | 3 +++ src/integration/Privatisation.spec.ts | 3 +++ src/integration/Reviews.spec.ts | 3 +++ src/services/identity/__tests__/SitesService.spec.ts | 3 +++ src/services/identity/__tests__/UsersService.spec.ts | 2 +- src/services/utilServices/Sanitizer.ts | 5 ++++- 7 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/integration/NotificationOnEditHandler.spec.ts b/src/integration/NotificationOnEditHandler.spec.ts index 4f2c7de53..b84a2b07a 100644 --- a/src/integration/NotificationOnEditHandler.spec.ts +++ b/src/integration/NotificationOnEditHandler.spec.ts @@ -41,6 +41,7 @@ import { UnlinkedPageService } from "@root/services/fileServices/MdPageServices/ import { CollectionYmlService } from "@root/services/fileServices/YmlFileServices/CollectionYmlService" import { ConfigService } from "@root/services/fileServices/YmlFileServices/ConfigService" import { FooterYmlService } from "@root/services/fileServices/YmlFileServices/FooterYmlService" +import PreviewService from "@root/services/identity/PreviewService" import { SitesCacheService } from "@root/services/identity/SitesCacheService" import { GitHubService } from "@services/db/GitHubService" import * as ReviewApi from "@services/db/review" @@ -125,6 +126,7 @@ const reviewRequestService = new ReviewRequestService( const MockSitesCacheService = { getLastUpdated: jest.fn(), } +const MockPreviewService = {} const sitesService = new SitesService({ siteRepository: Site, gitHubService: (mockGithubService as unknown) as GitHubService, @@ -133,6 +135,7 @@ const sitesService = new SitesService({ isomerAdminsService: (jest.fn() as unknown) as IsomerAdminsService, reviewRequestService, sitesCacheService: (MockSitesCacheService as unknown) as SitesCacheService, + previewService: (MockPreviewService as unknown) as PreviewService, }) const collaboratorsService = new CollaboratorsService({ siteRepository: Site, diff --git a/src/integration/Notifications.spec.ts b/src/integration/Notifications.spec.ts index a6f563b2f..ece332ae8 100644 --- a/src/integration/Notifications.spec.ts +++ b/src/integration/Notifications.spec.ts @@ -47,6 +47,7 @@ import { ConfigService } from "@root/services/fileServices/YmlFileServices/Confi import { ConfigYmlService } from "@root/services/fileServices/YmlFileServices/ConfigYmlService" import { FooterYmlService } from "@root/services/fileServices/YmlFileServices/FooterYmlService" import CollaboratorsService from "@root/services/identity/CollaboratorsService" +import PreviewService from "@root/services/identity/PreviewService" import { SitesCacheService } from "@root/services/identity/SitesCacheService" import SitesService from "@root/services/identity/SitesService" import ReviewRequestService from "@root/services/review/ReviewRequestService" @@ -118,6 +119,7 @@ const reviewRequestService = new ReviewRequestService( const MockSitesCacheService = { getLastUpdated: jest.fn(), } +const MockPreviewService = {} const sitesService = new SitesService({ siteRepository: Site, gitHubService, @@ -126,6 +128,7 @@ const sitesService = new SitesService({ isomerAdminsService, reviewRequestService, sitesCacheService: (MockSitesCacheService as unknown) as SitesCacheService, + previewService: (MockPreviewService as unknown) as PreviewService, }) const collaboratorsService = new CollaboratorsService({ siteRepository: Site, diff --git a/src/integration/Privatisation.spec.ts b/src/integration/Privatisation.spec.ts index f6a1dc7ef..4c3fec07c 100644 --- a/src/integration/Privatisation.spec.ts +++ b/src/integration/Privatisation.spec.ts @@ -58,6 +58,7 @@ import { FooterYmlService } from "@root/services/fileServices/YmlFileServices/Fo import { NavYmlService } from "@root/services/fileServices/YmlFileServices/NavYmlService" import CollaboratorsService from "@root/services/identity/CollaboratorsService" import DeploymentsService from "@root/services/identity/DeploymentsService" +import PreviewService from "@root/services/identity/PreviewService" import { SitesCacheService } from "@root/services/identity/SitesCacheService" import AuthorizationMiddlewareService from "@root/services/middlewareServices/AuthorizationMiddlewareService" import { GitHubService } from "@services/db/GitHubService" @@ -147,6 +148,7 @@ const reviewRequestService = new ReviewRequestService( const MockSitesCacheService = { getLastUpdated: jest.fn(), } +const MockPreviewService = {} const sitesService = new SitesService({ siteRepository: Site, gitHubService, @@ -155,6 +157,7 @@ const sitesService = new SitesService({ isomerAdminsService, reviewRequestService, sitesCacheService: (MockSitesCacheService as unknown) as SitesCacheService, + previewService: (MockPreviewService as unknown) as PreviewService, }) const navYmlService = new NavYmlService({ gitHubService, diff --git a/src/integration/Reviews.spec.ts b/src/integration/Reviews.spec.ts index 174036139..26e4b1fe9 100644 --- a/src/integration/Reviews.spec.ts +++ b/src/integration/Reviews.spec.ts @@ -85,6 +85,7 @@ import { UnlinkedPageService } from "@root/services/fileServices/MdPageServices/ import { CollectionYmlService } from "@root/services/fileServices/YmlFileServices/CollectionYmlService" import { ConfigService } from "@root/services/fileServices/YmlFileServices/ConfigService" import { FooterYmlService } from "@root/services/fileServices/YmlFileServices/FooterYmlService" +import PreviewService from "@root/services/identity/PreviewService" import { SitesCacheService } from "@root/services/identity/SitesCacheService" import { ReviewRequestDto } from "@root/types/dto/review" import { GitHubService } from "@services/db/GitHubService" @@ -150,6 +151,7 @@ const reviewRequestService = new ReviewRequestService( const MockSitesCacheService = { getLastUpdated: jest.fn(), } +const MockPreviewService = {} const sitesService = new SitesService({ siteRepository: Site, gitHubService, @@ -158,6 +160,7 @@ const sitesService = new SitesService({ isomerAdminsService, reviewRequestService, sitesCacheService: (MockSitesCacheService as unknown) as SitesCacheService, + previewService: (MockPreviewService as unknown) as PreviewService, }) const collaboratorsService = new CollaboratorsService({ siteRepository: Site, diff --git a/src/services/identity/__tests__/SitesService.spec.ts b/src/services/identity/__tests__/SitesService.spec.ts index 4b79ed6f2..818358c20 100644 --- a/src/services/identity/__tests__/SitesService.spec.ts +++ b/src/services/identity/__tests__/SitesService.spec.ts @@ -41,6 +41,7 @@ import MissingSiteError from "@root/errors/MissingSiteError" import { NotFoundError } from "@root/errors/NotFoundError" import RequestNotFoundError from "@root/errors/RequestNotFoundError" import { UnprocessableError } from "@root/errors/UnprocessableError" +import PreviewService from "@root/services/identity/PreviewService" import ReviewRequestService from "@root/services/review/ReviewRequestService" import { GitHubCommitData } from "@root/types/commitData" import { ConfigYmlData } from "@root/types/configYml" @@ -84,6 +85,7 @@ const MockSitesCacheService = { getLastUpdated: jest.fn(), } +const MockPreviewService = {} const SitesService = new _SitesService({ siteRepository: (MockRepository as unknown) as ModelStatic, gitHubService: (MockGithubService as unknown) as GitHubService, @@ -92,6 +94,7 @@ const SitesService = new _SitesService({ isomerAdminsService: (MockIsomerAdminsService as unknown) as IsomerAdminsService, reviewRequestService: (MockReviewRequestService as unknown) as ReviewRequestService, sitesCacheService: (MockSitesCacheService as unknown) as SitesCacheService, + previewService: (MockPreviewService as unknown) as PreviewService, }) const SpySitesService = { diff --git a/src/services/identity/__tests__/UsersService.spec.ts b/src/services/identity/__tests__/UsersService.spec.ts index c0e7b0286..c74397dd1 100644 --- a/src/services/identity/__tests__/UsersService.spec.ts +++ b/src/services/identity/__tests__/UsersService.spec.ts @@ -1,5 +1,5 @@ -import { Sequelize } from "sequelize-typescript" import { ModelStatic } from "sequelize/types" +import { Sequelize } from "sequelize-typescript" import { Otp, User, Whitelist } from "@root/database/models" import SmsClient from "@services/identity/SmsClient" diff --git a/src/services/utilServices/Sanitizer.ts b/src/services/utilServices/Sanitizer.ts index b83d7e64f..7b349c773 100644 --- a/src/services/utilServices/Sanitizer.ts +++ b/src/services/utilServices/Sanitizer.ts @@ -25,7 +25,10 @@ DOMPurify.addHook("uponSanitizeElement", (node, data) => { // Adapted from https://github.com/cure53/DOMPurify/blob/e0970d88053c1c564b6ccd633b4af7e7d9a10375/src/purify.js#L719-L736 DOMPurify.removed.push({ element: node }) try { - node.parentNode?.removeChild(node) + if (!node.parentNode) { + throw new Error() + } + node.parentNode.removeChild(node) } catch (e) { try { // eslint-disable-next-line no-param-reassign From 1f7a9947a88af11039bd19f9f4f0cafe9eb5519a Mon Sep 17 00:00:00 2001 From: caando Date: Wed, 26 Jul 2023 11:40:24 +0800 Subject: [PATCH 5/7] fix(preview): minor fixes according to review --- src/services/identity/PreviewService.ts | 1 - src/services/identity/SitesService.ts | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/services/identity/PreviewService.ts b/src/services/identity/PreviewService.ts index 364ea6486..b54645169 100644 --- a/src/services/identity/PreviewService.ts +++ b/src/services/identity/PreviewService.ts @@ -1,4 +1,3 @@ -// import { getLinkPreview, getPreviewFromContent } from "link-preview-js"; import axios from "axios" import { JSDOM } from "jsdom" import { ResultAsync, okAsync, errAsync } from "neverthrow" diff --git a/src/services/identity/SitesService.ts b/src/services/identity/SitesService.ts index 65086b108..128da7535 100644 --- a/src/services/identity/SitesService.ts +++ b/src/services/identity/SitesService.ts @@ -2,8 +2,6 @@ import _ from "lodash" import { errAsync, okAsync, ResultAsync } from "neverthrow" import { ModelStatic } from "sequelize" -import PreviewParsingError from "@errors/PreviewParsingError" - import { Deployment, Repo, Site } from "@database/models" import type UserSessionData from "@root/classes/UserSessionData" import UserWithSiteSessionData from "@root/classes/UserWithSiteSessionData" @@ -463,10 +461,8 @@ class SitesService { const urls = await this.getUrlsOfSite( new UserWithSiteSessionData({ siteName, ...userSessionData }) ) - if (urls.isOk()) { - if (urls.value.prod) { - return await this.previewService.getPreviewInfo(urls.value.prod) - } + if (urls.isOk() && urls.value.prod) { + return this.previewService.getPreviewInfo(urls.value.prod) } return { imageUrl: undefined, From 71301376115c2e3719137aa944480703b1d2869a Mon Sep 17 00:00:00 2001 From: caando Date: Thu, 27 Jul 2023 14:59:52 +0800 Subject: [PATCH 6/7] ref(preview): added error description and comments --- src/routes/v2/authenticated/sites.ts | 9 +++++++++ src/services/identity/PreviewService.ts | 5 +++++ src/services/utilServices/Sanitizer.ts | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/routes/v2/authenticated/sites.ts b/src/routes/v2/authenticated/sites.ts index 2164e9bd9..3f47f5344 100644 --- a/src/routes/v2/authenticated/sites.ts +++ b/src/routes/v2/authenticated/sites.ts @@ -211,6 +211,15 @@ export class SitesRouter { this.authorizationMiddleware.verifySiteMember, attachReadRouteHandlerWrapper(this.launchSite) ) + + // The /sites/preview is a POST endpoint as the frontend sends + // a list of sites to obtain previews for. This is to support + // GitHub login users who we don't have the list of sites for + // users in the db. However, using GET endpoint without sending + // a list of sites is ideal for caching of responses. Should all + // users be migrated to email based login in the future, a db + // query with session data can be used to obtain list of sites + // and endpoint can be changed to GET. router.post("/preview", attachReadRouteHandlerWrapper(this.getPreviewInfo)) return router diff --git a/src/services/identity/PreviewService.ts b/src/services/identity/PreviewService.ts index b54645169..4506a8231 100644 --- a/src/services/identity/PreviewService.ts +++ b/src/services/identity/PreviewService.ts @@ -17,6 +17,11 @@ export default class PreviewService { .querySelector('[rel="shortcut icon"]') ?.getAttribute("href") if (faviconLink) { + // There is an option to verify that the image link is valid by + // making a GET request and ensuring status code is 2XX. The + // decision is to not include it so that we feedback to CMS users + // that their favicon is malfunctioning as well as not incur + // additional latency. return okAsync(siteUrl.concat(faviconLink)) } return errAsync(new PreviewParsingError(siteUrl)) diff --git a/src/services/utilServices/Sanitizer.ts b/src/services/utilServices/Sanitizer.ts index 7b349c773..d3e744f80 100644 --- a/src/services/utilServices/Sanitizer.ts +++ b/src/services/utilServices/Sanitizer.ts @@ -26,7 +26,7 @@ DOMPurify.addHook("uponSanitizeElement", (node, data) => { DOMPurify.removed.push({ element: node }) try { if (!node.parentNode) { - throw new Error() + throw new Error("parent node is not defined") } node.parentNode.removeChild(node) } catch (e) { From 07654541e6b49c1230eae39d2abd16994521a82b Mon Sep 17 00:00:00 2001 From: caando Date: Thu, 27 Jul 2023 15:31:09 +0800 Subject: [PATCH 7/7] feat(previews): deny requests with more than limit no of sites and admin users --- src/services/identity/SitesService.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/services/identity/SitesService.ts b/src/services/identity/SitesService.ts index 128da7535..9aa7bdf9f 100644 --- a/src/services/identity/SitesService.ts +++ b/src/services/identity/SitesService.ts @@ -456,6 +456,18 @@ class SitesService { siteNames: string[], userSessionData: UserSessionData ): Promise { + const { isomerUserId: userId } = userSessionData + const isAdminUser = !!(await this.isomerAdminsService.getByUserId(userId)) + + // As fetching preview images is expensive and incurs high latency, + // we want to deny the request for admin users who don't require + // this feature and has hundreds of sites. We also deny requests + // with more than the limit number of sites as they are most likely + // admin users. + const SITES_NUMBER_LIMIT = 50 + if (isAdminUser || siteNames.length > SITES_NUMBER_LIMIT) { + return [] + } return Promise.all( siteNames.map(async (siteName) => { const urls = await this.getUrlsOfSite(