diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f458867b..ec060843b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,22 @@ All notable changes to this project will be documented in this file. Dates are d Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). +#### [v0.25.0](https://github.com/isomerpages/isomercms-backend/compare/v0.24.2...v0.25.0) + +- fix(markdown-utils): add check for falsy values [`#746`](https://github.com/isomerpages/isomercms-backend/pull/746) +- chore: add logging to endpoints being called [`#744`](https://github.com/isomerpages/isomercms-backend/pull/744) +- feat(datadog): add tracing for http requests out [`#745`](https://github.com/isomerpages/isomercms-backend/pull/745) +- feat(site creation): add support for email + github logins [`#739`](https://github.com/isomerpages/isomercms-backend/pull/739) +- Hotfix/0.24.2 [`#743`](https://github.com/isomerpages/isomercms-backend/pull/743) +- Fix: add svg to allowed image types [`#738`](https://github.com/isomerpages/isomercms-backend/pull/738) +- fix(site launch email): make table nicer [`#732`](https://github.com/isomerpages/isomercms-backend/pull/732) +- hotfix/0.24.1 [`#737`](https://github.com/isomerpages/isomercms-backend/pull/737) +- 0.24.0 [`#735`](https://github.com/isomerpages/isomercms-backend/pull/735) + #### [v0.24.2](https://github.com/isomerpages/isomercms-backend/compare/v0.24.1...v0.24.2) +> 3 May 2023 + - fix: token capacity alarm message ordering [`43ac12b`](https://github.com/isomerpages/isomercms-backend/commit/43ac12bab0114f7563428ce90cc9f55584a43422) #### [v0.24.1](https://github.com/isomerpages/isomercms-backend/compare/v0.23.1...v0.24.1) @@ -86,12 +100,12 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). - build(deps): bump vm2 from 3.9.12 to 3.9.15 in /microservices [`#688`](https://github.com/isomerpages/isomercms-backend/pull/688) - build(deps): bump vm2 from 3.9.11 to 3.9.15 [`#687`](https://github.com/isomerpages/isomercms-backend/pull/687) - release(0.19.0): merge to develop [`#684`](https://github.com/isomerpages/isomercms-backend/pull/684) -- fix(utils): sanitize empty string + trim [`#686`](https://github.com/isomerpages/isomercms-backend/pull/686) #### [v0.19.0](https://github.com/isomerpages/isomercms-backend/compare/v0.18.2...v0.19.0) > 6 April 2023 +- fix(utils): sanitize empty string + trim [`#686`](https://github.com/isomerpages/isomercms-backend/pull/686) - fix(utils): change order of ops and rec sanitization [`#680`](https://github.com/isomerpages/isomercms-backend/pull/680) - chore(review): fix tests for review router [`#683`](https://github.com/isomerpages/isomercms-backend/pull/683) - Feat(site launch): support for multiple sites [`#665`](https://github.com/isomerpages/isomercms-backend/pull/665) @@ -103,15 +117,14 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). > 3 April 2023 -- fix(review): return 200 for unmigrated sites [`bd69c29`](https://github.com/isomerpages/isomercms-backend/commit/bd69c29023554c5dcf8cb361227ba4ebf0d1ac08) -- fix(sanitize): use same setup for dompurify as FE [`c25f448`](https://github.com/isomerpages/isomercms-backend/commit/c25f448813e1addce67d0209375ec35ef9cb6c7b) - Fix: change response for github users accessing collaborator endpoints [`db1130f`](https://github.com/isomerpages/isomercms-backend/commit/db1130f96a6c9cda99df43d5774134aefffeee3e) #### [v0.18.1](https://github.com/isomerpages/isomercms-backend/compare/v0.18.0...v0.18.1) > 31 March 2023 -- fix(review): return 200 for unmigrated sites [`073cab8`](https://github.com/isomerpages/isomercms-backend/commit/073cab8c6704178ee5061b8582b4f999720dfc95) +- fix(review): return 200 for unmigrated sites [`bd69c29`](https://github.com/isomerpages/isomercms-backend/commit/bd69c29023554c5dcf8cb361227ba4ebf0d1ac08) +- fix(sanitize): use same setup for dompurify as FE [`c25f448`](https://github.com/isomerpages/isomercms-backend/commit/c25f448813e1addce67d0209375ec35ef9cb6c7b) #### [v0.18.0](https://github.com/isomerpages/isomercms-backend/compare/v0.17.0...v0.18.0) diff --git a/package-lock.json b/package-lock.json index 0c1793aac..280b72286 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "isomercms", - "version": "0.24.2", + "version": "0.25.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "isomercms", - "version": "0.24.1", + "version": "0.25.0", "dependencies": { "@aws-sdk/client-amplify": "^3.290.0", "@aws-sdk/client-cloudwatch-logs": "^3.290.0", diff --git a/package.json b/package.json index 2000b5fbb..82584b423 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "isomercms", - "version": "0.24.2", + "version": "0.25.0", "private": true, "scripts": { "build": "tsc -p tsconfig.build.json", diff --git a/src/routes/formsgSiteCreation.ts b/src/routes/formsgSiteCreation.ts index 881cc0e87..2c722287c 100644 --- a/src/routes/formsgSiteCreation.ts +++ b/src/routes/formsgSiteCreation.ts @@ -7,7 +7,6 @@ import { config } from "@config/config" import logger from "@logger/logger" import { BadRequestError } from "@errors/BadRequestError" -import InitializationError from "@errors/InitializationError" import { getField } from "@utils/formsg-utils" @@ -21,6 +20,7 @@ const REQUESTER_EMAIL_FIELD = "Government E-mail" const SITE_NAME_FIELD = "Site Name" const REPO_NAME_FIELD = "Repository Name" const OWNER_NAME_FIELD = "Site Owner E-mail" +const LOGIN_TYPE_FIELD = "Login Type" export interface FormsgRouterProps { usersService: UsersService @@ -55,7 +55,7 @@ export class FormsgRouter { const ownerEmail = getField(responses, OWNER_NAME_FIELD) ?.toLowerCase() .trim() - + const isEmailLogin = getField(responses, LOGIN_TYPE_FIELD) === "Email based" logger.info( `Create site form submission [${submissionId}] (repoName '${repoName}', siteName '${siteName}') requested by <${requesterEmail}>` ) @@ -87,15 +87,18 @@ export class FormsgRouter { await this.sendCreateError(requesterEmail, repoName, submissionId, err) return res.sendStatus(200) } - const foundOwner = await this.usersService.findOrCreateByEmail(ownerEmail) - + let foundOwner + if (isEmailLogin) { + foundOwner = await this.usersService.findOrCreateByEmail(ownerEmail) + } // 3. Use service to create site - const { deployment } = await this.infraService.createSite( - foundIsomerRequester, - foundOwner, + const { deployment } = await this.infraService.createSite({ + creator: foundIsomerRequester, + member: foundOwner, siteName, - repoName - ) + repoName, + isEmailLogin, + }) await this.sendCreateSuccess( requesterEmail, repoName, diff --git a/src/routes/formsgSiteLaunch.ts b/src/routes/formsgSiteLaunch.ts index 60fa3c3c8..617b000e2 100644 --- a/src/routes/formsgSiteLaunch.ts +++ b/src/routes/formsgSiteLaunch.ts @@ -11,6 +11,12 @@ import { getField, getFieldsFromTable } from "@utils/formsg-utils" import { attachFormSGHandler } from "@root/middleware" import { mailer } from "@root/services/utilServices/MailClient" +import { + DnsRecordsEmailProps, + LaunchFailureEmailProps, + getDNSRecordsEmailBody, + getErrorEmailBody, +} from "@root/services/utilServices/SendDNSRecordEmailClient" import UsersService from "@services/identity/UsersService" import InfraService from "@services/infra/InfraService" @@ -52,26 +58,6 @@ interface FormResponsesProps { siteLaunchDetails?: string[] | string[][] } -interface LaunchFailureEmailProps { - // The fields here are optional since a misconfiguration in our - // formSG can cause some or even all fields to be missing - requesterEmail?: string - repoName?: string - primaryDomain?: string - error: string -} - -interface DnsRecordsEmailProps { - requesterEmail: string - repoName: string - domainValidationSource: string - domainValidationTarget: string - primaryDomainSource: string - primaryDomainTarget: string - redirectionDomainSource?: string - redirectionDomainTarget?: string -} - export class FormsgSiteLaunchRouter { launchSiteFromForm = async (formResponses: FormResponsesProps) => { const { @@ -219,21 +205,7 @@ export class FormsgSiteLaunchRouter { const { requesterEmail } = failureResults[0] const email = requesterEmail || ISOMER_ADMIN_EMAIL const subject = `[Isomer] Launch site FAILURE` - let html = `

The following sites were NOT launched successfully. (Form submission id [${submissionId}])

- ` - - failureResults.forEach((failureResult) => { - const displayedRepoName = failureResult.repoName || "" - html += ` - - - - ` - }) - html += ` -
Repo NameError
${displayedRepoName}${failureResult.error}
-

This email was sent from the Isomer CMS backend.

` - + const html = getErrorEmailBody(submissionId, failureResults) await mailer.sendMail(email, subject, html) } @@ -244,56 +216,7 @@ export class FormsgSiteLaunchRouter { if (dnsRecordsEmailProps.length === 0) return const { requesterEmail } = dnsRecordsEmailProps[0] const subject = `[Isomer] DNS records for launching websites` - - let html = `

Isomer sites are in the process of launching. (Form submission id [${submissionId}])

- - - - - - - - - - ` - dnsRecordsEmailProps.forEach((dnsRecords) => { - // check if dnsRecords.redirectionDomain is undefined - const hasRedirection = !!dnsRecords.redirectionDomainSource - html += ` - - - - - - - - - - - - ` - - if (hasRedirection) { - html += ` - - - - - - ` - } - }) - html += `
Repo NameSourceTargetType
${dnsRecords.repoName}${dnsRecords.domainValidationSource}${dnsRecords.domainValidationTarget}CNAME
${dnsRecords.repoName}${ - hasRedirection - ? // if redirection, website will be hosted in the 'www' subdomain - `www.${dnsRecords.primaryDomainSource}` - : dnsRecords.primaryDomainSource - }${dnsRecords.primaryDomainTarget}CNAME
${dnsRecords.repoName}${ - // note that the source here is the primary domain source - // since the non-www will be the one pointing to our redirection server - dnsRecords.primaryDomainSource - }${dnsRecords.redirectionDomainTarget}A Record
-

This email was sent from the Isomer CMS backend.

` + const html = getDNSRecordsEmailBody(submissionId, dnsRecordsEmailProps) await mailer.sendMail(requesterEmail, subject, html) } diff --git a/src/server.js b/src/server.js index b8ac82953..4d8ae5db8 100644 --- a/src/server.js +++ b/src/server.js @@ -1,4 +1,5 @@ -import "dd-trace/init" +// NOTE: the import for tracer doesn't resolve with path aliasing +import "./utils/tracer" import "module-alias/register" import SequelizeStoreFactory from "connect-session-sequelize" import session from "express-session" diff --git a/src/services/api/AxiosInstance.ts b/src/services/api/AxiosInstance.ts index c0679a775..32624892b 100644 --- a/src/services/api/AxiosInstance.ts +++ b/src/services/api/AxiosInstance.ts @@ -5,6 +5,7 @@ import { config } from "@config/config" import logger from "@logger/logger" import { getAccessToken } from "@utils/token-retrieval-utils" +import tracer from "@utils/tracer" // Env vars const GITHUB_ORG_NAME = config.get("github.orgName") @@ -15,13 +16,33 @@ const requestFormatter = async (config: AxiosRequestConfig) => { const authMessage = config.headers.Authorization // If accessToken is missing, authMessage is `token ` - if ( + // NOTE: This also implies that the user has not provided + // their own github token and hence, are email login users. + const isEmailLoginUser = !authMessage || authMessage === "token " || authMessage === "token undefined" - ) { + + if (isEmailLoginUser) { const accessToken = await getAccessToken() config.headers.Authorization = `token ${accessToken}` + tracer.use("http", { + hooks: { + request: (span, req, res) => { + span?.setTag("user.type", "email") + }, + }, + }) + logger.info(`Email login user made call to Github API: ${config.url}`) + } else { + tracer.use("http", { + hooks: { + request: (span, req, res) => { + span?.setTag("user.type", "github") + }, + }, + }) + logger.info(`Github login user made call to Github API: ${config.url}`) } return { ...config, diff --git a/src/services/identity/ReposService.ts b/src/services/identity/ReposService.ts index 41f7a0cdd..21d17b314 100644 --- a/src/services/identity/ReposService.ts +++ b/src/services/identity/ReposService.ts @@ -57,13 +57,18 @@ export default class ReposService { setupGithubRepo = async ({ repoName, site, + isEmailLogin, }: { repoName: string site: Site + isEmailLogin: boolean }): Promise => { const repoUrl = `https://github.com/isomerpages/${repoName}` await this.createRepoOnGithub(repoName) + if (!isEmailLogin) { + await this.createTeamOnGitHub(repoName) + } await this.generateRepoAndPublishToGitHub(repoName, repoUrl) return this.create({ name: repoName, @@ -73,6 +78,15 @@ export default class ReposService { }) } + createTeamOnGitHub = ( + repoName: string + ): Promise => + octokit.teams.create({ + org: ISOMER_GITHUB_ORGANIZATION_NAME, + name: repoName, + privacy: "closed", + }) + modifyDeploymentUrlsOnRepo = async ( repoName: string, productionUrl: string, @@ -147,7 +161,10 @@ export default class ReposService { private: false, }) - setRepoAndTeamPermissions = async (repoName: string): Promise => { + setRepoAndTeamPermissions = async ( + repoName: string, + isEmailLogin: boolean + ): Promise => { await octokit.repos.updateBranchProtection({ owner: ISOMER_GITHUB_ORGANIZATION_NAME, repo: repoName, @@ -170,6 +187,15 @@ export default class ReposService { repo: repoName, permission: "admin", }) + if (!isEmailLogin) { + await octokit.teams.addOrUpdateRepoPermissionsInOrg({ + org: ISOMER_GITHUB_ORGANIZATION_NAME, + team_slug: repoName, + owner: ISOMER_GITHUB_ORGANIZATION_NAME, + repo: repoName, + permission: "push", + }) + } } generateRepoAndPublishToGitHub = async ( diff --git a/src/services/infra/InfraService.ts b/src/services/infra/InfraService.ts index 2f73a0935..72b843722 100644 --- a/src/services/infra/InfraService.ts +++ b/src/services/infra/InfraService.ts @@ -42,6 +42,16 @@ interface dnsRecordDto { target: string type: RedirectionTypes } + +type CreateSiteParams = { + creator: User + // this is ok, since we don't need this for github login flow + member: User | undefined + siteName: string + repoName: string + isEmailLogin: boolean +} + export default class InfraService { private readonly sitesService: InfraServiceProps["sitesService"] @@ -71,20 +81,22 @@ export default class InfraService { this.collaboratorsService = collaboratorsService } - createSite = async ( - creator: User, - member: User, - siteName: string, - repoName: string - ) => { + createSite = async ({ + creator, + member, + siteName, + repoName, + isEmailLogin, + }: CreateSiteParams) => { let site: Site | undefined // For error handling - const memberEmail = member.email - if (!memberEmail) { + const memberEmail = member?.email + const doesMemberEmailExistForEmailLogin = !memberEmail && isEmailLogin + if (doesMemberEmailExistForEmailLogin) { logger.error( - `createSite: initial member for ${siteName} does not have associated email` + `createSite: initial member for ${siteName} for email login flow does not have associated email` ) throw new Error( - `createSite: initial member for ${siteName} does not have associated email` + `createSite: initial member for ${siteName} for email login flow does not have associated email` ) } try { @@ -98,7 +110,11 @@ export default class InfraService { logger.info(`Created site record in database, site ID: ${site.id}`) // 2. Set up GitHub repo and branches using the ReposService - const repo = await this.reposService.setupGithubRepo({ repoName, site }) + const repo = await this.reposService.setupGithubRepo({ + repoName, + site, + isEmailLogin, + }) logger.info(`Created repo in GitHub, repo name: ${repoName}`) // 3. Set up the Amplify project using the DeploymentsService @@ -116,9 +132,10 @@ export default class InfraService { ) // 5. Set up permissions - await this.reposService.setRepoAndTeamPermissions(repoName) - await this.collaboratorsService.create(repoName, memberEmail, true) - + await this.reposService.setRepoAndTeamPermissions(repoName, isEmailLogin) + if (isEmailLogin && memberEmail) { + await this.collaboratorsService.create(repoName, memberEmail, true) + } // 6. Update status const updateSuccessSiteInitParams = { id: site.id, diff --git a/src/services/utilServices/SendDNSRecordEmailClient.ts b/src/services/utilServices/SendDNSRecordEmailClient.ts new file mode 100644 index 000000000..b0780f67f --- /dev/null +++ b/src/services/utilServices/SendDNSRecordEmailClient.ts @@ -0,0 +1,125 @@ +import { groupBy } from "lodash" + +export interface DnsRecordsEmailProps { + requesterEmail: string + repoName: string + domainValidationSource: string + domainValidationTarget: string + primaryDomainSource: string + primaryDomainTarget: string + redirectionDomainSource?: string + redirectionDomainTarget?: string +} + +export interface LaunchFailureEmailProps { + // The fields here are optional since a misconfiguration in our + // formSG can cause some or even all fields to be missing + requesterEmail?: string + repoName?: string + primaryDomain?: string + error: string +} + +const tableStyle = + "border-collapse: collapse; width: 100%; border: 1px solid #ddd; text-align: center;" + +const thStyle = "padding: 8px; text-align: center; border: 1px solid #ddd;" + +const tdStyle = "padding: 8px; text-align: left; border: 1px solid #ddd;" + +const headerRowStyle = + "background-color: #f2f2f2; border: 1px solid #ddd; text-align: center;" + +const repoNameStyle = + "padding: 8px; text-align: left; font-weight: bold; border: 1px solid #ddd;" + +const bodyTitleStyle = "font-size: 16px; font-weight: bold;" + +const bodyFooterStyle = "font-size: 14px;" + +export function getDNSRecordsEmailBody( + submissionId: string, + dnsRecordsEmailProps: DnsRecordsEmailProps[] +) { + let html = `

Isomer sites are in the process of launching. (Form submission id [${submissionId}])

+ + + + + + + + + + ` + const groupedDnsRecords = groupBy(dnsRecordsEmailProps, "repoName") + Object.keys(groupedDnsRecords).forEach((repoName) => { + const hasRedirection = !!groupedDnsRecords[repoName].some( + (dnsRecords) => !!dnsRecords.redirectionDomainSource + ) + html += ` + + ` + groupedDnsRecords[repoName].forEach((dnsRecords) => { + // check if dnsRecords.redirectionDomain is undefined + html += ` + + + + + + + + + + + ` + + if (hasRedirection) { + html += ` + + + + + ` + } + }) + }) + html += `
Repo NameSourceTargetType
${repoName}
${dnsRecords.domainValidationSource}${dnsRecords.domainValidationTarget}CNAME
${ + hasRedirection + ? `www.${dnsRecords.primaryDomainSource}` + : dnsRecords.primaryDomainSource + }${dnsRecords.primaryDomainTarget}CNAME
${dnsRecords.primaryDomainSource}${dnsRecords.redirectionDomainTarget}A Record
+

This email was sent from the Isomer CMS backend.

` + return html +} + +export function getErrorEmailBody( + submissionId: string, + failureResults: LaunchFailureEmailProps[] +) { + let html = `

The following sites were NOT launched successfully. (Form submission id [${submissionId}])

+ + + + + + + + ` + failureResults.forEach((failureResult) => { + const displayedRepoName = failureResult.repoName || "" + html += ` + + + + ` + }) + html += ` + +
Repo NameError
${displayedRepoName}${failureResult.error}
+

This email was sent from the Isomer CMS backend.

` + return html +} diff --git a/src/utils/file-upload-utils.js b/src/utils/file-upload-utils.js index d8e4cc68e..d15b12383 100644 --- a/src/utils/file-upload-utils.js +++ b/src/utils/file-upload-utils.js @@ -18,6 +18,7 @@ const ALLOWED_FILE_EXTENSIONS = [ "tif", "bmp", "ico", + "svg", ] const defaultCloudmersiveClient = CloudmersiveVirusApiClient.ApiClient.instance diff --git a/src/utils/markdown-utils.js b/src/utils/markdown-utils.js index d53364fd3..c37e5b72d 100644 --- a/src/utils/markdown-utils.js +++ b/src/utils/markdown-utils.js @@ -11,7 +11,7 @@ const getTrailingSlashWithPermalink = (permalink) => permalink.endsWith("/") ? permalink : `${permalink}/` const recursiveUnescape = (val) => { - if (val === "") return val + if (!val) return val if (Array.isArray(val)) { return val.map(recursiveUnescape) } diff --git a/src/utils/tracer.ts b/src/utils/tracer.ts new file mode 100644 index 000000000..d528082ea --- /dev/null +++ b/src/utils/tracer.ts @@ -0,0 +1,5 @@ +import tracer from "dd-trace" + +tracer.init() + +export default tracer