From 23f90dcc938c6d46c9d5aee18628a5017024def7 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Wed, 27 Mar 2024 23:14:20 +0800 Subject: [PATCH] fix(dig): dig not working --- package-lock.json | 19 +- package.json | 3 +- .../formsg/__tests__/formsgSiteLaunch.spec.ts | 202 ++++++++++++++++++ src/routes/formsg/formsgSiteLaunch.ts | 168 +++++++++------ src/services/identity/ReposService.ts | 6 +- .../utilServices/SendDNSRecordEmailClient.ts | 32 ++- src/types/dig.ts | 27 +-- src/types/node-dig-dns/node-dig-dns.d.ts | 1 - src/types/siteLaunch.ts | 13 ++ 9 files changed, 349 insertions(+), 122 deletions(-) create mode 100644 src/routes/formsg/__tests__/formsgSiteLaunch.spec.ts delete mode 100644 src/types/node-dig-dns/node-dig-dns.d.ts diff --git a/package-lock.json b/package-lock.json index 43206b780..76c6b593d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -67,7 +67,6 @@ "morgan": "~1.10.0", "neverthrow": "^6.1.0", "nocache": "^3.0.4", - "node-dig-dns": "^0.3.3", "otplib": "^12.0.1", "papaparse": "^5.4.1", "patch-package": "^8.0.0", @@ -2109,6 +2108,7 @@ "version": "7.23.9", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.9.tgz", "integrity": "sha512-0CX6F+BI2s9dkUqr08KFrAIZgNFj75rdBU/DjCyYLIaV/quFjkk6T+EJ2LkZHyZTbEV4L5p97mNkUsHl2wLFAw==", + "dev": true, "dependencies": { "regenerator-runtime": "^0.14.0" }, @@ -2119,7 +2119,8 @@ "node_modules/@babel/runtime/node_modules/regenerator-runtime": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", - "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "dev": true }, "node_modules/@babel/template": { "version": "7.23.9", @@ -14607,11 +14608,6 @@ "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" }, - "node_modules/lodash.compact": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/lodash.compact/-/lodash.compact-3.0.1.tgz", - "integrity": "sha512-2ozeiPi+5eBXW1CLtzjk8XQFhQOEMwwfxblqeq6EGyTxZJ1bPATqilY0e6g2SLQpP4KuMeuioBhEnWz5Pr7ICQ==" - }, "node_modules/lodash.find": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.find/-/lodash.find-4.6.0.tgz", @@ -15394,15 +15390,6 @@ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" }, - "node_modules/node-dig-dns": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/node-dig-dns/-/node-dig-dns-0.3.3.tgz", - "integrity": "sha512-l/57tMh/8D89MRiMF+iIYz3h5p2pGusxY75MM7gjsPegV/WqyVWC266DM9HTOeirmp4CdrQQBMbQssb2W7maAA==", - "dependencies": { - "@babel/runtime": "^7.18.9", - "lodash.compact": "^3.0.1" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", diff --git a/package.json b/package.json index fca8f7e87..042bd5f1a 100644 --- a/package.json +++ b/package.json @@ -27,9 +27,9 @@ "postinstall": "patch-package" }, "dependencies": { - "@aws-sdk/client-dynamodb": "^3.521.0", "@aws-sdk/client-amplify": "^3.521.0", "@aws-sdk/client-cloudwatch-logs": "^3.521.0", + "@aws-sdk/client-dynamodb": "^3.521.0", "@aws-sdk/client-secrets-manager": "^3.389.0", "@aws-sdk/lib-dynamodb": "^3.521.0", "@growthbook/growthbook": "^0.34.0", @@ -85,7 +85,6 @@ "morgan": "~1.10.0", "neverthrow": "^6.1.0", "nocache": "^3.0.4", - "node-dig-dns": "^0.3.3", "otplib": "^12.0.1", "papaparse": "^5.4.1", "patch-package": "^8.0.0", diff --git a/src/routes/formsg/__tests__/formsgSiteLaunch.spec.ts b/src/routes/formsg/__tests__/formsgSiteLaunch.spec.ts new file mode 100644 index 000000000..9ad88cbed --- /dev/null +++ b/src/routes/formsg/__tests__/formsgSiteLaunch.spec.ts @@ -0,0 +1,202 @@ +import { CaaRecord } from "node:dns" +import dnsPromises from "node:dns/promises" + +import UsersService from "@root/services/identity/UsersService" +import InfraService from "@root/services/infra/InfraService" +import { DigDNSRecord } from "@root/services/utilServices/SendDNSRecordEmailClient" +import { SiteLaunchResult } from "@root/types/siteLaunch" + +import { FormsgSiteLaunchRouter as _FormsgSiteLaunchRouter } from "../formsgSiteLaunch" + +const MockUsersService = ({ + findById: jest.fn(), + findByGitHubId: jest.fn(), + findByEmail: jest.fn(), +} as any) as UsersService + +const MockInfraService = ({ + getSite: jest.fn(), +} as any) as InfraService + +const FormsgSiteLaunch = new _FormsgSiteLaunchRouter({ + usersService: MockUsersService, + infraService: MockInfraService, +}) + +const mockResult: SiteLaunchResult = ({ + siteName: "testSite", + primaryDomain: "test.com", + primaryDomainSource: "test.com", +} as any) as SiteLaunchResult + +const mockResultWithRedirection: SiteLaunchResult = ({ + siteName: "testSite", + primaryDomain: "test.com", + primaryDomainSource: "test.com", + redirectionDomainSource: "test.com", + redirectionDomainTarget: "www.test.com", +} as any) as SiteLaunchResult + +describe("FormsgSiteLaunchRouter", () => { + it("should add quad A records if any exists", async () => { + // Arrange + jest + .spyOn(dnsPromises, "resolve6") + .mockResolvedValue(["2404:6800:4003:c0f::64", "2404:6800:4003:c0f::8b"]) + const expectedResult = [ + { + domain: "test.com", + value: "2404:6800:4003:c0f::64", + type: "AAAA", + }, + { + domain: "test.com", + value: "2404:6800:4003:c0f::8b", + type: "AAAA", + }, + ] as DigDNSRecord[] + + // Act + const actualResult = await FormsgSiteLaunch.digAAAADomainRecords(mockResult) + + // Assert + expect(actualResult).toEqual(expectedResult) + }) + + it("should not add quad A records if none exists", async () => { + // Arrange + jest.spyOn(dnsPromises, "resolve6").mockResolvedValue([]) + const expectedResult = [] as DigDNSRecord[] + + // Act + const actualResult = await FormsgSiteLaunch.digAAAADomainRecords(mockResult) + + // Assert + expect(actualResult).toEqual(expectedResult) + }) + + it("should add AWS CAA records if required", async () => { + // Arrange + const caaResult: CaaRecord[] = [ + { + critical: 0, + issue: "cloudflaressl.com", + }, + ] + jest.spyOn(dnsPromises, "resolveCaa").mockResolvedValue(caaResult) + const expectedResult = { + addAWSACMCertCAA: true, + addLetsEncryptCAA: false, + } + + // Act + const actualResult = await FormsgSiteLaunch.digCAADomainRecords(mockResult) + + // Assert + expect(actualResult).toEqual(expectedResult) + }) + + it("should not add AWS CAA if not required", async () => { + // Arrange + const caaResult: CaaRecord[] = [] + jest.spyOn(dnsPromises, "resolveCaa").mockResolvedValue(caaResult) + const expectedResult = { + addAWSACMCertCAA: false, + addLetsEncryptCAA: false, + } + + // Act + const actualResult = await FormsgSiteLaunch.digCAADomainRecords(mockResult) + + // Assert + expect(actualResult).toEqual(expectedResult) + }) + + it("should not add AWS CAA if already present", async () => { + // Arrange + const caaResult: CaaRecord[] = [ + { + critical: 0, + issue: "amazon.com", + }, + ] + jest.spyOn(dnsPromises, "resolveCaa").mockResolvedValue(caaResult) + const expectedResult = { + addAWSACMCertCAA: false, + addLetsEncryptCAA: false, + } + + // Act + const actualResult = await FormsgSiteLaunch.digCAADomainRecords(mockResult) + + // Assert + expect(actualResult).toEqual(expectedResult) + }) + + it("should add LetsEncrypt CAA records if required", async () => { + // Arrange + const caaResult: CaaRecord[] = [ + { + critical: 0, + issue: "amazon.com", + }, + ] + jest.spyOn(dnsPromises, "resolveCaa").mockResolvedValue(caaResult) + const expectedResult = { + addAWSACMCertCAA: false, + addLetsEncryptCAA: true, + } + + // Act + const actualResult = await FormsgSiteLaunch.digCAADomainRecords( + mockResultWithRedirection + ) + + // Assert + expect(actualResult).toEqual(expectedResult) + }) + + it("should not add LetsEncrypt CAA if not required", async () => { + // Arrange + const caaResult: CaaRecord[] = [] + jest.spyOn(dnsPromises, "resolveCaa").mockResolvedValue(caaResult) + const expectedResult = { + addAWSACMCertCAA: false, + addLetsEncryptCAA: false, + } + + // Act + const actualResult = await FormsgSiteLaunch.digCAADomainRecords( + mockResultWithRedirection + ) + + // Assert + expect(actualResult).toEqual(expectedResult) + }) + + it("should not add LetsEncrypt CAA if already present", async () => { + // Arrange + const caaResult: CaaRecord[] = [ + { + critical: 0, + issue: "letsencrypt.org", + }, + { + critical: 0, + issue: "amazon.com", + }, + ] + jest.spyOn(dnsPromises, "resolveCaa").mockResolvedValue(caaResult) + const expectedResult = { + addAWSACMCertCAA: false, + addLetsEncryptCAA: false, + } + + // Act + const actualResult = await FormsgSiteLaunch.digCAADomainRecords( + mockResultWithRedirection + ) + // Assert + expect(actualResult).toEqual(expectedResult) + }) +}) diff --git a/src/routes/formsg/formsgSiteLaunch.ts b/src/routes/formsg/formsgSiteLaunch.ts index f4906f9fe..b5b456689 100644 --- a/src/routes/formsg/formsgSiteLaunch.ts +++ b/src/routes/formsg/formsgSiteLaunch.ts @@ -1,9 +1,10 @@ +import { CaaRecord } from "node:dns" +import dnsPromises from "node:dns/promises" + import { DecryptedContentAndAttachments } from "@opengovsg/formsg-sdk/dist/types" import autoBind from "auto-bind" -import axios from "axios" import express, { RequestHandler } from "express" import { err, ok } from "neverthrow" -import dig from "node-dig-dns" import { config } from "@config/config" @@ -17,13 +18,14 @@ import { ISOMER_SUPPORT_EMAIL } from "@root/constants" import { attachFormSGHandler } from "@root/middleware" import { mailer } from "@root/services/utilServices/MailClient" import { + DigDNSRecord, DnsRecordsEmailProps, LaunchFailureEmailProps, getDNSRecordsEmailBody, getErrorEmailBody, } from "@root/services/utilServices/SendDNSRecordEmailClient" import TRUSTED_AMPLIFY_CAA_RECORDS from "@root/types/caaAmplify" -import { DigResponse, DigType } from "@root/types/dig" +import { SiteLaunchResult } from "@root/types/siteLaunch" import UsersService from "@services/identity/UsersService" import InfraService from "@services/infra/InfraService" @@ -225,23 +227,91 @@ export class FormsgSiteLaunchRouter { await mailer.sendMail(requesterEmail, subject, html) } - digDomainRecords = async ( - domain: string, - digType: DigType - ): Promise => - dig([domain, digType]) - .then((result: DigResponse) => { - logger.info(`Received DIG response: ${JSON.stringify(result)}`) + digAAAADomainRecords = async ( + launchResult: SiteLaunchResult + ): Promise => { + try { + // check for AAAA records + const quadADigResponses = await dnsPromises.resolve6( + launchResult.primaryDomainSource + ) + + if (quadADigResponses.length <= 0) { + return [] + } + return quadADigResponses.map((record) => ({ + domain: launchResult.primaryDomainSource, + type: "AAAA", + value: record, + })) + } catch (e: any) { + if (e.code && e.code === "ENODATA") { + logger.info( + `Unable to get dig response for domain: ${launchResult.primaryDomainSource}. Skipping check for AAAA records` + ) + } + throw e + } + } + + digCAADomainRecords = async ( + launchResult: SiteLaunchResult + ): Promise<{ + addAWSACMCertCAA: boolean + addLetsEncryptCAA: boolean + }> => { + try { + const caaDigResponses: CaaRecord[] = await dnsPromises.resolveCaa( + launchResult.primaryDomainSource + ) + + if (caaDigResponses.length > 0) { + const caaRecords = caaDigResponses + + /** + * NOTE: If there exists more than one CAA Record, we need to + * 1. check if they have whitelisted Amazon CAA && letsencrypt.org CAA (if using redir) + * 2. if not, send email to inform them to whitelist Amazon CAA and letsencrypt.org CAA (if using redir) + */ + const hasAmazonCAAWhitelisted = caaRecords.some((record) => { + const isAmazonCAA = TRUSTED_AMPLIFY_CAA_RECORDS.some( + (trustedCAA) => + trustedCAA === record.issue || trustedCAA === record.issuewild + ) + return isAmazonCAA + }) + + const isUsingRedirectionService = !!launchResult.redirectionDomainSource + + const needsLetsEncryptCAAWhitelisted = + isUsingRedirectionService && + !caaRecords.some((record) => { + const isLetsEncryptCAA = + record.issue === "letsencrypt.org" || + record.issuewild === "letsencrypt.org" + return isLetsEncryptCAA + }) + + const result = { + addAWSACMCertCAA: !hasAmazonCAAWhitelisted, + addLetsEncryptCAA: needsLetsEncryptCAAWhitelisted, + } + return result - }) - .catch((err: unknown) => { - logger.error( - `An error occurred while performing dig for domain: ${domain}: ${JSON.stringify( - err - )}` + } + return { + addAWSACMCertCAA: false, + addLetsEncryptCAA: false, + } + } catch (e: any) { + if (e.code && e.code === "ENODATA") { + logger.info( + `Unable to get dig response for domain: ${launchResult.primaryDomainSource}. Skipping check for CAA records` ) - return null - }) + } + throw e + } + } private async handleSiteLaunchResults( formResponses: FormResponsesProps[], @@ -254,60 +324,18 @@ export class FormsgSiteLaunchRouter { const successResults: DnsRecordsEmailProps[] = [] for (const launchResult of launchResults) { if (launchResult.isOk()) { - // check for AAAA records - const quadADigResponse = await this.digDomainRecords( - launchResult.value.primaryDomainSource, - "AAAA" - ) - const caaDigResponse = await this.digDomainRecords( - launchResult.value.primaryDomainSource, - "CAA" + const successResult: DnsRecordsEmailProps = launchResult.value + + successResult.quadARecords = await this.digAAAADomainRecords( + launchResult.value ) + const { + addAWSACMCertCAA, + addLetsEncryptCAA, + } = await this.digCAADomainRecords(launchResult.value) - const successResult: DnsRecordsEmailProps = launchResult.value - if (quadADigResponse && quadADigResponse.answer) { - const quadARecords = quadADigResponse.answer - successResult.quadARecords = quadARecords.map((record) => ({ - domain: record.domain, - class: record.class, - type: record.type, - value: record.value, - })) - } else { - logger.info( - `Unable to get dig response for domain: ${launchResult.value.primaryDomainSource}. Skipping check for AAAA records` - ) - } - - if (!caaDigResponse) { - logger.info( - `Unable to get dig response for domain: ${launchResult.value.primaryDomainSource}. Skipping check for CAA records` - ) - } else if (caaDigResponse.answer) { - const caaRecords = caaDigResponse.answer - - /** - * NOTE: If there exists more than one CAA Record, we need to - * 1. check if they have whitelisted Amazon CAA - * 2. if not, send email to inform them to whitelist Amazon CAA - */ - const hasAmazonCAAWhitelisted = caaRecords.some((record) => { - const isAmazonCAA = TRUSTED_AMPLIFY_CAA_RECORDS.some( - (trustedCAA) => trustedCAA === record.value - ) - return isAmazonCAA - }) - if (caaRecords.length > 0 && !hasAmazonCAAWhitelisted) { - successResult.addCAARecord = true - } else { - successResult.addCAARecord = false - } - } else { - logger.info( - `${launchResult.value.primaryDomainSource} Domain does not have any CAA records.` - ) - } - // Create better uptime monitor + successResult.addAWSACMCertCAA = addAWSACMCertCAA + successResult.addLetsEncryptCAA = addLetsEncryptCAA successResults.push(successResult) } } diff --git a/src/services/identity/ReposService.ts b/src/services/identity/ReposService.ts index 2c09de224..966c00e0f 100644 --- a/src/services/identity/ReposService.ts +++ b/src/services/identity/ReposService.ts @@ -315,12 +315,14 @@ export const createRecords = (zoneId: string): Record[] => { return records; }; ` - + const isProd = config.get("env") === "prod" + const branchName = isProd ? "staging" : "develop" return ResultAsync.fromPromise( octokit.repos.getContent({ owner: ISOMER_GITHUB_ORGANIZATION_NAME, repo: DNS_INDIRECTION_REPO, path: `dns/${primaryDomain}.ts`, + ref: branchName, }), () => errAsync(true) ) @@ -346,6 +348,7 @@ export const createRecords = (zoneId: string): Record[] => { path: `dns/${primaryDomain}.ts`, message: `Update ${primaryDomain}.ts`, content: Buffer.from(template).toString("base64"), + branch: branchName, sha, }), (error) => { @@ -369,6 +372,7 @@ export const createRecords = (zoneId: string): Record[] => { path: `dns/${primaryDomain}.ts`, message: `Create ${primaryDomain}.ts`, content: Buffer.from(template).toString("base64"), + branch: branchName, }), (error) => { logger.error( diff --git a/src/services/utilServices/SendDNSRecordEmailClient.ts b/src/services/utilServices/SendDNSRecordEmailClient.ts index 0218b669d..815591988 100644 --- a/src/services/utilServices/SendDNSRecordEmailClient.ts +++ b/src/services/utilServices/SendDNSRecordEmailClient.ts @@ -5,7 +5,6 @@ import { DigType } from "@root/types/dig" export interface DigDNSRecord { domain: string - class: string type: DigType value: string } @@ -21,7 +20,8 @@ export interface DnsRecordsEmailProps { redirectionDomainSource?: string redirectionDomainTarget?: string quadARecords?: DigDNSRecord[] - addCAARecord?: boolean + addLetsEncryptCAA?: boolean + addAWSACMCertCAA?: boolean } export interface LaunchFailureEmailProps { @@ -115,7 +115,7 @@ export function getDNSRecordsEmailBody( Object.keys(groupedDnsRecords).forEach((repoName) => { groupedDnsRecords[repoName].forEach((dnsRecord) => { - if (dnsRecord.addCAARecord) { + if (dnsRecord.addAWSACMCertCAA || dnsRecord.addLetsEncryptCAA) { html += `

Please add CAA records for the following repo: ${repoName}.` html += ` @@ -128,7 +128,10 @@ export function getDNSRecordsEmailBody( - + ` + + if (dnsRecord.addAWSACMCertCAA) { + html += ` @@ -142,7 +145,25 @@ export function getDNSRecordsEmailBody( - + ` + } + if (dnsRecord.addLetsEncryptCAA) { + html += ` + + + + + + + + + + + + + ` + } + html += `
Value
${repoName} ${dnsRecord.primaryDomainSource}issuewild amazontrust.com
${repoName}${dnsRecord.primaryDomainSource}0issueletsencrypt.org
${dnsRecord.primaryDomainSource}0issuewildletsencrypt.org
` } }) @@ -174,7 +195,6 @@ export function getDNSRecordsEmailBody( html += ` ${record.domain} - ${record.class} ${record.type} ${record.value} ` diff --git a/src/types/dig.ts b/src/types/dig.ts index 9cf4f352e..2588febbf 100644 --- a/src/types/dig.ts +++ b/src/types/dig.ts @@ -1,26 +1 @@ -export type DigResponse = { - question: string[][] - answer?: { - domain: string - ttl: string - class: string - type: DigType - value: string - }[] - time?: number - server?: string - datetime?: string - size?: number -} - -export type DigType = - | "A" - | "AAAA" - | "CNAME" - | "MX" - | "NS" - | "PTR" - | "SOA" - | "SRV" - | "TXT" - | "CAA" +export type DigType = "AAAA" | "CAA" diff --git a/src/types/node-dig-dns/node-dig-dns.d.ts b/src/types/node-dig-dns/node-dig-dns.d.ts deleted file mode 100644 index d14bacbf4..000000000 --- a/src/types/node-dig-dns/node-dig-dns.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module "node-dig-dns" diff --git a/src/types/siteLaunch.ts b/src/types/siteLaunch.ts index 90dc5a9e0..0965e7e8f 100644 --- a/src/types/siteLaunch.ts +++ b/src/types/siteLaunch.ts @@ -40,6 +40,19 @@ export interface SiteLaunchMessage { statusMetadata?: string } +export type SiteLaunchResult = Omit< + SiteLaunchMessage, + | "status" + | "statusMetadata" + | "redirectionDomain" + | "appId" + | "requestorEmail" + | "agencyEmail" +> & { + redirectionDomainSource?: string + redirectionDomainTarget?: string +} + export function isSiteLaunchMessage(obj: unknown): obj is SiteLaunchMessage { if (!obj) { return false