Release/0.67.0 (prod)

CHANGELOG.md

@@ -4,10 +4,28 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v0.67.0](https://github.com/isomerpages/isomercms-backend/compare/v0.66.4...v0.67.0)
+
+- fix(start-commands): update commands to be different by env [`#1159`](https://github.com/isomerpages/isomercms-backend/pull/1159)
+- fix(dompurify): further limit src [`#1156`](https://github.com/isomerpages/isomercms-backend/pull/1156)
+- fix(file-ext): prevent users from bypassing checks on file extensions  [`#1157`](https://github.com/isomerpages/isomercms-backend/pull/1157)
+- Chore/lock repos when repairing [`#1149`](https://github.com/isomerpages/isomercms-backend/pull/1149)
+- Fix/sanitise urls [`#1158`](https://github.com/isomerpages/isomercms-backend/pull/1158)
+- feat: add validation for homepage frontmatter [`#1151`](https://github.com/isomerpages/isomercms-backend/pull/1151)
+- build(deps-dev): bump ip from 2.0.0 to 2.0.1 [`#1155`](https://github.com/isomerpages/isomercms-backend/pull/1155)
+- release(0.66.4): merge to dev [`#1154`](https://github.com/isomerpages/isomercms-backend/pull/1154)
+- release(0.66.3): merge to dev [`#1146`](https://github.com/isomerpages/isomercms-backend/pull/1146)
+- Release/0.66.2 [`#1144`](https://github.com/isomerpages/isomercms-backend/pull/1144)
+- release(0.66.1): merge to develop [`#1142`](https://github.com/isomerpages/isomercms-backend/pull/1142)
+- 0.66.0 [`#1140`](https://github.com/isomerpages/isomercms-backend/pull/1140)
+- fix(tsak-def): add env vars [`fdf1230`](https://github.com/isomerpages/isomercms-backend/commit/fdf123001fc5fa6d8feaed0545eacb2670ff4ffb)
+
 #### [v0.66.4](https://github.com/isomerpages/isomercms-backend/compare/v0.66.3...v0.66.4)
 
+> 20 February 2024
+
 - fix(ci): use workflwo [`a5a225d`](https://github.com/isomerpages/isomercms-backend/commit/a5a225dd203826726ad9f8e39c420bdb4a8e0d2c)
-- fix(tsak-def): add env vars [`fdf1230`](https://github.com/isomerpages/isomercms-backend/commit/fdf123001fc5fa6d8feaed0545eacb2670ff4ffb)
+- fix(tsak-def): add env vars [`ad02a2f`](https://github.com/isomerpages/isomercms-backend/commit/ad02a2f630d657b468bb48fd48f0572051ae85e3)
 
 #### [v0.66.3](https://github.com/isomerpages/isomercms-backend/compare/v0.66.2...v0.66.3)
 
@@ -70,12 +88,12 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 - fix: remove unnecessary push logs [`#1109`](https://github.com/isomerpages/isomercms-backend/pull/1109)
 - fix(rr): skip checking the existence of review request [`#1102`](https://github.com/isomerpages/isomercms-backend/pull/1102)
 - release/0.61.0 [`#1104`](https://github.com/isomerpages/isomercms-backend/pull/1104)
+- fix(sl): include issuewild if CAA records are needed [`#1106`](https://github.com/isomerpages/isomercms-backend/pull/1106)
 
 #### [v0.61.0](https://github.com/isomerpages/isomercms-backend/compare/v0.60.0...v0.61.0)
 
-> 11 January 2024
+> 10 January 2024
 
-- fix(sl): include issuewild if CAA records are needed [`#1106`](https://github.com/isomerpages/isomercms-backend/pull/1106)
 - chore: upgrade axios [`#1100`](https://github.com/isomerpages/isomercms-backend/pull/1100)
 - build(deps): bump follow-redirects from 1.15.2 to 1.15.4 [`#1101`](https://github.com/isomerpages/isomercms-backend/pull/1101)
 - fix(ci): reverts ci changes to allow staging updates [`#1084`](https://github.com/isomerpages/isomercms-backend/pull/1084)
@@ -157,12 +175,12 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 - fix(siteCreate): add redirect rules [`#1036`](https://github.com/isomerpages/isomercms-backend/pull/1036)
 - chore: remove extra and unused submodules [`#1031`](https://github.com/isomerpages/isomercms-backend/pull/1031)
 - release/0.54.0 [`#1033`](https://github.com/isomerpages/isomercms-backend/pull/1033)
+- fix: use cTimeMs instead of birthtime due to EFS [`#1035`](https://github.com/isomerpages/isomercms-backend/pull/1035)
 
 #### [v0.54.0](https://github.com/isomerpages/isomercms-backend/compare/v0.53.0...v0.54.0)
 
 > 14 November 2023
 
-- fix: use cTimeMs instead of birthtime due to EFS [`#1035`](https://github.com/isomerpages/isomercms-backend/pull/1035)
 - fix(pagination): total length [`#1032`](https://github.com/isomerpages/isomercms-backend/pull/1032)
 - fix(staging-lite): apps were created for wrong br [`#1014`](https://github.com/isomerpages/isomercms-backend/pull/1014)
 - fix(cm): extra timeout [`#1027`](https://github.com/isomerpages/isomercms-backend/pull/1027)
@@ -289,12 +307,12 @@ Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 - build(deps-dev): bump @babel/traverse from 7.22.8 to 7.23.2 [`#984`](https://github.com/isomerpages/isomercms-backend/pull/984)
 - release/v0.48.0 [`#979`](https://github.com/isomerpages/isomercms-backend/pull/979)
 - feat(staging-id): add column to store the id [`#983`](https://github.com/isomerpages/isomercms-backend/pull/983)
+- Fix: collaborators service tests [`#978`](https://github.com/isomerpages/isomercms-backend/pull/978)
 
 #### [v0.48.0](https://github.com/isomerpages/isomercms-backend/compare/v0.47.0...v0.48.0)
 
 > 18 October 2023
 
-- Fix: collaborators service tests [`#978`](https://github.com/isomerpages/isomercms-backend/pull/978)
 - chore(commitService): proper naming [`#975`](https://github.com/isomerpages/isomercms-backend/pull/975)
 - Feat/is 585 govt sgid login rollout [`#976`](https://github.com/isomerpages/isomercms-backend/pull/976)
 - test(quickie): unit tests  [`#973`](https://github.com/isomerpages/isomercms-backend/pull/973)

Dockerfile

@@ -24,6 +24,7 @@ RUN echo "[user]" > /root/.gitconfig
 RUN echo "  name = Isomer Admin" >> /root/.gitconfig
 RUN echo "  email = [email protected]" >> /root/.gitconfig
 
+RUN chmod +x ./scripts/02_fetch_ssh_keys.sh
 
 EXPOSE "8081"
-CMD ["bash", "-c", "chmod +x ./scripts/02_fetch_ssh_keys.sh && bash ./scripts/02_fetch_ssh_keys.sh & npm run start:ecs"] 
+CMD ["bash", "-c", "bash ./scripts/02_fetch_ssh_keys.sh & npm run start:ecs:$NODE_ENV"] 

package.json

@@ -1,11 +1,13 @@
 {
   "name": "isomercms",
-  "version": "0.66.4",
+  "version": "0.67.0",
   "private": true,
   "scripts": {
     "build": "tsc -p tsconfig.build.json",
     "start": "node --unhandled-rejections=warn -r ts-node/register/transpile-only -r tsconfig-paths/register -r dotenv/config build/server.js dotenv_config_path=/efs/isomer/.isomer.env",
-    "start:ecs": "ts-node-dev --unhandled-rejections=warn --respawn src/server.js",
+    "start:ecs:prod": "node --unhandled-rejections=warn -r ts-node/register/transpile-only -r tsconfig-paths/register src/server.js",
+    "start:ecs:staging": "node --unhandled-rejections=warn -r ts-node/register/transpile-only -r tsconfig-paths/register src/server.js",
+    "start::ecs:dev": "ts-node-dev --respawn src/server.js",
     "dev": "source .env && docker compose -f docker-compose.dev.yml up",
     "test:docker": "docker run -d -p 54321:5432 --name postgres -e POSTGRES_USER=isomer -e POSTGRES_PASSWORD=password -e POSTGRES_DB=isomercms_test postgres:latest",
     "test": "source .env.test && jest --runInBand",
@@ -34,6 +36,7 @@
     "@octokit/rest": "^18.12.0",
     "@opengovsg/formsg-sdk": "^0.11.0",
     "@opengovsg/sgid-client": "^2.0.0",
+    "@types/dompurify": "^3.0.5",
     "auto-bind": "^4.0.0",
     "aws-lambda": "^1.0.7",
     "aws-sdk": "^2.1428.0",
@@ -53,6 +56,7 @@
     "crypto-js": "^4.2.0",
     "dd-trace": "^4.7.0",
     "debug": "~2.6.9",
+    "dompurify": "^3.0.9",
     "dotenv": "^16.3.1",
     "eventsource": "^2.0.2",
     "exponential-backoff": "^3.1.0",
@@ -91,6 +95,7 @@
     "ts-node": "^10.7.0",
     "type-fest": "^2.12.0",
     "umzug": "^3.0.0",
+    "url-template": "^2.0.8",
     "uuid": "^3.3.3",
     "validator": "^13.6.0",
     "winston": "^3.3.3",
@@ -113,6 +118,7 @@
     "@types/mock-fs": "^4.13.1",
     "@types/node": "^17.0.21",
     "@types/supertest": "^2.0.11",
+    "@types/url-template": "^2.0.31",
     "@types/validator": "^13.7.1",
     "@typescript-eslint/eslint-plugin": "^5.17.0",
     "@typescript-eslint/parser": "^5.17.0",

src/constants/constants.ts

@@ -2,24 +2,24 @@
 
 import { config } from "@config/config"
 
 export enum JobStatus {
   Ready = "READY", // Ready to run jobs
   Running = "RUNNING", // A job is running
   Failed = "FAILED", // A job has failed and recovery is needed
 }
 
 export enum SiteStatus {
   Empty = "EMPTY", // A site record site is being initialized
   Initialized = "INITIALIZED",
   Launched = "LAUNCHED",
 }
 
 export enum RedirectionTypes {
   CNAME = "CNAME",
   A = "A",
 }
 
 export enum CollaboratorRoles {
   Admin = "ADMIN",
   Contributor = "CONTRIBUTOR",
   IsomerAdmin = "ISOMERADMIN",
@@ -30,7 +30,7 @@
   CollaboratorRoles.IsomerAdmin
 >
 
 export enum ReviewRequestStatus {
   Approved = "APPROVED",
   Open = "OPEN",
   Merged = "MERGED",
@@ -90,3 +90,9 @@
 export const STAGING_LITE_BRANCH = "staging-lite"
 export const PLACEHOLDER_FILE_NAME = ".keep"
 export const GIT_SYSTEM_DIRECTORY = ".git"
+
+// Homepage blocks limits
+export const MAX_HERO_KEY_HIGHLIGHTS = 4
+export const MAX_ANNOUNCEMENT_ITEMS = 5
+export const MAX_TEXTCARDS_CARDS = 4
+export const MAX_INFOCOLS_BOXES = 4

src/routes/formsg/formsgGGsRepair.ts

@@ -10,12 +10,15 @@ import { ResultAsync, errAsync, fromPromise, okAsync } from "neverthrow"
 
 import { config } from "@config/config"
 
+import { lock, unlock } from "@utils/mutex-utils"
+
 import {
   EFS_VOL_PATH_STAGING,
   EFS_VOL_PATH_STAGING_LITE,
 } from "@root/constants"
 import GitFileSystemError from "@root/errors/GitFileSystemError"
 import InitializationError from "@root/errors/InitializationError"
+import LockedError from "@root/errors/LockedError"
 import { consoleLogger } from "@root/logger/console.logger"
 import logger from "@root/logger/logger"
 import { attachFormSGHandler } from "@root/middleware"
@@ -114,15 +117,20 @@ export class FormsgGGsRepairRouter {
   }
 
   handleGGsFormSubmission = (repoNames: string[], requesterEmail: string) => {
-    const repairs: ResultAsync<string, GitFileSystemError>[] = []
+    const repairs: ResultAsync<string, GitFileSystemError | LockedError>[] = []
 
     const clonedStagingRepos: string[] = []
     const syncedStagingAndStagingLiteRepos: string[] = []
+    const LOCK_TIME_SECONDS = 15 * 60 // 15 minutes
     repoNames.forEach((repoName) => {
       const repoUrl = `[email protected]:isomerpages/${repoName}.git`
 
       repairs.push(
-        this.doesRepoNeedClone(repoName)
+        ResultAsync.fromPromise(
+          lock(repoName, LOCK_TIME_SECONDS),
+          (err) => new LockedError(`Unable to lock repo ${repoName}`)
+        )
+          .andThen(() => this.doesRepoNeedClone(repoName))
           .andThen(() => {
             const isStaging = true
             return (
@@ -166,6 +174,15 @@ export class FormsgGGsRepairRouter {
                 return okAsync(result)
               })
           )
+          .andThen((result) => {
+            // Failure to unlock is not blocking
+            ResultAsync.fromPromise(unlock(repoName), () => {
+              logger.error(
+                "Failed to unlock repo - repo will unlock after at most 15 min"
+              )
+            })
+            return okAsync(result)
+          })
       )
     })
 

src/routes/v2/authenticatedSites/__tests__/Homepage.spec.js

@@ -118,25 +118,17 @@ describe("Homepage Router", () => {
       )
     })
 
-    it("accepts valid homepage update requests with additional unspecified fields and returns the details of the file updated", async () => {
+    it("rejects homepage update requests with additional unspecified fields", async () => {
       const extraUpdateDetails = { ...updatePageDetails }
       // Add extra unspecified field
       extraUpdateDetails.content.frontMatter.extra = ""
-      const expectedServiceInput = {
-        content: updatePageDetails.content.pageBody,
-        frontMatter: updatePageDetails.content.frontMatter,
-        sha: updatePageDetails.sha,
-      }
 
       await request(app)
         .post(`/${siteName}/homepage`)
         .send(extraUpdateDetails)
-        .expect(200)
+        .expect(400)
 
-      expect(mockHomepagePageService.update).toHaveBeenCalledWith(
-        mockUserWithSiteSessionData,
-        expectedServiceInput
-      )
+      expect(mockHomepagePageService.update).not.toHaveBeenCalled()
     })
   })
 })

src/routes/v2/authenticatedSites/homepage.js

@@ -33,9 +33,7 @@ class HomepageRouter {
   async updateHomepage(req, res, next) {
     const { userWithSiteSessionData } = res.locals
 
-    const { error } = UpdateHomepageSchema.validate(req.body, {
-      allowUnknown: true,
-    })
+    const { error } = UpdateHomepageSchema.validate(req.body)
     if (error) throw new BadRequestError(error.message)
     const {
       content: { frontMatter, pageBody },

src/services/api/NetlifyApi.ts

@@ -1,4 +1,5 @@
 import axios from "axios"
+import urlTemplate from "url-template"
 
 import { config } from "@config/config"
 
@@ -62,7 +63,10 @@ const updatePasswordAndStopBuildNetlifySite = async (
   repoId: string,
   password: string
 ) => {
-  const endpoint = `https://api.netlify.com/api/v1/sites/${repoId}`
+  const endpointTemplate = urlTemplate.parse(
+    `https://api.netlify.com/api/v1/sites/{repoId}`
+  )
+  const endpoint = endpointTemplate.expand({ repoId })
   const headers = {
     Authorization: `Bearer ${NETLIFY_ACCESS_TOKEN}`,
     "Content-Type": "application/json",