Merge branch 'feat/identity/database-models' into feat/identity/site-…

…token-middleware * feat/identity/database-models: (25 commits) fix: package.json & package-lock.json to reduce vulnerabilities (#285) fix: package.json & package-lock.json to reduce vulnerabilities (#311) Refactor/collection refactor (#281) build(deps-dev): bump jest from 26.6.3 to 27.0.6 (#222) build(deps): bump tmpl from 1.0.4 to 1.0.5 (#304) build(deps): bump axios from 0.21.1 to 0.21.4 (#295) Fix: change git tree object format (#305) Fix: handle 409 errors when editing files (#303) hotfix: properly handle subcollection file update Fix: prevent truncating of page body (#299) Feat: add ability to edit description in settings (#280) ci: remove .ebextensions from zip ci: update backend staging eb env ci: update production eb env name (#298) ci: include .platform dir in deployment chore: specify node version chore: use EB platform hooks instead of ebextensions chore: add health check endpoint chore: use new cicd user creds fix: update ebextension to fix EB deploy ...
isomerpages · Nov 8, 2021 · 0cfc7b3 · 0cfc7b3
2 parents 0308661 + c5de8b3
commit 0cfc7b3
Show file tree

Hide file tree

Showing 51 changed files with 5,995 additions and 3,260 deletions.
diff --git a/.ebextensions/nginx-max-body-size.config b/.ebextensions/nginx-max-body-size.config
diff --git a/.env-example b/.env-example
@@ -11,14 +11,20 @@ export GITHUB_ORG_NAME="isomerpages"
 export GITHUB_BUILD_ORG_NAME="opengovsg"
 export GITHUB_BUILD_REPO_NAME="isomer-build"
 export MUTEX_TABLE_NAME=""
-export LOCAL_SITE_ACCESS_TOKEN=""
 
 # Required to connect to DynamoDB
 export AWS_ACCESS_KEY_ID=""
 export AWS_SECRET_ACCESS_KEY=""
 
+# Required to run end-to-end tests
+export E2E_TEST_REPO="e2e-test-repo"
+export E2E_TEST_SECRET=""
+export E2E_TEST_GH_TOKEN=""
+
 # Database
 export DB_URI="postgres://isomer:password@localhost:15432/isomercms_dev"
 export DB_MIN_POOL="1"
 export DB_MAX_POOL="10"
-export DB_ENABLE_LOGGING=""
+export DB_ENABLE_LOGGING=""
+
+export LOCAL_SITE_ACCESS_TOKEN=""
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -78,7 +78,6 @@
                 ["@loaders", "./loaders"],
                 ["@database", "./database"],
                 ["@services", "./services"],
-                ["@controllers", "./controllers"],
                 ["@validators", "./validators"]
             ]
         }

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,11 +6,9 @@ on:
 env:
   PRODUCTION_BRANCH: refs/heads/master
   STAGING_BRANCH: refs/heads/staging
-  DEV_BRANCH: refs/heads/staging-dev
   EB_APP: isomer-cms
-  EB_ENV_PRODUCTION: isomercms-backend-prod
-  EB_ENV_STAGING: isomercms-backend-staging
-  EB_ENV_DEV: isomercms-backend-staging-dev
+  EB_ENV_PRODUCTION: cms-backend-prod
+  EB_ENV_STAGING: cms-backend-staging
   COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
 jobs:
   gatekeep:
@@ -27,8 +25,7 @@ jobs:
           ref = os.environ['GITHUB_REF']
           prod = os.environ['PRODUCTION_BRANCH']
           staging = os.environ['STAGING_BRANCH']
-          dev = os.environ['DEV_BRANCH']
-          if ref == prod or ref == staging or ref == dev:
+          if ref == prod or ref == staging:
             print('::set-output name=proceed::true')
           else:
             print('::set-output name=proceed::false')
@@ -55,7 +52,7 @@ jobs:
         - name: Install NPM
           run: npm ci
         - name: Zip application
-          run: zip -r "deploy.zip" * .ebextensions -x .env-example .gitignore package-lock.json
+          run: zip -r "deploy.zip" * .platform -x .env-example .gitignore package-lock.json
         - name: Get timestamp
           shell: bash
           run: echo "##[set-output name=timestamp;]$(env TZ=Asia/Singapore date '+%Y%m%d%H%M%S')"
@@ -81,26 +78,21 @@ jobs:
             branch = os.environ['GITHUB_REF']
             production = os.environ['PRODUCTION_BRANCH']
             staging = os.environ['STAGING_BRANCH']
-            dev = os.environ['DEV_BRANCH']
             eb_app = os.environ['EB_APP']
             eb_env_production = os.environ['EB_ENV_PRODUCTION']
             eb_env_staging = os.environ['EB_ENV_STAGING']
-            eb_env_dev = os.environ['EB_ENV_DEV']
             if branch == production:
               print('::set-output name=eb_app::' + eb_app)
               print('::set-output name=eb_env::' + eb_env_production)
             elif branch == staging:
               print('::set-output name=eb_app::' + eb_app)
               print('::set-output name=eb_env::' + eb_env_staging)
-            elif branch == dev:
-              print('::set-output name=eb_app::' + eb_app)
-              print('::set-output name=eb_env::' + eb_env_dev)
           id: select_eb_vars
         - name: Deploy to EB
           uses: opengovsg/beanstalk-deploy@v11
           with:
-            aws_access_key: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_DEPLOYMENT }}
-            aws_secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_DEPLOYMENT }}
+            aws_access_key: ${{ secrets.AWS_ACCESS_KEY_ID_FOR_CICD }}
+            aws_secret_key: ${{ secrets.AWS_SECRET_ACCESS_KEY_FOR_CICD }}
             application_name: ${{ steps.select_eb_vars.outputs.eb_app }}
             environment_name: ${{ steps.select_eb_vars.outputs.eb_env }}
             version_description: ${{ steps.get_desc.output.desc }}

diff --git a/.platform/nginx/conf.d/proxy.conf b/.platform/nginx/conf.d/proxy.conf
@@ -0,0 +1 @@
+client_max_body_size  10M;
diff --git a/README b/README
@@ -0,0 +1,9 @@
+## E2E Tests
+To run the E2E tests successfully, you will need to define the following environment variables:
+```
+export E2E_TEST_REPO="e2e-test-repo"
+export E2E_TEST_SECRET="blahblahblah" // this should match the value of CYPRESS_COOKIE_VALUE on 
+// the frontend
+export E2E_TEST_GH_TOKEN="" // this can be your own personal GH access token, or  the token from our 
+// specialized E2E test user
+```
diff --git a/classes/Collection.js b/classes/Collection.js
@@ -89,10 +89,24 @@ class Collection {
 
   async delete(collectionName, currentCommitSha, treeSha) {
     const commitMessage = `Delete collection ${collectionName}`
-    const gitTree = await getTree(this.siteName, this.accessToken, treeSha)
-    const newGitTree = gitTree.filter((item) => item.path !== `_${collectionName}`)
+    const gitTree = await getTree(
+      this.siteName,
+      this.accessToken,
+      treeSha,
+      true
+    )
+    const newGitTree = gitTree
+      .filter(
+        (item) =>
+          item.type !== "tree" && item.path.startsWith(`_${collectionName}/`)
+      )
+      .map((item) => ({
+        ...item,
+        sha: null,
+      }))
     await sendTree(
       newGitTree,
+      treeSha,
       currentCommitSha,
       this.siteName,
       this.accessToken,
@@ -125,20 +139,36 @@ class Collection {
   ) {
     const commitMessage = `Rename collection from ${oldCollectionName} to ${newCollectionName}`
 
-    const gitTree = await getTree(this.siteName, this.accessToken, treeSha)
+    const gitTree = await getTree(
+      this.siteName,
+      this.accessToken,
+      treeSha,
+      true
+    )
     const oldCollectionDirectoryName = `_${oldCollectionName}`
     const newCollectionDirectoryName = `_${newCollectionName}`
-    const newGitTree = gitTree.map((item) => {
-      if (item.path === oldCollectionDirectoryName) {
-        return {
+    const newGitTree = []
+    gitTree.forEach((item) => {
+      if (item.path === oldCollectionDirectoryName && item.type === "tree") {
+        // Rename old subdirectory to new name
+        newGitTree.push({
           ...item,
           path: newCollectionDirectoryName,
-        }
+        })
+      } else if (
+        item.path.startsWith(`${oldCollectionDirectoryName}/`) &&
+        item.type !== "tree"
+      ) {
+        // Delete old subdirectory items
+        newGitTree.push({
+          ...item,
+          sha: null,
+        })
       }
-      return item
     })
     await sendTree(
       newGitTree,
+      treeSha,
       currentCommitSha,
       this.siteName,
       this.accessToken,

diff --git a/classes/File.js b/classes/File.js
@@ -131,6 +131,10 @@ class File {
     } catch (err) {
       const { status } = err.response
       if (status === 404) throw new NotFoundError("File does not exist")
+      if (status === 409)
+        throw new ConflictError(
+          "File has been changed recently, please try again"
+        )
       throw err
     }
   }
@@ -155,6 +159,10 @@ class File {
     } catch (err) {
       const { status } = err.response
       if (status === 404) throw new NotFoundError("File does not exist")
+      if (status === 409)
+        throw new ConflictError(
+          "File has been changed recently, please try again"
+        )
       throw err
     }
   }

diff --git a/classes/MediaFile.js b/classes/MediaFile.js
@@ -157,14 +157,24 @@ class MediaFile {
       sha,
     }
 
-    const resp = await axios.put(endpoint, params, {
-      headers: {
-        Authorization: `token ${this.accessToken}`,
-        "Content-Type": "application/json",
-      },
-    })
+    try {
+      const resp = await axios.put(endpoint, params, {
+        headers: {
+          Authorization: `token ${this.accessToken}`,
+          "Content-Type": "application/json",
+        },
+      })
 
-    return { newSha: resp.data.commit.sha }
+      return { newSha: resp.data.commit.sha }
+    } catch (err) {
+      const { status } = err.response
+      if (status === 404) throw new NotFoundError("File does not exist")
+      if (status === 409)
+        throw new ConflictError(
+          "File has been changed recently, please try again"
+        )
+      throw err
+    }
   }
 
   async delete(fileName, sha) {
@@ -176,13 +186,23 @@ class MediaFile {
       sha,
     }
 
-    await axios.delete(endpoint, {
-      data: params,
-      headers: {
-        Authorization: `token ${this.accessToken}`,
-        "Content-Type": "application/json",
-      },
-    })
+    try {
+      await axios.delete(endpoint, {
+        data: params,
+        headers: {
+          Authorization: `token ${this.accessToken}`,
+          "Content-Type": "application/json",
+        },
+      })
+    } catch (err) {
+      const { status } = err.response
+      if (status === 404) throw new NotFoundError("File does not exist")
+      if (status === 409)
+        throw new ConflictError(
+          "File has been changed recently, please try again"
+        )
+      throw err
+    }
   }
 }
 

diff --git a/classes/MediaSubfolder.js b/classes/MediaSubfolder.js
@@ -36,17 +36,17 @@ class MediaSubfolder {
     )
     const directoryName = `${this.mediaFolderName}/${subfolderPath}`
     const newGitTree = gitTree
-      .filter((item) => {
-        return !item.path.includes(directoryName)
-      })
-      .filter((item) => {
-        return !(
-          item.type === "tree" && item.path.includes(this.mediaFolderName)
-        )
-      })
-
+      .filter(
+        (item) =>
+          item.type !== "tree" && item.path.startsWith(`${directoryName}/`)
+      )
+      .map((item) => ({
+        ...item,
+        sha: null,
+      }))
     await sendTree(
       newGitTree,
+      treeSha,
       currentCommitSha,
       this.siteName,
       this.accessToken,
@@ -67,24 +67,26 @@ class MediaSubfolder {
     const newDirectoryName = `${this.mediaFolderName}/${newSubfolderPath}`
     const newGitTree = []
     gitTree.forEach((item) => {
-      if (item.path === oldDirectoryName) {
+      if (item.path === oldDirectoryName && item.type === "tree") {
+        // Rename old subdirectory to new name
         newGitTree.push({
           ...item,
           path: newDirectoryName,
         })
-      } else if (item.path.includes(oldDirectoryName)) {
-        // We don't want to include these because they use the old path, they are included with the renamed tree
       } else if (
-        item.type === "tree" &&
-        item.path.includes(this.mediaFolderName)
+        item.path.startsWith(`${oldDirectoryName}/`) &&
+        item.type !== "tree"
       ) {
-        // We don't include any other trees - we reconstruct them by adding all their individual files instead
-      } else {
-        newGitTree.push(item)
+        // Delete old subdirectory items
+        newGitTree.push({
+          ...item,
+          sha: null,
+        })
       }
     })
     await sendTree(
       newGitTree,
+      treeSha,
       currentCommitSha,
       this.siteName,
       this.accessToken,

diff --git a/classes/Resource.js b/classes/Resource.js
@@ -59,38 +59,38 @@ class Resource {
       this.siteName,
       this.accessToken
     )
-    const gitTree = await getTree(this.siteName, this.accessToken, treeSha)
-    const newGitTree = []
-    let resourceRoomTreeSha
-    // Retrieve all git trees of other items
-    gitTree.forEach((item) => {
-      if (item.path === resourceRoomName) {
-        resourceRoomTreeSha = item.sha
-      } else {
-        newGitTree.push(item)
-      }
-    })
-    const resourceRoomTree = await getTree(
+    const gitTree = await getTree(
       this.siteName,
       this.accessToken,
-      resourceRoomTreeSha
+      treeSha,
+      true
     )
-    resourceRoomTree.forEach((item) => {
+    const newGitTree = []
+    gitTree.forEach((item) => {
       // We need to append resource room to the file path because the path is relative to the subtree
-      if (item.path === resourceName) {
+      if (
+        item.path === `${resourceRoomName}/${resourceName}` &&
+        item.type === "tree"
+      ) {
+        // Rename old subdirectory to new name
         newGitTree.push({
           ...item,
           path: `${resourceRoomName}/${newResourceName}`,
         })
-      } else {
+      } else if (
+        item.path.startsWith(`${resourceRoomName}/${resourceName}/`) &&
+        item.type !== "tree"
+      ) {
+        // Delete old subdirectory items
         newGitTree.push({
           ...item,
-          path: `${resourceRoomName}/${item.path}`,
+          sha: null,
         })
       }
     })
     await sendTree(
       newGitTree,
+      treeSha,
       currentCommitSha,
       this.siteName,
       this.accessToken,