From b3aa87d2013476cc423385d1dd03b2d10d1d7d5c Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Thu, 19 Oct 2023 17:19:09 +0800 Subject: [PATCH 1/6] feat(git clone): add branch --- src/routes/formsgSiteCreation.ts | 1 - src/services/identity/DeploymentsService.ts | 41 +++++++++++++---- src/services/identity/ReposService.ts | 51 +++++++++++++++------ 3 files changed, 69 insertions(+), 24 deletions(-) diff --git a/src/routes/formsgSiteCreation.ts b/src/routes/formsgSiteCreation.ts index 209d17c05..001e9564e 100644 --- a/src/routes/formsgSiteCreation.ts +++ b/src/routes/formsgSiteCreation.ts @@ -16,7 +16,6 @@ import UsersService from "@services/identity/UsersService" import InfraService from "@services/infra/InfraService" import { mailer } from "@services/utilServices/MailClient" -const SITE_CLONE_FORM_KEY = config.get("formSg.siteCloneFormKey") const SITE_CREATE_FORM_KEY = config.get("formSg.siteCreateFormKey") const REQUESTER_EMAIL_FIELD = "Government E-mail" const SITE_NAME_FIELD = "Site Name" diff --git a/src/services/identity/DeploymentsService.ts b/src/services/identity/DeploymentsService.ts index d1fa4fa83..c3c264b02 100644 --- a/src/services/identity/DeploymentsService.ts +++ b/src/services/identity/DeploymentsService.ts @@ -1,4 +1,4 @@ -import { errAsync, okAsync } from "neverthrow" +import { Result, errAsync, okAsync } from "neverthrow" import { ModelStatic } from "sequelize" import { config } from "@config/config" @@ -43,19 +43,28 @@ class DeploymentsService { repoName: string site: Site }): Promise => { - const amplifyResult = await this.createAmplifyAppOnAws(repoName) - if (amplifyResult.isErr()) { - logger.error(`Amplify set up error: ${amplifyResult.error}`) - throw amplifyResult.error + const [ + amplifyStagingResult, + amplifyStagingLiteResult, + ] = await this.createAmplifyAppsOnAws(repoName) + if (amplifyStagingResult.isErr()) { + logger.error(`Amplify set up error: ${amplifyStagingResult.error}`) + throw amplifyStagingResult.error } - const amplifyInfo = amplifyResult.value + + if (amplifyStagingLiteResult.isErr()) { + logger.error(`Amplify set up error: ${amplifyStagingLiteResult.error}`) + throw amplifyStagingLiteResult.error + } + + const amplifyInfo = amplifyStagingLiteResult.value return this.create({ stagingUrl: Brand.fromString( - `https://staging.${amplifyInfo.defaultDomain}` + `https://staging.${amplifyStagingLiteResult.value.defaultDomain}` ), productionUrl: Brand.fromString( - `https://master.${amplifyInfo.defaultDomain}` + `https://master.${amplifyStagingResult.value.defaultDomain}` ), site, siteId: site.id, @@ -63,12 +72,24 @@ class DeploymentsService { }) } - createAmplifyAppOnAws = async (repoName: string) => { + createAmplifyAppsOnAws = async (repoName: string) => { + const stagingApp = await this.createAmplifyAppOnAws(repoName, repoName) + const stagingLiteApp = await this.createAmplifyAppOnAws( + repoName, + `${repoName}-staging-lite` + ) + return [stagingApp, stagingLiteApp] + } + + createAmplifyAppOnAws = async ( + repoName: string, + appName: string + ): Promise> => { const repoUrl = `https://github.com/isomerpages/${repoName}` logger.info(`PublishToAmplify ${repoUrl}`) const createAppOptions = this.deploymentClient.generateCreateAppInput( - repoName, + appName, repoUrl ) // 1. Create Amplify app diff --git a/src/services/identity/ReposService.ts b/src/services/identity/ReposService.ts index 073771da0..04750e006 100644 --- a/src/services/identity/ReposService.ts +++ b/src/services/identity/ReposService.ts @@ -1,4 +1,6 @@ +import { exec } from "child_process" import fs from "fs" +import path from "path" import { retry } from "@octokit/plugin-retry" import { Octokit } from "@octokit/rest" @@ -13,7 +15,11 @@ import { config } from "@config/config" import { UnprocessableError } from "@errors/UnprocessableError" import { Repo, Site } from "@database/models" -import { DNS_INDIRECTION_REPO, EFS_VOL_PATH_STAGING } from "@root/constants" +import { + DNS_INDIRECTION_REPO, + EFS_VOL_PATH_STAGING, + EFS_VOL_PATH_STAGING_LITE, +} from "@root/constants" import GitHubApiError from "@root/errors/GitHubApiError" import logger from "@root/logger/logger" @@ -63,7 +69,11 @@ export default class ReposService { this.simpleGit = simpleGit } - getLocalRepoPath = (repoName: string) => `${EFS_VOL_PATH_STAGING}/${repoName}` + getLocalStagingRepoPath = (repoName: string) => + path.join(EFS_VOL_PATH_STAGING, repoName) + + getLocalStagingLiteRepoPath = (repoName: string) => + path.join(EFS_VOL_PATH_STAGING_LITE, repoName) create = (createParams: repoCreateParamsType): Promise => this.repository.create(createParams) @@ -107,7 +117,7 @@ export default class ReposService { productionUrl: string, stagingUrl: string ) => { - const dir = this.getLocalRepoPath(repoName) + const dir = this.getLocalStagingRepoPath(repoName) // 1. Set URLs in local _config.yml this.setUrlsInLocalConfig(dir, repoName, stagingUrl, productionUrl) @@ -206,45 +216,60 @@ export default class ReposService { repoName: string, repoUrl: string ): Promise => { - const dir = this.getLocalRepoPath(repoName) + const stgDir = this.getLocalStagingRepoPath(repoName) + const stgLiteDir = this.getLocalStagingLiteRepoPath(repoName) // Make sure the local path is empty, just in case dir was used on a previous attempt. - fs.rmSync(`${dir}`, { recursive: true, force: true }) + fs.rmSync(`${stgDir}`, { recursive: true, force: true }) // Clone base repo locally - fs.mkdirSync(dir) + fs.mkdirSync(stgDir) await this.simpleGit - .cwd(dir) - .clone(SITE_CREATION_BASE_REPO_URL, dir, ["-b", "staging"]) + .cwd(stgDir) + .clone(SITE_CREATION_BASE_REPO_URL, stgDir, ["-b", "staging"]) // Clear git - fs.rmSync(`${dir}/.git`, { recursive: true, force: true }) + fs.rmSync(`${stgDir}/.git`, { recursive: true, force: true }) // Prepare git repo await this.simpleGit - .cwd(dir) + .cwd(stgDir) .init(["--initial-branch=staging"]) .checkoutLocalBranch("staging") // Add all the changes - await this.simpleGit.cwd(dir).add(".") + await this.simpleGit.cwd(stgDir).add(".") // Commit await this.simpleGit - .cwd(dir) + .cwd(stgDir) .addConfig("user.name", "isomeradmin") .addConfig("user.email", ISOMER_GITHUB_EMAIL) .commit("Initial commit") // Push to origin await this.simpleGit - .cwd(dir) + .cwd(stgDir) .addRemote("origin", repoUrl) .checkout("staging") .push(["-u", "origin", "staging"]) // push to staging first to make it the default branch on GitHub .checkoutLocalBranch("master") .push(["-u", "origin", "master"]) .checkout("staging") // reset local branch back to staging + + // Create staging lite branch in other repo path + await this.simpleGit + .cwd(stgLiteDir) + .clone(repoUrl, stgLiteDir) + .checkout("staging") + .rm(["-r", "images"]) + .rm(["-r", "files"]) + + // DO NOT use execSync -> it tends to be blocking and causes BE to hang. + // From here, all the resulting commands are appended to avoid this ^ + exec( + `cd ${stgLiteDir} && rm -rf .git && cd ${stgLiteDir} init && git add . && git commit -m "initial commit for staging lite" && git remote add origin ${repoUrl} && git push origin staging-lite:staging-lite -f` + ) } createDnsIndirectionFile = ( From 5ef5f8ee313ea0f96f17200fc63b2886744bb9d0 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Thu, 19 Oct 2023 18:13:20 +0800 Subject: [PATCH 2/6] feat(db): update deployments table --- src/database/models/Deployment.ts | 6 ++++++ src/services/identity/DeploymentsService.ts | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/database/models/Deployment.ts b/src/database/models/Deployment.ts index 928cdc03a..0ff503a94 100644 --- a/src/database/models/Deployment.ts +++ b/src/database/models/Deployment.ts @@ -56,6 +56,12 @@ export class Deployment extends Model { }) hostingId!: string + @Column({ + allowNull: false, + type: DataType.TEXT, + }) + stagingLiteHostingId!: string + @Column({ allowNull: true, type: DataType.TEXT, diff --git a/src/services/identity/DeploymentsService.ts b/src/services/identity/DeploymentsService.ts index c3c264b02..0c7d95fa8 100644 --- a/src/services/identity/DeploymentsService.ts +++ b/src/services/identity/DeploymentsService.ts @@ -57,7 +57,8 @@ class DeploymentsService { throw amplifyStagingLiteResult.error } - const amplifyInfo = amplifyStagingLiteResult.value + const amplifyInfoStaging = amplifyStagingResult.value + const amplifyInfoStagingLite = amplifyStagingLiteResult.value return this.create({ stagingUrl: Brand.fromString( @@ -68,7 +69,8 @@ class DeploymentsService { ), site, siteId: site.id, - hostingId: amplifyInfo.id, + hostingId: amplifyInfoStaging.id, + stagingLiteHostingId: amplifyInfoStagingLite.id, }) } From 21902d4218972abeda09075c26d1ed4074b809f8 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Fri, 20 Oct 2023 08:55:42 +0800 Subject: [PATCH 3/6] fix(reposService): fix git commands --- src/services/identity/ReposService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/identity/ReposService.ts b/src/services/identity/ReposService.ts index 04750e006..0eda5957b 100644 --- a/src/services/identity/ReposService.ts +++ b/src/services/identity/ReposService.ts @@ -268,7 +268,7 @@ export default class ReposService { // DO NOT use execSync -> it tends to be blocking and causes BE to hang. // From here, all the resulting commands are appended to avoid this ^ exec( - `cd ${stgLiteDir} && rm -rf .git && cd ${stgLiteDir} init && git add . && git commit -m "initial commit for staging lite" && git remote add origin ${repoUrl} && git push origin staging-lite:staging-lite -f` + `cd ${stgLiteDir} && rm -rf .git && git init && git checkout -b staging-lite && git add . && git commit -m "initial commit for staging lite" && git remote add origin ${repoUrl} && git push origin staging-lite:staging-lite -f` ) } From 49ee755df3c39a1a8434ecb85bea2dabb0f3c2b5 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Tue, 24 Oct 2023 14:23:17 +0800 Subject: [PATCH 4/6] fix(reposService): cleaner code --- src/services/identity/ReposService.ts | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/services/identity/ReposService.ts b/src/services/identity/ReposService.ts index 0eda5957b..43dee47ad 100644 --- a/src/services/identity/ReposService.ts +++ b/src/services/identity/ReposService.ts @@ -257,6 +257,11 @@ export default class ReposService { .push(["-u", "origin", "master"]) .checkout("staging") // reset local branch back to staging + // Make sure the local path is empty, just in case dir was used on a previous attempt. + fs.rmSync(`${stgLiteDir}`, { recursive: true, force: true }) + // create a empty folder stgLiteDir + fs.mkdirSync(stgLiteDir) + // Create staging lite branch in other repo path await this.simpleGit .cwd(stgLiteDir) @@ -265,11 +270,19 @@ export default class ReposService { .rm(["-r", "images"]) .rm(["-r", "files"]) - // DO NOT use execSync -> it tends to be blocking and causes BE to hang. - // From here, all the resulting commands are appended to avoid this ^ - exec( - `cd ${stgLiteDir} && rm -rf .git && git init && git checkout -b staging-lite && git add . && git commit -m "initial commit for staging lite" && git remote add origin ${repoUrl} && git push origin staging-lite:staging-lite -f` - ) + // Clear git + fs.rmSync(`${stgLiteDir}/.git`, { recursive: true, force: true }) + + // Prepare git repo + + await this.simpleGit + .cwd(stgLiteDir) + .init() + .checkoutLocalBranch("staging-lite") + .add(".") + .commit("Initial commit") + .addRemote("origin", repoUrl) + .push(["origin", "staging-lite", "-f"]) } createDnsIndirectionFile = ( From fc87d484af7ddee00ffaf29c7a73c4dff3d21789 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:25:52 +0800 Subject: [PATCH 5/6] fix(tests): fix failing tests --- src/fixtures/sites.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fixtures/sites.ts b/src/fixtures/sites.ts index 14f5104a0..8f031dc2d 100644 --- a/src/fixtures/sites.ts +++ b/src/fixtures/sites.ts @@ -85,6 +85,7 @@ export const MOCK_DEPLOYMENT_DBENTRY_ONE = { hostingId: "1", encryptionIv: null, encryptedPassword: null, + stagingLiteHostingId: "2", } export const MOCK_DEPLOYMENT_DBENTRY_TWO: Attributes = { @@ -98,6 +99,7 @@ export const MOCK_DEPLOYMENT_DBENTRY_TWO: Attributes = { encryptionIv: "12345678901234561234567890123456", encryptedPassword: "1234567890123456789012345678901234567890123456789012345678901234", + stagingLiteHostingId: "2", } export const MOCK_SITEMEMBER_DBENTRY_ONE: Attributes = { From 9fbbd471bd21ea9251aaffb7e7f4eabfa8794afe Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:36:39 +0800 Subject: [PATCH 6/6] style(deploymentService): better error messages --- src/services/identity/DeploymentsService.ts | 8 ++++++-- src/services/identity/ReposService.ts | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/services/identity/DeploymentsService.ts b/src/services/identity/DeploymentsService.ts index 0c7d95fa8..33a12ae38 100644 --- a/src/services/identity/DeploymentsService.ts +++ b/src/services/identity/DeploymentsService.ts @@ -48,12 +48,16 @@ class DeploymentsService { amplifyStagingLiteResult, ] = await this.createAmplifyAppsOnAws(repoName) if (amplifyStagingResult.isErr()) { - logger.error(`Amplify set up error: ${amplifyStagingResult.error}`) + logger.error( + `Amplify set up error for main app: ${amplifyStagingResult.error}` + ) throw amplifyStagingResult.error } if (amplifyStagingLiteResult.isErr()) { - logger.error(`Amplify set up error: ${amplifyStagingLiteResult.error}`) + logger.error( + `Amplify set up error for staging-lite app: ${amplifyStagingLiteResult.error}` + ) throw amplifyStagingLiteResult.error } diff --git a/src/services/identity/ReposService.ts b/src/services/identity/ReposService.ts index 43dee47ad..1b5372c21 100644 --- a/src/services/identity/ReposService.ts +++ b/src/services/identity/ReposService.ts @@ -274,7 +274,6 @@ export default class ReposService { fs.rmSync(`${stgLiteDir}/.git`, { recursive: true, force: true }) // Prepare git repo - await this.simpleGit .cwd(stgLiteDir) .init()