From 0f22beb5aa41915cd1c137b1519bd43542944572 Mon Sep 17 00:00:00 2001 From: Alexander Lee Date: Mon, 19 Feb 2024 13:47:02 +0800 Subject: [PATCH 1/3] chore: lock repos during repair --- src/routes/formsg/formsgGGsRepair.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/routes/formsg/formsgGGsRepair.ts b/src/routes/formsg/formsgGGsRepair.ts index 2bf41a5e9..c010fbbd2 100644 --- a/src/routes/formsg/formsgGGsRepair.ts +++ b/src/routes/formsg/formsgGGsRepair.ts @@ -10,12 +10,15 @@ import { ResultAsync, errAsync, fromPromise, okAsync } from "neverthrow" import { config } from "@config/config" +import { lock, unlock } from "@utils/mutex-utils" + import { EFS_VOL_PATH_STAGING, EFS_VOL_PATH_STAGING_LITE, } from "@root/constants" import GitFileSystemError from "@root/errors/GitFileSystemError" import InitializationError from "@root/errors/InitializationError" +import LockedError from "@root/errors/LockedError" import { consoleLogger } from "@root/logger/console.logger" import logger from "@root/logger/logger" import { attachFormSGHandler } from "@root/middleware" @@ -114,7 +117,7 @@ export class FormsgGGsRepairRouter { } handleGGsFormSubmission = (repoNames: string[], requesterEmail: string) => { - const repairs: ResultAsync[] = [] + const repairs: ResultAsync[] = [] const clonedStagingRepos: string[] = [] const syncedStagingAndStagingLiteRepos: string[] = [] @@ -122,7 +125,11 @@ export class FormsgGGsRepairRouter { const repoUrl = `git@github.com:isomerpages/${repoName}.git` repairs.push( - this.doesRepoNeedClone(repoName) + ResultAsync.fromPromise( + lock(repoName), + (err) => new LockedError(`Unable to lock repo ${repoName}`) + ) + .andThen(() => this.doesRepoNeedClone(repoName)) .andThen(() => { const isStaging = true return ( @@ -166,6 +173,15 @@ export class FormsgGGsRepairRouter { return okAsync(result) }) ) + .andThen((result) => { + // Failure to unlock is not blocking + ResultAsync.fromPromise(unlock(repoName), () => { + logger.error( + "Failed to unlock repo - repo will unlock after 1 min" + ) + }) + return okAsync(result) + }) ) }) From 436a7b0661a5b984d9a7283a6bb6208bdfe8b4e3 Mon Sep 17 00:00:00 2001 From: Alexander Lee Date: Tue, 20 Feb 2024 13:35:29 +0800 Subject: [PATCH 2/3] chore: lock repo for 15 minutes during repair --- src/routes/formsg/formsgGGsRepair.ts | 5 +++-- src/utils/mutex-utils.js | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/routes/formsg/formsgGGsRepair.ts b/src/routes/formsg/formsgGGsRepair.ts index c010fbbd2..f8a6fce4b 100644 --- a/src/routes/formsg/formsgGGsRepair.ts +++ b/src/routes/formsg/formsgGGsRepair.ts @@ -121,12 +121,13 @@ export class FormsgGGsRepairRouter { const clonedStagingRepos: string[] = [] const syncedStagingAndStagingLiteRepos: string[] = [] + const LOCK_TIME_SECONDS = 15 * 60 // 15 minutes repoNames.forEach((repoName) => { const repoUrl = `git@github.com:isomerpages/${repoName}.git` repairs.push( ResultAsync.fromPromise( - lock(repoName), + lock(repoName, LOCK_TIME_SECONDS), (err) => new LockedError(`Unable to lock repo ${repoName}`) ) .andThen(() => this.doesRepoNeedClone(repoName)) @@ -177,7 +178,7 @@ export class FormsgGGsRepairRouter { // Failure to unlock is not blocking ResultAsync.fromPromise(unlock(repoName), () => { logger.error( - "Failed to unlock repo - repo will unlock after 1 min" + "Failed to unlock repo - repo will unlock after at most 15 min" ) }) return okAsync(result) diff --git a/src/utils/mutex-utils.js b/src/utils/mutex-utils.js index fa623c383..ef24ad6c3 100644 --- a/src/utils/mutex-utils.js +++ b/src/utils/mutex-utils.js @@ -35,10 +35,10 @@ const mockUnlock = (siteName) => { mockMutexObj[siteName] = false } -const lock = async (siteName) => { +const lock = async (siteName, lockLengthSeconds = 60) => { try { const ONE_MIN_FROM_CURR_DATE_IN_SECONDS_FROM_EPOCH_TIME = - Math.floor(new Date().valueOf() / 1000) + 60 + Math.floor(new Date().valueOf() / 1000) + lockLengthSeconds if (isE2eTestRepo(siteName)) return if (!IS_DEV) { From 0cc6344c3cb654f3f7fd5dc3b009b514ccf9aa23 Mon Sep 17 00:00:00 2001 From: Alexander Lee Date: Thu, 22 Feb 2024 11:30:22 +0800 Subject: [PATCH 3/3] chore: change var name --- src/utils/mutex-utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/mutex-utils.js b/src/utils/mutex-utils.js index ef24ad6c3..b0f19805b 100644 --- a/src/utils/mutex-utils.js +++ b/src/utils/mutex-utils.js @@ -37,7 +37,7 @@ const mockUnlock = (siteName) => { const lock = async (siteName, lockLengthSeconds = 60) => { try { - const ONE_MIN_FROM_CURR_DATE_IN_SECONDS_FROM_EPOCH_TIME = + const expiryTime = Math.floor(new Date().valueOf() / 1000) + lockLengthSeconds if (isE2eTestRepo(siteName)) return @@ -46,7 +46,7 @@ const lock = async (siteName, lockLengthSeconds = 60) => { TableName: MUTEX_TABLE_NAME, Item: { repo_id: siteName, - expdate: ONE_MIN_FROM_CURR_DATE_IN_SECONDS_FROM_EPOCH_TIME, + expdate: expiryTime, }, ConditionExpression: "attribute_not_exists(repo_id)", }