From b768fea74e9a496fefc237e7af1626b24ff88d46 Mon Sep 17 00:00:00 2001 From: Kishore <42832651+kishore03109@users.noreply.github.com> Date: Wed, 24 May 2023 14:19:16 +0800 Subject: [PATCH] feat(dynamo db): integrate with dynamo db (#770) * feat(step functions service): add step functions service * feat(dynamo cb client): add get all items function * feat(config): add env var for step functions * feat(dynamodb): integration with infra service * test(dynamodb service): add test cases * fix(infra service): fix wrong feature flag * style(infra service): use env var rather than inline const * feat(dynamodb service): add type guard * fix(typeguard): fix type guard * style(env var): change naming * fix(typeguard): check for undefined values * style(env var): better naming * fix(config): give default value for step functions arn * style(dynamoDb service): add args to constructor for ease of testing * fix(infra service): add awaits * fix(step functions): region should come from env vars * fix(infra service): handle error appropriately --- microservices/site-launch/shared/types.ts | 41 +++++++ src/config/config.ts | 28 +++-- src/server.js | 13 ++- src/services/infra/DynamoDBClient.ts | 43 ++------ src/services/infra/DynamoDBService.ts | 88 ++++++--------- src/services/infra/InfraService.ts | 89 ++++++++++------ src/services/infra/StepFunctionsService.ts | 24 +++++ .../infra/__tests__/DynamoDBService.spec.ts | 100 ++++++++++++++++++ 8 files changed, 292 insertions(+), 134 deletions(-) create mode 100644 src/services/infra/StepFunctionsService.ts create mode 100644 src/services/infra/__tests__/DynamoDBService.spec.ts diff --git a/microservices/site-launch/shared/types.ts b/microservices/site-launch/shared/types.ts index 4582c8594..c98c434f5 100644 --- a/microservices/site-launch/shared/types.ts +++ b/microservices/site-launch/shared/types.ts @@ -38,3 +38,44 @@ export interface SiteLaunchMessage { status?: SiteLaunchStatus statusMetadata?: string } + +export function isSiteLaunchMessage(obj: unknown): obj is SiteLaunchMessage { + if (!obj) { + return false + } + + const message = obj as SiteLaunchMessage + + return ( + typeof message.repoName === "string" && + typeof message.appId === "string" && + typeof message.primaryDomainSource === "string" && + typeof message.primaryDomainTarget === "string" && + typeof message.domainValidationSource === "string" && + typeof message.domainValidationTarget === "string" && + typeof message.requestorEmail === "string" && + typeof message.agencyEmail === "string" && + (typeof message.githubRedirectionUrl === "undefined" || + typeof message.githubRedirectionUrl === "string") && + (typeof message.redirectionDomain === "undefined" || + (Array.isArray(message.redirectionDomain) && + message.redirectionDomain.every( + (rd) => + typeof rd.source === "string" && + typeof rd.target === "string" && + typeof rd.type === "string" + ))) && + (typeof message.status === "undefined" || + (typeof message.status === "object" && + typeof message.status.state === "string" && + (message.status.state === "success" || + message.status.state === "failure" || + message.status.state === "pending") && + typeof message.status.message === "string" && + Object.keys(SiteLaunchLambdaStatus).includes( + message.status.message as SiteLaunchLambdaStatus + ))) && + (typeof message.statusMetadata === "undefined" || + typeof message.statusMetadata === "string") + ) +} diff --git a/src/config/config.ts b/src/config/config.ts index 33ead7f38..264a9864e 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -137,13 +137,13 @@ const config = convict({ }, }, aws: { + region: { + doc: "AWS region", + env: "AWS_REGION", + format: "required-string", + default: "ap-southeast-1", + }, amplify: { - region: { - doc: "AWS region", - env: "AWS_REGION", - format: "required-string", - default: "ap-southeast-1", - }, accountNumber: { doc: "AWS account number (microservices)", env: "AWS_ACCOUNT_NUMBER", @@ -180,6 +180,14 @@ const config = convict({ default: "site-launch", }, }, + stepFunctions: { + stepFunctionsArn: { + doc: "Amazon Resource Name (ARN) of the Step Functions state machine", + env: "STEP_FUNCTIONS_ARN", + format: "required-string", + default: "SiteLaunchStepFunctions-dev", + }, + }, sqs: { incomingQueueUrl: { doc: "URL of the incoming SQS queue", @@ -193,6 +201,14 @@ const config = convict({ format: "required-string", default: "", }, + featureFlags: { + shouldDeprecateSiteQueues: { + doc: "Whether the queues are deprecated", + env: "FF_DEPRECATE_SITE_QUEUES", + format: "required-boolean", + default: false, + }, + }, }, }, github: { diff --git a/src/server.js b/src/server.js index 4d8ae5db8..baca6a64c 100644 --- a/src/server.js +++ b/src/server.js @@ -43,6 +43,7 @@ import { SubcollectionPageService } from "@root/services/fileServices/MdPageServ import { UnlinkedPageService } from "@root/services/fileServices/MdPageServices/UnlinkedPageService" import { CollectionYmlService } from "@root/services/fileServices/YmlFileServices/CollectionYmlService" import { FooterYmlService } from "@root/services/fileServices/YmlFileServices/FooterYmlService" +import DynamoDBService from "@root/services/infra/DynamoDBService" import { isomerRepoAxiosInstance } from "@services/api/AxiosInstance" import { ResourceRoomDirectoryService } from "@services/directoryServices/ResourceRoomDirectoryService" import { ConfigYmlService } from "@services/fileServices/YmlFileServices/ConfigYmlService" @@ -58,6 +59,7 @@ import ReposService from "@services/identity/ReposService" import SitesService from "@services/identity/SitesService" import InfraService from "@services/infra/InfraService" import { statsService } from "@services/infra/StatsService" +import StepFunctionsService from "@services/infra/StepFunctionsService" import ReviewRequestService from "@services/review/ReviewRequestService" import { apiLogger } from "./middleware/apiLogger" @@ -71,6 +73,7 @@ import { PageService } from "./services/fileServices/MdPageServices/PageService" import CollaboratorsService from "./services/identity/CollaboratorsService" import LaunchClient from "./services/identity/LaunchClient" import LaunchesService from "./services/identity/LaunchesService" +import DynamoDBDocClient from "./services/infra/DynamoDBClient" import { rateLimiter } from "./services/utilServices/RateLimiter" import { isSecure } from "./utils/auth-utils" @@ -205,6 +208,10 @@ const launchesService = new LaunchesService({ launchClient, }) const queueService = new QueueService() +const stepFunctionsService = new StepFunctionsService() +const dynamoDBService = new DynamoDBService({ + dynamoDBClient: new DynamoDBDocClient(), +}) const identityAuthService = getIdentityAuthService(gitHubService) const collaboratorsService = new CollaboratorsService({ @@ -222,9 +229,11 @@ const infraService = new InfraService({ launchesService, queueService, collaboratorsService, + stepFunctionsService, + dynamoDBService, }) -// poller for incoming queue -infraService.pollQueue() +// poller site launch updates +infraService.pollMessages() const authenticationMiddleware = getAuthenticationMiddleware() const authorizationMiddleware = getAuthorizationMiddleware({ diff --git a/src/services/infra/DynamoDBClient.ts b/src/services/infra/DynamoDBClient.ts index 8cda27fe1..be1f67279 100644 --- a/src/services/infra/DynamoDBClient.ts +++ b/src/services/infra/DynamoDBClient.ts @@ -1,15 +1,11 @@ -import { DynamoDBClient } from "@aws-sdk/client-dynamodb" +import { DynamoDBClient, ScanCommandOutput } from "@aws-sdk/client-dynamodb" import { DynamoDBDocumentClient, PutCommand, - GetCommand, - UpdateCommand, DeleteCommand, - UpdateCommandInput, PutCommandOutput, - UpdateCommandOutput, DeleteCommandOutput, - GetCommandOutput, + ScanCommand, } from "@aws-sdk/lib-dynamodb" import autoBind from "auto-bind" @@ -43,7 +39,7 @@ export default class DynamoDBDocClient { } this.dynamoDBDocClient = DynamoDBDocumentClient.from( - new DynamoDBClient({ region: config.get("aws.amplify.region") }), + new DynamoDBClient({ region: config.get("aws.region") }), { marshallOptions, unmarshallOptions } ) @@ -52,7 +48,7 @@ export default class DynamoDBDocClient { private withLogger = async ( promise: Promise, - type: "create" | "get" | "delete" | "update" + type: "create" | "scan" | "delete" | "update" ): Promise => { try { return await promise @@ -82,38 +78,13 @@ export default class DynamoDBDocClient { ) } - getItem = async ( - tableName: string, - key: string - ): Promise => { + getAllItems = async (tableName: string): Promise => { const params = { TableName: tableName, - Key: { appId: key }, - } - - return this.withLogger( - this.dynamoDBDocClient.send(new GetCommand(params)), - "get" - ) - } - - updateItem = async ({ - TableName, - Key, - UpdateExpression, - ExpressionAttributeNames, - ExpressionAttributeValues, - }: UpdateParams): Promise => { - const params: UpdateCommandInput = { - TableName, - Key, - UpdateExpression, - ExpressionAttributeNames, - ExpressionAttributeValues, } return this.withLogger( - this.dynamoDBDocClient.send(new UpdateCommand(params)), - "update" + this.dynamoDBDocClient.send(new ScanCommand(params)), + "scan" ) } diff --git a/src/services/infra/DynamoDBService.ts b/src/services/infra/DynamoDBService.ts index eb93ac9a8..bc766a059 100644 --- a/src/services/infra/DynamoDBService.ts +++ b/src/services/infra/DynamoDBService.ts @@ -1,82 +1,56 @@ -import { - DeleteCommandOutput, - GetCommandOutput, - UpdateCommandOutput, -} from "@aws-sdk/lib-dynamodb" +import { AttributeValue } from "@aws-sdk/client-dynamodb" +import { DeleteCommandOutput } from "@aws-sdk/lib-dynamodb" import autoBind from "auto-bind" import { config } from "@config/config" -import { SiteLaunchMessage } from "@root/../microservices/site-launch/shared/types" +import { + SiteLaunchMessage, + isSiteLaunchMessage, +} from "@root/../microservices/site-launch/shared/types" -import DynamoDBClient, { UpdateParams } from "./DynamoDBClient" +import DynamoDBClient from "./DynamoDBClient" -const MOCK_LAUNCH: SiteLaunchMessage = { - repoName: "my-repo", - appId: "my-app", - primaryDomainSource: "example.com", - primaryDomainTarget: "myapp.example.com", - domainValidationSource: "example.com", - domainValidationTarget: "myapp.example.com", - requestorEmail: "john@example.com", - agencyEmail: "agency@example.com", - githubRedirectionUrl: "https://github.com/my-repo", - redirectionDomain: [ - { - source: "example.com", - target: "myapp.example.com", - type: "A", - }, - ], - status: { state: "pending", message: "PENDING_DURING_SITE_LAUNCH" }, -} export default class DynamoDBService { private readonly dynamoDBClient: DynamoDBClient private readonly TABLE_NAME: string - constructor() { - this.dynamoDBClient = new DynamoDBClient() - this.TABLE_NAME = config.get("aws.dynamodb.siteLaunchTableName") + constructor({ + dynamoDBClient, + dynamoDbTableName = config.get("aws.dynamodb.siteLaunchTableName"), + }: { + dynamoDBClient: DynamoDBClient + dynamoDbTableName?: string + }) { + this.dynamoDBClient = dynamoDBClient + this.TABLE_NAME = dynamoDbTableName autoBind(this) } async createItem(message: SiteLaunchMessage): Promise { - await this.dynamoDBClient.createItem(this.TABLE_NAME, MOCK_LAUNCH) + await this.dynamoDBClient.createItem(this.TABLE_NAME, message) } - async getItem(message: SiteLaunchMessage): Promise { - return this.dynamoDBClient.getItem(this.TABLE_NAME, MOCK_LAUNCH.appId) - } + async getAllCompletedLaunches(): Promise { + const entries = (( + await this.dynamoDBClient.getAllItems(this.TABLE_NAME) + ).Items?.filter(isSiteLaunchMessage) as unknown) as SiteLaunchMessage[] + + const completedEntries = + entries?.filter( + (entry) => + entry.status?.state === "success" || entry.status?.state === "failure" + ) || [] - async updateItem(message: SiteLaunchMessage): Promise { - // TODO: delete mocking after integration in IS-186 - MOCK_LAUNCH.status = { state: "success", message: "SUCCESS_SITE_LIVE" } - const updateParams: UpdateParams = { - TableName: this.TABLE_NAME, - Key: { appId: MOCK_LAUNCH.appId }, - // The update expression to apply to the item, - // in this case setting the "status" attribute to a value - UpdateExpression: - "set #status.#state = :state, #status.#message = :message", - ExpressionAttributeNames: { - "#status": "status", - "#state": "state", - "#message": "message", - }, - // A map of expression attribute values used in the update expression, - // in this case mapping ":state" to the value of the Launch status state and ":message" to the value of the Launch status message - ExpressionAttributeValues: { - ":state": MOCK_LAUNCH.status.state, - ":message": MOCK_LAUNCH.status.message, - }, - } - return this.dynamoDBClient.updateItem(updateParams) + // Delete after retrieving the items + Promise.all(completedEntries.map((entry) => this.deleteItem(entry))) + return completedEntries } async deleteItem(message: SiteLaunchMessage): Promise { return this.dynamoDBClient.deleteItem(this.TABLE_NAME, { - appId: MOCK_LAUNCH.appId, + appId: message.appId, }) } } diff --git a/src/services/infra/InfraService.ts b/src/services/infra/InfraService.ts index 1eba0431f..05692ddab 100644 --- a/src/services/infra/InfraService.ts +++ b/src/services/infra/InfraService.ts @@ -6,10 +6,7 @@ import { config } from "@config/config" import { Site } from "@database/models" import { User } from "@database/models/User" -import { - SiteLaunchMessage, - SiteLaunchLambdaStatus, -} from "@root/../microservices/site-launch/shared/types" +import { SiteLaunchMessage } from "@root/../microservices/site-launch/shared/types" import { SiteStatus, JobStatus, RedirectionTypes } from "@root/constants" import logger from "@root/logger/logger" import { AmplifyError } from "@root/types/amplify" @@ -25,6 +22,9 @@ import { mailer } from "@services/utilServices/MailClient" import CollaboratorsService from "../identity/CollaboratorsService" import QueueService from "../identity/QueueService" +import DynamoDBService from "./DynamoDBService" +import StepFunctionsService from "./StepFunctionsService" + const SITE_LAUNCH_UPDATE_INTERVAL = 30000 export const REDIRECTION_SERVER_IP = "18.136.36.203" @@ -35,6 +35,8 @@ interface InfraServiceProps { launchesService: LaunchesService queueService: QueueService collaboratorsService: CollaboratorsService + stepFunctionsService: StepFunctionsService + dynamoDBService: DynamoDBService } interface dnsRecordDto { @@ -52,6 +54,9 @@ type CreateSiteParams = { isEmailLogin: boolean } +const DEPRECATE_SITE_QUEUES = config.get( + "aws.sqs.featureFlags.shouldDeprecateSiteQueues" +) export default class InfraService { private readonly sitesService: InfraServiceProps["sitesService"] @@ -65,6 +70,10 @@ export default class InfraService { private readonly collaboratorsService: InfraServiceProps["collaboratorsService"] + private readonly stepFunctionsService: InfraServiceProps["stepFunctionsService"] + + private readonly dynamoDBService: InfraServiceProps["dynamoDBService"] + constructor({ sitesService, reposService, @@ -72,6 +81,8 @@ export default class InfraService { launchesService, queueService, collaboratorsService, + stepFunctionsService, + dynamoDBService, }: InfraServiceProps) { this.sitesService = sitesService this.reposService = reposService @@ -79,6 +90,8 @@ export default class InfraService { this.launchesService = launchesService this.queueService = queueService this.collaboratorsService = collaboratorsService + this.stepFunctionsService = stepFunctionsService + this.dynamoDBService = dynamoDBService } createSite = async ({ @@ -373,8 +386,12 @@ export default class InfraService { message.redirectionDomain = [redirectionDomainObject] } - this.queueService.sendMessage(message) - + if (DEPRECATE_SITE_QUEUES) { + await this.dynamoDBService.createItem(message) + await this.stepFunctionsService.triggerFlow(message) + } else { + await this.queueService.sendMessage(message) + } return okAsync(newLaunchParams) } catch (error) { return errAsync( @@ -387,39 +404,45 @@ export default class InfraService { } siteUpdate = async () => { - const messages = await this.queueService.pollMessages() - await Promise.all( - messages.map(async (message) => { - const site = await this.sitesService.getBySiteName(message.repoName) - if (site.isErr()) { - return - } - const isSuccess = message.status?.state === "success" - - let updateSiteLaunchParams - - if (isSuccess) { - updateSiteLaunchParams = { - id: site.value.id, - siteStatus: SiteStatus.Launched, - jobStatus: JobStatus.Running, + try { + const messages = DEPRECATE_SITE_QUEUES + ? await this.dynamoDBService.getAllCompletedLaunches() + : await this.queueService.pollMessages() + await Promise.all( + messages.map(async (message) => { + const site = await this.sitesService.getBySiteName(message.repoName) + if (site.isErr()) { + return } - } else { - updateSiteLaunchParams = { - id: site.value.id, - siteStatus: SiteStatus.Initialized, - jobStatus: JobStatus.Failed, + const isSuccess = message.status?.state === "success" + + let updateSiteLaunchParams + + if (isSuccess) { + updateSiteLaunchParams = { + id: site.value.id, + siteStatus: SiteStatus.Launched, + jobStatus: JobStatus.Running, + } + } else { + updateSiteLaunchParams = { + id: site.value.id, + siteStatus: SiteStatus.Initialized, + jobStatus: JobStatus.Failed, + } } - } - await this.sitesService.update(updateSiteLaunchParams) + await this.sitesService.update(updateSiteLaunchParams) - await this.sendEmailUpdate(message, isSuccess) - }) - ) + await this.sendEmailUpdate(message, isSuccess) + }) + ) + } catch (error) { + logger.error(`Error in site update: ${error}`) + } } - pollQueue = async () => { + pollMessages = async () => { setInterval(this.siteUpdate, SITE_LAUNCH_UPDATE_INTERVAL) } diff --git a/src/services/infra/StepFunctionsService.ts b/src/services/infra/StepFunctionsService.ts new file mode 100644 index 000000000..2a4ce3c15 --- /dev/null +++ b/src/services/infra/StepFunctionsService.ts @@ -0,0 +1,24 @@ +import { StepFunctions } from "aws-sdk" + +import config from "@root/config/config" + +import { SiteLaunchMessage } from "../../../microservices/site-launch/shared/types" + +export default class StepFunctionsService { + private client: StepFunctions + + constructor(private stateMachineArn: string) { + this.client = new StepFunctions({ region: config.get("aws.region") }) + } + + async triggerFlow( + message: SiteLaunchMessage + ): Promise { + const params: StepFunctions.StartExecutionInput = { + stateMachineArn: this.stateMachineArn, + input: JSON.stringify(message), + } + const response = await this.client.startExecution(params).promise() + return response + } +} diff --git a/src/services/infra/__tests__/DynamoDBService.spec.ts b/src/services/infra/__tests__/DynamoDBService.spec.ts new file mode 100644 index 000000000..b23297975 --- /dev/null +++ b/src/services/infra/__tests__/DynamoDBService.spec.ts @@ -0,0 +1,100 @@ +import { ScanCommandOutput } from "@aws-sdk/lib-dynamodb" + +import { SiteLaunchMessage } from "@root/../microservices/site-launch/shared/types" +import DynamoDBClient from "@services/infra/DynamoDBClient" +import DynamoDBService from "@services/infra/DynamoDBService" + +jest.mock("@services/infra/DynamoDBClient") + +const mockLaunch: SiteLaunchMessage = { + repoName: "my-repo", + appId: "my-app", + primaryDomainSource: "example.com", + primaryDomainTarget: "myapp.example.com", + domainValidationSource: "example.com", + domainValidationTarget: "myapp.example.com", + requestorEmail: "john@example.com", + agencyEmail: "agency@example.com", + githubRedirectionUrl: "https://github.com/my-repo", + redirectionDomain: [ + { + source: "example.com", + target: "myapp.example.com", + type: "A", + }, + ], + status: { state: "pending", message: "PENDING_DURING_SITE_LAUNCH" }, +} + +const mockSuccessLaunch: SiteLaunchMessage = { + ...mockLaunch, + appId: "success-app-id", + status: { state: "success", message: "SUCCESS_SITE_LIVE" }, +} + +const mockFailureLaunch: SiteLaunchMessage = { + ...mockLaunch, + appId: "failure-app-id", + status: { + state: "failure", + message: "FAILURE_WRONG_CLOUDFRONT_DISTRIBUTION", + }, +} + +const tableName = "site-launch" +const mockDynamoDBClient = { + createItem: jest.fn(), + getAllItems: jest.fn(), + deleteItem: jest.fn(), + withLogger: jest.fn(), +} + +const dynamoDBClient = (mockDynamoDBClient as unknown) as DynamoDBClient +const dynamoDBService = new DynamoDBService({ dynamoDBClient }) + +const spyDynamoDBService = { + deleteItem: jest.spyOn(dynamoDBService, "deleteItem"), +} +describe("DynamoDBService", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + describe("createItem", () => { + it("should call the createItem method of the DynamoDBClient with the correct arguments", async () => { + await dynamoDBService.createItem(mockLaunch) + expect(dynamoDBClient.createItem).toHaveBeenCalledWith( + tableName, + mockLaunch + ) + }) + }) + + describe("getAllSuccessOrFailureLaunches", () => { + it("should call the getAllItems method of the DynamoDBClient with the correct arguments", async () => { + const scanCommandOutput: ScanCommandOutput = { + $metadata: { httpStatusCode: 200 }, + Items: [mockSuccessLaunch, mockLaunch, mockFailureLaunch], + } + mockDynamoDBClient.getAllItems.mockReturnValueOnce(scanCommandOutput) + const result: SiteLaunchMessage[] = await dynamoDBService.getAllCompletedLaunches() + expect(dynamoDBClient.getAllItems).toHaveBeenCalledWith(tableName) + expect(spyDynamoDBService.deleteItem).toHaveBeenCalledWith( + mockSuccessLaunch + ) + expect(spyDynamoDBService.deleteItem).toHaveBeenCalledWith( + mockFailureLaunch + ) + expect(result).toEqual([mockSuccessLaunch, mockFailureLaunch]) + }) + }) + + describe("deleteItem", () => { + it("should call the deleteItem method of the DynamoDBClient with the correct arguments", async () => { + await dynamoDBService.deleteItem(mockLaunch) + expect(dynamoDBClient.deleteItem).toHaveBeenCalledWith(tableName, { + appId: "my-app", + }) + expect(dynamoDBClient.deleteItem).not.toThrow() + }) + }) +})