From c90d5fefd60aba264a5513befed5bb245affc51e Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Thu, 5 Oct 2023 22:03:59 +0900 Subject: [PATCH 01/10] feat: test code for forum ads api --- .eslintrc.yaml | 50 ++++---- lib/constructs/business/rest-api-service.ts | 135 ++++++++++++++++++++ lib/constructs/business/service.ts | 2 + lib/constructs/persistence/data-pipeline.ts | 4 + lib/stacks/business.ts | 5 + 5 files changed, 171 insertions(+), 25 deletions(-) diff --git a/.eslintrc.yaml b/.eslintrc.yaml index 58fd80a82..52f461f2d 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -4,39 +4,39 @@ env: node: true root: true plugins: - - "@typescript-eslint" + - '@typescript-eslint' - import -parser: "@typescript-eslint/parser" +parser: '@typescript-eslint/parser' parserOptions: ecmaVersion: 2018 sourceType: module - project: "./tsconfig.json" + project: './tsconfig.json' extends: - - "eslint:recommended" - - "plugin:@typescript-eslint/eslint-recommended" - - "plugin:@typescript-eslint/recommended" - - "plugin:import/typescript" + - 'eslint:recommended' + - 'plugin:@typescript-eslint/eslint-recommended' + - 'plugin:@typescript-eslint/recommended' + - 'plugin:import/typescript' settings: import/parsers: - "@typescript-eslint/parser": - - ".ts" - - ".tsx" + '@typescript-eslint/parser': + - '.ts' + - '.tsx' import/resolver: - node: { } + node: {} typescript: - project: "./tsconfig.json" + project: './tsconfig.json' alwaysTryTypes: true ignorePatterns: - - "*.js" - - "!.projenrc.js" - - "*.d.ts" + - '*.js' + - '!.projenrc.js' + - '*.d.ts' - node_modules/ - - "*.generated.ts" + - '*.generated.ts' - coverage rules: indent: - 'off' - "@typescript-eslint/indent": + '@typescript-eslint/indent': - error - 2 quotes: @@ -81,7 +81,7 @@ rules: - error - multi-line - consistent - "@typescript-eslint/member-delimiter-style": + '@typescript-eslint/member-delimiter-style': - error semi: - error @@ -97,13 +97,13 @@ rules: quote-props: - error - consistent-as-needed - "@typescript-eslint/no-require-imports": + '@typescript-eslint/no-require-imports': - error import/no-extraneous-dependencies: - error - devDependencies: - - "**/test/**" - - "**/build-tools/**" + - '**/test/**' + - '**/build-tools/**' optionalDependencies: false peerDependencies: true import/no-unresolved: @@ -120,17 +120,17 @@ rules: - error no-shadow: - 'off' - "@typescript-eslint/no-shadow": + '@typescript-eslint/no-shadow': - error key-spacing: - error no-multiple-empty-lines: - error - "@typescript-eslint/no-floating-promises": + '@typescript-eslint/no-floating-promises': - error no-return-await: - 'off' - "@typescript-eslint/return-await": + '@typescript-eslint/return-await': - error no-trailing-spaces: - error @@ -138,7 +138,7 @@ rules: - error no-bitwise: - error - "@typescript-eslint/member-ordering": + '@typescript-eslint/member-ordering': - error - default: - public-static-field diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index 5269d4379..a228d7865 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -51,6 +51,141 @@ export class RestApiService extends Construct { } } +//! New code for adsImgs +export class ForumAdsApiService extends RestApiService { + readonly resourceMapping: { + [path: string]: { [method in apigw2.HttpMethod]?: apigw.Method }; + }; + + constructor( + scope: AbstractRestApiEndpoint, + id: string, + props: RestApiServiceProps, + ) { + super(scope, id, props); + + // Create resources for the api + const root = scope.apiEndpoint.root.addResource('imgs'); + const imgsUrl = root.addResource('{url}'); + + const getRespModel = scope.apiEndpoint.addModel('review-get-resp-model', { + schema: courseReviewGetRespSchema, + contentType: 'application/json', + description: 'HTTP GET response body schema for fetching reviews.', + modelName: 'GetReviewsResp', + }); + + // const postRespModel = + // const deleteRespModel = ; + + const apiGatewayRole = new iam.Role(this, 'rest-api-s3', { + assumedBy: new iam.ServicePrincipal(AwsServicePrincipal.API_GATEWAY), + description: 'Allow API Gateway to fetch objects from s3 buckets.', + path: `/service-role/${AwsServicePrincipal.API_GATEWAY}/`, + roleName: 's3-apigateway-read', + managedPolicies: [ + iam.ManagedPolicy.fromManagedPolicyArn( + this, + 's3-read-only', + 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess', + ), + ], + }); + + const getIntegration = new apigw.AwsIntegration({ + service: 's3', + integrationHttpMethod: apigw2.HttpMethod.GET, + path: 'ads.json', //TODO insert the actual name + subdomain: props.dataSource, + options: { + credentialsRole: apiGatewayRole, + requestParameters: { + ['integration.request.path.school']: 'method.request.path.school', //TODO insert the actual pathway + }, + integrationResponses: [ + { + statusCode: '200', + responseParameters: s3RespMapping, + }, + ], + }, + }); + + // const postIntegration = ss; + // const deleteIntegration = ss; + + const optionsAdsImgs = root.addCorsPreflight({ + allowOrigins: allowOrigins, + allowHeaders: allowHeaders, + allowMethods: [ + apigw2.HttpMethod.GET, + apigw2.HttpMethod.POST, + apigw2.HttpMethod.DELETE, + ], + }); + + const getImgsUrl = imgsUrl.addMethod( + apigw2.HttpMethod.GET, + getIntegration, + { + requestParameters: { ['method.request.path.school']: true }, //TODO change the parameter + operationName: 'GetImgsUrl', + methodResponses: [ + { + statusCode: '200', + responseModels: { ['application/json']: getRespModel }, + responseParameters: syllabusRespParams, //TODO change this + }, + ], + requestValidator: props.validator, + }, + ); + + const postImgsUrl = imgsUrl.addMethod( + apigw2.HttpMethod.POST, + postIntegration, + { + requestParameters: { ['method.request.path.school']: true }, + operationName: 'PostImgsUrl', + methodResponses: [ + { + statusCode: '200', + responseModels: { ['application/json']: getRespModel }, + responseParameters: syllabusRespParams, //TODO change this + }, + ], + requestValidator: props.validator, + }, + ); + + const deleteImgsUrl = imgsUrl.addMethod( + apigw2.HttpMethod.DELETE, + deleteIntegration, + { + requestParameters: { ['method.request.path.school']: true }, + operationName: 'DeleteImgsUrl', + methodResponses: [ + { + statusCode: '200', + responseModels: { ['application/json']: getRespModel }, + responseParameters: syllabusRespParams, //TODO change this + }, + ], + requestValidator: props.validator, + }, + ); + + this.resourceMapping = { + '/imgs/{key}': { + [apigw2.HttpMethod.GET]: getImgsUrl, + [apigw2.HttpMethod.OPTIONS]: optionsAdsImgs, + [apigw2.HttpMethod.POST]: postImgsUrl, + [apigw2.HttpMethod.DELETE]: deleteImgsUrl, + }, + }; + } +} + export class SyllabusApiService extends RestApiService { readonly resourceMapping: { [path: string]: { [method in apigw2.HttpMethod]?: apigw.Method }; diff --git a/lib/constructs/business/service.ts b/lib/constructs/business/service.ts index c72e61c23..ed7c958d3 100644 --- a/lib/constructs/business/service.ts +++ b/lib/constructs/business/service.ts @@ -14,6 +14,7 @@ export type RestApiServiceId = | 'timetable' | 'thread' | 'comment' + | 'ads' | 'graphql'; export const restApiServiceMap: { @@ -25,6 +26,7 @@ export const restApiServiceMap: { 'timetable': rest.TimetableApiService, 'thread': rest.ForumThreadsApiService, 'comment': rest.ForumCommentsApiService, + 'ads': rest.ForumAdsApiService, //TODO Add service in construct/business/restapi 'graphql': rest.GraphqlApiService, }; diff --git a/lib/constructs/persistence/data-pipeline.ts b/lib/constructs/persistence/data-pipeline.ts index 9832abf3c..02cdbbde9 100644 --- a/lib/constructs/persistence/data-pipeline.ts +++ b/lib/constructs/persistence/data-pipeline.ts @@ -299,5 +299,9 @@ export class AdsDataPipeline extends AbstractDataPipeline { // filters: [{ prefix: "syllabus/" }], // }) // ); + + this.processor = new sfn.StateMachine(this, 'state-machine', { + stateMachineName: '', + }); } } diff --git a/lib/stacks/business.ts b/lib/stacks/business.ts index aa4a225f6..26b33c66d 100644 --- a/lib/stacks/business.ts +++ b/lib/stacks/business.ts @@ -74,6 +74,11 @@ export class WasedaTimeBusinessLayer extends BusinessLayer { 'comment', this.dataInterface.getEndpoint(DataEndpoint.COMMENT), true, + ) + .addService( + 'ads', + this.dataInterface.getEndpoint(DataEndpoint.ADS), + true, ); // .addService("graphql", graphqlApiEndpoint.apiEndpoint.graphqlUrl); restApiEndpoint.deploy(); From b4f97f5345ad84fc2a51788bba1671067b06733f Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Sat, 7 Oct 2023 17:11:39 +0900 Subject: [PATCH 02/10] feat: test code for lambda function syncadsimgs --- lib/constructs/business/rest-api-service.ts | 112 ++++---------------- lib/constructs/common/lambda-functions.ts | 28 ++--- lib/constructs/persistence/data-pipeline.ts | 29 +++-- src/lambda/sync-imscomsage/index.py | 40 +++++++ src/lambda/sync-imscomsage/utils.py | 68 ++++++++++++ 5 files changed, 153 insertions(+), 124 deletions(-) create mode 100644 src/lambda/sync-imscomsage/index.py create mode 100644 src/lambda/sync-imscomsage/utils.py diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index a228d7865..08d52bbb2 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -28,6 +28,7 @@ import { TimetableFunctions, ForumThreadFunctions, ForumCommentFunctions, + AdsImageProcessFunctions, } from '../common/lambda-functions'; import { AbstractRestApiEndpoint } from './api-endpoint'; @@ -65,8 +66,7 @@ export class ForumAdsApiService extends RestApiService { super(scope, id, props); // Create resources for the api - const root = scope.apiEndpoint.root.addResource('imgs'); - const imgsUrl = root.addResource('{url}'); + const root = scope.apiEndpoint.root.addResource('adsImgs'); const getRespModel = scope.apiEndpoint.addModel('review-get-resp-model', { schema: courseReviewGetRespSchema, @@ -75,44 +75,10 @@ export class ForumAdsApiService extends RestApiService { modelName: 'GetReviewsResp', }); - // const postRespModel = - // const deleteRespModel = ; - - const apiGatewayRole = new iam.Role(this, 'rest-api-s3', { - assumedBy: new iam.ServicePrincipal(AwsServicePrincipal.API_GATEWAY), - description: 'Allow API Gateway to fetch objects from s3 buckets.', - path: `/service-role/${AwsServicePrincipal.API_GATEWAY}/`, - roleName: 's3-apigateway-read', - managedPolicies: [ - iam.ManagedPolicy.fromManagedPolicyArn( - this, - 's3-read-only', - 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess', - ), - ], - }); - - const getIntegration = new apigw.AwsIntegration({ - service: 's3', - integrationHttpMethod: apigw2.HttpMethod.GET, - path: 'ads.json', //TODO insert the actual name - subdomain: props.dataSource, - options: { - credentialsRole: apiGatewayRole, - requestParameters: { - ['integration.request.path.school']: 'method.request.path.school', //TODO insert the actual pathway - }, - integrationResponses: [ - { - statusCode: '200', - responseParameters: s3RespMapping, - }, - ], - }, - }); - - // const postIntegration = ss; - // const deleteIntegration = ss; + const getIntegration = new apigw.LambdaIntegration( + AdsImageProcessFunctions.getFunction, + { proxy: true }, + ); const optionsAdsImgs = root.addCorsPreflight({ allowOrigins: allowOrigins, @@ -124,63 +90,23 @@ export class ForumAdsApiService extends RestApiService { ], }); - const getImgsUrl = imgsUrl.addMethod( - apigw2.HttpMethod.GET, - getIntegration, - { - requestParameters: { ['method.request.path.school']: true }, //TODO change the parameter - operationName: 'GetImgsUrl', - methodResponses: [ - { - statusCode: '200', - responseModels: { ['application/json']: getRespModel }, - responseParameters: syllabusRespParams, //TODO change this - }, - ], - requestValidator: props.validator, - }, - ); - - const postImgsUrl = imgsUrl.addMethod( - apigw2.HttpMethod.POST, - postIntegration, - { - requestParameters: { ['method.request.path.school']: true }, - operationName: 'PostImgsUrl', - methodResponses: [ - { - statusCode: '200', - responseModels: { ['application/json']: getRespModel }, - responseParameters: syllabusRespParams, //TODO change this - }, - ], - requestValidator: props.validator, - }, - ); - - const deleteImgsUrl = imgsUrl.addMethod( - apigw2.HttpMethod.DELETE, - deleteIntegration, - { - requestParameters: { ['method.request.path.school']: true }, - operationName: 'DeleteImgsUrl', - methodResponses: [ - { - statusCode: '200', - responseModels: { ['application/json']: getRespModel }, - responseParameters: syllabusRespParams, //TODO change this - }, - ], - requestValidator: props.validator, - }, - ); + const getImgsUrl = root.addMethod(apigw2.HttpMethod.GET, getIntegration, { + requestParameters: { ['method.request.path.school']: true }, //TODO change the parameter + operationName: 'GetImgsUrl', + methodResponses: [ + { + statusCode: '200', + responseModels: { ['application/json']: getRespModel }, + responseParameters: syllabusRespParams, //TODO change this + }, + ], + requestValidator: props.validator, + }); this.resourceMapping = { - '/imgs/{key}': { + '/adsImgs': { [apigw2.HttpMethod.GET]: getImgsUrl, [apigw2.HttpMethod.OPTIONS]: optionsAdsImgs, - [apigw2.HttpMethod.POST]: postImgsUrl, - [apigw2.HttpMethod.DELETE]: deleteImgsUrl, }, }; } diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index 85ca97d75..975f98986 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -859,18 +859,18 @@ export class ThreadImageProcessFunctions extends Construct { }, ); - // this.syncImageFunction = new lambda_py.PythonFunction(this, 'sync-image', { - // entry: 'src/lambda/sync-image', - // description: - // 'post image to dyanamo db database when image inputed in s3 bucket', - // functionName: 'sync-image', - // logRetention: logs.RetentionDays.ONE_MONTH, - // memorySize: 256, - // role: DBSyncRole, - // runtime: lambda.Runtime.PYTHON_3_9, - // timeout: Duration.seconds(5), - // environment: props.envVars, - // }); + this.syncImageFunction = new lambda_py.PythonFunction(this, 'sync-image', { + entry: 'src/lambda/sync-image', + description: + 'post image to dyanamo db database when image inputed in s3 bucket', + functionName: 'sync-image', + logRetention: logs.RetentionDays.ONE_MONTH, + memorySize: 256, + role: DBSyncRole, + runtime: lambda.Runtime.PYTHON_3_9, + timeout: Duration.seconds(5), + environment: props.envVars, + }); this.resizeImageFunction = new lambda_py.PythonFunction( this, @@ -916,7 +916,7 @@ export class ThreadImageProcessFunctions extends Construct { } export class AdsImageProcessFunctions extends Construct { - // readonly getFunction: lambda.Function; + readonly getFunction: lambda.Function; readonly syncImageFunction: lambda.Function; readonly resizeImageFunction: lambda.Function; // readonly deleteFunction: lambda.Function; @@ -983,7 +983,7 @@ export class AdsImageProcessFunctions extends Construct { ); this.syncImageFunction = new lambda_py.PythonFunction(this, 'sync-image', { - entry: 'src/lambda/sync-image', + entry: 'src/lambda/sync-imscomsage', description: 'post image to dyanamo db database when image inputed in s3 bucket', functionName: 'sync-image', diff --git a/lib/constructs/persistence/data-pipeline.ts b/lib/constructs/persistence/data-pipeline.ts index 02cdbbde9..af55f23d4 100644 --- a/lib/constructs/persistence/data-pipeline.ts +++ b/lib/constructs/persistence/data-pipeline.ts @@ -285,23 +285,18 @@ export class AdsDataPipeline extends AbstractDataPipeline { this.dataWarehouse = props.dataWarehouse!; - // this.processor = new AdsImageProcessFunctions(this, "image-process-func", { - // envVars: { - // ["BUCKET_NAME"]: this.dataSource.bucketName, - // ["TABLE_NAME"]: this.dataWarehouse.tableName, - // ["OBJECT_PATH"]: "syllabus/", - // }, - // }).syncImageFunction; - - // this.processor.addEventSource( - // new event_sources.S3EventSource(this.dataSource, { - // events: [s3.EventType.OBJECT_CREATED_PUT], - // filters: [{ prefix: "syllabus/" }], - // }) - // ); + this.processor = new AdsImageProcessFunctions(this, 'image-process-func', { + envVars: { + ['BUCKET_NAME']: this.dataSource.bucketName, + ['TABLE_NAME']: this.dataWarehouse.tableName, + ['OBJECT_PATH']: 'keys/', + }, + }).syncImageFunction; - this.processor = new sfn.StateMachine(this, 'state-machine', { - stateMachineName: '', - }); + this.processor.addEventSource( + new event_sources.S3EventSource(this.dataSource, { + events: [s3.EventType.OBJECT_CREATED_PUT], + }), + ); } } diff --git a/src/lambda/sync-imscomsage/index.py b/src/lambda/sync-imscomsage/index.py new file mode 100644 index 000000000..20764c9dd --- /dev/null +++ b/src/lambda/sync-imscomsage/index.py @@ -0,0 +1,40 @@ +import json +from datetime import datetime +from utils import JsonPayloadBuilder +from utils import resp_handler, table, output_table + + + + +@resp_handler +def post_imgskey(key): + # Get the crrent time + dt_now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + + # Creaet board_id, ads_id from the event payload we got + board_id, thread_id, _ = key.split('/') + + # Create new item in the dynamoDB + item = { + 'board_id': {'S': board_id}, + 'ads_id': {'S': thread_id}, + 'timestamp': {'S': dt_now} + } + + table.put_item( + TableName=output_table, + Item=item + ) + + + + body = JsonPayloadBuilder().add_status(True).add_data(None).add_message('Imgs key load to table successfully.').compile() + return body + + +def handler(event, context): + + # Get event payload and get imgs information + key = event['Records'][0]['s3']['object']['key'] + + return post_imgskey(key) diff --git a/src/lambda/sync-imscomsage/utils.py b/src/lambda/sync-imscomsage/utils.py new file mode 100644 index 000000000..58f994eb5 --- /dev/null +++ b/src/lambda/sync-imscomsage/utils.py @@ -0,0 +1,68 @@ +import boto3 +import json +import logging +import os +from decimal import Decimal + +db = boto3.resource("dynamodb", region_name="ap-northeast-1") +table = db.Table(os.getenv('TABLE_NAME')) # Use in index to post ads info + +output_table = os.environ['TABLE_NAME'] + + +class DecimalEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder.default(self, obj) + + +class JsonPayloadBuilder: + payload = {} + + def add_status(self, success): + self.payload['success'] = success + return self + + def add_data(self, data): + self.payload['data'] = data + return self + + def add_message(self, msg): + self.payload['message'] = msg + return self + + def compile(self): + return json.dumps(self.payload, cls=DecimalEncoder, ensure_ascii=False).encode('utf8') + + +def api_response(code, body): + return { + "isBase64Encoded": False, + "statusCode": code, + 'headers': { + "Access-Control-Allow-Origin": '*', + "Content-Type": "application/json", + "Referrer-Policy": "origin" + }, + "multiValueHeaders": {"Access-Control-Allow-Methods": ["POST", "OPTIONS", "GET", "PATCH", "DELETE"]}, + "body": body + } + + +def resp_handler(func): + def handle(*args, **kwargs): + try: + resp = func(*args, **kwargs) + return api_response(200, resp) + except LookupError: + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Not found").compile() + return api_response(404, resp) + except Exception as e: + logging.error(str(e)) + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Internal error, please contact bugs@wasedatime.com.").compile() + return api_response(500, resp) + + return handle From 7efdc2a168429e6aba01bb839f7dd4a7ee8a5849 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Sat, 7 Oct 2023 23:47:40 +0900 Subject: [PATCH 03/10] feat: new test code for lambda function syncadsimgs --- lib/constructs/persistence/data-pipeline.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/constructs/persistence/data-pipeline.ts b/lib/constructs/persistence/data-pipeline.ts index af55f23d4..e54260895 100644 --- a/lib/constructs/persistence/data-pipeline.ts +++ b/lib/constructs/persistence/data-pipeline.ts @@ -295,7 +295,7 @@ export class AdsDataPipeline extends AbstractDataPipeline { this.processor.addEventSource( new event_sources.S3EventSource(this.dataSource, { - events: [s3.EventType.OBJECT_CREATED_PUT], + events: [s3.EventType.OBJECT_CREATED_POST], }), ); } From 8927e25dff6559f7332b0c6ab682b9030e092169 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Sat, 7 Oct 2023 23:50:44 +0900 Subject: [PATCH 04/10] feat: new test code for lambda function syncadsimgs --- lib/constructs/business/rest-api-service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index 08d52bbb2..cda72b182 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -75,10 +75,10 @@ export class ForumAdsApiService extends RestApiService { modelName: 'GetReviewsResp', }); - const getIntegration = new apigw.LambdaIntegration( - AdsImageProcessFunctions.getFunction, - { proxy: true }, - ); + // const getIntegration = new apigw.LambdaIntegration( + // AdsImageProcessFunctions.getFunction, + // { proxy: true }, + // ); const optionsAdsImgs = root.addCorsPreflight({ allowOrigins: allowOrigins, From f2083e3d268936a28882a3b2d6b3e06b2fbe9dd9 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Sun, 8 Oct 2023 21:11:51 +0900 Subject: [PATCH 05/10] feat: test code for lambda function get-omgs-list --- lib/constructs/business/rest-api-service.ts | 32 ++++------ lib/constructs/common/lambda-functions.ts | 24 ++++---- src/lambda/get-imgs-list/index.py | 25 ++++++++ src/lambda/get-imgs-list/utils.py | 66 +++++++++++++++++++++ 4 files changed, 113 insertions(+), 34 deletions(-) create mode 100644 src/lambda/get-imgs-list/index.py create mode 100644 src/lambda/get-imgs-list/utils.py diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index cda72b182..379ce3723 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -68,44 +68,32 @@ export class ForumAdsApiService extends RestApiService { // Create resources for the api const root = scope.apiEndpoint.root.addResource('adsImgs'); - const getRespModel = scope.apiEndpoint.addModel('review-get-resp-model', { - schema: courseReviewGetRespSchema, - contentType: 'application/json', - description: 'HTTP GET response body schema for fetching reviews.', - modelName: 'GetReviewsResp', - }); - - // const getIntegration = new apigw.LambdaIntegration( - // AdsImageProcessFunctions.getFunction, - // { proxy: true }, - // ); + const getIntegration = new apigw.LambdaIntegration( + AdsImageProcessFunctions.getFunction, //* No f idea why this is an error + { proxy: true }, + ); const optionsAdsImgs = root.addCorsPreflight({ allowOrigins: allowOrigins, allowHeaders: allowHeaders, - allowMethods: [ - apigw2.HttpMethod.GET, - apigw2.HttpMethod.POST, - apigw2.HttpMethod.DELETE, - ], + allowMethods: [apigw2.HttpMethod.GET, apigw2.HttpMethod.POST], }); - const getImgsUrl = root.addMethod(apigw2.HttpMethod.GET, getIntegration, { - requestParameters: { ['method.request.path.school']: true }, //TODO change the parameter - operationName: 'GetImgsUrl', + const getImgsList = root.addMethod(apigw2.HttpMethod.GET, getIntegration, { + operationName: 'GetImgsList', methodResponses: [ { statusCode: '200', - responseModels: { ['application/json']: getRespModel }, - responseParameters: syllabusRespParams, //TODO change this + responseParameters: lambdaRespParams, }, ], + authorizer: props.authorizer, requestValidator: props.validator, }); this.resourceMapping = { '/adsImgs': { - [apigw2.HttpMethod.GET]: getImgsUrl, + [apigw2.HttpMethod.GET]: getImgsList, [apigw2.HttpMethod.OPTIONS]: optionsAdsImgs, }, }; diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index 975f98986..e4f221690 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -995,6 +995,18 @@ export class AdsImageProcessFunctions extends Construct { environment: props.envVars, }); + this.getFunction = new lambda_py.PythonFunction(this, 'get-imgs-list', { + entry: 'src/lambda/get-imgs-list', + description: 'get imgs list from the database.', + functionName: 'get-imgs-list', + logRetention: logs.RetentionDays.ONE_MONTH, + memorySize: 128, + role: DBReadRole, + runtime: lambda.Runtime.PYTHON_3_9, + timeout: Duration.seconds(3), + environment: props.envVars, + }); + // this.resizeImageFunction = new lambda_py.PythonFunction( // this, // "resize-image", @@ -1012,18 +1024,6 @@ export class AdsImageProcessFunctions extends Construct { // } // ); - // this.getFunction = new lambda_py.PythonFunction(this, "get-comment", { - // entry: "src/lambda/get-comments", - // description: "get forum comments from the database.", - // functionName: "get-forum-comments", - // logRetention: logs.RetentionDays.ONE_MONTH, - // memorySize: 128, - // role: DBReadRole, - // runtime: lambda.Runtime.PYTHON_3_9, - // timeout: Duration.seconds(3), - // environment: props.envVars, - // }); - // this.deleteFunction = new lambda_py.PythonFunction(this, "delete-comment", { // entry: "src/lambda/delete-comment", // description: "Delete forum comment in the database.", diff --git a/src/lambda/get-imgs-list/index.py b/src/lambda/get-imgs-list/index.py new file mode 100644 index 000000000..655e414ec --- /dev/null +++ b/src/lambda/get-imgs-list/index.py @@ -0,0 +1,25 @@ +from boto3.dynamodb.conditions import Key +from utils import JsonPayloadBuilder +from utils import resp_handler +from utils import table + + +@resp_handler +def get_imgs_list(): + response = table.scan() + results = response.get('Items', []) + + + body = JsonPayloadBuilder().add_status( + True).add_data(results).add_message('').compile() + return body + + +def handler(event, context): + params = { + "thread_id": event["pathParameters"]["thread_id"], + } + if "uid" in event["queryStringParameters"]: + params["uid"] = event["queryStringParameters"]["uid"] + + return get_imgs_list(**params) \ No newline at end of file diff --git a/src/lambda/get-imgs-list/utils.py b/src/lambda/get-imgs-list/utils.py new file mode 100644 index 000000000..ec9ba2cb7 --- /dev/null +++ b/src/lambda/get-imgs-list/utils.py @@ -0,0 +1,66 @@ +import boto3 +import json +import logging +import os +from decimal import Decimal + +db = boto3.resource("dynamodb", region_name="ap-northeast-1") +table = db.Table(os.getenv('TABLE_NAME')) + + +class DecimalEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder.default(self, obj) + + +class JsonPayloadBuilder: + payload = {} + + def add_status(self, success): + self.payload['success'] = success + return self + + def add_data(self, data): + self.payload['data'] = data + return self + + def add_message(self, msg): + self.payload['message'] = msg + return self + + def compile(self): + return json.dumps(self.payload, cls=DecimalEncoder, ensure_ascii=False).encode('utf8') + + +def api_response(code, body): + return { + "isBase64Encoded": False, + "statusCode": code, + 'headers': { + "Access-Control-Allow-Origin": '*', + "Content-Type": "application/json", + "Referrer-Policy": "origin" + }, + "multiValueHeaders": {"Access-Control-Allow-Methods": ["POST", "OPTIONS", "GET", "PATCH", "DELETE"]}, + "body": body + } + + +def resp_handler(func): + def handle(*args, **kwargs): + try: + resp = func(*args, **kwargs) + return api_response(200, resp) + except LookupError: + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Not found").compile() + return api_response(404, resp) + except Exception as e: + logging.error(str(e)) + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Internal error, please contact bugs@wasedatime.com.").compile() + return api_response(500, resp) + + return handle \ No newline at end of file From dabd71ea66a190ce1b81a510d72c317f5d2c1c74 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Sun, 8 Oct 2023 21:20:21 +0900 Subject: [PATCH 06/10] feat: test code for lambda function get-omgs-list 2 --- lib/constructs/business/rest-api-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index 379ce3723..200f6d115 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -69,7 +69,7 @@ export class ForumAdsApiService extends RestApiService { const root = scope.apiEndpoint.root.addResource('adsImgs'); const getIntegration = new apigw.LambdaIntegration( - AdsImageProcessFunctions.getFunction, //* No f idea why this is an error + AdsImageProcessFunctions.getFunction, //* No idea { proxy: true }, ); From a84f3913e6ea20875211580f346983d6781f7dba Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Mon, 9 Oct 2023 20:54:01 +0900 Subject: [PATCH 07/10] feat: test code for lambda get imgs list --- lib/constructs/business/rest-api-service.ts | 14 +++++-- lib/constructs/common/lambda-functions.ts | 2 +- lib/constructs/persistence/data-pipeline.ts | 2 +- src/lambda/get-imgs-list/index.py | 23 ++++++---- src/lambda/sync-image/index.py | 42 ++++++++++++++++++- .../{sync-imscomsage => sync-image}/utils.py | 0 src/lambda/sync-imscomsage/index.py | 40 ------------------ 7 files changed, 67 insertions(+), 56 deletions(-) rename src/lambda/{sync-imscomsage => sync-image}/utils.py (100%) delete mode 100644 src/lambda/sync-imscomsage/index.py diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index 200f6d115..02de75e6b 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -68,8 +68,18 @@ export class ForumAdsApiService extends RestApiService { // Create resources for the api const root = scope.apiEndpoint.root.addResource('adsImgs'); + const adsImageProcessFunctions = new AdsImageProcessFunctions( + this, + 'crud-functions', + { + envVars: { + TABLE_NAME: props.dataSource!, + }, + }, + ); + const getIntegration = new apigw.LambdaIntegration( - AdsImageProcessFunctions.getFunction, //* No idea + adsImageProcessFunctions.getFunction, { proxy: true }, ); @@ -87,7 +97,6 @@ export class ForumAdsApiService extends RestApiService { responseParameters: lambdaRespParams, }, ], - authorizer: props.authorizer, requestValidator: props.validator, }); @@ -99,7 +108,6 @@ export class ForumAdsApiService extends RestApiService { }; } } - export class SyllabusApiService extends RestApiService { readonly resourceMapping: { [path: string]: { [method in apigw2.HttpMethod]?: apigw.Method }; diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index e4f221690..278c2d737 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -983,7 +983,7 @@ export class AdsImageProcessFunctions extends Construct { ); this.syncImageFunction = new lambda_py.PythonFunction(this, 'sync-image', { - entry: 'src/lambda/sync-imscomsage', + entry: 'src/lambda/sync-image', description: 'post image to dyanamo db database when image inputed in s3 bucket', functionName: 'sync-image', diff --git a/lib/constructs/persistence/data-pipeline.ts b/lib/constructs/persistence/data-pipeline.ts index e54260895..7bb1b9987 100644 --- a/lib/constructs/persistence/data-pipeline.ts +++ b/lib/constructs/persistence/data-pipeline.ts @@ -289,7 +289,7 @@ export class AdsDataPipeline extends AbstractDataPipeline { envVars: { ['BUCKET_NAME']: this.dataSource.bucketName, ['TABLE_NAME']: this.dataWarehouse.tableName, - ['OBJECT_PATH']: 'keys/', + ['OBJECT_PATH']: 'board_id/', }, }).syncImageFunction; diff --git a/src/lambda/get-imgs-list/index.py b/src/lambda/get-imgs-list/index.py index 655e414ec..dd8d87c6a 100644 --- a/src/lambda/get-imgs-list/index.py +++ b/src/lambda/get-imgs-list/index.py @@ -5,10 +5,18 @@ @resp_handler -def get_imgs_list(): - response = table.scan() +def get_imgs_list(board_id): + + if board_id: + response = table.query(KeyConditionExpression=Key( + "board_id").eq(board_id), ScanIndexForward=False) + else: + response = table.scan(ConsistentRead=False) + results = response.get('Items', []) - + + # response = table.scan() + # results = response.get('Items', []) body = JsonPayloadBuilder().add_status( True).add_data(results).add_message('').compile() @@ -16,10 +24,7 @@ def get_imgs_list(): def handler(event, context): - params = { - "thread_id": event["pathParameters"]["thread_id"], - } - if "uid" in event["queryStringParameters"]: - params["uid"] = event["queryStringParameters"]["uid"] - return get_imgs_list(**params) \ No newline at end of file + params = event["queryStringParameters"] + board_id = params.get("board_id", "") + return get_imgs_list(board_id) diff --git a/src/lambda/sync-image/index.py b/src/lambda/sync-image/index.py index 2959113be..20764c9dd 100644 --- a/src/lambda/sync-image/index.py +++ b/src/lambda/sync-image/index.py @@ -1,2 +1,40 @@ -def handler(event): - pass +import json +from datetime import datetime +from utils import JsonPayloadBuilder +from utils import resp_handler, table, output_table + + + + +@resp_handler +def post_imgskey(key): + # Get the crrent time + dt_now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + + # Creaet board_id, ads_id from the event payload we got + board_id, thread_id, _ = key.split('/') + + # Create new item in the dynamoDB + item = { + 'board_id': {'S': board_id}, + 'ads_id': {'S': thread_id}, + 'timestamp': {'S': dt_now} + } + + table.put_item( + TableName=output_table, + Item=item + ) + + + + body = JsonPayloadBuilder().add_status(True).add_data(None).add_message('Imgs key load to table successfully.').compile() + return body + + +def handler(event, context): + + # Get event payload and get imgs information + key = event['Records'][0]['s3']['object']['key'] + + return post_imgskey(key) diff --git a/src/lambda/sync-imscomsage/utils.py b/src/lambda/sync-image/utils.py similarity index 100% rename from src/lambda/sync-imscomsage/utils.py rename to src/lambda/sync-image/utils.py diff --git a/src/lambda/sync-imscomsage/index.py b/src/lambda/sync-imscomsage/index.py deleted file mode 100644 index 20764c9dd..000000000 --- a/src/lambda/sync-imscomsage/index.py +++ /dev/null @@ -1,40 +0,0 @@ -import json -from datetime import datetime -from utils import JsonPayloadBuilder -from utils import resp_handler, table, output_table - - - - -@resp_handler -def post_imgskey(key): - # Get the crrent time - dt_now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' - - # Creaet board_id, ads_id from the event payload we got - board_id, thread_id, _ = key.split('/') - - # Create new item in the dynamoDB - item = { - 'board_id': {'S': board_id}, - 'ads_id': {'S': thread_id}, - 'timestamp': {'S': dt_now} - } - - table.put_item( - TableName=output_table, - Item=item - ) - - - - body = JsonPayloadBuilder().add_status(True).add_data(None).add_message('Imgs key load to table successfully.').compile() - return body - - -def handler(event, context): - - # Get event payload and get imgs information - key = event['Records'][0]['s3']['object']['key'] - - return post_imgskey(key) From 677d967e7afe87ff3cc7b3ed98db36626bc5ba12 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Mon, 9 Oct 2023 21:06:09 +0900 Subject: [PATCH 08/10] feat: test code for lambda get imgs list 2 --- lib/constructs/persistence/data-pipeline.ts | 3 +-- src/lambda/get-imgs-list/utils.py | 2 +- src/lambda/sync-image/index.py | 26 +++++++++------------ src/lambda/sync-image/utils.py | 4 +--- 4 files changed, 14 insertions(+), 21 deletions(-) diff --git a/lib/constructs/persistence/data-pipeline.ts b/lib/constructs/persistence/data-pipeline.ts index 7bb1b9987..5862b2f70 100644 --- a/lib/constructs/persistence/data-pipeline.ts +++ b/lib/constructs/persistence/data-pipeline.ts @@ -289,13 +289,12 @@ export class AdsDataPipeline extends AbstractDataPipeline { envVars: { ['BUCKET_NAME']: this.dataSource.bucketName, ['TABLE_NAME']: this.dataWarehouse.tableName, - ['OBJECT_PATH']: 'board_id/', }, }).syncImageFunction; this.processor.addEventSource( new event_sources.S3EventSource(this.dataSource, { - events: [s3.EventType.OBJECT_CREATED_POST], + events: [s3.EventType.OBJECT_CREATED], }), ); } diff --git a/src/lambda/get-imgs-list/utils.py b/src/lambda/get-imgs-list/utils.py index ec9ba2cb7..05f0218dc 100644 --- a/src/lambda/get-imgs-list/utils.py +++ b/src/lambda/get-imgs-list/utils.py @@ -63,4 +63,4 @@ def handle(*args, **kwargs): .add_message("Internal error, please contact bugs@wasedatime.com.").compile() return api_response(500, resp) - return handle \ No newline at end of file + return handle diff --git a/src/lambda/sync-image/index.py b/src/lambda/sync-image/index.py index 20764c9dd..8dc8889eb 100644 --- a/src/lambda/sync-image/index.py +++ b/src/lambda/sync-image/index.py @@ -1,39 +1,35 @@ import json from datetime import datetime from utils import JsonPayloadBuilder -from utils import resp_handler, table, output_table - - +from utils import resp_handler, table @resp_handler def post_imgskey(key): # Get the crrent time dt_now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' - + # Creaet board_id, ads_id from the event payload we got - board_id, thread_id, _ = key.split('/') - + board_id, ads_id, _ = key.split('/') + # Create new item in the dynamoDB item = { - 'board_id': {'S': board_id}, - 'ads_id': {'S': thread_id}, - 'timestamp': {'S': dt_now} + 'board_id': {board_id}, + 'ads_id': {ads_id}, + 'timestamp': {dt_now} } - + table.put_item( - TableName=output_table, Item=item ) - - - body = JsonPayloadBuilder().add_status(True).add_data(None).add_message('Imgs key load to table successfully.').compile() + body = JsonPayloadBuilder().add_status(True).add_data( + None).add_message('Imgs key load to table successfully.').compile() return body def handler(event, context): - + # Get event payload and get imgs information key = event['Records'][0]['s3']['object']['key'] diff --git a/src/lambda/sync-image/utils.py b/src/lambda/sync-image/utils.py index 58f994eb5..3feed09fb 100644 --- a/src/lambda/sync-image/utils.py +++ b/src/lambda/sync-image/utils.py @@ -5,9 +5,7 @@ from decimal import Decimal db = boto3.resource("dynamodb", region_name="ap-northeast-1") -table = db.Table(os.getenv('TABLE_NAME')) # Use in index to post ads info - -output_table = os.environ['TABLE_NAME'] +table = db.Table(os.getenv('TABLE_NAME')) # Use in index to post ads info class DecimalEncoder(json.JSONEncoder): From 4150d2945a64539b9aed9b73ae6d44bd97205460 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Mon, 9 Oct 2023 21:46:40 +0900 Subject: [PATCH 09/10] feat: test code for lambda get imgs list 3 --- lib/constructs/common/lambda-functions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index 278c2d737..13c348089 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -809,7 +809,7 @@ export class ThreadImageProcessFunctions extends Construct { description: 'Allow lambda function to perform read operation on dynamodb and s3', path: `/service-role/${AwsServicePrincipal.LAMBDA}/`, - roleName: 'dynamodb-s3-lambda-read-thread', + roleName: 'dynamodb-s3-lambda-read-thread-imgs', // Changed the role name managedPolicies: [ iam.ManagedPolicy.fromManagedPolicyArn( this, @@ -932,7 +932,7 @@ export class AdsImageProcessFunctions extends Construct { description: 'Allow lambda function to perform read operation on dynamodb and s3', path: `/service-role/${AwsServicePrincipal.LAMBDA}/`, - roleName: 'dynamodb-s3-lambda-read-thread', + roleName: 'dynamodb-s3-lambda-read-ads-imgs', // Changed the role name managedPolicies: [ iam.ManagedPolicy.fromManagedPolicyArn( this, From 86e424f304536064137b00f10fa9f4269c3d5761 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Mon, 9 Oct 2023 21:54:49 +0900 Subject: [PATCH 10/10] chore: solve the existed name problem of DBReadRole --- lib/constructs/common/lambda-functions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index 13c348089..cbf6ab950 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -932,7 +932,7 @@ export class AdsImageProcessFunctions extends Construct { description: 'Allow lambda function to perform read operation on dynamodb and s3', path: `/service-role/${AwsServicePrincipal.LAMBDA}/`, - roleName: 'dynamodb-s3-lambda-read-ads-imgs', // Changed the role name + roleName: 'dynamodb-s3-lambda-read-ads-imgs', // Changed the new role name managedPolicies: [ iam.ManagedPolicy.fromManagedPolicyArn( this,