From b4cdc337cd4cffa12e2ae0310e599b27af30edd5 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Fri, 20 Oct 2023 02:35:19 +0900 Subject: [PATCH 1/2] feat: improve getads lambda --- lib/constructs/business/rest-api-service.ts | 7 +- lib/constructs/common/lambda-functions.ts | 29 ++++---- lib/stacks/persistence.ts | 2 +- src/lambda/get-ads/index.py | 56 +++++++++++++++ src/lambda/get-ads/utils.py | 80 +++++++++++++++++++++ 5 files changed, 156 insertions(+), 18 deletions(-) create mode 100644 src/lambda/get-ads/index.py create mode 100644 src/lambda/get-ads/utils.py diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index b321ef4ec..f48be6286 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -74,6 +74,7 @@ export class ForumAdsApiService extends RestApiService { { envVars: { TABLE_NAME: props.dataSource!, + BUCKET_NAME: 'wasedatime-ads', }, }, ); @@ -89,8 +90,8 @@ export class ForumAdsApiService extends RestApiService { allowMethods: [apigw2.HttpMethod.GET, apigw2.HttpMethod.POST], }); - const getImgsList = root.addMethod(apigw2.HttpMethod.GET, getIntegration, { - operationName: 'GetImgsList', + const getAds = root.addMethod(apigw2.HttpMethod.GET, getIntegration, { + operationName: 'GetAds', methodResponses: [ { statusCode: '200', @@ -102,7 +103,7 @@ export class ForumAdsApiService extends RestApiService { this.resourceMapping = { '/adsImgs': { - [apigw2.HttpMethod.GET]: getImgsList, + [apigw2.HttpMethod.GET]: getAds, [apigw2.HttpMethod.OPTIONS]: optionsAdsImgs, }, }; diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index 5c8cc9eb3..0209f4867 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -984,42 +984,43 @@ export class AdsImageProcessFunctionsAPI extends Construct { constructor(scope: Construct, id: string, props: FunctionsProps) { super(scope, id); - const DBReadRole: iam.LazyRole = new iam.LazyRole( + //! Update to put role + const DBPutRole: iam.LazyRole = new iam.LazyRole( this, - 'dynamodb-s3-lambda-ads-imgs-read', + 'dynamo-s3-put-role', { assumedBy: new iam.ServicePrincipal(AwsServicePrincipal.LAMBDA), description: - 'Allow lambda function to perform read operation on dynamodb and s3', + 'Allow lambda function to perform crud operation on dynamodb and s3', path: `/service-role/${AwsServicePrincipal.LAMBDA}/`, - roleName: 'dynamodb-s3-lambda-ads-imgs-read', + roleName: 'dynamodb-s3-put-role', managedPolicies: [ iam.ManagedPolicy.fromManagedPolicyArn( this, - 'basic-exec', + 'basic-exec1', 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', ), iam.ManagedPolicy.fromManagedPolicyArn( this, - 'db-read-only', - 'arn:aws:iam::aws:policy/AmazonDynamoDBReadOnlyAccess', + 'db-full-access', + 'arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess', ), iam.ManagedPolicy.fromManagedPolicyArn( this, - 's3-read-only', - 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess', + 's3-full-access', + 'arn:aws:iam::aws:policy/AmazonS3FullAccess', ), ], }, ); - this.getFunction = new lambda_py.PythonFunction(this, 'get-imgs-list', { - entry: 'src/lambda/get-imgs-list', - description: 'get imgs list from the database.', - functionName: 'get-imgs-list', + this.getFunction = new lambda_py.PythonFunction(this, 'get-ads', { + entry: 'src/lambda/get-ads', + description: 'get ads list or specific url from the database.', + functionName: 'get-ads', logRetention: logs.RetentionDays.ONE_MONTH, memorySize: 128, - role: DBReadRole, + role: DBPutRole, //! Change to put role since we now have to read and write runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(3), environment: props.envVars, diff --git a/lib/stacks/persistence.ts b/lib/stacks/persistence.ts index b59a2efac..d624dc4dc 100644 --- a/lib/stacks/persistence.ts +++ b/lib/stacks/persistence.ts @@ -85,7 +85,7 @@ export class WasedaTimePersistenceLayer extends PersistenceLayer { dynamoDatabase.tables[Collection.COMMENT].tableName, ); - //! new endpoint for adsPipeline + //! new endpoint for ads this.dataInterface.setEndpoint( DataEndpoint.ADS, dynamoDatabase.tables[Collection.ADS].tableName, diff --git a/src/lambda/get-ads/index.py b/src/lambda/get-ads/index.py new file mode 100644 index 000000000..05d8892d8 --- /dev/null +++ b/src/lambda/get-ads/index.py @@ -0,0 +1,56 @@ +from boto3.dynamodb.conditions import Key +from utils import JsonPayloadBuilder +from utils import resp_handler, table, bucket, generate_url + +# typeI api call => no query parameter -> scan and return the whole table +# typeII api call => only have board_id -> return all items with matching board_id. +# typeIII api call => have both board and ad ids -> return only the url. + + +@resp_handler +def get_imgs_list(board_id, ad_id): + + results = [] + + # typeIII + if board_id and ad_id: + # Create the key and url when typeII api call + key = "/".join([board_id, ad_id]) + bucket_name = bucket + ad_url = generate_url(bucket_name, key) + + # typeII + elif board_id: + response = table.query(KeyConditionExpression=Key( + "board_id").eq(board_id), ScanIndexForward=False) + + # typeI + else: + response = table.scan(ConsistentRead=False) + + if response: + results = response.get('Items', []) + elif ad_url: + results = ad_url + + # response = table.scan() + # results = response.get('Items', []) + + body = JsonPayloadBuilder().add_status( + True).add_data(results).add_message('').compile() + return body + + +def handler(event, context): + + # params = event["queryStringParameters"] + # board_id = params.get("board_id", "") + + params = { + "board_id": event["queryStringParameters"]["board_id"], + } + if "ad_id" in event["queryStringParameters"]: + params["ad_id"] = event["queryStringParameters"]["ad_id"] + + + return get_imgs_list(**params) diff --git a/src/lambda/get-ads/utils.py b/src/lambda/get-ads/utils.py new file mode 100644 index 000000000..3f080c82d --- /dev/null +++ b/src/lambda/get-ads/utils.py @@ -0,0 +1,80 @@ +import boto3 +import json +import logging +import os +from decimal import Decimal + +db = boto3.resource("dynamodb", region_name="ap-northeast-1") +table = db.Table(os.getenv('TABLE_NAME')) + +s3_client = boto3.client('s3') +bucket = os.getenv('BUCKET_NAME') + +class DecimalEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder.default(self, obj) + + +class JsonPayloadBuilder: + payload = {} + + def add_status(self, success): + self.payload['success'] = success + return self + + def add_data(self, data): + self.payload['data'] = data + return self + + def add_message(self, msg): + self.payload['message'] = msg + return self + + def compile(self): + return json.dumps(self.payload, cls=DecimalEncoder, ensure_ascii=False).encode('utf8') + + +def api_response(code, body): + return { + "isBase64Encoded": False, + "statusCode": code, + 'headers': { + "Access-Control-Allow-Origin": '*', + "Content-Type": "application/json", + "Referrer-Policy": "origin" + }, + "multiValueHeaders": {"Access-Control-Allow-Methods": ["POST", "OPTIONS", "GET", "PATCH", "DELETE"]}, + "body": body + } + + +def resp_handler(func): + def handle(*args, **kwargs): + try: + resp = func(*args, **kwargs) + return api_response(200, resp) + except LookupError: + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Not found").compile() + return api_response(404, resp) + except Exception as e: + logging.error(str(e)) + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Internal error, please contact bugs@wasedatime.com.").compile() + return api_response(500, resp) + + return handle + +def generate_url(bucket_name, object_key, expiration=3600): + try: + response = s3_client.generate_presigned_url('get_object', + Params={'Bucket': bucket_name, + 'Key': object_key}, + ExpiresIn=expiration) + except Exception as e: + logging.error(str(e)) + return None + + return response \ No newline at end of file From 1e571c2d67c86cb341f88e2d2d7f80d02b55fdd1 Mon Sep 17 00:00:00 2001 From: "Y.H LIEN" Date: Fri, 20 Oct 2023 02:42:59 +0900 Subject: [PATCH 2/2] feat: improve getads lambda2 --- src/lambda/get-ads/index.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/lambda/get-ads/index.py b/src/lambda/get-ads/index.py index 05d8892d8..0995af89d 100644 --- a/src/lambda/get-ads/index.py +++ b/src/lambda/get-ads/index.py @@ -9,8 +9,6 @@ @resp_handler def get_imgs_list(board_id, ad_id): - - results = [] # typeIII if board_id and ad_id: @@ -18,20 +16,18 @@ def get_imgs_list(board_id, ad_id): key = "/".join([board_id, ad_id]) bucket_name = bucket ad_url = generate_url(bucket_name, key) + results = ad_url # typeII elif board_id: response = table.query(KeyConditionExpression=Key( "board_id").eq(board_id), ScanIndexForward=False) + results = response # typeI else: response = table.scan(ConsistentRead=False) - - if response: - results = response.get('Items', []) - elif ad_url: - results = ad_url + results = response # response = table.scan() # results = response.get('Items', [])