From 2df6f4af36ac46cac7650b12beb2e1f54337754e Mon Sep 17 00:00:00 2001 From: Jason Park <93040528+JasonNotJson@users.noreply.github.com> Date: Fri, 22 Sep 2023 02:22:01 +0900 Subject: [PATCH] feat: connecting thread forum bucket and dynamodb (#326) * feat: adding connection to db for forum images and commenting out graphql stuff * chore: commenting out graphql stuff * chore: fixing identical names * chore: fixing identical construct names in forumthreadfunctions * chore: identical identifier in forum thread functions --- lib/constructs/business/api-endpoint.ts | 178 ++++++++----- .../business/graphql-api-service.ts | 236 +++++++++--------- lib/constructs/business/rest-api-service.ts | 102 ++++---- lib/constructs/business/service.ts | 20 +- lib/constructs/common/lambda-functions.ts | 54 +++- lib/constructs/persistence/data-pipeline.ts | 105 ++++++-- lib/stacks/persistence.ts | 15 +- 7 files changed, 430 insertions(+), 280 deletions(-) diff --git a/lib/constructs/business/api-endpoint.ts b/lib/constructs/business/api-endpoint.ts index 1af9f7bce..ec1895fa2 100644 --- a/lib/constructs/business/api-endpoint.ts +++ b/lib/constructs/business/api-endpoint.ts @@ -12,10 +12,20 @@ import * as flatted from 'flatted'; import { defaultHeaders } from '../../configs/api-gateway/cors'; import { STAGE } from '../../configs/common/aws'; import { API_DOMAIN } from '../../configs/route53/domain'; -import { GraphqlApiService } from './graphql-api-service'; +// import { GraphqlApiService } from './graphql-api-service'; import { AbstractHttpApiService } from './http-api-service'; import { RestApiService } from './rest-api-service'; -import { GraphqlApiServiceId, graphqlApiServiceMap, RestApiServiceId, restApiServiceMap } from './service'; +import { + // GraphqlApiServiceId, + // graphqlApiServiceMap, + RestApiServiceId, + restApiServiceMap, +} from './service'; + +export interface ServiceDataSources { + tableName?: string; + bucketName?: string; +} export interface ApiEndpointProps { zone: route53.IHostedZone; @@ -23,9 +33,18 @@ export interface ApiEndpointProps { } export abstract class AbstractApiEndpoint extends Construct { - abstract readonly apiEndpoint: apigw.RestApi | apigw.LambdaRestApi | apigw.SpecRestApi | apigw2.HttpApi | appsync.GraphqlApi; + abstract readonly apiEndpoint: + | apigw.RestApi + | apigw.LambdaRestApi + | apigw.SpecRestApi + | apigw2.HttpApi + | appsync.GraphqlApi; - protected constructor(scope: Construct, id: string, props?: ApiEndpointProps) { + protected constructor( + scope: Construct, + id: string, + props?: ApiEndpointProps, + ) { super(scope, id); } } @@ -44,7 +63,8 @@ export abstract class AbstractRestApiEndpoint extends AbstractApiEndpoint { } public getDomain(): string { - const domainName: apigw.DomainName | undefined = this.apiEndpoint.domainName; + const domainName: apigw.DomainName | undefined = + this.apiEndpoint.domainName; if (typeof domainName === 'undefined') { throw RangeError('Domain not configured for this API endpoint.'); @@ -52,8 +72,12 @@ export abstract class AbstractRestApiEndpoint extends AbstractApiEndpoint { return domainName.domainName; } - public addService(name: RestApiServiceId, dataSource?: string, auth = false): this { - this.apiServices[name] = new restApiServiceMap[name](this, `${ name }-api`, { + public addService( + name: RestApiServiceId, + dataSource?: ServiceDataSources, + auth = false, + ): this { + this.apiServices[name] = new restApiServiceMap[name](this, `${name}-api`, { dataSource: dataSource, authorizer: auth ? this.authorizer : undefined, validator: this.reqValidator, @@ -61,36 +85,50 @@ export abstract class AbstractRestApiEndpoint extends AbstractApiEndpoint { return this; } - public abstract deploy(): void + public abstract deploy(): void; } -export abstract class AbstractGraphqlEndpoint extends AbstractApiEndpoint { - abstract readonly apiEndpoint: appsync.GraphqlApi; +// export abstract class AbstractGraphqlEndpoint extends AbstractApiEndpoint { +// abstract readonly apiEndpoint: appsync.GraphqlApi; - readonly apiServices: { [name: string]: GraphqlApiService }; +// readonly apiServices: { [name: string]: GraphqlApiService }; - protected authMode: { [mode: string]: appsync.AuthorizationMode } = {}; +// protected authMode: { [mode: string]: appsync.AuthorizationMode } = {}; - protected constructor(scope: Construct, id: string, props: ApiEndpointProps) { - super(scope, id, props); - } +// protected constructor(scope: Construct, id: string, props: ApiEndpointProps) { +// super(scope, id, props); +// } - public addService(name: GraphqlApiServiceId, dataSource: string, auth = 'apiKey'): this { - this.apiServices[name] = new graphqlApiServiceMap[name](this, `${ name }-api`, { - dataSource: dynamodb.Table.fromTableName(this, `${ name }-table`, dataSource), - auth: this.authMode[auth], - }); - return this; - } +// public addService( +// name: GraphqlApiServiceId, +// dataSource: string, +// auth = "apiKey" +// ): this { +// this.apiServices[name] = new graphqlApiServiceMap[name]( +// this, +// `${name}-api`, +// { +// dataSource: dynamodb.Table.fromTableName( +// this, +// `${name}-table`, +// dataSource +// ), +// auth: this.authMode[auth], +// } +// ); +// return this; +// } - public getDomain(): string { - const domain = this.apiEndpoint.graphqlUrl.match(/https:\/\/(.*)\/graphql/g); - if (domain === null) { - return ''; - } - return domain[1]; - } -} +// public getDomain(): string { +// const domain = this.apiEndpoint.graphqlUrl.match( +// /https:\/\/(.*)\/graphql/g +// ); +// if (domain === null) { +// return ""; +// } +// return domain[1]; +// } +// } export abstract class AbstractHttpApiEndpoint extends AbstractApiEndpoint { abstract readonly apiEndpoint: apigw2.HttpApi; @@ -171,7 +209,9 @@ export class WasedaTimeRestApiEndpoint extends AbstractRestApiEndpoint { }); new route53.ARecord(this, 'alias-record', { zone: props.zone, - target: route53.RecordTarget.fromAlias(new route53_targets.ApiGatewayDomain(this.domain)), + target: route53.RecordTarget.fromAlias( + new route53_targets.ApiGatewayDomain(this.domain), + ), recordName: API_DOMAIN, }); } @@ -186,7 +226,10 @@ export class WasedaTimeRestApiEndpoint extends AbstractRestApiEndpoint { api: this.apiEndpoint, retainDeployments: false, }); - const hash = Buffer.from(flatted.stringify(this.apiServices), 'binary').toString('base64'); + const hash = Buffer.from( + flatted.stringify(this.apiServices), + 'binary', + ).toString('base64'); if (STAGE === 'dev') { devDeployment.addToLogicalId(hash); } else if (STAGE === 'prod') { @@ -227,42 +270,41 @@ export class WasedaTimeRestApiEndpoint extends AbstractRestApiEndpoint { } } -export class WasedaTimeGraphqlEndpoint extends AbstractGraphqlEndpoint { - readonly apiEndpoint: appsync.GraphqlApi; - readonly apiServices: { [name: string]: GraphqlApiService } = {}; +// export class WasedaTimeGraphqlEndpoint extends AbstractGraphqlEndpoint { +// readonly apiEndpoint: appsync.GraphqlApi; +// readonly apiServices: { [name: string]: GraphqlApiService } = {}; - constructor(scope: Construct, id: string, props: ApiEndpointProps) { - - super(scope, id, props); +// constructor(scope: Construct, id: string, props: ApiEndpointProps) { +// super(scope, id, props); - const apiKeyAuth: appsync.AuthorizationMode = { - authorizationType: appsync.AuthorizationType.API_KEY, - apiKeyConfig: { - name: 'dev', - expires: Expiration.after(Duration.days(365)), - description: 'API Key for development environment.', - }, - }; - this.authMode.apiKey = apiKeyAuth; - const cognitoAuth: appsync.AuthorizationMode = { - authorizationType: appsync.AuthorizationType.USER_POOL, - userPoolConfig: { - userPool: props.authProvider!, - appIdClientRegex: 'web-app', - }, - }; - this.authMode.userPool = cognitoAuth; +// const apiKeyAuth: appsync.AuthorizationMode = { +// authorizationType: appsync.AuthorizationType.API_KEY, +// apiKeyConfig: { +// name: "dev", +// expires: Expiration.after(Duration.days(365)), +// description: "API Key for development environment.", +// }, +// }; +// this.authMode.apiKey = apiKeyAuth; +// const cognitoAuth: appsync.AuthorizationMode = { +// authorizationType: appsync.AuthorizationType.USER_POOL, +// userPoolConfig: { +// userPool: props.authProvider!, +// appIdClientRegex: "web-app", +// }, +// }; +// this.authMode.userPool = cognitoAuth; - this.apiEndpoint = new appsync.GraphqlApi(this, 'graphql-api', { - name: 'wasedatime-gql-api', - authorizationConfig: { - defaultAuthorization: apiKeyAuth, - additionalAuthorizationModes: [cognitoAuth], - }, - logConfig: { - fieldLogLevel: appsync.FieldLogLevel.ALL, - }, - xrayEnabled: true, - }); - } -} +// this.apiEndpoint = new appsync.GraphqlApi(this, "graphql-api", { +// name: "wasedatime-gql-api", +// authorizationConfig: { +// defaultAuthorization: apiKeyAuth, +// additionalAuthorizationModes: [cognitoAuth], +// }, +// logConfig: { +// fieldLogLevel: appsync.FieldLogLevel.ALL, +// }, +// xrayEnabled: true, +// }); +// } +// } diff --git a/lib/constructs/business/graphql-api-service.ts b/lib/constructs/business/graphql-api-service.ts index 403fe3089..64b167daf 100644 --- a/lib/constructs/business/graphql-api-service.ts +++ b/lib/constructs/business/graphql-api-service.ts @@ -1,130 +1,130 @@ -import * as appsync from '@aws-cdk/aws-appsync-alpha'; -import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; -import { Construct } from 'constructs'; +// import * as appsync from '@aws-cdk/aws-appsync-alpha'; +// import * as dynamodb from 'aws-cdk-lib/aws-dynamodb'; +// import { Construct } from 'constructs'; -import { - generateConnectionAndEdge, - int, - list_int, - list_of, - list_string, - PageInfo, - required, - required_string, - string, -} from '../../utils/appsync'; -import { AbstractGraphqlEndpoint } from './api-endpoint'; +// import { +// generateConnectionAndEdge, +// int, +// list_int, +// list_of, +// list_string, +// PageInfo, +// required, +// required_string, +// string, +// } from '../../utils/appsync'; +// import { AbstractGraphqlEndpoint } from './api-endpoint'; -export interface GraphqlApiServiceProps { - dataSource: dynamodb.ITable; - auth?: appsync.AuthorizationMode; -} +// export interface GraphqlApiServiceProps { +// dataSource: dynamodb.ITable; +// auth?: appsync.AuthorizationMode; +// } -export class GraphqlApiService extends Construct { - readonly resolvers: { [name: string]: appsync.Resolver }; +// export class GraphqlApiService extends Construct { +// readonly resolvers: { [name: string]: appsync.Resolver }; - constructor(scope: AbstractGraphqlEndpoint, id: string, props: GraphqlApiServiceProps) { - super(scope, id); - } -} +// constructor(scope: AbstractGraphqlEndpoint, id: string, props: GraphqlApiServiceProps) { +// super(scope, id); +// } +// } -export class CourseApiService extends GraphqlApiService { - readonly resolvers: { [name: string]: appsync.Resolver } = {}; +// export class CourseApiService extends GraphqlApiService { +// readonly resolvers: { [name: string]: appsync.Resolver } = {}; - constructor(scope: AbstractGraphqlEndpoint, id: string, props: GraphqlApiServiceProps) { - super(scope, id, props); +// constructor(scope: AbstractGraphqlEndpoint, id: string, props: GraphqlApiServiceProps) { +// super(scope, id, props); - const dataSource = scope.apiEndpoint.addDynamoDbDataSource('dynamo-db', props.dataSource, { - description: 'Syllabus table from DynamoDB.', - name: 'SyllabusTable', - }); +// const dataSource = scope.apiEndpoint.addDynamoDbDataSource('dynamo-db', props.dataSource, { +// description: 'Syllabus table from DynamoDB.', +// name: 'SyllabusTable', +// }); - const School = new appsync.EnumType('School', { - definition: [ - 'PSE', 'FSE', 'SSS', 'SILS', 'CSE', 'ASE', 'LAW', 'CMS', 'HSS', 'EDU', 'SOC', 'HUM', 'SPS', 'CJL', - 'GEC', 'CIE', 'ART', 'G_SPS', 'G_SE', 'G_LAW', 'G_LAS', 'G_SC', 'G_EDU', 'G_HUM', 'G_SSS', 'G_SAPS', - 'G_ITS', 'G_SJAL', 'G_IPS', 'G_WLS', 'G_SA', 'G_SPS', 'G_FSE', 'G_ASE', 'G_CSE', 'G_SEEE', 'G_WBS', - 'G_SICCS', - ], - }); - const Eval = new appsync.ObjectType('Evaluation', { - definition: { - type: int, - percent: int, - criteria: string, - }, - }); - const Occurrence = new appsync.ObjectType('Occurrence', { - definition: { - day: int, - period: int, - location: string, - }, - }); - const Course = new appsync.ObjectType('Course', { - definition: { - id: string, - category: string, - code: string, - credit: int, - evals: list_of(Eval), - instructor: string, - lang: list_int, - level: int, - minYear: int, - occurrences: list_of(Occurrence), - school: School.attribute(), - subtitle: string, - term: string, - title: string, - titleJp: string, - type: int, - }, - }); +// const School = new appsync.EnumType('School', { +// definition: [ +// 'PSE', 'FSE', 'SSS', 'SILS', 'CSE', 'ASE', 'LAW', 'CMS', 'HSS', 'EDU', 'SOC', 'HUM', 'SPS', 'CJL', +// 'GEC', 'CIE', 'ART', 'G_SPS', 'G_SE', 'G_LAW', 'G_LAS', 'G_SC', 'G_EDU', 'G_HUM', 'G_SSS', 'G_SAPS', +// 'G_ITS', 'G_SJAL', 'G_IPS', 'G_WLS', 'G_SA', 'G_SPS', 'G_FSE', 'G_ASE', 'G_CSE', 'G_SEEE', 'G_WBS', +// 'G_SICCS', +// ], +// }); +// const Eval = new appsync.ObjectType('Evaluation', { +// definition: { +// type: int, +// percent: int, +// criteria: string, +// }, +// }); +// const Occurrence = new appsync.ObjectType('Occurrence', { +// definition: { +// day: int, +// period: int, +// location: string, +// }, +// }); +// const Course = new appsync.ObjectType('Course', { +// definition: { +// id: string, +// category: string, +// code: string, +// credit: int, +// evals: list_of(Eval), +// instructor: string, +// lang: list_int, +// level: int, +// minYear: int, +// occurrences: list_of(Occurrence), +// school: School.attribute(), +// subtitle: string, +// term: string, +// title: string, +// titleJp: string, +// type: int, +// }, +// }); - const FilterForm = new appsync.InputType('FilterForm', { - definition: { - semester: list_string, - lang: list_int, - day: list_int, - period: list_int, - minYear: list_int, - credit: list_int, - evalType: int, - percent: int, - type: list_int, - level: list_int, - }, - }); +// const FilterForm = new appsync.InputType('FilterForm', { +// definition: { +// semester: list_string, +// lang: list_int, +// day: list_int, +// period: list_int, +// minYear: list_int, +// credit: list_int, +// evalType: int, +// percent: int, +// type: list_int, +// level: list_int, +// }, +// }); - const CourseConnection = generateConnectionAndEdge({ base: Course, target: Course }).connection; - const CourseEdge = generateConnectionAndEdge({ base: Course, target: Course }).edge; +// const CourseConnection = generateConnectionAndEdge({ base: Course, target: Course }).connection; +// const CourseEdge = generateConnectionAndEdge({ base: Course, target: Course }).edge; - [School, Eval, Occurrence, Course, CourseConnection, CourseEdge, PageInfo, FilterForm].forEach( - (type) => scope.apiEndpoint.addType(type), - ); +// [School, Eval, Occurrence, Course, CourseConnection, CourseEdge, PageInfo, FilterForm].forEach( +// (type) => scope.apiEndpoint.addType(type), +// ); - scope.apiEndpoint.addQuery('getCourse', new appsync.ResolvableField({ - returnType: Course.attribute(), - dataSource: dataSource, - args: { - id: required_string, - }, - requestMappingTemplate: appsync.MappingTemplate.dynamoDbGetItem('id', 'id'), - responseMappingTemplate: appsync.MappingTemplate.dynamoDbResultItem(), - })); - scope.apiEndpoint.addQuery('filterCourses', new appsync.ResolvableField({ - returnType: CourseConnection.attribute(), - dataSource: dataSource, - args: { - form: required(FilterForm), - after: string, - first: int, - before: string, - last: int, - }, - requestMappingTemplate: appsync.MappingTemplate.fromFile('src/appsync/mapping/syllabus-filter-req.vtl'), - responseMappingTemplate: appsync.MappingTemplate.dynamoDbResultList(), - })); - } -} +// scope.apiEndpoint.addQuery('getCourse', new appsync.ResolvableField({ +// returnType: Course.attribute(), +// dataSource: dataSource, +// args: { +// id: required_string, +// }, +// requestMappingTemplate: appsync.MappingTemplate.dynamoDbGetItem('id', 'id'), +// responseMappingTemplate: appsync.MappingTemplate.dynamoDbResultItem(), +// })); +// scope.apiEndpoint.addQuery('filterCourses', new appsync.ResolvableField({ +// returnType: CourseConnection.attribute(), +// dataSource: dataSource, +// args: { +// form: required(FilterForm), +// after: string, +// first: int, +// before: string, +// last: int, +// }, +// requestMappingTemplate: appsync.MappingTemplate.fromFile('src/appsync/mapping/syllabus-filter-req.vtl'), +// responseMappingTemplate: appsync.MappingTemplate.dynamoDbResultList(), +// })); +// } +// } diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index b12f1760c..ebdd32c93 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -29,10 +29,10 @@ import { ForumThreadFunctions, ForumCommentFunctions, } from '../common/lambda-functions'; -import { AbstractRestApiEndpoint } from './api-endpoint'; +import { AbstractRestApiEndpoint, ServiceDataSources } from './api-endpoint'; export interface RestApiServiceProps { - dataSource?: string; + dataSource?: ServiceDataSources; authorizer?: apigw.IAuthorizer; validator?: apigw.RequestValidator; } @@ -92,7 +92,7 @@ export class SyllabusApiService extends RestApiService { service: 's3', integrationHttpMethod: apigw2.HttpMethod.GET, path: 'syllabus/{school}.json', - subdomain: props.dataSource, + subdomain: props.dataSource?.bucketName, options: { credentialsRole: apiGatewayRole, requestParameters: { @@ -111,7 +111,7 @@ export class SyllabusApiService extends RestApiService { service: 's3', integrationHttpMethod: apigw2.HttpMethod.HEAD, path: 'syllabus/{school}.json', - subdomain: props.dataSource, + subdomain: props.dataSource?.bucketName, options: { credentialsRole: apiGatewayRole, requestParameters: { @@ -277,7 +277,7 @@ export class CourseReviewsApiService extends RestApiService { 'crud-functions', { envVars: { - TABLE_NAME: props.dataSource!, + TABLE_NAME: props.dataSource?.tableName ?? 'course-review', }, }, ); @@ -527,7 +527,7 @@ export class TimetableApiService extends RestApiService { const timetableFunctions = new TimetableFunctions(this, 'crud-functions', { envVars: { - TABLE_NAME: props.dataSource!, + TABLE_NAME: props.dataSource?.tableName ?? 'timetable', }, }); const getIntegration = new apigw.LambdaIntegration( @@ -659,47 +659,50 @@ export class TimetableApiService extends RestApiService { } } -export class GraphqlApiService extends RestApiService { - readonly resourceMapping: { - [path: string]: { [method in apigw2.HttpMethod]?: apigw.Method }; - }; - - constructor( - scope: AbstractRestApiEndpoint, - id: string, - props: RestApiServiceProps, - ) { - super(scope, id, props); - - const root = scope.apiEndpoint.root.addResource('graphql'); - - const postIntegration = new apigw.HttpIntegration(props.dataSource!, { - proxy: true, - httpMethod: apigw2.HttpMethod.POST, - }); - - const optionsGql = root.addCorsPreflight({ - allowOrigins: allowOrigins, - allowHeaders: allowHeaders, - allowMethods: [apigw2.HttpMethod.POST, apigw2.HttpMethod.OPTIONS], - }); - const postGql = root.addMethod(apigw2.HttpMethod.POST, postIntegration, { - operationName: 'PostGraphQL', - methodResponses: [ - { - statusCode: '200', - }, - ], - }); - - this.resourceMapping = { - '/graphql': { - [apigw2.HttpMethod.OPTIONS]: optionsGql, - [apigw2.HttpMethod.POST]: postGql, - }, - }; - } -} +// export class GraphqlApiService extends RestApiService { +// readonly resourceMapping: { +// [path: string]: { [method in apigw2.HttpMethod]?: apigw.Method }; +// }; + +// constructor( +// scope: AbstractRestApiEndpoint, +// id: string, +// props: RestApiServiceProps +// ) { +// super(scope, id, props); + +// const root = scope.apiEndpoint.root.addResource("graphql"); + +// const postIntegration = new apigw.HttpIntegration( +// props.dataSource?.tableName!, +// { +// proxy: true, +// httpMethod: apigw2.HttpMethod.POST, +// } +// ); + +// const optionsGql = root.addCorsPreflight({ +// allowOrigins: allowOrigins, +// allowHeaders: allowHeaders, +// allowMethods: [apigw2.HttpMethod.POST, apigw2.HttpMethod.OPTIONS], +// }); +// const postGql = root.addMethod(apigw2.HttpMethod.POST, postIntegration, { +// operationName: "PostGraphQL", +// methodResponses: [ +// { +// statusCode: "200", +// }, +// ], +// }); + +// this.resourceMapping = { +// "/graphql": { +// [apigw2.HttpMethod.OPTIONS]: optionsGql, +// [apigw2.HttpMethod.POST]: postGql, +// }, +// }; +// } +// } export class ForumThreadsApiService extends RestApiService { readonly resourceMapping: { @@ -790,7 +793,8 @@ export class ForumThreadsApiService extends RestApiService { 'crud-functions', { envVars: { - TABLE_NAME: props.dataSource!, + TABLE_NAME: props.dataSource?.tableName ?? 'forum-threads', + BUKET_NAME: props.dataSource?.bucketName ?? 'wasedatime-thread-img', }, }, ); @@ -975,7 +979,7 @@ export class ForumCommentsApiService extends RestApiService { 'crud-functions', { envVars: { - TABLE_NAME: props.dataSource!, + TABLE_NAME: props.dataSource?.tableName ?? 'forum-comments', }, }, ); diff --git a/lib/constructs/business/service.ts b/lib/constructs/business/service.ts index c72e61c23..a3a789dd8 100644 --- a/lib/constructs/business/service.ts +++ b/lib/constructs/business/service.ts @@ -1,4 +1,4 @@ -import * as gql from './graphql-api-service'; +// import * as gql from './graphql-api-service'; import * as rest from './rest-api-service'; export enum ApiEndpoint { @@ -13,8 +13,8 @@ export type RestApiServiceId = | 'career' | 'timetable' | 'thread' - | 'comment' - | 'graphql'; + | 'comment'; +// | 'graphql'; export const restApiServiceMap: { [name in RestApiServiceId]: typeof rest.RestApiService; @@ -25,13 +25,13 @@ export const restApiServiceMap: { 'timetable': rest.TimetableApiService, 'thread': rest.ForumThreadsApiService, 'comment': rest.ForumCommentsApiService, - 'graphql': rest.GraphqlApiService, + // 'graphql': rest.GraphqlApiService, }; -export type GraphqlApiServiceId = 'course'; +// export type GraphqlApiServiceId = "course"; -export const graphqlApiServiceMap: { - [name in GraphqlApiServiceId]: typeof gql.GraphqlApiService; -} = { - course: gql.CourseApiService, -}; +// export const graphqlApiServiceMap: { +// [name in GraphqlApiServiceId]: typeof gql.GraphqlApiService; +// } = { +// course: gql.CourseApiService, +// }; diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index a51645a03..0a77f2eb4 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -519,15 +519,15 @@ export class ForumThreadFunctions extends Construct { }, ); - const dynamoDBPutRole: iam.LazyRole = new iam.LazyRole( + const fullDBAccessRole: iam.LazyRole = new iam.LazyRole( this, - 'dynamo-put-role', + 'full-dynamo-s3-access-role', { assumedBy: new iam.ServicePrincipal(AwsServicePrincipal.LAMBDA), description: - 'Allow lambda function to perform crud operation on dynamodb', + 'Allow lambda function to fully access s3 buckets and dynamodb', path: `/service-role/${AwsServicePrincipal.LAMBDA}/`, - roleName: 'dynamodb-lambda-write-thread', + roleName: 's3-dynamodb-lambda-full-access-forums', managedPolicies: [ iam.ManagedPolicy.fromManagedPolicyArn( this, @@ -536,13 +536,47 @@ export class ForumThreadFunctions extends Construct { ), iam.ManagedPolicy.fromManagedPolicyArn( this, - 'db-full-access', + 's3-full-access1', + 'arn:aws:iam::aws:policy/AmazonS3FullAccess', + ), + iam.ManagedPolicy.fromManagedPolicyArn( + this, + 'db-full-access1', 'arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess', ), ], }, ); + const readOnlyDBAccessRole: iam.LazyRole = new iam.LazyRole( + this, + 'readOnly-dynamo-s3-access-role', + { + assumedBy: new iam.ServicePrincipal(AwsServicePrincipal.LAMBDA), + description: + 'Allow lambda function to read access s3 buckets and dynamodb', + path: `/service-role/${AwsServicePrincipal.LAMBDA}/`, + roleName: 's3-dynamodb-lambda-full-access-forums', + managedPolicies: [ + iam.ManagedPolicy.fromManagedPolicyArn( + this, + 'basic-exec2', + 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole', + ), + iam.ManagedPolicy.fromManagedPolicyArn( + this, + 's3-read-only2', + 'arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess', + ), + iam.ManagedPolicy.fromManagedPolicyArn( + this, + 'db-read-only2', + 'arn:aws:iam::aws:policy/AmazonDynamoDBReadOnlyAccess', + ), + ], + }, + ); + this.getAllFunction = new lambda_py.PythonFunction( this, 'get-all-threads', @@ -552,7 +586,7 @@ export class ForumThreadFunctions extends Construct { functionName: 'get-all-threads', logRetention: logs.RetentionDays.ONE_MONTH, memorySize: 128, - role: dynamoDBReadRole, + role: readOnlyDBAccessRole, runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(3), environment: props.envVars, @@ -584,7 +618,7 @@ export class ForumThreadFunctions extends Construct { functionName: 'get-single-thread', logRetention: logs.RetentionDays.ONE_MONTH, memorySize: 128, - role: dynamoDBPutRole, + role: fullDBAccessRole, runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(3), environment: props.envVars, @@ -597,7 +631,7 @@ export class ForumThreadFunctions extends Construct { functionName: 'post-forum-thread', logRetention: logs.RetentionDays.ONE_MONTH, memorySize: 256, - role: dynamoDBPutRole, + role: fullDBAccessRole, runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(5), environment: props.envVars, @@ -612,7 +646,7 @@ export class ForumThreadFunctions extends Construct { functionName: 'patch-forum-thread', logRetention: logs.RetentionDays.ONE_MONTH, memorySize: 256, - role: dynamoDBPutRole, + role: fullDBAccessRole, runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(5), environment: props.envVars, @@ -627,7 +661,7 @@ export class ForumThreadFunctions extends Construct { functionName: 'delete-forum-thread', logRetention: logs.RetentionDays.ONE_MONTH, memorySize: 128, - role: dynamoDBPutRole, + role: fullDBAccessRole, runtime: lambda.Runtime.PYTHON_3_9, timeout: Duration.seconds(3), environment: props.envVars, diff --git a/lib/constructs/persistence/data-pipeline.ts b/lib/constructs/persistence/data-pipeline.ts index 6ef0daf2a..1fc2ea542 100644 --- a/lib/constructs/persistence/data-pipeline.ts +++ b/lib/constructs/persistence/data-pipeline.ts @@ -11,12 +11,16 @@ import { Construct } from 'constructs'; import { syllabusSchedule } from '../../configs/event/schedule'; import { prodCorsRule } from '../../configs/s3/cors'; import { allowApiGatewayPolicy, allowLambdaPolicy } from '../../utils/s3'; -import { SyllabusScraper, SyllabusUpdateFunction } from '../common/lambda-functions'; +import { + SyllabusScraper, + SyllabusUpdateFunction, +} from '../common/lambda-functions'; export enum Worker { SYLLABUS, CAREER, - FEEDS + FEEDS, + THREAD_IMG, } export interface DataPipelineProps { @@ -52,14 +56,21 @@ export class SyllabusDataPipeline extends AbstractDataPipeline { allowApiGatewayPolicy(this.dataWarehouse); allowLambdaPolicy(this.dataWarehouse); - const scraperBaseFunction: lambda.Function = new SyllabusScraper(this, 'scraper-base-function', { - envVars: { - ['BUCKET_NAME']: this.dataWarehouse.bucketName, - ['OBJECT_PATH']: 'syllabus/', + const scraperBaseFunction: lambda.Function = new SyllabusScraper( + this, + 'scraper-base-function', + { + envVars: { + ['BUCKET_NAME']: this.dataWarehouse.bucketName, + ['OBJECT_PATH']: 'syllabus/', + }, }, - }).baseFunction; + ).baseFunction; - function getLambdaTaskInstance(schools: string[], num: string): sfn_tasks.LambdaInvoke { + function getLambdaTaskInstance( + schools: string[], + num: string, + ): sfn_tasks.LambdaInvoke { return new sfn_tasks.LambdaInvoke(scope, 'task-' + num, { lambdaFunction: scraperBaseFunction, comment: 'Scrape the syllabus info of school(s).', @@ -79,9 +90,35 @@ export class SyllabusDataPipeline extends AbstractDataPipeline { .next(getLambdaTaskInstance(['PSE', 'G_ASE', 'LAW'], '4')) .next(getLambdaTaskInstance(['G_FSE', 'SOC', 'SSS'], '5')) .next(getLambdaTaskInstance(['G_LAS', 'G_CSE', 'G_EDU', 'HUM'], '6')) - .next(getLambdaTaskInstance(['SILS', 'G_HUM', 'CJL', 'SPS', 'G_WBS', 'G_PS'], '7')) - .next(getLambdaTaskInstance(['G_SPS', 'G_IPS', 'G_WLS', 'G_E', 'G_SSS', 'G_SC', 'G_LAW', - 'G_SAPS', 'G_SA', 'G_SJAL', 'G_SICCS', 'G_SEEE', 'EHUM', 'ART', 'CIE', 'G_ITS'], '8')) + .next( + getLambdaTaskInstance( + ['SILS', 'G_HUM', 'CJL', 'SPS', 'G_WBS', 'G_PS'], + '7', + ), + ) + .next( + getLambdaTaskInstance( + [ + 'G_SPS', + 'G_IPS', + 'G_WLS', + 'G_E', + 'G_SSS', + 'G_SC', + 'G_LAW', + 'G_SAPS', + 'G_SA', + 'G_SJAL', + 'G_SICCS', + 'G_SEEE', + 'EHUM', + 'ART', + 'CIE', + 'G_ITS', + ], + '8', + ), + ) .next(new sfn.Succeed(this, 'success', {})), }); @@ -141,17 +178,43 @@ export class SyllabusSyncPipeline extends AbstractDataPipeline { //Use exsisting s3 bucket this.dataSource = props.dataSource!; - this.processor = new SyllabusUpdateFunction(this, 'syllabus-update-function', { - envVars: { - ['BUCKET_NAME']: this.dataSource.bucketName, - ['TABLE_NAME']: this.dataWarehouse.tableName, - ['OBJECT_PATH']: 'syllabus/', + this.processor = new SyllabusUpdateFunction( + this, + 'syllabus-update-function', + { + envVars: { + ['BUCKET_NAME']: this.dataSource.bucketName, + ['TABLE_NAME']: this.dataWarehouse.tableName, + ['OBJECT_PATH']: 'syllabus/', + }, }, - }).updateFunction; + ).updateFunction; + + this.processor.addEventSource( + new event_sources.S3EventSource(this.dataSource, { + events: [s3.EventType.OBJECT_CREATED_PUT], + filters: [{ prefix: 'syllabus/' }], + }), + ); + } +} + +export class ThreadImgDataPipeline extends AbstractDataPipeline { + readonly dataSource: s3.Bucket; + readonly processor: lambda.Function; + readonly dataWarehouse: s3.Bucket; + + constructor(scope: Construct, id: string, props?: DataPipelineProps) { + super(scope, id); - this.processor.addEventSource(new event_sources.S3EventSource(this.dataSource, { - events: [s3.EventType.OBJECT_CREATED_PUT], - filters: [{ prefix: 'syllabus/' }], - })); + this.dataSource = new s3.Bucket(this, 'thread-img-bucket', { + accessControl: s3.BucketAccessControl.PRIVATE, + blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, + bucketName: 'wasedatime-thread-img', + encryption: s3.BucketEncryption.S3_MANAGED, + publicReadAccess: false, + removalPolicy: RemovalPolicy.RETAIN, + versioned: true, + }); } } diff --git a/lib/stacks/persistence.ts b/lib/stacks/persistence.ts index ae2c94a98..424b726a2 100644 --- a/lib/stacks/persistence.ts +++ b/lib/stacks/persistence.ts @@ -7,6 +7,7 @@ import { CareerDataPipeline, SyllabusDataPipeline, SyllabusSyncPipeline, + ThreadImgDataPipeline, Worker, } from '../constructs/persistence/data-pipeline'; import { Collection, DynamoDatabase } from '../constructs/persistence/database'; @@ -33,6 +34,12 @@ export class WasedaTimePersistenceLayer extends PersistenceLayer { }, ); + const threadImgDataPipeline = new ThreadImgDataPipeline( + this, + 'thread-img-data-pipeline', + ); + this.dataPipelines[Worker.THREAD_IMG] = threadImgDataPipeline; + const dynamoDatabase = new DynamoDatabase(this, 'dynamo-db'); this.databases['dynamo-main'] = dynamoDatabase; @@ -60,10 +67,10 @@ export class WasedaTimePersistenceLayer extends PersistenceLayer { DataEndpoint.SYLLABUS, syllabusDataPipeline.dataWarehouse.bucketName, ); - this.dataInterface.setEndpoint( - DataEndpoint.THREAD, - dynamoDatabase.tables[Collection.THREAD].tableName, - ); + this.dataInterface.setEndpoint(DataEndpoint.THREAD, { + dynamoTableName: dynamoDatabase.tables[Collection.THREAD].tableName, + s3BucketName: threadImgDataPipeline.dataSource.bucketName, + }); this.dataInterface.setEndpoint( DataEndpoint.COMMENT, dynamoDatabase.tables[Collection.COMMENT].tableName,