diff --git a/.eslintrc.yaml b/.eslintrc.yaml index 58fd80a82..52f461f2d 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -4,39 +4,39 @@ env: node: true root: true plugins: - - "@typescript-eslint" + - '@typescript-eslint' - import -parser: "@typescript-eslint/parser" +parser: '@typescript-eslint/parser' parserOptions: ecmaVersion: 2018 sourceType: module - project: "./tsconfig.json" + project: './tsconfig.json' extends: - - "eslint:recommended" - - "plugin:@typescript-eslint/eslint-recommended" - - "plugin:@typescript-eslint/recommended" - - "plugin:import/typescript" + - 'eslint:recommended' + - 'plugin:@typescript-eslint/eslint-recommended' + - 'plugin:@typescript-eslint/recommended' + - 'plugin:import/typescript' settings: import/parsers: - "@typescript-eslint/parser": - - ".ts" - - ".tsx" + '@typescript-eslint/parser': + - '.ts' + - '.tsx' import/resolver: - node: { } + node: {} typescript: - project: "./tsconfig.json" + project: './tsconfig.json' alwaysTryTypes: true ignorePatterns: - - "*.js" - - "!.projenrc.js" - - "*.d.ts" + - '*.js' + - '!.projenrc.js' + - '*.d.ts' - node_modules/ - - "*.generated.ts" + - '*.generated.ts' - coverage rules: indent: - 'off' - "@typescript-eslint/indent": + '@typescript-eslint/indent': - error - 2 quotes: @@ -81,7 +81,7 @@ rules: - error - multi-line - consistent - "@typescript-eslint/member-delimiter-style": + '@typescript-eslint/member-delimiter-style': - error semi: - error @@ -97,13 +97,13 @@ rules: quote-props: - error - consistent-as-needed - "@typescript-eslint/no-require-imports": + '@typescript-eslint/no-require-imports': - error import/no-extraneous-dependencies: - error - devDependencies: - - "**/test/**" - - "**/build-tools/**" + - '**/test/**' + - '**/build-tools/**' optionalDependencies: false peerDependencies: true import/no-unresolved: @@ -120,17 +120,17 @@ rules: - error no-shadow: - 'off' - "@typescript-eslint/no-shadow": + '@typescript-eslint/no-shadow': - error key-spacing: - error no-multiple-empty-lines: - error - "@typescript-eslint/no-floating-promises": + '@typescript-eslint/no-floating-promises': - error no-return-await: - 'off' - "@typescript-eslint/return-await": + '@typescript-eslint/return-await': - error no-trailing-spaces: - error @@ -138,7 +138,7 @@ rules: - error no-bitwise: - error - "@typescript-eslint/member-ordering": + '@typescript-eslint/member-ordering': - error - default: - public-static-field diff --git a/lib/constructs/business/rest-api-service.ts b/lib/constructs/business/rest-api-service.ts index c279f03f1..ae9faebf7 100644 --- a/lib/constructs/business/rest-api-service.ts +++ b/lib/constructs/business/rest-api-service.ts @@ -28,6 +28,7 @@ import { TimetableFunctions, ForumThreadFunctions, ForumCommentFunctions, + AdsImageProcessFunctions, } from '../common/lambda-functions'; import { AbstractRestApiEndpoint } from './api-endpoint'; @@ -51,6 +52,62 @@ export class RestApiService extends Construct { } } +//! New code for adsImgs +export class ForumAdsApiService extends RestApiService { + readonly resourceMapping: { + [path: string]: { [method in apigw2.HttpMethod]?: apigw.Method }; + }; + + constructor( + scope: AbstractRestApiEndpoint, + id: string, + props: RestApiServiceProps, + ) { + super(scope, id, props); + + // Create resources for the api + const root = scope.apiEndpoint.root.addResource('adsImgs'); + + const adsImageProcessFunctions = new AdsImageProcessFunctions( + this, + 'crud-functions', + { + envVars: { + TABLE_NAME: props.dataSource!, + }, + }, + ); + + const getIntegration = new apigw.LambdaIntegration( + adsImageProcessFunctions.getFunction, + { proxy: true }, + ); + + const optionsAdsImgs = root.addCorsPreflight({ + allowOrigins: allowOrigins, + allowHeaders: allowHeaders, + allowMethods: [apigw2.HttpMethod.GET, apigw2.HttpMethod.POST], + }); + + const getImgsList = root.addMethod(apigw2.HttpMethod.GET, getIntegration, { + operationName: 'GetImgsList', + methodResponses: [ + { + statusCode: '200', + responseParameters: lambdaRespParams, + }, + ], + requestValidator: props.validator, + }); + + this.resourceMapping = { + '/adsImgs': { + [apigw2.HttpMethod.GET]: getImgsList, + [apigw2.HttpMethod.OPTIONS]: optionsAdsImgs, + }, + }; + } +} export class SyllabusApiService extends RestApiService { readonly resourceMapping: { [path: string]: { [method in apigw2.HttpMethod]?: apigw.Method }; diff --git a/lib/constructs/business/service.ts b/lib/constructs/business/service.ts index c72e61c23..ed7c958d3 100644 --- a/lib/constructs/business/service.ts +++ b/lib/constructs/business/service.ts @@ -14,6 +14,7 @@ export type RestApiServiceId = | 'timetable' | 'thread' | 'comment' + | 'ads' | 'graphql'; export const restApiServiceMap: { @@ -25,6 +26,7 @@ export const restApiServiceMap: { 'timetable': rest.TimetableApiService, 'thread': rest.ForumThreadsApiService, 'comment': rest.ForumCommentsApiService, + 'ads': rest.ForumAdsApiService, //TODO Add service in construct/business/restapi 'graphql': rest.GraphqlApiService, }; diff --git a/lib/constructs/common/lambda-functions.ts b/lib/constructs/common/lambda-functions.ts index 70bae0636..9c2e19c63 100644 --- a/lib/constructs/common/lambda-functions.ts +++ b/lib/constructs/common/lambda-functions.ts @@ -877,18 +877,18 @@ export class ThreadImageProcessFunctions extends Construct { }, ); - // this.syncImageFunction = new lambda_py.PythonFunction(this, 'sync-image', { - // entry: 'src/lambda/sync-image', - // description: - // 'post image to dyanamo db database when image inputed in s3 bucket', - // functionName: 'sync-image', - // logRetention: logs.RetentionDays.ONE_MONTH, - // memorySize: 256, - // role: DBSyncRole, - // runtime: lambda.Runtime.PYTHON_3_9, - // timeout: Duration.seconds(5), - // environment: props.envVars, - // }); + this.syncImageFunction = new lambda_py.PythonFunction(this, 'sync-image', { + entry: 'src/lambda/sync-image', + description: + 'post image to dyanamo db database when image inputed in s3 bucket', + functionName: 'sync-image', + logRetention: logs.RetentionDays.ONE_MONTH, + memorySize: 256, + role: DBSyncRole, + runtime: lambda.Runtime.PYTHON_3_9, + timeout: Duration.seconds(5), + environment: props.envVars, + }); this.resizeImageFunction = new lambda_py.PythonFunction( this, @@ -934,7 +934,7 @@ export class ThreadImageProcessFunctions extends Construct { } export class AdsImageProcessFunctions extends Construct { - // readonly getFunction: lambda.Function; + readonly getFunction: lambda.Function; readonly syncImageFunction: lambda.Function; readonly resizeImageFunction: lambda.Function; // readonly deleteFunction: lambda.Function; @@ -1013,6 +1013,18 @@ export class AdsImageProcessFunctions extends Construct { environment: props.envVars, }); + this.getFunction = new lambda_py.PythonFunction(this, 'get-imgs-list', { + entry: 'src/lambda/get-imgs-list', + description: 'get imgs list from the database.', + functionName: 'get-imgs-list', + logRetention: logs.RetentionDays.ONE_MONTH, + memorySize: 128, + role: DBReadRole, + runtime: lambda.Runtime.PYTHON_3_9, + timeout: Duration.seconds(3), + environment: props.envVars, + }); + // this.resizeImageFunction = new lambda_py.PythonFunction( // this, // "resize-image", @@ -1030,18 +1042,6 @@ export class AdsImageProcessFunctions extends Construct { // } // ); - // this.getFunction = new lambda_py.PythonFunction(this, "get-comment", { - // entry: "src/lambda/get-comments", - // description: "get forum comments from the database.", - // functionName: "get-forum-comments", - // logRetention: logs.RetentionDays.ONE_MONTH, - // memorySize: 128, - // role: DBReadRole, - // runtime: lambda.Runtime.PYTHON_3_9, - // timeout: Duration.seconds(3), - // environment: props.envVars, - // }); - // this.deleteFunction = new lambda_py.PythonFunction(this, "delete-comment", { // entry: "src/lambda/delete-comment", // description: "Delete forum comment in the database.", diff --git a/lib/constructs/persistence/data-pipeline.ts b/lib/constructs/persistence/data-pipeline.ts index 9832abf3c..5862b2f70 100644 --- a/lib/constructs/persistence/data-pipeline.ts +++ b/lib/constructs/persistence/data-pipeline.ts @@ -285,19 +285,17 @@ export class AdsDataPipeline extends AbstractDataPipeline { this.dataWarehouse = props.dataWarehouse!; - // this.processor = new AdsImageProcessFunctions(this, "image-process-func", { - // envVars: { - // ["BUCKET_NAME"]: this.dataSource.bucketName, - // ["TABLE_NAME"]: this.dataWarehouse.tableName, - // ["OBJECT_PATH"]: "syllabus/", - // }, - // }).syncImageFunction; + this.processor = new AdsImageProcessFunctions(this, 'image-process-func', { + envVars: { + ['BUCKET_NAME']: this.dataSource.bucketName, + ['TABLE_NAME']: this.dataWarehouse.tableName, + }, + }).syncImageFunction; - // this.processor.addEventSource( - // new event_sources.S3EventSource(this.dataSource, { - // events: [s3.EventType.OBJECT_CREATED_PUT], - // filters: [{ prefix: "syllabus/" }], - // }) - // ); + this.processor.addEventSource( + new event_sources.S3EventSource(this.dataSource, { + events: [s3.EventType.OBJECT_CREATED], + }), + ); } } diff --git a/lib/stacks/business.ts b/lib/stacks/business.ts index aa4a225f6..26b33c66d 100644 --- a/lib/stacks/business.ts +++ b/lib/stacks/business.ts @@ -74,6 +74,11 @@ export class WasedaTimeBusinessLayer extends BusinessLayer { 'comment', this.dataInterface.getEndpoint(DataEndpoint.COMMENT), true, + ) + .addService( + 'ads', + this.dataInterface.getEndpoint(DataEndpoint.ADS), + true, ); // .addService("graphql", graphqlApiEndpoint.apiEndpoint.graphqlUrl); restApiEndpoint.deploy(); diff --git a/src/lambda/get-imgs-list/index.py b/src/lambda/get-imgs-list/index.py new file mode 100644 index 000000000..dd8d87c6a --- /dev/null +++ b/src/lambda/get-imgs-list/index.py @@ -0,0 +1,30 @@ +from boto3.dynamodb.conditions import Key +from utils import JsonPayloadBuilder +from utils import resp_handler +from utils import table + + +@resp_handler +def get_imgs_list(board_id): + + if board_id: + response = table.query(KeyConditionExpression=Key( + "board_id").eq(board_id), ScanIndexForward=False) + else: + response = table.scan(ConsistentRead=False) + + results = response.get('Items', []) + + # response = table.scan() + # results = response.get('Items', []) + + body = JsonPayloadBuilder().add_status( + True).add_data(results).add_message('').compile() + return body + + +def handler(event, context): + + params = event["queryStringParameters"] + board_id = params.get("board_id", "") + return get_imgs_list(board_id) diff --git a/src/lambda/get-imgs-list/utils.py b/src/lambda/get-imgs-list/utils.py new file mode 100644 index 000000000..05f0218dc --- /dev/null +++ b/src/lambda/get-imgs-list/utils.py @@ -0,0 +1,66 @@ +import boto3 +import json +import logging +import os +from decimal import Decimal + +db = boto3.resource("dynamodb", region_name="ap-northeast-1") +table = db.Table(os.getenv('TABLE_NAME')) + + +class DecimalEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder.default(self, obj) + + +class JsonPayloadBuilder: + payload = {} + + def add_status(self, success): + self.payload['success'] = success + return self + + def add_data(self, data): + self.payload['data'] = data + return self + + def add_message(self, msg): + self.payload['message'] = msg + return self + + def compile(self): + return json.dumps(self.payload, cls=DecimalEncoder, ensure_ascii=False).encode('utf8') + + +def api_response(code, body): + return { + "isBase64Encoded": False, + "statusCode": code, + 'headers': { + "Access-Control-Allow-Origin": '*', + "Content-Type": "application/json", + "Referrer-Policy": "origin" + }, + "multiValueHeaders": {"Access-Control-Allow-Methods": ["POST", "OPTIONS", "GET", "PATCH", "DELETE"]}, + "body": body + } + + +def resp_handler(func): + def handle(*args, **kwargs): + try: + resp = func(*args, **kwargs) + return api_response(200, resp) + except LookupError: + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Not found").compile() + return api_response(404, resp) + except Exception as e: + logging.error(str(e)) + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Internal error, please contact bugs@wasedatime.com.").compile() + return api_response(500, resp) + + return handle diff --git a/src/lambda/sync-image/index.py b/src/lambda/sync-image/index.py index 2959113be..8dc8889eb 100644 --- a/src/lambda/sync-image/index.py +++ b/src/lambda/sync-image/index.py @@ -1,2 +1,36 @@ -def handler(event): - pass +import json +from datetime import datetime +from utils import JsonPayloadBuilder +from utils import resp_handler, table + + +@resp_handler +def post_imgskey(key): + # Get the crrent time + dt_now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + + # Creaet board_id, ads_id from the event payload we got + board_id, ads_id, _ = key.split('/') + + # Create new item in the dynamoDB + item = { + 'board_id': {board_id}, + 'ads_id': {ads_id}, + 'timestamp': {dt_now} + } + + table.put_item( + Item=item + ) + + body = JsonPayloadBuilder().add_status(True).add_data( + None).add_message('Imgs key load to table successfully.').compile() + return body + + +def handler(event, context): + + # Get event payload and get imgs information + key = event['Records'][0]['s3']['object']['key'] + + return post_imgskey(key) diff --git a/src/lambda/sync-image/utils.py b/src/lambda/sync-image/utils.py new file mode 100644 index 000000000..3feed09fb --- /dev/null +++ b/src/lambda/sync-image/utils.py @@ -0,0 +1,66 @@ +import boto3 +import json +import logging +import os +from decimal import Decimal + +db = boto3.resource("dynamodb", region_name="ap-northeast-1") +table = db.Table(os.getenv('TABLE_NAME')) # Use in index to post ads info + + +class DecimalEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return float(obj) + return json.JSONEncoder.default(self, obj) + + +class JsonPayloadBuilder: + payload = {} + + def add_status(self, success): + self.payload['success'] = success + return self + + def add_data(self, data): + self.payload['data'] = data + return self + + def add_message(self, msg): + self.payload['message'] = msg + return self + + def compile(self): + return json.dumps(self.payload, cls=DecimalEncoder, ensure_ascii=False).encode('utf8') + + +def api_response(code, body): + return { + "isBase64Encoded": False, + "statusCode": code, + 'headers': { + "Access-Control-Allow-Origin": '*', + "Content-Type": "application/json", + "Referrer-Policy": "origin" + }, + "multiValueHeaders": {"Access-Control-Allow-Methods": ["POST", "OPTIONS", "GET", "PATCH", "DELETE"]}, + "body": body + } + + +def resp_handler(func): + def handle(*args, **kwargs): + try: + resp = func(*args, **kwargs) + return api_response(200, resp) + except LookupError: + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Not found").compile() + return api_response(404, resp) + except Exception as e: + logging.error(str(e)) + resp = JsonPayloadBuilder().add_status(False).add_data(None) \ + .add_message("Internal error, please contact bugs@wasedatime.com.").compile() + return api_response(500, resp) + + return handle