From 6dce99eec384eecbc6676276011cac35400ea1f7 Mon Sep 17 00:00:00 2001 From: Daniel Emery Date: Thu, 2 Nov 2023 21:09:10 +0100 Subject: [PATCH] #67 Convert s3 and sqs service into DI classes --- src/file/s3.service.ts | 40 +++++++++++++++++++ src/file/s3.ts | 35 ----------------- src/index.ts | 5 +-- src/queue/sqs.service.ts | 81 +++++++++++++++++++++++++++++++++++++++ src/queue/sqs.ts | 75 ------------------------------------ src/quiz/quizResolvers.ts | 9 ++--- src/service.locator.ts | 8 ++++ 7 files changed, 135 insertions(+), 118 deletions(-) create mode 100644 src/file/s3.service.ts delete mode 100644 src/file/s3.ts create mode 100644 src/queue/sqs.service.ts delete mode 100644 src/queue/sqs.ts diff --git a/src/file/s3.service.ts b/src/file/s3.service.ts new file mode 100644 index 0000000..ac4cd65 --- /dev/null +++ b/src/file/s3.service.ts @@ -0,0 +1,40 @@ +import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; +import { S3RequestPresigner } from '@aws-sdk/s3-request-presigner'; +import { createRequest } from '@aws-sdk/util-create-request'; +import { formatUrl } from '@aws-sdk/util-format-url'; + +import config from '../config/config'; + +export class S3FileService { + #s3Client: S3Client; + constructor() { + this.#s3Client = new S3Client({ region: config.AWS_REGION }); + } + + async generateSignedUploadUrl(key: string): Promise { + const request = await createRequest( + this.#s3Client, + new PutObjectCommand({ + Key: key, + Bucket: config.AWS_BUCKET_NAME, + }), + ); + + const signer = new S3RequestPresigner({ + ...this.#s3Client.config, + }); + + const url = await signer.presign(request, { + expiresIn: 3600, + }); + return formatUrl(url); + } + + keyToUrl(key: string): string { + return `${config.FILE_ACCESS_BASE_URL}/${key}`; + } + + createKey(resourceId: string, fileName: string) { + return `${resourceId}/${fileName}`; + } +} diff --git a/src/file/s3.ts b/src/file/s3.ts deleted file mode 100644 index 4e62119..0000000 --- a/src/file/s3.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; -import { S3RequestPresigner } from '@aws-sdk/s3-request-presigner'; -import { createRequest } from '@aws-sdk/util-create-request'; -import { formatUrl } from '@aws-sdk/util-format-url'; - -import config from '../config/config'; - -const s3Client = new S3Client({ region: config.AWS_REGION }); - -export async function generateSignedUploadUrl(key: string): Promise { - const request = await createRequest( - s3Client, - new PutObjectCommand({ - Key: key, - Bucket: config.AWS_BUCKET_NAME, - }), - ); - - const signer = new S3RequestPresigner({ - ...s3Client.config, - }); - - const url = await signer.presign(request, { - expiresIn: 3600, - }); - return formatUrl(url); -} - -export function keyToUrl(key: string): string { - return `${config.FILE_ACCESS_BASE_URL}/${key}`; -} - -export function createKey(resourceId: string, fileName: string) { - return `${resourceId}/${fileName}`; -} diff --git a/src/index.ts b/src/index.ts index c11c0c9..555375a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,13 +9,12 @@ import { GraphQLScalarType, Kind } from 'graphql'; import http from 'http'; import * as Sentry from '@sentry/node'; -import { authenticationService } from './service.locator'; +import { authenticationService, queueService } from './service.locator'; import config from './config/config'; import typeDefs from './gql'; import { persistence } from './persistence/persistence'; import { createQuiz, quiz, quizzes, completeQuiz } from './quiz/quizResolvers'; import { me, users } from './user/userResolvers'; -import { subscribeToFileUploads } from './queue/sqs'; const QUIZLORD_VERSION_HEADER = 'X-Quizlord-Api-Version'; @@ -143,7 +142,7 @@ async function initialise() { ); app.use(Sentry.Handlers.errorHandler()); - subscribeToFileUploads(); + queueService.subscribeToFileUploads(); await new Promise((resolve) => httpServer.listen({ port: 4000 }, resolve)); console.log(`🚀 Server ready at http://localhost:4000/`); diff --git a/src/queue/sqs.service.ts b/src/queue/sqs.service.ts new file mode 100644 index 0000000..8b5ec72 --- /dev/null +++ b/src/queue/sqs.service.ts @@ -0,0 +1,81 @@ +import { SQSClient, ReceiveMessageCommand, Message, DeleteMessageCommand } from '@aws-sdk/client-sqs'; + +import config from '../config/config'; +import { persistence } from '../persistence/persistence'; + +interface S3MessageContent { + Records?: S3MessageContentRecord[]; +} + +interface S3MessageContentRecord { + eventTime: string; + eventName: string; + s3: { + object: { + key: string; + size: number; + }; + }; +} + +export class SQSQueueService { + #client: SQSClient; + + constructor() { + this.#client = new SQSClient({ region: config.AWS_REGION }); + } + + async subscribeToFileUploads() { + // todo exit this loop when app entering shutdown state. + // eslint-disable-next-line no-constant-condition + while (true) { + console.log(`Polling ${config.AWS_FILE_UPLOADED_SQS_QUEUE_URL} for messages`); + const result = await this.#client.send( + new ReceiveMessageCommand({ + QueueUrl: config.AWS_FILE_UPLOADED_SQS_QUEUE_URL, + WaitTimeSeconds: 10, + }), + ); + if (result.Messages) { + await Promise.all(result.Messages.map((message) => this.processMessage(message))); + } + } + } + + async processMessage(message: Message) { + if (message.Body) { + const messageBody = JSON.parse(message.Body); + if (messageBody.Message) { + const messageData: S3MessageContent = JSON.parse(messageBody.Message); + if (messageData.Records) { + await Promise.all(messageData.Records.map((record) => this.processUploadedItem(record))); + } + } else { + console.warn(`Unexpected empty inner message body`, message); + } + } else { + console.warn(`Unexpected empty message body`, message); + } + + await this.#client.send( + new DeleteMessageCommand({ + QueueUrl: config.AWS_FILE_UPLOADED_SQS_QUEUE_URL, + ReceiptHandle: message.ReceiptHandle, + }), + ); + } + + async processUploadedItem(record: S3MessageContentRecord) { + console.log('Processing uploaded item'); + if (record.eventName !== 'ObjectCreated:Put') { + console.warn(`Unexpected event name <${record.eventName}>`); + } + const key = record.s3.object.key; + const quiz = await persistence.getQuizImage(key); + if (quiz) { + await persistence.markQuizImageReady(key); + } else { + console.error(`Invalid file upload at key: ${key}`); + } + } +} diff --git a/src/queue/sqs.ts b/src/queue/sqs.ts deleted file mode 100644 index c865b33..0000000 --- a/src/queue/sqs.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { SQSClient, ReceiveMessageCommand, Message, DeleteMessageCommand } from '@aws-sdk/client-sqs'; - -import config from '../config/config'; -import { persistence } from '../persistence/persistence'; - -const client = new SQSClient({ region: config.AWS_REGION }); - -export async function subscribeToFileUploads() { - // todo exit this loop when app entering shutdown state. - // eslint-disable-next-line no-constant-condition - while (true) { - console.log(`Polling ${config.AWS_FILE_UPLOADED_SQS_QUEUE_URL} for messages`); - const result = await client.send( - new ReceiveMessageCommand({ - QueueUrl: config.AWS_FILE_UPLOADED_SQS_QUEUE_URL, - WaitTimeSeconds: 10, - }), - ); - if (result.Messages) { - await Promise.all(result.Messages.map((message) => processMessage(message))); - } - } -} - -interface S3MessageContent { - Records?: S3MessageContentRecord[]; -} - -interface S3MessageContentRecord { - eventTime: string; - eventName: string; - s3: { - object: { - key: string; - size: number; - }; - }; -} - -async function processMessage(message: Message) { - if (message.Body) { - const messageBody = JSON.parse(message.Body); - if (messageBody.Message) { - const messageData: S3MessageContent = JSON.parse(messageBody.Message); - if (messageData.Records) { - await Promise.all(messageData.Records.map((record) => processUploadedItem(record))); - } - } else { - console.warn(`Unexpected empty inner message body`, message); - } - } else { - console.warn(`Unexpected empty message body`, message); - } - - await client.send( - new DeleteMessageCommand({ - QueueUrl: config.AWS_FILE_UPLOADED_SQS_QUEUE_URL, - ReceiptHandle: message.ReceiptHandle, - }), - ); -} - -async function processUploadedItem(record: S3MessageContentRecord) { - console.log('Processing uploaded item'); - if (record.eventName !== 'ObjectCreated:Put') { - console.warn(`Unexpected event name <${record.eventName}>`); - } - const key = record.s3.object.key; - const quiz = await persistence.getQuizImage(key); - if (quiz) { - await persistence.markQuizImageReady(key); - } else { - console.error(`Invalid file upload at key: ${key}`); - } -} diff --git a/src/quiz/quizResolvers.ts b/src/quiz/quizResolvers.ts index 9d7754f..aa712be 100644 --- a/src/quiz/quizResolvers.ts +++ b/src/quiz/quizResolvers.ts @@ -12,13 +12,12 @@ import { v4 as uuidv4 } from 'uuid'; import { QuizlordContext } from '..'; import { Quiz, QuizDetails, QuizCompletion, QuizImage, CreateQuizResult, QuizFilters } from '../models'; import { persistence } from '../persistence/persistence'; -import { createKey, generateSignedUploadUrl, keyToUrl } from '../file/s3'; import { base64Decode, base64Encode, PagedResult } from '../util/paging-helpers'; -import { authorisationService } from '../service.locator'; +import { authorisationService, fileService } from '../service.locator'; function quizImagePersistenceToQuizImage(quizImage: QuizImagePersistence): QuizImage { return { - imageLink: keyToUrl(quizImage.imageKey), + imageLink: fileService.keyToUrl(quizImage.imageKey), state: quizImage.state, type: quizImage.type, }; @@ -114,7 +113,7 @@ export async function quiz(_: unknown, { id }: { id: string }, context: Quizlord } async function populateFileWithUploadLink(file: { fileName: string; type: QuizImageType; imageKey: string }) { - const uploadLink = await generateSignedUploadUrl(file.imageKey); + const uploadLink = await fileService.generateSignedUploadUrl(file.imageKey); return { ...file, uploadLink, @@ -128,7 +127,7 @@ export async function createQuiz( ): Promise { authorisationService.requireUserRole(context, 'USER'); const uuid = uuidv4(); - const filesWithKeys = files.map((file) => ({ ...file, imageKey: createKey(uuid, file.fileName) })); + const filesWithKeys = files.map((file) => ({ ...file, imageKey: fileService.createKey(uuid, file.fileName) })); const [createdQuiz, ...uploadLinks] = await Promise.all([ persistence.createQuizWithImages( { diff --git a/src/service.locator.ts b/src/service.locator.ts index 01d2cea..3a44706 100644 --- a/src/service.locator.ts +++ b/src/service.locator.ts @@ -1,6 +1,14 @@ import { AuthenticationService } from './auth/authentication.service'; import { AuthorisationService } from './auth/authorisation.service'; +import { S3FileService } from './file/s3.service'; +import { SQSQueueService } from './queue/sqs.service'; // auth export const authenticationService = new AuthenticationService(); export const authorisationService = new AuthorisationService(); + +// file +export const fileService = new S3FileService(); + +// queue +export const queueService = new SQSQueueService();