From 480f9346a2895f99f9311137baec5d646a03d2fc Mon Sep 17 00:00:00 2001 From: David Wyss Date: Thu, 1 Sep 2022 08:53:11 +0200 Subject: [PATCH] feat: File module (#422) * Basic typing etc. * Fix basic upload * Fix basic upload * Fix Private file upload * Cleanup * cleanup * Fix userpool * Add delete command * Add mutations for file deletion * env cleanup * Better docs --- backend/src/flox/modules/file/config.ts | 4 +- .../dto/{ => args}/get-private-file.args.ts | 0 .../dto/{ => args}/get-public-file.args.ts | 0 .../file/dto/input/delete-file.input.ts | 9 ++ .../src/flox/modules/file/file.controller.ts | 75 ++++++------- .../src/flox/modules/file/file.resolver.ts | 44 +++++++- backend/src/flox/modules/file/file.service.ts | 104 +++++++++++++----- backend/src/schema.gql | 6 + frontend/yarn.lock | 43 +++++++- 9 files changed, 209 insertions(+), 76 deletions(-) rename backend/src/flox/modules/file/dto/{ => args}/get-private-file.args.ts (100%) rename backend/src/flox/modules/file/dto/{ => args}/get-public-file.args.ts (100%) create mode 100644 backend/src/flox/modules/file/dto/input/delete-file.input.ts diff --git a/backend/src/flox/modules/file/config.ts b/backend/src/flox/modules/file/config.ts index 227506cf5..3fff48a9f 100644 --- a/backend/src/flox/modules/file/config.ts +++ b/backend/src/flox/modules/file/config.ts @@ -8,12 +8,12 @@ import { MODULES } from '../../MODULES'; */ type FileModuleConfig = { - // TODO: once file module is set up, add options here + // File module has no options }; // Default configuration set; will get merged with custom config from flox.config.json const defaultConfig: FileModuleConfig = { - // TODO: once file module is set up, add options here + // File module has no options }; /** diff --git a/backend/src/flox/modules/file/dto/get-private-file.args.ts b/backend/src/flox/modules/file/dto/args/get-private-file.args.ts similarity index 100% rename from backend/src/flox/modules/file/dto/get-private-file.args.ts rename to backend/src/flox/modules/file/dto/args/get-private-file.args.ts diff --git a/backend/src/flox/modules/file/dto/get-public-file.args.ts b/backend/src/flox/modules/file/dto/args/get-public-file.args.ts similarity index 100% rename from backend/src/flox/modules/file/dto/get-public-file.args.ts rename to backend/src/flox/modules/file/dto/args/get-public-file.args.ts diff --git a/backend/src/flox/modules/file/dto/input/delete-file.input.ts b/backend/src/flox/modules/file/dto/input/delete-file.input.ts new file mode 100644 index 000000000..6fd0a7dd8 --- /dev/null +++ b/backend/src/flox/modules/file/dto/input/delete-file.input.ts @@ -0,0 +1,9 @@ +import { Field, ID, InputType } from '@nestjs/graphql'; +import { IsUUID } from 'class-validator'; + +@InputType() +export class DeleteFileInput { + @Field(() => ID) + @IsUUID() + uuid: string; +} diff --git a/backend/src/flox/modules/file/file.controller.ts b/backend/src/flox/modules/file/file.controller.ts index 8ff203062..4c04add00 100644 --- a/backend/src/flox/modules/file/file.controller.ts +++ b/backend/src/flox/modules/file/file.controller.ts @@ -4,72 +4,61 @@ import { Post, Req, Res, + UnauthorizedException, + UploadedFile, + UseInterceptors, } from '@nestjs/common'; import { FileService } from './file.service'; import { LoggedIn, Public } from '../auth/authentication.decorator'; +import { Response, Request } from 'express'; +import { FileInterceptor } from '@nestjs/platform-express'; @Controller() export class FileController { - constructor(private readonly taskService: FileService) {} + constructor(private readonly fileService: FileService) {} @Public() @Post('/uploadPublicFile') + @UseInterceptors(FileInterceptor('file')) async uploadPublicFile( - @Req() req: Express.Request, - @Res() res: unknown, - ): Promise { - // Verify that request is multipart - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - if (!req.isMultipart()) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - res.send(new BadRequestException('File expected on this endpoint')); // TODO + @Req() req: Request, + @Res() res: Response, + @UploadedFile() file: Express.Multer.File, + ): Promise { + // Verify that request contains file + if (!file) { + res.send(new BadRequestException('File expected on this endpoint')); return; } - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const file = await req.file(); - const fileBuffer = await file.toBuffer(); - const newFile = await this.taskService.uploadPublicFile( - fileBuffer, - file.filename, - ); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore + + // Actually upload via FileService + const newFile = await this.fileService.uploadPublicFile(file); + res.send(newFile); } @Post('/uploadPrivateFile') @LoggedIn() + @UseInterceptors(FileInterceptor('file')) async uploadPrivateFile( - @Req() req: Express.Request, - @Res() res: unknown, - ): Promise { - // Verify that request is multipart - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - if (!req.isMultipart()) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore + @Req() req: Request, + @Res() res: Response, + @UploadedFile() file: Express.Multer.File, + ): Promise { + // Verify that request contains file + if (!file) { res.send(new BadRequestException('File expected on this endpoint')); return; } - // Get user, as determined by JWT Strategy - const owner = req['user'].userId; + // Ensure userID is given + if (!req['user']?.userId) { + res.send(new UnauthorizedException()); + } - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const file = await req.file(); - const fileBuffer = await file.toBuffer(); - const newFile = await this.taskService.uploadPrivateFile( - fileBuffer, - file.filename, - owner, - ); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore + // Get user, as determined by JWT Strategy + const owner = req['user']?.userId; + const newFile = await this.fileService.uploadPrivateFile(file, owner); res.send(newFile); } } diff --git a/backend/src/flox/modules/file/file.resolver.ts b/backend/src/flox/modules/file/file.resolver.ts index c88fa7363..f3ff1a24f 100644 --- a/backend/src/flox/modules/file/file.resolver.ts +++ b/backend/src/flox/modules/file/file.resolver.ts @@ -1,10 +1,12 @@ -import { Args, Resolver, Query } from '@nestjs/graphql'; +import { Args, Resolver, Query, Mutation } from '@nestjs/graphql'; import PublicFile from './entities/public_file.entity'; import { FileService } from './file.service'; -import { GetPublicFileArgs } from './dto/get-public-file.args'; -import { GetPrivateFileArgs } from './dto/get-private-file.args'; +import { GetPublicFileArgs } from './dto/args/get-public-file.args'; +import { GetPrivateFileArgs } from './dto/args/get-private-file.args'; import PrivateFile from './entities/private_file.entity'; import { LoggedIn, Public } from '../auth/authentication.decorator'; +import { User } from '../auth/entities/user.entity'; +import { DeleteFileInput } from './dto/input/delete-file.input'; @Resolver(() => PublicFile) export class FileResolver { @@ -35,4 +37,40 @@ export class FileResolver { ): Promise { return this.fileService.getPrivateFile(getPrivateFileArgs); } + + /** + * Deletes a private file + * @param {DeleteFileInput} deleteFileInput - contains UUID + * @returns {Promise} - the file that was deleted + */ + @LoggedIn() // TODO application specific: set appropriate guards here + @Mutation(() => User) + async deletePrivateFile( + @Args('deleteFileInput') + deleteFileInput: DeleteFileInput, + ): Promise { + // TODO application specific: Ensure only allowed person (usually admin or file owner) is allowed to delete + return this.fileService.deleteFile( + deleteFileInput, + false, + ) as unknown as PrivateFile; + } + + /** + * Deletes a public file + * @param {DeleteFileInput} deleteFileInput - contains UUID + * @returns {Promise} - the file that was deleted + */ + @LoggedIn() // TODO application specific: set appropriate guards here + @Mutation(() => User) + async deletePublicFile( + @Args('deleteFileInput') + deleteFileInput: DeleteFileInput, + ): Promise { + // TODO application specific: Ensure only allowed person (usually admin ) is allowed to delete + return this.fileService.deleteFile( + deleteFileInput, + true, + ) as unknown as PublicFile; + } } diff --git a/backend/src/flox/modules/file/file.service.ts b/backend/src/flox/modules/file/file.service.ts index 93a21d3bb..fbd56bdc1 100644 --- a/backend/src/flox/modules/file/file.service.ts +++ b/backend/src/flox/modules/file/file.service.ts @@ -3,25 +3,33 @@ import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import PublicFile from './entities/public_file.entity'; import PrivateFile from './entities/private_file.entity'; -import { GetObjectCommand, PutObjectCommand, S3 } from '@aws-sdk/client-s3'; +import { + DeleteObjectCommand, + GetObjectCommand, + PutObjectCommand, + S3, +} from '@aws-sdk/client-s3'; import { ConfigService } from '@nestjs/config'; import { v4 as uuid } from 'uuid'; -import { GetPublicFileArgs } from './dto/get-public-file.args'; -import { GetPrivateFileArgs } from './dto/get-private-file.args'; +import { GetPublicFileArgs } from './dto/args/get-public-file.args'; +import { GetPrivateFileArgs } from './dto/args/get-private-file.args'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { DeleteFileInput } from './dto/input/delete-file.input'; @Injectable() export class FileService { - // TODO: When implementing file module, solve via .env / Terraform // S3 credentials - // private readonly credentials = { - // region: this.configService.get('AWS_MAIN_REGION'), - // accessKeyId: this.configService.get('AWS_S3_ACCESS_KEY_ID'), - // secretAccessKey: this.configService.get('AWS_S3_SECRET_ACCESS_KEY'), - // }; + private readonly credentials = { + region: this.configService.get('AWS_MAIN_REGION'), + accessKeyId: this.configService.get('AWS_ACCESS_KEY_ID'), + secretAccessKey: this.configService.get('AWS_SECRET_ACCESS_KEY'), + }; // AWS S3 instance - private s3: S3 = new S3({}); + private s3: S3 = new S3({ + credentials: this.credentials, + region: this.credentials.region, + }); constructor( @InjectRepository(PublicFile) private publicFilesRepository: Repository, @@ -34,28 +42,28 @@ export class FileService { /** * Uploads a file to the public S3 bucket - * @param {Buffer} dataBuffer - data buffer representation of the file to upload - * @param {string} filename - the file's name + * @param {Express.Multer.File} file - the file to upload * @returns {Promise} - the newly uploaded file */ - async uploadPublicFile( - dataBuffer: Buffer, - filename: string, - ): Promise { + async uploadPublicFile(file: Express.Multer.File): Promise { // File upload - const key = `${uuid()}-${filename}`; + const key = `${uuid()}-${file.originalname}`; const uploadParams = { Bucket: this.configService.get('AWS_PUBLIC_BUCKET_NAME'), Key: key, - Body: dataBuffer, + Body: file.buffer, + ContentType: file.mimetype, }; await this.s3.send(new PutObjectCommand(uploadParams)); const configService = new ConfigService(); + + const url = `https://${configService.get( + 'AWS_PUBLIC_BUCKET_NAME', + )}.s3.${configService.get('AWS_MAIN_REGION')}.amazonaws.com/${key}`; + const newFile = this.publicFilesRepository.create({ key: key, - url: `https://${configService.get( - 'AWS_PUBLIC_BUCKET_NAME', - )}.s3.${configService.get('AWS_MAIN_REGION')}.amazonaws.com/${key}`, + url: url, }); await this.publicFilesRepository.save(newFile); return newFile; @@ -63,22 +71,20 @@ export class FileService { /** * Uploads a file to the private S3 bucket - * @param {Buffer} dataBuffer - data buffer representation of the file to upload - * @param {string} filename - the file's name + * @param {Express.Multer.File} file - the file to upload * @param {string} owner - the file owner's UUID * @returns {Promise} - the newly uploaded file */ async uploadPrivateFile( - dataBuffer: Buffer, - filename: string, + file: Express.Multer.File, owner: string, ): Promise { //File upload - const key = `${uuid()}-${filename}`; + const key = `${uuid()}-${file.originalname}`; const uploadParams = { Bucket: this.configService.get('AWS_PRIVATE_BUCKET_NAME'), Key: key, - Body: dataBuffer, + Body: file.buffer, }; await this.s3.send(new PutObjectCommand(uploadParams)); const newFile = this.privateFilesRepository.create({ @@ -143,6 +149,50 @@ export class FileService { return { ...result, url }; } + // File not found: throw error + throw new NotFoundException(); + } + + /** + * Deletes a private or public file + * @param {DeleteFileInput} deleteFileInput - contains UUID + * @param {boolean} isPublic - whether the file is public (otherwise, is private) + * @returns {Promise} - the file that was deleted + */ + async deleteFile( + deleteFileInput: DeleteFileInput, + isPublic: boolean, + ): Promise { + const repository = isPublic + ? this.publicFilesRepository + : this.privateFilesRepository; + + const file: PrivateFile | PublicFile = await repository.findOne({ + where: { + uuid: deleteFileInput.uuid, + }, + }); + + if (file) { + // Delete on S3 + await this.s3.send( + new DeleteObjectCommand({ + Bucket: this.configService.get( + isPublic ? 'AWS_PUBLIC_BUCKET_NAME' : 'AWS_PRIVATE_BUCKET_NAME', + ), + Key: file.key, + }), + ); + + // Delete in database (TypeScript does not understand variable typing between PrivateFile / PublicFile here) + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const deletedFile = await repository.remove(file); + deletedFile.uuid = deleteFileInput.uuid; + return deletedFile; + } + + // File not found: throw error throw new NotFoundException(); } } diff --git a/backend/src/schema.gql b/backend/src/schema.gql index 2a5bf5836..7441e58b8 100644 --- a/backend/src/schema.gql +++ b/backend/src/schema.gql @@ -14,12 +14,18 @@ A date-time string at UTC, such as 2019-12-03T09:54:33Z, compliant with the date """ scalar DateTime +input DeleteFileInput { + uuid: ID! +} + input DeleteUserInput { uuid: ID! } type Mutation { createUser(createUserInput: CreateUserInput!): User! + deletePrivateFile(deleteFileInput: DeleteFileInput!): User! + deletePublicFile(deleteFileInput: DeleteFileInput!): User! deleteUser(deleteUserInput: DeleteUserInput!): User! updateUser(updateUserInput: UpdateUserInput!): User! } diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 8e9738d23..b2d5a01f1 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1721,6 +1721,19 @@ exec-sh "^0.3.2" minimist "^1.2.0" +"@esbuild-plugins/node-globals-polyfill@^0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@esbuild-plugins/node-globals-polyfill/-/node-globals-polyfill-0.1.1.tgz#a313ab3efbb2c17c8ce376aa216c627c9b40f9d7" + integrity sha512-MR0oAA+mlnJWrt1RQVQ+4VYuRJW/P2YmRTv1AsplObyvuBMnPHiizUF95HHYiSsMGLhyGtWufaq2XQg6+iurBg== + +"@esbuild-plugins/node-modules-polyfill@^0.1.4": + version "0.1.4" + resolved "https://registry.yarnpkg.com/@esbuild-plugins/node-modules-polyfill/-/node-modules-polyfill-0.1.4.tgz#eb2f55da11967b2986c913f1a7957d1c868849c0" + integrity sha512-uZbcXi0zbmKC/050p3gJnne5Qdzw8vkXIv+c2BW0Lsc1ji1SkrxbKPUy5Efr0blbTu1SL8w4eyfpnSdPg3G0Qg== + dependencies: + escape-string-regexp "^4.0.0" + rollup-plugin-node-polyfills "^0.2.1" + "@esbuild/linux-loong64@0.14.54": version "0.14.54" resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz#de2a4be678bd4d0d1ffbb86e6de779cde5999028" @@ -6641,6 +6654,11 @@ estraverse@^5.1.0, estraverse@^5.2.0: resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== +estree-walker@^0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362" + integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w== + estree-walker@^2.0.1, estree-walker@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/estree-walker/-/estree-walker-2.0.2.tgz#52f010178c2a4c117a7757cfe942adb7d2da4cac" @@ -9880,7 +9898,7 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -magic-string@^0.25.7: +magic-string@^0.25.3, magic-string@^0.25.7: version "0.25.9" resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.9.tgz#de7f9faf91ef8a1c91d02c2e5314c8277dbcdd1c" integrity sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ== @@ -11979,6 +11997,22 @@ ripemd160@^2.0.0, ripemd160@^2.0.1: hash-base "^3.0.0" inherits "^2.0.1" +rollup-plugin-inject@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rollup-plugin-inject/-/rollup-plugin-inject-3.0.2.tgz#e4233855bfba6c0c12a312fd6649dff9a13ee9f4" + integrity sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w== + dependencies: + estree-walker "^0.6.1" + magic-string "^0.25.3" + rollup-pluginutils "^2.8.1" + +rollup-plugin-node-polyfills@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/rollup-plugin-node-polyfills/-/rollup-plugin-node-polyfills-0.2.1.tgz#53092a2744837164d5b8a28812ba5f3ff61109fd" + integrity sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA== + dependencies: + rollup-plugin-inject "^3.0.0" + rollup-plugin-visualizer@^5.5.4: version "5.8.0" resolved "https://registry.yarnpkg.com/rollup-plugin-visualizer/-/rollup-plugin-visualizer-5.8.0.tgz#32f2fe23d4299e977c06c59c07255590354e3445" @@ -11989,6 +12023,13 @@ rollup-plugin-visualizer@^5.5.4: source-map "^0.7.3" yargs "^17.5.1" +rollup-pluginutils@^2.8.1: + version "2.8.2" + resolved "https://registry.yarnpkg.com/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz#72f2af0748b592364dbd3389e600e5a9444a351e" + integrity sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ== + dependencies: + estree-walker "^0.6.1" + "rollup@>=2.59.0 <2.78.0": version "2.77.3" resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.77.3.tgz#8f00418d3a2740036e15deb653bed1a90ee0cc12"