From 4e5972e8f444d1a6d1558167dff6acd2020e1f0e Mon Sep 17 00:00:00 2001 From: Joel Barmettler Date: Wed, 12 Oct 2022 12:55:51 +0200 Subject: [PATCH 01/17] feat: Image module including some AI features (#433) * file abstraction * image * image endpoints * image endpoints * add file fetching * file data in frontend * update schema * file module in frontend * basic security for files * wrapped file as image * why is null never accepted :( * fix not null problem * start with rekognition * label display * cleanup * add config and rename S3File back to File in comments and stringss * new image module comment * fix gps coordinate type * fix linting issues * small ux improvements that allow easier demo * Apply suggestions from code review Co-authored-by: David Wyss * add message to i18n * remove ownership teswt from service * add old CognitoUser decorator * satisfy linter * Update backend/src/flox/modules/image/image.resolver.ts Co-authored-by: David Wyss Co-authored-by: David Wyss --- backend/flox.config.json | 1 + backend/package.json | 2 + backend/src/flox/MODULES.ts | 1 + backend/src/flox/flox.ts | 4 + .../src/flox/modules/auth/user.resolver.ts | 4 +- backend/src/flox/modules/auth/user.service.ts | 13 +- backend/src/flox/modules/email/config.ts | 2 +- backend/src/flox/modules/file/config.ts | 6 +- .../file/dto/args/get-all-files.args.ts | 16 + .../file/dto/args/get-private-file.args.ts | 6 +- .../flox/modules/file/entities/file.entity.ts | 38 ++ ...e_file.entity.ts => privateFile.entity.ts} | 11 +- ...ic_file.entity.ts => publicFile.entity.ts} | 11 +- .../src/flox/modules/file/file.controller.ts | 9 +- backend/src/flox/modules/file/file.module.ts | 8 +- .../src/flox/modules/file/file.resolver.ts | 74 ++- backend/src/flox/modules/file/file.service.ts | 58 +- backend/src/flox/modules/image/config.ts | 30 + .../image/dto/args/get-all-images.args.ts | 16 + .../image/dto/args/get-image-for-file.args.ts | 17 + .../modules/image/dto/args/get-image.args.ts | 17 + .../image/dto/input/create-image.input.ts | 16 + .../image/dto/input/create-labels.input.ts | 9 + .../image/dto/input/delete-image.input.ts | 9 + .../image/entities/bounding-box.entity.ts | 38 ++ .../modules/image/entities/image.entity.ts | 84 +++ .../modules/image/entities/label.entity.ts | 42 ++ .../src/flox/modules/image/image.module.ts | 14 + .../src/flox/modules/image/image.resolver.ts | 116 ++++ .../src/flox/modules/image/image.service.ts | 211 +++++++ .../modules/roles/authorization.decorator.ts | 12 +- backend/src/flox/modules/roles/roles.guard.ts | 1 + backend/src/schema.gql | 154 ++++- backend/yarn.lock | 585 ++++++++++++++++++ frontend/flox.config.json | 1 + frontend/package.json | 2 +- frontend/src/data/mutations/FILES.ts | 28 + frontend/src/data/mutations/IMAGE.ts | 43 ++ frontend/src/data/queries/FILES.ts | 78 +++ frontend/src/data/queries/IMAGE.ts | 58 ++ frontend/src/data/types/BaseEntity.ts | 20 +- frontend/src/data/types/ImageFile.ts | 23 + frontend/src/data/types/PrivateFile.ts | 22 + frontend/src/data/types/PublicFile.ts | 20 + frontend/src/data/types/S3File.ts | 21 + frontend/src/data/types/User.ts | 10 +- .../components/forms/fields/FileUpload.vue | 2 +- .../file/components/tables/FilesTable.vue | 116 ++++ frontend/src/flox/modules/file/index.ts | 4 +- .../modules/image/components/LabeledImage.vue | 118 ++++ frontend/src/helpers/data/fetch-helpers.ts | 87 ++- frontend/src/helpers/data/mutation-helpers.ts | 53 +- frontend/src/helpers/tools/file-helpers.ts | 18 +- .../src/helpers/tools/image-helpers.spec.ts | 2 +- frontend/src/i18n/de/index.ts | 4 + frontend/src/i18n/en/index.ts | 4 + frontend/src/pages/SamplePage.vue | 19 +- 57 files changed, 2281 insertions(+), 107 deletions(-) create mode 100644 backend/src/flox/modules/file/dto/args/get-all-files.args.ts create mode 100644 backend/src/flox/modules/file/entities/file.entity.ts rename backend/src/flox/modules/file/entities/{private_file.entity.ts => privateFile.entity.ts} (65%) rename backend/src/flox/modules/file/entities/{public_file.entity.ts => publicFile.entity.ts} (53%) create mode 100644 backend/src/flox/modules/image/config.ts create mode 100644 backend/src/flox/modules/image/dto/args/get-all-images.args.ts create mode 100644 backend/src/flox/modules/image/dto/args/get-image-for-file.args.ts create mode 100644 backend/src/flox/modules/image/dto/args/get-image.args.ts create mode 100644 backend/src/flox/modules/image/dto/input/create-image.input.ts create mode 100644 backend/src/flox/modules/image/dto/input/create-labels.input.ts create mode 100644 backend/src/flox/modules/image/dto/input/delete-image.input.ts create mode 100644 backend/src/flox/modules/image/entities/bounding-box.entity.ts create mode 100644 backend/src/flox/modules/image/entities/image.entity.ts create mode 100644 backend/src/flox/modules/image/entities/label.entity.ts create mode 100644 backend/src/flox/modules/image/image.module.ts create mode 100644 backend/src/flox/modules/image/image.resolver.ts create mode 100644 backend/src/flox/modules/image/image.service.ts create mode 100644 frontend/src/data/mutations/FILES.ts create mode 100644 frontend/src/data/mutations/IMAGE.ts create mode 100644 frontend/src/data/queries/FILES.ts create mode 100644 frontend/src/data/queries/IMAGE.ts create mode 100644 frontend/src/data/types/ImageFile.ts create mode 100644 frontend/src/data/types/PrivateFile.ts create mode 100644 frontend/src/data/types/PublicFile.ts create mode 100644 frontend/src/data/types/S3File.ts create mode 100644 frontend/src/flox/modules/file/components/tables/FilesTable.vue create mode 100644 frontend/src/flox/modules/image/components/LabeledImage.vue diff --git a/backend/flox.config.json b/backend/flox.config.json index e6f76ffb0..9f928fa9f 100644 --- a/backend/flox.config.json +++ b/backend/flox.config.json @@ -3,6 +3,7 @@ "auth": true, "roles": true, "file": true, + "image": true, "sharing": false }, "moduleOptions": { diff --git a/backend/package.json b/backend/package.json index 929b5a16c..cd8b489bf 100644 --- a/backend/package.json +++ b/backend/package.json @@ -24,6 +24,7 @@ "docker:dev": "docker-compose up --build -V" }, "dependencies": { + "@aws-sdk/client-rekognition": "^3.185.0", "@aws-sdk/client-s3": "^3.41.0", "@aws-sdk/client-ses": "^3.112.0", "@aws-sdk/s3-request-presigner": "^3.42.0", @@ -41,6 +42,7 @@ "@vendia/serverless-express": "^4.9.0", "apollo-server-express": "^3.10.0", "class-validator": "0.13.2", + "exifr": "^7.1.3", "express": "4.18.1", "graphql": "^15.6.1", "graphql-subscriptions": "^1.2.1", diff --git a/backend/src/flox/MODULES.ts b/backend/src/flox/MODULES.ts index af26aeaf8..419051666 100644 --- a/backend/src/flox/MODULES.ts +++ b/backend/src/flox/MODULES.ts @@ -3,6 +3,7 @@ export enum MODULES { AUTH = 'auth', ROLES = 'roles', FILE = 'file', + IMAGE = 'image', SHARING = 'sharing', EMAIL = 'email', } diff --git a/backend/src/flox/flox.ts b/backend/src/flox/flox.ts index 3b2a3f4d8..c35003d86 100644 --- a/backend/src/flox/flox.ts +++ b/backend/src/flox/flox.ts @@ -3,6 +3,7 @@ import { RolesGuard } from './modules/roles/roles.guard'; import { JwtStrategy } from './modules/auth/jwt.strategy'; import { JwtAuthGuard } from './modules/auth/auth.guard'; import { FileModule } from './modules/file/file.module'; +import { ImageModule } from './modules/image/image.module'; import { UserModule } from './modules/auth/user.module'; import { MODULES } from './MODULES'; import { EmailModule } from './modules/email/email.module'; @@ -23,6 +24,9 @@ export function floxModules() { case MODULES.FILE: modules.push(FileModule); break; + case MODULES.IMAGE: + modules.push(ImageModule); + break; case MODULES.AUTH: modules.push(UserModule); break; diff --git a/backend/src/flox/modules/auth/user.resolver.ts b/backend/src/flox/modules/auth/user.resolver.ts index 05edb2b32..0da8ba080 100644 --- a/backend/src/flox/modules/auth/user.resolver.ts +++ b/backend/src/flox/modules/auth/user.resolver.ts @@ -1,4 +1,4 @@ -import { Resolver, Query, Mutation, Args } from '@nestjs/graphql'; +import { Args, Mutation, Query, Resolver } from '@nestjs/graphql'; import { UserService } from './user.service'; import { CreateUserInput } from './dto/input/create-user.input'; import { UpdateUserInput } from './dto/input/update-user.input'; @@ -91,7 +91,7 @@ export class UserResolver { */ @LoggedIn() @Query(() => User, { name: 'myUser' }) - async myUser(@CurrentUser() user: Record): Promise { + async myUser(@CurrentUser() user: User): Promise { // Get user where user's UUID matches Cognito ID return this.userService.getMyUser(user); } diff --git a/backend/src/flox/modules/auth/user.service.ts b/backend/src/flox/modules/auth/user.service.ts index f4c489605..84f4172c6 100644 --- a/backend/src/flox/modules/auth/user.service.ts +++ b/backend/src/flox/modules/auth/user.service.ts @@ -99,19 +99,14 @@ export class UserService { /** * Return current user given the Cognito user from the request - * @param {Record} cognitoUser - cognito user from request + * @param {User} user - database user from request * @returns {Promise} - user */ - async getMyUser(cognitoUser: Record): Promise { - const myUser = await this.userRepository.findOne({ + async getMyUser(user: User): Promise { + return this.userRepository.findOneOrFail({ where: { - cognitoUuid: cognitoUser.userId, + uuid: user.uuid, }, }); - - if (!myUser) { - throw new Error(`No user found for ${cognitoUser.userId}`); - } - return myUser; } } diff --git a/backend/src/flox/modules/email/config.ts b/backend/src/flox/modules/email/config.ts index bdd71a0e4..089b7d3af 100644 --- a/backend/src/flox/modules/email/config.ts +++ b/backend/src/flox/modules/email/config.ts @@ -5,7 +5,7 @@ import { import { MODULES } from '../../MODULES'; /** - * The file module handles file up/download using a database table each for private and public files, as well as storing + * The file module handles file up/download using a database tables each for private and public files, as well as storing * the files in S3 and requesting corresponding URLs. */ diff --git a/backend/src/flox/modules/file/config.ts b/backend/src/flox/modules/file/config.ts index 3fff48a9f..2c70c76bd 100644 --- a/backend/src/flox/modules/file/config.ts +++ b/backend/src/flox/modules/file/config.ts @@ -1,5 +1,7 @@ -import { mergeConfigurations } from '../../core/flox-helpers'; -import { floxModuleOptions } from '../../core/flox-helpers'; +import { + floxModuleOptions, + mergeConfigurations, +} from '../../core/flox-helpers'; import { MODULES } from '../../MODULES'; /** diff --git a/backend/src/flox/modules/file/dto/args/get-all-files.args.ts b/backend/src/flox/modules/file/dto/args/get-all-files.args.ts new file mode 100644 index 000000000..d7a2d92af --- /dev/null +++ b/backend/src/flox/modules/file/dto/args/get-all-files.args.ts @@ -0,0 +1,16 @@ +import { ArgsType, Field, Int } from '@nestjs/graphql'; + +@ArgsType() +export class GetAllFilesArgs { + @Field(() => Int, { + defaultValue: 500, + description: 'Number of files to load', + }) + limit = 500; + + @Field(() => Int, { + defaultValue: 0, + description: 'How many files to skip', + }) + skip = 0; +} diff --git a/backend/src/flox/modules/file/dto/args/get-private-file.args.ts b/backend/src/flox/modules/file/dto/args/get-private-file.args.ts index 3f2522952..70d22b9c4 100644 --- a/backend/src/flox/modules/file/dto/args/get-private-file.args.ts +++ b/backend/src/flox/modules/file/dto/args/get-private-file.args.ts @@ -1,4 +1,4 @@ -import { ArgsType, Field, ID } from '@nestjs/graphql'; +import { ArgsType, Field, ID, Int } from '@nestjs/graphql'; import { IsNumber, IsOptional, IsUUID } from 'class-validator'; @ArgsType() @@ -7,11 +7,11 @@ export class GetPrivateFileArgs { @IsUUID() uuid: string; - @Field(() => [Number], { + @Field(() => Int, { nullable: true, description: 'URL expiration duration (in seconds)', }) @IsOptional() @IsNumber() - expires; + expires?: number; } diff --git a/backend/src/flox/modules/file/entities/file.entity.ts b/backend/src/flox/modules/file/entities/file.entity.ts new file mode 100644 index 000000000..b5b597941 --- /dev/null +++ b/backend/src/flox/modules/file/entities/file.entity.ts @@ -0,0 +1,38 @@ +import { Column, Entity } from 'typeorm'; +import { BaseEntity } from '../../../core/base-entity/entities/base-entity.entity'; +import { Field, ObjectType } from '@nestjs/graphql'; +import { IsNumber, IsOptional, IsString } from 'class-validator'; + +/** + * Defines a file within an AWS S3 bucket + */ + +@Entity() +@ObjectType() +export abstract class S3File extends BaseEntity { + @Field(() => String, { description: 'S3 File Key' }) + @Column() + @IsString() + public key: string; + + @Field(() => String, { description: 'Files mime type' }) + @Column() + @IsString() + public mimetype: string; + + @Field(() => String, { + nullable: true, + description: 'Name of File', + }) + @Column() + @IsOptional() + @IsString() + public filename: string; + + @Field(() => Number, { description: 'Filesize in bytes' }) + @Column() + @IsNumber() + public size: number; +} + +export default S3File; diff --git a/backend/src/flox/modules/file/entities/private_file.entity.ts b/backend/src/flox/modules/file/entities/privateFile.entity.ts similarity index 65% rename from backend/src/flox/modules/file/entities/private_file.entity.ts rename to backend/src/flox/modules/file/entities/privateFile.entity.ts index 7a0f30de1..be797dcce 100644 --- a/backend/src/flox/modules/file/entities/private_file.entity.ts +++ b/backend/src/flox/modules/file/entities/privateFile.entity.ts @@ -1,7 +1,7 @@ import { Column, Entity } from 'typeorm'; -import { BaseEntity } from '../../../core/base-entity/entities/base-entity.entity'; import { Field, ObjectType } from '@nestjs/graphql'; -import { IsOptional, IsString, IsUrl, IsUUID } from 'class-validator'; +import { IsOptional, IsUrl, IsUUID } from 'class-validator'; +import S3File from './file.entity'; /** * Defines a private file within a restricted AWS S3 bucket. @@ -10,17 +10,12 @@ import { IsOptional, IsString, IsUrl, IsUUID } from 'class-validator'; @Entity() @ObjectType() -export class PrivateFile extends BaseEntity { +export class PrivateFile extends S3File { @Field(() => String, { description: 'File owner' }) @Column() @IsUUID() public owner: string; - @Field(() => String, { description: 'S3 File Key' }) - @Column() - @IsString() - public key: string; - @Field(() => String, { nullable: true, description: 'Pre-signed download URL', diff --git a/backend/src/flox/modules/file/entities/public_file.entity.ts b/backend/src/flox/modules/file/entities/publicFile.entity.ts similarity index 53% rename from backend/src/flox/modules/file/entities/public_file.entity.ts rename to backend/src/flox/modules/file/entities/publicFile.entity.ts index 3621457dc..344b1ca31 100644 --- a/backend/src/flox/modules/file/entities/public_file.entity.ts +++ b/backend/src/flox/modules/file/entities/publicFile.entity.ts @@ -1,7 +1,7 @@ import { Column, Entity } from 'typeorm'; -import { BaseEntity } from '../../../core/base-entity/entities/base-entity.entity'; import { Field, ObjectType } from '@nestjs/graphql'; -import { IsString, IsUrl } from 'class-validator'; +import { IsUrl } from 'class-validator'; +import S3File from './file.entity'; /** * Defines a public file within a public AWS S3 bucket @@ -9,16 +9,11 @@ import { IsString, IsUrl } from 'class-validator'; @Entity() @ObjectType() -export class PublicFile extends BaseEntity { +export class PublicFile extends S3File { @Field(() => String, { description: 'Public download URL' }) @Column() @IsUrl() public url: string; - - @Field(() => String, { description: 'S3 File Key' }) - @Column() - @IsString() - public key: string; } export default PublicFile; diff --git a/backend/src/flox/modules/file/file.controller.ts b/backend/src/flox/modules/file/file.controller.ts index 4c04add00..8168ac960 100644 --- a/backend/src/flox/modules/file/file.controller.ts +++ b/backend/src/flox/modules/file/file.controller.ts @@ -10,8 +10,10 @@ import { } from '@nestjs/common'; import { FileService } from './file.service'; import { LoggedIn, Public } from '../auth/authentication.decorator'; -import { Response, Request } from 'express'; +import { Request, Response } from 'express'; import { FileInterceptor } from '@nestjs/platform-express'; +import { CurrentUser } from '../roles/authorization.decorator'; +import { User } from '../auth/entities/user.entity'; @Controller() export class FileController { @@ -44,6 +46,7 @@ export class FileController { @Req() req: Request, @Res() res: Response, @UploadedFile() file: Express.Multer.File, + @CurrentUser() user: User, ): Promise { // Verify that request contains file if (!file) { @@ -56,9 +59,7 @@ export class FileController { res.send(new UnauthorizedException()); } - // Get user, as determined by JWT Strategy - const owner = req['user']?.userId; - const newFile = await this.fileService.uploadPrivateFile(file, owner); + const newFile = await this.fileService.uploadPrivateFile(file, user); res.send(newFile); } } diff --git a/backend/src/flox/modules/file/file.module.ts b/backend/src/flox/modules/file/file.module.ts index 28d5892bc..7c37fdad9 100644 --- a/backend/src/flox/modules/file/file.module.ts +++ b/backend/src/flox/modules/file/file.module.ts @@ -2,13 +2,15 @@ import { Module } from '@nestjs/common'; import { FileService } from './file.service'; import { FileController } from './file.controller'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { PublicFile } from './entities/public_file.entity'; +import { PublicFile } from './entities/publicFile.entity'; import { FileResolver } from './file.resolver'; -import { PrivateFile } from './entities/private_file.entity'; +import { PrivateFile } from './entities/privateFile.entity'; +import S3File from './entities/file.entity'; @Module({ - imports: [TypeOrmModule.forFeature([PublicFile, PrivateFile])], + imports: [TypeOrmModule.forFeature([PublicFile, PrivateFile, S3File])], providers: [FileService, FileResolver], controllers: [FileController], + exports: [FileService], }) export class FileModule {} diff --git a/backend/src/flox/modules/file/file.resolver.ts b/backend/src/flox/modules/file/file.resolver.ts index f3ff1a24f..cced6999b 100644 --- a/backend/src/flox/modules/file/file.resolver.ts +++ b/backend/src/flox/modules/file/file.resolver.ts @@ -1,24 +1,57 @@ -import { Args, Resolver, Query, Mutation } from '@nestjs/graphql'; -import PublicFile from './entities/public_file.entity'; +import { Args, Mutation, Query, Resolver } from '@nestjs/graphql'; +import PublicFile from './entities/publicFile.entity'; import { FileService } from './file.service'; import { GetPublicFileArgs } from './dto/args/get-public-file.args'; import { GetPrivateFileArgs } from './dto/args/get-private-file.args'; -import PrivateFile from './entities/private_file.entity'; +import PrivateFile from './entities/privateFile.entity'; import { LoggedIn, Public } from '../auth/authentication.decorator'; -import { User } from '../auth/entities/user.entity'; import { DeleteFileInput } from './dto/input/delete-file.input'; +import { GetAllFilesArgs } from './dto/args/get-all-files.args'; +import { AdminOnly, CurrentUser } from '../roles/authorization.decorator'; +import { User } from '../auth/entities/user.entity'; +import { DEFAULT_ROLES } from '../roles/config'; +import { ForbiddenError } from 'apollo-server-express'; @Resolver(() => PublicFile) export class FileResolver { constructor(private readonly fileService: FileService) {} + /** + * Returns all public files stored within database + * @param {GetAllFilesArgs} getAllFilesArgs - contains limit and skip parameters + * @returns {Promise} List of public files + */ + @AdminOnly() + @Query(() => [PublicFile], { name: 'allPublicFiles' }) + async getAllPublicFiles( + @Args() getAllFilesArgs: GetAllFilesArgs, + ): Promise { + return this.fileService.getAllPublicFiles(getAllFilesArgs); + } + + /** + * Returns private files of logged in user + * @param {GetAllFilesArgs} getAllFilesArgs - contains limit and skip parameters + * @param {User} user - currently logged in user + * @returns {Promise} Users private files + */ + @LoggedIn() // TODO application specific: set appropriate guards here + @Query(() => [PrivateFile], { name: 'allMyFiles' }) + async getAllMyFiles( + @Args() getAllFilesArgs: GetAllFilesArgs, + @CurrentUser() user: User, + ): Promise { + console.log(user); + return this.fileService.getAllMyFiles(getAllFilesArgs, user); + } + /** * Gets a public file * @param {GetPublicFileArgs} getPublicFileArgs - contains UUID * @returns {Promise} - the file */ @Public() - @Query(() => PublicFile, { name: 'getPublicFile' }) + @Query(() => PublicFile, { name: 'publicFile' }) async getPublicFile( @Args() getPublicFileArgs: GetPublicFileArgs, ): Promise { @@ -28,28 +61,42 @@ export class FileResolver { /** * Gets a private file * @param {GetPrivateFileArgs} getPrivateFileArgs - contains UUID and optionally, expiration time + * @param {User} user - logged-in user * @returns {Promise} - the file, if the user is allowed to access it */ @LoggedIn() // TODO application specific: set appropriate guards here - @Query(() => PrivateFile, { name: 'getPrivateFile' }) + @Query(() => PrivateFile, { name: 'privateFile' }) async getPrivateFile( @Args() getPrivateFileArgs: GetPrivateFileArgs, + @CurrentUser() user: User, ): Promise { - return this.fileService.getPrivateFile(getPrivateFileArgs); + const file = await this.fileService.getPrivateFile(getPrivateFileArgs); + if (user.role !== DEFAULT_ROLES.ADMIN || file.owner !== user.uuid) { + throw new ForbiddenError('File does not belong to logged in user'); + } + return file; } /** * Deletes a private file * @param {DeleteFileInput} deleteFileInput - contains UUID + * @param {User} user - logged-in user * @returns {Promise} - the file that was deleted */ @LoggedIn() // TODO application specific: set appropriate guards here - @Mutation(() => User) + @Mutation(() => PrivateFile) async deletePrivateFile( - @Args('deleteFileInput') - deleteFileInput: DeleteFileInput, + @Args('deleteFileInput') deleteFileInput: DeleteFileInput, + @CurrentUser() user: User, ): Promise { - // TODO application specific: Ensure only allowed person (usually admin or file owner) is allowed to delete + const file = await this.fileService.getPrivateFile({ + uuid: deleteFileInput.uuid, + } as GetPrivateFileArgs); + if (user.role !== DEFAULT_ROLES.ADMIN || file.owner !== user.uuid) { + throw new ForbiddenError( + 'Cannot delete file that does not belong to user', + ); + } return this.fileService.deleteFile( deleteFileInput, false, @@ -61,13 +108,12 @@ export class FileResolver { * @param {DeleteFileInput} deleteFileInput - contains UUID * @returns {Promise} - the file that was deleted */ - @LoggedIn() // TODO application specific: set appropriate guards here - @Mutation(() => User) + @AdminOnly() // TODO application specific: set appropriate guards here + @Mutation(() => PublicFile) async deletePublicFile( @Args('deleteFileInput') deleteFileInput: DeleteFileInput, ): Promise { - // TODO application specific: Ensure only allowed person (usually admin ) is allowed to delete return this.fileService.deleteFile( deleteFileInput, true, diff --git a/backend/src/flox/modules/file/file.service.ts b/backend/src/flox/modules/file/file.service.ts index fbd56bdc1..80c179b62 100644 --- a/backend/src/flox/modules/file/file.service.ts +++ b/backend/src/flox/modules/file/file.service.ts @@ -1,8 +1,8 @@ import { Injectable, NotFoundException } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; -import PublicFile from './entities/public_file.entity'; -import PrivateFile from './entities/private_file.entity'; +import PublicFile from './entities/publicFile.entity'; +import PrivateFile from './entities/privateFile.entity'; import { DeleteObjectCommand, GetObjectCommand, @@ -15,6 +15,8 @@ import { GetPublicFileArgs } from './dto/args/get-public-file.args'; import { GetPrivateFileArgs } from './dto/args/get-private-file.args'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { DeleteFileInput } from './dto/input/delete-file.input'; +import { GetAllFilesArgs } from './dto/args/get-all-files.args'; +import { User } from '../auth/entities/user.entity'; @Injectable() export class FileService { @@ -62,8 +64,11 @@ export class FileService { )}.s3.${configService.get('AWS_MAIN_REGION')}.amazonaws.com/${key}`; const newFile = this.publicFilesRepository.create({ - key: key, - url: url, + key, + url, + mimetype: file.mimetype, + size: file.size, + filename: file.filename || file.originalname, }); await this.publicFilesRepository.save(newFile); return newFile; @@ -72,12 +77,12 @@ export class FileService { /** * Uploads a file to the private S3 bucket * @param {Express.Multer.File} file - the file to upload - * @param {string} owner - the file owner's UUID + * @param {User} owner - the file owner * @returns {Promise} - the newly uploaded file */ async uploadPrivateFile( file: Express.Multer.File, - owner: string, + owner: User, ): Promise { //File upload const key = `${uuid()}-${file.originalname}`; @@ -88,8 +93,11 @@ export class FileService { }; await this.s3.send(new PutObjectCommand(uploadParams)); const newFile = this.privateFilesRepository.create({ - key: key, - owner: owner, + key, + owner: owner.uuid, + mimetype: file.mimetype, + size: file.size, + filename: file.filename || file.originalname, }); await this.privateFilesRepository.save(newFile); return newFile; @@ -110,6 +118,40 @@ export class FileService { }); } + /** + * Returns all public files stored within database + * @param {GetAllFilesArgs} getAllFilesArgs - contains limit and skip parameters + * @returns {Promise} List of public files + */ + async getAllPublicFiles( + getAllFilesArgs: GetAllFilesArgs, + ): Promise> { + return this.publicFilesRepository.find({ + take: getAllFilesArgs.limit, + skip: getAllFilesArgs.skip, + }); + } + + /** + * Returns private files of logged in user + * @param {GetAllFilesArgs} getAllFilesArgs - contains limit and skip parameters + * @param {User} user - currently logged in user + * @returns {Promise} Users private files + */ + async getAllMyFiles( + getAllFilesArgs: GetAllFilesArgs, + user: User, + ): Promise> { + const usersFileUUIDs = await this.privateFilesRepository.find({ + where: { + owner: user.uuid, + }, + }); + return Promise.all( + usersFileUUIDs.map((file) => this.getPrivateFile({ uuid: file.uuid })), + ); + } + /** * Gets a private file * @param {GetPrivateFileArgs} getPrivateFileArgs - contains UUID and optionally, expiration time diff --git a/backend/src/flox/modules/image/config.ts b/backend/src/flox/modules/image/config.ts new file mode 100644 index 000000000..e2c785bd8 --- /dev/null +++ b/backend/src/flox/modules/image/config.ts @@ -0,0 +1,30 @@ +import { + floxModuleOptions, + mergeConfigurations, +} from '../../core/flox-helpers'; +import { MODULES } from '../../MODULES'; + +/** + * The image module handles files that contain images and contains functionality to extract information from these + * images, such as object recognition. + */ + +type ImageModuleConfig = { + // Image module has no options +}; + +// Default configuration set; will get merged with custom config from flox.config.json +const defaultConfig: ImageModuleConfig = { + // Image module has no options +}; + +/** + * Gets the module's actual configuration + * @returns {FileModuleConfig} - configuration + */ +export function moduleConfig() { + return mergeConfigurations( + defaultConfig, + floxModuleOptions(MODULES.IMAGE), + ) as ImageModuleConfig; +} diff --git a/backend/src/flox/modules/image/dto/args/get-all-images.args.ts b/backend/src/flox/modules/image/dto/args/get-all-images.args.ts new file mode 100644 index 000000000..66ba4b28c --- /dev/null +++ b/backend/src/flox/modules/image/dto/args/get-all-images.args.ts @@ -0,0 +1,16 @@ +import { ArgsType, Field, Int } from '@nestjs/graphql'; + +@ArgsType() +export class GetAllImagesArgs { + @Field(() => Int, { + defaultValue: 500, + description: 'Number of images to load', + }) + limit = 500; + + @Field(() => Int, { + defaultValue: 0, + description: 'How many images to skip', + }) + skip = 0; +} diff --git a/backend/src/flox/modules/image/dto/args/get-image-for-file.args.ts b/backend/src/flox/modules/image/dto/args/get-image-for-file.args.ts new file mode 100644 index 000000000..0470d2303 --- /dev/null +++ b/backend/src/flox/modules/image/dto/args/get-image-for-file.args.ts @@ -0,0 +1,17 @@ +import { ArgsType, Field, ID } from '@nestjs/graphql'; +import { IsNumber, IsOptional, IsUUID } from 'class-validator'; + +@ArgsType() +export class GetImageForFileArgs { + @Field(() => ID) + @IsUUID() + file: string; + + @Field(() => [Number], { + nullable: true, + description: 'URL expiration duration (in seconds)', + }) + @IsOptional() + @IsNumber() + expires; +} diff --git a/backend/src/flox/modules/image/dto/args/get-image.args.ts b/backend/src/flox/modules/image/dto/args/get-image.args.ts new file mode 100644 index 000000000..3b2dc77e5 --- /dev/null +++ b/backend/src/flox/modules/image/dto/args/get-image.args.ts @@ -0,0 +1,17 @@ +import { ArgsType, Field, ID } from '@nestjs/graphql'; +import { IsNumber, IsOptional, IsUUID } from 'class-validator'; + +@ArgsType() +export class GetImageArgs { + @Field(() => ID) + @IsUUID() + uuid: string; + + @Field(() => [Number], { + nullable: true, + description: 'URL expiration duration (in seconds)', + }) + @IsOptional() + @IsNumber() + expires; +} diff --git a/backend/src/flox/modules/image/dto/input/create-image.input.ts b/backend/src/flox/modules/image/dto/input/create-image.input.ts new file mode 100644 index 000000000..6eeb0cde7 --- /dev/null +++ b/backend/src/flox/modules/image/dto/input/create-image.input.ts @@ -0,0 +1,16 @@ +import { Field, ID, InputType } from '@nestjs/graphql'; +import { IsBoolean, IsOptional, IsUUID } from 'class-validator'; + +@InputType() +export class CreateImageInput { + @Field(() => ID) + @IsUUID() + file: string; + + @Field(() => Boolean, { + defaultValue: false, + }) + @IsOptional() + @IsBoolean() + objectRecognition = false; +} diff --git a/backend/src/flox/modules/image/dto/input/create-labels.input.ts b/backend/src/flox/modules/image/dto/input/create-labels.input.ts new file mode 100644 index 000000000..6b5664756 --- /dev/null +++ b/backend/src/flox/modules/image/dto/input/create-labels.input.ts @@ -0,0 +1,9 @@ +import { Field, ID, InputType } from '@nestjs/graphql'; +import { IsUUID } from 'class-validator'; + +@InputType() +export class CreateLabelsInput { + @Field(() => ID) + @IsUUID() + image: string; +} diff --git a/backend/src/flox/modules/image/dto/input/delete-image.input.ts b/backend/src/flox/modules/image/dto/input/delete-image.input.ts new file mode 100644 index 000000000..d6e37936f --- /dev/null +++ b/backend/src/flox/modules/image/dto/input/delete-image.input.ts @@ -0,0 +1,9 @@ +import { Field, ID, InputType } from '@nestjs/graphql'; +import { IsUUID } from 'class-validator'; + +@InputType() +export class DeleteImageInput { + @Field(() => ID) + @IsUUID() + uuid: string; +} diff --git a/backend/src/flox/modules/image/entities/bounding-box.entity.ts b/backend/src/flox/modules/image/entities/bounding-box.entity.ts new file mode 100644 index 000000000..d34d7ee99 --- /dev/null +++ b/backend/src/flox/modules/image/entities/bounding-box.entity.ts @@ -0,0 +1,38 @@ +import { Column, Entity } from 'typeorm'; +import { Field, ObjectType } from '@nestjs/graphql'; +import { BaseEntity } from '../../../core/base-entity/entities/base-entity.entity'; +import { IsNumber } from 'class-validator'; + +@Entity() +@ObjectType() +export class BoundingBox extends BaseEntity { + @Field(() => Number, { + description: 'Bounding-Box width in percentage of image width', + }) + @Column('float8') + @IsNumber() + public width: number; + + @Field(() => Number, { + description: 'Bounding-Box height in percentage of image height', + }) + @Column('float8') + @IsNumber() + public height: number; + + @Field(() => Number, { + description: + 'Bounding-Box position from the left side of the image, in percentage', + }) + @Column('float8') + @IsNumber() + public left: number; + + @Field(() => Number, { + description: + 'Bounding-Box position from the top side of the image, in percentage', + }) + @Column('float8') + @IsNumber() + public top: number; +} diff --git a/backend/src/flox/modules/image/entities/image.entity.ts b/backend/src/flox/modules/image/entities/image.entity.ts new file mode 100644 index 000000000..970c101e8 --- /dev/null +++ b/backend/src/flox/modules/image/entities/image.entity.ts @@ -0,0 +1,84 @@ +import { Column, Entity, JoinColumn, OneToMany, OneToOne } from 'typeorm'; +import { BaseEntity } from '../../../core/base-entity/entities/base-entity.entity'; +import { Field, ObjectType } from '@nestjs/graphql'; +import PrivateFile from '../../file/entities/privateFile.entity'; +import { IsDate, IsNumber, IsOptional } from 'class-validator'; +import { Label } from './label.entity'; + +/** + * Defines an image that wraps an S3 File + */ + +@Entity() +@ObjectType() +export class Image extends BaseEntity { + @Field(() => PrivateFile, { description: 'File' }) + @OneToOne(() => PrivateFile) + @JoinColumn() + public file: PrivateFile; + + @Field(() => Number, { + nullable: true, + description: 'Image Width in Pixels', + }) + @Column({ + nullable: true, + }) + @IsOptional() + @IsNumber() + public width?: number; + + @Field(() => Number, { + nullable: true, + description: 'Image Height in Pixels', + }) + @Column({ + nullable: true, + }) + @IsOptional() + @IsNumber() + public height?: number; + + @Field(() => Number, { + nullable: true, + description: 'GPS Latitude', + }) + @Column({ + type: 'float8', + nullable: true, + }) + @IsOptional() + @IsNumber() + public latitude?: number; + + @Field(() => Number, { + nullable: true, + description: 'GPS Longitude', + }) + @Column({ + type: 'float8', + nullable: true, + }) + @IsOptional() + @IsNumber() + public longitude?: number; + + @Field(() => Date, { + nullable: true, + description: 'Capture Date', + }) + @Column({ + nullable: true, + }) + @IsOptional() + @IsDate() + public capturedAt?: Date; + + @Field(() => [Label], { + description: 'Labels of detected objects on image', + }) + @OneToMany(() => Label, (label) => label.image) + public labels: Label[]; +} + +export default Image; diff --git a/backend/src/flox/modules/image/entities/label.entity.ts b/backend/src/flox/modules/image/entities/label.entity.ts new file mode 100644 index 000000000..9ae2a691b --- /dev/null +++ b/backend/src/flox/modules/image/entities/label.entity.ts @@ -0,0 +1,42 @@ +import { Column, Entity, JoinColumn, ManyToOne, OneToOne } from 'typeorm'; +import { Field, ObjectType } from '@nestjs/graphql'; +import { BaseEntity } from '../../../core/base-entity/entities/base-entity.entity'; +import { IsNumber, IsString } from 'class-validator'; +import { BoundingBox } from './bounding-box.entity'; +import Image from './image.entity'; + +@Entity() +@ObjectType() +export class Label extends BaseEntity { + @Field(() => Image, { + description: 'Image on which this label was detected', + }) + @ManyToOne(() => Image, (image) => image.labels) + public image: Image; + + @Field(() => String, { + description: 'Label Name', + }) + @Column() + @IsString() + public name: string; + + @Field(() => Number, { description: 'Confidence between 0 and 100' }) + @Column('float8') + @IsNumber() + public confidence: number; + + @Field(() => [String], { + description: 'Parent labels', + }) + @Column('text', { array: true }) + @IsString({ each: true }) + public parents: string[]; + + @Field(() => BoundingBox, { + description: 'Bounding box for every instance of this label on image', + }) + @OneToOne(() => BoundingBox) + @JoinColumn() + public boundingBox: BoundingBox; +} diff --git a/backend/src/flox/modules/image/image.module.ts b/backend/src/flox/modules/image/image.module.ts new file mode 100644 index 000000000..f543c877f --- /dev/null +++ b/backend/src/flox/modules/image/image.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { ImageService } from './image.service'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { Image } from './entities/image.entity'; +import { ImageResolver } from './image.resolver'; +import { FileModule } from '../file/file.module'; +import { BoundingBox } from './entities/bounding-box.entity'; +import { Label } from './entities/label.entity'; + +@Module({ + imports: [FileModule, TypeOrmModule.forFeature([Image, BoundingBox, Label])], + providers: [ImageService, ImageResolver], +}) +export class ImageModule {} diff --git a/backend/src/flox/modules/image/image.resolver.ts b/backend/src/flox/modules/image/image.resolver.ts new file mode 100644 index 000000000..13620ba21 --- /dev/null +++ b/backend/src/flox/modules/image/image.resolver.ts @@ -0,0 +1,116 @@ +import { Args, Mutation, Query, Resolver } from '@nestjs/graphql'; +import { ImageService } from './image.service'; +import { LoggedIn } from '../auth/authentication.decorator'; +import Image from './entities/image.entity'; +import { GetImageArgs } from './dto/args/get-image.args'; +import { GetImageForFileArgs } from './dto/args/get-image-for-file.args'; +import { DeleteImageInput } from './dto/input/delete-image.input'; +import { CreateImageInput } from './dto/input/create-image.input'; +import { AdminOnly, CurrentUser } from '../roles/authorization.decorator'; +import { User } from '../auth/entities/user.entity'; +import { DEFAULT_ROLES } from '../roles/config'; +import { ForbiddenError } from 'apollo-server-express'; +import { FileService } from '../file/file.service'; +import { GetAllImagesArgs } from './dto/args/get-all-images.args'; + +@Resolver(() => Image) +export class ImageResolver { + constructor( + private readonly imageService: ImageService, + private readonly fileService: FileService, + ) {} + + /** + * Returns all images stored in database. Only accessible to admins + * @param {GetAllImagesArgs} getAllImagesArgs - limit and skip parameters + * @returns {Promise} All Images + */ + @AdminOnly() + @Query(() => [Image], { name: 'images' }) + async getImages(getAllImagesArgs: GetAllImagesArgs): Promise { + return this.imageService.getAllImages(getAllImagesArgs); + } + + /** + * Returns an Image that wraps an s3 bucket file + * @param {GetImageArgs }getImageArgs - contains uuid of image + * @param {User} user - Currently logged-in user + * @returns {Promise} Requested image + */ + @LoggedIn() + @Query(() => Image, { name: 'image' }) + async getImage( + @Args() getImageArgs: GetImageArgs, + @CurrentUser() user: User, + ): Promise { + const image = await this.imageService.getImage(getImageArgs); + if (user.role !== DEFAULT_ROLES.ADMIN && image.file.owner !== user.uuid) { + throw new ForbiddenError('Image does not belong to logged in user'); + } + return image; + } + + /** + * Gets the image wrapper for a specified file. Useful if you know the file but not the + * corresponding image wrapper + * @param {GetImageForFileArgs} getImageForFileArgs - contains the uuid of the file + * @param {User} user - Currently logged-in user + * @returns {Promise} Requested image + */ + @LoggedIn() + @Query(() => Image, { name: 'imageForFile' }) + async getImageForFile( + @Args() getImageForFileArgs: GetImageForFileArgs, + @CurrentUser() user: User, + ): Promise { + const image = await this.imageService.getImageForFile(getImageForFileArgs); + if (user.role !== DEFAULT_ROLES.ADMIN && image.file.owner !== user.uuid) { + throw new ForbiddenError('File does not belong to logged in user'); + } + return image; + } + + /** + * Creates a new image for an already existing file + * @param {CreateImageInput} createImageInput - contains uuid of file to wrap + * @param {User} user - Currently logged-in user + * @returns {Promise} Requested image + */ + @LoggedIn() + @Mutation(() => Image) + async createImage( + @Args('createImageInput') createImageInput: CreateImageInput, + @CurrentUser() user: User, + ): Promise { + const file = await this.fileService.getPrivateFile({ + uuid: createImageInput.file, + }); + if (user.role !== DEFAULT_ROLES.ADMIN && file.owner !== user.uuid) { + throw new ForbiddenError( + 'Cannot create image for file that belongs to someone else', + ); + } + return this.imageService.createImage(createImageInput); + } + + /** + * Deletes an image (without deleting the corresponding file) + * @param {DeleteImageInput} deleteImageInput - contains uuid of image + * @param {User} user - Currently logged-in user + * @returns {Promise} Requested image + */ + @LoggedIn() + @Mutation(() => Image) + async deleteImage( + @Args('deleteImageInput') deleteImageInput: DeleteImageInput, + @CurrentUser() user: User, + ): Promise { + const image = await this.imageService.getImage({ + uuid: deleteImageInput.uuid, + } as GetImageArgs); + if (user.role !== DEFAULT_ROLES.ADMIN && image.file.owner !== user.uuid) { + throw new ForbiddenError('Image does not belong to logged in user'); + } + return this.imageService.deleteImage(deleteImageInput); + } +} diff --git a/backend/src/flox/modules/image/image.service.ts b/backend/src/flox/modules/image/image.service.ts new file mode 100644 index 000000000..fc2971ee8 --- /dev/null +++ b/backend/src/flox/modules/image/image.service.ts @@ -0,0 +1,211 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { + DetectLabelsCommand, + RekognitionClient, +} from '@aws-sdk/client-rekognition'; +import exifr from 'exifr'; +import Image from './entities/image.entity'; +import { Repository } from 'typeorm'; +import { GetImageArgs } from './dto/args/get-image.args'; +import { GetImageForFileArgs } from './dto/args/get-image-for-file.args'; +import { CreateImageInput } from './dto/input/create-image.input'; +import { FileService } from '../file/file.service'; +import { DeleteImageInput } from './dto/input/delete-image.input'; +import { GetPrivateFileArgs } from '../file/dto/args/get-private-file.args'; +import { GetAllImagesArgs } from './dto/args/get-all-images.args'; +import { DeleteFileInput } from '../file/dto/input/delete-file.input'; +import { ConfigService } from '@nestjs/config'; +import { CreateLabelsInput } from './dto/input/create-labels.input'; +import { Label } from './entities/label.entity'; +import { BoundingBox } from './entities/bounding-box.entity'; + +@Injectable() +export class ImageService { + // Rekognition credentials + private readonly credentials = { + region: this.configService.get('AWS_MAIN_REGION'), + accessKeyId: this.configService.get('AWS_ACCESS_KEY_ID'), + secretAccessKey: this.configService.get('AWS_SECRET_ACCESS_KEY'), + }; + + // Rekognition instance + private rekognition: RekognitionClient = new RekognitionClient({ + credentials: this.credentials, + region: this.credentials.region, + }); + + constructor( + @InjectRepository(Image) + private imageRepository: Repository, + + @InjectRepository(Label) + private labelRepository: Repository