diff --git a/nx.json b/nx.json index 8cb6074..001d957 100644 --- a/nx.json +++ b/nx.json @@ -77,5 +77,5 @@ } } ], - "nxCloudAccessToken": "OGJlMTgwN2ItNzI3Yy00YzU4LWFjM2MtYzcxZjE2MWYwMTgxfHJlYWQtd3JpdGU=" + "nxCloudAccessToken": "OGYyNDIzN2ItNDZmZC00NjFmLTg3MDgtYzc0Nzc0NzdkMDk3fHJlYWQtd3JpdGU=" } diff --git a/package.json b/package.json index f653709..c9a0903 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,8 @@ ], "lint-staged": { "*.{ts,tsx}": [ - "eslint" + "eslint --fix", + "prettier --write" ] } } diff --git a/packages/file-storage/src/lib/file-storage-fs.class.ts b/packages/file-storage/src/lib/file-storage-fs.class.ts index 84192eb..cc72fc8 100644 --- a/packages/file-storage/src/lib/file-storage-fs.class.ts +++ b/packages/file-storage/src/lib/file-storage-fs.class.ts @@ -1,5 +1,5 @@ import { createReadStream, createWriteStream, ObjectEncodingOptions, stat, unlink } from 'node:fs'; -import { access, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { access, mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { normalize, resolve as resolvePath, sep } from 'node:path'; import { finished, Readable } from 'node:stream'; @@ -72,6 +72,13 @@ export class FileStorageLocal implements FileStorage { return new Promise((resolve) => stat(fileName, (err) => (err ? resolve(false) : resolve(true)))); } + async moveFile(args: FileStorageBaseArgs & { newFilePath: string }): Promise { + const { filePath, newFilePath, request } = args; + const oldFileName = await this.transformFilePath(filePath, MethodTypes.READ, request); + const newFileName = await this.transformFilePath(newFilePath, MethodTypes.WRITE, request); + await rename(oldFileName, newFileName); + } + async uploadFile(args: FileStorageLocalUploadFile): Promise { const { filePath, content, options, request } = args; const fileName = await this.transformFilePath(filePath, MethodTypes.WRITE, request, options); diff --git a/packages/file-storage/src/lib/file-storage-fs.types.ts b/packages/file-storage/src/lib/file-storage-fs.types.ts index 763956e..b747024 100644 --- a/packages/file-storage/src/lib/file-storage-fs.types.ts +++ b/packages/file-storage/src/lib/file-storage-fs.types.ts @@ -24,6 +24,10 @@ export interface FileStorageLocalFileExists extends FileStorageBaseArgs { options?: StatOptions | BigIntOptions; } +export interface FileStorageLocalMoveFile extends FileStorageBaseArgs { + newFilePath: string; +} + export interface FileStorageLocalUploadFile extends FileStorageBaseArgs { content: string | Uint8Array | Buffer; options?: WriteFileOptions; diff --git a/packages/file-storage/src/lib/file-storage-google.class.ts b/packages/file-storage/src/lib/file-storage-google.class.ts index efe9540..a4ccd25 100644 --- a/packages/file-storage/src/lib/file-storage-google.class.ts +++ b/packages/file-storage/src/lib/file-storage-google.class.ts @@ -10,6 +10,7 @@ import type { FileStorageGoogleDownloadFile, FileStorageGoogleDownloadStream, FileStorageGoogleFileExists, + FileStorageGoogleMoveFile, FileStorageGoogleReadDir, FileStorageGoogleSetup, FileStorageGoogleUploadFile, @@ -66,6 +67,14 @@ export class FileStorageGoogle implements FileStorage { return exists; } + async moveFile(args: FileStorageGoogleMoveFile): Promise { + const { storage, bucket } = this.config; + const { options = {}, request } = args; + const oldFilePath = await this.transformFilePath(args.filePath, MethodTypes.READ, request, options); + const newFilePath = await this.transformFilePath(args.newFilePath, MethodTypes.WRITE, request, options); + await storage.bucket(bucket).file(oldFilePath).move(newFilePath, options); + } + async uploadFile(args: FileStorageGoogleUploadFile & { content: Buffer | string }): Promise { const { storage, bucket } = this.config; const { options = {}, request } = args; diff --git a/packages/file-storage/src/lib/file-storage-google.types.ts b/packages/file-storage/src/lib/file-storage-google.types.ts index 3713611..a7d2854 100644 --- a/packages/file-storage/src/lib/file-storage-google.types.ts +++ b/packages/file-storage/src/lib/file-storage-google.types.ts @@ -5,6 +5,7 @@ import type { DownloadOptions, FileOptions, GetFilesOptions, + MoveOptions, SaveOptions, Storage, } from '@google-cloud/storage'; @@ -30,6 +31,11 @@ export interface FileStorageGoogleFileExists extends FileStorageBaseArgs { options?: ExistsOptions & FileOptions; } +export interface FileStorageGoogleMoveFile extends FileStorageBaseArgs { + newFilePath: string; + options?: FileOptions & MoveOptions; +} + export interface FileStorageGoogleUploadFile extends FileStorageBaseArgs { content: string | Uint8Array | Buffer; options?: FileOptions & SaveOptions; diff --git a/packages/file-storage/src/lib/file-storage-s3.class.ts b/packages/file-storage/src/lib/file-storage-s3.class.ts index 4839fc9..4f91e80 100644 --- a/packages/file-storage/src/lib/file-storage-s3.class.ts +++ b/packages/file-storage/src/lib/file-storage-s3.class.ts @@ -1,4 +1,4 @@ -import { DeleteObjectsCommandInput, ListObjectsV2CommandInput, S3 } from '@aws-sdk/client-s3'; +import { DeleteObjectsCommandInput, ListObjectsCommandInput, S3 } from '@aws-sdk/client-s3'; import { Upload } from '@aws-sdk/lib-storage'; import { PassThrough, Readable } from 'node:stream'; @@ -15,6 +15,7 @@ import type { FileStorageS3DownloadFile, FileStorageS3DownloadStream, FileStorageS3FileExists, + FileStorageS3MoveFile, FileStorageS3Setup, FileStorageS3UploadFile, FileStorageS3UploadStream, @@ -95,6 +96,15 @@ export class FileStorageS3 implements FileStorage { } } + async moveFile(args: FileStorageS3MoveFile): Promise { + const { filePath, newFilePath, options = {}, request } = args; + const { s3, bucket: Bucket } = this.config; + const Key = await this.transformFilePath(filePath, MethodTypes.READ, request, options); + const newKey = await this.transformFilePath(newFilePath, MethodTypes.WRITE, request, options); + await s3.copyObject({ ...options, Bucket, CopySource: `${Bucket}/${Key}`, Key: newKey }); + await s3.deleteObject({ ...options, Key, Bucket }); + } + async uploadFile(args: FileStorageS3UploadFile): Promise { const { filePath, content, options = {}, request } = args; const { s3, bucket: Bucket } = this.config; @@ -162,12 +172,12 @@ export class FileStorageS3 implements FileStorage { const { dirPath, options = {}, request } = args; const { s3, bucket: Bucket } = this.config; const listKey = await this.transformFilePath(dirPath, MethodTypes.DELETE, request); - const listParams: ListObjectsV2CommandInput = { + const listParams: ListObjectsCommandInput = { Bucket, Prefix: listKey, }; // get list of objects in a dir, limited to 1000 items - const listedObjects = await s3.listObjectsV2(listParams); + const listedObjects = await s3.listObjects(listParams); if (!listedObjects.Contents?.length) { return; } @@ -193,7 +203,7 @@ export class FileStorageS3 implements FileStorage { const { dirPath, request } = args; const { s3, bucket: Bucket } = this.config; const Key = await this.transformFilePath(dirPath, MethodTypes.READ, request); - const listParams: ListObjectsV2CommandInput = { + const listParams: ListObjectsCommandInput = { Bucket, Delimiter: '/', }; @@ -201,8 +211,8 @@ export class FileStorageS3 implements FileStorage { if (Key !== '/' && Key !== '') { listParams.Prefix = addTrailingForwardSlash(Key); } - const listedObjects = await s3.listObjectsV2(listParams); - const filesAndFilders = []; + const listedObjects = await s3.listObjects(listParams); + const filesAndFilders: string[] = []; // add nested folders, CommonPrefixes contains / if (listedObjects.CommonPrefixes?.length) { const folders = listedObjects.CommonPrefixes.map((prefixObject) => { @@ -213,6 +223,7 @@ export class FileStorageS3 implements FileStorage { }); filesAndFilders.push(...folders); } + // adds filenames if (listedObjects.Contents?.length && listedObjects.Prefix) { const files = listedObjects.Contents.filter((file) => !!file.Key).map((file) => diff --git a/packages/file-storage/src/lib/file-storage-s3.types.ts b/packages/file-storage/src/lib/file-storage-s3.types.ts index 1656878..ee2ef9c 100644 --- a/packages/file-storage/src/lib/file-storage-s3.types.ts +++ b/packages/file-storage/src/lib/file-storage-s3.types.ts @@ -35,6 +35,11 @@ export interface FileStorageS3FileExists extends FileStorageBaseArgs { options?: Omit; } +export interface FileStorageS3MoveFile extends FileStorageBaseArgs { + newFilePath: string; + options?: Omit; +} + export interface FileStorageS3UploadFile extends FileStorageBaseArgs { content: string | Uint8Array | Buffer; options?: Omit; diff --git a/packages/file-storage/src/lib/file-storage.class.ts b/packages/file-storage/src/lib/file-storage.class.ts index 05edb01..bef68af 100644 --- a/packages/file-storage/src/lib/file-storage.class.ts +++ b/packages/file-storage/src/lib/file-storage.class.ts @@ -53,6 +53,15 @@ export abstract class FileStorage { throw new Error(defaultErrorMessage); } + moveFile( + args: FileStorageBaseArgs & { + newFilePath: string; + options?: string | any; + }, + ): Promise { + throw new Error(defaultErrorMessage); + } + uploadFile( args: FileStorageBaseArgs & { content: Buffer | Uint8Array | string; @@ -86,6 +95,11 @@ export abstract class FileStorage { throw new Error(defaultErrorMessage); } + // TODO: + // createDir(args: FileStorageDirBaseArgs): Promise { + // throw new Error(defaultErrorMessage); + // } + deleteDir(args: FileStorageDirBaseArgs): Promise { throw new Error(defaultErrorMessage); } diff --git a/packages/file-storage/src/lib/file-storage.service.ts b/packages/file-storage/src/lib/file-storage.service.ts index 69aa5e7..5e53c00 100644 --- a/packages/file-storage/src/lib/file-storage.service.ts +++ b/packages/file-storage/src/lib/file-storage.service.ts @@ -6,6 +6,7 @@ import type { FileStorageLocalDownloadFile, FileStorageLocalDownloadStream, FileStorageLocalFileExists, + FileStorageLocalMoveFile, FileStorageLocalUploadFile, FileStorageLocalUploadStream, } from './file-storage-fs.types'; @@ -15,6 +16,7 @@ import type { FileStorageGoogleDownloadFile, FileStorageGoogleDownloadStream, FileStorageGoogleFileExists, + FileStorageGoogleMoveFile, FileStorageGoogleReadDir, FileStorageGoogleUploadFile, FileStorageGoogleUploadStream, @@ -25,6 +27,7 @@ import type { FileStorageS3DownloadFile, FileStorageS3DownloadStream, FileStorageS3FileExists, + FileStorageS3MoveFile, FileStorageS3UploadFile, FileStorageS3UploadStream, } from './file-storage-s3.types'; @@ -41,6 +44,10 @@ export class FileStorageService implements Omit { + return this.fileStorage.moveFile(args); + } + uploadFile(args: FileStorageLocalUploadFile | FileStorageS3UploadFile | FileStorageGoogleUploadFile): Promise { return this.fileStorage.uploadFile(args); } diff --git a/packages/file-storage/src/lib/types.ts b/packages/file-storage/src/lib/types.ts index 41ca853..1eb99a4 100644 --- a/packages/file-storage/src/lib/types.ts +++ b/packages/file-storage/src/lib/types.ts @@ -1,5 +1,5 @@ import type { InjectionToken, ModuleMetadata } from '@nestjs/common'; -import type { Writable } from 'node:stream'; +import type { Stream } from 'node:stream'; import type { FileStorage, FileStorageConfigFactory } from './file-storage.class'; import type { FileStorageLocalSetup } from './file-storage-fs.types'; @@ -60,4 +60,4 @@ interface WritableWithDoneEvent { removeListener(event: 'done', listener: () => void): this; } -export type FileStorageWritable = Writable & WritableWithDoneEvent; +export type FileStorageWritable = NodeJS.WritableStream & Stream & WritableWithDoneEvent; diff --git a/packages/file-storage/test/file-storage-cases.ts b/packages/file-storage/test/file-storage-cases.ts new file mode 100644 index 0000000..149fd10 --- /dev/null +++ b/packages/file-storage/test/file-storage-cases.ts @@ -0,0 +1,88 @@ +import * as dotenv from 'dotenv'; +import { randomUUID } from 'node:crypto'; +import { resolve } from 'node:path'; +import { setTimeout } from 'node:timers/promises'; + +import { FileStorage, StorageType } from '../src'; + +dotenv.config({ path: resolve(__dirname, '../.env.test') }); + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace NodeJS { + interface ProcessEnv { + S3_BUCKET: string; + S3_REGION: string; + AWS_ACCESS_KEY_ID?: string; + AWS_SECRET_ACCESS_KEY?: string; + AWS_SECRET_SESSION_TOKEN?: string; + AWS_PROFILE?: string; + GC_BUCKET: string; + GC_PROJECT_ID?: string; + CI?: string; + } + } +} + +export const fsStoragePath = resolve('store'); + +export const testMap = [ + { + description: 'file-storage-fs', + storageType: StorageType.FS, + options: { + [StorageType.FS]: { setup: { storagePath: fsStoragePath, maxPayloadSize: 1 } }, + }, + }, + { + description: 'file-storage-S3', + storageType: StorageType.S3, + options: { + [StorageType.S3]: { + setup: { + maxPayloadSize: 1, + bucket: process.env.S3_BUCKET, + region: process.env.S3_REGION, + ...(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY + ? { + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + sessionToken: process.env.AWS_SECRET_SESSION_TOKEN, + }, + } + : {}), + }, + }, + }, + }, + { + description: 'file-storage-GC', + storageType: StorageType.GC, + options: { + [StorageType.GC]: { + setup: { + maxPayloadSize: 1, + projectId: process.env.GC_PROJECT_ID, + bucketName: process.env.GC_BUCKET, + }, + }, + }, + }, +] as const; + +export const delay = async (ms = 100) => { + if (process.env.CI) { + await setTimeout(ms); + } +}; + +export const createDummyFile = async ( + fileStorage: FileStorage, + filePath: string = randomUUID(), + content = 'this is a test content', +) => { + await fileStorage.uploadFile({ filePath, content }); + await delay(100); + return { filePath, content }; +}; diff --git a/packages/file-storage/test/file-storage-fs.spec.ts b/packages/file-storage/test/file-storage-fs.spec.ts new file mode 100644 index 0000000..f616a0c --- /dev/null +++ b/packages/file-storage/test/file-storage-fs.spec.ts @@ -0,0 +1,159 @@ +/* eslint-disable max-nested-callbacks */ +/* eslint-disable max-lines-per-function */ +import { Test, TestingModule } from '@nestjs/testing'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { once, Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; + +import { FileStorage, FileStorageModule } from '../src'; +import { FILE_STORAGE_STRATEGY_TOKEN } from '../src/lib/constants'; +import { createDummyFile, fsStoragePath, testMap } from './file-storage-cases'; + +const { description, storageType, options } = testMap[0]; + +describe(description, () => { + let fileStorage: FileStorage; + + beforeAll(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [FileStorageModule.forRoot(storageType, options)], + }).compile(); + + fileStorage = module.get(FILE_STORAGE_STRATEGY_TOKEN); + await mkdir(fsStoragePath, { recursive: true }); + }); + + afterAll(async () => { + await rm(fsStoragePath, { recursive: true, force: true }).catch(() => { + // ignore error + }); + }); + + it('calling fileExists on a filepath that exists returns true', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it("calling fileExists on a filepath that doesn't exist return false", async () => { + const fileExists = await fileStorage.fileExists({ filePath: 'fileDoesntExist' }); + // + expect(fileExists).toBe(false); + }); + + it('readDir returns an empty array when no files exist', async () => { + const res = await fileStorage.readDir({ dirPath: '' }); + expect(res.length).toBe(0); + }); + + it('uploadFile uploads a file', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it('moveFile moves a file to a new location and remove the previous one', async () => { + const oldFileName = 'oldFileName.txt'; + const newFileName = 'newFileName.txt'; + await createDummyFile(fileStorage, oldFileName); + // + await fileStorage.moveFile({ filePath: oldFileName, newFilePath: newFileName }); + // + const oldFileExists = await fileStorage.fileExists({ filePath: oldFileName }); + expect(oldFileExists).toBe(false); + const newFileExists = await fileStorage.fileExists({ filePath: newFileName }); + expect(newFileExists).toBe(true); + await fileStorage.deleteFile({ filePath: newFileName }); + }); + + it('deleteFile deletes a file', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + await fileStorage.deleteFile({ filePath }); + // + const oldFileExists = await fileStorage.fileExists({ filePath }); + expect(oldFileExists).toBe(false); + }); + + it('uploadStream uploads a file', async () => { + const filePath = randomUUID(); + const content = randomBytes(1024); + // + const upload = await fileStorage.uploadStream({ filePath }); + const entry = Readable.from(content); + const ac = new AbortController(); + const t = setTimeout(() => ac.abort(), 2000); + const listener = once(upload, 'done', { signal: ac.signal }).finally(() => clearTimeout(t)); + await pipeline(entry, upload); + await listener; + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it('downloadFile downloads a file', async () => { + const { filePath, content } = await createDummyFile(fileStorage); + // + const file = await fileStorage.downloadFile({ filePath }); + // + expect(file.toString()).toEqual(content.toString()); + await fileStorage.deleteFile({ filePath }); + }); + + it('downloadStream downloads a file', async () => { + const { filePath, content } = await createDummyFile(fileStorage); + // + const stream = await fileStorage.downloadStream({ filePath }); + // + expect(stream).toBeInstanceOf(Readable); + // this makes the assumption that when the stream is readable, all the data is available in one read + await once(stream, 'readable'); + const chunk = stream.read(); + expect(chunk.toString()).toBe(content); + await fileStorage.deleteFile({ filePath }); + }); + + it('uploads a file to a nested directory', async () => { + const nestedDir = 'nested'; + const nestedFileName = 'nested.txt'; + const nestedFilePath = `${fsStoragePath}/${nestedDir}`; + await mkdir(nestedFilePath, { recursive: true }); + // + await fileStorage.uploadFile({ filePath: `${nestedDir}/${nestedFileName}`, content: 'this is a nested file' }); + // + const result = await fileStorage.readDir({ dirPath: nestedDir }); + expect(result.find((fileName) => fileName === nestedFileName)).not.toBeUndefined(); + await fileStorage.deleteDir({ dirPath: nestedDir }); + }); + + it('readDir returns an array of files and folders in a directory', async () => { + const dirPath = ''; + const filePath = randomUUID(); + const nestedFilePath = `nest/${randomUUID()}`; + const content = randomBytes(1024); + await mkdir(`${fsStoragePath}/nest`, { recursive: true }); + await fileStorage.uploadFile({ filePath, content }); + await fileStorage.uploadFile({ filePath: nestedFilePath, content }); + // + const result = await fileStorage.readDir({ dirPath }); + // + expect(result.length).toBe(2); + const parts = nestedFilePath.split('/'); + const expected = [parts.shift(), filePath]; + expect(result.every((item) => expected.includes(item))).toBe(true); + }); + + it('deleteDir deletes a directory', async () => { + const dirPath = ''; + // + await fileStorage.deleteDir({ dirPath }); + // + expect(await fileStorage.readDir({ dirPath })).toEqual([]); + }); +}); diff --git a/packages/file-storage/test/file-storage-google.spec.ts b/packages/file-storage/test/file-storage-google.spec.ts new file mode 100644 index 0000000..da59bf3 --- /dev/null +++ b/packages/file-storage/test/file-storage-google.spec.ts @@ -0,0 +1,156 @@ +/* eslint-disable max-nested-callbacks */ +/* eslint-disable max-lines-per-function */ +import { Test, TestingModule } from '@nestjs/testing'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { once, Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; + +import { FileStorage, FileStorageModule } from '../src'; +import { FILE_STORAGE_STRATEGY_TOKEN } from '../src/lib/constants'; +import { createDummyFile, delay, testMap } from './file-storage-cases'; + +const { description, storageType, options } = testMap[2]; + +describe(description, () => { + let fileStorage: FileStorage; + + beforeAll(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [FileStorageModule.forRoot(storageType, options)], + }).compile(); + + fileStorage = module.get(FILE_STORAGE_STRATEGY_TOKEN); + + await fileStorage.deleteDir({ dirPath: '' }); + }); + + afterAll(async () => { + await fileStorage.deleteDir({ dirPath: '' }); + }); + + it('calling fileExists on a filepath that exists returns true', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it("calling fileExists on a filepath that doesn't exist return false", async () => { + const fileExists = await fileStorage.fileExists({ filePath: 'fileDoesntExist' }); + // + expect(fileExists).toBe(false); + }); + + it('readDir returns an empty array when no files exist', async () => { + const res = await fileStorage.readDir({ dirPath: '' }); + expect(res.length).toBe(0); + }); + + it('uploadFile uploads a file', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it('moveFile moves a file to a new location and remove the previous one', async () => { + const oldFileName = 'oldFileName.txt'; + const newFileName = 'newFileName.txt'; + await createDummyFile(fileStorage, oldFileName); + // + await fileStorage.moveFile({ filePath: oldFileName, newFilePath: newFileName }); + // + const oldFileExists = await fileStorage.fileExists({ filePath: oldFileName }); + expect(oldFileExists).toBe(false); + const newFileExists = await fileStorage.fileExists({ filePath: newFileName }); + expect(newFileExists).toBe(true); + await fileStorage.deleteFile({ filePath: newFileName }); + }, 7000); + + it('deleteFile deletes a file', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + await fileStorage.deleteFile({ filePath }); + // + const oldFileExists = await fileStorage.fileExists({ filePath }); + expect(oldFileExists).toBe(false); + }); + + it('uploadStream uploads a file', async () => { + const filePath = randomUUID(); + const content = randomBytes(1024); + // + const upload = await fileStorage.uploadStream({ filePath }); + const entry = Readable.from(content); + const ac = new AbortController(); + const t = setTimeout(() => ac.abort(), 2500); + const listener = once(upload, 'done', { signal: ac.signal }).finally(() => clearTimeout(t)); + await pipeline(entry, upload); + await listener; + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it('downloadFile downloads a file', async () => { + const { filePath, content } = await createDummyFile(fileStorage); + // + const file = await fileStorage.downloadFile({ filePath }); + // + expect(file.toString()).toEqual(content.toString()); + await fileStorage.deleteFile({ filePath }); + }); + + it('downloadStream downloads a file', async () => { + const { filePath, content } = await createDummyFile(fileStorage); + // + const stream = await fileStorage.downloadStream({ filePath }); + // + expect(stream).toBeInstanceOf(Readable); + // this makes the assumption that when the stream is readable, all the data is available in one read + await once(stream, 'readable'); + const chunk = stream.read(); + expect(chunk.toString()).toBe(content); + await fileStorage.deleteFile({ filePath }); + }); + + it('uploads a file to a nested directory', async () => { + const nestedDir = 'nested'; + const nestedFileName = 'nested.txt'; + // + await fileStorage.uploadFile({ filePath: `${nestedDir}/${nestedFileName}`, content: 'this is a nested file' }); + // + const result = await fileStorage.readDir({ dirPath: nestedDir }); + expect(result.find((fileName) => fileName === nestedFileName)).not.toBeUndefined(); + await fileStorage.deleteDir({ dirPath: nestedDir }); + }); + + it('readDir returns an array of files and folders in a directory', async () => { + const dirPath = ''; + const filePath = randomUUID(); + const nestedFilePath = `nest/${randomUUID()}`; + const content = randomBytes(1024); + await fileStorage.uploadFile({ filePath, content }); + await fileStorage.uploadFile({ filePath: nestedFilePath, content }); + await delay(100); + // + const result = await fileStorage.readDir({ dirPath }); + // + expect(result.length).toBeGreaterThanOrEqual(2); + console.warn('GC storage readDir is not completely implemented yet. Skipping test.'); + // const expected = [parts.shift(), filePath]; + // expect(result.every((item) => expected.includes(item))).toBe(true); + }, 7000); + + it('deleteDir deletes a directory', async () => { + const dirPath = ''; + // + await fileStorage.deleteDir({ dirPath }); + await delay(1000); + // + expect(await fileStorage.readDir({ dirPath })).toEqual([]); + }); +}); diff --git a/packages/file-storage/test/file-storage-s3.spec.ts b/packages/file-storage/test/file-storage-s3.spec.ts new file mode 100644 index 0000000..06a065c --- /dev/null +++ b/packages/file-storage/test/file-storage-s3.spec.ts @@ -0,0 +1,157 @@ +/* eslint-disable max-nested-callbacks */ +/* eslint-disable max-lines-per-function */ +import { Test, TestingModule } from '@nestjs/testing'; +import { randomBytes, randomUUID } from 'node:crypto'; +import { once, Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; + +import { FileStorage, FileStorageModule } from '../src'; +import { FILE_STORAGE_STRATEGY_TOKEN } from '../src/lib/constants'; +import { createDummyFile, delay, testMap } from './file-storage-cases'; + +const { description, storageType, options } = testMap[1]; + +describe(description, () => { + let fileStorage: FileStorage; + + beforeAll(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [FileStorageModule.forRoot(storageType, options)], + }).compile(); + + fileStorage = module.get(FILE_STORAGE_STRATEGY_TOKEN); + await fileStorage.deleteDir({ dirPath: '' }); + }); + + afterAll(async () => { + await fileStorage.deleteDir({ dirPath: '' }); + }); + + it('calling fileExists on a filepath that exists returns true', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it("calling fileExists on a filepath that doesn't exist return false", async () => { + const fileExists = await fileStorage.fileExists({ filePath: 'fileDoesntExist' }); + // + expect(fileExists).toBe(false); + }); + + it('readDir returns an empty array when no files exist', async () => { + const res = await fileStorage.readDir({ dirPath: '' }); + expect(res.length).toBe(0); + }); + + it('uploadFile uploads a file', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it('moveFile moves a file to a new location and remove the previous one', async () => { + const oldFileName = 'oldFileName.txt'; + const newFileName = 'newFileName.txt'; + await createDummyFile(fileStorage, oldFileName); + // + await fileStorage.moveFile({ filePath: oldFileName, newFilePath: newFileName }); + // + const oldFileExists = await fileStorage.fileExists({ filePath: oldFileName }); + expect(oldFileExists).toBe(false); + const newFileExists = await fileStorage.fileExists({ filePath: newFileName }); + expect(newFileExists).toBe(true); + await fileStorage.deleteFile({ filePath: newFileName }); + }, 7000); + + it('deleteFile deletes a file', async () => { + const { filePath } = await createDummyFile(fileStorage); + // + await fileStorage.deleteFile({ filePath }); + // + const oldFileExists = await fileStorage.fileExists({ filePath }); + expect(oldFileExists).toBe(false); + }); + + it('uploadStream uploads a file', async () => { + const filePath = randomUUID(); + const content = randomBytes(1024); + // + const upload = await fileStorage.uploadStream({ filePath }); + const entry = Readable.from(content); + const ac = new AbortController(); + const t = setTimeout(() => ac.abort(), 2500); + const listener = once(upload, 'done', { signal: ac.signal }).finally(() => clearTimeout(t)); + await pipeline(entry, upload); + await listener; + // + const fileExists = await fileStorage.fileExists({ filePath }); + expect(fileExists).toBe(true); + await fileStorage.deleteFile({ filePath }); + }); + + it('downloadFile downloads a file', async () => { + const { filePath, content } = await createDummyFile(fileStorage); + // + const file = await fileStorage.downloadFile({ filePath }); + // + expect(file.toString()).toEqual(content.toString()); + await fileStorage.deleteFile({ filePath }); + }); + + it('downloadStream downloads a file', async () => { + const { filePath, content } = await createDummyFile(fileStorage); + // + const stream = await fileStorage.downloadStream({ filePath }); + // + expect(stream).toBeInstanceOf(Readable); + // this makes the assumption that when the stream is readable, all the data is available in one read + await once(stream, 'readable'); + const chunk = stream.read(); + expect(chunk.toString()).toBe(content); + await fileStorage.deleteFile({ filePath }); + }); + + it('uploads a file to a nested directory', async () => { + const nestedDir = 'nested'; + const nestedFileName = 'nested.txt'; + const filePath = `${nestedDir}/${nestedFileName}`; + // + await fileStorage.uploadFile({ filePath, content: 'this is a nested file' }); + // + const result = await fileStorage.readDir({ dirPath: nestedDir }); + expect(result.find((fileName) => fileName === nestedFileName)).not.toBeUndefined(); + await fileStorage.deleteFile({ filePath }); + }); + + it('readDir returns an array of files and folders in a directory', async () => { + const dirPath = ''; + const filePath = randomUUID(); + const nestedFilePath = `nest/${randomUUID()}`; + const content = randomBytes(1024); + await fileStorage.uploadFile({ filePath, content }); + await fileStorage.uploadFile({ filePath: nestedFilePath, content }); + await delay(100); + // + const result = await fileStorage.readDir({ dirPath }); + // + expect(result.length).toBeGreaterThanOrEqual(2); + const parts = nestedFilePath.split('/'); + const expected = [parts.shift(), filePath]; + expect(result.every((item) => expected.includes(item))).toBe(true); + }, 7000); + + it('deleteDir deletes a directory', async () => { + const dirPath = ''; + // + await fileStorage.deleteDir({ dirPath }); + await delay(1000); + + // + expect(await fileStorage.readDir({ dirPath })).toEqual([]); + }); +}); diff --git a/packages/file-storage/test/file-storage.spec.ts b/packages/file-storage/test/file-storage.spec.ts deleted file mode 100644 index ffc95af..0000000 --- a/packages/file-storage/test/file-storage.spec.ts +++ /dev/null @@ -1,193 +0,0 @@ -/* eslint-disable max-nested-callbacks */ -/* eslint-disable max-lines-per-function */ -import { Test, TestingModule } from '@nestjs/testing'; -import * as dotenv from 'dotenv'; -import { mkdir, rm } from 'node:fs/promises'; -import { resolve } from 'node:path'; -import { once, Readable } from 'node:stream'; -import { pipeline } from 'node:stream/promises'; - -import { FileStorage, FileStorageModule, FileStorageModuleOptions, StorageType } from '../src'; -import { FILE_STORAGE_STRATEGY_TOKEN } from '../src/lib/constants'; - -dotenv.config({ path: resolve(__dirname, '../.env.test') }); - -declare global { - // eslint-disable-next-line @typescript-eslint/no-namespace - namespace NodeJS { - interface ProcessEnv { - S3_BUCKET: string; - S3_REGION: string; - AWS_ACCESS_KEY_ID?: string; - AWS_SECRET_ACCESS_KEY?: string; - AWS_SECRET_SESSION_TOKEN?: string; - AWS_PROFILE?: string; - GC_BUCKET: string; - GC_PROJECT_ID?: string; - } - } -} - -const storagePath = resolve('store'); -// const path = resolve(storagePath); -const testFileName = 'test.txt'; -const testFileContent = 'this is a test'; -const dirPath = ''; -const nestedDir = 'nested'; -const nestedFileName = 'nested.txt'; -const nestedFilePath = `${storagePath}/${nestedDir}`; - -const testMap: { - description: string; - storageType: StorageType; - options: Partial; -}[] = [ - { - description: 'file-storage-fs', - storageType: StorageType.FS, - options: { - [StorageType.FS]: { setup: { storagePath, maxPayloadSize: 1 } }, - }, - }, - { - description: 'file-storage-S3', - storageType: StorageType.S3, - options: { - [StorageType.S3]: { - setup: { - maxPayloadSize: 1, - bucket: process.env.S3_BUCKET, - region: process.env.S3_REGION, - ...(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY - ? { - credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, - sessionToken: process.env.AWS_SECRET_SESSION_TOKEN, - }, - } - : {}), - }, - }, - }, - }, - { - description: 'file-storage-GC', - storageType: StorageType.GC, - options: { - [StorageType.GC]: { - setup: { - maxPayloadSize: 1, - projectId: process.env.GC_PROJECT_ID, - bucketName: process.env.GC_BUCKET, - }, - }, - }, - }, -]; - -testMap.forEach((testSuite) => { - const { description, storageType, options } = testSuite; - - describe(description, () => { - let fileStorage: FileStorage; - - beforeAll(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [FileStorageModule.forRoot(storageType, options)], - }).compile(); - - fileStorage = module.get(FILE_STORAGE_STRATEGY_TOKEN); - if (storageType === StorageType.FS) await mkdir(storagePath, { recursive: true }); - // ensure S3 and GC buckets are empty - if ([StorageType.S3, StorageType.GC].includes(storageType)) await fileStorage.deleteDir({ dirPath }); - }); - - afterAll(async () => { - if (storageType === StorageType.FS) - await rm(storagePath, { recursive: true, force: true }).catch(() => { - // ignore error - }); - if ([StorageType.S3, StorageType.GC].includes(storageType)) await fileStorage.deleteDir({ dirPath }); - }); - - it('readDir returns an empty array when no files exist', async () => { - const res = await fileStorage.readDir({ dirPath }); - expect(res.length).toBe(0); - }); - - it('uploadFile uploads a file', async () => { - await fileStorage.uploadFile({ filePath: testFileName, content: 'this is a test' }); - const result = await fileStorage.readDir({ dirPath }); - expect(result.length).toBe(1); - expect(result[0]).toBe(testFileName); - }); - - it('calling fileExists on a filepath that exists returns true', async () => { - const fileExists = await fileStorage.fileExists({ filePath: testFileName }); - expect(fileExists).toBe(true); - }); - - it('calling fileExists on a filepath that doesnt exist return false', async () => { - const fileExists = await fileStorage.fileExists({ filePath: 'fileDoesntExist' }); - expect(fileExists).toBe(false); - }); - - it('deleteFile deletes a file', async () => { - await fileStorage.deleteFile({ filePath: testFileName }); - const result = await fileStorage.readDir({ dirPath }); - expect(result.length).toBe(0); - }); - - it('uploadStream uploads a file', async () => { - const upload = await fileStorage.uploadStream({ filePath: testFileName }); - const entry = Readable.from(Buffer.from(testFileContent)); - const ac = new AbortController(); - const t = setTimeout(() => ac.abort(), 1000); - const listener = once(upload, 'done', { signal: ac.signal }).finally(() => clearTimeout(t)); - await pipeline(entry, upload); - await listener; - - const result = await fileStorage.readDir({ dirPath }); - expect(result.length).toBe(1); - }); - - it('downloadFile downloads a file', async () => { - const file = await fileStorage.downloadFile({ filePath: testFileName }); - expect(file.toString()).toBe(testFileContent); - }); - - it('downloadStream downloads a file', async () => { - const download = await fileStorage.downloadStream({ filePath: testFileName }); - expect(download).toBeInstanceOf(Readable); - // this makes the assumption that the stream is readable and all the data is available in one read - for await (const chunk of download) { - expect(chunk.toString()).toBe(testFileContent); - } - }); - - it('uploads a file to a nested folder', async () => { - if (storageType === StorageType.FS) await mkdir(nestedFilePath, { recursive: true }); - - await fileStorage.uploadFile({ filePath: `${nestedDir}/${nestedFileName}`, content: 'this is a nested file' }); - const result = await fileStorage.readDir({ dirPath: nestedDir }); - expect(result.length).toBe(1); - expect(result[0]).toBe(nestedFileName); - }); - - it('readDir returns an array of files and folders in a dir', async () => { - if (storageType === StorageType.GC) { - console.warn('GC storage readDir is not completely implemented yet. Skipping test.'); - return; - } - const result = await fileStorage.readDir({ dirPath }); - expect(result.length).toBe(2); - expect(result).toEqual([nestedDir, testFileName]); - }); - - it('deleteDir deletes a dir', async () => { - await fileStorage.deleteDir({ dirPath }); - expect(await fileStorage.readDir({ dirPath })).toEqual([]); - }); - }); -});