Merge pull request #81 from getlarge/80-featfile-storage-add-method-t…

…o-move-rename-file

feat(file-storage): add method to move (rename) file

nx.json

@@ -77,5 +77,5 @@
       }
     }
   ],
-  "nxCloudAccessToken": "OGJlMTgwN2ItNzI3Yy00YzU4LWFjM2MtYzcxZjE2MWYwMTgxfHJlYWQtd3JpdGU="
+  "nxCloudAccessToken": "OGYyNDIzN2ItNDZmZC00NjFmLTg3MDgtYzc0Nzc0NzdkMDk3fHJlYWQtd3JpdGU="
 }

package.json

@@ -106,7 +106,8 @@
   ],
   "lint-staged": {
     "*.{ts,tsx}": [
-      "eslint"
+      "eslint --fix",
+      "prettier --write"
     ]
   }
 }

packages/file-storage/src/lib/file-storage-fs.class.ts

@@ -1,5 +1,5 @@
 import { createReadStream, createWriteStream, ObjectEncodingOptions, stat, unlink } from 'node:fs';
-import { access, mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { access, mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
 import { normalize, resolve as resolvePath, sep } from 'node:path';
 import { finished, Readable } from 'node:stream';
 
@@ -72,6 +72,13 @@ export class FileStorageLocal implements FileStorage {
     return new Promise<boolean>((resolve) => stat(fileName, (err) => (err ? resolve(false) : resolve(true))));
   }
 
+  async moveFile(args: FileStorageBaseArgs & { newFilePath: string }): Promise<void> {
+    const { filePath, newFilePath, request } = args;
+    const oldFileName = await this.transformFilePath(filePath, MethodTypes.READ, request);
+    const newFileName = await this.transformFilePath(newFilePath, MethodTypes.WRITE, request);
+    await rename(oldFileName, newFileName);
+  }
+
   async uploadFile(args: FileStorageLocalUploadFile): Promise<void> {
     const { filePath, content, options, request } = args;
     const fileName = await this.transformFilePath(filePath, MethodTypes.WRITE, request, options);

packages/file-storage/src/lib/file-storage-fs.types.ts

@@ -24,6 +24,10 @@ export interface FileStorageLocalFileExists extends FileStorageBaseArgs {
   options?: StatOptions | BigIntOptions;
 }
 
+export interface FileStorageLocalMoveFile extends FileStorageBaseArgs {
+  newFilePath: string;
+}
+
 export interface FileStorageLocalUploadFile extends FileStorageBaseArgs {
   content: string | Uint8Array | Buffer;
   options?: WriteFileOptions;

packages/file-storage/src/lib/file-storage-google.class.ts

@@ -10,6 +10,7 @@ import type {
   FileStorageGoogleDownloadFile,
   FileStorageGoogleDownloadStream,
   FileStorageGoogleFileExists,
+  FileStorageGoogleMoveFile,
   FileStorageGoogleReadDir,
   FileStorageGoogleSetup,
   FileStorageGoogleUploadFile,
@@ -66,6 +67,14 @@ export class FileStorageGoogle implements FileStorage {
     return exists;
   }
 
+  async moveFile(args: FileStorageGoogleMoveFile): Promise<void> {
+    const { storage, bucket } = this.config;
+    const { options = {}, request } = args;
+    const oldFilePath = await this.transformFilePath(args.filePath, MethodTypes.READ, request, options);
+    const newFilePath = await this.transformFilePath(args.newFilePath, MethodTypes.WRITE, request, options);
+    await storage.bucket(bucket).file(oldFilePath).move(newFilePath, options);
+  }
+
   async uploadFile(args: FileStorageGoogleUploadFile & { content: Buffer | string }): Promise<void> {
     const { storage, bucket } = this.config;
     const { options = {}, request } = args;

packages/file-storage/src/lib/file-storage-google.types.ts

@@ -5,6 +5,7 @@ import type {
   DownloadOptions,
   FileOptions,
   GetFilesOptions,
+  MoveOptions,
   SaveOptions,
   Storage,
 } from '@google-cloud/storage';
@@ -30,6 +31,11 @@ export interface FileStorageGoogleFileExists extends FileStorageBaseArgs {
   options?: ExistsOptions & FileOptions;
 }
 
+export interface FileStorageGoogleMoveFile extends FileStorageBaseArgs {
+  newFilePath: string;
+  options?: FileOptions & MoveOptions;
+}
+
 export interface FileStorageGoogleUploadFile extends FileStorageBaseArgs {
   content: string | Uint8Array | Buffer;
   options?: FileOptions & SaveOptions;

packages/file-storage/src/lib/file-storage-s3.class.ts

@@ -1,4 +1,4 @@
-import { DeleteObjectsCommandInput, ListObjectsV2CommandInput, S3 } from '@aws-sdk/client-s3';
+import { DeleteObjectsCommandInput, ListObjectsCommandInput, S3 } from '@aws-sdk/client-s3';
 import { Upload } from '@aws-sdk/lib-storage';
 import { PassThrough, Readable } from 'node:stream';
 
@@ -15,6 +15,7 @@ import type {
   FileStorageS3DownloadFile,
   FileStorageS3DownloadStream,
   FileStorageS3FileExists,
+  FileStorageS3MoveFile,
   FileStorageS3Setup,
   FileStorageS3UploadFile,
   FileStorageS3UploadStream,
@@ -95,6 +96,15 @@ export class FileStorageS3 implements FileStorage {
     }
   }
 
+  async moveFile(args: FileStorageS3MoveFile): Promise<void> {
+    const { filePath, newFilePath, options = {}, request } = args;
+    const { s3, bucket: Bucket } = this.config;
+    const Key = await this.transformFilePath(filePath, MethodTypes.READ, request, options);
+    const newKey = await this.transformFilePath(newFilePath, MethodTypes.WRITE, request, options);
+    await s3.copyObject({ ...options, Bucket, CopySource: `${Bucket}/${Key}`, Key: newKey });
+    await s3.deleteObject({ ...options, Key, Bucket });
+  }
+
   async uploadFile(args: FileStorageS3UploadFile): Promise<void> {
     const { filePath, content, options = {}, request } = args;
     const { s3, bucket: Bucket } = this.config;
@@ -162,12 +172,12 @@ export class FileStorageS3 implements FileStorage {
     const { dirPath, options = {}, request } = args;
     const { s3, bucket: Bucket } = this.config;
     const listKey = await this.transformFilePath(dirPath, MethodTypes.DELETE, request);
-    const listParams: ListObjectsV2CommandInput = {
+    const listParams: ListObjectsCommandInput = {
       Bucket,
       Prefix: listKey,
     };
     // get list of objects in a dir, limited to 1000 items
-    const listedObjects = await s3.listObjectsV2(listParams);
+    const listedObjects = await s3.listObjects(listParams);
     if (!listedObjects.Contents?.length) {
       return;
     }
@@ -193,16 +203,16 @@ export class FileStorageS3 implements FileStorage {
     const { dirPath, request } = args;
     const { s3, bucket: Bucket } = this.config;
     const Key = await this.transformFilePath(dirPath, MethodTypes.READ, request);
-    const listParams: ListObjectsV2CommandInput = {
+    const listParams: ListObjectsCommandInput = {
       Bucket,
       Delimiter: '/',
     };
     // Passing in / as Key breaks the folder matching
     if (Key !== '/' && Key !== '') {
       listParams.Prefix = addTrailingForwardSlash(Key);
     }
-    const listedObjects = await s3.listObjectsV2(listParams);
-    const filesAndFilders = [];
+    const listedObjects = await s3.listObjects(listParams);
+    const filesAndFilders: string[] = [];
     //  add nested folders, CommonPrefixes contains <prefix>/<next nested dir>
     if (listedObjects.CommonPrefixes?.length) {
       const folders = listedObjects.CommonPrefixes.map((prefixObject) => {
@@ -213,6 +223,7 @@ export class FileStorageS3 implements FileStorage {
       });
       filesAndFilders.push(...folders);
     }
+
     // adds filenames
     if (listedObjects.Contents?.length && listedObjects.Prefix) {
       const files = listedObjects.Contents.filter((file) => !!file.Key).map((file) =>

packages/file-storage/src/lib/file-storage-s3.types.ts

@@ -35,6 +35,11 @@ export interface FileStorageS3FileExists extends FileStorageBaseArgs {
   options?: Omit<HeadObjectCommandInput, 'Bucket' | 'Key'>;
 }
 
+export interface FileStorageS3MoveFile extends FileStorageBaseArgs {
+  newFilePath: string;
+  options?: Omit<DeleteObjectCommandInput, 'Bucket' | 'Key'>;
+}
+
 export interface FileStorageS3UploadFile extends FileStorageBaseArgs {
   content: string | Uint8Array | Buffer;
   options?: Omit<PutObjectCommandInput, 'Body' | 'Bucket' | 'Key'>;

packages/file-storage/src/lib/file-storage.class.ts

@@ -53,6 +53,15 @@ export abstract class FileStorage {
     throw new Error(defaultErrorMessage);
   }
 
+  moveFile(
+    args: FileStorageBaseArgs & {
+      newFilePath: string;
+      options?: string | any;
+    },
+  ): Promise<void> {
+    throw new Error(defaultErrorMessage);
+  }
+
   uploadFile(
     args: FileStorageBaseArgs & {
       content: Buffer | Uint8Array | string;
@@ -86,6 +95,11 @@ export abstract class FileStorage {
     throw new Error(defaultErrorMessage);
   }
 
+  // TODO:
+  // createDir(args: FileStorageDirBaseArgs): Promise<void> {
+  //   throw new Error(defaultErrorMessage);
+  // }
+
   deleteDir(args: FileStorageDirBaseArgs): Promise<void> {
     throw new Error(defaultErrorMessage);
   }

packages/file-storage/src/lib/file-storage.service.ts

@@ -6,6 +6,7 @@ import type {
   FileStorageLocalDownloadFile,
   FileStorageLocalDownloadStream,
   FileStorageLocalFileExists,
+  FileStorageLocalMoveFile,
   FileStorageLocalUploadFile,
   FileStorageLocalUploadStream,
 } from './file-storage-fs.types';
@@ -15,6 +16,7 @@ import type {
   FileStorageGoogleDownloadFile,
   FileStorageGoogleDownloadStream,
   FileStorageGoogleFileExists,
+  FileStorageGoogleMoveFile,
   FileStorageGoogleReadDir,
   FileStorageGoogleUploadFile,
   FileStorageGoogleUploadStream,
@@ -25,6 +27,7 @@ import type {
   FileStorageS3DownloadFile,
   FileStorageS3DownloadStream,
   FileStorageS3FileExists,
+  FileStorageS3MoveFile,
   FileStorageS3UploadFile,
   FileStorageS3UploadStream,
 } from './file-storage-s3.types';
@@ -41,6 +44,10 @@ export class FileStorageService implements Omit<FileStorage, 'transformFilePath'
     return this.fileStorage.fileExists(args);
   }
 
+  moveFile(args: FileStorageLocalMoveFile | FileStorageS3MoveFile | FileStorageGoogleMoveFile): Promise<void> {
+    return this.fileStorage.moveFile(args);
+  }
+
   uploadFile(args: FileStorageLocalUploadFile | FileStorageS3UploadFile | FileStorageGoogleUploadFile): Promise<void> {
     return this.fileStorage.uploadFile(args);
   }

packages/file-storage/src/lib/types.ts

@@ -1,5 +1,5 @@
 import type { InjectionToken, ModuleMetadata } from '@nestjs/common';
-import type { Writable } from 'node:stream';
+import type { Stream } from 'node:stream';
 
 import type { FileStorage, FileStorageConfigFactory } from './file-storage.class';
 import type { FileStorageLocalSetup } from './file-storage-fs.types';
@@ -60,4 +60,4 @@ interface WritableWithDoneEvent {
   removeListener(event: 'done', listener: () => void): this;
 }
 
-export type FileStorageWritable = Writable & WritableWithDoneEvent;
+export type FileStorageWritable = NodeJS.WritableStream & Stream & WritableWithDoneEvent;

packages/file-storage/test/file-storage-cases.ts

@@ -0,0 +1,88 @@
+import * as dotenv from 'dotenv';
+import { randomUUID } from 'node:crypto';
+import { resolve } from 'node:path';
+import { setTimeout } from 'node:timers/promises';
+
+import { FileStorage, StorageType } from '../src';
+
+dotenv.config({ path: resolve(__dirname, '../.env.test') });
+
+declare global {
+  // eslint-disable-next-line @typescript-eslint/no-namespace
+  namespace NodeJS {
+    interface ProcessEnv {
+      S3_BUCKET: string;
+      S3_REGION: string;
+      AWS_ACCESS_KEY_ID?: string;
+      AWS_SECRET_ACCESS_KEY?: string;
+      AWS_SECRET_SESSION_TOKEN?: string;
+      AWS_PROFILE?: string;
+      GC_BUCKET: string;
+      GC_PROJECT_ID?: string;
+      CI?: string;
+    }
+  }
+}
+
+export const fsStoragePath = resolve('store');
+
+export const testMap = [
+  {
+    description: 'file-storage-fs',
+    storageType: StorageType.FS,
+    options: {
+      [StorageType.FS]: { setup: { storagePath: fsStoragePath, maxPayloadSize: 1 } },
+    },
+  },
+  {
+    description: 'file-storage-S3',
+    storageType: StorageType.S3,
+    options: {
+      [StorageType.S3]: {
+        setup: {
+          maxPayloadSize: 1,
+          bucket: process.env.S3_BUCKET,
+          region: process.env.S3_REGION,
+          ...(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY
+            ? {
+                credentials: {
+                  accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+                  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+                  sessionToken: process.env.AWS_SECRET_SESSION_TOKEN,
+                },
+              }
+            : {}),
+        },
+      },
+    },
+  },
+  {
+    description: 'file-storage-GC',
+    storageType: StorageType.GC,
+    options: {
+      [StorageType.GC]: {
+        setup: {
+          maxPayloadSize: 1,
+          projectId: process.env.GC_PROJECT_ID,
+          bucketName: process.env.GC_BUCKET,
+        },
+      },
+    },
+  },
+] as const;
+
+export const delay = async (ms = 100) => {
+  if (process.env.CI) {
+    await setTimeout(ms);
+  }
+};
+
+export const createDummyFile = async (
+  fileStorage: FileStorage,
+  filePath: string = randomUUID(),
+  content = 'this is a test content',
+) => {
+  await fileStorage.uploadFile({ filePath, content });
+  await delay(100);
+  return { filePath, content };
+};