Introduce MinIO for object storage (#365)

* feat: integrate MinIO for file storage
- Add MinIO support in FilesService
- Update docker-compose.yml to include MinIO service
- Implement environment variable switching between S3 and MinIO
- Add MinIO endpoint configuration
- Update documentation for MinIO setup and usage

* feat: Add storage module for file upload

- Add storage module to handle file upload functionality
- Configure S3Client for storage service
- Update files service to use injected S3Client
- Update files module to import storage module

* refactor: storage module to remove unnecessary credentials case by aws_s3

* refactor: storage module update condition check logic to use file_upload variable

* feat: set default configuration values for Minio

* refactor: update storage module to remove default value for AWS_REGION

* refactor: remove unused StorageType enum

* refactor: update .env.development to set default values for Minio

* feat: enhance file upload configuration for MinIO support

* refactor: update .env.development with detailed comments for MinIO configuration

* refactor: update .env.development for MinIO configuration with specific values

* refactor: update docker-compose configuration for MinIO file upload

* refactor: update file upload configuration type check in storage module

* refactor: update file upload configuration to enable based on false check

* feat: integrate MinIO for file storage
- Add MinIO support in FilesService
- Update docker-compose.yml to include MinIO service
- Implement environment variable switching between S3 and MinIO
- Add MinIO endpoint configuration
- Update documentation for MinIO setup and usage

* Feat configure MinIO for file storage and update environment variables

* Reformat code

* Remove redundant `links` field in `docker-compose`

* Add dependant condition

* Remove unused functions

* Remove unused file

---------

Co-authored-by: devleejb <[email protected]>

backend/.env.development

@@ -79,11 +79,26 @@ LANGCHAIN_API_KEY=your_langsmith_api_key_here
 # To create a LangSmith project, visit LangSmith: https://www.langchain.com/langsmith
 LANGCHAIN_PROJECT=your_langsmith_project_name_here
 
-
-# FILE_UPLOAD: Whether to enable file upload to storage
-# Set to true if file upload is required.
-# If set to false, AWS_S3_BUCKET_NAME is not required.
-FILE_UPLOAD=false
-# AWS_S3_BUCKET_NAME: S3 Bucket name 
-# This is the name of the S3 Bucket
-AWS_S3_BUCKET_NAME="your_s3_bucket_name"
+# FILE_UPLOAD: Whether to enable file upload to storage.  
+# Available options: false, s3, minio.  
+# Set to "false" if file upload is not required.  
+# Set to "s3" or "minio" to enable file uploads.  
+FILE_UPLOAD=minio
+# BUCKET_NAME: The name of the S3 or MinIO bucket to use.  
+# Required only if FILE_UPLOAD is set to "s3" or "minio".  
+BUCKET_NAME="default-storage"
+# MINIO_ENDPOINT: The endpoint URL for the MinIO server.  
+# Format: http(s)://<host>:<port>.  
+# Example: http://localhost:9000 (For development mode).  
+MINIO_ENDPOINT="http://localhost:9000"
+# MINIO_ACCESS_KEY: Access key for authentication.  
+# Default: minioadmin (for development only).  
+# Warning: Use a strong, unique value in production.  
+MINIO_ACCESS_KEY="minioadmin"
+# MINIO_SECRET_KEY: Secret key for authentication.  
+# Default: minioadmin (for development only).  
+# Warning: Keep this value secret and never commit to version control.  
+MINIO_SECRET_KEY="minioadmin"
+# AWS Region: only required for AWS S3  
+# Example: us-east-1 
+AWS_REGION="your_aws_region_here"

backend/docker/docker-compose-full.yml

@@ -26,16 +26,18 @@ services:
             LANGCHAIN_TRACING_V2: "false"
             LANGCHAIN_API_KEY: "your_langsmith_api_key_here"
             LANGCHAIN_PROJECT: "your_langsmith_project_name_here"
-            FILE_UPLOAD: false
-            AWS_S3_BUCKET_NAME: "your_s3_bucket_name"
+            FILE_UPLOAD: minio
+            AWS_REGION: "your_aws_region_here"
+            BUCKET_NAME: "default-storage"
+            MINIO_ENDPOINT: "http://localhost:9000"
+            MINIO_ACCESS_KEY: "minioadmin"
+            MINIO_SECRET_KEY: "minioadmin"
         ports:
             - "3000:3000"
         depends_on:
             - mongo
+            - minio
         restart: unless-stopped
-        links:
-            - "mongo:mongo"
-            - "yorkie:yorkie"
 
     yorkie:
         image: "yorkieteam/yorkie:0.5.6"
@@ -70,3 +72,31 @@ services:
             interval: 5s
             timeout: 2s
             retries: 20
+
+    # You can remove the following content if you're using S3 or not using Minio.
+    minio:
+        image: minio/minio
+        ports:
+            - "9000:9000"
+            - "9001:9001"
+        environment:
+            MINIO_ROOT_USER: "minioadmin"
+            MINIO_ROOT_PASSWORD: "minioadmin"
+        command: server --console-address ":9001" --address ":9000" /data
+        healthcheck:
+            test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+            interval: 30s
+            timeout: 20s
+            retries: 3
+
+    init_minio:
+        image: minio/mc
+        depends_on:
+            minio:
+                condition: service_healthy
+        entrypoint: >
+            /bin/sh -c "
+            mc alias set codepair http://minio:9000 minioadmin minioadmin --api S3v4;
+            mc mb codepair/default-storage;
+            exit 0;
+            "

backend/docker/docker-compose.yml

@@ -34,3 +34,30 @@ services:
             interval: 5s
             timeout: 2s
             retries: 20
+
+    minio:
+        image: minio/minio
+        ports:
+            - "9000:9000"
+            - "9001:9001"
+        environment:
+            MINIO_ROOT_USER: "minioadmin"
+            MINIO_ROOT_PASSWORD: "minioadmin"
+        command: server --console-address ":9001" --address ":9000" /data
+        healthcheck:
+            test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+            interval: 30s
+            timeout: 20s
+            retries: 3
+
+    init_minio:
+        image: minio/mc
+        depends_on:
+            minio:
+                condition: service_healthy
+        entrypoint: >
+            /bin/sh -c "
+            mc alias set codepair http://minio:9000 minioadmin minioadmin --api S3v4;
+            mc mb codepair/default-storage;
+            exit 0;
+            "

backend/src/app.module.ts

@@ -1,19 +1,20 @@
 import { Module } from "@nestjs/common";
-import { PrismaService } from "./db/prisma.service";
-import { UsersModule } from "./users/users.module";
-import { AuthModule } from "./auth/auth.module";
 import { ConfigModule } from "@nestjs/config";
 import { APP_GUARD } from "@nestjs/core/constants";
+import { AuthModule } from "./auth/auth.module";
 import { JwtAuthGuard } from "./auth/jwt.guard";
-import { WorkspacesModule } from "./workspaces/workspaces.module";
-import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module";
-import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module";
-import { DocumentsModule } from "./documents/documents.module";
 import { CheckModule } from "./check/check.module";
+import { PrismaService } from "./db/prisma.service";
+import { DocumentsModule } from "./documents/documents.module";
+import { FilesModule } from "./files/files.module";
 import { IntelligenceModule } from "./intelligence/intelligence.module";
 import { LangchainModule } from "./langchain/langchain.module";
-import { FilesModule } from "./files/files.module";
 import { SettingsModule } from "./settings/settings.module";
+import { StorageModule } from "./storage/storage.module";
+import { UsersModule } from "./users/users.module";
+import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module";
+import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module";
+import { WorkspacesModule } from "./workspaces/workspaces.module";
 
 @Module({
 	imports: [
@@ -34,6 +35,7 @@ import { SettingsModule } from "./settings/settings.module";
 		FilesModule,
 		ConfigModule,
 		SettingsModule,
+		StorageModule,
 	],
 	controllers: [],
 	providers: [

backend/src/files/files.module.ts

@@ -1,9 +1,11 @@
 import { Module } from "@nestjs/common";
+import { PrismaService } from "src/db/prisma.service";
+import { StorageModule } from "src/storage/storage.module";
 import { FilesController } from "./files.controller";
 import { FilesService } from "./files.service";
-import { PrismaService } from "src/db/prisma.service";
 
 @Module({
+	imports: [StorageModule],
 	controllers: [FilesController],
 	providers: [FilesService, PrismaService],
 })

backend/src/files/files.service.ts

@@ -2,6 +2,7 @@ import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import {
 	BadRequestException,
+	Inject,
 	Injectable,
 	NotFoundException,
 	UnauthorizedException,
@@ -18,14 +19,13 @@ import { ExportFileRequestBody, ExportFileResponse } from "./types/export-file.t
 
 @Injectable()
 export class FilesService {
-	private s3Client: S3Client;
 	private readonly markdown: MarkdownIt;
 
 	constructor(
+		@Inject("STORAGE_CLIENT") private s3Client: S3Client,
 		private configService: ConfigService,
 		private prismaService: PrismaService
 	) {
-		this.s3Client = new S3Client();
 		this.markdown = new MarkdownIt({
 			html: true,
 			breaks: true,
@@ -55,7 +55,7 @@ export class FilesService {
 
 		const fileKey = `${workspace.slug}-${generateRandomKey()}.${contentType.split("/")[1]}`;
 		const command = new PutObjectCommand({
-			Bucket: this.configService.get("AWS_S3_BUCKET_NAME"),
+			Bucket: this.configService.get("BUCKET_NAME"),
 			Key: fileKey,
 			StorageClass: "INTELLIGENT_TIERING",
 			ContentType: contentType,
@@ -70,7 +70,7 @@ export class FilesService {
 	async createDownloadPresignedUrl(fileKey: string) {
 		try {
 			const command = new GetObjectCommand({
-				Bucket: this.configService.get("AWS_S3_BUCKET_NAME"),
+				Bucket: this.configService.get("BUCKET_NAME"),
 				Key: fileKey,
 			});
 			return getSignedUrl(this.s3Client, command, { expiresIn: 3600 });

backend/src/settings/settings.service.ts

@@ -16,7 +16,7 @@ export class SettingsService {
 				},
 			},
 			fileUpload: {
-				enable: this.configService.get("FILE_UPLOAD") === "true",
+				enable: this.configService.get("FILE_UPLOAD") !== "false",
 			},
 		};
 	}

backend/src/storage/storage.module.ts

@@ -0,0 +1,39 @@
+import { S3Client, S3ClientConfig } from "@aws-sdk/client-s3";
+import { Module } from "@nestjs/common";
+import { ConfigService } from "@nestjs/config";
+
+const s3ClientFactory = {
+	provide: "STORAGE_CLIENT",
+	useFactory: (configService: ConfigService): S3Client | null => {
+		const fileUpload = configService.get<"false" | "s3" | "minio">("FILE_UPLOAD");
+		if (fileUpload === "false") {
+			return null;
+		}
+
+		const region = configService.get<string>("AWS_REGION");
+		const endpoint = configService.get<string>("MINIO_ENDPOINT");
+		const accessKeyId = configService.get<string>("MINIO_ACCESS_KEY");
+		const secretAccessKey = configService.get<string>("MINIO_SECRET_KEY");
+
+		const config: S3ClientConfig = {
+			region,
+			...(fileUpload === "minio" && {
+				endpoint,
+				forcePathStyle: true,
+				credentials: {
+					accessKeyId,
+					secretAccessKey,
+				},
+			}),
+		};
+
+		return new S3Client(config);
+	},
+	inject: [ConfigService],
+};
+
+@Module({
+	providers: [s3ClientFactory],
+	exports: [s3ClientFactory],
+})
+export class StorageModule {}