Add Migration Scripts (#117)

* Add module for admin

* Add migration APIs

* Enable is_forward options

* Remove workspace slug options

* Remove unused code

* Update package-lock.json

* Fix lint

backend/package.json

@@ -33,6 +33,7 @@
 		"@prisma/client": "^5.8.1",
 		"class-transformer": "^0.5.1",
 		"class-validator": "^0.14.1",
+		"markdown-to-txt": "^2.0.1",
 		"langchain": "^0.1.9",
 		"moment": "^2.30.1",
 		"passport-github": "^1.1.0",

backend/src/admin/admin.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from "@nestjs/testing";
+import { AdminController } from "./admin.controller";
+
+describe("AdminController", () => {
+	let controller: AdminController;
+
+	beforeEach(async () => {
+		const module: TestingModule = await Test.createTestingModule({
+			controllers: [AdminController],
+		}).compile();
+
+		controller = module.get<AdminController>(AdminController);
+	});
+
+	it("should be defined", () => {
+		expect(controller).toBeDefined();
+	});
+});

backend/src/admin/admin.controller.ts

@@ -0,0 +1,21 @@
+import { Controller, Post, Query } from "@nestjs/common";
+import { AdminService } from "./admin.service";
+import { Public } from "src/utils/decorators/auth.decorator";
+import { ApiQuery } from "@nestjs/swagger";
+
+@Controller("admin")
+export class AdminController {
+	constructor(private adminService: AdminService) {}
+
+	@Public()
+	@Post("migrate")
+	@ApiQuery({
+		type: String,
+		name: "last_document_id",
+		description: "The last document ID of previous API Call",
+		required: false,
+	})
+	async migrate(@Query("last_document_id") lastDocumentId: string) {
+		this.adminService.migrateData("backup", lastDocumentId);
+	}
+}

backend/src/admin/admin.module.ts

@@ -0,0 +1,10 @@
+import { Module } from "@nestjs/common";
+import { AdminController } from "./admin.controller";
+import { AdminService } from "./admin.service";
+import { PrismaService } from "src/db/prisma.service";
+
+@Module({
+	controllers: [AdminController],
+	providers: [AdminService, PrismaService],
+})
+export class AdminModule {}

backend/src/admin/admin.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from "@nestjs/testing";
+import { AdminService } from "./admin.service";
+
+describe("AdminService", () => {
+	let service: AdminService;
+
+	beforeEach(async () => {
+		const module: TestingModule = await Test.createTestingModule({
+			providers: [AdminService],
+		}).compile();
+
+		service = module.get<AdminService>(AdminService);
+	});
+
+	it("should be defined", () => {
+		expect(service).toBeDefined();
+	});
+});

backend/src/admin/admin.service.ts

@@ -0,0 +1,148 @@
+import { Injectable } from "@nestjs/common";
+import { ConfigService } from "@nestjs/config";
+import { connect } from "http2";
+import { ListDocuments, YorkieDocument } from "./types/list-documents.type";
+import { FindDocumentFromYorkieResponse } from "src/workspace-documents/types/find-document-from-yorkie-response.type";
+import { PrismaService } from "src/db/prisma.service";
+import markdownToTxt from "markdown-to-txt";
+
+@Injectable()
+export class AdminService {
+	constructor(
+		private configService: ConfigService,
+		private prismaService: PrismaService
+	) {}
+
+	private async postHttp2<T>(path: string, requestBody: string) {
+		return new Promise<T>((resolve, reject) => {
+			const client = connect(`${this.configService.get<string>("YORKIE_API_ADDR")}`);
+
+			client.on("error", (err) => reject(err));
+
+			const req = client.request({
+				":method": "POST",
+				":path": path,
+				"Content-Type": "application/json",
+				"content-length": Buffer.byteLength(requestBody),
+				Authorization: this.configService.get<string>("YORKIE_PROJECT_SECRET_KEY"),
+			});
+
+			req.write(requestBody);
+			req.setEncoding("utf8");
+			let data = "";
+
+			req.on("data", (chunk) => {
+				data += chunk;
+			});
+
+			req.on("end", () => {
+				client.close();
+				resolve(JSON.parse(data) as T);
+			});
+
+			req.end();
+		});
+	}
+
+	async migrateData(
+		workspaceSlug: string,
+		prevLastDocumentId?: string
+	): Promise<Array<YorkieDocument>> {
+		let lastDocumentId: string = prevLastDocumentId;
+		let mergedDocumentList: Array<YorkieDocument> = [];
+
+		while (true) {
+			const pagedDocumentsList = await this.postHttp2<ListDocuments>(
+				"/yorkie.v1.AdminService/ListDocuments",
+				JSON.stringify({
+					project_name: this.configService.get<string>("YORKIE_PROJECT_NAME"),
+					page_size: 101,
+					previous_id: lastDocumentId,
+					is_forward: true,
+				})
+			);
+
+			if (pagedDocumentsList.documents?.length) {
+				lastDocumentId =
+					pagedDocumentsList.documents[pagedDocumentsList.documents.length - 1].id;
+			}
+
+			if (!pagedDocumentsList?.documents?.length) {
+				break;
+			}
+
+			mergedDocumentList = mergedDocumentList.concat(pagedDocumentsList.documents);
+		}
+
+		const workspace = await this.prismaService.workspace.findFirstOrThrow({
+			where: {
+				slug: workspaceSlug,
+			},
+		});
+
+		let cnt = 0,
+			skippedCnt = 0;
+		for (const document of mergedDocumentList) {
+			const foundDocumentInYorkie = await this.postHttp2<FindDocumentFromYorkieResponse>(
+				"/yorkie.v1.AdminService/GetDocument",
+				JSON.stringify({
+					project_name: this.configService.get<string>("YORKIE_PROJECT_NAME"),
+					document_key: document.key,
+				})
+			);
+			if (!foundDocumentInYorkie.document?.snapshot) continue;
+
+			let parsedSnapshot: { content: Array<{ val: string }> };
+			try {
+				parsedSnapshot = JSON.parse(foundDocumentInYorkie.document.snapshot);
+
+				if (!parsedSnapshot || !parsedSnapshot?.content) continue;
+			} catch (err) {
+				console.log(err);
+				console.log(foundDocumentInYorkie.document);
+			}
+
+			const content = parsedSnapshot.content.reduce((prev, cur) => {
+				return prev + cur.val;
+			}, "");
+			const title = markdownToTxt(content.slice(0, 200))
+				.trim()
+				.slice(0, 50)
+				.replaceAll("\n", " ")
+				.replace(/\s+/g, " ");
+
+			if (title) {
+				console.log(
+					`${++cnt}(${skippedCnt} Skipped)`,
+					"Doc Key: ",
+					document.key,
+					", Doc ID: ",
+					document.id,
+					", Title: ",
+					title
+				);
+
+				const duplicatedDocument = await this.prismaService.document.findFirst({
+					where: {
+						workspaceId: workspace.id,
+						yorkieDocumentId: document.key,
+					},
+				});
+
+				if (!duplicatedDocument) {
+					await this.prismaService.document.create({
+						data: {
+							title,
+							workspaceId: workspace.id,
+							yorkieDocumentId: document.key,
+						},
+					});
+				}
+			} else {
+				skippedCnt++;
+			}
+		}
+
+		return mergedDocumentList;
+	}
+}

backend/src/admin/types/list-documents.type.ts

@@ -0,0 +1,11 @@
+export class YorkieDocument {
+	id: string;
+	key: string;
+	createdAt: string;
+	accessedAt: string;
+	updatedAt: string;
+}
+
+export class ListDocuments {
+	documents: Array<YorkieDocument>;
+}

backend/src/app.module.ts

@@ -10,6 +10,7 @@ import { WorkspaceUsersModule } from "./workspace-users/workspace-users.module";
 import { WorkspaceDocumentsModule } from "./workspace-documents/workspace-documents.module";
 import { DocumentsModule } from "./documents/documents.module";
 import { CheckModule } from "./check/check.module";
+import { AdminModule } from "./admin/admin.module";
 import { IntelligenceModule } from "./intelligence/intelligence.module";
 import { LangchainModule } from "./langchain/langchain.module";
 
@@ -23,6 +24,7 @@ import { LangchainModule } from "./langchain/langchain.module";
 		WorkspaceDocumentsModule,
 		DocumentsModule,
 		CheckModule,
+		AdminModule,
 		IntelligenceModule,
 		LangchainModule,
 	],

frontend/src/hooks/api/workspaceDocument.ts

@@ -29,7 +29,7 @@ export const useGetWorkspaceDocumentListQuery = (workspaceId?: string) => {
 				{
 					params: {
 						cursor: pageParam,
-						page_size: 20,
+						page_size: 30,
 					},
 				}
 			);