From 35271099f676fb0dfcb41d96e0b99c9cef5b50e9 Mon Sep 17 00:00:00 2001 From: miensoap Date: Tue, 19 Nov 2024 19:57:11 +0900 Subject: [PATCH] =?UTF-8?q?docs:=20=EB=8D=B0=EC=9D=B4=ED=84=B0=20=EA=B4=80?= =?UTF-8?q?=EB=A0=A8=20=EC=8A=A4=ED=81=AC=EB=A6=BD=ED=8A=B8=20=EC=97=85?= =?UTF-8?q?=EB=8D=B0=EC=9D=B4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/data/exportToESFormat.js | 89 +++++++++++++++++++ backend/resources/scripts/data/toCsv.js | 9 +- backend/src/place/dto/PlaceListResponse.ts | 3 +- backend/src/place/dto/PlaceSearchResponse.ts | 3 +- 4 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 backend/resources/scripts/data/exportToESFormat.js diff --git a/backend/resources/scripts/data/exportToESFormat.js b/backend/resources/scripts/data/exportToESFormat.js new file mode 100644 index 00000000..56d51b54 --- /dev/null +++ b/backend/resources/scripts/data/exportToESFormat.js @@ -0,0 +1,89 @@ +const mysql = require("mysql2/promise"); +const { Client } = require("ssh2"); +const fs = require("fs"); + +const sshConfig = { + host: "example.example", + port: 22, + username: "example", + privateKey: fs.readFileSync( + "example" + ), +}; + +const dbConfig = { + host: "example", + user: "example", + password: "example", + database: "example", + port: "3306", +}; + +// Elasticsearch Bulk 파일 생성 함수 +async function generateBulkFile(query, outputFilePath, indexName) { + const sshClient = new Client(); + + return new Promise((resolve, reject) => { + sshClient + .on("ready", () => { + console.log("SSH Connection Established"); + + sshClient.forwardOut( + "127.0.0.1", + 3000, + dbConfig.host, + dbConfig.port, + async (err, stream) => { + if (err) { + sshClient.end(); + return reject(err); + } + + const connection = await mysql.createConnection({ + ...dbConfig, + stream, + }); + + console.log("DB Connection Established"); + + try { + const [rows] = await connection.execute(query); + + const bulkLines = + rows + .map((row) => { + const id = row.id; + delete row.id; + + const meta = JSON.stringify({ + index: { _index: indexName, _id: id }, + }); + const data = JSON.stringify(row); + return `${meta}\n${data}`; + }) + .join("\n") + "\n"; + + fs.writeFileSync(outputFilePath, bulkLines, "utf8"); + console.log(`Bulk file created: ${outputFilePath}`); + resolve(); + } catch (error) { + reject(error); + } finally { + await connection.end(); + sshClient.end(); + } + } + ); + }) + .on("error", reject) + .connect(sshConfig); + }); +} + +const query = "SELECT * FROM PLACE"; +const outputFilePath = "./bulk_place_data.json"; +const indexName = "place"; + +generateBulkFile(query, outputFilePath, indexName) + .then(() => console.log("Bulk file generation completed.")) + .catch((err) => console.error("Error:", err)); diff --git a/backend/resources/scripts/data/toCsv.js b/backend/resources/scripts/data/toCsv.js index 2e3aaff3..1a8d9ed4 100644 --- a/backend/resources/scripts/data/toCsv.js +++ b/backend/resources/scripts/data/toCsv.js @@ -1,6 +1,13 @@ const fs = require("fs"); const { createObjectCsvWriter } = require("csv-writer"); +const category = { + 100: "명소", + 300: "맛집", + 400: "카페", + 500: "숙소", +}; + async function jsonToCsv(inputJsonPath, outputCsvPath) { const jsonData = JSON.parse(fs.readFileSync(inputJsonPath, "utf-8")); @@ -28,7 +35,7 @@ async function jsonToCsv(inputJsonPath, outputCsvPath) { longitude: parseFloat(item.longitude), latitude: parseFloat(item.latitude), formatted_address: item.address || null, - category: item.type || null, + category: category[item.type] || null, description: item.description || null, detail_page_url: item.googleURL || null, })); diff --git a/backend/src/place/dto/PlaceListResponse.ts b/backend/src/place/dto/PlaceListResponse.ts index 24e71e59..e7e600b3 100644 --- a/backend/src/place/dto/PlaceListResponse.ts +++ b/backend/src/place/dto/PlaceListResponse.ts @@ -15,8 +15,7 @@ export class PlaceListResponse { readonly thumbnail_url?: string, readonly rating?: number, readonly formed_address?: string, - ) { - } + ) {} static from(place: Place): PlaceListResponse { return new PlaceListResponse( diff --git a/backend/src/place/dto/PlaceSearchResponse.ts b/backend/src/place/dto/PlaceSearchResponse.ts index 8f2753f3..40f39d0a 100644 --- a/backend/src/place/dto/PlaceSearchResponse.ts +++ b/backend/src/place/dto/PlaceSearchResponse.ts @@ -15,8 +15,7 @@ export class PlaceSearchResponse { readonly thumbnail_url?: string, readonly rating?: number, readonly formed_address?: string, - ) { - } + ) {} static from(place: Place): PlaceSearchResponse { return new PlaceSearchResponse(