Skip to content

Commit

Permalink
docs: 데이터 관련 스크립트 업데이트
Browse files Browse the repository at this point in the history
  • Loading branch information
Miensoap committed Nov 19, 2024
1 parent eb16120 commit 8920536
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 1 deletion.
89 changes: 89 additions & 0 deletions backend/resources/scripts/data/exportToESFormat.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
const mysql = require("mysql2/promise");
const { Client } = require("ssh2");
const fs = require("fs");

const sshConfig = {
host: "example.example",
port: 22,
username: "example",
privateKey: fs.readFileSync(
"example"
),
};

const dbConfig = {
host: "example",
user: "example",
password: "example",
database: "example",
port: "3306",
};

// Elasticsearch Bulk 파일 생성 함수
async function generateBulkFile(query, outputFilePath, indexName) {
const sshClient = new Client();

return new Promise((resolve, reject) => {
sshClient
.on("ready", () => {
console.log("SSH Connection Established");

sshClient.forwardOut(
"127.0.0.1",
3000,
dbConfig.host,
dbConfig.port,
async (err, stream) => {
if (err) {
sshClient.end();
return reject(err);
}

const connection = await mysql.createConnection({
...dbConfig,
stream,
});

console.log("DB Connection Established");

try {
const [rows] = await connection.execute(query);

const bulkLines =
rows
.map((row) => {
const id = row.id;
delete row.id;

const meta = JSON.stringify({
index: { _index: indexName, _id: id },
});
const data = JSON.stringify(row);
return `${meta}\n${data}`;
})
.join("\n") + "\n";

fs.writeFileSync(outputFilePath, bulkLines, "utf8");
console.log(`Bulk file created: ${outputFilePath}`);
resolve();
} catch (error) {
reject(error);
} finally {
await connection.end();
sshClient.end();
}
}
);
})
.on("error", reject)
.connect(sshConfig);
});
}

const query = "SELECT * FROM PLACE";
const outputFilePath = "./bulk_place_data.json";
const indexName = "place";

generateBulkFile(query, outputFilePath, indexName)
.then(() => console.log("Bulk file generation completed."))
.catch((err) => console.error("Error:", err));
9 changes: 8 additions & 1 deletion backend/resources/scripts/data/toCsv.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
const fs = require("fs");
const { createObjectCsvWriter } = require("csv-writer");

const category = {
100: "명소",
300: "맛집",
400: "카페",
500: "숙소",
};

async function jsonToCsv(inputJsonPath, outputCsvPath) {
const jsonData = JSON.parse(fs.readFileSync(inputJsonPath, "utf-8"));

Expand Down Expand Up @@ -28,7 +35,7 @@ async function jsonToCsv(inputJsonPath, outputCsvPath) {
longitude: parseFloat(item.longitude),
latitude: parseFloat(item.latitude),
formatted_address: item.address || null,
category: item.type || null,
category: category[item.type] || null,
description: item.description || null,
detail_page_url: item.googleURL || null,
}));
Expand Down

0 comments on commit 8920536

Please sign in to comment.