검색을 위한 ElasticSearch 설정 및 장소 검색 성능 개선

backend/package.json

@@ -32,7 +32,9 @@
     "nestjs-pino": "^4.1.0",
     "pino-http": "^10.3.0",
     "rxjs": "^7.8.1",
-    "typeorm": "^0.3.20"
+    "typeorm": "^0.3.20",
+    "@nestjs/elasticsearch": "^10.0.2",
+    "@elastic/elasticsearch": "^8.16.0"
   },
   "devDependencies": {
     "@nestjs/cli": "^10.0.0",

backend/resources/elasticsearch/Dockerfile

@@ -0,0 +1,3 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:8.16.0
+
+RUN elasticsearch-plugin install --batch analysis-nori

backend/resources/elasticsearch/docker-compose.production.yml

@@ -0,0 +1,82 @@
+services:
+  setup:
+    profiles:
+      - setup
+    build:
+      context: .
+    init: true
+    user: root
+    volumes:
+      - ./index/place-index.json:/usr/share/elasticsearch/place-index.json
+      - ./index/synonyms.txt:/usr/share/elasticsearch/config/synonyms.txt
+      - ./init-es.prod.sh:/usr/share/elasticsearch/init-es.prod.sh
+    command: >
+      /bin/bash -c '
+        until curl -s -XGET "http://elasticsearch:9200/_cluster/health" -u "elastic:${ELASTIC_PASSWORD}" | grep "\"status\":\"green\"" > /dev/null; do
+          echo "Waiting for Elasticsearch to be ready..."
+          sleep 5
+        done &&
+        curl -X POST "http://elasticsearch:9200/_security/role/custom_role" -H "Content-Type: application/json" -u elastic:${ELASTIC_PASSWORD} -d "{
+                  \"cluster\": [\"all\"],
+                  \"indices\": [
+                    {
+                      \"names\": [\"*\"],
+                      \"privileges\": [\"all\"]
+                    }
+                  ],
+                  \"applications\": [
+                    {
+                      \"application\": \"kibana-.kibana\",
+                      \"privileges\": [\"all\"],
+                      \"resources\": [\"*\"]
+                    }
+                  ],
+                  \"run_as\": [\"*\"]
+                }" &&
+
+        curl -X POST "http://elasticsearch:9200/_security/user/${CUSTOM_USERNAME}" -H "Content-Type: application/json" -u elastic:${ELASTIC_PASSWORD} -d "{\"password\": \"${ELASTIC_PASSWORD}\", \"roles\": [\"custom_role\", \"kibana_system\"]}" &&
+        chmod +x /usr/share/elasticsearch/init-es.prod.sh &&
+        /usr/share/elasticsearch/init-es.prod.sh
+      '
+    environment:
+      - CUSTOM_USERNAME=${CUSTOM_USERNAME}
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+    depends_on:
+      - elasticsearch
+
+  elasticsearch:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: elasticsearch
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata:/usr/share/elasticsearch/data
+      - ./index/synonyms.txt:/usr/share/elasticsearch/config/synonyms.txt
+    ports:
+      - "9200:9200"
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.16.0
+    container_name: kibana
+    depends_on:
+      - elasticsearch
+    environment:
+      - SERVER_NAME=kibana
+      - ELASTICSEARCH_USERNAME=${CUSTOM_USERNAME}
+      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+    ports:
+      - "5601:5601"
+
+volumes:
+  esdata:
+    driver: local

backend/resources/elasticsearch/docker-compose.yml

@@ -0,0 +1,42 @@
+services:
+  elasticsearch:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: elasticsearch
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - esdata:/usr/share/elasticsearch/data
+      - ./index/place-index.json:/usr/share/elasticsearch/place-index.json
+      - ./index/synonyms.txt:/usr/share/elasticsearch/config/synonyms.txt
+      - ./init-es.local.sh:/usr/share/elasticsearch/init-es.local.sh
+    ports:
+      - "9200:9200"
+    command: >
+      /bin/bash -c "
+      chmod +x /usr/share/elasticsearch/init-es.local &&
+      /usr/share/elasticsearch/init-es.local &
+      exec /usr/local/bin/docker-entrypoint.sh
+      "
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:8.16.0
+    container_name: kibana
+    depends_on:
+      - elasticsearch
+    environment:
+      - SERVER_NAME=kibana
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+    ports:
+      - "5601:5601"
+
+volumes:
+  esdata:
+    driver: local

backend/resources/elasticsearch/index/place-index.json

@@ -0,0 +1,83 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "number_of_replicas": 0,
+    "analysis": {
+      "analyzer": {
+        "nori_with_synonym": {
+          "type": "custom",
+          "tokenizer": "nori_tokenizer",
+          "filter": [
+            "lowercase",
+            "nori_readingform",
+            "synonym_filter"
+          ]
+        }
+      },
+      "filter": {
+        "synonym_filter": {
+          "type": "synonym",
+          "synonyms_path": "synonyms.txt"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties": {
+      "id": {
+        "type": "integer"
+      },
+      "googlePlaceId": {
+        "type": "keyword"
+      },
+      "name": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword"
+          }
+        },
+        "analyzer": "nori_with_synonym"
+      },
+      "thumbnailUrl": {
+        "type": "text",
+        "index": false
+      },
+      "rating": {
+        "type": "scaled_float",
+        "scaling_factor": 100
+      },
+      "location": {
+        "type": "geo_point"
+      },
+      "formattedAddress": {
+        "type": "text",
+        "analyzer": "nori_with_synonym"
+      },
+      "category": {
+        "type": "keyword",
+        "index": true
+      },
+      "description": {
+        "type": "text"
+      },
+      "detailPageUrl": {
+        "type": "text",
+        "index": false
+      },
+      "createdAt": {
+        "type": "date",
+        "format": "strict_date_optional_time||epoch_millis"
+      },
+      "updatedAt": {
+        "type": "date",
+        "format": "strict_date_optional_time||epoch_millis"
+      },
+      "deletedAt": {
+        "type": "date",
+        "format": "strict_date_optional_time||epoch_millis",
+        "null_value": null
+      }
+    }
+  }
+}

backend/resources/elasticsearch/index/synonyms.txt

@@ -0,0 +1,17 @@
+서울특별시, 서울, 서울시
+부산광역시, 부산, 부산시
+대구광역시, 대구, 대구시
+인천광역시, 인천, 인천시
+광주광역시, 광주, 광주시
+대전광역시, 대전, 대전시
+울산광역시, 울산, 울산시
+세종특별자치시, 세종, 세종시
+경기도, 경기, 경기도청
+강원도, 강원, 강원도청
+충청북도, 충북, 충청북도청
+충청남도, 충남, 충청남도청
+전라북도, 전북, 전북특별자치도, 전라북도청
+전라남도, 전남, 전라남도청
+경상북도, 경북, 경상북도청
+경상남도, 경남, 경상남도청
+제주특별자치도, 제주, 제주도

backend/resources/elasticsearch/init-es.local.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+until curl -s -XGET "http://localhost:9200/_cluster/health" | grep '"status":"green"' > /dev/null; do
+  echo "Waiting for Elasticsearch to be ready..."
+  sleep 5
+done
+# Place 인덱스 확인 및 생성
+echo "Checking if 'place' index exists..."
+response=$(curl -s -o /dev/null -w "%{http_code}" -XGET "http://localhost:9200/place")
+if [ "$response" -eq 404 ]; then
+  echo "Index 'place' does not exist. Creating it..."
+  curl -X PUT "http://localhost:9200/place" \
+       -H "Content-Type: application/json" \
+       --data-binary @/usr/share/elasticsearch/place-index.json
+  echo "'place' index created successfully."
+elif [ "$response" -eq 200 ]; then
+  echo "Index 'place' already exists."
+else
+  echo "Unexpected response while checking for 'place' index: HTTP $response"
+fi

backend/resources/elasticsearch/init-es.prod.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+echo "Checking if 'place' index exists..."
+response=$(curl -s -o /dev/null -w "%{http_code}" -XGET "http://elasticsearch:9200/place" -u "elastic:${ELASTIC_PASSWORD}")
+if [ "$response" -eq 404 ]; then
+  echo "Index 'place' does not exist. Creating it..."
+  curl -X PUT "http://elasticsearch:9200/place" \
+       -H "Content-Type: application/json" \
+       -u "elastic:${ELASTIC_PASSWORD}" \
+       --data-binary @/usr/share/elasticsearch/place-index.json
+  echo "'place' index created successfully."
+elif [ "$response" -eq 200 ]; then
+  echo "Index 'place' already exists."
+else
+  echo "Unexpected response while checking for 'place' index: HTTP $response"
+fi

backend/resources/elasticsearch/sample_env

@@ -0,0 +1,2 @@
+CUSTOM_USERNAME=
+ELASTIC_PASSWORD=

backend/resources/nginx/nginx.conf

@@ -61,4 +61,26 @@ http {
             include /etc/nginx/proxy_headers.conf;
         }
     }
+
+    # Elasticsearch 서브도메인
+    server {
+        listen 443 ssl;
+        server_name es.dailyroad.site;
+
+        location / {
+            proxy_pass http://<SEARCH_PRIVATE_IP>:9200;
+            include /etc/nginx/proxy_headers.conf;
+        }
+    }
+
+    # Search(Kibana) 서브도메인
+    server {
+        listen 443 ssl;
+        server_name search.dailyroad.site;
+
+        location / {
+            proxy_pass http://<SEARCH_PRIVATE_IP>:5601;
+            include /etc/nginx/proxy_headers.conf;
+        }
+    }
 }

backend/resources/scripts/elk/elasticsearch/log_index_template.json

@@ -0,0 +1,47 @@
+{
+  "index_patterns": [
+    "dailyroad-*"
+  ],
+  "template": {
+    "settings": {
+      "number_of_shards": 1,
+      "number_of_replicas": 1
+    },
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "log_level": {
+          "type": "keyword"
+        },
+        "log_message": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "req": {
+          "properties": {
+            "id": {
+              "type": "integer"
+            },
+            "method": {
+              "type": "keyword"
+            },
+            "url": {
+              "type": "keyword"
+            },
+            "headers": {
+              "type": "object",
+              "enabled": false
+            }
+          }
+        }
+      }
+    }
+  }
+}

backend/resources/scripts/elk/logstash/logstash.conf

@@ -0,0 +1,34 @@
+input {
+    beats {
+        port => 5044
+    }
+
+    tcp {
+        port => 50000
+        codec => json_lines # JSON 형태의 데이터를 처리
+    }
+}
+
+filter {
+    json {
+        source => "message"
+        target => "parsed_message"
+    }
+
+    mutate {
+        rename => { "[parsed_message][level]" => "log_level" }
+        rename => { "[parsed_message][msg]" => "log_message" }
+        rename => { "[parsed_message][time]" => "@timestamp" }
+
+        remove_field => ["message", "parsed_message"]
+    }
+}
+
+output {
+    elasticsearch {
+        index => "%{[@metadata][service_name]}-%{+YYYY.MM.dd}"
+        hosts => "elasticsearch:9200"
+        user => "logstash_internal"
+        password => "${LOGSTASH_INTERNAL_PASSWORD}"
+    }
+}