Crawling_bobae&instiz.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium.webdriver.common.by import By # 셀레니움4에서 나타나는 오류 잡기위함\n",
    "import requests\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "\n",
    "### DB연결용\n",
    "import pymysql\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### web driver test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Chaaaaabbo\\AppData\\Local\\Temp\\ipykernel_29292\\1533073334.py:1: DeprecationWarning: executable_path has been deprecated, please pass in a Service object\n",
      "  driver = webdriver.Chrome(\"C:/Users/Chaaaaabbo/Desktop/chromedriver_win32/chromedriver.exe\")\n"
     ]
    }
   ],
   "source": [
    "def setup_chrome_driver():\n",
    "    chrome_options = Options()\n",
    "    chrome_options.add_argument(\"--headless\")  # Run Chrome in headless mode (without GUI)\n",
    "    chrome_options.add_argument('--blink-settings=imagesEnabled=false') # Not load image\n",
    "\n",
    "    # install Chrome webdriver\n",
    "    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)\n",
    "    return driver\n",
    "\n",
    "driver = setup_chrome_driver()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 보배드림"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: {'title': '맥북 사전예약중 참고하세요', 'url': 'https://www.bobaedream.co.kr//view?code=freeb&No=2454362', 'date': '21. 11. 15'}, 1: {'title': '맥북인데 윈도깔아쓰는 제 맥북..ㅠ ㅠ', 'url': 'https://www.bobaedream.co.kr//view?code=national&No=334380', 'date': '11. 12. 12'}, 2: {'title': '애플맥북', 'url': 'https://www.bobaedream.co.kr//view?code=freeb&No=2451286', 'date': '21. 11. 11'}}\n"
     ]
    }
   ],
   "source": [
    "# Set up Chrome options\n",
    "chrome_options = Options()\n",
    "chrome_options.add_argument(\"--headless\")  # Run Chrome in headless mode (without GUI)\n",
    "chrome_options.add_argument('--blink-settings=imagesEnabled=false') # Not load image\n",
    "\n",
    "# Set up Chrome webdriver\n",
    "driver = webdriver.Chrome(options=chrome_options)\n",
    "\n",
    "#Selenium_driver로 url 접속\n",
    "url = \"https://www.bobaedream.co.kr/search\"\n",
    "driver.get(url)\n",
    "#Selenium_접속하는 데 시간이 걸릴 수 있으므로 1초 기다림\n",
    "time.sleep(1)\n",
    "\n",
    "x_path_search = '/html/body/div/div[2]/span/form/input[6]'\n",
    "searchbox = driver.find_element(By.XPATH, x_path_search)\n",
    "searchbox.click()\n",
    "element = driver.find_element(By.NAME, \"keyword\")\n",
    "element.send_keys(\"맥북\")\n",
    "element.submit()\n",
    "\n",
    "# 검색 정렬 조건 정확도로 바꿈\n",
    "x_path_acc = '/html/body/div/div[3]/div[1]/div[1]/ul/li[1]/a' \n",
    "acc_box = driver.find_element(By.XPATH, x_path_acc)\n",
    "acc_box.click()\n",
    "\n",
    "#BeautifulSoup_html을 Parsing 함\n",
    "soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
    "\n",
    "# title은 \n",
    "search_in_commu = soup.find('div', \"search_Community\")\n",
    "search_results = search_in_commu.findAll(\"a\")\n",
    "\n",
    "data = {}\n",
    "cnt = 0\n",
    "for idx, result in enumerate(search_results):\n",
    "    if idx >= 3: #0:전체, 1:제목, 2:글쓴이\n",
    "        data_ = {}\n",
    "        data_['title'] = result.text\n",
    "        data_['url'] = f\"https://www.bobaedream.co.kr/{result['href']}\"\n",
    "        if data_['title'] != '':\n",
    "            data[cnt] = data_\n",
    "            cnt += 1\n",
    "        if cnt == 3:\n",
    "            break\n",
    "\n",
    "search_results = search_in_commu.findAll(\"span\",\"next\")\n",
    "cnt = 0\n",
    "for idx, result in enumerate(search_results):\n",
    "    if idx % 2 == 1:\n",
    "        data[cnt][\"date\"] = result.text\n",
    "        cnt += 1\n",
    "        if cnt == 3:\n",
    "            break\n",
    "print(data)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 인스티즈\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def instiz_crawling(word_list):\n",
    "    for word in word_list:\n",
    "        url = f\"https://www.instiz.net/popup_search.htm#gsc.tab=0&gsc.q={word}&gsc.sort=\"\n",
    "        driver.get(url)\n",
    "\n",
    "        # Parse the HTML response using Beautiful Soup to extract the data of each result\n",
    "        soup = BeautifulSoup(driver.page_source, \"html.parser\")\n",
    "        results = soup.find_all(\"a\", class_=\"gs-title\")\n",
    "\n",
    "        data = {}\n",
    "        temp = None\n",
    "        cnt = 0\n",
    "        try: \n",
    "            for result in results:\n",
    "                data_ = {}\n",
    "                title = result.text.split('-')[0][:-1]\n",
    "                if title == temp: # 이전 내용과 동일하면 pass\n",
    "                    pass\n",
    "                else:\n",
    "                    if title == '':\n",
    "                        break\n",
    "                    data_['title'] = temp = title\n",
    "                    data_['url'] = result['href']\n",
    "                    data[cnt] = data_\n",
    "                    cnt += 1\n",
    "        except Exception as e:\n",
    "            temp = None\n",
    "            print('검색결과가 없습니다.', e)\n",
    "        # Close the Chrome webdriver\n",
    "        driver.quit()\n",
    "\n",
    "    return data # 나중에 MySQL 어떻게 넣을지 고민하고 형태 바꿔야함"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'title': '맥북에 청포도에이슬 쏟은 트위터리안 후기.JPG',\n",
       "  'url': 'https://www.instiz.net/pt/7004209'},\n",
       " 1: {'title': '미친듯이 갈리는 애플 맥북 색상', 'url': 'https://www.instiz.net/pt/7215722'},\n",
       " 2: {'title': '업자한테 맥북 중고거래(판매)한 후기',\n",
       "  'url': 'https://www.instiz.net/pt/6541040'},\n",
       " 3: {'title': '맥북 꼭 사라 ..... 나도 노트북은 당연히 윈도우가 짱일줄',\n",
       "  'url': 'https://www.instiz.net/name/43138973'},\n",
       " 4: {'title': '와 류수정 인스타 맥북 감성 오진다',\n",
       "  'url': 'https://www.instiz.net/name_enter/75500207'},\n",
       " 5: {'title': '오빠한테 맥북사달랬더니 진짜사줌.Manhwa',\n",
       "  'url': 'https://www.instiz.net/pt/7054551'},\n",
       " 6: {'title': '우리집 고양이련이 맥북 떨궛다', 'url': 'https://www.instiz.net/pt/7064012'},\n",
       " 7: {'title': '애플 가로수길 매장에서 맥북 박살낸 썰',\n",
       "  'url': 'https://www.instiz.net/pt/6876481'},\n",
       " 8: {'title': '판에서 핫한 맥북 보상사건...jpg',\n",
       "  'url': 'https://www.instiz.net/pt/4010864'},\n",
       " 9: {'title': '맥북 제스처 모음.gif', 'url': 'https://www.instiz.net/pt/4668416'}}"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_list = ['맥북']\n",
    "instiz_crawling(word_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "613be4257a235896c62c6cddf239c19cb31941bb1e932190aafc389901e7e74e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}