Comment Scraper V2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List of modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "import pickle\n",
    "\n",
    "from googleapiclient.discovery import build\n",
    "\n",
    "from operator import itemgetter\n",
    "\n",
    "import os\n",
    "from os import listdir\n",
    "from os.path import isfile, join"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read video URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readdata():\n",
    "    \n",
    "    datapath = os.getcwd() +\"\\\\yourpath\\\\trainfile\"\n",
    "    onlyfiles = [f for f in listdir(datapath) if isfile(join(datapath, f))]\n",
    "\n",
    "    return onlyfiles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read personality label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readdatapersonality():\n",
    "    \n",
    "    labelpath = os.getcwd() + \"\\\\yourpath\\\\annotation_training.pkl\"\n",
    "    with open(labelpath, 'rb') as f:\n",
    "        u = pickle._Unpickler(f)\n",
    "        u.encoding = 'latin1'\n",
    "        labeldata = u.load()\n",
    "    \n",
    "    return labeldata"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "YouTube function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def youtube_build():\n",
    "\n",
    "    youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION,developerKey=DEVELOPER_KEY)\n",
    "\n",
    "    return youtube"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract video's comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_video_comments(service, **kwargs):\n",
    "    \n",
    "    comments = []\n",
    "    usernames = []\n",
    "    dates = []\n",
    "    likes = []\n",
    "    \n",
    "    try:\n",
    "        results = service.commentThreads().list(**kwargs).execute()\n",
    "    except Exception:\n",
    "        return False\n",
    " \n",
    "    while results:\n",
    "        if len(comments) == 300:\n",
    "            break\n",
    "        \n",
    "        for item in results['items']:\n",
    "            comment = item['snippet']['topLevelComment']['snippet']['textDisplay']\n",
    "            comments.append(comment)\n",
    "            username = item['snippet']['topLevelComment']['snippet']['authorDisplayName']\n",
    "            usernames.append(username)\n",
    "            date = item['snippet']['topLevelComment']['snippet']['publishedAt']\n",
    "            dates.append(date)\n",
    "            like = item['snippet']['topLevelComment']['snippet']['likeCount']\n",
    "            likes.append(like)\n",
    " \n",
    "        if 'nextPageToken' in results:\n",
    "            kwargs['pageToken'] = results['nextPageToken']\n",
    "            results = service.commentThreads().list(**kwargs).execute()\n",
    "        else:\n",
    "            break\n",
    "    \n",
    "    if len(comments) == 0:\n",
    "        return False\n",
    "            \n",
    "    d = {'Comment':comments,'Username':usernames,'Date':dates,'Like':likes}\n",
    "    \n",
    "    return d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get personalities value for each video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_personalities(vid, lab):\n",
    "\n",
    "    d = {'Agreeableness':lab['agreeableness'][vid], 'Conscientiousness':lab['conscientiousness'][vid],\n",
    "         'Extraversion':lab['extraversion'][vid], 'Neuroticism':lab['neuroticism'][vid],\n",
    "         'Openness':lab['openness'][vid]}\n",
    "    \n",
    "    return d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine comment and personality data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractdata(listvideo, listlabel):\n",
    "    \n",
    "    cmlist = []\n",
    "    vidchecker = '';\n",
    "    \n",
    "    for video in listvideo:\n",
    "        if video[:-8] == vidchecker:\n",
    "            continue\n",
    "        vidchecker = video[:-8]\n",
    "        comment_data = get_video_comments(youtube_build(), part='snippet', videoId=video[:-8], textFormat='plainText')\n",
    "        person_data = get_personalities(video, listlabel)\n",
    "        if comment_data == False:\n",
    "            continue\n",
    "        d = {'Video':video[:-8],'Personality':person_data, 'Data':comment_data}\n",
    "        cmlist.append(d)\n",
    "    \n",
    "    return cmlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Write and save combined data in JSON file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def writefile(cmlist):\n",
    "    \n",
    "    with open('ssp2.json', 'a') as fp: #your file may be saved in here (print this) -> os.getcwd()\n",
    "        json.dump(cmlist, fp)\n",
    "    fp.close()\n",
    "    \n",
    "    f = open('ssp2.json','r')\n",
    "    old_data = f.read()\n",
    "    f.close()\n",
    "    \n",
    "    new_data = old_data.replace(\"][\", \",\")\n",
    "    \n",
    "    f = open('ssp2_new.json','w')\n",
    "    f.write(new_data)\n",
    "    f.close()\n",
    "    \n",
    "    os.remove('ssp2.json') \n",
    "    os.rename('ssp2_new.json', 'ssp2.json')  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read JSON file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readfile():\n",
    "    with open('ssp2.json', 'r') as fp: \n",
    "        complistnew = json.load(fp)\n",
    "\n",
    "    cmlistnew = sorted(complistnew, key=itemgetter('Video'))\n",
    "    \n",
    "    return cmlistnew"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Main code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEVELOPER_KEY = \"Your API here\"\n",
    "YOUTUBE_API_SERVICE_NAME = \"youtube\"\n",
    "YOUTUBE_API_VERSION = \"v3\"\n",
    "\n",
    "list_of_videonames = readdata()\n",
    "list_of_labels = readdatapersonality()\n",
    "\n",
    "complist = extractdata(list_of_videonames, list_of_labels)\n",
    "\n",
    "writefile(complist)\n",
    "\n",
    "complistnew = readfile()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}