SearchByKeywordAcademicAPI_test_Igarashi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "charitable-airplane",
   "metadata": {},
   "source": [
    "# Read data from file (emotion keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "narrative-november",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import os, sys\n",
    "\n",
    "\n",
    "\n",
    "wordlist_df = pd.read_excel('~/Documents/working/research/covid19sns/data/WordList.xlsx')\n",
    "wordlist = wordlist_df[\"orignal form\"].to_list() #store original words in df\n",
    "\n",
    "search_list = wordlist_df[~wordlist_df['Unnamed: 4'].isnull()]['Unnamed: 4'].to_list()\n",
    "search_list_cat = wordlist_df[~wordlist_df['Unnamed: 4'].isnull()]['file name'].to_list()\n",
    "\n",
    "csv_save_path = '/Users/admin/Documents/working/research/covid19sns/FullArchiveData_2021Apr/'\n",
    "\n",
    "def QueryWordsReg(searchwords):\n",
    "    return re.sub('%20',' ', searchwords) +  ' -is:reply -is:retweet lang:ja'\n",
    "\n",
    "querylist = [QueryWordsReg(searchwords) for searchwords in search_list ]\n",
    "\n",
    "#      modify query words in case of changes in API v2 \n",
    "#\n",
    "#      eg. '心配です%20OR%20懸念%20OR%20落ち着かない%20OR%20緊張する%20OR%20心配だ%20OR%20不安です%20OR%20不安だ'\n",
    "#     \n",
    "#     ----->>>    '心配です OR 懸念 OR 落ち着かない OR 緊張する OR 心配だ OR 不安です OR 不安だ lang:ja'\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "educated-classification",
   "metadata": {},
   "source": [
    "# Preparation of authentication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "polyphonic-orientation",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "\n",
    "CK = '' # Consumer Key\n",
    "CS = '' # Consumer Secret\n",
    "AT = '' # Access Token\n",
    "AS = '' # Accesss Token Secert\n",
    "\n",
    "BTpath = '/Users/admin/Documents/working/research/covid19sns/Li_code/BearerToken'\n",
    "\n",
    "with open(BTpath, 'r') as file:\n",
    "    BT = file.read()\n",
    "# Bearer Token\n",
    "\n",
    "\n",
    "\n",
    "search_url = \"https://api.twitter.com/2/tweets/search/all\"\n",
    "\n",
    "#keyword = querylist[2] #query words for test\n",
    "keyword = \"テニス\" #query words for test\n",
    "\n",
    "# Optional params: start_time,end_time,since_id,until_id,max_results,next_token,\n",
    "# expansions,tweet.fields,media.fields,poll.fields,place.fields,user.fields\n",
    "\n",
    "query_params = {'query': keyword ,\n",
    "                'tweet.fields': 'created_at',\n",
    "                #'expansions': 'author_id',\n",
    "                'start_time': '2021-02-15T00:00:00Z',\n",
    "                'end_time': '2021-02-15T09:00:00Z',\n",
    "                #'user.fields': 'description', # profile information of author\n",
    "                'max_results':500,  #一回のqueryは５００で上限らしい\n",
    "                'next_token' : {}   #次のページにいくためのparam？\n",
    "               }\n",
    "\n",
    "\n",
    "def create_headers(BT):\n",
    "    headers = {\"Authorization\": \"Bearer {}\".format(BT)}\n",
    "    return headers\n",
    "\n",
    "\n",
    "def connect_to_endpoint(url, headers, params):\n",
    "    response = requests.request(\"GET\", search_url, headers=headers, params=params)\n",
    "    #print(response.status_code)\n",
    "    if response.status_code != 200:\n",
    "        raise Exception(response.status_code, response.text)\n",
    "    return response.json()\n",
    "\n",
    "\n",
    "#    300 requests per 15-minute window (app auth)\n",
    "#\n",
    "#    1 request per second (app auth)\n",
    "#\n",
    "#    ----->>> sleep 1 sec between every query\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "source": [
    "# Define the main function"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 7,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "hairy-pursuit",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "\n",
    "    count = 0\n",
    "    flag = True\n",
    "    TWEET_LIMIT = 1000000    \n",
    "    cols = ['created_at', 'id', ' text']#[ここ改良]\n",
    "    normalized_data_old = pd.DataFrame(index=[], columns=cols)#[ここ改良]\n",
    "    \n",
    "    while flag:\n",
    "        #print('Collected Tweets: %.2f%% \\r' %(count*100/TWEET_LIMIT), end = '')\n",
    "        if count >= TWEET_LIMIT:\n",
    "            break\n",
    "    \n",
    "        headers = create_headers(BT)\n",
    "        time.sleep(1)\n",
    "        json_response = connect_to_endpoint(search_url, headers, query_params)\n",
    "        normalized_data = pd.json_normalize(json_response['data'])#[ここ改良]\n",
    "        normalized_data_new = pd.concat([normalized_data＿old, normalized_data])#[ここ改良]\n",
    "        normalized_data＿old=normalized_data_new#[ここ改良]\n",
    "        print(\"total:\"+str(len(normalized_data＿old))+\"tweets\")\n",
    "        #print(json_response)\n",
    "        #print(len(json_response['data']))\n",
    "    \n",
    "        result_count = json_response['meta']['result_count']\n",
    "        if 'next_token' in json_response['meta']:\n",
    "            next_token = json_response['meta']['next_token']\n",
    "            \n",
    "            query_params['next_token'] = next_token\n",
    "            \n",
    "            count += result_count\n",
    "            \n",
    "            time.sleep(3)  # rate limit = 1 request/1 sec\n",
    "            \n",
    "            json_response = connect_to_endpoint(search_url, headers, query_params)\n",
    "            \n",
    "            normalized_data = pd.json_normalize(json_response['data'])#[ここ改良]\n",
    "            normalized_data_new = pd.concat([normalized_data＿old, normalized_data])#[ここ改良]\n",
    "            normalized_data＿old=normalized_data_new#[ここ改良]\n",
    "            print(\"total:\"+str(len(normalized_data＿old))+\"tweets\")\n",
    "            #import pdb; pdb.set_trace()\n",
    "\n",
    "        else:\n",
    "            flag = False\n",
    "            \n",
    "    normalized_data_last=normalized_data_old#[ここ改良]\n",
    "    #normalized_data = pd.json_normalize(json_response['data'])\n",
    "    #normalized_users = pd.json_normalize(json_response['includes']['users']).set_index('id')\n",
    "    #normalized_places = pd.json_normalize(json_response['includes']['places']).set_index('id')\n",
    "    #print(len(normalized_data_last))#[ここ改良]\n",
    "            \n",
    "    #normalized_users.rename({'id': 'author_id'}, inplace = True, axis = 1)\n",
    "    #normalized_data.rename({'geo.place_id': 'place_id'}, inplace = True, axis = 1)\n",
    "    #normalized_places.rename({'id': 'place_id'}, inplace = True, axis = 1)\n",
    "            \n",
    "    #normalized = normalized_data.join(normalized_users, on = 'author_id', how = 'outer', rsuffix = '_user')\n",
    "    #normalized = normalized.join(normalized_places, on = 'place_id', how = 'outer', rsuffix = '_place')\n",
    "\n",
    "    normalized_data_last.to_csv('SearchTestAcademicAPI.csv')  #[ここ改良]\n",
    "\n",
    "    # csv_save_path = '/Users/admin/Documents/working/research/covid19sns/FullArchiveData_2021Apr/'\n",
    "    # normalized_data_last.to_csv(csv_save_path + 'SearchTestAcademicAPI.csv') \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "congressional-culture",
   "metadata": {},
   "source": [
    "# Building queries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "active-assignment",
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "total:110tweets\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "completed-utility",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = pd.read_csv(csv_save_path + 'SearchTestAcademicAPI.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "amateur-blood",
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "     Unnamed: 0                created_at                   id   text  \\\n",
       "0             0  2021-02-15T00:15:46.000Z  1361106893947609088    NaN   \n",
       "1             1  2021-02-15T00:15:39.000Z  1361106867108257793    NaN   \n",
       "2             2  2021-02-15T00:15:29.000Z  1361106821461680129    NaN   \n",
       "3             3  2021-02-15T00:15:22.000Z  1361106794731347969    NaN   \n",
       "4             4  2021-02-15T00:15:15.000Z  1361106764784046082    NaN   \n",
       "..          ...                       ...                  ...    ...   \n",
       "105         105  2021-02-15T00:00:59.000Z  1361103172618448896    NaN   \n",
       "106         106  2021-02-15T00:00:49.000Z  1361103133045362688    NaN   \n",
       "107         107  2021-02-15T00:00:26.000Z  1361103035695394817    NaN   \n",
       "108         108  2021-02-15T00:00:16.000Z  1361102995727912961    NaN   \n",
       "109         109  2021-02-15T00:00:16.000Z  1361102992586526721    NaN   \n",
       "\n",
       "                                                  text  \n",
       "0    無料レポート！！\\n◆テニスのライブベットのやり方と\\n少額をドカンと増やす為のアンダー、オ...  \n",
       "1       テニスのプレーヤーってラケットにあたる人が多いね。ラケット壊れて試合がより不利にならんのか？  \n",
       "2    RT @asahi_gdigimo: 五輪開催、テニスOKで柔道ダメ?　科学的に判断すると\\...  \n",
       "3                          @p_nuts_b テニスとか卓球とかはダメそうな……  \n",
       "4    駄目ですよ！暑いからと言って部屋でごろごろしてばっかりじゃ！さあ！早くブチャラティ！外にでて...  \n",
       "..                                                 ...  \n",
       "105  【拡散希望】越前 リョーマ グッズ を収集しています。ラバストや缶バッチなど身に付けられるグ...  \n",
       "106  ショッキングな初戦敗退で\\n涙を見せたモンフィス🇫🇷\\n\\n「僕は必ず復活する自信がある。僕...  \n",
       "107                            テニスコート広いしキレイと他校の子から言われた  \n",
       "108  RT @hochi_hanbai: #テニス の #全豪オープン 女子シングルス第3シードで...  \n",
       "109  →そういう人が、どうやったら前向きに試合に向き合えるかな。。誰か、乗り越えた経験とか、アドバ...  \n",
       "\n",
       "[110 rows x 5 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Unnamed: 0</th>\n      <th>created_at</th>\n      <th>id</th>\n      <th>text</th>\n      <th>text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>2021-02-15T00:15:46.000Z</td>\n      <td>1361106893947609088</td>\n      <td>NaN</td>\n      <td>無料レポート！！\\n◆テニスのライブベットのやり方と\\n少額をドカンと増やす為のアンダー、オ...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2021-02-15T00:15:39.000Z</td>\n      <td>1361106867108257793</td>\n      <td>NaN</td>\n      <td>テニスのプレーヤーってラケットにあたる人が多いね。ラケット壊れて試合がより不利にならんのか？</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>2021-02-15T00:15:29.000Z</td>\n      <td>1361106821461680129</td>\n      <td>NaN</td>\n      <td>RT @asahi_gdigimo: 五輪開催、テニスOKで柔道ダメ?　科学的に判断すると\\...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>2021-02-15T00:15:22.000Z</td>\n      <td>1361106794731347969</td>\n      <td>NaN</td>\n      <td>@p_nuts_b テニスとか卓球とかはダメそうな……</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>2021-02-15T00:15:15.000Z</td>\n      <td>1361106764784046082</td>\n      <td>NaN</td>\n      <td>駄目ですよ！暑いからと言って部屋でごろごろしてばっかりじゃ！さあ！早くブチャラティ！外にでて...</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>105</th>\n      <td>105</td>\n      <td>2021-02-15T00:00:59.000Z</td>\n      <td>1361103172618448896</td>\n      <td>NaN</td>\n      <td>【拡散希望】越前 リョーマ グッズ を収集しています。ラバストや缶バッチなど身に付けられるグ...</td>\n    </tr>\n    <tr>\n      <th>106</th>\n      <td>106</td>\n      <td>2021-02-15T00:00:49.000Z</td>\n      <td>1361103133045362688</td>\n      <td>NaN</td>\n      <td>ショッキングな初戦敗退で\\n涙を見せたモンフィス🇫🇷\\n\\n「僕は必ず復活する自信がある。僕...</td>\n    </tr>\n    <tr>\n      <th>107</th>\n      <td>107</td>\n      <td>2021-02-15T00:00:26.000Z</td>\n      <td>1361103035695394817</td>\n      <td>NaN</td>\n      <td>テニスコート広いしキレイと他校の子から言われた</td>\n    </tr>\n    <tr>\n      <th>108</th>\n      <td>108</td>\n      <td>2021-02-15T00:00:16.000Z</td>\n      <td>1361102995727912961</td>\n      <td>NaN</td>\n      <td>RT @hochi_hanbai: #テニス の #全豪オープン 女子シングルス第3シードで...</td>\n    </tr>\n    <tr>\n      <th>109</th>\n      <td>109</td>\n      <td>2021-02-15T00:00:16.000Z</td>\n      <td>1361102992586526721</td>\n      <td>NaN</td>\n      <td>→そういう人が、どうやったら前向きに試合に向き合えるかな。。誰か、乗り越えた経験とか、アドバ...</td>\n    </tr>\n  </tbody>\n</table>\n<p>110 rows × 5 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "logical-threshold",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}