Non-Relational Database for Tweets with MongoDB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "40ddc3dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pinged your deployment. You successfully connected to MongoDB!\n"
     ]
    }
   ],
   "source": [
    "from pymongo import MongoClient\n",
    "\n",
    "\n",
    "\n",
    "# Specify the connection URI for your MongoDB deployment\n",
    "uri = \"mongodb+srv://dbHung:KPacAYLtMd4EEaQ8@atlascluster.tec2lmw.mongodb.net/?retryWrites=true&w=majority&appName=AtlasCluster\"\n",
    "\n",
    "# Create a new client and connect to the server\n",
    "client = MongoClient(uri)\n",
    "\n",
    "# Send a ping to confirm a successful connection\n",
    "try:\n",
    "    client.admin.command('ping')\n",
    "    print(\"Pinged your deployment. You successfully connected to MongoDB!\")\n",
    "except Exception as e:\n",
    "    print(e)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6b571c5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting utils\n",
      "  Downloading utils-1.0.2.tar.gz (13 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hBuilding wheels for collected packages: utils\n",
      "  Building wheel for utils (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for utils: filename=utils-1.0.2-py2.py3-none-any.whl size=13906 sha256=1c1e74368aed39aed7cfce1d0842c71951b4beb0a7b0f32ae5db06e4b3aff1f9\n",
      "  Stored in directory: /Users/hsiao-chunhung/Library/Caches/pip/wheels/15/0c/b3/674aea8c5d91c642c817d4d630bd58faa316724b136844094d\n",
      "Successfully built utils\n",
      "Installing collected packages: utils\n",
      "Successfully installed utils-1.0.2\n"
     ]
    }
   ],
   "source": [
    "\n",
    "! pip install utils \n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import pandas as pd \n",
    "import json "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "db2b2581",
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'connNoSQL' from 'utils' (/Users/hsiao-chunhung/anaconda3/lib/python3.11/site-packages/utils/__init__.py)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m connNoSQL\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minsert_data_to_mongodb\u001b[39m(db):\n\u001b[1;32m      4\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m      5\u001b[0m         \u001b[38;5;66;03m# Insert documents into MongoDB collection\u001b[39;00m\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'connNoSQL' from 'utils' (/Users/hsiao-chunhung/anaconda3/lib/python3.11/site-packages/utils/__init__.py)"
     ]
    }
   ],
   "source": [
    "from utils import connNoSQL\n",
    "\n",
    "def insert_data_to_mongodb(db):\n",
    "    try:\n",
    "        # Insert documents into MongoDB collection\n",
    "        db.twitter_collection.insert_many(docs)\n",
    "        print('MongoDB: *** Data Successfully Inserted ***')\n",
    "    except Exception as e:\n",
    "        print(f'MongoDB: *** Data Insertion Unsuccessful as {e} ***')\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    client = connNoSQL()\n",
    "\n",
    "    # Create MongoDB database instance\n",
    "    db = client.twitter_db\n",
    "\n",
    "    # Load the Twitter data\n",
    "    twitterdf = pd.concat([\n",
    "        pd.read_json(\"corona-out-2\", lines=True),\n",
    "        pd.read_json(\"corona-out-3\", lines=True)\n",
    "    ])\n",
    "        # Create new required columns\n",
    "    twitterdf[\"user_id\"] = twitterdf[\"user\"].apply(\n",
    "        lambda x: x[\"id_str\"]\n",
    "    )\n",
    "\n",
    "    # Reset indices\n",
    "    twitterdf.reset_index(inplace=True, drop=True)\n",
    "\n",
    "    twitterdf = twitterdf[\n",
    "        [\n",
    "            'contributors',\n",
    "            'text',\n",
    "            'source',\n",
    "            'id_str',\n",
    "            'created_at',\n",
    "            'user_id',\n",
    "            'truncated',\n",
    "            'lang',\n",
    "            'quote_count',\n",
    "            'reply_count',\n",
    "            'retweet_count',\n",
    "            'favorite_count',\n",
    "            'favorited',\n",
    "            'retweeted',\n",
    "            'possibly_sensitive',\n",
    "            'withheld_in_countries',\n",
    "            'place',\n",
    "            'entities',\n",
    "            'extended_entities',\n",
    "            'quoted_status',\n",
    "            'retweeted_status'\n",
    "        ]\n",
    "    ]\n",
    "\n",
    "    twitterdf['place'] = twitterdf['place'].map(str)\n",
    "    twitterdf['id_str'] = twitterdf['id_str'].map(str)\n",
    "    twitterdf['extended_entities'] = twitterdf['extended_entities'].map(str)\n",
    "    twitterdf['quoted_status'] = twitterdf['quoted_status'].map(str)\n",
    "    twitterdf['retweeted_status'] = twitterdf['retweeted_status'].map(str)\n",
    "\n",
    "    # Create document list\n",
    "    docs = json.loads(twitterdf.to_json(orient='records'))\n",
    "\n",
    "    # Insert data into MongoDB\n",
    "    insert_data_to_mongodb(db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdbd2f5c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}