Importing Data in Python II.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing data from web"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** with links **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import package\n",
    "from urllib.request import urlretrieve \n",
    "\n",
    "# Import pandas\n",
    "import pandas as pd\n",
    "\n",
    "# Assign url of file: url\n",
    "url = 'https://s3.amazonaws.com/assets.datacamp.com/production/course_1606/datasets/winequality-red.csv'\n",
    "\n",
    "# Save file locally\n",
    "urlretrieve(url, \"winequality-red.csv\")\n",
    "\n",
    "# Read file into a DataFrame and print its head\n",
    "df = pd.read_csv('winequality-red.csv', sep=';')\n",
    "print(df.head())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Importing using pandas **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import package\n",
    "import pandas as pd\n",
    "\n",
    "# Assign url of file: url\n",
    "url = 'http://s3.amazonaws.com/assets.datacamp.com/course/importing_data_into_r/latitude.xls'\n",
    "\n",
    "# Read in all sheets of Excel file: xl\n",
    "xl = pd.read_excel(url, sheetname=None)\n",
    "\n",
    "# Print the sheetnames to the shell\n",
    "print(xl.keys())\n",
    "\n",
    "# Print the head of the first sheet (using its name, NOT its index)\n",
    "print(xl['1700'].head())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Importing with request package **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "from urllib.request import urlopen, Request\n",
    "\n",
    "# Specify the url\n",
    "url = \"http://www.datacamp.com/teach/documentation\"\n",
    "\n",
    "# This packages the request: request\n",
    "request = Request(url)\n",
    "\n",
    "# Sends the request and catches the response: response\n",
    "response = urlopen(request)\n",
    "\n",
    "# Print the datatype of response\n",
    "print(type(response))\n",
    "\n",
    "# Be polite and close the response!\n",
    "response.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Printing HTTP request results in Python using urllib**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "from urllib.request import urlopen, Request\n",
    "\n",
    "# Specify the url\n",
    "url = \"http://www.datacamp.com/teach/documentation\"\n",
    "\n",
    "# This packages the request\n",
    "request = Request(url)\n",
    "\n",
    "# Sends the request and catches the response: response\n",
    "response = urlopen(request)\n",
    "\n",
    "# Extract the response: html\n",
    "html = response.read()\n",
    "\n",
    "# Print the html\n",
    "print(html)\n",
    "\n",
    "# Be polite and close the response!\n",
    "response.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Performing HTTP requests in Python using requests**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import package\n",
    "import requests\n",
    "\n",
    "# Specify the url: url\n",
    "url = \"http://www.datacamp.com/teach/documentation\"\n",
    "\n",
    "# Packages the request, send the request and catch the response: r\n",
    "r = requests.get(url)\n",
    "\n",
    "# Extract the response: text\n",
    "text = r.text\n",
    "\n",
    "# Print the html\n",
    "print(text)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Parsing HTML with BeautifulSoup**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Specify url: url\n",
    "url = 'https://www.python.org/~guido/'\n",
    "\n",
    "# Package the request, send the request and catch the response: r\n",
    "r = requests.get(url)\n",
    "\n",
    "# Extracts the response as html: html_doc\n",
    "html_doc = r.text\n",
    "\n",
    "# Create a BeautifulSoup object from the HTML: soup\n",
    "soup = BeautifulSoup(html_doc)\n",
    "\n",
    "# Prettify the BeautifulSoup object: pretty_soup\n",
    "pretty_soup = soup.prettify()\n",
    "\n",
    "# Print the response\n",
    "print(pretty_soup)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Turning a webpage into data using BeautifulSoup: getting the text**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Specify url: url\n",
    "url = 'https://www.python.org/~guido/'\n",
    "\n",
    "# Package the request, send the request and catch the response: r\n",
    "r = requests.get(url)\n",
    "\n",
    "# Extract the response as html: html_doc\n",
    "html_doc = r.text\n",
    "\n",
    "# Create a BeautifulSoup object from the HTML: soup\n",
    "soup = BeautifulSoup(html_doc)\n",
    "\n",
    "# Get the title of Guido's webpage: guido_title\n",
    "guido_title = soup.title\n",
    "\n",
    "# Print the title of Guido's webpage to the shell\n",
    "print(guido_title)\n",
    "\n",
    "# Get Guido's text: guido_text\n",
    "guido_text = soup.get_text()\n",
    "\n",
    "# Print Guido's text to the shell\n",
    "print(guido_text)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Turning a webpage into data using BeautifulSoup: getting the hyperlinks**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this exercise, you'll figure out how to extract the URLs of the hyperlinks from the BDFL's webpage. In the process, you'll become close friends with the soup method find_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Specify url\n",
    "url = 'https://www.python.org/~guido/'\n",
    "\n",
    "# Package the request, send the request and catch the response: r\n",
    "r = requests.get(url)\n",
    "\n",
    "# Extracts the response as html: html_doc\n",
    "html_doc = r.text\n",
    "\n",
    "# create a BeautifulSoup object from the HTML: soup\n",
    "soup = BeautifulSoup(html_doc)\n",
    "\n",
    "# Print the title of Guido's webpage\n",
    "print(soup.title)\n",
    "\n",
    "# Find all 'a' tags (which define hyperlinks): a_tags\n",
    "a_tags = soup.find_all(\"a\")\n",
    "\n",
    "# Print the URLs to the shell\n",
    "for link in a_tags:\n",
    "    print(link.get(\"href\"))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interacting with APIs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "** JSON files **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load JSON: json_data\n",
    "with open(\"a_movie.json\") as json_file:\n",
    "    json_data = json.load(json_file)\n",
    "\n",
    "# Print each key-value pair in json_data\n",
    "for k in json_data.keys():\n",
    "    print(k + ': ',json_data[k])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** API requests **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import requests package\n",
    "import requests\n",
    "\n",
    "# Assign URL to variable: url\n",
    "url = 'http://www.omdbapi.com/?apikey=ff21610b&t=social+network'\n",
    "\n",
    "# Package the request, send the request and catch the response: r\n",
    "r = requests.get(url)\n",
    "\n",
    "# Print the text of the response\n",
    "print(r.text)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** JSON from web to Python **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import package\n",
    "import requests\n",
    "\n",
    "# Assign URL to variable: url\n",
    "url = 'http://www.omdbapi.com/?apikey=ff21610b&t=social+network'\n",
    "\n",
    "# Package the request, send the request and catch the response: r\n",
    "r = requests.get(url)\n",
    "\n",
    "# Decode the JSON data into a dictionary: json_data\n",
    "json_data = r.json()\n",
    "\n",
    "# Print each key-value pair in json_data\n",
    "for k in json_data.keys():\n",
    "    print(k + ': ', json_data[k])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Wikipidea API **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import package\n",
    "import requests\n",
    "\n",
    "# Assign URL to variable: url\n",
    "url = \"https://en.wikipedia.org/w/api.php?action=query&prop=extracts&format=json&exintro=&titles=pizza\"\n",
    "\n",
    "# Package the request, send the request and catch the response: r\n",
    "r = requests.get(url)\n",
    "\n",
    "# Decode the JSON data into a dictionary: json_data\n",
    "json_data = r.json()\n",
    "\n",
    "# Print the Wikipedia page extract\n",
    "pizza_extract = json_data['query']['pages']['24768']['extract']\n",
    "print(pizza_extract)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}