Data Manipulation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data that looks like:\n",
    "\n",
    "data = [\n",
    "    {'closing_price': 102.06,\n",
    "    'date': datetime.datetime(2014, 8, 29, 0, 0),\n",
    "    'symbol': 'AAPL'},\n",
    "    # ...\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# find max AAPL closing price with list comprehension\n",
    "max_aapl_price = max(row[\"closing price\"]\n",
    "                    for row in data\n",
    "                    if row[\"symbol\"] == \"AAPL\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "# group rows by symbol\n",
    "by_symbol = defaultdict(list) # will insert empty list as value for any key that is accessed but not in dict\n",
    "for row in data:\n",
    "    by_symbol[row[\"symbol\"]].append(row) # in the defaultdict, access the row's \"symbol\" value and\n",
    "    # if the symbol value is in the dict, add the row to the dict as {row[\"symbol\"] : [existing_row, new_row]}\n",
    "    # otherwise create a new key: list-value entry in the default dict like {row[\"symbol\"] : [new_row]}\n",
    "\n",
    "# use a dict comprehension to find the max for each symbol\n",
    "max_price_by_symbol = { symbol : max(row[\"closing_price\"]                 # creates new dict with symbol : max price\n",
    "                                    for row in grouped_rows)\n",
    "                      for symbol, grouped_rows in by_symbol.iteritems() } # for each symbol and its associated list of\n",
    "                                                                          # rows (daily stock prices)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AAPL\n"
     ]
    }
   ],
   "source": [
    "def picker(field_name):\n",
    "    \"\"\"returns a function that picks a field out of a dict\"\"\"\n",
    "    return lambda row: row[field_name] # so the output is a function that takes row as an arg but always accesses\n",
    "                                       # the given field_name\n",
    "\n",
    "def pluck(field_name, rows):\n",
    "    \"\"\"turn a list of dicts into the list of field_name values\"\"\"\n",
    "    return map(picker(field_name), rows) # takes a list of row dicts, returns a list of picker(field_name)\n",
    "                                         # applied to the elements in rows, i.e. a list of row[field_name]\n",
    "# e.g.\n",
    "import datetime    \n",
    "sample_row = {'closing_price': 102.06,\n",
    "    'date': datetime.datetime(2014, 8, 29, 0, 0),\n",
    "    'symbol': 'AAPL'}\n",
    "\n",
    "print picker(\"symbol\")(sample_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def group_by(grouper, rows, value_transform=None):\n",
    "    # key is output of grouper, value is list of rows\n",
    "    grouped = defaultdict(list)\n",
    "    for row in rows:\n",
    "        grouped[grouper(row)].append(row) #for each row, apply grouper to row and access as key; if present, append row\n",
    "        # so say grouper sorts rows into types. Then grouped is a dict with keys = types and values = list of rows\n",
    "        # of that type\n",
    "    \n",
    "    if value_transform is None:\n",
    "        return grouped\n",
    "    else:\n",
    "        return {key: value_transform(rows)\n",
    "               for key, rows in grouped.iteritems() } # so value_transform takes a list as an arg\n",
    "    \n",
    "# rewriting previous example:\n",
    "max_price_by_symbol = group_by(picker(\"symbol\"),\n",
    "                              data,\n",
    "                              lambda rows: max(pluck(\"closing_price\", rows)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p> Now we ask, what are the largest and smallest one-day percent changes in our data set? <br>\n",
    "Percent change: <br>\n",
    "Today = (1 + percent change) * Yesterday OR <br>\n",
    "Today / Yesterday - 1 = percent change </p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def percent_price_change(yesterday, today):\n",
    "    return today[\"closing_price\"] / yesterday[\"closing_price\"] - 1\n",
    "\n",
    "def day_over_day_changes(grouped_rows): # grouped_rows is a list of dicts, each with a given symbol\n",
    "    # sort the rows by date\n",
    "    ordered = sorted(grouped_rows, key = picker(\"date\")) # for each row dict in grouped_rows, sort on \"date\"-key value\n",
    "    \n",
    "    # zip with an offset to get pairs of consecutive days\n",
    "    return [{ \"symbol\": today[\"symbol\"],\n",
    "              \"date\" : today[\"date\"],\n",
    "              \"change\" : percent_price_change(yesterday, today) }\n",
    "           for yesterday, today in zip(ordered, ordered[1:])] # returns a dict for each pair of (yesterday, today)\n",
    "\n",
    "# key is symbol, value is list of \"change\" dicts\n",
    "changes_by_symbol = group_by(picker(\"symbol\"), data, day_over_day_changes)\n",
    "# this f'n first creates a dict with keys = symbols and values a list of row dicts. Then it applies\n",
    "# day_over_day_changes to each symbol's list of row dicts, creating a list of \n",
    "# dicts, one for each symbol, of [{percent change dict 1}, ...]}\n",
    "\n",
    "# collect all \"change\" dicts into one big list\n",
    "all_changes = [change\n",
    "              for changes in changes_by_symbol.values() # changes are [{percent change dict 1}, ...]\n",
    "              for change in changes] # change is a single {percent change dict}\n",
    "\n",
    "max(all_changes, key=picker(\"change\")) # for each dict in all_changes, apply picker(\"change\") (i.e. access the change\n",
    "# value) and return the dict with the max \"change\" value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}