ch12.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=379207b8b8f44b8c8e923bccddb74e1b\n",
    "%pushd book-materials\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.random.seed(12345)\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rc('figure', figsize=(10, 6))\n",
    "PREVIOUS_MAX_ROWS = pd.options.display.max_rows\n",
    "pd.options.display.max_columns = 20\n",
    "pd.options.display.max_rows = 20\n",
    "pd.options.display.max_colwidth = 80\n",
    "np.set_printoptions(precision=4, suppress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6b91a5404025497cb8caca631d473d94\n",
    "data = pd.DataFrame({\n",
    "    'x0': [1, 2, 3, 4, 5],\n",
    "    'x1': [0.01, -0.01, 0.25, -4.1, 0.],\n",
    "    'y': [-1.5, 0., 3.6, 1.3, -2.]})\n",
    "data\n",
    "data.columns\n",
    "data.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=bd1bb62fd48f4d1d8c0fa538775d33fa\n",
    "df2 = pd.DataFrame(data.to_numpy(), columns=['one', 'two', 'three'])\n",
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=eb72464446b24feab231e6d154a13d68\n",
    "df3 = data.copy()\n",
    "df3['strings'] = ['a', 'b', 'c', 'd', 'e']\n",
    "df3\n",
    "df3.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0284bd81d9734d62a0d2dcb8fc65e935\n",
    "model_cols = ['x0', 'x1']\n",
    "data.loc[:, model_cols].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=7b7d579912624331ac4bd2bf08470533\n",
    "data['category'] = pd.Categorical(['a', 'b', 'a', 'a', 'b'],\n",
    "                                  categories=['a', 'b'])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=361dec779b57482b94be56e2d062e793\n",
    "dummies = pd.get_dummies(data.category, prefix='category')\n",
    "data_with_dummies = data.drop('category', axis=1).join(dummies)\n",
    "data_with_dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0f377070f0d14b769232b62284a278f4\n",
    "data = pd.DataFrame({\n",
    "    'x0': [1, 2, 3, 4, 5],\n",
    "    'x1': [0.01, -0.01, 0.25, -4.1, 0.],\n",
    "    'y': [-1.5, 0., 3.6, 1.3, -2.]})\n",
    "data\n",
    "import patsy\n",
    "y, X = patsy.dmatrices('y ~ x0 + x1', data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5252acd3c6db4a3d9af0ebbb309c28c1\n",
    "y\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=03adb0f4a90247f6aba11e6763b806d0\n",
    "np.asarray(y)\n",
    "np.asarray(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a3a80cb56c54468b9e26c4cb052b5f6a\n",
    "patsy.dmatrices('y ~ x0 + x1 + 0', data)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=db616433095b4c758545ec95c5ba56d4\n",
    "coef, resid, _, _ = np.linalg.lstsq(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a02f213f2f10477b8b7907c83746bbc6\n",
    "coef\n",
    "coef = pd.Series(coef.squeeze(), index=X.design_info.column_names)\n",
    "coef"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=0da24c4c7d374bb0b2cac75072a4dd04\n",
    "y, X = patsy.dmatrices('y ~ x0 + np.log(np.abs(x1) + 1)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=eed36a4a619c4d9f9d65c0d8cb19d34e\n",
    "y, X = patsy.dmatrices('y ~ standardize(x0) + center(x1)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=88739eb5eaa94ab0bb19779111888158\n",
    "new_data = pd.DataFrame({\n",
    "    'x0': [6, 7, 8, 9],\n",
    "    'x1': [3.1, -0.5, 0, 2.3],\n",
    "    'y': [1, 2, 3, 4]})\n",
    "new_X = patsy.build_design_matrices([X.design_info], new_data)\n",
    "new_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=766fcf4390d24922a9f4cbbf86da6870\n",
    "y, X = patsy.dmatrices('y ~ I(x0 + x1)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5fa5d310160a4cd7a5862986a6be9c3d\n",
    "data = pd.DataFrame({\n",
    "    'key1': ['a', 'a', 'b', 'b', 'a', 'b', 'a', 'b'],\n",
    "    'key2': [0, 1, 0, 1, 0, 1, 0, 0],\n",
    "    'v1': [1, 2, 3, 4, 5, 6, 7, 8],\n",
    "    'v2': [-1, 0, 2.5, -0.5, 4.0, -1.2, 0.2, -1.7]\n",
    "})\n",
    "y, X = patsy.dmatrices('v2 ~ key1', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d795962e76ab48f7bbd08fa3adc3f481\n",
    "y, X = patsy.dmatrices('v2 ~ key1 + 0', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=2619ef96053c43eeb4f0fbaa5aed3836\n",
    "y, X = patsy.dmatrices('v2 ~ C(key2)', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=6efd1f19da93449d8ae591df85a826c9\n",
    "data['key2'] = data['key2'].map({0: 'zero', 1: 'one'})\n",
    "data\n",
    "y, X = patsy.dmatrices('v2 ~ key1 + key2', data)\n",
    "X\n",
    "y, X = patsy.dmatrices('v2 ~ key1 + key2 + key1:key2', data)\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=2c6c970ee0234156b7dea4ebb8b16790\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=15d197f427b344fa9f456841b129b9e4\n",
    "# To make the example reproducible\n",
    "rng = np.random.default_rng(seed=12345)\n",
    "\n",
    "def dnorm(mean, variance, size=1):\n",
    "    if isinstance(size, int):\n",
    "        size = size,\n",
    "    return mean + np.sqrt(variance) * rng.standard_normal(*size)\n",
    "\n",
    "N = 100\n",
    "X = np.c_[dnorm(0, 0.4, size=N),\n",
    "          dnorm(0, 0.6, size=N),\n",
    "          dnorm(0, 0.2, size=N)]\n",
    "eps = dnorm(0, 0.1, size=N)\n",
    "beta = [0.1, 0.3, 0.5]\n",
    "\n",
    "y = np.dot(X, beta) + eps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=81c61bc21b6c409aa2aaa99a3f575bfb\n",
    "X[:5]\n",
    "y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8ae5538884bb4d05bff507be6a9c3d53\n",
    "X_model = sm.add_constant(X)\n",
    "X_model[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=5ee479f40b0d49f1b8725a3ff80d5794\n",
    "model = sm.OLS(y, X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=dab4d3edbde744b0ae331d4fac14a62b\n",
    "results = model.fit()\n",
    "results.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=f6cacdd9108340b7b7888be9cccb8c66\n",
    "print(results.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e7dc7a51f95240d690847760a4bce30e\n",
    "data = pd.DataFrame(X, columns=['col0', 'col1', 'col2'])\n",
    "data['y'] = y\n",
    "data[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=44341d4bc2984b1f9eefdf0518c00b32\n",
    "results = smf.ols('y ~ col0 + col1 + col2', data=data).fit()\n",
    "results.params\n",
    "results.tvalues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=c1e9fd6361f747fcbdd7054fdea0b3e0\n",
    "results.predict(data[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython verbatim id=271bfcc9f59a4d2096cc41fdcf9f7fef\n",
    "init_x = 4\n",
    "\n",
    "values = [init_x, init_x]\n",
    "N = 1000\n",
    "\n",
    "b0 = 0.8\n",
    "b1 = -0.4\n",
    "noise = dnorm(0, 0.1, N)\n",
    "for i in range(N):\n",
    "    new_x = values[-1] * b0 + values[-2] * b1 + noise[i]\n",
    "    values.append(new_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=e576aacedffa4dbab3be7517065e0b78\n",
    "from statsmodels.tsa.ar_model import AutoReg\n",
    "MAXLAGS = 5\n",
    "model = AutoReg(values, MAXLAGS)\n",
    "results = model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=a43500bc16854e43977c0a4c5fc2a2de\n",
    "results.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=9e54440f129e4624889185d002432371\n",
    "train = pd.read_csv('datasets/titanic/train.csv')\n",
    "test = pd.read_csv('datasets/titanic/test.csv')\n",
    "train.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ea68d69bba434e0c9665c794accee8d8\n",
    "train.isna().sum()\n",
    "test.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=ca1197b7e36e4af8bb0c7087960dca2e\n",
    "impute_value = train['Age'].median()\n",
    "train['Age'] = train['Age'].fillna(impute_value)\n",
    "test['Age'] = test['Age'].fillna(impute_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=52d11da1850c463783b6e088f0f662d3\n",
    "train['IsFemale'] = (train['Sex'] == 'female').astype(int)\n",
    "test['IsFemale'] = (test['Sex'] == 'female').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=450811815c5f40d0af45346441c2e3c2\n",
    "predictors = ['Pclass', 'IsFemale', 'Age']\n",
    "\n",
    "X_train = train[predictors].to_numpy()\n",
    "X_test = test[predictors].to_numpy()\n",
    "y_train = train['Survived'].to_numpy()\n",
    "X_train[:5]\n",
    "y_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8a75f2aaf25b449b9347d5a71a1fa643\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "model = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=d15acba4d6674ea58746c2258c28bf44\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=27541b6b54c54a37b01dcb07921952b2\n",
    "y_predict = model.predict(X_test)\n",
    "y_predict[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=8a33d87df19f49ec9e62bcb26d88b3b7\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "model_cv = LogisticRegressionCV(Cs=10)\n",
    "model_cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython id=36245c79ec40498f8bcb84258c1020cd\n",
    "from sklearn.model_selection import cross_val_score\n",
    "model = LogisticRegression(C=10)\n",
    "scores = cross_val_score(model, X_train, y_train, cv=4)\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=ce2a8a40e5074d149cd991bc1062381c\n",
    "%popd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! ipython suppress id=de44cca014a54e88a10784f41350688a\n",
    "pd.options.display.max_rows = PREVIOUS_MAX_ROWS"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}