07_EvaluatingRegressors_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup\n",
    "from mlwpy import *\n",
    "%matplotlib inline\n",
    "\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "tts = skms.train_test_split(diabetes.data,\n",
    "                            diabetes.target, \n",
    "                            test_size=.25,\n",
    "                            random_state=42)\n",
    "\n",
    "(diabetes_train_ftrs, diabetes_test_ftrs, \n",
    " diabetes_train_tgt,  diabetes_test_tgt) = tts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline = dummy.DummyRegressor(strategy='median')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "strategies = ['constant', 'quantile', 'mean', 'median', ]\n",
    "baseline_args = [{\"strategy\":s} for s in strategies]\n",
    "\n",
    "# additional args for constant and quantile\n",
    "baseline_args[0]['constant'] = 50.0\n",
    "baseline_args[1]['quantile'] =  0.75\n",
    "\n",
    "# similar to ch 5, but using a list comp\n",
    "# process a single argument package (a dict)\n",
    "def do_one(**args):\n",
    "    baseline = dummy.DummyRegressor(**args)\n",
    "    baseline.fit(diabetes_train_ftrs, diabetes_train_tgt)\n",
    "    base_preds = baseline.predict(diabetes_test_ftrs)\n",
    "    return metrics.mean_squared_error(base_preds, diabetes_test_tgt)\n",
    "\n",
    "# gather all results via a list comprehension\n",
    "mses = [do_one(**bla) for bla in baseline_args]\n",
    "\n",
    "display(pd.DataFrame({'mse':mses}, \n",
    "                     index=strategies))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rms_error(actual, predicted):\n",
    "    ' root-mean-squared-error function '\n",
    "    # lesser values are better (a<b ... a is better)\n",
    "    mse = metrics.mean_squared_error(actual, predicted)    \n",
    "    return np.sqrt(mse)\n",
    "\n",
    "def neg_rmse_score(actual, predicted):\n",
    "    ' rmse based score function '\n",
    "    #  greater values are better  (a<b ... b better)\n",
    "    return -rms_error(actual, predicted)\n",
    "\n",
    "def neg_rmse_scorer(mod, ftrs, tgt_actual):\n",
    "    ' rmse scorer suitable for scoring arg '\n",
    "    tgt_pred = mod.predict(ftrs)\n",
    "    return neg_rmse_score(tgt_actual, tgt_pred)\n",
    "\n",
    "\n",
    "knn = neighbors.KNeighborsRegressor(n_neighbors=3)\n",
    "skms.cross_val_score(knn, diabetes.data, diabetes.target, \n",
    "                     cv=skms.KFold(5, shuffle=True),\n",
    "                     scoring=neg_rmse_scorer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = linear_model.LinearRegression()\n",
    "\n",
    "# help(lr.score) #for full output\n",
    "print(lr.score.__doc__.splitlines()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "our_preds  = np.array([1,2,3])\n",
    "mean_preds = np.array([2,2,2])\n",
    "actual     = np.array([2,3,4])\n",
    "\n",
    "sse_ours = np.sum(( our_preds - actual)**2)\n",
    "sse_mean = np.sum((mean_preds - actual)**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_2 = 1 - (sse_ours / sse_mean)\n",
    "print(\"manual r2:{:5.2f}\".format(r_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseline = dummy.DummyRegressor(strategy='mean')\n",
    "\n",
    "baseline.fit(diabetes_train_ftrs, diabetes_train_tgt)\n",
    "base_preds = baseline.predict(diabetes_test_ftrs)\n",
    "\n",
    "# r2 is not symmetric b/c true values have priority \n",
    "# and used to compute target mean\n",
    "base_r2_sklearn = metrics.r2_score(diabetes_test_tgt, base_preds)\n",
    "print(base_r2_sklearn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sklearn-train-mean to predict test tgts\n",
    "base_errors    = base_preds - diabetes_test_tgt\n",
    "sse_base_preds = np.dot(base_errors, base_errors)\n",
    "\n",
    "# train-mean to predict test targets\n",
    "train_mean_errors = np.mean(diabetes_train_tgt) - diabetes_test_tgt\n",
    "sse_mean_train    = np.dot(train_mean_errors, train_mean_errors)\n",
    "\n",
    "# test-mean to predict test targets (Danger Will Robinson!)\n",
    "test_mean_errors = np.mean(diabetes_test_tgt) - diabetes_test_tgt\n",
    "sse_mean_test    = np.dot(test_mean_errors, test_mean_errors)\n",
    "\n",
    "print(\"sklearn train-mean model SSE(on test):\", sse_base_preds)\n",
    "print(\" manual train-mean model SSE(on test):\", sse_mean_train)\n",
    "print(\" manual test-mean  model SSE(on test):\", sse_mean_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "1 - (sse_base_preds / sse_mean_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(base_r2_sklearn)\n",
    "print(1 - (sse_base_preds / sse_mean_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "# WARNING!  Don't try this at home boys & girls!\n",
    "# we are fitting on the *test* set ... to mimic the behavior \n",
    "# of sklearn R^2.\n",
    "#\n",
    "testbase = dummy.DummyRegressor(strategy='mean')\n",
    "testbase.fit(diabetes_test_ftrs, diabetes_test_tgt)\n",
    "testbase_preds = testbase.predict(diabetes_test_ftrs)\n",
    "testbase_mse = metrics.mean_squared_error(testbase_preds, \n",
    "                                          diabetes_test_tgt)\n",
    "\n",
    "models = [neighbors.KNeighborsRegressor(n_neighbors=3),\n",
    "          linear_model.LinearRegression()]\n",
    "results = co.defaultdict(dict)\n",
    "for m in models:\n",
    "    preds = (m.fit(diabetes_train_ftrs, diabetes_train_tgt)\n",
    "              .predict(diabetes_test_ftrs))\n",
    "             \n",
    "    mse = metrics.mean_squared_error(preds, diabetes_test_tgt)\n",
    "    r2  = metrics.r2_score(diabetes_test_tgt, preds)\n",
    "    results[get_model_name(m)]['R^2'] = r2\n",
    "    results[get_model_name(m)]['MSE'] = mse\n",
    "\n",
    "print(testbase_mse)\n",
    "\n",
    "df = pd.DataFrame(results).T\n",
    "df['Norm_MSE'] = df['MSE'] / testbase_mse\n",
    "df['1-R^2'] = 1-df['R^2']\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ape_df = pd.DataFrame({'predicted' : [4, 2, 9],\n",
    "                       'actual'    : [3, 5, 7]})\n",
    "\n",
    "ape_df['error'] = ape_df['predicted'] - ape_df['actual']\n",
    "\n",
    "ape_df.index.name = 'example'\n",
    "display(ape_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def regression_errors(figsize, predicted, actual, errors='all'):\n",
    "    ''' figsize -> subplots; \n",
    "        predicted/actual data -> columns in a DataFrame\n",
    "        errors -> \"all\" or sequence of indices '''\n",
    "    fig, axes = plt.subplots(1, 2, figsize=figsize, \n",
    "                             sharex=True, sharey=True)\n",
    "    df = pd.DataFrame({'actual':actual, \n",
    "                       'predicted':predicted})\n",
    "\n",
    "    for ax, (x,y) in zip(axes, it.permutations(['actual', \n",
    "                                                'predicted'])):\n",
    "        # plot the data as '.'; perfect as y=x line\n",
    "        ax.plot(df[x], df[y], '.', label='data')\n",
    "        ax.plot(df['actual'], df['actual'], '-', \n",
    "                label='perfection')\n",
    "        ax.legend()\n",
    "\n",
    "        ax.set_xlabel('{} Value'.format(x.capitalize()))\n",
    "        ax.set_ylabel('{} Value'.format(y.capitalize()))\n",
    "        ax.set_aspect('equal')\n",
    "\n",
    "    axes[1].yaxis.tick_right()\n",
    "    axes[1].yaxis.set_label_position(\"right\")\n",
    "        \n",
    "    # show connecting bars from data to perfect\n",
    "    # for all or only those specified?\n",
    "    if errors == 'all':\n",
    "        errors = range(len(df))\n",
    "    if errors:\n",
    "        acts  = df.actual.iloc[errors]\n",
    "        preds = df.predicted.iloc[errors]\n",
    "        axes[0].vlines(acts, preds, acts, 'r')\n",
    "        axes[1].hlines(acts, preds, acts, 'r')\n",
    "        \n",
    "    \n",
    "regression_errors((6,3), ape_df.predicted, ape_df.actual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr  = linear_model.LinearRegression()\n",
    "preds = (lr.fit(diabetes_train_ftrs, diabetes_train_tgt)\n",
    "           .predict(diabetes_test_ftrs))\n",
    "\n",
    "regression_errors((8,4), preds, diabetes_test_tgt, errors=[-20]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ape_df = pd.DataFrame({'predicted' : [4, 2, 9],\n",
    "                       'actual'    : [3, 5, 7]})\n",
    "\n",
    "ape_df['error'] = ape_df['predicted'] - ape_df['actual']\n",
    "ape_df['resid'] = ape_df['actual'] - ape_df['predicted']\n",
    "\n",
    "ape_df.index.name = 'example'\n",
    "display(ape_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def regression_residuals(ax, predicted, actual, \n",
    "                         show_errors=None, right=False):\n",
    "    ''' figsize -> subplots; \n",
    "        predicted/actual data -> columns of a DataFrame\n",
    "        errors -> \"all\" or sequence of indices '''\n",
    "    df = pd.DataFrame({'actual':actual, \n",
    "                       'predicted':predicted})\n",
    "    df['error'] = df.actual - df.predicted\n",
    "    ax.plot(df.predicted, df.error, '.')\n",
    "    ax.plot(df.predicted, np.zeros_like(predicted), '-')\n",
    "    \n",
    "    if right:\n",
    "        ax.yaxis.tick_right()\n",
    "        ax.yaxis.set_label_position(\"right\")\n",
    "    \n",
    "    ax.set_xlabel('Predicted Value')\n",
    "    ax.set_ylabel('Residual')\n",
    "    \n",
    "    if show_errors == 'all':\n",
    "        show_errors = range(len(df))\n",
    "    if show_errors:\n",
    "        preds = df.predicted.iloc[show_errors]\n",
    "        errors = df.error.iloc[show_errors]\n",
    "        ax.vlines(preds, 0, errors, 'r')\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(1,2,figsize=(8,4))\n",
    "\n",
    "ax1.plot(ape_df.predicted, ape_df.actual, 'r.', # pred v actual\n",
    "         [0,10], [0,10], 'b-')                  # perfect line\n",
    "ax1.set_xlabel('Predicted')\n",
    "ax1.set_ylabel('Actual')\n",
    "regression_residuals(ax2, ape_df.predicted, ape_df.actual, \n",
    "                     'all', right=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr  = linear_model.LinearRegression()\n",
    "knn = neighbors.KNeighborsRegressor()\n",
    "\n",
    "models = [lr, knn]\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(10,5), \n",
    "                         sharex=True, sharey=True)\n",
    "fig.tight_layout()\n",
    "\n",
    "for model, ax, on_right in zip(models, axes, [False, True]):\n",
    "    preds = (model.fit(diabetes_train_ftrs, diabetes_train_tgt)\n",
    "                  .predict(diabetes_test_ftrs))\n",
    "    \n",
    "    regression_residuals(ax, preds, diabetes_test_tgt, [-20], on_right)\n",
    "\n",
    "axes[0].set_title('Linear Regression Residuals')\n",
    "axes[1].set_title('kNN-Regressor Rediduals');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(diabetes_test_tgt[-20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1-D standardization\n",
    "# place evenly spaced values in a dataframe\n",
    "xs = np.linspace(-5, 10, 20)\n",
    "df = pd.DataFrame(xs, columns=['x'])\n",
    "\n",
    "# center ( - mean) and scale (/ std)\n",
    "df['std-ized'] = (df.x - df.x.mean()) / df.x.std()\n",
    "\n",
    "# show original and new data; compute statistics\n",
    "fig, ax = plt.subplots(1,1,figsize=(3,3))\n",
    "sns.stripplot(data=df)\n",
    "display(df.describe().loc[['mean', 'std']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2 1-D standardizations\n",
    "xs = np.linspace(-5, 10, 20)\n",
    "ys = 3*xs + 2 + np.random.uniform(20, 40, 20)\n",
    "\n",
    "df = pd.DataFrame({'x':xs, 'y':ys})\n",
    "df_std_ized = (df - df.mean()) / df.std()\n",
    "\n",
    "display(df_std_ized.describe().loc[['mean', 'std']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(2,2, figsize=(5,5))\n",
    "\n",
    "ax[0,0].plot(df.x, df.y, '.')\n",
    "ax[0,1].plot(df_std_ized.x, df_std_ized.y, '.')\n",
    "ax[0,0].set_ylabel('\"Natural\" Scale')\n",
    "\n",
    "ax[1,0].plot(df.x, df.y, '.')\n",
    "ax[1,1].plot(df_std_ized.x, df_std_ized.y, '.')\n",
    "\n",
    "ax[1,0].axis([-10, 50, -10, 50])\n",
    "ax[1,1].axis([-10, 50, -10, 50])\n",
    "\n",
    "ax[1,0].set_ylabel('Fixed/Shared Scale')\n",
    "ax[1,0].set_xlabel('Original Data')\n",
    "ax[1,1].set_xlabel('Standardized Data');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_xs, test_xs = skms.train_test_split(xs.reshape(-1,1), test_size=.5)\n",
    "\n",
    "scaler = skpre.StandardScaler()\n",
    "scaler.fit(train_xs).transform(test_xs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(train_xs, test_xs,\n",
    " train_ys, test_ys)= skms.train_test_split(xs.reshape(-1,1), \n",
    "                                           ys.reshape(-1,1),\n",
    "                                           test_size=.5)\n",
    "\n",
    "scaler = skpre.StandardScaler()\n",
    "lr  = linear_model.LinearRegression()\n",
    "\n",
    "std_lr_pipe  = pipeline.make_pipeline(scaler, lr)\n",
    "\n",
    "std_lr_pipe.fit(train_xs, train_ys).predict(test_xs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "student_df = pd.read_csv('data/portugese_student_numeric.csv')\n",
    "display(student_df[['absences']].describe().T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "student_ftrs = student_df[student_df.columns[:-1]]\n",
    "student_tgt  = student_df['G3']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = skpre.StandardScaler()\n",
    "\n",
    "lr  = linear_model.LinearRegression()\n",
    "knn_3 = neighbors.KNeighborsRegressor(n_neighbors=3)\n",
    "knn_10 =  neighbors.KNeighborsRegressor(n_neighbors=10)\n",
    "\n",
    "std_lr_pipe  = pipeline.make_pipeline(scaler, lr)\n",
    "std_knn3_pipe  = pipeline.make_pipeline(scaler, knn_3)\n",
    "std_knn10_pipe  = pipeline.make_pipeline(scaler, knn_10)\n",
    "\n",
    "# mean with/without Standardization should give same results\n",
    "regressors = {'baseline'  : dummy.DummyRegressor(strategy='mean'),\n",
    "              'std_knn3'  : std_knn3_pipe,\n",
    "              'std_knn10' : std_knn10_pipe,\n",
    "              'std_lr'    : std_lr_pipe}\n",
    "\n",
    "msrs = {'MAE'  : metrics.make_scorer(metrics.mean_absolute_error),\n",
    "        'RMSE' : metrics.make_scorer(rms_error)}\n",
    "\n",
    "fig, axes = plt.subplots(2, 1, figsize=(6,4))\n",
    "fig.tight_layout()\n",
    "for mod_name, model in regressors.items():\n",
    "    cv_results = skms.cross_validate(model, \n",
    "                                     student_ftrs, student_tgt, \n",
    "                                     scoring = msrs, cv=10)\n",
    "\n",
    "    for ax, msr in zip(axes, msrs):\n",
    "        msr_results = cv_results[\"test_\" + msr]\n",
    "        my_lbl = \"{:12s} {:.3f} {:.2f}\".format(mod_name, \n",
    "                                               msr_results.mean(), \n",
    "                                               msr_results.std())\n",
    "        ax.plot(msr_results, 'o--', label=my_lbl)\n",
    "        ax.set_title(msr)\n",
    "        # ax.legend() # uncomment for summary stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(1,1,figsize=(6,3))\n",
    "baseline_results = skms.cross_val_score(regressors['baseline'], \n",
    "                                        student_ftrs, student_tgt, \n",
    "                                        scoring = msrs['RMSE'], cv=10)\n",
    "\n",
    "for mod_name, model in regressors.items():\n",
    "    if mod_name.startswith(\"std_\"):\n",
    "        cv_results = skms.cross_val_score(model, \n",
    "                                          student_ftrs, student_tgt, \n",
    "                                          scoring = msrs['RMSE'], cv=10)\n",
    "\n",
    "        my_lbl = \"{:12s} {:.3f} {:.2f}\".format(mod_name, \n",
    "                                               cv_results.mean(), \n",
    "                                               cv_results.std())\n",
    "\n",
    "        ax.plot(cv_results / baseline_results, 'o--', label=my_lbl)\n",
    "ax.set_title(\"RMSE(model) / RMSE(baseline)\\n$<1$ is better than baseline\")\n",
    "ax.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(6,3))\n",
    "for mod_name, model in regressors.items():\n",
    "        cv_results = skms.cross_val_score(model, \n",
    "                                          student_ftrs, student_tgt, \n",
    "                                          cv=10)\n",
    "        my_lbl = \"{:12s} {:.3f} {:.2f}\".format(mod_name, \n",
    "                                               cv_results.mean(), \n",
    "                                               cv_results.std())\n",
    "\n",
    "        ax.plot(cv_results, 'o--', label=my_lbl)\n",
    "ax.set_title(\"$R^2$\");\n",
    "# ax.legend(); #uncomment for summary stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "msrs = {'MAD'  : metrics.mean_absolute_error,\n",
    "        'RMSE' : rms_error} # not scorer, no model\n",
    "\n",
    "results = {}\n",
    "for mod_name, model in regressors.items():\n",
    "    cv_preds = skms.cross_val_predict(model, \n",
    "                                      student_ftrs, student_tgt,\n",
    "                                      cv=10)\n",
    "    for ax, msr in zip(axes, msrs):\n",
    "        msr_results = msrs[msr](student_tgt, cv_preds)\n",
    "        results.setdefault(msr, []).append(msr_results)\n",
    "df = pd.DataFrame(results, index=regressors.keys())\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 4, figsize=(10,5), \n",
    "                         sharex=True, sharey=True)\n",
    "fig.tight_layout()\n",
    "\n",
    "for model_name, ax in zip(regressors, axes):\n",
    "    model = regressors[model_name]\n",
    "    preds = skms.cross_val_predict(model, \n",
    "                                   student_ftrs, student_tgt,\n",
    "                                   cv=10)\n",
    "\n",
    "    regression_residuals(ax, preds, student_tgt)\n",
    "    ax.set_title(model_name + \" residuals\")\n",
    "pd.DataFrame(student_tgt).describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "student_url = ('https://archive.ics.uci.edu/' + \n",
    "               'ml/machine-learning-databases/00320/student.zip')\n",
    "def grab_student_numeric():\n",
    "    # download zip file and unzip\n",
    "    # unzipping unknown files can be a security hazard\n",
    "    import urllib.request, zipfile\n",
    "    urllib.request.urlretrieve(student_url,\n",
    "                               'port_student.zip')\n",
    "    zipfile.ZipFile('port_student.zip').extract('student-mat.csv')\n",
    "\n",
    "    # preprocessing\n",
    "    df = pd.read_csv('student-mat.csv', sep=';')\n",
    "    \n",
    "    # g1 & g2 are highly correlated with g3;\n",
    "    # dropping them makes the problem sig. harder\n",
    "    # we also remove all non-numeric columns\n",
    "    df = df.drop(columns=['G1', 'G2']).select_dtypes(include=['number'])\n",
    "\n",
    "    # save as\n",
    "    df.to_csv('portugese_student_numeric.csv', index=False)\n",
    "\n",
    "# grab_student_numeric()"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 2
}