05_EvaluatingAndComparingLearners_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup\n",
    "from mlwpy import *\n",
    "diabetes = datasets.load_diabetes()\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 20\n",
    "ftr = np.linspace(-10, 10, num=N)                # ftr values\n",
    "tgt = 2*ftr**2 - 3 + np.random.uniform(-2, 2, N) # tgt = func(ftr)\n",
    "\n",
    "(train_ftr, test_ftr,\n",
    " train_tgt, test_tgt) = skms.train_test_split(ftr, tgt, test_size=N//2)\n",
    "\n",
    "display(pd.DataFrame({\"ftr\":train_ftr, \n",
    "                      \"tgt\":train_tgt}).T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(train_ftr, train_tgt, 'bo')\n",
    "plt.plot(test_ftr,  np.zeros_like(test_ftr), 'r+');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note: sklearn *really* wants 2D inputs (a table)\n",
    "# so we use rehape here.\n",
    "sk_model = linear_model.LinearRegression()\n",
    "sk_model.fit(train_ftr.reshape(-1,1), train_tgt)\n",
    "sk_preds = sk_model.predict(test_ftr.reshape(-1,1))\n",
    "sk_preds[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit predict evaluate a 1-D polynomial (a line)\n",
    "model_one = np.poly1d(np.polyfit(train_ftr, train_tgt, 1))\n",
    "preds_one = model_one(test_ftr)\n",
    "print(preds_one[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the predictions come back the same\n",
    "print(\"all close?\", np.allclose(sk_preds, preds_one))\n",
    "\n",
    "# and we can still use sklearn to evaluate it\n",
    "mse = metrics.mean_squared_error\n",
    "print(\"RMSE:\", np.sqrt(mse(test_tgt, preds_one)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit predict evaluate a 2-D polynomial (a parabola)\n",
    "model_two = np.poly1d(np.polyfit(train_ftr, train_tgt, 2))\n",
    "preds_two = model_two(test_ftr)\n",
    "print(\"RMSE:\", np.sqrt(mse(test_tgt, preds_two)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_three = np.poly1d(np.polyfit(train_ftr, train_tgt, 9))\n",
    "preds_three = model_three(test_ftr)\n",
    "print(\"RMSE:\", np.sqrt(mse(test_tgt, preds_three)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1,2, figsize=(6,3), sharey=True)\n",
    "\n",
    "labels = ['line', 'parabola', 'nonic']\n",
    "models = [model_one, model_two, model_three]\n",
    "train = (train_ftr, train_tgt)\n",
    "test  = (test_ftr, test_tgt)\n",
    "\n",
    "for ax, (ftr, tgt) in zip(axes, [train, test]):\n",
    "    ax.plot(ftr, tgt, 'k+')\n",
    "    for m, lbl in zip(models, labels):\n",
    "        ftr = sorted(ftr)\n",
    "        ax.plot(ftr, m(ftr), '-', label=lbl)\n",
    "\n",
    "axes[1].set_ylim(-20, 200)\n",
    "axes[0].set_title(\"Train\")\n",
    "axes[1].set_title(\"Test\");\n",
    "axes[0].legend(loc='upper center');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []\n",
    "for complexity in [1,2,6,9]:\n",
    "    model = np.poly1d(np.polyfit(train_ftr, train_tgt, complexity))\n",
    "    train_error = np.sqrt(mse(train_tgt, model(train_ftr)))\n",
    "    test_error = np.sqrt(mse(test_tgt, model(test_ftr)))\n",
    "    results.append((complexity, train_error, test_error))\n",
    "   \n",
    "columns = [\"Complexity\", \"Train Error\", \"Test Error\"]\n",
    "results_df = pd.DataFrame.from_records(results, \n",
    "                                       columns=columns,\n",
    "                                       index=\"Complexity\")\n",
    "\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df.plot();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_loss(loss, model, training_data):\n",
    "    ' total training_loss on train_data with model under loss'\n",
    "    return sum(loss(model.predict(x.reshape(1,-1)), y) \n",
    "                                 for x,y in training_data)\n",
    "def squared_error(prediction, actual):\n",
    "    ' squared error on a single example '\n",
    "    return (prediction-actual)**2\n",
    "\n",
    "# could be used like:\n",
    "# my_training_loss = training_loss(squared_error, model, training_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn   = neighbors.KNeighborsRegressor(n_neighbors=3)\n",
    "fit   = knn.fit(diabetes.data, diabetes.target)\n",
    "\n",
    "training_data = zip(diabetes.data, diabetes.target)\n",
    "\n",
    "my_training_loss = training_loss(squared_error, \n",
    "                                 knn, \n",
    "                                 training_data)\n",
    "print(my_training_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mse = metrics.mean_squared_error(diabetes.target,\n",
    "                                 knn.predict(diabetes.data))\n",
    "print(mse*len(diabetes.data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def complexity(model):\n",
    "    return model.complexity\n",
    "\n",
    "def cost(model, training_data, loss, _lambda):\n",
    "    return training_loss(m,D) + _lambda * complexity(m) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data, model, fit & cv-score\n",
    "model = neighbors.KNeighborsRegressor(10)\n",
    "skms.cross_val_score(model, \n",
    "                     diabetes.data, \n",
    "                     diabetes.target,\n",
    "                     cv=5,\n",
    "                     scoring='neg_mean_squared_error')\n",
    "# notes:\n",
    "# defaults for cross_val_score are\n",
    "# cv=3 fold, no shuffle, stratified if classifier\n",
    "# model.score by default (regressors: r2, classifiers: accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iris = datasets.load_iris()\n",
    "model = neighbors.KNeighborsClassifier(10)\n",
    "skms.cross_val_score(model, iris.data, iris.target, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# not stratified\n",
    "pet = np.array(['cat', 'dog', 'cat', \n",
    "                'dog', 'dog', 'dog'])\n",
    "list_folds = list(skms.KFold(2).split(pet))\n",
    "training_idxs = np.array(list_folds)[:,0,:]\n",
    "\n",
    "print(pet[training_idxs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stratified\n",
    "# note: typically this is behind the scenes\n",
    "# making StratifiedKFold produce readable output \n",
    "# requries some trickery.  feel free to ignore.\n",
    "pet = np.array(['cat', 'dog', 'cat', 'dog', 'dog', 'dog'])\n",
    "idxs = np.array(list(skms.StratifiedKFold(2)\n",
    "                         .split(np.ones_like(pet), pet)))\n",
    "training_idxs = idxs[:,0,:]\n",
    "print(pet[training_idxs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# running non-stratified CV\n",
    "iris = datasets.load_iris()\n",
    "model = neighbors.KNeighborsClassifier(10)\n",
    "non_strat_kf = skms.KFold(5)\n",
    "skms.cross_val_score(model, \n",
    "                     iris.data, \n",
    "                     iris.target, \n",
    "                     cv=non_strat_kf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# as a reminder, these are some of the imports \n",
    "# that are hidden behind: from mlwpy import *\n",
    "# from sklearn import (datasets, neighbors, \n",
    "#                      model_selection as skms,\n",
    "#                      linear_model, metrics)\n",
    "# see Appendix A for details\n",
    "\n",
    "linreg   = linear_model.LinearRegression()\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "scores = []\n",
    "for r in range(10):\n",
    "    tts = skms.train_test_split(diabetes.data, \n",
    "                                diabetes.target, \n",
    "                                test_size=.25)\n",
    "    \n",
    "    (diabetes_train_ftrs, diabetes_test_ftrs, \n",
    "     diabetes_train_tgt,  diabetes_test_tgt) = tts\n",
    "    \n",
    "    \n",
    "    fit   = linreg.fit(diabetes_train_ftrs, diabetes_train_tgt)\n",
    "    preds = fit.predict(diabetes_test_ftrs)\n",
    "\n",
    "    score = metrics.mean_squared_error(diabetes_test_tgt, preds)\n",
    "    scores.append(score)\n",
    "\n",
    "scores = pd.Series(np.sqrt(sorted(scores)))\n",
    "df = pd.DataFrame({'RMSE':scores})\n",
    "df.index.name = 'Repeat'\n",
    "display(df.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = plt.figure(figsize=(4,3)).gca()\n",
    "sns.swarmplot(y='RMSE', data=df, ax=ax)\n",
    "ax.set_xlabel('Over Repeated\\nTrain-Test Splits');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(df.describe().T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tts_fit_score(model, data, msr, test_size=.25):\n",
    "    ' apply a train-test split to fit model on data and eval with msr '\n",
    "    tts = skms.train_test_split(data.data, \n",
    "                                data.target, \n",
    "                                test_size=test_size)\n",
    "    \n",
    "    (train_ftrs, test_ftrs, train_tgt,  test_tgt) = tts\n",
    "    \n",
    "    fit   = linreg.fit(train_ftrs, train_tgt)\n",
    "    preds = fit.predict(test_ftrs)\n",
    "\n",
    "    score = msr(test_tgt, preds)\n",
    "    return score\n",
    "\n",
    "linreg   = linear_model.LinearRegression()\n",
    "diabetes = datasets.load_diabetes()\n",
    "scores = [tts_fit_score(linreg, diabetes, \n",
    "                       metrics.mean_squared_error) for i in range(10)]\n",
    "print(np.mean(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg   = linear_model.LinearRegression()\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "# non-default cv= argument\n",
    "ss = skms.ShuffleSplit(test_size=.25) # default, 10 splits\n",
    "scores = skms.cross_val_score(linreg, \n",
    "                              diabetes.data, diabetes.target,\n",
    "                              cv=ss, \n",
    "                              scoring='neg_mean_squared_error')\n",
    "\n",
    "scores = pd.Series(np.sqrt(-scores))\n",
    "df = pd.DataFrame({'RMSE':scores})\n",
    "df.index.name = 'Repeat'\n",
    "\n",
    "display(df.describe().T)\n",
    "\n",
    "ax = sns.swarmplot(y='RMSE', data=df)\n",
    "ax.set_xlabel('Over Repeated\\nTrain-Test Splits');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = skms.ShuffleSplit(test_size=.25, random_state=42)\n",
    "\n",
    "# note:\n",
    "# look at the first split (next)\n",
    "# look at training set\n",
    "# look at first 10 examples\n",
    "train, test = 0,1\n",
    "next(ss.split(diabetes.data))[train][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ss = skms.ShuffleSplit(test_size=.25, random_state=42)\n",
    "next(ss.split(diabetes.data))[train][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = 0, 1\n",
    "kf = skms.KFold(5)\n",
    "next(kf.split(diabetes.data))[train][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = skms.KFold(5)\n",
    "next(kf.split(diabetes.data))[train][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pet = np.array(['cat', 'dog', 'cat', \n",
    "                'dog', 'dog', 'dog'])\n",
    "\n",
    "kf = skms.KFold(3, shuffle=True)\n",
    "\n",
    "train, test = 0, 1\n",
    "split_1_group_1 = next(kf.split(pet))[train]\n",
    "split_2_group_1 = next(kf.split(pet))[train]\n",
    "\n",
    "print(split_1_group_1, \n",
    "      split_2_group_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kf = skms.KFold(3, shuffle=True, random_state=42)\n",
    "\n",
    "split_1_group_1 = next(kf.split(pet))[train]\n",
    "split_2_group_1 = next(kf.split(pet))[train]\n",
    "print(split_1_group_1, \n",
    "      split_2_group_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linreg   = linear_model.LinearRegression()\n",
    "diabetes = datasets.load_diabetes()\n",
    "\n",
    "loo = skms.LeaveOneOut()\n",
    "scores = skms.cross_val_score(linreg, \n",
    "                              diabetes.data, diabetes.target,\n",
    "                              cv=loo, \n",
    "                              scoring='neg_mean_squared_error')\n",
    "\n",
    "scores = pd.Series(np.sqrt(-scores))\n",
    "df = pd.DataFrame({'RMSE':scores})\n",
    "df.index.name = 'Repeat'\n",
    "\n",
    "display(df.describe().T)\n",
    "\n",
    "ax = sns.swarmplot(y='RMSE', data=df)\n",
    "ax.set_xlabel('Over LOO\\nTrain-Test Splits');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "iris = datasets.load_iris()\n",
    "\n",
    "# 10 data set sizes:  10% - 100% \n",
    "# (that much data is piped to a 5-fold CV)\n",
    "train_sizes = np.linspace(.1,1.0,10)\n",
    "nn = neighbors.KNeighborsClassifier()\n",
    "\n",
    "(train_N, \n",
    " train_scores, \n",
    " test_scores) = skms.learning_curve(nn, \n",
    "                                    iris.data, iris.target, \n",
    "                                    cv=5, \n",
    "                                    train_sizes=train_sizes)\n",
    "\n",
    "# collapse across the 5 CV scores; one result for each data set size\n",
    "df = pd.DataFrame(test_scores, index=(train_sizes*100).astype(np.int))\n",
    "df['Mean 5-CV'] = df.mean(axis='columns')\n",
    "df.index.name = \"% Data Used\"\n",
    "\n",
    "display(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neat_sizes = (train_sizes*100).astype(np.int)\n",
    "labels = np_cartesian_product(neat_sizes, [0,1], np.arange(5))\n",
    "score = np.concatenate([train_scores.flatten(), \n",
    "                        test_scores.flatten()], axis=0)\n",
    "assert len(score) == len(labels)\n",
    "\n",
    "df = pd.DataFrame.from_records(labels)\n",
    "df.columns = ['pct', 'set', 'fold']\n",
    "df.set = df.set.replace({0:'Train', 1:'Test'})\n",
    "df['score'] = score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = sns.lineplot(x='pct', y='score', hue='set', data=df)\n",
    "\n",
    "ax.set_title(\"Learning Curve for 5-NN Classifier\")\n",
    "ax.set_xlabel(\"Percent of Data used for Training\")\n",
    "ax.set_ylabel(\"Accuracy\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tidying the numpy array is a bit of a pain\n",
    "# xarray is designed to do this \"natively\" but\n",
    "# i don't want to introduce that dependency\n",
    "# [seems like there could be a better broadcasting \n",
    "#  solution to this]\n",
    "def sk_graph_to_tidy(train_test_scores, # y values\n",
    "                     eval_points,       # x values\n",
    "                     eval_label,        # x column name\n",
    "                     num_folds):        # could be inferred\n",
    "    train_scores, test_scores = train_test_scores\n",
    "    # humph, didn't know np_cartesian was order sensitive\n",
    "    labels = np_cartesian_product(eval_points, \n",
    "                                  [0,1], # surrogates for train/test \n",
    "                                  np.arange(num_folds))\n",
    "    score = np.concatenate([train_scores.flatten(), \n",
    "                            test_scores.flatten()], axis=0)\n",
    "    \n",
    "    df = pd.DataFrame.from_records(labels)\n",
    "    df.columns = [eval_label, 'set', 'fold']\n",
    "    df.set = df.set.replace({0:'Train', 1:'Test'})\n",
    "    df['score'] = score\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "neat_sizes = (train_sizes*100).astype(np.int)\n",
    "tidy_df = sk_graph_to_tidy([train_scores, test_scores],\n",
    "                           neat_sizes, 'pct', 5)\n",
    "\n",
    "ax = sns.lineplot(x='pct', y='score', hue='set', data=tidy_df)\n",
    "\n",
    "ax.set_title(\"Learning Curve for 5-NN Classifier\")\n",
    "ax.set_xlabel(\"Percent of Data used for Training\")\n",
    "ax.set_ylabel(\"Accuracy\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# deprecated in recent seaborn; kept for comparison with above\n",
    "if False:\n",
    "    joined = np.array([train_scores, test_scores]).transpose()\n",
    "    ax = sns.tsplot(joined, \n",
    "                   time=train_sizes, \n",
    "                   condition = ['Train', 'Test'], \n",
    "                   interpolate=False)\n",
    "\n",
    "    ax.set_title(\"Learning Curve for 5-NN Classifier\")\n",
    "    ax.set_xlabel(\"Percent of Data used for Training\")\n",
    "    ax.set_ylabel(\"Accuracy\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_neigh = [1,3,5,10,15,20]\n",
    "KNC = neighbors.KNeighborsClassifier\n",
    "tt = skms.validation_curve(KNC(), \n",
    "                           iris.data, iris.target, \n",
    "                           param_name='n_neighbors',\n",
    "                           param_range=num_neigh,\n",
    "                           cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tidy_df = sk_graph_to_tidy(tt, num_neigh, 'k', 5)\n",
    "ax = sns.lineplot(x='k', y='score', hue='set', data=tidy_df)\n",
    "\n",
    "ax.set_title('5-fold CV Performance for k-NN')\n",
    "ax.set_xlabel(\"\\n\".join(['k for k-NN',\n",
    "                         'lower k, more complex',\n",
    "                         'higher k, less complex']))\n",
    "ax.set_ylim(.9, 1.01)\n",
    "ax.set_ylabel('Accuracy');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tsplot deprecated; kept for comparison with above\n",
    "if False:\n",
    "    # stack and transpose trick (as above)\n",
    "    ax = sns.tsplot(np.array(tt).transpose(), \n",
    "                    time=num_neigh, \n",
    "                    condition=['Train', 'Test'], \n",
    "                    interpolate=False)\n",
    "\n",
    "    ax.set_title('5-fold CV Performance for k-NN')\n",
    "    ax.set_xlabel(\"\\n\".join(['k for k-NN',\n",
    "                             'lower k, more complex',\n",
    "                             'higher k, less complex']))\n",
    "    ax.set_ylim(.9, 1.01)\n",
    "    ax.set_ylabel('Accuracy');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifiers = {'gnb' : naive_bayes.GaussianNB(),\n",
    "               '5-NN' : neighbors.KNeighborsClassifier(n_neighbors=5)}\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(6,4))\n",
    "for name, model in classifiers.items():\n",
    "    cv_scores = skms.cross_val_score(model, \n",
    "                                     iris.data, iris.target, \n",
    "                                     cv=10, \n",
    "                                     scoring='accuracy', \n",
    "                                     n_jobs=-1) # use all cores\n",
    "    my_lbl = \"{} {:.3f}\".format(name, cv_scores.mean())\n",
    "    ax.plot(cv_scores, '-o', label=my_lbl) # marker=next(markers)\n",
    "ax.set_ylim(0.0, 1.1)\n",
    "ax.set_xlabel('Fold')\n",
    "ax.set_ylabel('Accuracy')\n",
    "ax.legend(ncol=2);"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 2
}