15_Connections_Between_Learners_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "from mlwpy import *\nimport logging, warnings\n\nimport tensorflow as tf\n# suppress excessive TensorFlow warnings\ntf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n\n# getting really hard to convince toolkits to be less verbose\nimport pymc3 as pm\npymc3_log = logging.getLogger('pymc3')\npymc3_log.setLevel(2**20)\n\n# for sampling reproducibility and less verbose:\nsampling_args = {'random_seed':42, 'progressbar':False}"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "xs = np.linspace(-5,5)\nys = xs**2\n\nfig, ax = plt.subplots(figsize=(4,3))\nax.plot(xs, ys)\n\n# better Python:\n# pt = co.namedtuple('Point', ['x', 'y'])(3,3**2)\npt_x, pt_y = 3, 3**2\nax.plot(pt_x, pt_y, 'ro')\n\nline_xs = pt_x + np.array([-2, 2])\n# line ys = mid_point + (x amount) * slope_of_line\n# one step right gets one \"slope of line\" increase in that line's up\nline_ys = 3**2 + (line_xs - pt_x) * (2 * pt_x)\nax.plot(line_xs, line_ys, 'r-')\nax.set_xlabel('weight')\nax.set_ylabel('cost');"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "weights = np.linspace(-5,5)\ncosts   = weights**2\n\nfig, ax = plt.subplots(figsize=(4,3))\nax.plot(weights, costs, 'b')\n\n# current best guess at the minimum\nweight_min = 3\n\n# we can follow the path, downhill from our starting point\n# and find out the weight value where our initial, blue graph is\n# (approximately) the smallest\nfor i in range(10):\n    # for a weight, we can figure out the cost\n    cost_at_min = weight_min**2\n    ax.plot(weight_min, cost_at_min, 'ro')\n\n    # also, we can figure out the slope (steepness)\n    # (via a magic incantation called a \"derivative\")\n    slope_at_min = 2*weight_min\n\n    # new best guess made by walking downhill\n    step_size = .25\n    weight_min = weight_min - step_size * slope_at_min\n\nax.set_xlabel('weight value')\nax.set_ylabel('cost')\nprint(\"Appoximate location of blue graph minimum:\", weight_min)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "from scipy.optimize import fmin as magical_minimum_finder\ndef f(x):\n    return x**2\n\nmagical_minimum_finder(f, [3], disp=False)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "linreg_ftrs_p1 = np.c_[np.arange(10), np.ones(10)] # +1 trick in data\n\ntrue_wgts  = m,b = w_1, w_0 = 3,2\nlinreg_tgt = rdot(true_wgts, linreg_ftrs_p1)\n\nlinreg_table = pd.DataFrame(linreg_ftrs_p1,\n                            columns=['ftr_1', 'ones'])\n# recent pymc3 was very grumpy with an \"exact\" target ... added noise\nlinreg_table['tgt'] = linreg_tgt + np.random.normal(size=linreg_tgt.size)\nlinreg_table[:3]"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "def linreg_model(weights, ftrs):\n    return rdot(weights, ftrs)\n\ndef linreg_loss(predicted, actual):\n    errors = predicted - actual\n    return np.dot(errors, errors) # sum-of-squares\n\ndef no_penalty(weights):\n    return 0.0"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "def make_cost(ftrs, tgt,\n              model_func, loss_func,\n              c_tradeoff, complexity_penalty):\n    ' build an optimization problem from data, model, loss, penalty '\n    def cost(weights):\n        return (loss_func(model_func(weights, ftrs), tgt) +\n                c_tradeoff * complexity_penalty(weights))\n    return cost"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# build linear regression optimization problem\nlinreg_cost = make_cost(linreg_ftrs_p1, linreg_tgt,\n                        linreg_model, linreg_loss,\n                        0.0, no_penalty)\nlearned_wgts = magical_minimum_finder(linreg_cost, [5,5], disp=False)\n\nprint(\"   true weights:\", true_wgts)\nprint(\"learned weights:\", learned_wgts)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "def L1_penalty(weights):\n    return np.abs(weights).sum()\n\ndef L2_penalty(weights):\n    return np.dot(weights, weights)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# linear regression with L1 regularization (lasso regression)\nlinreg_L1_pen_cost = make_cost(linreg_ftrs_p1, linreg_tgt,\n                               linreg_model, linreg_loss,\n                               1.0, L1_penalty)\nlearned_wgts = magical_minimum_finder(linreg_L1_pen_cost, [5,5], disp=False)\n\nprint(\"   true weights:\", true_wgts)\nprint(\"learned weights:\", learned_wgts)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# linear regression with L2 regularization (ridge regression)\nlinreg_L2_pen_cost = make_cost(linreg_ftrs_p1, linreg_tgt,\n                               linreg_model, linreg_loss,\n                               1.0, L2_penalty)\nlearned_wgts = magical_minimum_finder(linreg_L2_pen_cost, [5,5], disp=False)\n\nprint(\"   true weights:\", true_wgts)\nprint(\"learned weights:\", learned_wgts)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "logreg_ftr = np.random.uniform(5,15, size=(100,))\n\ntrue_wgts  = m,b = -2, 20\nline_of_logodds = m*logreg_ftr + b\nprob_at_x = np.exp(line_of_logodds) / (1 + np.exp(line_of_logodds))\n\nlogreg_tgt = np.random.binomial(1, prob_at_x, len(logreg_ftr))\n\nlogreg_ftrs_p1 = np.c_[logreg_ftr,\n                       np.ones_like(logreg_ftr)]\n\nlogreg_table = pd.DataFrame(logreg_ftrs_p1,\n                            columns=['ftr_1','ones'])\nlogreg_table['tgt'] = logreg_tgt\ndisplay(logreg_table.head())"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "fig, ax = plt.subplots(figsize=(6,4))\nax.plot(logreg_ftr, prob_at_x, 'r.')\nax.scatter(logreg_ftr, logreg_tgt, c=logreg_tgt);"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# for logistic regression\ndef logreg_model(weights, ftrs):\n    return rdot(weights, ftrs)\n\ndef logreg_loss_01(predicted, actual):\n    # sum(-actual log(predicted) - (1-actual) log(1-predicted))\n    # for 0/1 target works out to\n    return np.sum(- predicted * actual + np.log(1+np.exp(predicted)))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "logreg_cost = make_cost(logreg_ftrs_p1, logreg_tgt,\n                        logreg_model, logreg_loss_01,\n                        0.0, no_penalty)\nlearned_wgts = magical_minimum_finder(logreg_cost, [5,5], disp=False)\n\nprint(\"   true weights:\", true_wgts)\nprint(\"learned weights:\", learned_wgts)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# logistic regression with penalty\nlogreg_pen_cost = make_cost(logreg_ftrs_p1, logreg_tgt,\n                            logreg_model, logreg_loss_01,\n                            0.5, L1_penalty)\nlearned_wgts = magical_minimum_finder(logreg_pen_cost, [5,5], disp=False)\nprint(\"   true weights:\", true_wgts)\nprint(\"learned weights:\", learned_wgts)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "def binary_to_pm1(b):\n    ' map {0,1} or {False,True} to {-1, +1} '\n    return (b*2)-1\nbinary_to_pm1(0), binary_to_pm1(1)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# for logistic regression\ndef logreg_model(weights, ftrs):\n    return rdot(weights, ftrs)\n\ndef logreg_loss_pm1(predicted, actual):\n    # -actual log(predicted) - (1-actual) log(1-predicted)\n    # for +1/-1 targets, works out to:\n    return np.sum(np.log(1+np.exp(-predicted*actual)))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "logreg_cost = make_cost(logreg_ftrs_p1, binary_to_pm1(logreg_tgt),\n                        logreg_model, logreg_loss_pm1,\n                        0.0, no_penalty)\nlearned_wgts = magical_minimum_finder(logreg_cost, [5,5], disp=False)\n\nprint(\"   true weights:\", true_wgts)\nprint(\"learned weights:\", learned_wgts)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "def predict_with_logreg_weights_to_pm1(w_hat, x):\n    prob = 1 / (1 + np.exp(rdot(w_hat, x)))\n    thresh = prob < .5\n    return binary_to_pm1(thresh)\n\npreds = predict_with_logreg_weights_to_pm1(learned_wgts, logreg_ftrs_p1)\nprint(metrics.accuracy_score(preds, binary_to_pm1(logreg_tgt)))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# for SVC\ndef hinge_loss(predicted, actual):\n    hinge = np.maximum(1-predicted*actual, 0.0)\n    return np.sum(hinge)\n\ndef predict_with_svm_weights(w_hat, x):\n    return np.sign(rdot(w_hat,x)).astype(np.int)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "svm_ftrs = logreg_ftrs_p1\nsvm_tgt  = binary_to_pm1(logreg_tgt)  # svm \"demands\" +/- 1\n\n# svm model is \"just\" rdot, so we don't define it separately now\nsvc_cost = make_cost(svm_ftrs, svm_tgt, rdot,\n                     hinge_loss, 0.0, no_penalty)\nlearned_weights = magical_minimum_finder(svc_cost, [5,5], disp=False)\n\npreds = predict_with_svm_weights(learned_weights, svm_ftrs)\nprint('no penalty accuracy:',\n      metrics.accuracy_score(preds, svm_tgt))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# svc with penalty\nsvc_pen_cost = make_cost(svm_ftrs, svm_tgt, rdot,\n                         hinge_loss, 1.0, L1_penalty)\nlearned_weights = magical_minimum_finder(svc_pen_cost, [5,5], disp=False)\n\npreds = predict_with_svm_weights(learned_weights, svm_ftrs)\nprint('accuracy with penalty:',\n      metrics.accuracy_score(preds, svm_tgt))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "import keras.layers as kl\nimport keras.models as km\nimport keras.optimizers as ko"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "def Keras_LinearRegression(n_ftrs):\n    model = km.Sequential()\n    # Dense layer defaults includes a \"bias\" (a +1 trick)\n    model.add(kl.Dense(1,\n                       activation='linear',\n                       input_dim=n_ftrs))\n    model.compile(optimizer=ko.SGD(lr=0.01), loss='mse')\n    return model"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "# for various reasons, are going to let Keras do the +1\n# trick.  we will *not* send the `ones` feature\nlinreg_ftrs = linreg_ftrs_p1[:,0]\n\nlinreg_nn = Keras_LinearRegression(1)\nhistory = linreg_nn.fit(linreg_ftrs, linreg_tgt, epochs=1000, verbose=0)\npreds = linreg_nn.predict(linreg_ftrs)\n\nmse = metrics.mean_squared_error(preds, linreg_tgt)\n\nprint(\"Training MSE: {:5.4f}\".format(mse))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "history.history['loss'][:5]"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "def Keras_LogisticRegression(n_ftrs):\n    model = km.Sequential()\n    model.add(kl.Dense(1,\n                       activation='sigmoid',\n                       input_dim=n_ftrs))\n    model.compile(optimizer=ko.SGD(), loss='binary_crossentropy')\n    return model\n\n\nlogreg_nn = Keras_LogisticRegression(1)\nhistory = logreg_nn.fit(logreg_ftr, logreg_tgt, epochs=1000, verbose=0)\n\n# output is a probability\npreds = logreg_nn.predict(logreg_ftr) > .5\nprint('accuracy:', metrics.accuracy_score(preds, logreg_tgt))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "from keras.utils import to_categorical as k_to_categorical\ndef Keras_MultiLogisticRegression(n_ftrs, n_classes):\n    model = km.Sequential()\n    model.add(kl.Dense(n_classes,\n                       activation='softmax',\n                       input_dim=n_ftrs))\n    model.compile(optimizer=ko.SGD(), loss='categorical_crossentropy')\n    return model\n\nlogreg_nn2 = Keras_MultiLogisticRegression(1, 2)\nhistory = logreg_nn2.fit(logreg_ftr,\n                         k_to_categorical(logreg_tgt),\n                         epochs=1000, verbose=0)\n\n# predict gives \"probability table\" by class\n# we just need the bigger one\npreds = logreg_nn2.predict(logreg_ftr).argmax(axis=1)\nprint(metrics.accuracy_score(logreg_tgt, preds))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "logreg_ftr = np.random.uniform(5,15, size=(100,))\n\ntrue_wgts  = m,b = -2, 20\nline_of_logodds = m*logreg_ftr + b\nprob_at_x = np.exp(line_of_logodds) / (1 + np.exp(line_of_logodds))\n\nlogreg_tgt = np.random.binomial(1, prob_at_x, len(logreg_ftr))\n\nlogreg_ftrs_p1 = np.c_[logreg_ftr,\n                       np.ones_like(logreg_ftr)]\n\nlogreg_table = pd.DataFrame(logreg_ftrs_p1,\n                            columns=['ftr_1','ones'])\nlogreg_table['tgt'] = logreg_tgt\ndisplay(logreg_table.head())"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "with pm.Model() as model:\n    # boilder plate-ish setup of the distributions of our\n    # guesses for things we don't know\n    sd      = pm.HalfNormal('sd', sd=1)\n    intercept  = pm.Normal('Intercept', 0, sd=20)\n    ftr_1_wgt  = pm.Normal('ftr_1_wgt', 0, sd=20)\n\n    # outcomes made from inital guess and input data\n    # this is y = m x + b in an alternate form\n    preds = ftr_1_wgt * linreg_table['ftr_1'] + intercept\n\n    # relationship between guesses, input data, and actual outputs\n    # target = preds + noise(sd)  (noise == tolerance around the line)\n    target = pm.Normal('tgt',\n                       mu=preds, sd=sd,\n                       observed=linreg_table['tgt'])\n\n    linreg_trace = pm.sample(1000, **sampling_args)\n\n    # pymc3 complains when this is outside the with:\n    display(pm.summary(linreg_trace)[['mean']])"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "with pm.Model() as model:\n    pm.glm.GLM.from_formula('tgt ~ ftr_1', linreg_table,\n                           family=pm.glm.families.Normal())\n    linreg_trace = pm.sample(5000, **sampling_args)\n\n    # pymc3 now complains when these are outside with:\n    display(pm.summary(linreg_trace)[['mean']])\n    pm.traceplot(linreg_trace)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "with pm.Model() as model:\n    pm.glm.GLM.from_formula('tgt ~ ftr_1', logreg_table,\n                            family=pm.glm.families.Binomial())\n    logreg_trace = pm.sample(10000, **sampling_args)\n    display(pm.summary(logreg_trace)[['mean']])"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": "df_trace = pm.trace_to_dataframe(logreg_trace)\nsns.jointplot(x='ftr_1', y='Intercept', data=df_trace,\n              kind='kde', stat_func=None, height=4);"
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 2
}