rosenbrock_plot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from befree.model import get_model\n",
    "from befree.datasets import get_dataset\n",
    "from befree.optimizers import get_optimizer\n",
    "from befree.config import get_config\n",
    "from befree.examples.train import train\n",
    "\n",
    "config = get_config('befree/config/mlp.yaml')\n",
    "\n",
    "from befree.examples.print_stats import print_stats\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_stats({config['optimizer']['name']: stats})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import rc\n",
    "rc('font',**{'family':'sans-serif','sans-serif':['Helvetica']})\n",
    "rc('text', usetex=True)\n",
    "DEFAULT_FONTSIZE = 18\n",
    "def beauty_plt(figsize=(10, 7), fontsize=DEFAULT_FONTSIZE): \n",
    "    plt.figure(figsize=figsize)\n",
    "    plt.xticks(fontsize=fontsize)\n",
    "    _ = plt.yticks(fontsize=fontsize)\n",
    "\n",
    "rc('xtick', labelsize=DEFAULT_FONTSIZE) \n",
    "rc('ytick', labelsize=DEFAULT_FONTSIZE) \n",
    "\n",
    "beauty_plt()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uv_hf = np.load('hessianFree-0.1-0.99-0.1-uv.npy', allow_pickle=True)\n",
    "loss_hf = np.load('hessianFree-0.1-0.99-0.1-loss.npy', allow_pickle=True)\n",
    "\n",
    "uv_sm = np.load('simplifiedHessian-0.0001-0.99-0.1-uv.npy', allow_pickle=True)\n",
    "loss_sm = np.load('simplifiedHessian-0.0001-0.99-0.1-loss.npy', allow_pickle=True)\n",
    "\n",
    "uv_newt = np.load('newton-0.0001-0.99-0.1-uv.npy', allow_pickle=True)\n",
    "loss_newt = np.load('newton-0.0001-0.99-0.1-loss.npy', allow_pickle=True)\n",
    "                  \n",
    "uv_curv = np.load('curveball-0.001-0.99-0.1-uv.npy', allow_pickle=True)\n",
    "loss_curv = np.load('curveball-0.001-0.99-0.1-loss.npy', allow_pickle=True)\n",
    "\n",
    "uv_adam = np.load('adam-0.001-0.99-0.1-uv.npy', allow_pickle=True)\n",
    "loss_adam = np.load('adam-0.001-0.99-0.1-loss.npy', allow_pickle=True)\n",
    "\n",
    "uv_sgd = np.load('sgd-0.001-0.99-0.1-uv.npy', allow_pickle=True)\n",
    "loss_sgd = np.load('sgd-0.001-0.99-0.1-loss.npy', allow_pickle=True)\n",
    "\n",
    "\n",
    "uv_sgd_m = np.load('sgd-momentum-0.001-0.9-0.1-uv.npy', allow_pickle=True)\n",
    "loss_sgd_m = np.load('sgd-momentum-0.001-0.9-0.1-loss.npy', allow_pickle=True)\n",
    "\n",
    "uv_lbfgs = np.load('lbfgs-0.2-0.99-0.1-uv.npy', allow_pickle=True)\n",
    "loss_lbfgs = np.load('lbfgs-0.2-0.99-0.1-loss.npy', allow_pickle=True)\n",
    "\n",
    "uv_sm[0] = torch.tensor([0., 0.])\n",
    "uv_hf[0] = torch.tensor([0., 0.])\n",
    "uv_newt[0] = torch.tensor([0., 0.])\n",
    "uv_curv[0] = torch.tensor([0., 0.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12, 5))\n",
    "beauty_plt(figsize=(12, 5))    \n",
    "plt.plot([i for i in range(1, len(loss_curv)+1)], loss_curv, label='Curveball', c='k')\n",
    "plt.plot([i for i in range(1, len(loss_sm)+1)], loss_sm, label='Simplified Hessian', c='g')\n",
    "plt.plot([i for i in range(1, len(loss_hf)+1)], loss_hf, label='Hessian Free', c='r')\n",
    "plt.plot([i for i in range(1, len(loss_adam)+1)], loss_adam, label='Adam', c='c')\n",
    "plt.plot([i for i in range(1, len(loss_sgd)+1)], loss_sgd, label='SGD', c='m')\n",
    "plt.plot([i for i in range(1, len(loss_sgd_m)+1)], loss_sgd_m, label='Momentum', c='#4070a0')\n",
    "plt.plot([i for i in range(1, len(loss_lbfgs)+1)], loss_lbfgs, label='LBFGS', c='b')\n",
    "# plt.plot([i+0.5 for i in range(1, len(loss_newt)+1)], loss_newt, label='newton')\n",
    "plt.yscale('log')\n",
    "plt.xscale('log')\n",
    "plt.ylim(10**(-5), 10**2+50)\n",
    "# plt.xlim(1, 10**5)\n",
    "# plt.tick_params(axis='both', which='major', labelsize=DEFAULT_FONTSIZE)\n",
    "# plt.tick_params(axis='both', which='minor', labelsize=D EFAULT_FONTSIZE)\n",
    "ax = plt.subplot(111)\n",
    "box = ax.get_position()\n",
    "ax.set_position([box.x0+0., box.y0, box.width*0.7, box.height])\n",
    "legend_x = 1\n",
    "legend_y = 1\n",
    "plt.legend(fontsize=DEFAULT_FONTSIZE, bbox_to_anchor=(legend_x,legend_y), framealpha=1)\n",
    "plt.title('Rosenbrock-$U$[0, 1]', fontsize=DEFAULT_FONTSIZE)\n",
    "plt.xlabel('Number of iterations', fontsize=DEFAULT_FONTSIZE)\n",
    "plt.ylabel('Loss', fontsize=DEFAULT_FONTSIZE)\n",
    "plt.savefig('for_nikita_loss_300.png', dpi=300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rosen_func(x, y, noise=0.1):\n",
    "    noise = np.random.randn(x.shape[0], x.shape[1]) * noise + 1\n",
    "    return (1 - x) ** 2 + 100 * noise * (x - y ** 2)**2\n",
    " \n",
    "x = np.linspace(-0.5, 1.5, 100)\n",
    "y = np.linspace(-0.5, 1.5, 100)\n",
    "X,Y = np.meshgrid(x, y) # grid of point\n",
    "Z = rosen_func(X, Y)\n",
    "\n",
    "\n",
    "plt.figure(figsize=(7, 7))\n",
    "cm = plt.cm.get_cmap('viridis')\n",
    "layers = np.concatenate([np.arange(0, 5, 1),\n",
    "                         np.arange(5, 20, 5),\n",
    "                         np.arange(20, 100, 20),\n",
    "                         np.arange(100, 1000, 50)])\n",
    "plt.contour(X.T, Y.T, Z, layers, cmap=cm, linewidths=0.5)\n",
    "plt.plot([0, 1], [0, 1], '*', ms=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.figure(figsize=(6, 6))\n",
    "plt.figure(figsize=(7, 7))\n",
    "cm = plt.cm.get_cmap('viridis')\n",
    "layers = np.concatenate([np.arange(0, 1, 0.3),\n",
    "                         np.arange(1, 5, 1),\n",
    "                         np.arange(5, 20, 5),\n",
    "                         np.arange(20, 100, 20),\n",
    "                         np.arange(100, 1000, 50)])\n",
    "plt.contour(X.T, Y.T, Z, layers, cmap=cm, linewidths=0.4, alpha=0.8)\n",
    "plt.plot([0, 1], [0, 1], '*', ms=20)\n",
    "\n",
    "\n",
    "plt.plot([x[0] for x in uv_newt], [x[1] for x in uv_newt], 'y.-', label='Newton')\n",
    "plt.plot([x[0] for x in uv_adam], [x[1] for x in uv_adam], 'c.-', label='Adam')\n",
    "plt.plot([x[0] for x in uv_sgd], [x[1] for x in uv_sgd], 'm.-', label='SGD')\n",
    "plt.plot([x[0] for x in uv_hf], [x[1] for x in uv_hf], 'r.-', label='Hessian Free')\n",
    "plt.plot([x[0] for x in uv_lbfgs], [x[1] for x in uv_lbfgs], 'b.-', label='LBFGS')\n",
    "plt.plot([x[0] for x in uv_lbfgs], [x[1] for x in uv_lbfgs], '.-', color='#4070a0', label='Momentum')\n",
    "plt.plot([x[0] for x in uv_curv], [x[1] for x in uv_curv], 'ko--', label='Curveball')\n",
    "plt.plot([x[0] for x in uv_sm], [x[1] for x in uv_sm], 'g.-', label='Simplified Hessian')\n",
    "\n",
    "plt.xlim([-0.5, 1.5])\n",
    "plt.ylim([-0.5, 1.5])\n",
    "plt.legend(fontsize=16)\n",
    "plt.title('Rosenbrock-$U$[0, 1]', fontsize=DEFAULT_FONTSIZE)\n",
    "plt.xlabel('u', fontsize=DEFAULT_FONTSIZE)\n",
    "plt.ylabel('v', fontsize=DEFAULT_FONTSIZE)\n",
    "# plt.show()\n",
    "ax = plt.subplot(111)\n",
    "box = ax.get_position()\n",
    "ax.set_position([box.x0+0.02, box.y0, box.width*0.99, box.height])\n",
    "plt.savefig('for_nikita_300.png', dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}