no-quant.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda_convnet (faster)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 1: Tesla K40c (CNMeM is enabled with initial size: 49.0% of memory, cuDNN 5005)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "# os.environ['THEANO_FLAGS']='contexts=dev0->cuda0;dev1->cuda1'\n",
    "os.environ['THEANO_FLAGS']='device=gpu1'\n",
    "import time\n",
    "import matplotlib\n",
    "import lasagne\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import pylab as P\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import cPickle as pickle\n",
    "from helpers.DiscreteLayer import DiscreteLayer\n",
    "try:\n",
    "    from lasagne.layers.dnn import Conv2DDNNLayer as conv\n",
    "    from lasagne.layers.dnn import MaxPool2DDNNLayer as pool\n",
    "    print('Using cuda_convnet (faster)')\n",
    "except ImportError:\n",
    "    from lasagne.layers import Conv2DLayer as conv\n",
    "    from lasagne.layers import MaxPool2DLayer as pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Constants\n",
    "NUM_EPOCHS = 1500\n",
    "BATCH_SIZE = 256\n",
    "LEARNING_RATE = 0.001\n",
    "DIM = 60\n",
    "NUM_CLASSES = 10\n",
    "mnist_cluttered = \"data/mnist_cluttered_60x60_6distortions.npz\"\n",
    "\n",
    "# Disc. Layer Settings\n",
    "DISC = False\n",
    "ADD_NOISE = True\n",
    "QUANT = np.array([0.125, 0.125, 0.125, 0.125, 0.125, 0.125], dtype='float32')\n",
    "VARIANCE_DEVIDER = 16.0\n",
    "\n",
    "# Test Specs\n",
    "TEST_NAME = 'no-quant'\n",
    "\n",
    "# Additional Settings\n",
    "lasagne.random.set_rng(np.random.RandomState(12345))  # Set random state so we can investigate results\n",
    "np.random.seed(1234)\n",
    "#theano.config.exception_verbosity = 'high'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train samples: (60000, 1, 60, 60)\n",
      "Test samples: (10000, 1, 60, 60)\n"
     ]
    }
   ],
   "source": [
    "def load_data():\n",
    "    data = np.load(mnist_cluttered)\n",
    "    X_train, y_train = data['x_train'], np.argmax(data['y_train'], axis=-1)\n",
    "    X_valid, y_valid = data['x_valid'], np.argmax(data['y_valid'], axis=-1)\n",
    "    X_test, y_test = data['x_test'], np.argmax(data['y_test'], axis=-1)\n",
    "    \n",
    "    # reshape for convolutions\n",
    "    X_train = X_train.reshape((X_train.shape[0], 1, DIM, DIM))\n",
    "    X_valid = X_valid.reshape((X_valid.shape[0], 1, DIM, DIM))\n",
    "    X_test = X_test.reshape((X_test.shape[0], 1, DIM, DIM))\n",
    "    \n",
    "    # stack train and valid, we wont be using valid\n",
    "    X_train = np.concatenate([X_train, X_valid], axis=0)\n",
    "    y_train = np.concatenate([y_train, y_valid], axis=0)\n",
    "        \n",
    "    print \"Train samples:\", X_train.shape\n",
    "    print \"Test samples:\", X_test.shape\n",
    "\n",
    "    return dict(\n",
    "        X_train=lasagne.utils.floatX(X_train),\n",
    "        y_train=y_train.astype('int32'),\n",
    "        X_test=lasagne.utils.floatX(X_test),\n",
    "        y_test=y_test.astype('int32'),\n",
    "        num_examples_train=X_train.shape[0],\n",
    "        num_examples_valid=X_valid.shape[0],\n",
    "        num_examples_test=X_test.shape[0],\n",
    "        input_height=X_train.shape[2],\n",
    "        input_width=X_train.shape[3],\n",
    "        output_dim=10,)\n",
    "data = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Disc. Layer\n",
      "Transformer network output shape:  (None, 1, 20, 20)\n"
     ]
    }
   ],
   "source": [
    "def build_model(input_width, input_height, output_dim,\n",
    "                batch_size=BATCH_SIZE, withdisc=True):\n",
    "    ini = lasagne.init.HeUniform()\n",
    "    l_in = lasagne.layers.InputLayer(shape=(None, 1, input_width, input_height),)\n",
    "    #l_in = lasagne.layers.DropoutLayer(l_in, p=0.1)\n",
    "\n",
    "    # Localization network\n",
    "    b = np.zeros((2, 3), dtype=theano.config.floatX)\n",
    "    b[0, 0] = 1\n",
    "    b[1, 1] = 1\n",
    "    b = b.flatten()\n",
    "    loc_l1 = pool(l_in, pool_size=(2, 2))\n",
    "    loc_l2 = conv(\n",
    "        loc_l1, num_filters=20, filter_size=(5, 5), W=ini)\n",
    "    loc_l3 = pool(loc_l2, pool_size=(2, 2))\n",
    "    loc_l4 = conv(loc_l3, num_filters=20, filter_size=(5, 5), W=ini)\n",
    "    loc_l5 = lasagne.layers.DenseLayer(\n",
    "        loc_l4, num_units=50, W=lasagne.init.HeUniform('relu'))\n",
    "    loc_out = lasagne.layers.DenseLayer(\n",
    "        loc_l5, num_units=6, b=b, W=lasagne.init.Constant(0.0), \n",
    "        nonlinearity=lasagne.nonlinearities.identity, name='param_regressor')\n",
    "    \n",
    "    if withdisc:\n",
    "        sharedBins = theano.shared(None, name='sharedBins')\n",
    "        l_dis = DiscreteLayer(loc_out, sharedBins=sharedBins, name='disclayer')\n",
    "        print(\"Using Discret. Layer\")\n",
    "    else:\n",
    "        l_dis = loc_out\n",
    "        print(\"No Disc. Layer\")\n",
    "    \n",
    "    # Transformer network\n",
    "    l_trans1 = lasagne.layers.TransformerLayer(l_in, l_dis, downsample_factor=3.0)\n",
    "    print \"Transformer network output shape: \", l_trans1.output_shape\n",
    "    \n",
    "    # Classification network\n",
    "    class_l1 = conv(\n",
    "        l_trans1,\n",
    "        num_filters=32,\n",
    "        filter_size=(3, 3),\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "    class_l2 = pool(class_l1, pool_size=(2, 2))\n",
    "    class_l3 = conv(\n",
    "        class_l2,\n",
    "        num_filters=32,\n",
    "        filter_size=(3, 3),\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "    class_l4 = pool(class_l3, pool_size=(2, 2))\n",
    "    class_l5 = lasagne.layers.DenseLayer(\n",
    "        class_l4,\n",
    "        num_units=256,\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "\n",
    "    l_out = lasagne.layers.DenseLayer(\n",
    "        class_l5,\n",
    "        num_units=output_dim,\n",
    "        nonlinearity=lasagne.nonlinearities.softmax,\n",
    "        W=ini,\n",
    "    )\n",
    "    if withdisc:\n",
    "        return l_out, l_trans1, sharedBins\n",
    "    else:\n",
    "        return l_out, l_trans1\n",
    "\n",
    "if DISC:\n",
    "    model, l_transform, sharedBins = build_model(DIM, DIM, NUM_CLASSES, withdisc=DISC)\n",
    "else:\n",
    "    model, l_transform = build_model(DIM, DIM, NUM_CLASSES, withdisc=DISC)\n",
    "\n",
    "model_params = lasagne.layers.get_all_params(model, trainable=True)\n",
    "# params = lasagne.layers.get_all_params(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Input: dist, dist.shape = (-1, num_units)\n",
    "Find quantization bins of a given dist history\n",
    "Returns a list of (x, num_units), where x's length is a random variable\n",
    "\"\"\"\n",
    "def find_quantization_bins(dist, sharedBins):\n",
    "    # Quantizer function\n",
    "    def Q(x, y):\n",
    "        return y * np.floor((x/y) + .5)\n",
    "    \n",
    "    shape = dist.shape\n",
    "    init_Q = QUANT\n",
    "    final_Q = []\n",
    "    \n",
    "    # Theta iterator\n",
    "    for i in range(shape[1]):\n",
    "        theta_i = dist[:, i]\n",
    "        \n",
    "        # Whats is the error threshold for this distribution\n",
    "        Q_eps = np.var(theta_i) / VARIANCE_DEVIDER\n",
    "        \n",
    "        # Batch Iterator\n",
    "        final_Q_i = []\n",
    "        for j in range(shape[0]):\n",
    "            theta = theta_i[j]\n",
    "            \n",
    "            # Quantized theta = Quantization bins\n",
    "            q = init_Q[i]\n",
    "            x_i = theta\n",
    "            x_o = Q(x_i, q)\n",
    "            \n",
    "            # Optimize x_o\n",
    "\n",
    "            while(np.abs(x_o - x_i) > Q_eps):\n",
    "                q = q / 2\n",
    "                x_o = Q(x_i, q)\n",
    "            \n",
    "            # End of optimisation\n",
    "            final_Q_i.append(x_o)\n",
    "        \n",
    "        # Append to outer list\n",
    "        uniques = np.unique(np.array(final_Q_i))\n",
    "        final_Q.append(uniques.astype(theano.config.floatX))\n",
    "        \n",
    "    # Report\n",
    "    print \"New Bin Sizes: [\" + \", \".join([str(final_Q[x].shape[0]) for x in range(shape[1])] ) + \"]\"\n",
    "    sharedBins.set_value(final_Q)\n",
    "    return final_Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def build_functions():\n",
    "    X = T.tensor4(dtype=theano.config.floatX)\n",
    "    y = T.ivector()\n",
    "\n",
    "    ## Layer History\n",
    "    if DISC:\n",
    "        l_disc = next(l for l in lasagne.layers.get_all_layers(model) if l.name is 'disclayer')\n",
    "        l_paramreg = next(l for l in lasagne.layers.get_all_layers(model) if l.name is 'param_regressor')\n",
    "        l_disc_output, l_paramreg_output = lasagne.layers.get_output([l_disc, l_paramreg], X, deterministic=False)\n",
    "    ## Layer History\n",
    "\n",
    "    # training output\n",
    "    output_train = lasagne.layers.get_output(model, X, deterministic=False)\n",
    "\n",
    "    # evaluation output. Also includes output of transform for plotting\n",
    "    output_eval, transform_eval = lasagne.layers.get_output([model, l_transform], X, deterministic=True)\n",
    "\n",
    "    sh_lr = theano.shared(lasagne.utils.floatX(LEARNING_RATE))\n",
    "    cost = T.mean(T.nnet.categorical_crossentropy(output_train, y))\n",
    "    updates = lasagne.updates.adam(cost, model_params, learning_rate=sh_lr)\n",
    "\n",
    "    if DISC:\n",
    "        train = theano.function([X, y], [cost, output_train, l_disc_output, l_paramreg_output], updates=updates)\n",
    "    else:\n",
    "        train = theano.function([X, y], [cost, output_train], updates=updates)\n",
    "    eval = theano.function([X], [output_eval, transform_eval])\n",
    "    \n",
    "    return train, eval, sh_lr\n",
    "\n",
    "train, eval, sh_lr = build_functions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_epoch(X, y):\n",
    "    # History Keeping\n",
    "    param_output = []\n",
    "    disc_output = []\n",
    "    # History\n",
    "    num_samples = X.shape[0]\n",
    "    num_batches = int(np.ceil(num_samples / float(BATCH_SIZE)))\n",
    "    costs = []\n",
    "    correct = 0\n",
    "    for i in range(num_batches):\n",
    "        idx = range(i*BATCH_SIZE, np.minimum((i+1)*BATCH_SIZE, num_samples))\n",
    "        X_batch = X[idx]\n",
    "        y_batch = y[idx]\n",
    "        if DISC:\n",
    "            cost, output_train, l_disc_output, l_paramreg_output = train(X_batch, y_batch)\n",
    "            param_output = np.append(param_output, l_paramreg_output)\n",
    "            disc_output = np.append(disc_output, l_disc_output)\n",
    "        else:\n",
    "            cost, output_train = train(X_batch, y_batch)\n",
    "        costs += [cost]\n",
    "        preds = np.argmax(output_train, axis=-1)\n",
    "        correct += np.sum(y_batch == preds)\n",
    "    \n",
    "    return np.mean(costs), correct / float(num_samples), param_output, disc_output\n",
    "\n",
    "\n",
    "def eval_epoch(X, y):\n",
    "    output_eval, transform_eval = eval(X)\n",
    "    preds = np.argmax(output_eval, axis=-1)\n",
    "    acc = np.mean(preds == y)\n",
    "    return acc, transform_eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: T.cost 1.611110, Train acc 0.431300, test acc 0.701400, took 5.85 sec.\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(formatter={'float': '{: 0.4f}'.format}, suppress=True)\n",
    "train_accs, test_accs = [], []\n",
    "total_time = 0\n",
    "param_outputs, disc_outputs = [], []\n",
    "disc_dist_t_1 = None\n",
    "quantized_bins = []\n",
    "try:\n",
    "    for n in range(NUM_EPOCHS):\n",
    "        start_time = time.time()\n",
    "        train_cost, train_acc, param_output, disc_output = train_epoch(data['X_train'], data['y_train'])\n",
    "        test_acc, test_transform = eval_epoch(data['X_test'], data['y_test'])\n",
    "        test_accs += [test_acc]\n",
    "        train_accs += [train_acc]\n",
    "\n",
    "        if DISC:\n",
    "            param_outputs = np.append(param_outputs, param_output)\n",
    "            disc_outputs = np.append(disc_outputs, disc_output)\n",
    "\n",
    "        if (n+1) % 20 == 0:\n",
    "            new_lr = sh_lr.get_value() * 0.99\n",
    "            print \"New LR:\", new_lr\n",
    "            sh_lr.set_value(lasagne.utils.floatX(new_lr))\n",
    "        \n",
    "        # Non-uniform Quantization\n",
    "        if DISC:\n",
    "            if n>0:\n",
    "                dist = disc_output.reshape((-1, 6))\n",
    "                q_bins = find_quantization_bins(dist, sharedBins=sharedBins)\n",
    "                quantized_bins.append(q_bins)\n",
    "\n",
    "        time_spent = time.time() - start_time\n",
    "        total_time += time_spent\n",
    "        print \"Epoch {0}: T.cost {1:0.6f}, Train acc {2:0.6f}, test acc {3:0.6f}, took {4:.3} sec.\".format(\n",
    "                n, train_cost, train_acc, test_acc, time_spent)\n",
    "\n",
    "except KeyboardInterrupt:\n",
    "    pass\n",
    "print \"\\nTotal time spent: {0:.5} seconds\\nTraing Acc: {1}\\nTest Acc: {2}\\n\".format(total_time, train_acc, test_acc) \n",
    "\n",
    "if DISC:\n",
    "    story = {'train_accs': train_accs,\n",
    "             'test_accs': test_accs,\n",
    "             'epoch_reached': n, \n",
    "             'total_time': total_time,\n",
    "             'disc_enabled': DISC,\n",
    "             'learning_rate': LEARNING_RATE,\n",
    "             'batch_size': BATCH_SIZE,\n",
    "             'dense_params': param_output,\n",
    "             'disc_params': disc_output,\n",
    "             'quantized_bins': quantized_bin}\n",
    "else:\n",
    "    story = {'train_accs': train_accs,\n",
    "             'test_accs': test_accs,\n",
    "             'epoch_reached': n, \n",
    "             'total_time': total_time,\n",
    "             'disc_enabled': DISC,\n",
    "             'learning_rate': LEARNING_RATE,\n",
    "             'batch_size': BATCH_SIZE,\n",
    "             'disc_params': disc_output}   \n",
    "\n",
    "with open(TEST_NAME + '.model', 'wb') as fp:\n",
    "  pickle.dump(story, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(5,5))\n",
    "plt.plot(1-np.array(train_accs), label='Training Error')\n",
    "plt.plot(1-np.array(test_accs), label='Test Error')\n",
    "plt.legend(fontsize=20)\n",
    "plt.xlabel('Epoch', fontsize=8)\n",
    "plt.ylabel('Error', fontsize=8)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Histograms\n",
    "dense_params = param_outputs.reshape((-1, 6))\n",
    "disc_params = disc_outputs.reshape((-1, 6))\n",
    "\n",
    "bin_count = 100\n",
    "plot_over = True\n",
    "\n",
    "for i in range(0, 6):\n",
    "    dns = dense_params[:, i]\n",
    "    \n",
    "    \n",
    "    dsc = disc_params[:, i]\n",
    "    print len(np.unique(dsc))\n",
    "    #PS: Using normed histograms to plot them over\n",
    "    # Theta x Dense\n",
    "    plt.figure()\n",
    "    n, bins, patches = plt.hist(dns, bins=bin_count, normed=plot_over, histtype='stepfilled')\n",
    "    plt.setp(patches, 'facecolor', 'r', 'alpha', 0.55)\n",
    "    if not plot_over:\n",
    "        plt.xlabel(('Theta({0}) - Discrete Output').format(i+1))\n",
    "        plt.ylabel('Frequency (Consider bin size)')\n",
    "        plt.grid(True)\n",
    "        plt.figure()\n",
    "    \n",
    "    # Theta x Discrete\n",
    "    n, bins, patches = plt.hist(dsc, bins=np.unique(dsc), normed=plot_over, histtype='stepfilled')\n",
    "    plt.setp(patches, 'facecolor', 'g', 'alpha', 0.55)\n",
    "    if not plot_over:\n",
    "        plt.xlabel(('Theta({0}) - Discrete Output').format(i+1))\n",
    "    else:\n",
    "        plt.xlabel(('Theta({0})').format(i+1))\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.grid(True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(7,14))\n",
    "for i in range(3):\n",
    "    plt.subplot(321+i*2)\n",
    "    plt.imshow(data['X_test'][i].reshape(DIM, DIM), cmap='gray', interpolation='none')\n",
    "    if i == 0:\n",
    "        plt.title('Original 60x60', fontsize=20)\n",
    "    plt.axis('off')\n",
    "    plt.subplot(322+i*2)\n",
    "    plt.imshow(test_transform[i].reshape(DIM//3, DIM//3), cmap='gray', interpolation='none')\n",
    "    if i == 0:\n",
    "        plt.title('Transformed 20x20', fontsize=20)\n",
    "    plt.axis('off')\n",
    "plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}