experiment_error_convergence_2_cifar10_simplecnn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Bka_bK83VFHh"
   },
   "source": [
    "## Experiment Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8D5NSPs_cJZe"
   },
   "source": [
    "### Random seed / PyTorch / CUDA related"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 347
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5979,
     "status": "ok",
     "timestamp": 1575227654566,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "pHbfpytEVFHu",
    "outputId": "fc6e59f5-fe07-4f25-b85a-0c465d4a7fb7"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import datetime\n",
    "import os\n",
    "import sys\n",
    "import itertools\n",
    "\n",
    "# Use Google Colab\n",
    "use_colab = True\n",
    "\n",
    "# Is this notebook running on Colab?\n",
    "# If so, then google.colab package (github.com/googlecolab/colabtools)\n",
    "# should be available in this environment\n",
    "\n",
    "# Previous version used importlib, but we could do the same thing with\n",
    "# just attempting to import google.colab\n",
    "try:\n",
    "    from google.colab import drive\n",
    "    colab_available = True\n",
    "except:\n",
    "    colab_available = False\n",
    "\n",
    "if use_colab and colab_available:\n",
    "    drive.mount('/content/drive')\n",
    "    \n",
    "    # If there's a package I need to install separately, do it here\n",
    "    !pip install pyro-ppl\n",
    "    \n",
    "    # Install ronald_bdl package\n",
    "    !pip install git+https://github.com/ronaldseoh/ronald_bdl.git\n",
    "\n",
    "    # cd to the appropriate working directory under my Google Drive\n",
    "    %cd 'drive/My Drive/Colab Notebooks/bayesian-dl-experiments'\n",
    "    \n",
    "    # List the directory contents\n",
    "    !ls\n",
    "\n",
    "# IPython reloading magic\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Random seeds\n",
    "# Based on https://pytorch.org/docs/stable/notes/randomness.html\n",
    "random_seed = 682"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tqlpuws9Y-8U"
   },
   "source": [
    "### Third party libraries (NumPy, PyTorch, Pyro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6455,
     "status": "ok",
     "timestamp": 1575227655051,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "2zNVvKmZY-8X",
    "outputId": "e36d839c-aea0-4560-d542-9290e420c9fd"
   },
   "outputs": [],
   "source": [
    "# Third party libraries import\n",
    "import numpy as np\n",
    "import torch\n",
    "import pyro\n",
    "import matplotlib.pyplot as plt\n",
    "import tqdm\n",
    "\n",
    "# Print version information\n",
    "print(\"Python Version: \" + sys.version)\n",
    "print(\"NumPy Version: \" + np.__version__)\n",
    "print(\"PyTorch Version: \" + torch.__version__)\n",
    "print(\"Pyro Version: \" + pyro.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6992,
     "status": "ok",
     "timestamp": 1575227655599,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "uyRIfCC5Y-8g",
    "outputId": "8f130afa-cd6e-4ee3-bc0a-dbd6af2c8070"
   },
   "outputs": [],
   "source": [
    "# More imports...\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import random_split, DataLoader, RandomSampler\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from pyro.infer import SVI, Trace_ELBO, HMC, MCMC\n",
    "\n",
    "# Import model and dataset classes from ronald_bdl\n",
    "from ronald_bdl import models, datasets\n",
    "from ronald_bdl.models import utils\n",
    "\n",
    "# pyplot setting\n",
    "%matplotlib inline\n",
    "\n",
    "# torch.device / CUDA Setup\n",
    "use_cuda = True\n",
    "\n",
    "if use_cuda and torch.cuda.is_available():\n",
    "    torch_device = torch.device('cuda')\n",
    "\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "    # Disable 'benchmark' mode\n",
    "    # Note: https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    use_pin_memory = True # Faster Host to GPU copies with page-locked memory\n",
    "\n",
    "    # CUDA libraries version information\n",
    "    print(\"CUDA Version: \" + str(torch.version.cuda))\n",
    "    print(\"cuDNN Version: \" + str(torch.backends.cudnn.version()))\n",
    "    print(\"CUDA Device Name: \" + str(torch.cuda.get_device_name()))\n",
    "    print(\"CUDA Capabilities: \"+ str(torch.cuda.get_device_capability()))\n",
    "else:\n",
    "    torch_device = torch.device('cpu')\n",
    "    use_pin_memory = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XIFRoH3AcJZn"
   },
   "source": [
    "### Variable settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DGRg2u0Q_I3n"
   },
   "source": [
    "#### Data prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f2gc_i7T_HVw"
   },
   "outputs": [],
   "source": [
    "# CIFAR10 data transformation setting\n",
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "# Set the proportion of the original dataset to be available as a whole\n",
    "subset_proportions = [0.01, 0.1, 1]\n",
    "\n",
    "# Proportion of the dataset to be used for training\n",
    "dataset_train_size = 0.8\n",
    "\n",
    "# Number of dataset splits\n",
    "n_splits = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fzxIZiUcA8D8"
   },
   "source": [
    "#### NN settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g_pzGq1_cJZp"
   },
   "outputs": [],
   "source": [
    "# Dropout\n",
    "dropout_rates = [0.1, 0.3, 0.5]\n",
    "\n",
    "# Length-scale\n",
    "length_scale_values = [1e-2]\n",
    "\n",
    "# Model Precision\n",
    "tau_values = [0.1, 0.15, 0.2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DuTnXABzVFKI"
   },
   "source": [
    "\n",
    "### Training setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 830,
     "status": "ok",
     "timestamp": 1575227752260,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "p19qFgSAVFKS",
    "outputId": "3895eaea-67d8-4780-91f5-8d4be308a397"
   },
   "outputs": [],
   "source": [
    "# Epochs\n",
    "n_epoch_values = [40, 400, 4000]\n",
    "\n",
    "# Optimizer learning rate\n",
    "optimizer_learning_rate = 0.01 # PyTorch default value is 0.001\n",
    "\n",
    "# Data batch sizes\n",
    "n_training_batch = 512\n",
    "\n",
    "# Number of test predictions (for each data point)\n",
    "n_prediction = 500\n",
    "\n",
    "# Cross Entropy to minimize\n",
    "objective = nn.CrossEntropyLoss()\n",
    "\n",
    "# Test start time\n",
    "test_start_time = datetime.datetime.today().strftime('%Y%m%d%H%M')\n",
    "\n",
    "print(test_start_time)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1PpzPMI8VFKE"
   },
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11504639,
     "status": "error",
     "timestamp": 1575239261379,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "m4kavCiTVFKf",
    "outputId": "02b9b726-8f0d-47e0-d92e-d86d31bd2dd1",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for subset_prop in subset_proportions:\n",
    "    \n",
    "    \"\"\"\n",
    "    Dataset multiple splits prep\n",
    "    \"\"\"\n",
    "    # Prepare new subset of the original dataset\n",
    "    subset = datasets.CIFAR10(\n",
    "        root='./datasets_files', limit_size=subset_prop, transform=transform, download=True)\n",
    "\n",
    "    # Determine sizes of training and testing set\n",
    "    train_size = int(dataset_train_size * len(subset))\n",
    "    test_size = len(subset) - train_size\n",
    "\n",
    "    # Print the size of the subset\n",
    "    print(\"subset size = \" + str(subset.data.shape))\n",
    "    print(\"training set size = %d\" % train_size)\n",
    "    print(\"test set size = %d\" % test_size)\n",
    "    print()\n",
    "\n",
    "    # Prepare multiple sets of random train-test splits \n",
    "    # to test the parameter combination\n",
    "    subset_splits = []\n",
    "\n",
    "    for _ in range(n_splits):\n",
    "        train, test = random_split(subset, lengths=[train_size, test_size])\n",
    "        subset_splits.append((train, test))\n",
    "\n",
    "    # With all the splits, test out each combination of hyperparameters\n",
    "    for dropout_rate, length_scale, tau in itertools.product(\n",
    "        dropout_rates, length_scale_values, tau_values,\n",
    "    ):  \n",
    "        # Reset the random number generator for each method (to produce identical results)\n",
    "        torch.manual_seed(random_seed)\n",
    "        np.random.seed(random_seed)\n",
    "        pyro.set_rng_seed(random_seed)\n",
    "\n",
    "        # Print parameter combinations being tested\n",
    "        print(\n",
    "            \"subset %f, dropout_rate %f, length_scale %f, tau %f\"\n",
    "            % (subset_prop, dropout_rate, length_scale, tau))\n",
    "\n",
    "        \"\"\"\n",
    "        Training & testing\n",
    "        \"\"\"\n",
    "\n",
    "        # Try learning with different splits\n",
    "        for s, (train, test) in enumerate(subset_splits):\n",
    "            train_loader = DataLoader(train, batch_size=n_training_batch, pin_memory=use_pin_memory)\n",
    "\n",
    "            # Prepare network\n",
    "            network = models.SimpleCIFAR10(\n",
    "                dropout_rate=dropout_rate,\n",
    "                dropout_type='bernoulli',\n",
    "            )\n",
    "\n",
    "            # Send the whole model to the selected torch.device\n",
    "            network.to(torch_device)\n",
    "\n",
    "            # Model to train mode\n",
    "            network.train()\n",
    "\n",
    "            # Adam optimizer\n",
    "            # https://pytorch.org/docs/stable/optim.html?highlight=adam#torch.optim.Adam\n",
    "\n",
    "            # NOTE: Need to set L2 regularization from here\n",
    "            reg_strength = utils.reg_strength(dropout_rate, length_scale, train_size, tau)\n",
    "\n",
    "            print('reg_strength = ' + str(reg_strength))\n",
    "\n",
    "            optimizer = optim.Adam(\n",
    "                network.parameters(),\n",
    "                lr=optimizer_learning_rate,\n",
    "                weight_decay=reg_strength, # L2 regularization\n",
    "            )\n",
    "\n",
    "            accumulated_epochs = 0\n",
    "\n",
    "            for n_epoch in n_epoch_values:\n",
    "\n",
    "                \"\"\"\n",
    "                Training\n",
    "                \"\"\"\n",
    "\n",
    "                print('Training with split %d' % s)\n",
    "                print(\"Training for %d epochs total.\" % n_epoch)\n",
    "            \n",
    "                # Record training start time (for this split)\n",
    "                tic = time.time()\n",
    "\n",
    "                progress_bar = tqdm.tqdm(range(n_epoch-accumulated_epochs))\n",
    "                \n",
    "                for epoch in progress_bar: # loop over the dataset multiple times\n",
    "                    # Mini-batches\n",
    "                    for data in train_loader:\n",
    "                        # get the inputs; data is a list of [inputs, labels]\n",
    "                        inputs, targets = data\n",
    "\n",
    "                        # Store the batch to torch_device's memory\n",
    "                        inputs = inputs.to(torch_device)\n",
    "                        targets = targets.to(torch_device)\n",
    "\n",
    "                        # zero the parameter gradients\n",
    "                        optimizer.zero_grad()\n",
    "\n",
    "                        # forward + backward + optimize\n",
    "                        outputs = network(inputs)\n",
    "\n",
    "                        loss = objective(outputs, targets)\n",
    "                        loss.backward()\n",
    "\n",
    "                        optimizer.step()\n",
    "\n",
    "                    progress_bar.set_description(\"[iteration %04d] loss: %.4f\" % (epoch + 1, loss.item()))\n",
    "\n",
    "                # Record training end time\n",
    "                toc = time.time()\n",
    "                \n",
    "                # Track the number of epochs done so far\n",
    "                accumulated_epochs += (n_epoch - accumulated_epochs)\n",
    "                \n",
    "                # Report the total training time\n",
    "                print(\"training time = \" + str(toc - tic) + \" seconds\")\n",
    "                \n",
    "                # Report the final loss\n",
    "                print(\"final loss = %f\" % (loss.item()))\n",
    "                print()\n",
    "\n",
    "                \"\"\"\n",
    "                Testing\n",
    "                \"\"\"\n",
    "\n",
    "                # Model to eval mode\n",
    "                network.eval()\n",
    "\n",
    "                # Store the batch to torch_device's memory\n",
    "                test_loader = DataLoader(test, batch_size=n_training_batch, pin_memory=use_pin_memory)\n",
    "\n",
    "                # Record testing start time\n",
    "                tic_testing = time.time()\n",
    "\n",
    "                _, mean, metrics = network.predict_dist(test_loader, n_prediction)\n",
    "\n",
    "                # Record testing end time\n",
    "                toc_testing = time.time()\n",
    "                \n",
    "                # Report the total testing time\n",
    "                print(\"testing time = \" + str(toc_testing - tic_testing) + \" seconds\")\n",
    "\n",
    "                # Record all the scores to the score files\n",
    "                \"\"\"\n",
    "                Results file storage\n",
    "                \"\"\"\n",
    "\n",
    "                # Create directory to store results for the current test configuration\n",
    "                test_results_path = os.path.join(\n",
    "                    './test_results',\n",
    "                    'error_convergence_2',\n",
    "                    'CIFAR-10',\n",
    "                    test_start_time,\n",
    "                    (\n",
    "                        str(subset_prop)\n",
    "                        + '_' + str(dropout_rate) \n",
    "                        + '_' + str(length_scale)\n",
    "                        + '_' + str(tau)\n",
    "                        + '_' + str(n_epoch)),\n",
    "                )\n",
    "\n",
    "                os.makedirs(test_results_path, exist_ok=True)\n",
    "\n",
    "                test_results_accuracy_mc_path = os.path.join(\n",
    "                    test_results_path,\n",
    "                    \"accuracy_mc.txt\"\n",
    "                )\n",
    "\n",
    "                test_results_accuracy_non_mc_path = os.path.join(\n",
    "                    test_results_path,\n",
    "                    \"accuracy_non_mc.txt\"\n",
    "                )\n",
    "\n",
    "                test_results_lls_mc_path = os.path.join(\n",
    "                    test_results_path,\n",
    "                    \"lls_mc.txt\"\n",
    "                )\n",
    "\n",
    "                if len(metrics) > 0:\n",
    "                    for key, value in metrics.items():\n",
    "                        print(str(key) + \" = \" + str(value))\n",
    "\n",
    "                        if key == 'accuracy_mc':\n",
    "                            with open(test_results_accuracy_mc_path, 'a+') as accuracy_mc_file:\n",
    "                                accuracy_mc_file.write('%d %f \\n' % (s, value))\n",
    "\n",
    "                        elif key == 'accuracy_non_mc':\n",
    "                            with open(test_results_accuracy_non_mc_path, 'a+') as accuracy_non_mc_file:\n",
    "                                accuracy_non_mc_file.write('%d %f \\n' % (s, value))\n",
    "\n",
    "                        elif key == 'test_ll_mc':\n",
    "                            with open(test_results_lls_mc_path, 'a+') as lls_mc_file:\n",
    "                                lls_mc_file.write('%d %f \\n' % (s, value))\n",
    "\n",
    "                print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "whZbn3VTTUFa"
   },
   "source": [
    "## Results visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pDxkRM5aVrdf"
   },
   "outputs": [],
   "source": [
    "# from mpl_toolkits.mplot3d import Axes3D\n",
    "\n",
    "# experiment_root_directory = os.path.join(\n",
    "#     './test_results',\n",
    "#     'error_convergence_2',\n",
    "#     'CIFAR-10',\n",
    "#     test_start_time,\n",
    "# )\n",
    "\n",
    "# for subset_prop, dropout_rate, length_scale, tau, n_epoch in itertools.product(\n",
    "#     subset_proportions, dropout_rates, length_scale_values, tau_values, n_epochs\n",
    "# ):\n",
    "#     for metric_name in ['lls_mc', 'accuracy_mc', 'accuracy_non_mc']:\n",
    "#         figure_file_name = (\n",
    "#             '3d_' + str(subset_prop)\n",
    "#             + '_' + str(dropout_rate)\n",
    "#             + '_' + str(length_scale)\n",
    "#             + '_' + str(tau)\n",
    "#             + '_' + str(n_epoch)\n",
    "#             + '_' + metric_name + '.png'\n",
    "#         )\n",
    "        \n",
    "#         figure_title = (\n",
    "#             metric_name \n",
    "#             + (' subset %f, dropout rate = %f, length_scale %f, tau %f, n_epoch = %d' \n",
    "#                % (subset_prop, dropout_rate, length_scale, tau, n_epoch))\n",
    "#         )\n",
    "\n",
    "#         fig = plt.figure()\n",
    "#         ax = fig.add_subplot(111, projection='3d')\n",
    "\n",
    "#         fig.tight_layout(pad=2, rect=[0, 0.00, 2, 2])\n",
    "        \n",
    "#         hidden_dims_data = []\n",
    "#         hidden_layers_data = []\n",
    "#         scores_data = []\n",
    "        \n",
    "#         for hidden_dim in network_hidden_dims:\n",
    "#             for n_layer in network_hidden_layers:\n",
    "#                 # Open the score file\n",
    "#                 score_file_path = os.path.join(\n",
    "#                     experiment_root_directory,\n",
    "#                     (\n",
    "#                         str(subset_prop) \n",
    "#                         + '_' + str(hidden_dim)\n",
    "#                         + '_' + str(n_layer) \n",
    "#                         + '_' + str(dropout_rate) \n",
    "#                         + '_' + str(length_scale)\n",
    "#                         + '_' + str(tau)\n",
    "#                         + '_' + str(n_epoch)\n",
    "#                     ),\n",
    "#                     metric_name + '.txt',\n",
    "#                 )\n",
    "\n",
    "#                 scores = np.loadtxt(score_file_path).T\n",
    "                \n",
    "#                 for s in scores[1]:\n",
    "#                     # Multiple values (for each split) for\n",
    "#                     # each (hidden_dim, n_layer) combination\n",
    "#                     hidden_dims_data.append(hidden_dim)\n",
    "#                     hidden_layers_data.append(n_layer)\n",
    "#                     scores_data.append(s)\n",
    "                    \n",
    "#                 mean = np.mean(scores[1])\n",
    "#                 var = np.var(scores[1])\n",
    "\n",
    "#         ax.set_xlabel('hidden layer units')\n",
    "#         ax.set_ylabel('number of hidden layers')\n",
    "        \n",
    "#         #if metric_name in ('rmse_mc', 'rmse_non_mc'):\n",
    "#         #    ax.set_zlim([5, 20])\n",
    "#         #elif metric_name == 'lls_mc':\n",
    "#         #    ax.set_zlim([-10, 0])\n",
    "    \n",
    "#         ax.scatter3D(hidden_dims_data, hidden_layers_data, scores_data, c=scores_data)\n",
    "\n",
    "#         fig.suptitle(figure_title, y=2.05)        \n",
    "        \n",
    "#         plt.savefig(\n",
    "#             os.path.join(experiment_root_directory, figure_file_name),\n",
    "#             dpi=600,\n",
    "#             bbox_inches='tight',\n",
    "#         )\n",
    "        \n",
    "#         plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "experiment_error_convergence_2_cifar10_simplecnn.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}