experiment_error_convergence_1_uci_fcnet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Bka_bK83VFHh"
   },
   "source": [
    "## Experiment Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8D5NSPs_cJZe"
   },
   "source": [
    "### Random seed / PyTorch / CUDA related"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 225
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 15161,
     "status": "ok",
     "timestamp": 1575002224413,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "pHbfpytEVFHu",
    "outputId": "d522b7bc-6293-4e78-9378-33fbd698b1be"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import datetime\n",
    "import os\n",
    "import sys\n",
    "import itertools\n",
    "\n",
    "# Use Google Colab\n",
    "use_colab = True\n",
    "\n",
    "# Is this notebook running on Colab?\n",
    "# If so, then google.colab package (github.com/googlecolab/colabtools)\n",
    "# should be available in this environment\n",
    "\n",
    "# Previous version used importlib, but we could do the same thing with\n",
    "# just attempting to import google.colab\n",
    "try:\n",
    "    from google.colab import drive\n",
    "    colab_available = True\n",
    "except:\n",
    "    colab_available = False\n",
    "\n",
    "if use_colab and colab_available:\n",
    "    drive.mount('/content/drive')\n",
    "    \n",
    "    # If there's a package I need to install separately, do it here\n",
    "    !pip install pyro-ppl\n",
    "    \n",
    "    # Install ronald_bdl package\n",
    "    !pip install git+https://github.com/ronaldseoh/ronald_bdl.git\n",
    "\n",
    "    # cd to the appropriate working directory under my Google Drive\n",
    "    %cd 'drive/My Drive/Colab Notebooks/bayesian-dl-experiments'\n",
    "    \n",
    "    # List the directory contents\n",
    "    !ls\n",
    "\n",
    "# IPython reloading magic\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Random seeds\n",
    "# Based on https://pytorch.org/docs/stable/notes/randomness.html\n",
    "random_seed = 682"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tqlpuws9Y-8U"
   },
   "source": [
    "### Third party libraries (NumPy, PyTorch, Pyro)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 15123,
     "status": "ok",
     "timestamp": 1575002224422,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "2zNVvKmZY-8X",
    "outputId": "1671e2b7-39d8-4a11-bc96-ab78ecd4a04a"
   },
   "outputs": [],
   "source": [
    "# Third party libraries import\n",
    "import numpy as np\n",
    "import torch\n",
    "import pyro\n",
    "import matplotlib.pyplot as plt\n",
    "import tqdm\n",
    "\n",
    "# Print version information\n",
    "print(\"Python Version: \" + sys.version)\n",
    "print(\"NumPy Version: \" + np.__version__)\n",
    "print(\"PyTorch Version: \" + torch.__version__)\n",
    "print(\"Pyro Version: \" + pyro.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 15109,
     "status": "ok",
     "timestamp": 1575002224428,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "uyRIfCC5Y-8g",
    "outputId": "23ca46c7-d6e8-455a-82b1-d03df6ebc121"
   },
   "outputs": [],
   "source": [
    "# More imports...\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import random_split, DataLoader, RandomSampler\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from pyro.infer import SVI, Trace_ELBO, HMC, MCMC\n",
    "\n",
    "# Import model and dataset classes from ronald_bdl\n",
    "from ronald_bdl import models, datasets\n",
    "from ronald_bdl.models import utils\n",
    "\n",
    "# pyplot setting\n",
    "%matplotlib inline\n",
    "\n",
    "# torch.device / CUDA Setup\n",
    "use_cuda = True\n",
    "\n",
    "if use_cuda and torch.cuda.is_available():\n",
    "    torch_device = torch.device('cuda')\n",
    "\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    \n",
    "    # Disable 'benchmark' mode\n",
    "    # Note: https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "    use_pin_memory = True # Faster Host to GPU copies with page-locked memory\n",
    "\n",
    "    # CUDA libraries version information\n",
    "    print(\"CUDA Version: \" + str(torch.version.cuda))\n",
    "    print(\"cuDNN Version: \" + str(torch.backends.cudnn.version()))\n",
    "    print(\"CUDA Device Name: \" + str(torch.cuda.get_device_name()))\n",
    "    print(\"CUDA Capabilities: \"+ str(torch.cuda.get_device_capability()))\n",
    "else:\n",
    "    torch_device = torch.device('cpu')\n",
    "    use_pin_memory = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XIFRoH3AcJZn"
   },
   "source": [
    "### Variable settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "P9xnP4vI_0yw"
   },
   "source": [
    "#### Data prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OcY4iCbk_0UU"
   },
   "outputs": [],
   "source": [
    "# Dataset to use\n",
    "uci_dataset_name = 'yacht'\n",
    "\n",
    "# Set the proportion of the original dataset to be available as a whole\n",
    "subset_proportions = [1]\n",
    "\n",
    "# Proportion of the dataset to be used for training\n",
    "dataset_train_size = 0.9\n",
    "\n",
    "# Number of dataset splits\n",
    "n_splits = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4GRYK9MqBbvu"
   },
   "source": [
    "#### NN settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "g_pzGq1_cJZp"
   },
   "outputs": [],
   "source": [
    "# Fully-Connected network hidden layers\n",
    "network_hidden_dims = [25, 50, 100]\n",
    "network_hidden_layers = [1, 3, 5]\n",
    "\n",
    "# Dropout\n",
    "dropout_rates = [0.1, 0.3, 0.5]\n",
    "\n",
    "# Length-scale\n",
    "length_scale_values = [1e-2]\n",
    "\n",
    "# Model Precision\n",
    "tau_values = [0.1, 0.15, 0.2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DuTnXABzVFKI"
   },
   "source": [
    "\n",
    "### Training setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p19qFgSAVFKS"
   },
   "outputs": [],
   "source": [
    "# Epochs\n",
    "n_epoch_values = [40, 400, 4000]\n",
    "\n",
    "# Optimizer learning rate\n",
    "learning_rate = 0.001 # PyTorch default value is 0.001\n",
    "\n",
    "# Training data batch sizes\n",
    "if uci_dataset_name in ('protein-tertiary-structure'): # Note: 45730 rows originally\n",
    "    n_training_batch = 256\n",
    "elif uci_dataset_name in ('yacht', 'bostonHousing'):\n",
    "    n_training_batch = 32\n",
    "\n",
    "# Number of test predictions (for each data point)\n",
    "n_prediction = 1000\n",
    "\n",
    "# Mean Squared Error for loss function to minimize\n",
    "objective = nn.MSELoss()\n",
    "\n",
    "# Test start time\n",
    "test_start_time = datetime.datetime.today().strftime('%Y%m%d%H%M')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1PpzPMI8VFKE"
   },
   "source": [
    "## Train the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6522898,
     "status": "ok",
     "timestamp": 1574969868282,
     "user": {
      "displayName": "Ronald Seoh",
      "photoUrl": "",
      "userId": "10284188050297676522"
     },
     "user_tz": 300
    },
    "id": "m4kavCiTVFKf",
    "outputId": "bdbb6c6c-dbd7-419b-beb6-eae907fe780c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for subset_prop in subset_proportions:\n",
    "\n",
    "    \"\"\"\n",
    "    Dataset multiple splits prep\n",
    "    \"\"\"\n",
    "\n",
    "    # Prepare new subset of the original dataset\n",
    "    subset = datasets.UCIDatasets(\n",
    "        uci_dataset_name, root_dir='./datasets_files', \n",
    "        limit_size=subset_prop,\n",
    "        transform=lambda X, mean, std: (X - mean) / std,\n",
    "        target_transform=lambda y, mean, std: (y - mean) / std,\n",
    "        download=True)\n",
    "\n",
    "    # Determine sizes of training and testing set\n",
    "    train_size = int(dataset_train_size * len(subset))\n",
    "    test_size = len(subset) - train_size\n",
    "    \n",
    "    # Print the size of the subset\n",
    "    print(\"subset size = \" + str((len(subset), subset.n_features)))\n",
    "    print(\"training set size = %d\" % train_size)\n",
    "    print(\"test set size = %d\" % test_size)\n",
    "    print()\n",
    "    \n",
    "    # Prepare multiple sets of random train-test splits \n",
    "    # to test the parameter combination\n",
    "    subset_splits = []\n",
    "\n",
    "    for _ in range(n_splits):\n",
    "        train, test = random_split(subset, lengths=[train_size, test_size])\n",
    "        subset_splits.append((train, test))\n",
    "\n",
    "    # With all the splits, test out each combination of hyperparameters\n",
    "    for hidden_dim, n_hidden, dropout_rate, length_scale, tau in itertools.product(\n",
    "        network_hidden_dims, network_hidden_layers,\n",
    "        dropout_rates, length_scale_values, tau_values,\n",
    "    ):\n",
    "        # Reset the random number generator for each method (to produce identical results)\n",
    "        torch.manual_seed(random_seed)\n",
    "        np.random.seed(random_seed)\n",
    "        pyro.set_rng_seed(random_seed)\n",
    "\n",
    "        # Print parameter combinations being tested\n",
    "        print(\n",
    "            \"subset %f, n_hidden %d, hidden_dim %d, dropout_rate %f, length_scale %f, tau %f\"\n",
    "            % (subset_prop, n_hidden, hidden_dim, dropout_rate, length_scale, tau))   \n",
    "\n",
    "        # Try learning with different splits\n",
    "        for s, (train, test) in enumerate(subset_splits):\n",
    "            train_loader = DataLoader(train, batch_size=n_training_batch, pin_memory=use_pin_memory)\n",
    "\n",
    "            # Prepare network\n",
    "            network = models.FCNet(\n",
    "              input_dim=subset.n_features, \n",
    "              output_dim=subset.n_targets,\n",
    "              hidden_dim=hidden_dim,\n",
    "              n_hidden=n_hidden,\n",
    "              dropout_rate=dropout_rate,\n",
    "              dropout_type='bernoulli',\n",
    "            )\n",
    "\n",
    "            # Send the whole model to the selected torch.device\n",
    "            network.to(torch_device)\n",
    "\n",
    "            # Model to train mode\n",
    "            network.train()\n",
    "\n",
    "            # Adam optimizer\n",
    "            # https://pytorch.org/docs/stable/optim.html?highlight=adam#torch.optim.Adam\n",
    "\n",
    "            # NOTE: Need to set L2 regularization from here\n",
    "            reg_strength = utils.reg_strength(dropout_rate, length_scale, train_size, tau)\n",
    "\n",
    "            print('reg_strength = ' + str(reg_strength))\n",
    "\n",
    "            optimizer = optim.Adam(\n",
    "                network.parameters(),\n",
    "                lr=learning_rate,\n",
    "                weight_decay=reg_strength, # L2 regularization\n",
    "            )\n",
    "\n",
    "            accumulated_epochs = 0\n",
    "\n",
    "            for n_epoch in n_epoch_values:\n",
    "\n",
    "                \"\"\"\n",
    "                Training\n",
    "                \"\"\"\n",
    "\n",
    "                print('Training with split %d' % s)\n",
    "                print(\"Training for %d epochs total.\" % n_epoch)\n",
    "\n",
    "                # Record training start time\n",
    "                tic = time.time()\n",
    "\n",
    "                progress_bar = tqdm.tqdm(range(n_epoch-accumulated_epochs))\n",
    "                \n",
    "                for epoch in progress_bar: # loop over the dataset multiple times\n",
    "                    # Mini-batches\n",
    "                    for data in train_loader:\n",
    "                        # get the inputs; data is a list of [inputs, labels]\n",
    "                        inputs, targets = data\n",
    "\n",
    "                        # Store the batch to torch_device's memory\n",
    "                        inputs = inputs.to(torch_device)\n",
    "                        targets = targets.to(torch_device)\n",
    "\n",
    "                        # zero the parameter gradients\n",
    "                        optimizer.zero_grad()\n",
    "\n",
    "                        # forward + backward + optimize\n",
    "                        outputs = network(inputs)\n",
    "\n",
    "                        loss = objective(outputs, targets)\n",
    "                        loss.backward()\n",
    "\n",
    "                        optimizer.step()\n",
    "\n",
    "                    progress_bar.set_description(\"[iteration %04d] loss: %.4f\" % (epoch + 1, loss.item()))\n",
    "                \n",
    "                # Record training end time\n",
    "                toc = time.time()\n",
    "                \n",
    "                # Track the number of epochs done so far\n",
    "                accumulated_epochs += (n_epoch - accumulated_epochs)\n",
    "                \n",
    "                # Report the total training time\n",
    "                print(\"training time = \" + str(toc - tic) + \" seconds\")\n",
    "                \n",
    "                # Report the final loss\n",
    "                print(\"final loss = %f\" % (loss.item()))\n",
    "                print()\n",
    "\n",
    "                \"\"\"\n",
    "                Testing\n",
    "                \"\"\"\n",
    "\n",
    "                # Model to eval mode\n",
    "                network.eval()\n",
    "\n",
    "                # Get the test data\n",
    "                inputs, targets = test.dataset[test.indices]\n",
    "\n",
    "                # Store the batch to torch_device's memory\n",
    "                inputs = inputs.to(torch_device)\n",
    "                targets = targets.to(torch_device)\n",
    "\n",
    "                # Record testing start time\n",
    "                tic_testing = time.time()\n",
    "\n",
    "                _, mean, var, metrics = network.predict_dist(\n",
    "                    inputs, n_prediction,\n",
    "                    y_test=targets,\n",
    "                    reg_strength=reg_strength, train_size=train_size, length_scale=length_scale)\n",
    "\n",
    "                # Record testing end time\n",
    "                toc_testing = time.time()\n",
    "                \n",
    "                # Report the total testing time\n",
    "                print(\"testing time = \" + str(toc_testing - tic_testing) + \" seconds\")\n",
    "\n",
    "                # Record all the scores to the score files\n",
    "                \"\"\"\n",
    "                Results file storage\n",
    "                \"\"\"\n",
    "\n",
    "                # Create directory to store results for the current test configuration\n",
    "                test_results_path = os.path.join(\n",
    "                    './test_results',\n",
    "                    'error_convergence_1',\n",
    "                    uci_dataset_name,\n",
    "                    test_start_time,\n",
    "                    (\n",
    "                        str(subset_prop) \n",
    "                        + '_' + str(hidden_dim) \n",
    "                        + '_' + str(n_hidden) \n",
    "                        + '_' + str(dropout_rate) \n",
    "                        + '_' + str(length_scale)\n",
    "                        + '_' + str(tau)\n",
    "                        + '_' + str(n_epoch)),\n",
    "                )\n",
    "\n",
    "                os.makedirs(test_results_path, exist_ok=True)\n",
    "\n",
    "                test_results_rmse_mc_path = os.path.join(\n",
    "                    test_results_path,\n",
    "                    \"rmse_mc.txt\"\n",
    "                )\n",
    "\n",
    "                test_results_lls_mc_path = os.path.join(\n",
    "                    test_results_path,\n",
    "                    \"lls_mc.txt\"\n",
    "                )\n",
    "\n",
    "                test_results_rmse_non_mc_path = os.path.join(\n",
    "                    test_results_path,\n",
    "                    \"rmse_non_mc.txt\"\n",
    "                ) \n",
    "\n",
    "                if len(metrics) > 0:\n",
    "                    for key, value in metrics.items():\n",
    "                        print(str(key) + \" = \" + str(value))\n",
    "\n",
    "                        if key == 'rmse_mc':\n",
    "                            with open(test_results_rmse_mc_path, 'a+') as rmse_mc_file:\n",
    "                                rmse_mc_file.write('%d %f \\n' % (s, value))\n",
    "\n",
    "                        elif key == 'test_ll_mc':\n",
    "                            with open(test_results_lls_mc_path, 'a+') as lls_mc_file:\n",
    "                                lls_mc_file.write('%d %f \\n' % (s, value))\n",
    "\n",
    "                        elif key == 'rmse_non_mc':\n",
    "                            with open(test_results_rmse_non_mc_path, 'a+') as rmse_non_mc_file:\n",
    "                                rmse_non_mc_file.write('%d %f \\n' % (s, value))\n",
    "                print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Results visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pDxkRM5aVrdf",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# experiment_root_directory = os.path.join(\n",
    "#     './test_results',\n",
    "#     'error_convergence_1',\n",
    "#     uci_dataset_name,\n",
    "#     test_start_time,\n",
    "# )\n",
    "\n",
    "# for subset_prop, dropout_rate, reg_strength, n_epoch in itertools.product(\n",
    "#     subset_proportions, dropout_rates, reg_strengths, n_epochs\n",
    "# ):\n",
    "#     for metric_name in ['lls_mc', 'rmse_mc', 'rmse_non_mc']:\n",
    "#         figure_file_name = (\n",
    "#             str(subset_prop)\n",
    "#             + '_' + str(dropout_rate)\n",
    "#             + '_' + str(reg_strength)\n",
    "#             + '_' + str(n_epoch)\n",
    "#             + '_' + metric_name + '.png'\n",
    "#         )\n",
    "        \n",
    "#         figure_title = (\n",
    "#             metric_name \n",
    "#             + (' subset %f, dropout rate = %f, reg strength = %f, n_epoch = %d' % (subset_prop, dropout_rate, reg_strength, n_epoch))\n",
    "#         )\n",
    "\n",
    "#         fig, ax = plt.subplots(len(network_hidden_dims), len(network_hidden_layers))\n",
    "\n",
    "#         fig.tight_layout(pad=2, rect=[0, 0.00, 2, 2])    \n",
    "\n",
    "#         plt_subplot_current_row = 0\n",
    "        \n",
    "#         for hidden_dim in network_hidden_dims:\n",
    "#             plt_subplot_current_column = 0\n",
    "\n",
    "#             for n_layer in network_hidden_layers:\n",
    "#                 # Open the score file\n",
    "#                 score_file_path = os.path.join(\n",
    "#                     experiment_root_directory,\n",
    "#                     (\n",
    "#                         str(subset_prop) \n",
    "#                         + '_' + str(hidden_dim)\n",
    "#                         + '_' + str(n_layer) \n",
    "#                         + '_' + str(dropout_rate) \n",
    "#                         + '_' + str(reg_strength)\n",
    "#                         + '_' + str(n_epoch)\n",
    "#                     ),\n",
    "#                     metric_name + '.txt',\n",
    "#                 )\n",
    "\n",
    "#                 scores = np.loadtxt(score_file_path).T\n",
    "                \n",
    "#                 #if metric_name in ('rmse_mc', 'rmse_non_mc'):\n",
    "#                 #    ax[plt_subplot_current_row, plt_subplot_current_column].set_ylim([0, 20])\n",
    "#                 #elif metric_name == 'lls_mc':\n",
    "#                 #    ax[plt_subplot_current_row, plt_subplot_current_column].set_ylim([-20, 0])\n",
    "\n",
    "#                 ax[plt_subplot_current_row, plt_subplot_current_column].scatter(scores[0], scores[1])\n",
    "\n",
    "#                 ax[plt_subplot_current_row, plt_subplot_current_column].set_title(str((hidden_dim, n_layer)))\n",
    "\n",
    "#                 plt_subplot_current_column += 1\n",
    "\n",
    "#             plt_subplot_current_row += 1\n",
    "\n",
    "#         fig.suptitle(figure_title, y=2.05)\n",
    "        \n",
    "#         plt.savefig(\n",
    "#             os.path.join(experiment_root_directory, figure_file_name),\n",
    "#             dpi=600,\n",
    "#             bbox_inches='tight',\n",
    "#         )\n",
    "        \n",
    "#         plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# from mpl_toolkits.mplot3d import Axes3D\n",
    "\n",
    "# experiment_root_directory = os.path.join(\n",
    "#     './test_results',\n",
    "#     'error_convergence_1',\n",
    "#     uci_dataset_name,\n",
    "#     test_start_time,\n",
    "# )\n",
    "\n",
    "# for subset_prop, dropout_rate, length_scale, tau, n_epoch in itertools.product(\n",
    "#     subset_proportions, dropout_rates, length_scale_values, tau_values, n_epochs\n",
    "# ):\n",
    "#     for metric_name in ['lls_mc', 'rmse_mc', 'rmse_non_mc']:\n",
    "#         figure_file_name = (\n",
    "#             '3d_' + str(subset_prop)\n",
    "#             + '_' + str(dropout_rate)\n",
    "#             + '_' + str(length_scale)\n",
    "#             + '_' + str(tau)\n",
    "#             + '_' + str(n_epoch)\n",
    "#             + '_' + metric_name + '.png'\n",
    "#         )\n",
    "        \n",
    "#         figure_title = (\n",
    "#             metric_name \n",
    "#             + (' subset %f, dropout rate = %f, length_scale %f, tau %f, n_epoch = %d' \n",
    "#                % (subset_prop, dropout_rate, length_scale, tau, n_epoch))\n",
    "#         )\n",
    "\n",
    "#         fig = plt.figure()\n",
    "#         ax = fig.add_subplot(111, projection='3d')\n",
    "\n",
    "#         fig.tight_layout(pad=2, rect=[0, 0.00, 2, 2])\n",
    "        \n",
    "#         hidden_dims_data = []\n",
    "#         hidden_layers_data = []\n",
    "#         scores_data = []\n",
    "        \n",
    "#         for hidden_dim in network_hidden_dims:\n",
    "#             for n_layer in network_hidden_layers:\n",
    "#                 # Open the score file\n",
    "#                 score_file_path = os.path.join(\n",
    "#                     experiment_root_directory,\n",
    "#                     (\n",
    "#                         str(subset_prop) \n",
    "#                         + '_' + str(hidden_dim)\n",
    "#                         + '_' + str(n_layer) \n",
    "#                         + '_' + str(dropout_rate) \n",
    "#                         + '_' + str(length_scale)\n",
    "#                         + '_' + str(tau)\n",
    "#                         + '_' + str(n_epoch)\n",
    "#                     ),\n",
    "#                     metric_name + '.txt',\n",
    "#                 )\n",
    "\n",
    "#                 scores = np.loadtxt(score_file_path).T\n",
    "                \n",
    "#                 for s in scores[1]:\n",
    "#                     # Multiple values (for each split) for\n",
    "#                     # each (hidden_dim, n_layer) combination\n",
    "#                     hidden_dims_data.append(hidden_dim)\n",
    "#                     hidden_layers_data.append(n_layer)\n",
    "#                     scores_data.append(s)\n",
    "                    \n",
    "#                 mean = np.mean(scores[1])\n",
    "#                 var = np.var(scores[1])\n",
    "\n",
    "#         ax.set_xlabel('hidden layer units')\n",
    "#         ax.set_ylabel('number of hidden layers')\n",
    "        \n",
    "#         #if metric_name in ('rmse_mc', 'rmse_non_mc'):\n",
    "#         #    ax.set_zlim([0, 5])\n",
    "#         #elif metric_name == 'lls_mc':\n",
    "#         #    ax.set_zlim([-5, 0])\n",
    "    \n",
    "#         ax.scatter3D(hidden_dims_data, hidden_layers_data, scores_data, c=scores_data)\n",
    "\n",
    "#         fig.suptitle(figure_title, y=2.05)        \n",
    "        \n",
    "#         plt.savefig(\n",
    "#             os.path.join(experiment_root_directory, figure_file_name),\n",
    "#             dpi=300,\n",
    "#             bbox_inches='tight',\n",
    "#         )\n",
    "        \n",
    "#         plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "experiment_convergence_1.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}