FS_sandbox.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FS_sandbox.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "GNFQ0-MpQuBm",
        "q4JYRICVBtjl",
        "AS5zhoWCDZ8E",
        "IVa9K3vmAGKQ"
      ],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TomMaullin/BLMM-sandbox/blob/master/FS_sandbox.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TIMtrhFKB3Ay",
        "colab_type": "text"
      },
      "source": [
        "# FS implementation in python\n",
        "\n",
        "This code implements the Fisher Scoring algorithm for estimating the parameters of linear mixed effects models as described in [Demidenko 2013](https://www.wiley.com/en-us/Mixed+Models%3A+Theory+and+Applications+with+R%2C+2nd+Edition-p-9781118091579)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GNFQ0-MpQuBm",
        "colab_type": "text"
      },
      "source": [
        "## Pip Installations\n",
        "\n",
        "Pip install everything."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UX02P0KvBWJr",
        "colab_type": "code",
        "outputId": "ed069a4b-a20d-4629-c389-032d6952c17e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "!pip install numpy\n",
        "!pip install pandas\n",
        "!pip install scipy\n",
        "!pip install matplotlib\n",
        "!pip install sparse\n",
        "!pip install nibabel\n",
        "!pip install nilearn"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (1.16.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (0.24.2)\n",
            "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.5.3)\n",
            "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from pandas) (1.16.5)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.5.0->pandas) (1.12.0)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (1.3.1)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy) (1.16.5)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (3.0.3)\n",
            "Requirement already satisfied: numpy>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.16.5)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (1.1.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.5.3)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib) (2.4.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from cycler>=0.10->matplotlib) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from kiwisolver>=1.0.1->matplotlib) (41.2.0)\n",
            "Requirement already satisfied: sparse in /usr/local/lib/python3.6/dist-packages (0.8.0)\n",
            "Requirement already satisfied: numba>=0.45 in /usr/local/lib/python3.6/dist-packages (from sparse) (0.46.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from sparse) (1.16.5)\n",
            "Requirement already satisfied: scipy>=0.19 in /usr/local/lib/python3.6/dist-packages (from sparse) (1.3.1)\n",
            "Requirement already satisfied: llvmlite>=0.30.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba>=0.45->sparse) (0.30.0)\n",
            "Requirement already satisfied: nibabel in /usr/local/lib/python3.6/dist-packages (2.3.3)\n",
            "Requirement already satisfied: bz2file in /usr/local/lib/python3.6/dist-packages (from nibabel) (0.98)\n",
            "Requirement already satisfied: six>=1.3 in /usr/local/lib/python3.6/dist-packages (from nibabel) (1.12.0)\n",
            "Requirement already satisfied: numpy>=1.7.1 in /usr/local/lib/python3.6/dist-packages (from nibabel) (1.16.5)\n",
            "Requirement already satisfied: nilearn in /usr/local/lib/python3.6/dist-packages (0.5.2)\n",
            "Requirement already satisfied: nibabel>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from nilearn) (2.3.3)\n",
            "Requirement already satisfied: bz2file in /usr/local/lib/python3.6/dist-packages (from nibabel>=2.0.2->nilearn) (0.98)\n",
            "Requirement already satisfied: numpy>=1.7.1 in /usr/local/lib/python3.6/dist-packages (from nibabel>=2.0.2->nilearn) (1.16.5)\n",
            "Requirement already satisfied: six>=1.3 in /usr/local/lib/python3.6/dist-packages (from nibabel>=2.0.2->nilearn) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q4JYRICVBtjl",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## Python Imports\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YkTBWbRKQ5ah",
        "colab_type": "text"
      },
      "source": [
        "We need:\n",
        " - `numpy` for matrix handling.\n",
        " - `scipy` for sparse matrix functions.\n",
        " - `pandas` for quick reading and writing of csv files.\n",
        " - `os` and `sys` for basic commandline functions\n",
        " - `time` for timing functions.\n",
        " - `matplotlib` for making displays."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tebSlxvBBruv",
        "colab_type": "code",
        "outputId": "24740895-4d06-46f0-cbd8-69620154a5ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "import numpy as np\n",
        "import cvxopt\n",
        "import pandas as pd\n",
        "import os\n",
        "import time\n",
        "import scipy.sparse\n",
        "import scipy.sparse.linalg\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.pyplot import imshow\n",
        "import sys\n",
        "import nibabel as nib\n",
        "import nilearn"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
            "  warnings.warn(msg, category=DeprecationWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AS5zhoWCDZ8E",
        "colab_type": "text"
      },
      "source": [
        "## Toy Dataset\n",
        "\n",
        "This section read ins and formats a toy dataset. The files used here were generated in `R` and with **True** values (those with postfix `True`) being those used to generate the data and **Estimated** (those with postfix `REst`) values being the estimates `R`'s `lmer` package generated from this data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qy39zwuhkn4C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import shutil and remove old testdata\n",
        "#import shutil\n",
        "#shutil.rmtree('/Data')\n",
        "\n",
        "# Make a data directory\n",
        "if not os.path.isdir('/Data'):\n",
        "  os.mkdir('/Data')\n",
        "  \n",
        "os.chdir('/Data')\n",
        "\n",
        "# Clone small git repo containg some csv files.\n",
        "if not os.path.isdir('/Data/BLMM-testdata'):\n",
        "  !git clone https://github.com/TomMaullin/BLMM-testdata.git\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2EWsCZjCQcc9",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "### Z matrix\n",
        "\n",
        "The below reads in Z and makes an image of Z transpose.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aoKOwHqcEKDT",
        "colab_type": "code",
        "outputId": "c90990b5-26f7-4444-dda1-8d7713eae33e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "# Read in random effects design matrix and convert it into it's sparse format in\n",
        "# cvxopt.\n",
        "Z_3col=pd.read_csv('/Data/BLMM-testdata/Z_3col.csv',header=None).values#pd.read_csv('/Data/BLMM-testdata/Z_3col_1factor.csv',header=None).values#\n",
        "Z = scipy.sparse.csr_matrix((Z_3col[:,2].tolist(), \\\n",
        "                            ((Z_3col[:,0]-1).astype(np.int64), \\\n",
        "                             (Z_3col[:,1]-1).astype(np.int64))))\n",
        "\n",
        "# Create an image of Z'\n",
        "imshow(Z.toarray().transpose(), \\\n",
        "       interpolation='nearest', vmin=-5, vmax=5, aspect='auto')\n",
        "\n",
        "# Number of subjects\n",
        "n = Z.shape[0]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFnNJREFUeJzt3X2QXFWZx/Hfk5kkM3khLxrHJENM\nhoQgIhswQCIuOyVi2Ihi1VIqo27czW5E1zWIL4C6peyqBborYolAMO5EpAwIrLCRZWAx6KIYSEyD\ngQA9BJAkkPAOCQKZ5Nk/+nbP7Z5+f5mZzPl+qqa677n3nnPu7dPP3LndzxxzdwEAwjBqqDsAABg8\nBH0ACAhBHwACQtAHgIAQ9AEgIAR9AAgIQR8AAkLQB4CA1BT0zexUM3vIzHrN7Lx6dQoA0BhWbUau\nmTVJeljSKZK2S7pH0pnu/kChfZomjvfmaZOrai+jjz9OAITl9Se2P+Pu0+pRV3MN+x4vqdfdt0mS\nma2VdLqkgkG/edpktX/r0zU0KR3Y1VLT/gBwsHns7C88Xq+6arlsninpidjy9qgMADBMNfxeiZmt\nMLONZrbxwMt7G90cAKCIWoL+DkmHxpbbo7Is7r7K3Re6+8JRE8fX0BwAoFa13NO/R9I8M5ujVLD/\niKSuYju8feKzGv+T0Rpzyz3q2ZnQfa+/qqueW6z7jnX17ExoyYwFkqSenQlJ0pIZCzLP08u9Fy+q\nocsAELaqg76795nZZyT1SGqS9GN3v79uPQMA1F0tV/py95sl3VynvgAAGowvvQNAQKpOzqrG2FmH\n+ozPn11THaPaXq1pf77nD+Bg89jZX9jk7gvrURdX+gAQEII+AASEoA8AASHoA0BAavrK5lBou2Gs\nxl+3QZIyCV09OxN63wmnqe+J7ZnytCUzFmj3p9+pzV/9IcldAILHlT4ABISgDwABIegDQEAI+gAQ\nkIMuI7dWtWb0SmT1AhhcZOQCAKpC0AeAgBD0ASAgBH0ACEhwQb910zh1dCU0/q7U44eP3KSOroSS\nnd3q6ErouLc8rv1Pt2TKJt3emlmXLgOAg1VwQR8AQkbQB4CAEPQBICDBJWfVAwleAAYTyVkAgKoQ\n9AEgIAR9AAgIQR8AAnLQTZc4HMz73E7dvPlWLZmxQFJqesbP7DhBP5i5ITN9Y/pRkj65fbF+88tj\nNOuC32XKDrvmrCHrP4BwcaUPAAEh6ANAQAj6ABAQgj4ABISM3CFCVi+AcpGRCwCoCkEfAAJC0AeA\ngBD0ASAgZOQOkY6uxICydLaupEy2b/vvJ2j1rDslSae+5Xj5vtclSa+edry2nzwIHQUwonClDwAB\nIegDQEBKBn0z+7GZ7TazLbGyqWZ2m5klo8cpje0mAKAeSiZnmdlJkvZI+om7HxWVfVvSc+5+oZmd\nJ2mKu59bqjGSs+qr1gQvkruAg8OgJme5+28kPZdTfLqkNdHzNZI+WI/OAAAaq9p7+m3u/mT0/ClJ\nbXXqDwCggWr+INdT94cK3iMysxVmttHMNu7fs7fW5gAANag26O8ys+mSFD3uLrShu69y94XuvrBp\nwvgqmwMA1EO1yVk3SVom6cLo8ca69Qhlm/nT0brjR1dqn+/X0b/7hLaeeFXWdI3SwISv+LreixcN\nSb8BDJ1yvrL5M0l3SZpvZtvNbLlSwf4UM0tKek+0DAAY5kpe6bv7mQVW8U8AAOAgQ0YuAASEoA8A\nAWG6xIAxZSNwcGC6RABAVQj6ABAQgj4ABISgDwABYbrEgCU7uzPPl8xYIN3erp63rutfVn9G75IZ\nC7Ti4W1adXhHpuydnztLu0jqBQ4qXOkDQEAI+gAQEII+AASE5CzUhAQvoPFIzgIAVIWgDwABIegD\nQEAI+gAQEJKzUJOOrlSiVs/OhE7Z+n6NOvmJrCka04740ae0+mOX6l87jpUkXfmnOzWreYKWzFjA\ntI3AIOJKHwACQtAHgIAQ9AEgIAR9AAgIGbkYcmT1AsWRkQsAqApBHwACQtAHgIAQ9AEgIGTkYsgl\nO7v1rWfm69dHt6pnZ0JLZizIPKb1XnWM5n58s6RU9u/Sh5bq5vk3Z7YhqxcoD1f6ABAQgj4ABISg\nDwABITkLI0KtCV4kd2E4IzkLAFAVgj4ABISgDwABIegDQEBIzsKIcGh3s16b1KTfXnJFWdunE8DS\nz0nuQii40geAgBD0ASAgBH0ACEjJoG9mh5rZejN7wMzuN7OVUflUM7vNzJLR45TGdxcAUIuSGblm\nNl3SdHf/g5lNlLRJ0gclfULSc+5+oZmdJ2mKu59brC4ycjFcMWUjhrNBzch19yfd/Q/R85clbZU0\nU9LpktZEm61R6hcBAGAYq+ievpnNlnSMpA2S2tz9yWjVU5LaCuyzwsw2mtnG/Xv21tBVAECtyg76\nZjZB0vWSznb3l+LrPHWPKO99Indf5e4L3X1h04TxNXUWAFCbsoK+mY1WKuBf7e43RMW7ovv96fv+\nuxvTRQBAvZTMyDUzk7Ra0lZ3/25s1U2Slkm6MHq8sSE9BAZBR1cqOzedpXvqrIV69OojNfvD92Vl\n7v759OPVeuPdWdM6ph12zVmD33GgQuX8G4YTJX1c0h/NLD3Cv6xUsL/WzJZLelzShxrTRQBAvZQM\n+u5+pyQrsPrk+nYHANBIZOQCQECYLhGoExK80ChMlwgAqApBHwACQtAHgIAQ9AEgIEyXCNRJsrNb\nUv9UjPFHqT/x68m+PfrErHdl9iPBC4OJK30ACAhBHwACQtAHgIAQ9AEgIGTkAsMIWb3Ih4xcAEBV\nCPoAEBCCPgAEhKAPDCPJzm4lO7vV0ZXI/CQ7u9X3WlOmfN43XpHvaM2sO7DfNG/lDiU7uzX5ttah\nPgQMcwR9AAgIQR8AAkLQB4CAEPQBICAkZwEjTK0JXiR3DT8kZwEAqkLQB4CAEPQBICAEfQAICNMl\nAiPMzxdfoXPnnKDP9j6o7889Qj07E7p2zyR9aMKLkpSZvjFt1FFH6H9uXZsp77140aD3GYOHK30A\nCAhBHwACQtAHgIAQ9AEgIGTkAsjClI3DDxm5AICqEPQBICAEfQAICMlZALIkO7uzltNJWz07E3nL\n4+tI8Br+uNIHgIAQ9AEgIAR9AAhIyaBvZi1mdreZ3Wtm95vZBVH5HDPbYGa9ZnaNmY1pfHcBALUo\nmZxlZiZpvLvvMbPRku6UtFLSOZJucPe1Zna5pHvd/bJidZGcBYSBBK/6GtTkLE/ZEy2Ojn5c0rsl\nXReVr5H0wXp0CADQOGXd0zezJjNLSNot6TZJj0h6wd37ok22S5rZmC4CAOqlrKDv7vvdfYGkdknH\nSzqi3AbMbIWZbTSzjfv37K2ymwCAeqjo2zvu/oKk9ZIWS5psZunkrnZJOwrss8rdF7r7wqYJ42vq\nLACgNiUzcs1smqR97v6CmbVKOkXSRUoF/zMkrZW0TNKNjewogIPH5ENe0Qltjyt53GuSpK9sS+ik\nllTGbuuv2/SLeT1aMmOBFt27TxdMu3/AFI49OxM67JqzhqLrI145/4ZhuqQ1Ztak1F8G17r7OjN7\nQNJaM/uGpM2SVjewnwCAOigZ9N39PknH5CnfptT9fQDAQYKMXAAICEEfAALCdIkAhiWyevsxXSIA\noCoEfQAICEEfAALCdIkAhqWOrtQUjPFpGuNTNy49+mRZc7P2zXmz7K57Zce9XbfceJUO7/6U5nz5\nLklM25gPV/oAEBCCPgAEhKAPAAEh6ANAQEjOAjBi1ZrgNVySu0jOAgBUhaAPAAEh6ANAQAj6ABAQ\ngj6AEaujK6FkZ7c6uhI6/Iu7M8+Tnd2acOe4zPP9z47N2jb9OBIR9AEgIAR9AAgIQR8AAkLQB4CA\nkJELAAUMlykbycgFAFSFoA8AASHoA0BAmC4RAAro6EoUnK7x5I8vV/Ptm9SzM5EpT645VpPvGqtp\nl6ema3zPlpd1+W2nDH7Hi+BKHwACQtAHgIAQ9AEgIAR9AAgIyVkA0ED1SPDaduZXSc4CAFSOoA8A\nASHoA0BACPoAEBAycgGggTq6Elr+8KP60IQXtbTzb3TzHddL6s/ujUtn97bddYh2LX4pkw3cVMf+\ncKUPAAEh6ANAQMoO+mbWZGabzWxdtDzHzDaYWa+ZXWNmYxrXTQBAPVRypb9S0tbY8kWSLnb3uZKe\nl7S8nh0DANRfWRm5ZtYuaY2kb0o6R9L7JT0t6c3u3mdmiyV93d2XFKtnbMdMn7HyHI1qe1UHdrVk\nHiVlPU8rtD5329x1kjL1p58XaqNYW7mZdKWmPSunX/mU079y+pzbx3xl8f0KrS+3L+W8JvHjLrS+\nktek1P6lXoN8fcqtq1RbpcZtNSo5jnhZ+hgqqa+S163Y8ZY6H4XGZDnv1XLayqfccVftfpW+TvmW\nc+vO149cQzFd4vckfUnSgWj5DZJecPe+aHm7pJn16BAAoHFKBn0zO03SbnffVE0DZrbCzDaa2cYD\nL++tpgoAQJ2U8z39EyV9wMyWSmqRdIikSyRNNrPm6Gq/XdKOfDu7+ypJq6TU7Z269BoAUJWSV/ru\nfr67t7v7bEkfkfQrd/+opPWSzog2WybpxlJ1vX3is5Kkud9K3dc6/EtPZ9YlO7sHbP/zxVfkXZ+7\nbUdXImtdS+vrmfL4tu+e+1DBvsW3a+ptzdp/8ZxtWW0U0vdK9u/Qf3tH/ylJdnaroyuhWaubNGbL\nOCU7uzVpfWtWu60bx5VsIy7ep46uxIA+5iuL75fs7Na8Tz+at+63zXyyZPvt057P25d03ZJ02Hf2\n5e1Hen2x83rIxFf6n/+6Ne/+uWNh80mXZy1P/E3/fvHjjvepo+2ZrLpGPZrd1vRrx2Ttnzve6iG3\nntzllsTAsZHs7Nbh5z9bVn2F+tzx/QN5t+voSqjlD+MG7JvveaH1fS+M0YTx2Z9l5fZj/3NjM89z\nx0LfS2OytpWkE2Y/pmLSdbRsHqf3zd8y4DzEY0pcfDxNvaVVb535VN56873Hii3nex2KjZl6jadi\navme/rmSzjGzXqXu8a+uT5cAAI1S0b9hcPc7JN0RPd8m6fj6dwkA0Chk5AJAQAj6ABCQYTddYr6E\nqtzlQskV5WwX3za+fal9Cu2fr45y+1FO3em6Kkk6K7S+VFJOvP58SWXFEpgKtVPu+nKSr8pN0MrX\nbr5jq0Y1bebuF5d7TnP3KbRdsUStUolG8f5Uc2zFjrnUe65UIlmpRLlS7RRan+/9E1dOW+WO60Jj\nN378lb6HhyI5CwAwAhD0ASAgBH0ACAhBHwACMuyCfm7GWjqjM23uRa/l3e+QO7KzW4tleqbbKJU9\nV0jfy6PV9+dmdXQl9C/HrhtQd6EMxXKk+xV/jJfnayO33/P++U9Z66fdlPowaP27flBwn6wM1Zy6\n563cMSADNt+5WnH0/w3YP7eNQvvmOuw7+wbUtXpR9n7pelo3Fc9k7uhKqO36VObn/qdr+4+Yb7ly\n1IB+FWpz9g+zl9OPJ8x+TMnO7gHnctL61qx6t/zVlfrliZeqbepLkqTf/uWlWdvHxV+jN/3X2Exb\nyc7urKzk3P7Ezf326wXrL0excxI/3nzr0pofzM4CnvWm5ypqp9D6ePsdXQm1r2keEAdO6ugt2rdC\n7bb/ZHTWtnMvfHXA9p9fcJvm/mMyszzv6y9nrd///NisOufP2FVx7CjXsAv6AIDGIegDQEAI+gAQ\nEII+AARk0DNy2799VsEsxFLZb8UyRYsplpFZTvZqoW2LlVcypV4lU6hVM61eOcdV6VR65WQWlttW\nJVPclfMaVDo2CmVTxusolcGaK19/y80uLZXZWuy9UklWbKFM4GLHWOg9W+6YyLe+UJuljjmfUhm/\nlSj0nsk3voq9D8t9TYod57Yzv0pGLgCgcgR9AAgIQR8AAjLoQT83QSeegPTGdf33vT4w/74B+65d\nvCqrnrEt+8pKYIgnYOSblm7mVaML9q9QPbnHlDud38fednfWfqX6V+4UapVOz5ZbXui40kY90pq3\nvFAduX3/xTsvyzzPl0j3H8ddm3n+9Xf8d1Z99lirDuxuKXpMcz/5SOEpILen+n7IHa0a/cA4zbx6\n9IDtcuUeU/z1P+zi/UWndYz3Pz2OOw9LasqkvVn9Su8779/2Dqjji8fcmv9YSvS3YALg9oGJWPF9\n8yX1JTu71fxw/gS3eFJZvO10+cyfjh7Qh3F3jyuaoFjofZQ275t/ztuHeHLTsqN+r46uhG498Qfy\nnS1FpyGsNtmsddxrefdPt3X4V/qnC537Dw9nbeNumeezLzMVkuzs1qTbU6/Zm6/rnzoy3la9k7S4\n0geAgBD0ASAgBH0ACAhBHwACMqjJWWb2sqSHBq3B4e2Nkp4Z6k4ME5yLfpyLfpyLfvPdfWI9Kmqu\nRyUVeKheWWUHOzPbyLlI4Vz041z041z0M7ON9aqL2zsAEBCCPgAEZLCD/qrSmwSDc9GPc9GPc9GP\nc9GvbudiUD/IBQAMLW7vAEBABiXom9mpZvaQmfWa2XmD0eZQMrNDzWy9mT1gZveb2cqofKqZ3WZm\nyehxSlRuZvb96PzcZ2bHDu0R1J+ZNZnZZjNbFy3PMbMN0TFfY2ZjovKx0XJvtH72UPa73sxsspld\nZ2YPmtlWM1sc6rgws89F748tZvYzM2sJZVyY2Y/NbLeZbYmVVTwOzGxZtH3SzJaV03bDg76ZNUm6\nVNJfSzpS0plmdmSj2x1ifZI+7+5HSlok6Z+iYz5P0u3uPk/S7dGylDo386KfFZIuG1jlQW+lpK2x\n5YskXezucyU9L2l5VL5c0vNR+cXRdiPJJZJucfcjJP2FUuckuHFhZjMlfVbSQnc/SlKTpI8onHHR\nLenUnLKKxoGZTZX0NUknSDpe0tfSvyiKcveG/khaLKkntny+pPMb3e5w+pF0o6RTlEpMmx6VTVcq\nb0GSrpB0Zmz7zHYj4UdSezSI3y1pnSRTKummOXeMSOqRtDh63hxtZ0N9DHU6D5MkPZp7PCGOC0kz\nJT0haWr0Oq+TtCSkcSFptqQt1Y4DSWdKuiJWnrVdoZ/BuL2TfnHTtkdlQYj+DD1G0gZJbe7+ZLTq\nKUlt0fORfo6+J+lLkg5Ey2+Q9IK790XL8ePNnIto/YvR9iPBHElPS/rP6FbXj8xsvAIcF+6+Q9K/\nS/qTpCeVep03KcxxkVbpOKhqfPBBbgOZ2QRJ10s6291fiq/z1K/mEf/VKTM7TdJud9801H0ZBpol\nHSvpMnc/RtJe9f8JLymocTFF0ulK/SKcIWm8Bt7uCFYjx8FgBP0dkg6NLbdHZSOamY1WKuBf7e43\nRMW7zGx6tH66pN1R+Ug+RydK+oCZPSZprVK3eC6RNNnM0v8GJH68mXMRrZ8k6dnB7HADbZe03d03\nRMvXKfVLIMRx8R5Jj7r70+6+T9INSo2VEMdFWqXjoKrxMRhB/x5J86JP5cco9WHNTYPQ7pAxM5O0\nWtJWd/9ubNVNktKfsC9T6l5/uvxvo0/pF0l6MfZn3kHN3c9393Z3n63Ua/8rd/+opPWSzog2yz0X\n6XN0RrT9iLjydfenJD1hZvOjopMlPaAAx4VSt3UWmdm46P2SPhfBjYuYSsdBj6T3mtmU6C+n90Zl\nxQ3SBxZLJT0s6RFJXxnqD1AG4XjfpdSfZvdJSkQ/S5W6B3m7pKSk/5U0NdrelPqG0yOS/qjUNxqG\n/DgacF46Ja2LnndIultSr6SfSxoblbdEy73R+o6h7nedz8ECSRujsfELSVNCHReSLpD0oKQtkq6S\nNDaUcSHpZ0p9lrFPqb8Al1czDiT9fXROeiX9XTltk5ELAAHhg1wACAhBHwACQtAHgIAQ9AEgIAR9\nAAgIQR8AAkLQB4CAEPQBICD/D6dLkr+Yz5X4AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JaiuxkypB7L8",
        "colab_type": "text"
      },
      "source": [
        "#### Dealing with ill-specified intercept models\n",
        "\n",
        "The above Random effects design matrix has multiple intercepts and is therefore of less than rank $q$; to resolve this we transform $Z$ by a matrix $c$ such that it's now has ($q-\\sum_{i=1}^r\\mathbb{1_{[\\text{Factor i has an intercept}]}}$) orthogonal columns (where $\\mathbb{1}$ is an indicator function). This is achieved in the below code by having $c$ as the unique matrix which subtracts, for each factor, the intercept for the last level of the factor away from the other levels of said factor. This may be clearer to understand in matrix format:\n",
        "\n",
        "Example:\n",
        "\n",
        "$$\\begin{bmatrix} z_1 & 0 & 0 \\\\ z_2 & 0 & 0 \\\\ z_3 & 0 & 0 \\\\0 & z_4 & 0 \\\\ 0 & z_5 & 0 \\\\ 0 & z_6 & 0 \\\\ 0 & 0 & z_7 \\\\ 0 & 0 & z_8 \\\\ 0 & 0 & z_9 \\\\ \\end{bmatrix} c = \\begin{bmatrix} z_1 & 0 \\\\ z_2 & 0 \\\\ z_3 & 0 \\\\0 & z_4 \\\\ 0 & z_5 \\\\ 0 & z_6 \\\\ -z_7 & -z_7 \\\\ -z_8 & -z_8  \\\\ -z_9 & -z_9  \\\\ \\end{bmatrix}$$\n",
        "\n",
        "In this example, $c$ would be given by the matrix:\n",
        "\n",
        "$$c = \\begin{bmatrix} 1 & 0 \\\\ 0 & 1 \\\\ -1 & -1\\end{bmatrix}$$\n",
        "\n",
        "In this code I have made the assumption WLOG that the intercept for each factor, should it be included, is the first parameter listed for each factor.\n",
        "\n",
        "UPDATE: SCRAPPED - RUINED BLOCK DIAGONAL STRUCTURE OF Z"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9Y29RLDB7af",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#import numba\n",
        "\n",
        "#@numba.jit\n",
        "#def obtainC(nlevels, nparams, interceptIndicator):\n",
        "  \n",
        "  # Columns per factor\n",
        "  # This array tells us how many columns each factor has in Z.\n",
        "  # i.e. colsPerFactor[k]=4 means the kth factor has 4 columns\n",
        "#  colsPerFactor = nlevels*nparams\n",
        "  \n",
        "  # List of first column of Z for each level of each factor (If an intercept has\n",
        "  # been included for factor k, then it is assumed that the intercept is in the \n",
        "  # first column of each level of factor k; therefore this list contains all \n",
        "  # intercept column indices by assumption). One additional entry is included at \n",
        "  # the end which is the number of columns in z; this can be useful looping.\n",
        "#  interceptCols = np.insert(np.cumsum(np.repeat(nparams, nlevels)),0,0)\n",
        "\n",
        "  # The below array marks the first entry for each factor in interceptCols\n",
        "  # i.e. interceptCols[factorChangepoints[k]] is the first intercept for\n",
        "  # factor k\n",
        "#  factorChangepoints = np.insert(np.cumsum(nlevels),0,0)\n",
        "\n",
        "  # Loop through each factor\n",
        "#  for k in np.arange(len(nlevels)):\n",
        "  \n",
        "    # If we have an intercept for this factor work out which\n",
        "    # columns hold the intercepts\n",
        "#    if interceptIndicator[k]==1:\n",
        "      \n",
        "      # Get the intercept columns for factor k\n",
        "#      interceptCols_k = interceptCols[factorChangepoints[k]:factorChangepoints[k+1]]-interceptCols[factorChangepoints[k]]\n",
        "      \n",
        "      # Get a vector with -1 in each position for all intercept columns\n",
        "#      negativeCols = np.zeros(colsPerFactor[k])\n",
        "#      negativeCols[interceptCols_k] = -1\n",
        "      \n",
        "      # Get an identity matrix which is number of columns for factor k by number\n",
        "      # of columns for factor k in size. This will become c for factor k\n",
        "#      c_k = np.eye(colsPerFactor[k])\n",
        "      \n",
        "      # This is the index of the column we are removing from Z with this transform\n",
        "#      toRemove = interceptCols_k[-1]\n",
        "      \n",
        "      # Place the negative Cols matrix in the row corresponding to the intercept \n",
        "      # column\n",
        "#      c_k[toRemove,:] = negativeCols\n",
        "#      c_k = np.delete(c_k,toRemove,axis=1)\n",
        "      \n",
        "#      if k == 0:\n",
        "        \n",
        "#        c = c_k\n",
        "        \n",
        "#      else:\n",
        "        \n",
        "#        c = scipy.linalg.block_diag(c, c_k)\n",
        "      \n",
        "#  imshow(c, \\\n",
        "#     interpolation='nearest', aspect='auto',vmax=2,vmin=-1)\n",
        "#  plt.colorbar()\n",
        "\n",
        "#  return(c)\n",
        "\n",
        "#t1 = time.time()\n",
        "#c=obtainC(nlevels, nparams, [1,1])      \n",
        "#t2 = time.time()\n",
        "#print(t2-t1)\n",
        "\n",
        "#t1 = time.time()\n",
        "#obtainC(nlevels, nparams, [1,1])      \n",
        "#t2 = time.time()\n",
        "#print(t2-t1)\n",
        "\n",
        "\n",
        "#print(np.cumsum(nlevels * nparams))\n",
        "\n",
        "#nparams = [2,2]\n",
        "#nparamstmp[1] = 3\n",
        "\n",
        "# List of columns of Z containing intercept\n",
        "#interceptCols = np.insert(np.cumsum(np.repeat(nparams, nlevels)),0,0)[:-1]\n",
        "\n",
        "# The below array marks the first entry for each factor in interceptCols\n",
        "# i.e. interceptCols[factorChangepoints[k]] is the first intercept for\n",
        "# factor k\n",
        "#factorChangepoints = np.insert(np.cumsum(nlevels),0,0)\n",
        "#print('factor change points')\n",
        "#print(factorChangepoints)\n",
        "\n",
        "# Columns per factor\n",
        "# This array tells us how many columns each factor has in Z.\n",
        "# i.e. colsPerFactor[k]=4 means the kth factor has 4 columns\n",
        "#colsPerFactor = nlevels*nparams\n",
        "#print('colsPerFactor')\n",
        "#print(colsPerFactor)\n",
        "\n",
        "#k = 0\n",
        "# Intercept columns for one factor\n",
        "#print(interceptCols[factorChangepoints[k]:factorChangepoints[k+1]])\n",
        "\n",
        "#print(interceptCols)\n",
        "#print(len(interceptCols))\n",
        "\n",
        "#print(Z[:, interceptCols].toarray())\n",
        "\n",
        "#print(np.cumsum(nlevels))\n",
        "\n",
        "#print(nparams)\n",
        "      \n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_chXrSrlWjra",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "#imshow((Z @ scipy.sparse.csr_matrix(c)).toarray(), \\\n",
        "#     interpolation='nearest', aspect='auto', vmax = 2, vmin = -1)\n",
        "#plt.colorbar()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tfRwOu46X44A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "#imshow(((Z @ scipy.sparse.csr_matrix(c)).transpose() @ (Z @ scipy.sparse.csr_matrix(c))).toarray(), \\\n",
        "#     interpolation='nearest', aspect='auto', vmax = 2, vmin = -1)\n",
        "#plt.colorbar()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9umWv0H7XGAV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#imshow((Z).toarray(), \\\n",
        "#     interpolation='nearest', aspect='auto', vmax = 2, vmin = -1)\n",
        "#plt.colorbar()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nBtxdq-gXtI3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#imshow((Z.transpose() @ Z).toarray(), \\\n",
        "#     interpolation='nearest', aspect='auto')\n",
        "#plt.colorbar()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ENn5FJVkU2ie",
        "colab_type": "text"
      },
      "source": [
        "#### Demean and rescale Z"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-RivNVCyU21E",
        "colab_type": "code",
        "outputId": "36802385-a019-477e-c226-13e6fcf1584b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "def demeanAndRescaleZ(Z, nparams, nlevels):\n",
        "  \n",
        "  # List of columns of Z containing intercept, assume if there is an intercept;\n",
        "  # that it is in the first column of each level of each factor\n",
        "  interceptCols = np.insert(np.cumsum(np.repeat(nparams, nlevels)),0,0)[:-1]\n",
        "  \n",
        "  # Make demeaned and rescaled Z\n",
        "  Z_dr = Z\n",
        "  \n",
        "  # Loop through each column\n",
        "  for i in np.arange(Z.shape[1]):\n",
        "    \n",
        "    # Check if column is an intercept\n",
        "    if i in interceptCols:\n",
        "      \n",
        "      # If the array doesn't contain only zero and one\n",
        "      if not np.array_equal(np.unique(Z[:,i]), np.array([0,1])):\n",
        "        \n",
        "        Z_dr[np.nonzero(Z_dr[:,i]),i] = (Z_dr[np.nonzero(Z_dr[:,i]),i]-np.mean(Z_dr[np.nonzero(Z_dr[:,i]),i]))/np.std(Z_dr[np.nonzero(Z_dr[:,i]),i])\n",
        "                                         \n",
        "    else:\n",
        "      \n",
        "      Z_dr[np.nonzero(Z_dr[:,i]),i] = (Z_dr[np.nonzero(Z_dr[:,i]),i]-np.mean(Z_dr[np.nonzero(Z_dr[:,i]),i]))/np.std(Z_dr[np.nonzero(Z_dr[:,i]),i])\n",
        "        \n",
        "  return(Z_dr)\n",
        "nlevels = np.array([20,3])#])#\n",
        "nparams = np.array([2,2])#])#\n",
        "Z_dr = demeanAndRescaleZ(Z.toarray(), nparams, nlevels)\n",
        "\n",
        "imshow(Z_dr, \\\n",
        "     interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.colorbar.Colorbar at 0x7f9e1add97b8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAD8CAYAAACxUoU3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuQXNV9J/Dvb3pGM6MR0gwaSegx\nIGzJ4uWADQHHeBPMwwGCo/XGxtheBwgp7VaZxFnHZYOJ197EroLyVmxqcRHPGmJIsZYxMStiaw0Y\ngwHbCCQQDwkEQqPH6D2ahx7znvntH/fKGdS/I/Xtvvf2ube/nyqVpk933z490/3r07/7O+eIqoKI\niPxWV+0OEBHRiTFYExFlAIM1EVEGMFgTEWUAgzURUQYwWBMRZUDqwVpErhSRTSKyWURuSfvxiYiy\nSNKssxaRAoA3AVwBoBvACwA+paobU+sEEVEGpT2yvhDAZlXdoqqjAFYCWJ5yH4iIMqc+5cdbCGDH\nlMvdAC6aegMRWQFgBQBIU8P5jQvbiw4ihwrmwSemxdVNIkpCXeNEUZsO2u9ntZudRru7e1R1Tjn9\nOuqPP9yiB3qL+2hZ98rIo6p6ZSWPF0XawfqEVLUTQCcANC9ZoIu/taLoNsPbTjLvK5w5T5SqxsWH\nzPa6OvvNOLRlZmJ96frCF7dVeowDvRN4/tFTS7ptYf5bxSPJBKUdrHcC6JhyeVHYFknLTjt7M7hg\nsrxeEVFZRrbaA6c4uD4ICr+eldhjKoBJ+BlH0g7WLwBYKiKnIwjS1wH4dNSDHF5sf02pG5WKOkdE\n/mvdMp7YsRWKMS0tDZK2VIO1qo6LyM0AHgVQAHCvqm6IfCDGZKKatee6YfuKh+M5PkfWIVVdDWB1\nJceY3m2feRie4+cvmYhitKUlsUMrFBOeLhvt3QnGUrRtsr+m7J7DITdR3rXsSvZ9PgkG69g03bzL\nvmL9wnQ7QkSp6z8ryZw1MMFgHZ/N2+ea7RHLMokog6b1JvtO58g6Rmcu3m22v9lbWn0kEVWHVY6n\nL9mleCNwlAU2JxdMFcAYc9bxef0tO93BkTVR9tS9f8C+oiu5Gm4XhTINUo7JkYJZdO8Kytan9ohr\nxpRjhhURpWv8Ncd7tKUK71EFJjwNDV4H6zjoKXZNpuxrTLknRPnjmmXY0jRqtve+Mdu4sT/RMZjB\n6Kf8B+tJu8yHRX5ElXNNNx+J4diuD4Kh7iTTI4IJT6ND7oO1HOBSfER50rwnuZWdgxOMDNZVMTnD\nnkBTcCyzSkTJsUbLk684qkFci0S1J5eoCOqsGayronl7g9k+2uZrZooov8wAPNN+LzrTIEeSPd80\nyZF1dTQcsdtH29LtBxHFY/ZTdrCueDFrcGRdVRMs+iDyhpkGeTVaGmTk7CQnxQgm0t9HvCT5D9ZN\n1e4BUX65UhWu3ZzMAHySXylJpkGqZGzpkH3FHg65iSrlGv3GEe5cHwRNjya3NZhCMBp188eU5D5Y\n/5dznzbbv7fnipR7QpQ/roA6uM9ec7owWHqKwZkGWZbs2iCTMaVBRKQJwNMAGhHE2odU9WvlHi9X\nwdr6435vqx2UnS+ynulme+Gwn5+2RD4qzLJnMGLQ/7xkjCcYRwBcqqqHRaQBwLMi8v9U9blyDpar\nYB2H5tl22mT08IyUe0Lkv6gb5lqDpNHNdlpD69Ofhq4qmNB4RtaqqgAOhxcbwn9lPykG62Ncctpm\ns/2xbeel3BMi/7m+oU57wq7wOGQte1qFoHw8kzGW7olIAcA6AEsAfFdV15R7LAbrYzzx6PvsKxJc\nQ5coq5x55XdXXuHh+iAYSXDp1OAEY8lhsV1E1k653Kmqne84nuoEgPNEpBXAwyJyjqq+Vk7fGKyP\nMdZhL0FT18M1RogqZQVg13kiZ4olwcq6iCcYe1T1gpKOq9ovIk8CuBIAg3UsBuzp6URULOroN8r6\n9NUyEVOdtYjMATAWBupmAFcAuKPc4zFYH0OnOdIdY34WyhN5ab5jkVTP5zfEPINxPoD7wrx1HYAH\nVfWn5R6MwfoYdUN+TjUlypL3LNhrtr+5x/99UifjqwZ5BYDjJFh0DNbHaNpn/6FGElyWkShvun61\n2L7Cs6nlxwoWcvJzwMZgfYyWXXYaZKQ95Y4QZYDzJGAMQdmVD9cE1+5QCMY43Twbhua5Xggs3SPy\nwZI5PWb7phiOrYrYJsXErWaDtXNEMC/aQuiuT/nRbZzxSHQs6300vD3CCn0ANkacNRmNxDopJk41\nG6zjoq84VgCb5XdujqgarADsCo3O2ZH19lZ9cVBwZJ1bwwvGzfbCET//4ERZ1/jj1kSPzxOMObVg\nsZ0/27thbso9IaoNvWcle4KRmw/kVO8he91eoloQdacYiXCe3nleqTHZ9azHSl8bJFV+9ipDpj1r\nv6DGTmXOmmrXZLP9+o+y+UB1CDfMzauJ5mr3gMg/Jy/qN9sH3jw55Z5Eo4hvBmPcGKwrNG3A/ko2\nPCfljhBVgXOJ1BiO7UqxDO1Mtiw2dyNrEekAcD+AeQg+kDpV9U4RORnAjwAsBrAVwLWq2iciAuBO\nAFcDGARwg6q+WFn3q298OifREKVJT7IrsGI5tkouR9bjAP5WVV8UkZMArBORxwHcAOAJVb1dRG4B\ncAuALwO4CsDS8N9FAO4O/880Se51Q5Q71mi58Bt7V5lBa1cZJLukanCCMWfTzVV1N4Dd4c+HROR1\nAAsBLAdwSXiz+wA8hSBYLwdwf7gv2XMi0ioi88PjZNa4vW46ERnMtMmCaLOGm1c7JqLFIr49GOMW\nS85aRBYjWApwDYB5UwLwHgRpEiAI5Dum3K07bHtHsBaRFQBWAEChrS2O7sXCWUbkWI3PeqHVP22P\nII6wcoQyyhVQR99ybILbUHp60JkPPyvZ0r3c1lmLyAwA/wrgb1T1YJCaDqiqikSprATCPcw6AaCx\noyNXid9D7x8227llGOXN+Nwxs73Q539NQy5nMIpIA4JA/YCq/iRs3ns0vSEi8wHsC9t3AuiYcvdF\nYVvNqN9p75IxmWCRP1GSXKPfOLK+rlH78ne9araXvV/WFLmcwRhWd9wD4HVV/ccpVz0C4HoAt4f/\nr5rSfrOIrERwYnEg6/nqqBqWHTTbnSkWIs+5AmrdGjvlN+RY1dLiel88uPWDjnv8xNEeTYQNc1NV\nycj6YgCfBfCqiKwP276CIEg/KCI3AdgG4NrwutUIyvY2Iyjdu7GCx86kk5rt6tMRx1lvIt/FsdRw\n02N2fnvgPel/41QFxiZzFqxV9Vm4Vze8zLi9AvhcuY+XBwtmDJjtPeA2NJRNzrVBdpS+RvVIFYKy\nS5AGyVmwpujWv/wus93PlwbRiblG1lHWqL7stDfN267+1fnldqsicc1gdE0cLPd4DNYpWrB0v9m+\nh8upUo2wgvvqrXZQdo3aXWmTOMRcumdOHFTVjeUcjME6RXtfnmdfUe/P10CiJFkBuPnndvDtd5zL\nSTZtEl8a5DgTBxmsfTfv3L1mO0fWVCvMnPUZdvB1jazHx5OdDh5hD8Z2EVk75XJnOE+kyDETB8vC\nYJ0Q60W5xzFScL0oR0Ya7IPvbiq7X0TVZFaDPOqoBqlClVRQDVLyh0GPql5wohsdO3Gw3L4xWHtM\ndtiLZSvTJpRR5sh6WbSR9eTLdg13HOKeFOOYOFgWBmuPuWY2SnKbOxMlygrAg/vsrfGcNdyzkl1L\nJ0Ia5LiOM3GwLAzWHjtpi32i4/BpXPiJsskKwK6kg2tkvajVnq/QVW6npoi5GsScOKiqq8s5GIO1\nx4YuPmxf0c11WckPzt1cdtm7udSNlh4IXSPrtxPOZcdYDXK8iYORMVh7LOKChUSpcwXUOMKd64Ng\nsDe5wYqqYJwzGCkq2eAYQcxkGoSyyQrAw4P2EsFJruh3PLlbdY+SN3kW0yCUf4UddilqNZYOzvXm\nA5Sc8xftMNuf716Wck+IkjN3nf1Ncc8HqxM0GawpsjVdi812P19KRCdmThZzLE/tylm3tgyZ7fFU\ng+Rw8wFKXvP0UbN9GPaOM0RpcwXU+mcc+412VL75wN6kq0E8HQ4xWHsg6k4xztXIptn73g28eXLk\nPhFV4vD59uhX9vk90FAFxvO2+QD5p3/Angnm5ziB8iDqetZRuAYlc046YrbHkQYBmLOmFDRtsNcS\nGWlnqR/5wQrAjQ3j5m0PvtVmtncnmAZhzppSMfMP7SVY92+ck3JPiGzmQk6O2zpXo9yS3OYDQDAx\nxkcM1jmyf4MjKPv52iMqiyT8RZEnGClx9YP2i2y8hdPWKT9mnNGX2LFVmbOmFMyw59Cg/4x0+0GU\npIGtrQkeXTDBahBK2qFTq90DouOz8tB1dfY3vyFHbjrpUMqcNSVupqN2qe/sdPtBFMXhvfZyqkkv\n2GTh2iCUisNXORZ+2m7XXxOlLY7NBwb77RLVWGiQt/YRg3WOTH/cHqEMOPa4I8qiUx+2EyHbYzo+\nq0EoNq5ZY1E2Hp3+Mzsf2Hc2Azv5redGewYj/q3yYytPMJJvBj7ieMHv5FrZVDpnqmKvnXorDJUe\nCKOumRMXpkHIKx3/u8Fs33Z1yh2hTIu6m4sV3KsVlF3iqgYRkXsBXANgn6qeU+nxGKxr1K4PuVY/\n4zoilJwogdk1atcX7eVX46Aaa+neDwDcBeD+OA7GYF2jxps9/a5HmeJMgwzYFRuFvtJDjjOwn5zs\ngCKu0j1VfVpEFsdyMDBY1yw5bdBs110JlkVR7iS5qa1zZL0+uZE1wJw1eWbStemuYzYZkcUVUIeH\n7B3LZW/pmw84R9atyY2sFYLJ0qtB2kVk7ZTLnaramUC3ADBY16zJRvsFXzfmZ40p+Snq5gNmcF9n\nj5RHZlfn/EmE4UqPql6QXE/eqeJgLSIFAGsB7FTVa0TkdAArAcwGsA7AZ1V1VEQaESTazwdwAMAn\nVXVrpY9P5akbcYweOLKmBJnB3RGUXaP2jrZ+sz2WnWLiPcEYqzhG1p8H8DqAo7Ms7gDwbVVdKSL/\nBOAmAHeH//ep6hIRuS683SdjeHwqw2STY2Q96ucLlfzkCqhXn77RbH/4yYtKPrZr1L456VK/mMYr\nIvJDAJcgSJd0A/iaqt5T7vEqCtYisgjAnwD4JoAviIgAuBTAp8Ob3Afg6wiC9fLwZwB4CMBdIiKq\nvqbzc87PSVqUMa6A+vDW0oOyi+uDoG5N0icYY6sG+VQsBwpVOrL+DoAvAb/bFG02gH5VPbqpWjeA\nheHPCwHsAABVHReRgfD2PVMPKCIrAKwAgEKbvQcbRWO9oVyx2vUGGRuzz+9PcsYjpazhQwfsK+6o\n/NgKYHLSz2+XZQdrETk6M2ediFwSV4fCs6mdANDY0cFRtyfGhuwZj9VYxpJq2/Co/VqMhQLIYc76\nYgB/KiJXA2hCkLO+E0CriNSHo+tFAHaGt98JoANAt4jUA5iF4EQjZUBdA2c2kh9Gt9mrS8bF18Rs\n2cFaVW8FcCsAhCPrL6rqZ0TkxwA+jqAi5HoAq8K7PBJe/m14/S+Zr86O079vjza2fjTljlCmWWk2\nV0121dYM8TQqJVFn/WUAK0XkGwBeAnD07Oc9AP5FRDYD6AVwXQKPTQnpvtQ1mcHTVzalwnWOY2iX\nPfq1ArBfSQfJdekeVPUpAE+FP28BcKFxm2EAn4jj8Sh9p/x23GzvvpxZayrWst1+XQydkoF0mqfj\nD85gpJIwKFMUw7M9jXgnooDmrRqEaovOHTHbZV/paz1Q7ZhoyfKkKz/7yGBNJZnxor0a35FFGfha\nS4lxnQSMUse/oPWgeduu9QvN9sR5+qWAwZpKMpZstRRllHMS1SZ7j08ruHfBr51iGKwp08ZaPH0F\nU1U5y+saS9+8ed7dTeZtt1+V4OQXl5xOiqEccr75Gkp/8wHA0J7ioXjdsJ9vAkqP9frafpV9W9dr\na+I1bj5AFJuWruLqkaH5zG9T5do2Jfw6YjUI1ZKZl+8pahvaMLcKPaG8OfB7jmD6QDzHF46sqZb0\nPXtKcWMbR9a1zkptjG8s/WQkAGdKLhYKnmCk2jLSXhyYZaIKHaFEufLKo29FCMDTfYqOwhOMVFvE\n2suRW4bVjIYljiCe8Ip5sfD0ZcpgTYmYmFW8lkjhEKes14qF37FDS9fHUu5IOTzN1jFYUyJmbiyu\nkT3S4em7gGLXsGGbfcXHlqXbkahYZ021ZsKe50A1QhfOq3YXysZqEKopR941VtTGNEjt2PwVx6f1\n7nT7UZb4dje/EsHuWQUA31fV2ys5HoM1JWLOb4pfWr3v9XTIQrGb6LNXY3Qt8JQ3IlIA8F0AVyDY\nOPwFEXlEVTeWe0wGa6qIqxZ2xAjMrjKv+qft6cNHTmWOO6s+9oEXzPZVTxXtS+KdmNIgFwLYHG7G\nAhFZCWA5AAZryq6DZ9q70BSO1Mo4LLtcH9artlYelF0f7oMHpld8bCdFlOnm7SKydsrlTlXtDH9e\nCGDHlOu6AVxUSdcYrKnqmrvtl+EoZzySoa7Z/nCPTekj6x5VvSDBnrwDgzVV3cwuOyj3tKXcEYrM\nNfrV9XZqa7S19A9g16g96cK6mNIgOwF0TLm8KGwrG4M1Vd3gJwbsK95OdilMSs54c4ZPJsfT9RcA\nLBWR0xEE6esAfLqSAzJYU9Ud2WoHZT+nJlAppp1hb9U13OXZrjCWGIK1qo6LyM0AHkVQunevqm6o\n5JgM1lR1DQP2icTxGcxZ+865Mp6Duere6/aiTxNVGJ2LxjcpRlVXA1gdz9EYrMkDjef2me3jTIN4\nz7lTULejpNMK7r6lTLj5AJGt+SFHnfX7Uu4IRRZ1d/MonKV7+1tiOLobp5sTOfQtc41kPH3XUFU1\n70g4bHn6smOwpqobnWPvSsBJMWQZXjac3MFjzFnHjcGaUuP6yuxa3sn5Nbi/2T5OP1/OaXP+jQ7a\nCzkVeouXznVJMsVyXAzWRJQ3UT+Ao3CevNyZ7G4z4mkREoM1Zc7Chb1m+55+7p7uOysAj47Yo+2q\njaw9xWBNmdP3tLFzOgDM9nRIRL8TpS7bNbLGuoRLOpkGIYpH3Wi1e0DlsgJw4Td28B2EI7An+aHM\nE4xE8akr3oSGMsIcWS+wg69rZD08NC3OLhVjsCaKx8FljvWvh2o1m5kdVgAe2mOfMKzWqnsM1kQx\nmfucXWtw4FxP32X0O1YAdn3EOkfW25NbDEqQ02oQEWkF8H0A5yD4PPoLAJsA/AjAYgBbAVyrqn0i\nIgg2j7wawCCAG1T1xUoen2pTyy47aX3g3NJreCm7JpuZsy7HnQB+rqofF5FpAKYD+AqAJ1T1dhG5\nBcAtAL4M4CoAS8N/FwG4GxVuc0O1af959masgKdDIopXXcLRNG/BWkRmAfhDADcAgKqOAhgVkeUA\nLglvdh+ApxAE6+UA7ldVBfCciLSKyHxVzcLm9OSRCz7+qtn+zK/PTrknFJWV2hh7y14iNckJN8eV\nt2AN4HQA+wH8s4icC2AdgM8DmDclAO8BMC/82dpAciGAdwRrEVkBYAUAFNq4r1Mtc71Zn9lqB2Xn\n7unPFJeGHengKLwazL9pgx0dXX/P0bft4B6XPKZB6gG8H8BfqeoaEbkTQcrjd1RVRaI99XB34E4A\naOzo8PTXRlnSvL84MB/pMG5IkbkCakuTfV6h943ZJR/bOYGmwDRIVN0AulV1TXj5IQTBeu/R9IaI\nzAewL7w+9g0kiUqx//eL28Re6I8icgXUkRiO7fymVEj4BKOnX7rKDtaqukdEdojIMlXdBOAyABvD\nf9cDuD38f1V4l0cA3CwiKxGcWBxgvprSMNlSHJkLBxPPfJLBrLPeFa3OOo4PguPK4cgaAP4KwANh\nJcgWADciKJt8UERuArANwLXhbVcjKNvbjKB078YKH5uoJO9aWTxU2nY1g7Uv2hbb27oNvHlyyj0J\npJGzFpFPAPg6gDMBXKiqa090n4qCtaquB3CBcdVlxm0VwOcqeTyicvQus0r9PP2um3PWaNk1Unal\nQd43386edpXbqWOlM7J+DcB/AvC9Uu/AGYyUeweXFAdmX8/4Z40roE6+bC/ONDar9A9JVxrkua1n\nlHyMyBSpBGtVfR0AgrmCpWGwpty75IOvFbX96tfnVKEntaNltx3x+j3fsF4Q6YO8XUSmpi86w2q2\nRDBYU+69fM97ixvP4NA6Ds6TgDH8fp1bhg3Y27rFJUKw7lFVKw0cHEfkFwCsxddvU9VVRvtxMVhT\n7k00WF81GayrwQrArkzAcFe2ZzCq6uXxHCnAYE25d+h0BmZfxLFTzIgjiMfG05cLgzXlhjMQGAv/\nuALB2Kj9lpjclexX71ph/d6bptm7SThL95Jc0DqlVfdE5GMA/heAOQB+JiLrVfWPj3cfBmuiKcZ7\nmsx2bmuQnJFn2+0r5lapvDKdapCHATwc5T4M1kRTFGbbVb+62w7iFI357ccRlF3ffoYOu5bIjUfu\nppsT5dFEjx0IOLK2ORdy+jd7Zbze95Y+bHWltZL+W/hag89gTURlc5buRQjKLq4PgkSlNCmmHAzW\nRFMsWTlstm/5M55gtDhTFYccuf8DpW+9FqVyJFYM1kT+2/wZR5CxY3jNi5qqsIL74EH7d17oTX9P\nzYgzGFPFYE00Rcs2e8rF0DxPzzpVmWtkPb6x9K26fFv/UCb9jNYM1kRTjM7y842aNQ1H7GLoieme\n/36ZsybKhvnPjpvt3Zf7Nv7zgzOvPKf0cryJDfYofLylOlGTaRCiDOg9y5UnZRokDmZwdwRl5yxT\nx27osWGwJvKHc0S4INoEjfZ/bjHbd364NiqzXb+XxsftgHpwaeV11q7d0OPCkTVRDo3f3GNfsWFu\nuh2pEmeddYSg7OKss16b8KLYDNZE+XNwiNPQ0zacZGVOHnc3JyJgZIRvobRNzrRPAseBddZEeaVJ\nrtfpP+ea0zHMPnQdI/nNB/yM1gzWRBUovGGfYJyc6el36ZglOSXc9UHQ8Ktkc9YcWRPl0PACe+H8\nwmHWZSdl5D84TjzeGcPBOSmGKJ9OXjBgtjt3Oalx1mh5dHPpU9PTwBOMRDnU19VmttdGlXV0ZgCu\njzYpxrXBblwYrInyqMZPMEZlBeAoiz4lTsETjERZFsdSoAAw1F18nLrkKtG8Y/4eHYs7uX6Hs35s\n/y26yu7VO/EEIxFhWn9xeB+f4en37gSY61n32Rs7uD4g9/2+4+A/LLdXx0hnd/NvAfgogFEAbwO4\nUVX7j3cfBmuiFDX2FbeNz0i/H9USZT1rZ876heRK91KcFPM4gFtVdVxE7gBwK4AvH+8ODNZEKTr4\nnuKcR2GIpyOjOLI4wbyRaiqbD6jqY1MuPgfg4ye6D4M1UYraXygeR/ad42mStASu0W/dGnv0G2XH\nnerNYCz5lu0isnbK5U5V7SzjEf8CwI9OdCMGa6IUjc6yqkeyG6xd6i/uta/Y3JpuR8oQIQ3So6oX\nOI8j8gsApxhX3aaqq8Lb3AZgHMADJ3owBmuiFI0mvLqnL/5gwVaz/bHN56XbkagUQExpEFW9/HjX\ni8gNAK4BcJnqiesFGayJUjTng7uL2na9Nq8KPYmHK1Xx2NbKg7IrxXLm3L1me1yleylVg1wJ4EsA\n/khVB0u5T0XBWkT+G4C/RPD0XgVwI4D5AFYCmA1gHYDPquqoiDQCuB/A+QAOAPikqm6t5PGJsubA\nr+YXN87ObumeK6DWF+zndOTt0r9auD4I1ic8WSalapC7ADQCeFyCKZnPqep/Pd4dyg7WIrIQwF8D\nOEtVh0TkQQDXAbgawLdVdaWI/BOAmwDcHf7fp6pLROQ6AHcA+GS5j0+URaPnGIOo3fnbwEB+aU/D\nx2n+fzClVA2yJOp9Kk2D1ANoFpExANMB7AZwKYBPh9ffB+DrCIL18vBnAHgIwF0iIqXkaojyon7T\n9KK2sQwvp+rc1iuGoOwatU+8lmDiP4+r7qnqThH5nwC2AxgC8BiCtEe/qh4thOwGsDD8eSGAHeF9\nx0VkAEGq5B2b2InICgArAKDQ5vh0JvKcc10LIzC7gtLSdnt/x9fWnl52v+Lm6vv0n9rrfUQpU3T+\nDhOc8RlMivEzWleSBmlDMFo+HUA/gB8DuLLSDoV1ip0A0NjR4edvjSgFr7y82Gz3aQqNc2TtCMpW\ncJ/eaK8J3repSsvMevpFp5I0yOUAulR1PwCIyE8AXAygVUTqw9H1IgA7w9vvBNABoFtE6gHMQnCi\nkYgMM049aLYPRjhJlzTXyFrX2320gvtIrD2qXO5G1gjSHx8QkekI0iCXAVgL4EkEUydXArgewKrw\n9o+El38bXv9L5quJ3A5vswOeTyNrl9EzHdVovp9MzWnOeo2IPATgRQQzcF5CkL74GYCVIvKNsO2e\n8C73APgXEdkMoBdB5QgROTT22WF57CRPv6dPMdE/zWz3f7OzdNYGKUdF1SCq+jUAXzumeQuAC43b\nDgP4RCWPR1RLZmy3g0bf2Sl35DiSXL8j6snL+CbF5DBYE1Fyej7k2Iy3r7bftsOzE9ydR7mtFxFF\n1PZ8g9l+cKmfI7+0nP2xN8z2jd+K6QE4siaiKGo9KLvsOJTwyn2e/toZrIk8Nel4d/q0Z6Nzr8md\n9vY3dWOlpzBc+fC9SHhtkEk/8yAM1kSeOvWxCbO9+1L/ayoKw3YlixY8HbYepcjlpBgiioFrBNl9\nqX1712h2cH+L2V44kn5ldsMhewQ92up3sBZoLifFEJFHFr/bXud5xyvGsqwxca7f0ZrcQk6NDQnn\ngRisiShJh//PAvuKDO/xaBlaOzvZB2CwJqIkLf1Lu6Tt+eeWJfaYzpRMb/FSsABQOFh6vt05ap+V\nYFKZOWsiStqrq860r4iwo3hU1ZjBWP90sgtZsRqEiBI1NNfPIDOVFYDr1tjBd8hRojdyapLPU5kG\nIaJkNe23qz6GPQri5kjcMfJ3nmD8hb02SCwUDNZElCzfy+IAx8j6uYgj6yUJP88UPttE5B8QbN4y\nCWAfgBtUddfx7sNgTZQTkw3+B2tzZH1KtJH1+BsJjqyR2uYD31LVrwKAiPw1gP8OIJndzYnIL+o4\nq5fkKnKugIq1jp1i2kvvjLMapCnhYJpCsFbVqdsAtaCEFUkYrIlyonVJr9k+8GZyexk6A6ojKFvB\nfc737TK/7surMK1eFZgo+QOITJLaAAAHZ0lEQVSlXUTWTrncGe4hWxIR+SaAPwcwAODDJ7o9gzVR\nxjg3qXXc3nmi7vHidELUlf6cddYH7e27rL53Xx7pIZNX+si6R1UvcF0pIr8AcIpx1W2qukpVbwNw\nm4jcCuBmFG/k8g4M1kQ1qv+DxeG9br+9HVdUbc/bxzmY9MnBOMSUBlHVUj+GHgCwGgzWRGRp2N5Y\n1DbRHC1QOUf5MQTlqAtWxUIBpLAHo4gsVdW3wovLAdjTT6dgsCaqUdN3Fa+Md+jdyaZBCr327jeW\nJGdHuimgqdSl3y4iyxCU7m3DCSpBAAZropp16EODxY277SAbVVNX8agdAMaSXNcjDoooJxjLfxjV\nP4t6HwZroho1MVCcV45t1PpeR0nf9gRTGHHhDEYi8kqDNYKMtlGBs3TPwUqbDPY4Vug7XKUdcRis\nicgn7accLGrrG0iuJhuwg7srJDv3d9xt7+8YDy7kRESe6XvDCMyl72dbFisAT2ywp4+7Ru2JblKm\nALhEKhFRsZGOUbM9SuVIrDiyJiKfTBprbNSNRBtau1IVp8yy27e9XLz1mF97tUeabp4qBmuinIuS\nTnAF3+af26mKocX2Yw53OvaDvMhu9oYCmk6ddWQM1kR0QpMftReJGnmrzWzf6wjK1ofB5Mv2Cn1V\nq8lOYQZjORisieiEDm+wq0Qa31NcUQIAWOdYItXaUMC3iTLMWRNRVjWd2W+2D77t2Lx2dulLpE68\nZh9jfEYVgrgqq0GIKLuOdNkBNY5KP1e6QyZiOHg5OLImoqyavtuubh5ybHbrYp3sdAV818nOsTeT\n3NZLoRPV+pQ4PgZrIjqhIwvtoNy8yA6oXz1ntdn+dz/9ZMmP6ZzKPi3BkW9KS6SWg8GaiE6oMGc4\n0u2/um55Qj1JQVZL90TkXgDXANinqueEbScD+BGAxQC2ArhWVftERADcCeBqAIMItld/MbzP9QD+\nLjzsN1T1vnifChElZdYTzWZ7/1nxLKlqcaVBOtrsk51dMTymAtAMj6x/AOAuAPdPabsFwBOqeruI\n3BJe/jKAqwAsDf9dBOBuABeFwf1rAC5A8PtYJyKPqGpfXE+EiJIzcs2AfcWWJPPHtrde6Uju4Jra\n5gORnTBYq+rTIrL4mOblAC4Jf74PwFMIgvVyAPerqgJ4TkRaRWR+eNvHVbUXAETkcQBXAvhhxc+A\niBJ3uNdexnS6Y/S7oNWuv+5av7Dkx3TlrBNeayp3Jxjnqeru8Oc9AOaFPy8EsGPK7brDNld7ERFZ\nAWBFePFw1xe+uCn8uR1AT5n9zZJaeZ5A7TzXmnueJ9xQMBmnVXqAQ+h79Bf6UHuJN0/1b1rxCUZV\nVRGJLcmjqp0AOo9tF5G1x9v2PS9q5XkCtfNc+TyzQ1WvrHYfXMpdGnZvmN5A+P++sH0ngKkJpUVh\nm6udiIhKUG6wfgTA9eHP1wNYNaX9zyXwAQADYbrkUQAfEZE2EWkD8JGwjYiISlBK6d4PEZwgbBeR\nbgRVHbcDeFBEbkKwjfq14c1XIyjb24ygdO9GAFDVXhH5BwAvhLf7+6MnGyMoSo3kVK08T6B2niuf\nJ1VM1NN58ERE9O8S3c6MiIjiwWBNRJQBmQjWInKliGwSkc3hjMlcEJF7RWSfiLw2pe1kEXlcRN4K\n/7e34sgQEekQkSdFZKOIbBCRz4ftuXquItIkIs+LyMvh8/wfYfvpIrImfP3+SESmVbuvcRCRgoi8\nJCI/DS/n8nn6wvtgLSIFAN9FMJX9LACfEpGzqtur2PwAwUzOqY5O5V8K4InwctaNA/hbVT0LwAcA\nfC78G+btuY4AuFRVzwVwHoArw6qoOwB8W1WXAOgDcFMV+xinzwN4fcrlvD5PL3gfrAFcCGCzqm5R\n1VEAKxFMa888VX0awLFVMcsRTOFH+P9/TLVTCVDV3UcX9FLVQwje4AuRs+eqgcPhxYbwnwK4FMBD\nYXvmnycAiMgiAH8C4PvhZUEOn6dPshCsS56qnhOuqfy5EK4z8z4Aa5DD5xqmBtYjmCj2OIC3AfSr\n6nh4k7y8fr8D4EsAjq56NBv5fJ7eyEKwrlnhgli5qa0UkRkA/hXA36jqO1b6yctzVdUJVT0PwSzd\nCwGcUeUuxU5Eji6ZvK7afaklWdh8oNamqu8VkfmquvuYqfyZJiINCAL1A6r6k7A5l88VAFS1X0Se\nBPAHAFpFpD4cdebh9XsxgD8VkasBNAGYiWAd+7w9T69kYWT9AoCl4ZnmaQCuQzCtPa9cU/kzK8xn\n3gPgdVX9xylX5eq5isgcEWkNf24GcAWC/PyTAD4e3izzz1NVb1XVRaq6GMH78Zeq+hnk7Hn6JhMz\nGMNP8O8AKAC4V1W/WeUuxWLqVH4AexFM5f+/AB4EcCrCqfxlTM33ioh8CMAzAF7Fv+c4v4Igb52b\n5yoiv4fgxFoBwUDoQVX9exF5F4IT4ycDeAnAf1bVker1ND4icgmAL6rqNXl+nj7IRLAmIqp1WUiD\nEBHVPAZrIqIMYLAmIsoABmsiogxgsCYiygAGayKiDGCwJiLKgP8P4r4jz6zgV/0AAAAASUVORK5C\nYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pzweedO6Yw09",
        "colab_type": "code",
        "outputId": "96b22285-6c9a-444d-a519-fdc14936fdaf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "imshow(Z.toarray(), \\\n",
        "     interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()\n",
        "\n",
        "#Z_orig = Z.toarray()\n",
        "#Z = scipy.sparse.csr_matrix(Z_dr)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.colorbar.Colorbar at 0x7f9e1ad03a20>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAD8CAYAAACB3pQWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuQXOV95vHvTzOjGd1HF3QXSAFh\nB+NgGwKOcTAGJ8GERNktjIkTBwhb7KawjXftMmBvVbJZU4V3U7bZssspLSaBlBNBsCtobTYYsLE3\nsRGIS4wBC4SQkITud2nuM7/945wxDfO+oznT53T36X4+VVMz/XbP6bdnep7zznvei7k7IiJSTlPq\nXQEREZk8hbiISIkpxEVESkwhLiJSYgpxEZESU4iLiJRYzUPczC4zs01mttnMbqn184uINBOr5Thx\nM2sDXgJ+C9gBPAn8obu/ULNKiIg0kVq3xM8HNrv7FncfANYBa2pcBxGRptFe4+dbBmyvuL0DuKDy\nAWZ2A3ADgE2dem7HooVjDtLWFz74cFdOtRSRmrGRcLlnbGIObN+x391PqaYuv/PBGX7g4PCEHvvU\nz/ofcvfLqnm+PNQ6xE/K3dcCawE6T13hS26+aeyDRqzGtRKRoCnh7ljrHgiW+8HOwqqy7VOf3Vbt\nMQ4cHOaJh06d0GPblry8oNrny0OtQ3wnsKLi9vK0LJNZW8On6GMrI6d0ESlGpEGVS1hHThAdh4vr\nBXZghHLlSK1D/ElgtZmtIgnvq4GPZT3I0TOHguU2oBGTIs2u62Bx/4k7zqBPrDulUdQ0xN19yMw+\nATwEtAF3ufvzmQ+k3hSRlnXsXZGLYjlRS/wk3P1B4MFqjjHj1XC1e5aW64cvItlN2T+1sGM7znDJ\nluduuAubE7HgucFg+WtL22pcExGptY6jxXabjqAQL9ziL7wSLH9t45k1romI1Frf4vA1sTw4MKwQ\nL97GrafVuwoiUidtJ8rTEjezbuBO4GySc8SfApuAe4GVwFbgKnc/NNnnKGWI/+bpm4PlP9r/jhrX\nREQyCQwbbI+E8tC08CFGOvKs0Js5MJhvn/gdwD+7+5VmNhWYDnweeNTdb0/Xj7oFuHmyT1DKEH/s\nhbcFyzVoRaR8hpdHRpscKG5iUIzjuXWnmNkc4CLgWoB0qZEBM1sDXJw+7G7gMZo6xAOTCexYpNqB\ns/yUgXC0jzT+KxdpCbYrvF6GT61D37TD8MSfdoGZbay4vTadcT5qFbAP+BszOwd4CrgJWOTuu9LH\n7AYWVVPlpo+y9hUnguUDu2bUuCYiTSgyq7Jtfn+wfHjf2MAeqUdYRyQzNidsv7ufN8797cB7gE+6\n+wYzu4Ok6+SN53N3M6vqB9D0IT6idVZEihP5+wqFdWaRE0Sx/abGcH5PsAPY4e4b0tv3k4T4HjNb\n4u67zGwJsLeaJ2n6EB/ePT18h7JdpJTaCxwnnlzYzCcc3H23mW03s7e5+ybgUuCF9OMa4Pb08wPV\nPE/Th7h3hycGcaTAS9wiEhZoXbf1hEM5trT00Iziul+SceK5tvA+CXwrHZmyBbiOZB+H+8zsemAb\ncFU1T9D0IT5tU/gKd+9iTdEXqblA98twVySUI90pPq3YBapGcmqJA7j7s0Co3/zSvJ6j6UN86tFw\nee/i2tZDRPIxfUuRa6fk3hIvXNOHeGzCgIjUQag7pTfSnRIZJt67qLj/oh1juPb7x1el+UM8cl1T\nRHIQG0ESGxUW6k7pbJwhhpBvd0otNH2ITznnSPiOHbNqWxGRZlTkEN7ICaJrT3GrlTrGgJdrNdSm\nD/F73vM3wfKP7PhUjWsi0oRiFx/bIntvDmboqoicIPpOKbI7BUbUnVJHgV/6R9ZHwjr25psafoNY\nX7nOziJ11RUJ2iwhXie6sFlyMxeGp+mfeG12jWsiUgKR1nKW9Y1sKLLZch3y3t0YrscTV0Eh/hb/\n/lf+LVj+d6/9Zo1rIlICkf9oO/eF/3Ptnz+2hd5omTmilni53fvAReE75jTWFXSRhhBpiYfCOrOs\nI19ykFzYLFcslqu2NTC4KrK28cHar20s0nQCwewdkYug/bVvouvCZhPwQ8XNBhNpOpHWsg1H+rlD\n+wP0N1b3xbDGiZebd2p0iki1RmaFNzO2440dOZqx2QSmnAiHdcnG/4vU1fTu3mB57/HGn2Q30mhX\nWk9CIf4W0/aEf4E9S7XqochEDbwSGZLbYFPs3ypZAEshXmqzt4XDumdpjSsiUgaxnX3yCOtYf3v3\nQPXHjnCMwZL9260Qf4vjS2NnYbXERRrB9Jnh/Tvz4I4m+5RGpAVx/LRIWEdaBdOWHg+W92qBLZGx\nQn9HsUZ75G+02NnTpsk+rWbohcgbanZj9/2J1EWWiTqRhtOU+QW2xFFLvOUMLI4MpeopV7+aSFl0\n/azYTQJ0YbPFLF1xIFi+a9PCGtdEpDUUuxStaVOIVnPouLYOkhZW5PomkWOMFJhaDgxq7ZTW0rYh\n0ie+XKNZpHV5R2Tmc8OvJ25aT7zVxDZzFWllbZGx3CP7umpck2wczdhsOR3HwuV9p9S2HiJ1Eevy\nyCOsY101BWdsni1xM2sDNgI73f0KM1sFrAPmA08BH3f3qmYvTfrHYWYrzOyHZvaCmT1vZjel5fPM\n7GEzezn9PDctNzP7X2a22cx+ZmbvqabijWJoevhDRIrhU0eCH7kc240RnzKhjwm6CXix4vaXgK+4\n+xnAIeD6autczTltCPiMu58FvBe40czOAm4BHnX31cCj6W2ADwOr048bgG9U8dwNY8pQ+ENEAqb4\nmI+Ooxb8YCT8YT1twY88JBc22yb0cTJmthz4XeDO9LYBlwD3pw+5G/iDaus86e4Ud98F7Eq/PmZm\nLwLLgDXAxenD7gYeA25Oy+9xdwceN7NuM1uSHqe0hqbVuwYiJRLofhmMTYyLdKd07S5yDkamPTYX\nmNnGittr3X1txe2vAp8DRqdvzwcOu/toM28HSWZWJZc+cTNbCbwb2AAsqgjm3cCi9OtlwPaKbxt9\nAW8KcTO7gaSlTtvcuXlULx9Zt6EKvAHn/Sz85jh4tmZ3SknFFqkajGwKkSV/I39zfQuLHCdOlnHi\n+939vNAdZnYFsNfdnzKzi3OqXlDVIW5mM4FvA59296PJfwwJd3czy5RQ6ZlsLUDnqSuaKt2OXtIT\nvmOvmvPSXEZmDgfLrbfxZzLnNGPzQuD3zexyoAuYDdwBdJtZe9oaXw7srPaJqgpxM+sgCfBvuft3\n0uI9o90kZrYE2JuW7wRWVHx7Li+gTGxL5IrnzKY6V0kribSWcwnrSCt/6ep9wfJt1T9jbjM23f1W\n4FaAtCX+WXf/IzP7R+BKkhEq1wAPVPtckw7xtJP+m8CL7v7lirvWk1Tudt5cyfXAJ8xsHXABcKTs\n/eFZLTh3T7BcU/SltCJB23Ek3JodnJWhwRI5Qbxe8N9LwRsl3wysM7MvAs+QZGhVqmmJXwh8HHjO\nzJ5Nyz5PEt73mdn1JCfHq9L7HgQuBzYDPcB1VTx3KS2aHh5UvguFuJRUJGijYR0I/a494VZ7kWuk\nxLjD4Ei+Ie7uj5EM8MDdtwDn53n8akan/AtER8VfGni8AzdO9vmawTlzwr1Hz3J6jWsikpPYhJyY\nQOjXI6xjku4UzdiUiL/91/cHy8u1UoNIhawLXQVCf+Hp4ZVA9768YDI1qprWTpGoC9/1UrD8J0++\nvcY1EamTQOhHwzo2TnxvcSNcMg4xbAgK8Rra8ONfDd8xTaNTpEWE+sQjk3di48H7FhTZ/aLuFBnH\nBRe9GCxXS1xaRqhPPDZ5J9bfPnswxwqNpT02JRF4s0bDOvZv45ITwfK+nTMnXS2RusqhJc7hqXnW\n6E2S0SmNPyGpkkK8gfVvj4R1uf7bE3lDDi3xtp7i/gC0PZvkaqSzrLujiEQEgtnbI+uvDITf58Nd\nxV5DUneK5GbW5vCv5/hpjTOuViSTQEvcBiKhmbGbMQ8anSK5sosOhe/YNqe2FRGJiV18jOXgcIaA\njK1iWPA1IY1OkdxMybYApEjt5bGrfUzkBJHXLj7BY7sxpBCXvPQ8F1lPPcsiQiKNJNQnPj2ybO3x\ncDxZX7GjR9SdIrnpPPtwsHxQ3SnSRNoOh2NopA7ppD5xydXFyzcHy7+77dwa10SkONN3hrsv6nUB\nXyEuufn+Fs3klCYT6EOPhnVsrfKFvXnW6E00TlxyNXtGX7B8PzNqXBORiEjQTj0Y7rce6M7Quo6t\nVb47skNWTjROXLKLvFn3b54ffnzkD+f0s14Plr/y86o31BbJpH/ZQLDcTjR25LjDUM6bQhStsX+i\nksn2A931roK0mtgem3mEdaSx0rk4suF4TtSdInXT8eSsYPnAcs3wlAYRCOa2+f3Bhw7v6wqW979e\nXHei+sSlrlZdsSVY/vNnV9a2IiIxgZZ7LKxjLXHLMutzElwhLvXywpMrw3d01LQaIsUqeK6bLmxK\n3XQcD1+Q6Z+r7hRpHrYoPGorD+7qE5c6mvNyOKz3nl/jiogUaORgZ4FHN4Y1OkXq5cjq2JtPLXFp\nEIF+7inzwsMRR/YXGdZxefWJm9kK4B5gEUkn0Fp3v8PM5gH3AiuBrcBV7h5ZsvTkFOJNZO6L4bDe\n/b4aV0Qkg+Fj4Ys29ejUyHntlCHgM+7+tJnNAp4ys4eBa4FH3f12M7sFuAW4ebJPohBvIv1/fDB8\nx5Z5ta2ISExoU4j+bJtCeGTHq1x40i+ey6HcdwG70q+PmdmLwDJgDXBx+rC7gcdQiAtAx72RsP71\n2tZDpEizXyh2uFWG0SkLzGxjxe217r429EAzWwm8G9gALEoDHmA3SXfLpCnEyygyS25vLKwDLZrZ\nL4XXtjh6hvrPpbEdP7fYBbAyXNjc7+7nnexBZjYT+DbwaXc/avbG36+7u1l1u78oxFvUwAeOhu8o\neOsraTKxLo+2yESdLJt8RxorXujolPy6UwDMrIMkwL/l7t9Ji/eY2RJ332VmS4C91TyHQrxFzfuH\n8NTl1y+qcUWk3GJrp8S2bQuFfpFbvE1CjqNTDPgm8KK7f7nirvXANcDt6ecHqnkehXiL2vvrsRaR\ntn6TAmUJ7Egrv/1EceO43XOddn8h8HHgOTN7Ni37PEl432dm1wPbgKuqeRKFeIsa7lRYSw4yjiCx\n3gz7Y0YCf2hase/dvIYYuvu/EB8peWkuT4JCvGV1rjgeLO9Tn7hkEetOyRLWMZETRFuBLXHIt0+8\nFhTiLapvl3YHkhzEWuIZd7APipwghgtsiTvGiKbdSxl4R2T0QGzihUhIrCUeC+tA6LdHFm4bml6f\nJnHJGuLVh7iZtQEbgZ3ufoWZrQLWAfOBp4CPu/uAmXWSrCNwLnAA+Ki7b632+WVybEBhLXUQCP1o\nWEda+dOWhrsCc5Hvhc2ayKMlfhPwIjA7vf0l4Cvuvs7M/hq4HvhG+vmQu59hZlenj/toDs8vkxC9\n8NSXQ1+mtI5I0C48/UCwfO/LCyZ+7Egrv3dHeAer3JSsKV5ViJvZcuB3gduA/5KOi7wE+Fj6kLuB\nvyAJ8TXp1wD3A18zM3Mv22WEJlGubj9pVLHZw1nCOiZygug4WvSFzdZqiX8V+BwwemqcDxx296H0\n9g6SBV9IP28HcPchMzuSPn5/5QHN7AbgBoC2uXOrrJ4A4UWHeiIt7sgfzqwV4Rmex7bNmXS1RCZj\nZFWR0+5hpMEmH53MpEPczK4A9rr7U2Z2cV4VSheQWQvQeeoKtdIbRE9PfdZ2FnmrkSL32HSghVri\nFwK/b2aXA10kfeJ3AN1m1p62xpcDO9PH7wRWADvMrB2YQ3KBU0qgvSM8ZCxcKlKcMq2dUguTDnF3\nvxW4FSBtiX/W3f/IzP4RuJJkhErlugCj6wX8NL3/B+oPL49lXw8v/7nl39W4IlJuge46nxEZU36s\nTiOgS5ZKRfyUbgbWmdkXgWdIFoAh/fx3ZrYZOAhcXcBzS0Fe+51Y66dk73jJV+QaSnSyeaArpG5h\nHWQtd2ETAHd/jGR3Ctx9CzBma1537wM+ksfzSe0t/fFQsPy1yzQkUcbqOBweQTI4qwQn/RJUsVIj\nnQKlgb12eWRYl/aQkIB6zbasmoO3yugUaS2di3uC5f2vaw0WGcunRkK8yJEluSlDHd+gEJcJad8Q\nniXXv0JN8ZaWtdUa6EPvWnIi+NC6rahZsn8iFOIyIYOzS/bOltqIXNicElmbZ6R9bHnDLX9csre6\nQlwmZGhGyd7ZUhuRlvhILFkCoT/nF+GL40fOrMN/eS022UeaUQ5/lADMCoxmORIeay4tJPD+ioZ1\nbFOIXm0KUUkhLoXofHXsuPL+eeo/l+p1Hii4pazRKSLwq5e+PKbs2WdOr0NNpNn0Liq2MWBqiYvA\nz39yxtjCgje4lRIIdJFM6Qt3j4xMDR/Ci5xf5ujCpgjA4PyxfeLR5W+lvCL91jYY7pLwwBjskdiY\n8rowXdgUAWCoXH8Iki9fMBC+41Cked1IcjqnmNllJCu7tgF3uvvt+Rz5zRTiUoi2OWP/iEf2ddWh\nJlIP3T8JL5h2+FcbqdUdkUOXe7r38NeB3yLZHOdJM1vv7i9Uf/Q3U4hLIaY+P31MWd9CjU5pFdMO\nhH/Xhxt9Snt+48TPBzanCwJiZutItqhUiEs5DHeVoMUlhRmcERvL3fjvi5xGp/xyO8rUDuCCXI78\nFgpxKcTw6YF9ENWd0jL2fSDcJ25HSzDha+IhvsDMNlbcXptuL1lTCnEpxKyfjO1OObJa3Smtwnpa\nIlr2u/t5kftGt6McVblVZa5a4ictBYrMbgsGdmQ4Wtee8NDDvlMU+mW1cFV4+9y9Ly+ocU2yy6k7\n5UlgtZmtIgnvq4GP5XLkt1CIS931ndkXvqMMw9FaXeQknktYR0763lFgv7qTy7R7dx8ys08AD5EM\nMbzL3Z+v+sABCnGpu7bXw8PRhjXDU0I6Cv4PLae3nbs/CDyYz9HiFOJSdzO3R7pkzlSIN7zYSoMn\nwqNTMp2YIy1iO15sbGntFJGMuq7YEyw/8tIpNa6J5KXQLo+ilazqCnGpu93b5wXLG3xaiIzDF/aH\n7zgY7jprKApxkWzajoVHp0Q3opDGEeny8FhYh1Yx7I9sRFKHIeXm6k4RyWzxO/YGy1/ftLDGNZHM\nYjs7xQRCvx5hPS5tCiGSzYn/szh8Rz32WJRsigy8egwxRC1xkcyOnq6wlonrOFzwuvQKcZGMTinx\nRTCpucFTBos7uPrERcaRw0UwAOZE/og1w7P2Yl0eneH/rqw3Qyu6TuPE1RIXKZhFmkol+9trDrGg\nzRLWMbGTeMHXHa1kvXsKcSmdc07bESx/9uDpNa6JZBYK5llj92MF4EijDVtpTApxKZ1N/3d1+I7F\nJWtCtaJQyz0W1pGWeHtkSn9uSvYvnUJcSqctch1USiAQzB2Hw6E8ODt8iKEiF0bThU2R4inESyzQ\nEh+cHUnN2EXTGcN51ijwBMUePm8KcSmdw78WHp1ifQWPH5bqBYLZ28KpaYPhFrod0+iUSgpxKZ0l\nPwiH9e731bgikl2gJW6xWZ9Zp/TnwGix0Slm1g3cCZxNcv76U2ATcC+wEtgKXOXuh8zMgDuAy4Ee\n4Fp3f7qa55fWNOvVE8Hy3e+bUeOaSD0UvbNPq/WJ3wH8s7tfaWZTgenA54FH3f12M7sFuAW4Gfgw\nsDr9uAD4RvpZJJPdF86K3FOyJpQ0plYJcTObA1wEXAvg7gPAgJmtAS5OH3Y38BhJiK8B7nF3Bx43\ns24zW+LuuyZde2lJH73u0WD5nT/8YI1rIpkFukhsMDKTNzKrxyJL1+amVUIcWAXsA/7GzM4BngJu\nAhZVBPNuYFH69TJge8X370jL3hTiZnYDcANA29y5VVRPSi/SVxoN60gf6qxXxvahH1ulVntdBH6n\nHrseHfl92nCxId5K3SntwHuAT7r7BjO7g6Tr5Jfc3S02RzrC3dcCawE6T11Rsh+nNKIZu8cG9rFV\ndahIM4rtsTk/PA50eF/XxI8dW2un6OW+S5Y61YT4DmCHu29Ib99PEuJ7RrtJzGwJMLri/05gRcX3\nL0/LRAq1+/2Bv8qCW3MtIxK0mcI6JnKCmBI5QeTCW2h0irvvNrPtZvY2d98EXAq8kH5cA9yefn4g\n/Zb1wCfMbB3JBc0j6g+XmpgZWJtD63LURyiYY7PohyLbtuVxghhPC7XEAT4JfCsdmbIFuI7kV3Kf\nmV0PbAOuSh/7IMnwws0kQwyvq/K5RSbk9DvHzvB75SMK8UbRNjeHrpcc1aJP3Mz+J/B7wADwCnCd\nux9O77sVuB4YBj7l7g+Nd6yqQtzdnwXOC9x1aeCxDtxYzfOJTMaBd04LlJasudUsAt0v0bCOdKd0\nn3Y4WL5t0pV6i9q8NR4GbnX3ITP7EnArcLOZnQVcDbwDWAo8YmZnunt0rQHN2JSmd/jtCuzCxC5s\n9oT7SIa7MvwuIv3th18tcNSaU5MQd/fvV9x8HLgy/XoNsM7d+4FXzWwzcD7w09ixFOLS9D72gX8d\nU/b3P7qwDjVpHR1HIxc8s4R4HRiZulMWmNnGittr09F1Wf0pySx3SIZdP15x3+hQ7CiFuDS9f/7a\n+8cWvrOxw6Q0Iq3lvoU5DPHIuPVbXjKE+H53D3UnJ8cxewRYHLjrC+7+QPqYLwBDwLcyVvOXFOLS\n9EZ0DbNxhGZszh0IPtQPhPdezWXrt/HkdH539w+Nd7+ZXQtcAVyaXjOESQzFVohL0zuyWq3uhhGa\nsRkJ6+gqhrFVD/NSm9EplwGfAz7g7j0Vd60H/t7MvkxyYXM18MR4x1KIS/OIzfALNdwiATFj+bFg\n+YnXItvMSDaBn3v7gr7gQ4f2hkYVFax2qxh+DegEHk4WeOVxd/9P7v68md1HMt9mCLhxvJEpoBAX\neZPje2YGyzW/szi2ZXr4jpl1+g+qNqNTzhjnvtuA2yZ6LIW4SIWZi44Hy9USz0loe7ZYWMcubE4r\ndnu2lpl2L9KM1BLPKBK0014PX3zsXZwhISPdY3ai2NhqpVUMRZpPyf6A6y4StJnCOia2FG13eDRL\nLmo02SdPCnGRCm+7M7z120t/EttNqMXFujy6wiFuPRmGB8YuVB+MjGbJi0JcpLw2/cfwRTbrrXFF\nyiLW5REL69Bu93kEfk4yzthsCApxkQoztoRnBvUsKdnVrlqJrfndF147ZWTq2LJ6hPV4bKRcKa4Q\nF6nQ312uP+BG1Rbptg6FeENRn7hIua14JJw+W39Pc/eDIt0pWYYNxlvt9UlTdaeIlNi+d8Uumqk7\nJReB0I+GdWx0ymD5p93nSSEurSnSgjyxIhLWkUA57bvhiSfbLm+RP63Iz6Vrb7ifu29B9ePEg8so\n5EgtcZEWcvH/+Emw/O7HLqpxTeokthRtlrCOiZwg2o/HNuXMiUJcpHW80rOg3lVoOUMzC+zaaqXd\n7kUEDvbPqHcVWk6Rm0JonLiItJYi1/yOTSQqfFOIcqW4QlykCpueWBm+o7NcQTBpRW7QEDlBTD1Y\nbIirJS7SQjrPOBos79mutVaKMrAqvIlELjTZR6S1XHLqS8Hy724/t8Y1KYnQHptDkaGEsQWADxU7\n7VMXNkVayPd+cXa9q1AuoT02YyMGY5N9Ihsr50UhLtJKSvavd91lmnYfPkR0Y+U8OLqwKdKUsq5t\nHRu1ETrMcAvtG5TDtPsZ23Vhs5JCXKSGOg6PbXUOzipZalQjtJ54ZNx3bCjhiWUF93eU7NehEBep\noc5DgY2CWynEAy3x6Ljv2LT7Y8VNu9dkHxEZ17HVQ2PKrL/gtUCazOC8Ane7d9emECISN3/j2Fbn\nwXeWKzTeJNJa7jgSPjFl+q8jNmOzv3mWojWzzwB/BZzi7vvNzIA7gMuBHuBad396vGMoxEVqqL87\nFEAlDvGI4diEnP0Fb3Kcg1p1p5jZCuC3gdcqij8MrE4/LgC+kX6OUoiL1NDg7OYL7JC5c04Eyw80\neog7ULvulK8AnwMeqChbA9zj7g48bmbdZrbE3XfFDqIQF6mhZe/bOabs1eeX1qEmOYl0eRx4ZV71\nx4501cxcEV7qIDcTz/AFZrax4vZad187kW80szXATnf/t6QH5ZeWAdsrbu9Iy4oJcTP7z8B/IHnZ\nzwHXAUuAdcB84Cng4+4+YGadwD3AucAB4KPuvrWa5xcpm10/Wj62MI8NFOol46zKTBN1IieI49vm\nTPwYk5ChO2W/u58XPY7ZI8DiwF1fAD5P0pVStUmHuJktAz4FnOXuvWZ2H3A1SYf8V9x9nZn9NXA9\nSb/O9cAhdz/DzK4GvgR8tOpXIFIiI+88NrZwV/OtST5187Rgef/cxj9h5TU6xd0/FDy+2TuBVcBo\nK3w58LSZnQ/sBFZUPHx5WhZVbXdKOzDNzAaB6SRN/kuAj6X33w38BUmIr0m/Brgf+JqZWdr3I9IS\nfNPMsYVl7iePtJZzCetIK7+tt8AhmTVYxdDdnwMWjt42s63AeenolPXAJ8xsHckFzSPj9YdDFSHu\n7jvN7K9Irqz2At8n6T457O6jg2FH+3Ogoq/H3YfM7AhJl8v+yuOa2Q3ADQBtc+dOtnoi9RUJt+CF\nzUhYnf+el4PlT2w8c9LVyl2k7tNeD0/g6V1c/UbJwwWu1Z5M9qnrSfVBkt6MzSRDDK872TdU050y\nl6R1vQo4DPwjcNlkjzcqvTCwFqDz1BUlbqKIVKehwjomErTRsA6EfvuC8HDEob3hLpnC1bjHx91X\nVnztwI1Zvr+a7pQPAa+6+z4AM/sOcCHQbWbtaWu8sj9ntK9nh5m1A3NILnCKSMCclYeD5Ue2dte4\nJuOIdXmcCHd5DAdyuW5hHVHnlnhm1YT4a8B7zWw6SXfKpcBG4IfAlSQjVK7hjTGQ69PbP03v/4H6\nw0Xijr4aCesSLHo4tHAwWG7HGnxUcyvt7OPuG8zsfuBpYAh4hqQb5HvAOjP7Ylr2zfRbvgn8nZlt\nBg6SjGQRkYjOA+HWbF8ZhiRG1ghvfC22doq7/znw528p3gKcH3hsH/CRap5PpJXM2hoOk74FNa7I\neGLrm+SxgXLsouku7XZfqcH/txFpXXs/WNIuiYINTS8wZF3bs4lITub/a0ew/OCvlaulmLeus8MX\nfHOjlriI5OHw2+tdg8bU2xdsZ1/SAAAJSklEQVQ+ueWmXBmuEBdpVN5RgjTJspcoZNtPNNKvPrRn\n+sSPMQk2Uq7+FIW4SINauT7cJ/7qmoJbojmwwcjG0o0+aMWp+WSfainEReot0uKMhnWk9evTwtuW\n2Yna/5lPiey+Mzytsf+7MLylJvuISAN595nbguXPPnN6cU8aW98kj7COnKymzO+v/tjjUYiLSD3s\n/N9nhO84r1yhdDK2veBp+gpxEamH3/j0k8Hy9f8S3begerGunchFWevP0Ckea+V3FTtOXH3iIlIX\nj947ZqJ0YlmBqVTkjvSRE8TUg8XO2NToFBGpi54lJQifQDB3HAm3zgdnhQ8x0F3k63R1p4hIfUzf\nFQ7DE0W2xLMKtNwHZ0VCM9IS79xXYEvcUYiLSH30zytB+OTQEu+fX/BJqYHOeROhEBdpEkVuW5ab\nHFriUwaKXVBd48RFpC68LRI+XmDoRYK2/Vi4dT00I0NARi6ajhSdWgpxEamH5W/bGyzf8YtFxT1p\nbH2TWFgHQn/2S+E+7qNn1KFfwx2Gy9WfohAXKZtIcEbDOtJanvvc2NbyoXdkbIXGxol3hoPQescG\ndl3CejxqiYtIGRy9uHds4b6uXI7dtSO87kvhFyXzoBAXkTKY8urY6evDMzMGWOS/glzCOuNs0Fw4\n0Ep7bIpIec3YMbYs80YUOXSnRBU5GzTKwWvz34KZfRK4ERgGvufun0vLbwWuT8s/5e4PjXcchbhI\ni+r54PGxhbtm5HLs9oPhaGn0pWhxanJh08w+CKwBznH3fjNbmJafBVwNvANYCjxiZme6e3idYRTi\nIi2r/9DY/u+82rjDSyLLxR6emtMzFKg2feJ/Btzu7v3JU/ro0KI1wLq0/FUz2wycD/w0diCFuEir\nmhpocfZlnNIe6fKIhnWg+yWXFQ/zNPEQX2BmGytur3X3tRP83jOB3zSz24A+4LPu/iSwDHi84nE7\n0rIohbhIizpt2YExZa8dXVzskwZCP9rHHdu/s9Bsz7QA1n53j67za2aPAKEf6BdIsnce8F7g14H7\nzOxXMlYWUIiLtKztPw/kS7GrvAaDeUpfOJVHYj0vBS9iSE5L0br7h2L3mdmfAd9xdweeMLMRYAGw\nE1hR8dDlaVlUo29bKiJNbrh7KPhRN+4T+6jOPwEfBDCzM4GpwH5gPXC1mXWa2SpgNfDEeAdSS1yk\nRY1MG9vitIGM7brYcrGLe4Ll/a+PHf1iPUU3/7Oo2bT7u4C7zOznwABwTdoqf97M7gNeAIaAG8cb\nmQIKcZHmFxtvHVoNMBLK858Oh/uB90QWwNoQXke2f0WDz9h08BqME3f3AeCPI/fdBtw20WMpxEXk\npKZdvTt8R2S9lhOxsA6cJNp6wieIQvfSHI9mbIpIs9n9TGTUyoxwWLcfjyxFO31sWd3COkZrp4hI\ns5l/TniZ2z0vnRIsH5o+8aVo23ojLfF6bHLhntvolFpRiIvIScXCOg/DgQusQHwiUdHUEheRZjNj\ne2QT5qwXKrMEc+Qiqw0WuwCWD487GKThKMRF5KROrIwEW6TR+vaztwfLf/GzUyf+pJHA9yJHJGop\nWhFpRl2LTwTL+yKrHr64dUmwvE4dJNnUaCnavJw0xM3sLuAKYK+7n52WzQPuBVYCW4Gr3P2QmRlw\nB3A50ANc6+5Pp99zDfBf08N+0d3vzveliEhRZq0Pj/vuOzfS5XE0vLNPJpHulGlLA0vo5sQBb8KW\n+N8CXwPuqSi7BXjU3W83s1vS2zcDHyaZJroauAD4BnBBGvp/DpxH8nN6yszWu/uhvF6IiBTHrt4X\nvuPlBbWtCND7+sziDu612xQiLycNcXf/sZmtfEvxGuDi9Ou7gcdIQnwNcE86ffRxM+s2syXpYx92\n94MAZvYwcBnwD1W/AhEp3N59s8N35DDtPqpOo1Na5cLmInfflX69GxidtrUMqLyiMboWbqx8DDO7\nAbghvXl826c+uyn9egHJAjHNrlVeJ7TOa9XrrI3Tqj3AMQ499IjfP9F/Lxrid1r1hU13dzPLrRMp\nXVR9zMLqZrZxvLV7m0WrvE5ondeq11ke7n5ZveuQ1WSXot2TdpOQfh6dzhVbCzfzGrkiInJykw3x\n9cA16dfXAA9UlP+JJd4LHEm7XR4CftvM5prZXOC30zIREanCRIYY/gPJhckFZraDZJTJ7STbCV0P\nbAOuSh/+IMnwws0kQwyvA3D3g2b234En08f95ehFzgwmundd2bXK64TWea16nVIY85KtEyAiIm/Q\n9mwiIiWmEBcRKbFShLiZXWZmm8xsczpDtCmY2V1mtjfdZ2+0bJ6ZPWxmL6ef59azjnkwsxVm9kMz\ne8HMnjezm9LypnqtZtZlZk+Y2b+lr/O/peWrzGxD+v6918xi+7iXipm1mdkzZvbd9HZTvs5G1/Ah\nbmZtwNdJpvSfBfyhmZ1V31rl5m9JZq5WGl3SYDXwaHq77IaAz7j7WcB7gRvT32GzvdZ+4BJ3Pwd4\nF3BZOkrrS8BX3P0M4BBwfR3rmKebgBcrbjfr62xoDR/iwPnAZnffkm4uuo5ken/pufuPgbeO0llD\nspQB6ec/qGmlCuDuu0YXQnP3YyR/+MtostfqidHVmTrSDwcuAe5Py0v/OgHMbDnwu8Cd6W2jCV9n\nGZQhxCc8Zb9JxJY0aArpOjzvBjbQhK817WJ4lmQC3MPAK8Bhdx9KH9Is79+vAp8DRleLmk9zvs6G\nV4YQb1npQmJNMwbUzGYC3wY+7e5HK+9rltfq7sPu/i6SWcnnA2+vc5VyZ2ajS1M/Ve+6SDk2hWi1\nKft7zGyJu+96y5IGpWZmHSQB/i13/05a3JSvFcDdD5vZD4HfALrNrD1tpTbD+/dC4PfN7HKgC5hN\nso9As73OUihDS/xJYHV65XsqcDXJ9P5mFVvSoLTS/tJvAi+6+5cr7mqq12pmp5hZd/r1NOC3SPr/\nfwhcmT6s9K/T3W919+XuvpLk7/EH7v5HNNnrLItSzNhMz/hfBdqAu9z9tjpXKReVSxoAe0iWNPgn\n4D7gVNIlDSaxREFDMbP3A/8PeI43+lA/T9Iv3jSv1cx+jeSCXhtJA+k+d/9LM/sVkgvy84BngD92\n9/761TQ/ZnYx8Fl3v6KZX2cjK0WIi4hIWBm6U0REJEIhLiJSYgpxEZESU4iLiJSYQlxEpMQU4iIi\nJaYQFxEpsf8PHQbrNMDW2DIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rkd2LdwKLWtP",
        "colab_type": "text"
      },
      "source": [
        "### Estimated Random Effects matrix\n",
        "\n",
        "The below reads in the Random effects variance predicted by `R`'s `lmer`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tOqQRAROqdkv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Read in estimated variance\n",
        "RFXVar_REst = pd.read_csv('/Data/BLMM-testdata/estd_rfxvar.csv',header=None).values#pd.read_csv('/Data/BLMM-testdata/estd_rfxvar_1factor.csv',header=None).values#"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RStOLwF_LlTE",
        "colab_type": "text"
      },
      "source": [
        "### Y vector\n",
        "\n",
        "The response vector is read in here:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DG8eWNdpPQOs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "Y=pd.read_csv('/Data/BLMM-testdata/Y.csv',header=None).values#pd.read_csv('/Data/BLMM-testdata/Y_1factor.csv',header=None).values#"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RHm6DjdbPTvg",
        "colab_type": "text"
      },
      "source": [
        "### X matrix\n",
        "\n",
        "The fixed effects design matrix is read in here. It consists of an intercept and two random (Gaussian) columns.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xqTKH4n_Po8e",
        "colab_type": "code",
        "outputId": "49bf60de-e993-49be-dd40-b3b6896a4c53",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "X=pd.read_csv('/Data/BLMM-testdata/X.csv',header=None).values#pd.read_csv('/Data/BLMM-testdata/X_1factor.csv',header=None).values#\n",
        "\n",
        "# Image of the first 20 rows of X\n",
        "imshow(X[1:20,:])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f9e1ac6cf28>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAFYAAAD8CAYAAADt0VN/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADBNJREFUeJztnXuMXVUVh78fbXnVhjeUIm9Kk0Kw\nkKaCqCkWEUZClaC2MVAERVQUjGhQIxiMRkOUaCCQSsvDFGjkZYXyqECCGEBK00JLqRQCgQIFWuiD\nAnXo8o+zBy+393b23HNXe+/p+pLJnHvOuufs+WbPvuexZm2ZGUH72WZLN6CqhFgnQqwTIdaJEOtE\niHUixDoRYp0IsU4M3tINaMQOu2xnO40Ymh2/z+B3smMXvr1Hdmzvirf4YO07yn5DDR0pdqcRQzn9\nxgnZ8b/Zc1527CF/Pzc79rVf/zE7tp5SQ4GkEyUtkbRU0kUNtm8naWba/pikA8ocr5toWaykQcCV\nwEnAaGCypNF1YWcDb5nZIcDlwO9aPV63UabHjgOWmtnzZrYeuBmYWBczEbg+Ld8CTJDU0pjVbZQR\nuw/wUs3rl9O6hjFm1gusAnYrccyuoWNOtySdI2mupLnr3np/SzenNGXELgP2rXn98bSuYYykwcBO\nwIpGOzOzqWY21szG7rjLdiWa1RmUEfs4MFLSgZK2BSYBs+piZgFT0vJpwAO2lTyyaPk81sx6JZ0H\n3AsMAqab2SJJlwJzzWwWMA34i6SlwEoK+VsFpS4QzGw2MLtu3cU1y+8BXylzjG6lI6+8BkrPPkdl\nx5427/Hs2BlD17XSHKCDzgqqRoh1IsQ6EWKdCLFOhFgnQqwTIdaJEOtEiHUixDpRiXsFM176V3bs\ncX/8cXbsqpWPtNIcIHqsGyHWiRDrRIh1IsQ6EWKdCLFOlMnd2lfSg5KelrRI0vkNYsZLWiVpfvq6\nuNG+qkiZC4Re4EdmNk/SMOAJSXPM7Om6uH+a2ckljtOVtNxjzexVM5uXltcAi9k4d2urpS2XtCnv\n9UjgsQabj5G0AHgFuNDMFrXjmLX0/OLC7Nj3R+Un4lgJO6XFSvoYcCtwgZmtrts8D9jfzNZK6gHu\nAEY22c85wDkAw/besWyztjhlM7qHUEidYWa31W83s9VmtjYtzwaGSNq90b4iKS6REoinAYvN7A9N\nYob3JRpLGpeO1zDbsGqUGQqOBU4HnpI0P637GbAfgJldTZFh+B1JvcC7wKTINuwHM3sY2GTau5ld\nAVzR6jG6mbjyciLEOhFinQixToRYJ0KsE5V4/P2Z7ze6RdGYOx4alx1rJbpd9FgnQqwTIdaJEOtE\niHUixDoRYp0IsU6EWCdCrBOVuKS955ajs2M3HPTf/B1v0/pTpOixToRYJ0qLlfSCpKdS0tvcBtsl\n6U+pWtyTkvKrNnQx7RpjjzOzN5tsO4ki+2Uk8EngqvS90myOoWAicIMVPArsLGnvzXDcLUo7xBpw\nn6QnUv5VPTkV5SpX0KwdQ8GnzWyZpD2BOZKeMbOHBroTM5sKTAUYftiuXZ8tU7rHmtmy9P114HaK\nYpK15FSUqxxlsw2HpmxuJA0FTgAW1oXNAs5IZwdHA6vM7NUyx+0Gyg4FewG3p4TCwcCNZnaPpHPh\nw8S42UAPsBRYB3yj5DG7grKV4p4HPtFg/dU1ywZ8r8xxupFK3CtYd2j+WcQZRz2aHTtt6NpWmgPE\nJa0bIdaJEOtEiHUixDoRYp0IsU6EWCdCrBMh1olKXNIOWb5tduwNjxybHbti7UaP8LKJHutEiHUi\nxDoRYp0IsU6EWCdCrBNlSpeMqilUNl/SakkX1MVEQbOBYmZLgDHw4YyfyyjyCuqJgmYlmAA8Z2Yv\ntml/XU+7xE4Cbmqy7RhJCyTdLemwNh2v42lHQbNtgVOAnzbYvFkKmvWOyH/8PeiN/PsKbGh9Ct12\n9NiTgHlmtrx+QxQ0K8dkmgwDUdCsRVIi3OeBb9esq83bioJmrWBm71A35XRd3lYUNAvaS4h1IsQ6\nEWKdCLFOhFgnKvH4+67P5p/RrdywfXbst659vZXmANFj3QixToRYJ0KsEyHWiRDrRIh1IsQ6EWKd\nCLFOhFgnKnGvoOf+H2THnnREfZ2K5qzond1Kc4DosW5kiZU0XdLrkhbWrNtV0hxJz6bvuzR575QU\n86ykKe1qeKeT22OvA06sW3cRcL+ZjQTuT68/gqRdgUsoCpiNAy5p9guoGlliU7mnlXWrJwLXp+Xr\ngS81eOsXgDlmttLM3gLmsPEvqJKUGWP3qqlG9BpF4Z16soqZVZG2fHil7JZSGS5VqxRXRuzyvhqF\n6Xuj5xjZxcwiKe7/zAL6PuWnAH9rEHMvcIKkXdKH1glpXeXJPd26CXgEGCXpZUlnA78FPi/pWeD4\n9BpJYyVdA2BmK4FfAY+nr0vTusqTdeVlZpObbJrQIHYu8M2a19OB6S21roupxCXtoLfzf4yHXjo4\nO3bt+tbH+rikdSLEOhFinQixToRYJ0KsEyHWiRDrRIh1IsQ6EWKdqMS9gtFjX8iOXfjk/tmxG9YP\naqE1BdFjnQixToRYJ0KsEyHWiRDrRL9im+RtXSbpmTS55O2Sdm7y3k1OVFllcnrsdWycFjQHONzM\njgD+Q+MKRn0cZ2ZjzGxsa03sTvoV2yhvy8zuM7Pe9PJRikSMoIZ2jLFnAXc32dbfRJWVpWwVo58D\nvcCMJiHZE1WWKWj21OL9smMPHZ0/XePKHdYPqB21lKnGeSZwMvD1ZiWfMiaqrI2N3C1JJwI/AU4x\ns3VNYnImqqwsOadbjfK2rgCGUfx5z5d0dYodIanvPyL2Ah6WtAD4N3CXmd3j8lN0IP2OsU3ytqY1\niX2FYkbPphNVbi3ElZcTIdaJEOtEiHUixDoRYp0IsU5U4vH3zgvzf4zn1uzbf1Di/XUDKJReR/RY\nJ0KsEyHWiRDrRIh1IsQ6EWKdCLFOhFgnQqwTlbikHfRuftWUH37xzuzYy65d1UpzgOixbrSaFPdL\nSctqZu/safLeEyUtkbRU0kZ1uapMq0lxAJenZLcxaXa5j5Bm/rySYoa60cBkSaPLNLabaCkpLpNx\nwFIze97M1gM3UxRB2yooM8ael/Jjpzcpq7fVFjOD1sVeBRxMMeHvq8DvyzYkCpoBZrbczD4wsw3A\nn2mc7JZdzCztM5Li+irEJb5M42S3x4GRkg5Mc9dOoiiCtlXQ7wVCSoobD+wu6WWKsqXjJY2hSCx+\ngTTLp6QRwDVm1mNmvZLOo6gMNwiYbmaLXH6KDsQtKS69ng20Xo+5i4krLycqca9gzYH5sUvWDc+O\nfW/DkBZaUxA91okQ60SIdSLEOhFinQixToRYJ0KsEyHWiRDrRCUuaZecdVV27EH/OCs7dtV7Df9R\nPYvosU6EWCdCrBMh1okQ60SIdSLEOpHzlHY6RVGd183s8LRuJjAqhewMvG1mYxq89wVgDfAB0Ls1\nFTXLuUC4jqIGzA19K8zsa33Lkn4PbCqR9Dgze7PVBnYrOY+/H5J0QKNtkgR8Ffhce5vV/ZQdYz8D\nLDezZ5tsz64UV7XcrbL3CiYDN21ie3alODObCkwFGH7YrgOaMfTYJ0/ND149gEfaH2ggzfgIZSrF\nDQZOBWY2ixlIpbiqUWYoOB54xsxebrQxKsX1Q5NKcVBkD95UFxuV4hKtJsVhZmc2WBeV4hJx5eVE\niHUixDoRYp0IsU6EWCfUpLz2FkXSG8CLdat3Bzb3XbJRZjaslTd2ZF6Bme1Rv07S3M19P7fMbCMx\nFDgRYp3oJrFTu+mYHfnhVQW6qcd2FR0ntr9yJ5K2kzQzbX+s2fO4ARxvX0kPSnpa0iJJ5zeIGS9p\nVU2plov73bGZdcwXxT8zPwccBGwLLABG18V8F7g6LU8CZpY85t7AUWl5GMX8ZPXHHA/cOZD9dlqP\nzSl3MhG4Pi3fAkxIT4tbwsxeNbN5aXkNsJg2VALpNLE55U4+jLFisrZVwG7tOHgaVo4EHmuw+RhJ\nCyTdLemw/vbVkVdeWwJJHwNuBS4ws9V1m+cB+5vZ2lQK6w5g5Kb212k9NqfcyYcx6UnxTsCKMgeV\nNIRC6gwzu61+u5mtNrO1aXk2METS7pvaZ6eJzSl3MguYkpZPAx6wEifjaXyeBiw2sz80iRneN45L\nGkfhbdO/zC19JtDgU7qH4pP5OeDnad2lFJOyAWwP/BVYSvH096CSx/s0RcbOk8D89NUDnAucm2LO\nAxZRnKU8Cnyqv/3GlZcTnTYUVIYQ60SIdSLEOhFinQixToRYJ0KsE/8D9NqNWg2rPVoAAAAASUVO\nRK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W_sXGD8qzskp",
        "colab_type": "text"
      },
      "source": [
        "### Number of Levels and Parameters\n",
        "\n",
        "The number of levels is given by a vector with one entry for each grouping factor. e.g. nlevels=[10,2] means there are 10 levels for factor 1 and 2 levels for factor 2. \n",
        "\n",
        "The number of parameters is given by a vector with one entry for each grouping factor. e.g. nparams=[3,4] means there are 3 variables for factor 1 and 4 variables for factor 2."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_e5bUA2DztCf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlevels = np.array([20,3])#])#\n",
        "nparams = np.array([2,2])#])#"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0byKXygpCa1P",
        "colab_type": "text"
      },
      "source": [
        "### True b values\n",
        "\n",
        "The true recorded values of the random effects b vector in this example."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LzwuuWSdCbCJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "b_True=pd.read_csv('/Data/BLMM-testdata/true_b.csv',header=None).values#pd.read_csv('/Data/BLMM-testdata/true_b_1factor.csv',header=None).values#"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bvfns6c-CbPd",
        "colab_type": "text"
      },
      "source": [
        "### True beta values\n",
        "\n",
        "The true fixed effects parameters used to generate this example."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlvolWwvCbbP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "beta_True=pd.read_csv('/Data/BLMM-testdata/true_beta.csv',header=None).values#pd.read_csv('/Data/BLMM-testdata/true_beta_1factor.csv',header=None).values#"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bVyBWvQK2Gw5",
        "colab_type": "text"
      },
      "source": [
        "### Product Matrices\n",
        "\n",
        "All products of matrices are calculated beforehand as it is both more computationally efficient and also similar to the setting we are interested in. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e53HYCsj2G7I",
        "colab_type": "code",
        "outputId": "0adbcef0-d857-4c6d-8378-2c1f5e1f9145",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        }
      },
      "source": [
        "# Z transpose Z\n",
        "print(Z.shape)\n",
        "ZtZ = np.matmul(Z.toarray().transpose(),Z.toarray()) # This works for products involving sparse\n",
        "# Sparse \n",
        "# ZtZ = Z.transpose() * Z\n",
        "\n",
        "# Z transpose X\n",
        "XtZ = np.matmul(X.transpose(),Z.toarray())\n",
        "\n",
        "# X transpose Z\n",
        "ZtX = np.matmul(Z.toarray().transpose(),X)\n",
        "\n",
        "# ZtY\n",
        "ZtY = np.matmul(Z.toarray().transpose(),Y)\n",
        "\n",
        "# YtZ\n",
        "YtZ = np.matmul(Y.transpose(),Z.toarray())\n",
        "\n",
        "# XtX\n",
        "XtX = np.matmul(X.transpose(),X)\n",
        "\n",
        "# XtY\n",
        "XtY = np.matmul(X.transpose(),Y)\n",
        "\n",
        "# YtX\n",
        "YtX = np.matmul(Y.transpose(),X)\n",
        "\n",
        "# YtX\n",
        "YtY = np.matmul(Y.transpose(),Y)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1000, 46)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ilAB3qmDMHa8",
        "colab_type": "text"
      },
      "source": [
        "## Helper Functions\n",
        "\n",
        "This section contains miscellaneous functions used to help the `FS` function including functions to work out the duplication matrix.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cVkJVMSo1fCF",
        "colab_type": "text"
      },
      "source": [
        "###Matrix to Vector function\n",
        "\n",
        "This function takes in a matrix and vectorizes it (i.e. transforms it to a vector of each of the columns of the matrix stacked on top of one another). Example:\n",
        "\n",
        "$$\\begin{bmatrix} a & b & c \\\\ d & e & f \\\\ g & h & i \\\\\\end{bmatrix} \\rightarrow \\begin{bmatrix} a \\\\ d \\\\ g \\\\ b \\\\ e \\\\ h \\\\ c \\\\ f \\\\ i \\end{bmatrix}$$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rBhQXRF01d9n",
        "colab_type": "code",
        "outputId": "19f8689f-a639-4e45-d2a4-8fde5c0007df",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 228
        }
      },
      "source": [
        "def mat2vec(matrix):\n",
        "  \n",
        "  #Return vectorised matrix\n",
        "  return(matrix.transpose().reshape(matrix.shape[0]*matrix.shape[1],1))\n",
        "\n",
        "# Example:\n",
        "matrix = np.random.randn(3,3)\n",
        "print(matrix)\n",
        "print(mat2vec(matrix))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 0.70105957  1.53122018 -1.61626081]\n",
            " [-0.56557918 -0.27036032 -1.01789735]\n",
            " [ 0.42650756  1.59044263 -0.4313635 ]]\n",
            "[[ 0.70105957]\n",
            " [-0.56557918]\n",
            " [ 0.42650756]\n",
            " [ 1.53122018]\n",
            " [-0.27036032]\n",
            " [ 1.59044263]\n",
            " [-1.61626081]\n",
            " [-1.01789735]\n",
            " [-0.4313635 ]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EcIJAyHi1eTj",
        "colab_type": "text"
      },
      "source": [
        "###Matrix to Vector function\n",
        "\n",
        "This function takes in a (symmetric, square) matrix and half-vectorizes it (i.e. transforms it to a vector of each of the columns of the matrix, below and including the diagonal, stacked on top of one another). Example:\n",
        "\n",
        "$$\\begin{bmatrix} a & b & c \\\\ b & d & e \\\\ c & e & f \\\\\\end{bmatrix} \\rightarrow \\begin{bmatrix} a \\\\ b \\\\ c \\\\ d \\\\ e \\\\ f \\end{bmatrix}$$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6fpQzdwA3QGq",
        "colab_type": "code",
        "outputId": "965942e6-9ef7-4999-ebbf-3ee0417c74ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "source": [
        "def mat2vech(matrix):\n",
        "  \n",
        "  # Get lower triangular indices\n",
        "  rowinds, colinds = np.tril_indices(matrix.shape[0]) #Try mat.transpose()[trilu]?\n",
        "  \n",
        "  # They're in the wrong order so we need to order them\n",
        "  # To do this we first hash them\n",
        "  indhash = colinds*matrix.shape[0]+rowinds\n",
        "  \n",
        "  # Sort permutation\n",
        "  perm=np.argsort(indhash)\n",
        "  \n",
        "  # Return vectorised half-matrix\n",
        "  return(np.array([matrix[rowinds[perm],colinds[perm]]]).transpose())\n",
        "\n",
        "# Example:\n",
        "matrix = np.random.randn(3,3)\n",
        "print(matrix*matrix.transpose())\n",
        "print(mat2vech(matrix*matrix.transpose()))\n",
        "\n",
        "#print(vech2mat(invDupMat(3) @ mat2vec(matrix*matrix.transpose())))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 4.11709191 -0.0507718   0.05208841]\n",
            " [-0.0507718   0.63279883 -1.18873472]\n",
            " [ 0.05208841 -1.18873472  1.58103632]]\n",
            "[[ 4.11709191]\n",
            " [-0.0507718 ]\n",
            " [ 0.05208841]\n",
            " [ 0.63279883]\n",
            " [-1.18873472]\n",
            " [ 1.58103632]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V79rfwHp9eNe",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1DmbjUOh9euy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def vec2mat(vec):\n",
        "  \n",
        "  # Return matrix\n",
        "  return(vec.reshape(np.int64(np.sqrt(vec.shape[0])),np.int64(np.sqrt(vec.shape[0]))).transpose())\n",
        "\n",
        "# Example\n",
        "#vec = np.array([[1,2,3,4]]).transpose()\n",
        "#mat = vec2mat(vec)\n",
        "#print(vec)\n",
        "#print(mat)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ijBvOZsu9iKg",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nhitZciM9hcz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def vech2mat(vech):\n",
        "  \n",
        "  # dimension of matrix\n",
        "  n = np.int64((-1+np.sqrt(1+8*vech.shape[0]))/2)\n",
        "  matrix = np.zeros((n,n))\n",
        "  \n",
        "  # Get lower triangular indices\n",
        "  rowinds, colinds = np.tril_indices(matrix.shape[0])\n",
        "  \n",
        "  # They're in the wrong order so we need to order them\n",
        "  # To do this we first hash them\n",
        "  indhash = colinds*matrix.shape[0]+rowinds\n",
        "  \n",
        "  # Sort permutation\n",
        "  perm=np.argsort(indhash)\n",
        "  \n",
        "  # Assign values to lower half\n",
        "  matrix[rowinds[perm],colinds[perm]] = vech.reshape(vech.shape[0])\n",
        "  \n",
        "  # Assign values to upper half\n",
        "  matrix[colinds[perm],rowinds[perm]] = vech.reshape(vech.shape[0])\n",
        "  \n",
        "  # Return vectorised half-matrix\n",
        "  return(matrix)\n",
        "\n",
        "# Example:\n",
        "#vech = np.array([[1],[2],[3],[4],[5],[6],[7],[8],[9],[10]])\n",
        "#matrix = vech2mat(vech)\n",
        "#print(vech)\n",
        "#print(matrix)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gd3_jaDe-MNy",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qutTs-8S-MuT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def vec2vech(vec):\n",
        "  \n",
        "  # Return vech\n",
        "  return(mat2vech(vec2mat(vec)))\n",
        "\n",
        "# Example\n",
        "#vec = np.array([[1],[2],[3],[2],[4],[5],[3],[5],[6]])\n",
        "#vech = vec2vech(vec)\n",
        "\n",
        "#print(vec)\n",
        "#print(vech)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z-Yp9RjT-PmL",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P8txTVtu-Pwc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def vech2vec(vech):\n",
        "  \n",
        "  # Return vec\n",
        "  return(mat2vec(vech2mat(vech)))\n",
        "\n",
        "# Example\n",
        "#vech = np.array([[1],[2],[3],[4],[5],[6],[7],[8],[9],[10]])\n",
        "#vec = vech2vec(vech)\n",
        "\n",
        "#print(vech)\n",
        "#print(vec)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kG22u6mq9mQT",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2PD6f9bD9mgx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def dupMat(n):\n",
        "  \n",
        "  # Make vech of 1:(n(n+1)/2)\n",
        "  vech = np.arange(n*(n+1)/2)\n",
        "  \n",
        "  # Convert to vec\n",
        "  vec = vech2vec(vech)\n",
        "  \n",
        "  # Make D (sparse one hot encoded vec)\n",
        "  D = scipy.sparse.csr_matrix((np.ones(n**2),(np.arange(n**2),np.int64(vec).reshape(vec.shape[0]))))\n",
        "  \n",
        "  return(D)\n",
        "\n",
        "# Example\n",
        "#print(dupMat(3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TVrM3CsGSIHL",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2FR-cwIkSIQJ",
        "colab_type": "code",
        "outputId": "c5835829-cafa-4290-a962-02cee51ecd27",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "source": [
        "def invDupMat(n):\n",
        "  \n",
        "  \n",
        "  # Make vech of 1:(n(n+1)/2)\n",
        "  vech = np.arange(n*(n+1)/2)\n",
        "  \n",
        "  # Convert to vec\n",
        "  vec = np.int64(vech2vec(vech))\n",
        "  vec = vec.reshape(vec.shape[0])\n",
        "  \n",
        "  # Work out frequency of each entry\n",
        "  freq = 1/np.bincount(vec)\n",
        "  \n",
        "  # Work out duplication matrix\n",
        "  D = scipy.sparse.csr_matrix((freq[vec],(vec,np.arange(n**2))))\n",
        "  \n",
        "  return(D)\n",
        "\n",
        "# Example\n",
        "print(invDupMat(3))"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  (0, 0)\t1.0\n",
            "  (1, 1)\t0.5\n",
            "  (1, 3)\t0.5\n",
            "  (2, 2)\t0.5\n",
            "  (2, 6)\t0.5\n",
            "  (3, 4)\t1.0\n",
            "  (4, 5)\t0.5\n",
            "  (4, 7)\t0.5\n",
            "  (5, 8)\t1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2byzJx5Kf5fS",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFCU4gxkf5r8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def blockInverse(matrix, blockSize, numBlocks):\n",
        "\n",
        "  invMatrix = scipy.sparse.csr_matrix((np.array([]), (np.array([]),np.array([]))),shape=matrix.shape)\n",
        "  \n",
        "  # For each level, invert the corresponding block on the diagonal\n",
        "  for i in range(numBlocks):\n",
        "    \n",
        "    # The block is nparams by nparams\n",
        "    blockInds = np.ix_(np.arange(i*blockSize,(i+1)*blockSize),np.arange(i*blockSize,(i+1)*blockSize))\n",
        "    \n",
        "    # Get the block\n",
        "    block = matrix[blockInds]\n",
        "    \n",
        "    # Replace it with it's inverse\n",
        "    invMatrix[blockInds]=scipy.sparse.linalg.inv(block)\n",
        "    \n",
        "  return(invMatrix)\n",
        "\n",
        "# Example - need to have loaded in data first\n",
        "\n",
        "# Get ZtZ just for the first grouping factor\n",
        "firstFactorIndices = np.ix_(np.arange(nlevels[0]*nparams[0]),np.arange(nlevels[0]*nparams[0]))\n",
        "ZtZ_f1 = ZtZ[firstFactorIndices]\n",
        "\n",
        "# Compute the block inverse for ZtZ_f1\n",
        "#t1 = time.time()\n",
        "#ZtZ_f1_inv = blockInverse(matrix=ZtZ_f1, blockSize=nparams[0], numBlocks=nlevels[0])\n",
        "#t2 = time.time()\n",
        "#blockInverse_time = t2-t1\n",
        "\n",
        "# Compare it to the inverse scipy would calculate\n",
        "#t1 = time.time()\n",
        "#ZtZ_f1_inv_sp = scipy.sparse.linalg.inv(ZtZ_f1)\n",
        "#t2 = time.time()\n",
        "#scipyInverse_time = t2-t1\n",
        "\n",
        "#print(blockInverse_time)\n",
        "#print(scipyInverse_time)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-BdQ3jh7x71k",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "77gBbWIAx8AZ",
        "colab_type": "code",
        "outputId": "126b7c33-9dd4-425e-f664-cc664d79967b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "source": [
        "def recursiveInverse(M, nparams, nlevels):\n",
        "  \n",
        "  # Check if we have a matrix we can partition into more than 1 block\n",
        "  if len(nparams) > 1:\n",
        "  \n",
        "    # Work out qc\n",
        "    qc = nparams[-1]*nlevels[-1]\n",
        "    # Make q\n",
        "    q = M.shape[0]\n",
        "\n",
        "    # Get A, B and C where M=[[A,B],[B',C]]\n",
        "    # A\n",
        "    A_inds = np.ix_(np.arange(0,(q-qc)),np.arange(0,(q-qc)))\n",
        "    A = M[A_inds]\n",
        "\n",
        "    # B\n",
        "    B_inds = np.ix_(np.arange(0,(q-qc)),np.arange((q-qc),q))\n",
        "    B = M[B_inds].toarray() # B is dense\n",
        "\n",
        "    # C\n",
        "    C_inds = np.ix_(np.arange((q-qc),q),np.arange((q-qc),q))\n",
        "    C = M[C_inds].toarray() # C is small and now only involved in dense mutliplys\n",
        "\n",
        "    # Recursive inverse A\n",
        "    if nparams[:-1].shape[0] > 1:\n",
        "\n",
        "      Ainv = recursiveInverse(A, nparams[:-1], nlevels[:-1]).toarray()\n",
        "\n",
        "    else:\n",
        "\n",
        "      #Ainv = blockInverse(A, nparams[0], nlevels[0]) - much slower\n",
        "      Ainv = scipy.sparse.linalg.inv(scipy.sparse.csc_matrix(A)).toarray()\n",
        "\n",
        "    # Schur complement\n",
        "    S = C-np.matmul(np.matmul(B.transpose(),Ainv),B)\n",
        "    Sinv = np.linalg.inv(S)\n",
        "\n",
        "    # Top Left Hand Side of inverse\n",
        "    TLHS = Ainv + np.matmul(np.matmul(np.matmul(np.matmul(Ainv,B),Sinv),B.transpose()),Ainv)\n",
        "\n",
        "\n",
        "    # Top Right Hand Side of inverse\n",
        "    TRHS = -np.matmul(np.matmul(Ainv,B),Sinv)\n",
        "\n",
        "\n",
        "    # Bottom Right Hand Side of inverse\n",
        "    BRHS = Sinv\n",
        "\n",
        "    # Join together\n",
        "    top = np.hstack((TLHS,TRHS))\n",
        "    bottom = np.hstack((TRHS.transpose(), BRHS))\n",
        "\n",
        "    # Make Minv\n",
        "    Minv = np.vstack((top, bottom))\n",
        "    \n",
        "  else:\n",
        "    \n",
        "    # If we have only one block; invert it\n",
        "    Minv = scipy.sparse.linalg.inv(scipy.sparse.csc_matrix(M)).toarray() \n",
        "  \n",
        "  return(Minv)\n",
        "\n",
        "# Example\n",
        "t1 = time.time()\n",
        "ZtZinv_rec = recursiveInverse(scipy.sparse.csc_matrix(ZtZ), nparams, nlevels)\n",
        "t2 = time.time()\n",
        "inv_rec_time = t2-t1\n",
        "\n",
        "t1 = time.time()\n",
        "ZtZinv_sp = scipy.sparse.linalg.inv(scipy.sparse.csc_matrix(ZtZ))\n",
        "t2 = time.time()\n",
        "inv_sp_time = t2-t1\n",
        "\n",
        "t1 = time.time()\n",
        "ZtZinv_np = np.linalg.inv(ZtZ)\n",
        "t2 = time.time()\n",
        "inv_np_time = t2-t1\n",
        "\n",
        "\n",
        "print('Distance (norm) from identity (scipy)')\n",
        "print(np.linalg.norm(np.matmul(ZtZinv_sp.toarray(),ZtZ)-np.eye(ZtZ.shape[0])))\n",
        "print('Distance (norm) from identity (numpy)')\n",
        "print(np.linalg.norm(np.matmul(ZtZinv_np,ZtZ)-np.eye(ZtZ.shape[0])))\n",
        "print('Distance (norm) from identity (rec)')\n",
        "print(np.linalg.norm(np.matmul(ZtZinv_rec,ZtZ)-np.eye(ZtZ.shape[0])))\n",
        "\n",
        "print(inv_sp_time)\n",
        "print(inv_np_time)\n",
        "print(inv_rec_time)\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Distance (norm) from identity (scipy)\n",
            "289.8902619901572\n",
            "Distance (norm) from identity (numpy)\n",
            "1955.403906774292\n",
            "Distance (norm) from identity (rec)\n",
            "381.4130003726553\n",
            "0.01392817497253418\n",
            "0.0003743171691894531\n",
            "0.015040397644042969\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AyCloufz0W3U",
        "colab_type": "text"
      },
      "source": [
        "#### Force symmetric\n",
        "\n",
        "The below function takes in a matrix $X$ and returns a symmetric matrix $X_s$ given by:\n",
        "\n",
        "$$X_s = (X+X^T)/2$$\n",
        "\n",
        "The effect of this is that if $X$ is expected to be symmetric but is not due to computational error, any antisymmetric errors will be resolved/reduced."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CR4xAjhO0W-f",
        "colab_type": "code",
        "outputId": "ccc3ef1a-a11a-4cda-d633-b431fa212ae4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "import numba\n",
        "@numba.jit\n",
        "def forceSym(x):\n",
        "  \n",
        "  # Force it to be symmetric\n",
        "  return((x+x.transpose())/2)\n",
        "\n",
        "#print(ZtZ-forceSym(ZtZ))\n",
        "\n",
        "t1 = time.time()\n",
        "ZtZ = forceSym(forceSym(ZtZ))\n",
        "print(time.time()-t1)\n",
        "\n",
        "t1 = time.time()\n",
        "ZtZ = forceSym(forceSym(ZtZ))\n",
        "print(time.time()-t1)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.33131909370422363\n",
            "6.747245788574219e-05\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MPgPFpnOLxdG",
        "colab_type": "text"
      },
      "source": [
        "### Sum of square residuals\n",
        "\n",
        "The function below calculates the sum of the square residuals, $e^Te$, using the below formula:\n",
        "\n",
        "$$e^Te = (Y-X\\beta)^T(Y-X\\beta)$$ \n",
        "$$=Y^TY - 2Y^TX\\beta + \\beta^T X^TX \\beta$$\n",
        "\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `YtX`: $Y$ transpose multiplied by $X$ ($Y^TX$ in the above notation).\n",
        " - `YtY`: $Y$ transpose multiplied by $Y$ ($Y^TY$ in the above notation).\n",
        " - `XtX`: $X$ transpose multiplied by $X$ ($X^TX$ in the above notation).\n",
        " - `beta`: An estimate of the parameter vector ($\\beta$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "\n",
        " - `ete`: The sum of square residuals ($e^Te$ in the above notation)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KK0i7TkgNNZD",
        "colab_type": "code",
        "outputId": "d9832c57-22dc-45fb-f5f7-e93aee974056",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def ssr(YtX, YtY, XtX, beta):\n",
        "  \n",
        "  # Return the sum of squared residuals\n",
        "  return(YtY - 2*YtX @ beta + beta.transpose() @ XtX @ beta)\n",
        "\n",
        "t1 = time.time()\n",
        "ete = ssr(YtX, YtY, XtX, np.array([[1],[2],[3]]))\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.00023102760314941406\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LO_68uYYxCvf",
        "colab_type": "text"
      },
      "source": [
        "#### Get Factor/Level Indices\n",
        "\n",
        "This function gives the indices of the columns of the $Z$ matrix which correspond to factor $k$ level $j$. \n",
        "\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `k`: The grouping factor we need the columns of.\n",
        " - `j`: The level of the grouping factor $k$ which we are interested in.\n",
        " - `nlevels`: A vector containing the number of levels for each factor, e.g. `nlevels=[3,4]` would mean the first factor has 3 levels and the second factor has 4 levels.\n",
        " - `nparams`: A vector containing the number of parameters for each factor, e.g. `nlevels=[2,1]` would mean the first factor has 2 parameters and the second factor has 1 parameter.\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `Ikj`: The indices of the columns of $Z$ corresponding to factor $k$ level $j$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iSrodNh0zI0_",
        "colab_type": "code",
        "outputId": "974d5f48-5b11-46f9-b630-476a183db75a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import numba\n",
        "\n",
        "@numba.jit\n",
        "# k and j are both zero indexed\n",
        "def faclev_indices(k, j, nlevels, nparams):\n",
        "  \n",
        "  # Work out the starting point of the indices\n",
        "  start = np.concatenate((np.array([0]), np.cumsum(nlevels*nparams)))[k] + nparams[k]*j\n",
        "  \n",
        "  # work out the end point of the indices\n",
        "  end = start + nparams[k]\n",
        "  \n",
        "  return(np.arange(start, end))\n",
        "\n",
        "t1 = time.time()\n",
        "faclev_indices(0, 1, nlevels, nparams)\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5358004570007324\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QYv59en2gpt3",
        "colab_type": "code",
        "outputId": "c2bc077d-c6d6-4f5a-e9dd-76d6a4ab0e2f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "t1 = time.time()\n",
        "faclev_indices(0, 1, nlevels, nparams)\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "6.556510925292969e-05\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ENLuSjo07vTW",
        "colab_type": "text"
      },
      "source": [
        "### Initial Beta\n",
        "\n",
        "The below function returns the OLS estimator for $\\hat{\\beta}$, given by:\n",
        "\n",
        "$$\\hat{\\beta}_{OLS}=(X'X)^{-1}X'Y$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `XtX`: The design matrix transposed and multiplied by itself ($X'X$ in the above notation)\n",
        " - `XtY`: The design matrix transposed and multiplied by the response vector ($X'Y$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `beta`: The OLS estimate of $\\beta$ ($\\hat{\\beta}_{OLS}$ in the above notation).\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Ue2gIHo7veU",
        "colab_type": "code",
        "outputId": "871245d5-c8bd-4f7e-f8bc-43147c881fd9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "def initBeta(XtX, XtY):\n",
        "  \n",
        "  # Get the beta estimator\n",
        "  beta = np.linalg.solve(XtX,XtY)\n",
        "  \n",
        "  # Return the result\n",
        "  return(beta)\n",
        "\n",
        "print(initBeta(XtX,XtY))\n",
        "print(np.linalg.inv(XtX) @ XtY)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[0.1591389 ]\n",
            " [3.83124154]\n",
            " [3.14595612]]\n",
            "[[0.1591389 ]\n",
            " [3.83124154]\n",
            " [3.14595612]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2xeRsXZBJUGd",
        "colab_type": "text"
      },
      "source": [
        "#### Initial Sigma\n",
        "\n",
        "The function below returns an initial estimate for the Fixed Effects Variance, $\\sigma^2$. The estimator used is based on the suggested OLS estimator in Demidenko (2012) and is given by:\n",
        "\n",
        "$$\\hat{\\sigma}^2_{OLS}=\\frac{1}{n}(Y-X\\beta)^T(Y-X\\beta)$$\n",
        "$$=\\frac{1}{n}e^Te$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `ete`: The sum of square residuals ($e^Te$ in the above notation).\n",
        " - `n`: The total number of observations ($n$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `sigma2`: The OLS estimate of $\\sigma^2$ ($\\hat{\\sigma}^2_{OLS}$ in the above notation)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_STdZ1mNLWbR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def initSigma2(ete, n):\n",
        "\n",
        "  # Return the OLS estimate of sigma\n",
        "  return(1/n*ete[0,0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rdtIeA0NNsZl",
        "colab_type": "text"
      },
      "source": [
        "#### Initial D_k\n",
        "\n",
        "The function below returns an initial estimate for the Random Effects Variance matrix for the $k^{th}$ grouping factor, $D_k$. The estimator used is an adaption of the suggested estimator in Demidenko (2012) and is given by:\n",
        "\n",
        "$$vec(\\hat{D}_{k})=\\bigg[\\sum_{j=1}^{l_k}(Z_{(k,j)}^TZ_{(k,j)}) \\otimes (Z_{(k,j)}^TZ_{(k,j)})\\bigg]^{-1}vec\\bigg(\\sum_{j=1}^{l_k}[\\hat{\\sigma}^{-2}_{OLS}Z_{(k,j)}^Tee^TZ_{(k,j)} - Z_{(k,j)}^TZ_{(k,j)}]\\bigg)$$\n",
        "\n",
        "Or:\n",
        "\n",
        "$$\\hat{D}_{k}=matrix\\bigg(\\bigg[\\sum_{j=1}^{l_k}(Z_{(k,j)}^TZ_{(k,j)}) \\otimes (Z_{(k,j)}^TZ_{(k,j)})\\bigg]^{-1}vec\\bigg(\\sum_{j=1}^{l_k}[\\hat{\\sigma}^{-2}_{OLS}Z_{(k,j)}^Tee^TZ_{(k,j)} - Z_{(k,j)}^TZ_{(k,j)}]\\bigg)\\bigg)$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `k`: The grouping factor we wish to estimate $D$ for ($k$ in the above notation)\n",
        " - `lk`: The number of levels belonging to grouping factor $k$ ($l_k$ in the above notation).\n",
        " - `ZtZ`: The $Z$ matrix transposed and then multiplied by itself ($Z^TZ$ in the above notation).\n",
        " - `Zte`: The $Z$ matrix transposed and then multiplied by the OLS residuals ($Z^Te=Z^T(Y-X\\beta)$ in the above notation).\n",
        " - `sigma2`: The OLS estimate of $\\sigma^2$ ($\\hat{\\sigma}^2_{OLS}$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `Dkest`: The inital estimate of $D_k$ ($\\hat{D}_k$ in the above notation).\n",
        "\n",
        "\n",
        "###CHECK DERIVATION"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I0J2CFTrNrhq",
        "colab_type": "code",
        "outputId": "251913f9-7d8a-4220-903e-b2b0b35b322c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "def initDk(k, lk, ZtZ, Zte, sigma2):\n",
        "  \n",
        "  # Initalize D to zeros\n",
        "  invSig2ZteetZminusZtZ = np.zeros((nparams[k],nparams[k]))\n",
        "  \n",
        "  # For each level j we need to add a term\n",
        "  for j in np.arange(nlevels[k]):\n",
        "    \n",
        "    Ikj = faclev_indices(k, j, nlevels, nparams)\n",
        "\n",
        "    # Work out Z_(k, j)'Z_(k, j)\n",
        "    ZkjtZkj = ZtZ[np.ix_(Ikj,Ikj)]\n",
        "    \n",
        "    # Work out Z_(k,j)'e\n",
        "    Zkjte = Zte[Ikj,:]\n",
        "    \n",
        "    if j==0:\n",
        "      \n",
        "      # Add first Z_(k,j)'Z_(k,j) kron Z_(k,j)'Z_(k,j)\n",
        "      ZtZkronZtZ = np.kron(ZkjtZkj,ZkjtZkj.transpose())\n",
        "      \n",
        "      # Add first \\sigma^{-2}Z'ee'Z - Z_(k,j)'Z_(k,j)\n",
        "      invSig2ZteetZminusZtZ = 1/sigma2*(Zkjte @ Zkjte.transpose()) - ZkjtZkj\n",
        "      \n",
        "    else:\n",
        "      \n",
        "      # Add next Z_(k,j)'Z_(k,j) kron Z_(k,j)'Z_(k,j)\n",
        "      ZtZkronZtZ = ZtZkronZtZ + np.kron(ZkjtZkj,ZkjtZkj.transpose())\n",
        "      \n",
        "      # Add next \\sigma^{-2}Z'ee'Z - Z_(k,j)'Z_(k,j)\n",
        "      invSig2ZteetZminusZtZ = invSig2ZteetZminusZtZ + 1/sigma2*(Zkjte @ Zkjte.transpose()) - ZkjtZkj\n",
        "  \n",
        "  # Work out the final term.\n",
        "  Dkest = vec2mat(np.linalg.inv(ZtZkronZtZ) @ mat2vec(invSig2ZteetZminusZtZ)) \n",
        "  \n",
        "  return(Dkest)\n",
        "\n",
        "Zte = ZtY-ZtX @ np.array([[1],[2],[3]])\n",
        "\n",
        "t1 = time.time()\n",
        "initDk(0, nlevels[0], ZtZ, Zte , 1)\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)\n",
        "\n",
        "t1 = time.time()\n",
        "initDk(0, nlevels[0], ZtZ, Zte , 1)\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.006935596466064453\n",
            "0.003425121307373047\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hjypr01QTJh5",
        "colab_type": "text"
      },
      "source": [
        "#### Non-negative Definite D\n",
        "\n",
        "The below function takes in a covariance matrix $D$ and finds nearest projection onto the space of non-negative definite matrices $\\mathbb{D}_+$. It uses the following method taken from Demidenko (2012), page 105:\n",
        "\n",
        "If $D$ is non-negative definite and has eigenvalue decomposition $D=P\\Lambda P^T$ it's closest projection into $\\mathbb{D}_+$ is defined by the matrix below:\n",
        "\n",
        "$$\\hat{D}_+ = P\\Lambda_+P'$$\n",
        "\n",
        "Where $\\Lambda_+$ is defined by the elementwise maximum of $\\Lambda$ and 0; i.e. $\\Lambda_{+(i,j)} = max(\\Lambda_{+(i,j)},0)$.\n",
        "\n",
        "Note: This is not to be confused with the generalized inverse of the duplication matrix $\\mathcal{D}^+$.\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `D`: A square symmetric matrix.\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `D_nnd`: The nearest projection of $D$ onto the space of non-negative definite matrices $\\mathbb{D}_+$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k2rW_uOBTJ0l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def makeDnnd(D):\n",
        "  \n",
        "  # Check if we have negative eigenvalues\n",
        "  if not np.all(np.linalg.eigvals(D)>0):\n",
        "  \n",
        "    # If we have negative eigenvalues\n",
        "    eigvals,eigvecs = np.linalg.eigh(D)\n",
        "    \n",
        "    # Work out elementwise max of lambda and 0\n",
        "    lamplus = np.diag(np.maximum(eigvals,0))\n",
        "    \n",
        "    # Work out D+\n",
        "    D_nnd = eigvecs @ lamplus @ np.linalg.inv(eigvecs)\n",
        "    \n",
        "  else:\n",
        "    \n",
        "    # D is already non-negative in this case\n",
        "    D_nnd = D\n",
        "    \n",
        "  return(D_nnd)\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fWz8h5WUFn7R",
        "colab_type": "text"
      },
      "source": [
        "#### Log likelihood of $(\\beta, \\sigma^2, D)$\n",
        "\n",
        "This function returns the log likelihood of $(\\beta, \\sigma^2, D)$ which is given by the below equation:\n",
        "\n",
        "$$l(\\beta,\\sigma^2,D) = -\\frac{1}{2}\\bigg\\{ n\\text{ln}(\\sigma^2) + \\text{ln}|I+Z'ZD| + \\sigma^{-2}(e'e-e'ZD(I+Z'ZD)^{-1}Z'e)\\bigg\\}$$\n",
        "\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `n`: The total number of observations.\n",
        " - `ZtZ`: The $Z$ matrix transposed and then multiplied by Z ($Z'Z$ in the above notation).\n",
        " - `Zte`: The $Z$ matrix transposed and then multiplied by the OLS residuals ($Z'e=Z'(Y-X\\beta)$ in the above notation).\n",
        " - `ete`: The OLS residuals transposed and then multiplied by themselves ($e'e=(Y-X\\beta)'(Y-X\\beta)$ in the above notation).\n",
        " - `sigma2`: The fixed effects variance ($\\sigma^2$ in the above notation).\n",
        " - `DinvIplusZtZD`: The product $D(I+Z'ZD)^{-1}$.\n",
        " - `D`: The random effects variance-covariance matrix ($D$ in the above notation)\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `llh`: The log likelihood of $(\\beta, \\sigma^2, D)$ ($l(\\beta,\\sigma^2,D)$ in the above notation).\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3oaXHL55Fnlv",
        "colab_type": "code",
        "outputId": "cd6e7545-885e-479a-d862-26aa99fbeee7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "source": [
        "def llh(n, ZtZ, Zte, ete, sigma2, DinvIplusZtZD,D):\n",
        "  \n",
        "  # Work out the log likelihood\n",
        "  llh = -0.5*(n*np.log(sigma2) + np.log(np.linalg.det(np.eye(ZtZ.shape[0]) + ZtZ @ D)) + (1/sigma2)*(ete - forceSym(Zte.transpose() @ DinvIplusZtZD @ Zte)))\n",
        "  \n",
        "  # Return result\n",
        "  return(llh)\n",
        "\n",
        "Ddict = dict()\n",
        "print(RFXVar_REst)\n",
        "for k in np.arange(len(nparams)):\n",
        "  \n",
        "  Ddict[k] = RFXVar_REst[np.ix_(np.arange(2*k,2*(k+1)),np.arange(2*k,2*(k+1)))]#makeDnnd(initDk(k, nlevels[k], ZtZ, Zte, sigma2))#\n",
        "  print(Ddict[k])\n",
        "\n",
        "# Matrix version\n",
        "D = np.array([])\n",
        "for i in np.arange(len(nparams)):\n",
        "  \n",
        "  for j in np.arange(nlevels[i]):\n",
        "    \n",
        "    if i == 0 and j == 0:\n",
        "\n",
        "      D = Ddict[i]\n",
        "\n",
        "    else:\n",
        "\n",
        "      D = scipy.linalg.block_diag(D, Ddict[i])\n",
        "\n",
        "beta = beta_True\n",
        "q = Z.shape[1]\n",
        "sigma2 = 1\n",
        "Zte = ZtY - ZtX @ beta\n",
        "ete = ssr(YtX, YtY, XtX, beta)\n",
        "\n",
        "DinvIplusZtZD = D @ np.linalg.inv(np.eye(q) + ZtZ @ D)\n",
        "t1 = time.time()\n",
        "loglh = llh(n, ZtZ, Zte, ete, sigma2, DinvIplusZtZD,D)\n",
        "t2 = time.time()\n",
        "\n",
        "print(loglh)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 1.24022707  0.36034323  0.          0.        ]\n",
            " [ 0.36034323  5.0719078   0.          0.        ]\n",
            " [ 0.          0.         20.68631165 -0.79595202]\n",
            " [ 0.          0.         -0.79595202  0.21015464]]\n",
            "[[1.24022707 0.36034323]\n",
            " [0.36034323 5.0719078 ]]\n",
            "[[20.68631165 -0.79595202]\n",
            " [-0.79595202  0.21015464]]\n",
            "[[-683.03796665]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "70shU6hKlDMR",
        "colab_type": "text"
      },
      "source": [
        "#### Derivative of $l$ with respect to $\\beta$\n",
        "\n",
        "The below function calculates the derivative of the log likelihood with respect to $\\beta$. This is given by the following equation:\n",
        "\n",
        "$$\\frac{\\delta l}{\\delta \\beta} = \\sigma^{-2}X'(I+ZDZ')^{-1}(Y-X\\beta)$$\n",
        "$$ = \\sigma^{-2}X'(I-ZD(I+Z'ZD)^{-1}Z')(Y-X\\beta)$$\n",
        "$$ = \\sigma^{-2}X'(Y-X\\beta)-X'ZD(I+Z'ZD)^{-1}Z'(Y-X\\beta) $$\n",
        "$$ = \\sigma^{-2}X'e-X'ZD(I+Z'ZD)^{-1}Z'e$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `XtZ`: The $X$ matrix transposed and then multiplied by Z ($X^TZ$ in the above notation).\n",
        " - `Zte`: The $Z$ matrix transposed and then multiplied by the OLS residuals ($Z^Te=Z^T(Y-X\\beta)$ in the above notation).\n",
        " - `sigma2`: The fixed effects variance ($\\sigma^2$ in the above notation).\n",
        " - `DinvIplusZtZD`: The product $D(I+Z'ZD)^{-1}$.\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `dldb`: The derivative of $l$ with respect to $\\beta$.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pX6LfLOBlDVn",
        "colab_type": "code",
        "outputId": "a6d2edf7-2838-48b9-c2d8-ce65f13f9af9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        }
      },
      "source": [
        "def get_dldB(sigma2, Xte, XtZ, DinvIplusZtZD, Zte):\n",
        "  \n",
        "  # Return the derivative\n",
        "  return(1/sigma2*(Xte - (XtZ @ DinvIplusZtZD @ Zte)))\n",
        "\n",
        "### Test example\n",
        "\n",
        "p = X.shape[1]\n",
        "beta = np.random.randn(p,1)\n",
        "\n",
        "sigma2=1.1\n",
        "D = np.array([])\n",
        "q=Z.shape[1]\n",
        "n=Z.shape[0]\n",
        "for i in np.arange(len(nparams)):\n",
        "  \n",
        "  for j in np.arange(nlevels[i]):\n",
        "    \n",
        "    if i == 0 and j == 0:\n",
        "\n",
        "      D = initDk(i, nlevels[i], ZtZ, Zte, sigma2)\n",
        "\n",
        "    else:\n",
        "\n",
        "      D = scipy.linalg.block_diag(D, initDk(i, nlevels[i], ZtZ, Zte, sigma2))\n",
        "      \n",
        "IplusZtZD = np.eye(q) + ZtZ @ D\n",
        "DinvIplusZtZD = D @ np.linalg.inv(IplusZtZD)\n",
        "\n",
        "print(X.shape)\n",
        "\n",
        "dldb1 = (sigma2)**(-1)*(X.transpose() @ np.linalg.inv(np.eye(n) + Z @ D @ Z.transpose()) @ (Y - X @ beta))\n",
        "\n",
        "Xte = XtY- XtX @ beta\n",
        "\n",
        "Zte = ZtY- ZtX @ beta\n",
        "\n",
        "DinvIplusZtZD = D @ np.linalg.inv(np.eye(q) + ZtZ @ D)\n",
        "\n",
        "dldb2 = get_dldB(sigma2, Xte, XtZ, DinvIplusZtZD, Zte)\n",
        "\n",
        "print(dldb1)\n",
        "print(dldb2)\n",
        "\n",
        "\n",
        "print(dldb2-dldb1)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1000, 3)\n",
            "[[7.68865739e-01]\n",
            " [2.12518659e+03]\n",
            " [2.35592937e+03]]\n",
            "[[7.68865739e-01]\n",
            " [2.12518659e+03]\n",
            " [2.35592937e+03]]\n",
            "[[-1.71237802e-10]\n",
            " [ 1.40969405e-07]\n",
            " [ 1.04723767e-08]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "pL8p641GAgxF"
      },
      "source": [
        "#### Derivative of $l$ with respect to $\\sigma^2$\n",
        "\n",
        "The below function calculates the derivative of the log likelihood with respect to $\\beta$. This is given by the following equation:\n",
        "\n",
        "$$\\frac{\\delta l}{\\delta \\sigma^2} = -\\frac{n}{2\\sigma^2} + \\frac{1}{2\\sigma^4}(Y-X\\beta)'(I+ZDZ')^{-1}(Y-X\\beta)$$\n",
        "$$  = -\\frac{n}{2\\sigma^2} + \\frac{1}{2\\sigma^4}e'(I+ZDZ')^{-1}e$$\n",
        "$$  = -\\frac{n}{2\\sigma^2} + \\frac{1}{2\\sigma^4}e'(I-ZD(I+ZZ'D)^{-1}Z')e$$\n",
        "$$  = -\\frac{n}{2\\sigma^2} + \\frac{1}{2\\sigma^4}(e'e-e'ZD(I+ZZ'D)^{-1}Z'e)$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `n`: The number of observations.\n",
        " - `ete`: The OLS residuals transposed and then multiplied by themselvess ($e^Te=(Y-X\\beta)^T(Y-X\\beta)$ in the above notation).\n",
        " - `Zte`: The $Z$ matrix transposed and then multiplied by the OLS residuals ($Z^Te=Z^T(Y-X\\beta)$ in the above notation).\n",
        " - `sigma2`: The fixed effects variance ($\\sigma^2$ in the above notation).\n",
        " - `DinvIplusZtZD`: The product $D(I+Z'ZD)^{-1}$.\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `dldsigma2`: The derivative of $l$ with respect to $\\sigma^2$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "R0T-_zfJAhRK",
        "outputId": "5797f8bd-2dac-4e87-c120-947ccdc6715d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "def get_dldsigma2(n, ete, Zte, sigma2, DinvIplusZtZD):\n",
        "  \n",
        "  # Return the bottom expression in the above derivation\n",
        "  return(-n/(2*sigma2) + 1/(2*(sigma2**2))*(ete - forceSym(Zte.transpose() @ DinvIplusZtZD @ Zte)))\n",
        "\n",
        "\n",
        "# Toy example - recalculate the required inputs\n",
        "Zte = Z.toarray().transpose() @ (Y - X @ beta)\n",
        "ete = (Y - X @ beta).transpose() @ (Y - X @ beta)\n",
        "DinvIplusZtZD = D @ np.linalg.inv(np.eye(q) + ZtZ @ D)\n",
        "\n",
        "# Time the code\n",
        "t1 = time.time()\n",
        "dldsigma2_1 = get_dldsigma2(n, ete, Zte, sigma2, DinvIplusZtZD)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "\n",
        "dldsigma2_2 = -n/(2*sigma2) + 1/(2*(sigma2**2))*(Y - X @ beta).transpose() @ np.linalg.inv(np.eye(n) + (Z @ D @ Z.transpose())) @ (Y - X @ beta)\n",
        "\n",
        "\n",
        "ZtZ = Z.toarray().transpose() @ Z.toarray()\n",
        "\n",
        "# Cannot test against n by n inversion as n by n inversion is extremely bad\n",
        "print(np.amax(np.eye(n) - Z.toarray() @ D @ np.linalg.inv(np.eye(q) + ZtZ @ D) @ Z.toarray().transpose()-np.linalg.inv(np.eye(n) + (Z.toarray() @ D @ Z.toarray().transpose()))))\n",
        "\n",
        "tmp1 = (Y - X @ beta).transpose() @ np.linalg.inv(np.eye(n) + (Z.toarray() @ D @ Z.toarray().transpose())) @ (Y - X @ beta)\n",
        "\n",
        "tmp2 = (Y - X @ beta).transpose() @ (np.eye(n) - Z.toarray() @ D @ np.linalg.inv(np.eye(q) + ZtZ @ D) @ Z.toarray().transpose()) @ (Y - X @ beta)\n",
        "\n",
        "#print(tmp1)\n",
        "#print(tmp2)\n",
        "print(tmp2-tmp1)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.0003077983856201172\n",
            "2.7351480260361105e-10\n",
            "[[-6.68573193e-07]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g0yUC5yE1qin",
        "colab_type": "text"
      },
      "source": [
        "#### Derivative of $l$ with respect to $D_k$\n",
        "\n",
        "The below function calculates the derivative of the log likelihood with respect to $D_k$, the random effects covariance matrix for factor $k$. This is given by the following equation:\n",
        "\n",
        "$$\\frac{\\delta l}{\\delta D_k} = \\frac{1}{2}\\sum_{j=1}^{l_k}(T_{(k,j)}u)(T_{(k,j)}u)'-\\frac{1}{2}\\sum_{j=1}^{l_k}T_{(k,j)}T_{(k,j)}'$$\n",
        "\n",
        "Where $T_{(i,j)}=Z'_{(i,j)}(I+ZDZ')^{-\\frac{1}{2}}$ and $u=\\sigma^{-1}(I+ZDZ')^{-\\frac{1}{2}}(Y-X\\beta)\\sim N(0,\\mathbb{I})$.\n",
        "\n",
        "$$= \\frac{1}{2\\sigma^2}\\sum_{j=1}^{l_k}Z'_{(k,j)}(I+ZDZ')^{-1}ee'(I+ZDZ')^{-1}Z_{(k,j)} - \\frac{1}{2}\\sum_{j=1}^{l_k}Z'_{(k,j)}(I+ZDZ')^{-1}Z_{(k,j)}$$\n",
        "\n",
        "$$= \\frac{1}{2\\sigma^2}\\sum_{j=1}^{l_k}Z'_{(k,j)}(I-ZD(I+Z'ZD)^{-1}Z')ee'(I-ZD(I+Z'ZD)^{-1}Z')'Z_{(k,j)} - \\frac{1}{2}\\sum_{j=1}^{l_k}Z'_{(k,j)}(I-ZD(I+Z'ZD)^{-1})Z_{(k,j)}$$\n",
        "\n",
        "$$= \\frac{1}{2\\sigma^2}\\sum_{j=1}^{l_k}(Z'_{(k,j)}e-Z'_{(k,j)}ZD(I+Z'ZD)^{-1}Z'e)(Z'_{(k,j)}e-Z'_{(k,j)}ZD(I+Z'ZD)^{-1}Z'e)' - \\frac{1}{2}\\sum_{j=1}^{l_k}Z'_{(k,j)}Z_{(k,j)}-Z'_{(k,j)}ZD(I+Z'ZD)^{-1}Z'Z_{(k,j)}$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `k`: The factor we wish to estimate the derivative of the covariance matrix of.\n",
        " - `nlevels`: A vector containing the number of levels for each factor, e.g. `nlevels=[3,4]` would mean the first factor has 3 levels and the second factor has 4 levels.\n",
        " - `nparams`: A vector containing the number of parameters for each factor, e.g. `nlevels=[2,1]` would mean the first factor has 2 parameters and the second factor has 1 parameter.\n",
        " - `ZtZ`: The $Z$ matrix transposed and then multiplied by itself ($Z^TZ$ in the above notation).\n",
        " - `Zte`: The $Z$ matrix transposed and then multiplied by the OLS residuals ($Z^Te=Z^T(Y-X\\beta)$ in the above notation).\n",
        " - `sigma2`: The fixed effects variance ($\\sigma^2$ in the above notation).\n",
        " - `DinvIplusZtZD`: The product $D(I+Z'ZD)^{-1}$.\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `dldD`: The derivative of $l$ with respect to $D_k$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_D43Adi61quy",
        "colab_type": "code",
        "outputId": "21e0502a-93d1-43c9-a859-e1fee9129fcd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        }
      },
      "source": [
        "def get_dldDk(k, nlevels, nparams, ZtZ, Zte, sigma2, DinvIplusZtZD):\n",
        "\n",
        "  # Initalize the derivative to zeros\n",
        "  dldDk = np.zeros((nparams[k],nparams[k]))\n",
        "\n",
        "  # For each level j we need to add a term\n",
        "  for j in np.arange(nlevels[k]):\n",
        "\n",
        "    # Get the indices for the kth factor jth level\n",
        "    Ikj = faclev_indices(k, j, nlevels, nparams)\n",
        "\n",
        "    # Get (the kj^th columns of Z)^T multiplied by Z\n",
        "    Z_kjtZ = ZtZ[Ikj,:]\n",
        "    Z_kjte = Zte[Ikj,:]\n",
        "\n",
        "    # Get the first term of the derivative\n",
        "    Z_kjtVinve = Z_kjte - (Z_kjtZ @ DinvIplusZtZD @ Zte)\n",
        "    firstterm = 1/sigma2 * forceSym(Z_kjtVinve @ Z_kjtVinve.transpose())\n",
        "    \n",
        "    # Get (the kj^th columns of Z)^T multiplied by (the kj^th columns of Z)\n",
        "    Z_kjtZ_kj = ZtZ[np.ix_(Ikj,Ikj)]\n",
        "    secondterm = forceSym(Z_kjtZ_kj) - forceSym(Z_kjtZ @ DinvIplusZtZD @ Z_kjtZ.transpose())\n",
        "    #print('TT')\n",
        "    #print(secondterm)\n",
        "    \n",
        "    if j == 0:\n",
        "      \n",
        "      # Start a running sum over j\n",
        "      dldDk = firstterm - secondterm\n",
        "      \n",
        "    else:\n",
        "    \n",
        "      # Add these to the running sum\n",
        "      dldDk = dldDk + firstterm - secondterm\n",
        "      \n",
        "    #print(j)\n",
        "    #print(dldDk)\n",
        "\n",
        "  # Halve the sum (the coefficient of a half was not included in the above)\n",
        "  dldDk = forceSym(dldDk/2)\n",
        "\n",
        "  # Store it in the dictionary\n",
        "  return(dldDk)\n",
        "\n",
        "IplusZtZD = np.eye(q) + ZtZ @ D\n",
        "DinvIplusZtZD = D @ np.linalg.inv(IplusZtZD)\n",
        "\n",
        "ZtZ = Z.toarray().transpose() @ Z.toarray()\n",
        "Zte = Z.toarray().transpose() @ (Y - X @ beta)\n",
        "\n",
        "k=0\n",
        "sigma2=0.5\n",
        "Zte = ZtY - ZtX @ beta\n",
        "\n",
        "D = np.eye(D.shape[0])\n",
        "\n",
        "DinvIplusZtZD = D @ np.linalg.inv(np.eye(q) + ZtZ @ D)\n",
        "\n",
        "t1 = time.time()\n",
        "dldDk = get_dldDk(k, nlevels, nparams, ZtZ, Zte, sigma2, DinvIplusZtZD)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "t1 = time.time()\n",
        "dldDk = get_dldDk(k, nlevels, nparams, ZtZ, Zte, sigma2, DinvIplusZtZD)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "sqrtinvIplusZDZt = forceSym(scipy.linalg.sqrtm(np.eye(n) - Z @ DinvIplusZtZD @ Z.transpose()))\n",
        "for j in np.arange(nlevels[k]):\n",
        "\n",
        "  Ikj = faclev_indices(k, j, nlevels, nparams)\n",
        "  \n",
        "  #print(Zte[Ikj,:] - (Z[:,Ikj].transpose() @ (Y - X @ beta)))\n",
        "  \n",
        "  Z_kjt = Z[:,Ikj].transpose()\n",
        "  Tkj = Z_kjt @ sqrtinvIplusZDZt\n",
        "\n",
        "  u = 1/np.sqrt(sigma2)*sqrtinvIplusZDZt @ (Y - X @ beta)\n",
        "  \n",
        "\n",
        "  Tkju = Tkj @ u\n",
        "\n",
        "  TkjuTkjut = Tkju @ Tkju.transpose()\n",
        "  \n",
        "  TkjTkjt = Tkj @ Tkj.transpose()\n",
        "\n",
        "  \n",
        "  if j == 0:\n",
        "    \n",
        "    sum1 = TkjuTkjut\n",
        "    sum2 = TkjTkjt\n",
        "    \n",
        "  else:\n",
        "    \n",
        "    sum1 = sum1 + TkjuTkjut\n",
        "    sum2 = sum2 + TkjTkjt\n",
        "    \n",
        "  \n",
        "dldDk2 = 0.5*(sum1-sum2)\n",
        "print(dldDk2)\n",
        "print(dldDk)\n",
        "print(dldDk-dldDk2)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.0024018287658691406\n",
            "0.00263214111328125\n",
            "[[17.14865514-1.87009500e-15j  8.61485578+3.04352884e-15j]\n",
            " [ 8.61485578+3.04352884e-15j 91.80894228+3.37081270e-14j]]\n",
            "[[17.14865514  8.61485578]\n",
            " [ 8.61485578 91.80894228]]\n",
            "[[ 4.85300689e-12+1.87009500e-15j -7.93640709e-11-3.04352884e-15j]\n",
            " [-7.93640709e-11-3.04352884e-15j -4.37694325e-10-3.37081270e-14j]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F-i38jdBJ8RR",
        "colab_type": "text"
      },
      "source": [
        "#### Covariance of $\\frac{\\delta l}{\\delta \\beta}$ \n",
        "\n",
        "The below function calculates the covariance between the derivative of the log likelihood with respect to $\\beta$, given by the below formula:\n",
        "\n",
        "$$\\text{cov}\\bigg(\\frac{\\delta l(\\theta | y)}{\\delta \\beta}\\bigg) = \\sigma^{-2} X'(I+ZDZ')^{-1}X$$\n",
        "$$= \\sigma^{-2} X'(I-ZD(I+Z'ZD)^{-1}Z')X$$\n",
        "$$= \\sigma^{-2} (X'X-X'ZD(I+Z'ZD)^{-1}Z'X)$$\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `XtZ`: $X$ transpose multiplied by $Z$.\n",
        " - `XtX`: $X$ transpose multiplied by $X$.\n",
        " - `ZtZ`: $Z$ transpose multiplied by $Z$.\n",
        " - `DinvIplusZtZD`: $D(I+Z'ZD)^{-1}$ in the above notation.\n",
        " - `sigma2`: The fixed effects variance ($\\sigma^2$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `covdldbeta`: The covariance of the derivative of the log likelihood with respect to $\\beta$."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EH4TsQPIJ8ZI",
        "colab_type": "code",
        "outputId": "a0784aa7-5709-4c8b-ff71-3671ca390de7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "def get_covdldbeta(XtZ, XtX, ZtZ, DinvIplusZtZD, sigma2):\n",
        "  \n",
        "  # Return the covariance of the derivative\n",
        "  return((1/sigma2)*(XtX - forceSym(XtZ @ DinvIplusZtZD @ XtZ.transpose())))\n",
        "\n",
        "## test\n",
        "\n",
        "IplusZtZD = np.eye(q) + ZtZ @ D\n",
        "DinvIplusZtZD = D @ np.linalg.inv(IplusZtZD)\n",
        "\n",
        "t1 = time.time()\n",
        "covdldbeta_1 = get_covdldbeta(XtZ, XtX, ZtZ, DinvIplusZtZD, sigma2)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "\n",
        "covdldbeta_2 = sigma2**(-1) * (X.transpose() @ np.linalg.inv(np.eye(n) + Z.toarray() @ D @ Z.toarray().transpose()) @ X)\n",
        "\n",
        "#print(covdldbeta_1)\n",
        "#print(covdldbeta_2)\n",
        "\n",
        "# Inverting the n by n is (perhaps obviously) awful though... this was just a\n",
        "# rough check... see below how far out from the identity the inversion mutliplied\n",
        "# by the original matrix is\n",
        "#print(np.linalg.inv(np.eye(n) + Z.toarray() @ D @ Z.toarray().transpose()) @ (np.eye(n) + Z.toarray() @ D @ Z.toarray().transpose()))\n",
        "\n",
        "#print(np.linalg.inv(IplusZtZD) @ IplusZtZD)\n",
        "#print(D)\n",
        "print('solve vs inv')\n",
        "#print((np.linalg.solve(IplusZtZD, D) @ IplusZtZD)-D)\n",
        "#print(DinvIplusZtZD)\n",
        "#print(np.linalg.solve(IplusZtZD, D)-DinvIplusZtZD)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.00014543533325195312\n",
            "solve vs inv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uh61qPojq_xg",
        "colab_type": "text"
      },
      "source": [
        "#### Covariance of $\\frac{\\delta l}{\\delta \\text{vech}(D_{k})}$ and $\\frac{\\delta l}{\\delta \\sigma^2}$\n",
        "\n",
        "The below function calculates the covariance between the derivative of the log likelihood with respect to $\\text{vech}(D_{k})$ and the derivative with respect to $\\sigma^2$.\n",
        "\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `k`: The number of the first factor ($k$ in the above notation).\n",
        " - `sigma2`: The fixed effects variance ($\\sigma^2$ in the above notation).\n",
        " - `nlevels`: A vector containing the number of levels for each factor, e.g. `nlevels=[3,4]` would mean the first factor has 3 levels and the second factor has 4 levels.\n",
        " - `nparams`: A vector containing the number of parameters for each factor, e.g. `nlevels=[2,1]` would mean the first factor has 2 parameters and the second factor has 1 parameter.\n",
        " - `ZtZ`: $Z$ transpose multiplied by $Z$.\n",
        " - `DinvIplusZtZD`: $D(I+Z'ZD)^{-1}$ in the above notation.\n",
        " - `invDupMatdict`: A dictionary of inverse duplication matrices such that `invDupMatdict[k]` = $\\mathcal{D}_k^+$.\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `covdldDdldsigma2`: The covariance between the derivative of the log likelihood with respect to $\\text{vech}(D_{k_1})$ and the derivative with respect to $\\sigma^2$.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OEx5GJJkq_8-",
        "colab_type": "code",
        "outputId": "fda05701-1902-40aa-b2c6-70fcb9ad8d82",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        }
      },
      "source": [
        "def get_covdldDkdsigma2(k, sigma2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict):\n",
        "  \n",
        "  # Sum of R_(k, j) over j\n",
        "  RkSum = np.zeros(nparams[k],nparams[k])\n",
        "\n",
        "  for j in np.arange(nlevels[k]):\n",
        "\n",
        "    # Get the indices for the kth factor jth level\n",
        "    Ikj = faclev_indices(k, j, nlevels, nparams)\n",
        "\n",
        "    # Work out R_(k, j)\n",
        "    Rkj = ZtZ[np.ix_(Ikj,Ikj)] - forceSym(ZtZ[Ikj,:] @ DinvIplusZtZD @ ZtZ[:,Ikj])\n",
        "\n",
        "    # Add together\n",
        "    RkSum = RkSum + Rkj\n",
        "\n",
        "  # Multiply by duplication matrices and save\n",
        "  covdldDdldsigma2 = 1/(2*sigma2) * invDupMatdict[k] @ mat2vec(RkSum)\n",
        "  \n",
        "  return(covdldDdldsigma2)\n",
        "\n",
        "\n",
        "\n",
        "invDupMatdict = dict()\n",
        "for i in np.arange(len(nparams)):\n",
        "  \n",
        "  invDupMatdict[i] = invDupMat(nparams[i])\n",
        "\n",
        "print(ZtZ)\n",
        "print(sigma2)\n",
        "k=0\n",
        "t1 = time.time()\n",
        "get_covdldDkdsigma2(k, sigma2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "\n",
        "t1 = time.time()\n",
        "covdldDdldsigma2=get_covdldDkdsigma2(k, sigma2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict)\n",
        "t2 = time.time()\n",
        "print(t2-t1)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 5.00000000e+01 -3.53848226e+01  0.00000000e+00 ... -1.89673372e+01\n",
            "   1.60000000e+01 -4.77010601e+01]\n",
            " [-3.53848226e+01  2.12177714e+04  0.00000000e+00 ...  5.72911850e+02\n",
            "  -9.70764641e+01  7.68081988e+02]\n",
            " [ 0.00000000e+00  0.00000000e+00  5.00000000e+01 ...  2.00888087e+01\n",
            "   1.90000000e+01 -8.31505044e+01]\n",
            " ...\n",
            " [-1.89673372e+01  5.72911850e+02  2.00888087e+01 ...  3.07902515e+04\n",
            "   0.00000000e+00  0.00000000e+00]\n",
            " [ 1.60000000e+01 -9.70764641e+01  1.90000000e+01 ...  0.00000000e+00\n",
            "   3.45000000e+02  8.14148559e+01]\n",
            " [-4.77010601e+01  7.68081988e+02 -8.31505044e+01 ...  0.00000000e+00\n",
            "   8.14148559e+01  3.27603629e+04]]\n",
            "0.5\n",
            "0.0019021034240722656\n",
            "0.0018661022186279297\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qW-p5bOy4K-u",
        "colab_type": "text"
      },
      "source": [
        "#### Covariance of $\\frac{\\delta l}{\\delta \\text{vech}(D_{k_1})}$ and $\\frac{\\delta l}{\\delta \\text{vech}(D_{k_2})}$\n",
        "\n",
        "The below function calculates the covariance between the derivative of the log likelihood with respect to $\\text{vech}(D_{k_1})$ and the derivative with respect to $\\text{vech}(D_{k_2})$.\n",
        "\n",
        "$$\\text{cov}\\bigg(\\frac{\\delta l(\\theta | y)}{\\delta \\text{vech}(D_{k_2})},\\frac{\\delta l(\\theta | y)}{\\delta \\text{vech}(D_{k_2})}\\bigg)=\\frac{1}{2}\\mathcal{D}_{k_1}^+\\sum_{j=1}^{l_{k_2}}\\sum_{i=1}^{l_{k_1}}(R_{(k_1,k_2,i,j)}\\otimes R_{(k_1,k_2, i,j)})\\mathcal{D}_{k_2}^{+'}$$\n",
        "\n",
        "\n",
        "\n",
        "Where $R_{(k_1,k_2,i,j)}=Z_{(k_1,i)}'(I+ZDZ')^{-1}Z_{(k_2,j)}=Z_{(k_1,i)}'Z_{(k_2,j)} - Z_{(k_1,i)}'ZD(I+Z'ZD)^{-1}Z_{(k_2,j)}$.\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `k1`: The number of the first factor ($k_1$ in the above notation).\n",
        " - `k2`: The number of the second factor ($k_2$ in the above notation).\n",
        " - `nlevels`: A vector containing the number of levels for each factor, e.g. `nlevels=[3,4]` would mean the first factor has 3 levels and the second factor has 4 levels.\n",
        " - `nparams`: A vector containing the number of parameters for each factor, e.g. `nlevels=[2,1]` would mean the first factor has 2 parameters and the second factor has 1 parameter.\n",
        " - `ZtZ`: $Z$ transpose multiplied by $Z$.\n",
        " - `DinvIplusZtZD`: $D(I+Z'ZD)^{-1}$ in the above notation.\n",
        " - `invDupMatdict`: A dictionary of inverse duplication matrices such that `invDupMatdict[k]` = $\\mathcal{D}_k^+$\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `covdldDk1dldk2`: The covariance between the derivative of the log likelihood with respect to $\\text{vech}(D_{k_1})$ and the derivative with respect to $\\text{vech}(D_{k_2})$.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VoAsQQtM4LLs",
        "colab_type": "code",
        "outputId": "03d908e6-821c-4405-e6b5-27186c4c58e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 545
        }
      },
      "source": [
        "def get_covdldDk1Dk2(k1, k2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict):\n",
        "  \n",
        "  # Sum of R_(k1, k2, i, j) kron R_(k1, k2, i, j) over i and j \n",
        "  for i in np.arange(nlevels[k1]):\n",
        "\n",
        "    for j in np.arange(nlevels[k2]):\n",
        "      \n",
        "      # Get the indices for the k1th factor jth level\n",
        "      Ik1i = faclev_indices(k1, i, nlevels, nparams)\n",
        "      Ik2j = faclev_indices(k2, j, nlevels, nparams)\n",
        "      \n",
        "      # Work out R_(k1, k2, i, j)\n",
        "      Rk1k2ij = ZtZ[np.ix_(Ik1i,Ik2j)] - (ZtZ[Ik1i,:] @ DinvIplusZtZD @ ZtZ[:,Ik2j])\n",
        "      \n",
        "      # Work out Rk1k2ij kron Rk1k2ij\n",
        "      RkRt = np.kron(Rk1k2ij,Rk1k2ij)\n",
        "      \n",
        "      # Add together\n",
        "      if (i == 0) and (j == 0):\n",
        "      \n",
        "        RkRtSum = RkRt\n",
        "      \n",
        "      else:\n",
        "        \n",
        "        RkRtSum = RkRtSum + RkRt\n",
        "    \n",
        "  # Multiply by duplication matrices and save\n",
        "  covdldDk1dldk2 = 1/2 * invDupMatdict[k1] @ RkRtSum @ invDupMatdict[k2].transpose()\n",
        "  \n",
        "  # Return the result\n",
        "  return(covdldDk1dldk2)\n",
        "\n",
        "\n",
        "# Example\n",
        "\n",
        "# Check against alternative expression\n",
        "Ztmp = Z.toarray()\n",
        "\n",
        "IplusZDZt = np.eye(n) + Z.toarray() @ D @ Z.toarray().transpose()\n",
        "\n",
        "invIplusZDZt = np.linalg.inv(IplusZDZt)\n",
        "\n",
        "DinvIplusZtZD = D @ np.linalg.inv(np.eye(q) + Z.toarray().transpose() @ Z.toarray() @ D)\n",
        "\n",
        "invhalfIplusZDZt = scipy.linalg.sqrtm(np.linalg.inv(IplusZDZt))\n",
        "\n",
        "\n",
        "invDupMatdict = dict()\n",
        "for i in np.arange(len(nparams)):\n",
        "  \n",
        "  invDupMatdict[i] = invDupMat(nparams[i])\n",
        "  \n",
        "t1 = time.time()\n",
        "examplecov = get_covdldDk1Dk2(0, 0, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict)\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)\n",
        "print(examplecov)\n",
        "k1 = 0\n",
        "k2 = 0\n",
        "\n",
        "t1 = time.time()\n",
        "examplecov = get_covdldDk1Dk2(k1, k2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict)\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)\n",
        "print(examplecov)\n",
        "\n",
        "print(invDupMatdict[k1])\n",
        "\n",
        "for j in np.arange(nlevels[k1]):\n",
        "  \n",
        "  Ikj = faclev_indices(k1, j, nlevels, nparams)\n",
        "\n",
        "  Tkj = Z[:,Ikj].transpose() @ invhalfIplusZDZt \n",
        "  \n",
        "  if j == 0:\n",
        "    \n",
        "    sumTkT = np.kron(Tkj,Tkj)\n",
        "  \n",
        "  else:\n",
        "    \n",
        "    sumTkT = np.kron(Tkj,Tkj) + sumTkT\n",
        "\n",
        "    \n",
        "for j in np.arange(nlevels[k2]):\n",
        "  \n",
        "  Ikj = faclev_indices(k2, j, nlevels, nparams)\n",
        "\n",
        "  Tkj = Z[:,Ikj].transpose() @ invhalfIplusZDZt \n",
        "  \n",
        "  Tkjt = Tkj.transpose()\n",
        "  \n",
        "  if j == 0:\n",
        "    \n",
        "    sumTtkTt = np.kron(Tkjt,Tkjt)\n",
        "  \n",
        "  else:\n",
        "    \n",
        "    sumTtkTt = np.kron(Tkjt,Tkjt) + sumTtkTt\n",
        "    \n",
        "\n",
        "# ------------------------------------------------------------------------------\n",
        "for i in np.arange(nlevels[k1]):\n",
        "  \n",
        "  for j in np.arange(nlevels[k2]):\n",
        "  \n",
        "    \n",
        "    Ik1i = faclev_indices(k1, i, nlevels, nparams)\n",
        "    Ik2j = faclev_indices(k2, j, nlevels, nparams)\n",
        "\n",
        "    Tk1iTk2jt = Z[:,Ik1i].transpose() @ invIplusZDZt @ Z[:,Ik2j]\n",
        "\n",
        "    if i==0 and j == 0:\n",
        "\n",
        "      sumTTtkTTt = np.kron(Tk1iTk2jt,Tk1iTk2jt)\n",
        "\n",
        "    else:\n",
        "\n",
        "      sumTTtkTTt = np.kron(Tk1iTk2jt,Tk1iTk2jt) + sumTTtkTTt\n",
        "\n",
        "    \n",
        "\n",
        "print('function')\n",
        "print(examplecov)\n",
        "print('rec1')\n",
        "print(1/2 * invDupMatdict[k1] @ sumTkT @ sumTtkTt @ invDupMatdict[k2].transpose())\n",
        "print('rec2')\n",
        "print(1/2 * invDupMatdict[k1] @ sumTTtkTTt @ invDupMatdict[k2].transpose())\n",
        "\n",
        "print(np.abs(1/2 * invDupMatdict[k1] @ sumTkT @ sumTtkTt @ invDupMatdict[k2].transpose() - examplecov))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.03927969932556152\n",
            "[[9.13266263e+00 1.72708615e-04 1.42758393e-07]\n",
            " [1.72708621e-04 4.68737815e+00 1.77855087e-04]\n",
            " [1.42758392e-07 1.77855093e-04 9.99895092e+00]]\n",
            "0.039875030517578125\n",
            "[[9.13266263e+00 1.72708615e-04 1.42758393e-07]\n",
            " [1.72708621e-04 4.68737815e+00 1.77855087e-04]\n",
            " [1.42758392e-07 1.77855093e-04 9.99895092e+00]]\n",
            "  (0, 0)\t1.0\n",
            "  (1, 1)\t0.5\n",
            "  (1, 2)\t0.5\n",
            "  (2, 3)\t1.0\n",
            "function\n",
            "[[9.13266263e+00 1.72708615e-04 1.42758393e-07]\n",
            " [1.72708621e-04 4.68737815e+00 1.77855087e-04]\n",
            " [1.42758392e-07 1.77855093e-04 9.99895092e+00]]\n",
            "rec1\n",
            "[[9.13266263e+00-4.63547667e-16j 1.72708620e-04+1.89074819e-15j\n",
            "  1.42758392e-07+9.00332051e-20j]\n",
            " [1.72708620e-04+1.89074819e-15j 4.68737815e+00-2.54905143e-15j\n",
            "  1.77855093e-04+1.86476330e-15j]\n",
            " [1.42758392e-07+9.00332051e-20j 1.77855093e-04+1.86476330e-15j\n",
            "  9.99895092e+00-1.11119836e-14j]]\n",
            "rec2\n",
            "[[9.13266263e+00 1.72708620e-04 1.42758392e-07]\n",
            " [1.72708620e-04 4.68737815e+00 1.77855093e-04]\n",
            " [1.42758392e-07 1.77855093e-04 9.99895092e+00]]\n",
            "[[1.95454229e-14 5.73889331e-12 7.57656286e-16]\n",
            " [7.40065622e-14 7.09032878e-12 6.15710741e-12]\n",
            " [8.75121346e-18 7.24620932e-14 3.02460299e-11]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IVa9K3vmAGKQ",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "#### KKT conditions\n",
        "\n",
        "The Karush–Kuhn–Tucker conditions are given by:\n",
        "\n",
        " 1. For all $k$, all eigenvalues of $\\hat{D_k}$ are non-negative.\n",
        " 2. For all $k$; there exists a non-positive $\\lambda_k$ such that $\\frac{\\delta l}{\\delta D_k}=\\lambda_k \\text{adj}(D_k)$\n",
        " \n",
        "These are assessed by the below function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rGeODb8gANs0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "### WIP: DIDNT WORK VERY WELL - MAY REVISIT SOMETIME\n",
        "\n",
        "def assessKKT(Dk, dldDk, tol):\n",
        "  \n",
        "  # Check condition 1 is satisfied\n",
        "  eigvalsD = np.linalg.eigvals(Dk)\n",
        "  \n",
        "  # If any of the eigenvalues are negative, we return false; the conditions are \n",
        "  # not satisfied.\n",
        "  if np.any(eigvalsD < -tol):\n",
        "    return(False)\n",
        "  \n",
        "  #print('1passed')\n",
        "  # Check condition 2 is satisfied\n",
        "  adjDk = np.matrix.getH(Dk)\n",
        "  \n",
        "  # Seperate into zero and non-zero elements #### NEED TO THINK ABOUT HOW TO ASSESS THIS BIT\n",
        "  #print('np.abs(adjDk)/np.mean(np.abs(adjDk))>tol')\n",
        "  #print(np.abs(adjDk))\n",
        "  #print(np.abs(adjDk)/np.mean(np.abs(adjDk))>tol)\n",
        "  dldDkNonZero = dldDk[np.abs(adjDk)/np.mean(np.abs(adjDk))>tol]\n",
        "  dldDkZero = dldDk[np.abs(adjDk)/np.mean(np.abs(adjDk))<tol]\n",
        "  \n",
        "  adjDkNonZero = adjDk[np.abs(adjDk)/np.mean(np.abs(adjDk))>tol]\n",
        "  adjDkZero = adjDk[np.abs(adjDk)/np.mean(np.abs(adjDk))<tol]\n",
        "  \n",
        "  # We need that  an element of adj(D) = 0 implies the corresponding element of\n",
        "  # dldDk = 0\n",
        "  #print(dldDkZero)\n",
        "  if np.any(np.abs(dldDkZero)>tol):\n",
        "    return(False)\n",
        "  \n",
        "  #print('2passed')\n",
        "  \n",
        "  # And we also need that the ratio between the nonzero elements must be \n",
        "  # constant\n",
        "  nonZeroRatio = np.divide(dldDkNonZero/np.std(dldDkNonZero), adjDkNonZero/np.std(adjDkNonZero))\n",
        "  nonZeroRatioRelativeError = (nonZeroRatio-np.mean(nonZeroRatio))\n",
        "  print(nonZeroRatio)\n",
        "  print(nonZeroRatioRelativeError)\n",
        "  if np.any(nonZeroRatioRelativeError>tol) or (np.mean(nonZeroRatio) >= tol):\n",
        "    #print(nonZeroRatioRelativeError)\n",
        "    return(False)\n",
        "  \n",
        "  else:\n",
        "    print('3passed')\n",
        "    return(True)\n",
        "  \n",
        "#print(Ddict[k])\n",
        "#print(dldDdict[k])\n",
        "\n",
        "#print(assessKKT(Ddict[k], dldDdict[k], 1e-6))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ze-tnNcl6cmK",
        "colab_type": "text"
      },
      "source": [
        "## Fisher Scoring implementation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gC2U0nts6sO_",
        "colab_type": "text"
      },
      "source": [
        "The below function is the \"one-voxel\" implementation of Fisher Scoring.\n",
        "\n",
        "---\n",
        "\n",
        "The following inputs are required for this function:\n",
        "\n",
        "---\n",
        "\n",
        " - **ZtX**: Z transpose multiplied by X.\n",
        " - **ZtY**: Z transpose multiplied by Y.\n",
        " - **XtX**: X transpose multiplied by X.\n",
        " - **ZtZ**: Z transpose multiplied by Z.\n",
        " - **XtY**: X transpose multiplied by Y.\n",
        " - **YtX**: Y transpose multiplied by X.\n",
        " - **YtZ**: Y transpose multiplied by Z.\n",
        " - **XtZ**: X transpose multiplied by Z.\n",
        " - **YtY**: Y transpose multiplied by Y.\n",
        " - **nlevels**: A vector containing the number of levels for each factor, e.g. `nlevels=[3,4]` would mean the first factor has 3 levels and the second factor has 4 levels.\n",
        " - **nparams**: A vector containing the number of parameters for each factor, e.g. `nlevels=[2,1]` would mean the first factor has 2 parameters and the second factor has 1 parameter.\n",
        " - **tol**: The tolerance for convergence.\n",
        " \n",
        " \n",
        "---\n",
        "\n",
        "The following outputs are given by this function:\n",
        "\n",
        "---\n",
        "\n",
        " - **paramVec**: A vector containing the $\\beta$, $\\sigma^2$ and $D$ estimates for the mixed model.\n",
        " - **bvals**: A vector of b values for the mixed model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Nj_W4Ls6btZ",
        "colab_type": "code",
        "outputId": "a87cbc03-2bcb-40fb-d419-e966e8582990",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def FS(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nparams, tol, n):\n",
        "  \n",
        "  # Useful scalars\n",
        "  # ------------------------------------------------------------------------------\n",
        "\n",
        "  # Number of factors, r\n",
        "  r = len(nlevels)\n",
        "\n",
        "  # Number of random effects, q\n",
        "  q = np.sum(np.dot(nparams,nlevels))\n",
        "\n",
        "  # Number of fixed effects, p\n",
        "  p = XtX.shape[0]\n",
        "\n",
        "  # Initial estimates\n",
        "  # ------------------------------------------------------------------------------\n",
        "\n",
        "  # Inital beta\n",
        "  beta = initBeta(XtX, XtY)\n",
        "\n",
        "  # Work out e'e\n",
        "  ete = ssr(YtX, YtY, XtX, beta)\n",
        "\n",
        "  # Initial sigma2\n",
        "  sigma2 = initSigma2(ete, n)\n",
        "\n",
        "  Zte = ZtY - (ZtX @ beta)\n",
        "\n",
        "  # Inital D\n",
        "  # Dictionary version\n",
        "  Ddict = dict()\n",
        "  for k in np.arange(len(nparams)):\n",
        "\n",
        "    Ddict[k] = makeDnnd(initDk(k, nlevels[k], ZtZ, Zte, sigma2))\n",
        "    \n",
        "  # Matrix version\n",
        "  D = np.array([])\n",
        "  for i in np.arange(len(nparams)):\n",
        "\n",
        "    for j in np.arange(nlevels[i]):\n",
        "\n",
        "      if i == 0 and j == 0:\n",
        "\n",
        "        D = Ddict[i]\n",
        "\n",
        "      else:\n",
        "\n",
        "        D = scipy.linalg.block_diag(D, Ddict[i])\n",
        "\n",
        "  # Duplication matrices\n",
        "  # ------------------------------------------------------------------------------\n",
        "  invDupMatdict = dict()\n",
        "  for i in np.arange(len(nparams)):\n",
        "\n",
        "    invDupMatdict[i] = invDupMat(nparams[i])\n",
        "\n",
        "  # Index variables\n",
        "  # ------------------------------------------------------------------------------\n",
        "  # Work out the total number of paramateres\n",
        "  tnp = np.int32(p + 1 + np.sum(nparams*(nparams+1)/2))\n",
        "\n",
        "  # Indices for submatrics corresponding to Dks\n",
        "  FishIndsDk = np.int32(np.cumsum(nparams*(nparams+1)/2) + p + 1)\n",
        "  FishIndsDk = np.insert(FishIndsDk,0,p+1)\n",
        "  print('inds',FishIndsDk)\n",
        "\n",
        "  Zte = ZtY - (ZtX @ beta)\n",
        "\n",
        "  # Inverse of (I+Z'ZD) multiplied by D\n",
        "  IplusZtZD = np.eye(q) + ZtZ @ D\n",
        "  DinvIplusZtZD = forceSym(D @ scipy.sparse.linalg.inv(scipy.sparse.csc_matrix(IplusZtZD)))\n",
        "\n",
        "  # Step size lambda\n",
        "  lam = 1\n",
        "  \n",
        "  # Initial log likelihoods\n",
        "  llhprev = np.inf\n",
        "  llhcurr = -np.inf\n",
        "  \n",
        "  counter = 0\n",
        "  while np.abs(llhprev-llhcurr)>tol:\n",
        "    \n",
        "    print('nit', counter)\n",
        "    counter = counter+1\n",
        "    \n",
        "    # Change current likelihood to previous\n",
        "    llhprev = llhcurr\n",
        "\n",
        "    # Matrices needed later by many calculations:\n",
        "    # ----------------------------------------------------------------------------\n",
        "    # X transpose e and Z transpose e\n",
        "    Xte = XtY - (XtX @ beta)\n",
        "    Zte = ZtY - (ZtX @ beta)\n",
        "\n",
        "    # Inverse of (I+Z'ZD) multiplied by D\n",
        "    IplusZtZD = np.eye(q) + (ZtZ @ D)\n",
        "    DinvIplusZtZD = forceSym(D @ scipy.sparse.linalg.inv(scipy.sparse.csc_matrix(IplusZtZD)))\n",
        "\n",
        "    # Sum of squared residuals\n",
        "    ete = ssr(YtX, YtY, XtX, beta)\n",
        "\n",
        "    # Derivatives\n",
        "    # ----------------------------------------------------------------------------\n",
        "\n",
        "    # Derivative wrt beta\n",
        "    dldB = get_dldB(sigma2, Xte, XtZ, DinvIplusZtZD, Zte)\n",
        "\n",
        "    # Derivative wrt sigma^2\n",
        "    dldsigma2 = get_dldsigma2(n, ete, Zte, sigma2, DinvIplusZtZD)\n",
        "    \n",
        "    # For each factor, factor k, work out dl/dD_k\n",
        "    dldDdict = dict()\n",
        "    for k in np.arange(len(nparams)):\n",
        "      # Store it in the dictionary\n",
        "      dldDdict[k] = get_dldDk(k, nlevels, nparams, ZtZ, Zte, sigma2, DinvIplusZtZD)\n",
        "\n",
        "    # Covariances\n",
        "    # ----------------------------------------------------------------------------\n",
        "\n",
        "    # Covariance of dl/dsigma2\n",
        "    covdldsigma2 = n/(2*(sigma2**2))\n",
        "\n",
        "    # Construct the Fisher Information matrix\n",
        "    # ----------------------------------------------------------------------------\n",
        "    FisherInfoMat = np.zeros((tnp,tnp))\n",
        "\n",
        "    # Add dl/dbeta covariance\n",
        "    FisherInfoMat[np.ix_(np.arange(p),np.arange(p))] = get_covdldbeta(XtZ, XtX, ZtZ, DinvIplusZtZD, sigma2)\n",
        "\n",
        "    # Add dl/dsigma2 covariance\n",
        "    FisherInfoMat[p,p] = covdldsigma2\n",
        "\n",
        "    # Add dl/dsigma2 dl/dD covariance\n",
        "    for k in np.arange(len(nparams)):\n",
        "\n",
        "      # Assign to the relevant block\n",
        "      FisherInfoMat[p, FishIndsDk[k]:FishIndsDk[k+1]] = get_covdldDkdsigma2(k, sigma2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict).reshape(FishIndsDk[k+1]-FishIndsDk[k])\n",
        "      FisherInfoMat[FishIndsDk[k]:FishIndsDk[k+1],p] = FisherInfoMat[p, FishIndsDk[k]:FishIndsDk[k+1]].transpose()\n",
        "\n",
        "    # Add dl/dD covariance\n",
        "    for k1 in np.arange(len(nparams)):\n",
        "\n",
        "      for k2 in np.arange(k1+1):\n",
        "\n",
        "        IndsDk1 = np.arange(FishIndsDk[k1],FishIndsDk[k1+1])\n",
        "        IndsDk2 = np.arange(FishIndsDk[k2],FishIndsDk[k2+1])\n",
        "\n",
        "        # Get covariance between D_k1 and D_k2 \n",
        "        FisherInfoMat[np.ix_(IndsDk1, IndsDk2)] = get_covdldDk1Dk2(k1, k2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict)\n",
        "        FisherInfoMat[np.ix_(IndsDk2, IndsDk1)] = FisherInfoMat[np.ix_(IndsDk1, IndsDk2)].transpose()\n",
        "\n",
        "    paramVector = np.concatenate((beta, np.array([[sigma2]])))\n",
        "    derivVector = np.concatenate((dldB, dldsigma2))\n",
        "\n",
        "    for k in np.arange(len(nparams)):\n",
        "\n",
        "      paramVector = np.concatenate((paramVector, mat2vech(Ddict[k])))\n",
        "      derivVector = np.concatenate((derivVector, mat2vech(dldDdict[k])))\n",
        "\n",
        "    FisherInfoMat = forceSym(FisherInfoMat)\n",
        "\n",
        "    paramVector = paramVector + lam*(np.linalg.inv(FisherInfoMat) @ derivVector)\n",
        "    \n",
        "    if sigma2<0:\n",
        "\n",
        "      sigspos[z]=1\n",
        "      sigma2 = np.maximum(sigma2,1e-6)\n",
        "\n",
        "    #print(paramVector)\n",
        "    beta = paramVector[0:p]\n",
        "    sigma2 = paramVector[p:(p+1)][0,0]\n",
        "\n",
        "    for k in np.arange(len(nparams)):\n",
        "\n",
        "      Ddict[k] = makeDnnd(vech2mat(paramVector[FishIndsDk[k]:FishIndsDk[k+1]]))\n",
        "      \n",
        "    for i in np.arange(len(nparams)):\n",
        "\n",
        "      for j in np.arange(nlevels[i]):\n",
        "\n",
        "\n",
        "        if i == 0 and j == 0:\n",
        "\n",
        "          D = Ddict[i]\n",
        "\n",
        "        else:\n",
        "\n",
        "          D = scipy.linalg.block_diag(D, Ddict[i])\n",
        "\n",
        "    # Update the step size\n",
        "    llhcurr = llh(n, ZtZ, Zte, ete, sigma2, DinvIplusZtZD,D)[0,0]\n",
        "    if llhprev>llhcurr:\n",
        "      lam = lam/2\n",
        "      \n",
        "  bvals = DinvIplusZtZD @ Zte\n",
        "  \n",
        "  return(paramVector, bvals)\n",
        "\n",
        "t1 = time.time()\n",
        "paramVec, bvals = FS(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nparams, 1e-6, n)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "t1 = time.time()\n",
        "paramVec, bvals = FS(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nparams, 1e-6, n)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "print(\"Predicted time on nifti of size 100x100x100 (in hours): \", 100*100*100*(t2-t1)/(60*60))\n",
        "\n",
        "\n",
        "\n",
        "print(\"U estimates (R)\")\n",
        "print(pd.read_csv('/Data/BLMM-testdata/estd_b.csv',header=None).values.reshape(23,2))\n",
        "print(\"U estimates (FS)\")\n",
        "print(bvals.reshape(23,2))\n",
        "print(\"U true\")\n",
        "print(pd.read_csv('/Data/BLMM-testdata/true_b.csv',header=None).values.reshape(23,2))"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "inds [ 4  7 10]\n",
            "nit 0\n",
            "nit 1\n",
            "nit 2\n",
            "nit 3\n",
            "nit 4\n",
            "nit 5\n",
            "nit 6\n",
            "nit 7\n",
            "nit 8\n",
            "nit 9\n",
            "nit 10\n",
            "nit 11\n",
            "nit 12\n",
            "nit 13\n",
            "nit 14\n",
            "nit 15\n",
            "nit 16\n",
            "nit 17\n",
            "nit 18\n",
            "nit 19\n",
            "nit 20\n",
            "nit 21\n",
            "nit 22\n",
            "nit 23\n",
            "nit 24\n",
            "nit 25\n",
            "nit 26\n",
            "nit 27\n",
            "nit 28\n",
            "2.5026297569274902\n",
            "inds [ 4  7 10]\n",
            "nit 0\n",
            "nit 1\n",
            "nit 2\n",
            "nit 3\n",
            "nit 4\n",
            "nit 5\n",
            "nit 6\n",
            "nit 7\n",
            "nit 8\n",
            "nit 9\n",
            "nit 10\n",
            "nit 11\n",
            "nit 12\n",
            "nit 13\n",
            "nit 14\n",
            "nit 15\n",
            "nit 16\n",
            "nit 17\n",
            "nit 18\n",
            "nit 19\n",
            "nit 20\n",
            "nit 21\n",
            "nit 22\n",
            "nit 23\n",
            "nit 24\n",
            "nit 25\n",
            "nit 26\n",
            "nit 27\n",
            "nit 28\n",
            "2.2574164867401123\n",
            "Predicted time on nifti of size 100x100x100 (in hours):  627.0601352055868\n",
            "U estimates (R)\n",
            "[[-0.16266309 -4.21724427]\n",
            " [ 1.41643004  1.41243783]\n",
            " [ 1.59527575 -1.58921871]\n",
            " [ 1.57314783  0.40497904]\n",
            " [ 1.29727291  0.93796984]\n",
            " [-1.20537013  2.19715236]\n",
            " [-0.62177368 -1.1845146 ]\n",
            " [-0.8982196  -1.34761407]\n",
            " [ 0.2105669   2.5381855 ]\n",
            " [-0.57561587 -3.43679857]\n",
            " [ 1.33808317  0.10724134]\n",
            " [-1.20676845 -2.43712662]\n",
            " [ 0.64262693  1.33629341]\n",
            " [-0.9916963  -0.08043681]\n",
            " [ 1.44782818  2.19618068]\n",
            " [-1.25815704  4.3074814 ]\n",
            " [ 0.27424557  0.87149959]\n",
            " [-0.78704685 -3.16062475]\n",
            " [-1.48916827 -0.24116661]\n",
            " [-0.47444214  3.13847411]\n",
            " [ 1.80633411  0.64368738]\n",
            " [ 2.06756827 -0.15742136]\n",
            " [-7.37272861  0.43752657]]\n",
            "U estimates (FS)\n",
            "[[-0.16277237 -4.21724426]\n",
            " [ 1.41631016  1.41243782]\n",
            " [ 1.59515855 -1.58921869]\n",
            " [ 1.5730278   0.40497904]\n",
            " [ 1.29715167  0.93796983]\n",
            " [-1.20549151  2.19715239]\n",
            " [-0.62188845 -1.18451457]\n",
            " [-0.8983342  -1.34761406]\n",
            " [ 0.21044476  2.53818554]\n",
            " [-0.57572665 -3.43679866]\n",
            " [ 1.33796518  0.10724135]\n",
            " [-1.20688117 -2.43712659]\n",
            " [ 0.64250659  1.33629342]\n",
            " [-0.99181298 -0.08043681]\n",
            " [ 1.44770551  2.19618072]\n",
            " [-1.25828042  4.30748132]\n",
            " [ 0.27412685  0.87149954]\n",
            " [-0.78715787 -3.16062473]\n",
            " [-1.48928447 -0.24116662]\n",
            " [-0.47456435  3.13847416]\n",
            " [ 1.78155856  0.64368706]\n",
            " [ 2.04279921 -0.15742126]\n",
            " [-7.39750587  0.43752703]]\n",
            "U true\n",
            "[[-0.19301143 -4.22666242]\n",
            " [ 1.43735476  1.40883489]\n",
            " [ 1.34711832 -1.58788242]\n",
            " [ 1.81825209  0.3910735 ]\n",
            " [ 1.25654173  0.94063103]\n",
            " [-1.22775617  2.19293973]\n",
            " [-0.79408731 -1.19281546]\n",
            " [-0.9898057  -1.35221872]\n",
            " [-0.03319243  2.53178969]\n",
            " [-0.65604937 -3.43139032]\n",
            " [ 1.42910277  0.11018046]\n",
            " [-0.96244844 -2.43071144]\n",
            " [ 0.70088294  1.33487455]\n",
            " [-1.3137399  -0.07784482]\n",
            " [ 1.45666816  2.19521647]\n",
            " [-1.3663195   4.30629966]\n",
            " [ 0.22757187  0.87208539]\n",
            " [-0.89571743 -3.14622337]\n",
            " [-1.51292639 -0.23242295]\n",
            " [-0.5588196   3.15054584]\n",
            " [ 1.85205881  0.64080503]\n",
            " [ 2.10133112 -0.1570577 ]\n",
            " [-7.27154738  0.44280999]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_Nazpuh-A5vx",
        "colab_type": "text"
      },
      "source": [
        "# Scaling up the computation (Random field)\n",
        "\n",
        "This section has several implemented ideas for scaling up the computation to compute several similar models at once. For simplicity it is assumed here that X and Z are the same across voxels."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WQqn-irnDVbw",
        "colab_type": "text"
      },
      "source": [
        "### Toy dataset (for a random field)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q70VwMBhDZpG",
        "colab_type": "text"
      },
      "source": [
        "#### Matrix Dimensions\n",
        "\n",
        "Below are the matrix dimensions used for **one voxel** in this example. If the model has form:\n",
        "\n",
        "$$Y=X\\beta+Zb+\\epsilon$$ With $\\epsilon \\sim N(0,\\sigma^2I_n)$ and $b \\sim N(0,\\sigma^2D)$, then the dimensions of each matrix are as follows:\n",
        "\n",
        " - $Y$: $(n \\times 1)$\n",
        " - $X$: $(n \\times p)$\n",
        " - $\\beta$: $(p \\times 1)$\n",
        " - $Z$: $(n \\times q)$\n",
        " - $b$: $(q \\times 1)$\n",
        " - $\\epsilon$: $(n \\times 1)$\n",
        " - $D$: $(p\\times p)$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wmvGEtodDXRF",
        "colab_type": "code",
        "outputId": "fd17c055-c8f4-4ec1-e19f-a9dc7f6269da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        }
      },
      "source": [
        "# Number of factors, random integer between 1 and 3\n",
        "r = np.random.randint(2,4)#np.random.randint(1,4)\n",
        "print(\"Number of grouping factors for random effects:\")\n",
        "print(r)\n",
        "\n",
        "# Number of levels, random number between 2 and 8\n",
        "nlevels = np.random.randint(2,8,size=(r))\n",
        "# Let the first number of levels be a little larger (typically like subjects)\n",
        "nlevels[0] = np.random.randint(2,35,size=1)\n",
        "nlevels = np.sort(nlevels)[::-1]\n",
        "print(\"Number of levels for each factor:\")\n",
        "print(nlevels)\n",
        "\n",
        "# Number of parameters, random number between 1 and 5\n",
        "nparams = np.random.randint(1,6,size=(r))\n",
        "print(\"Number of parameters for each factor:\")\n",
        "print(nparams)\n",
        "\n",
        "# Dimension of D\n",
        "print(\"Dimension of D, q:\")\n",
        "q = np.sum(nlevels*nparams)\n",
        "print(q)\n",
        "\n",
        "# Number of fixed effects, random number between 6 and 30\n",
        "p = np.random.randint(6,31)\n",
        "print(\"Number of fixed effects:\")\n",
        "print(p)\n",
        "\n",
        "# Number of subjects, n\n",
        "n = 1000\n",
        "print(\"Number of subjects:\")\n",
        "print(n)\n",
        "\n",
        "# Voxel dimensions\n",
        "dimv = [30,30,30]\n",
        "nv = np.prod(dimv)\n",
        "print(\"Number of voxels:\")\n",
        "print(nv)\n"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of grouping factors for random effects:\n",
            "2\n",
            "Number of levels for each factor:\n",
            "[11  5]\n",
            "Number of parameters for each factor:\n",
            "[5 3]\n",
            "Dimension of D, q:\n",
            "70\n",
            "Number of fixed effects:\n",
            "16\n",
            "Number of subjects:\n",
            "1000\n",
            "Number of voxels:\n",
            "27000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3NvSqz2ADc7X",
        "colab_type": "text"
      },
      "source": [
        "#### Fixed Effects matrix (X)\n",
        "\n",
        "For simplicity, in this example $X$ is the same for all voxels. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4689w6FrDaDO",
        "colab_type": "code",
        "outputId": "25345890-a4fd-4304-faca-2d5b93f189f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        }
      },
      "source": [
        "# Initialize empty x\n",
        "X = np.zeros((n,p))\n",
        "\n",
        "# First column is intercept\n",
        "X[:,0] = 1\n",
        "\n",
        "# Rest of the columns we will make random noise \n",
        "X[:,1:] = np.random.randn(n*(p-1)).reshape((n,(p-1)))\n",
        "\n",
        "# Image of the last 20 rows of X\n",
        "imshow(X[-20:-1,:])\n"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f9e1647fb00>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOYAAAD8CAYAAABjJ9hGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFBdJREFUeJzt3XuQ3WV9x/H3h91kyWVJQu43EDCl\nRtSoMeIUHCiKkKEGlFqwU7HaiXWgU2d6QzsDjv6j07F0LFQHNQN0VHBqIxmNQEQ6iAqyYLgHCCEx\n94Qk5L5JdvfbP/YXWTdns8+zZyHP7vm8Znb2nN/5nuf3nHP2e37n8t3nq4jAzMpy0omegJkdy4lp\nViAnplmBnJhmBXJimhXIiWlWICemWYGcmGYFcmKaFaj5RE+glqbWMdE8eXxS7En7855bYmxXcmxL\nc0fW2O3tI5JjTzqsrLG70oeGvKE56XDGPEal338AOpT++ERzZhVaU0Z8R97fiTpzgtNDj+zaSef+\n/f1eo8jEbJ48nhlfvjYpduxjo7LGPvje/cmxb562PWvs51bNSo4dsy7vrj8wM+MvJTMxR21qSo49\n9NaDWWM3v3Ry+tiTc7IBmsalP6N0bU+fB0DLrvRE7mxJf4LY8J83JcXV9VJW0iWSnpe0WtL1NS5v\nkXRXdfkjkt5Uz/7MGsWAE1NSE3ALcCkwF7ha0txeYZ8GdkXEm4GbgK8OdH9mjaSeI+YCYHVErImI\nw8CdwKJeMYuA26vT/wNcJCnzhZZZ46knMWcC63uc31BtqxkTER3AbmBiHfs0awjFfF0iabGkNklt\nnXvSP6AxG47qScyNwOwe52dV22rGSGoGxgE7ag0WEbdGxPyImN90ypg6pmU29NWTmI8CcySdIWkk\ncBWwrFfMMuCa6vSVwM/DSyaY9WvA32NGRIek64B7gSZgSUQ8I+lLQFtELAO+A/y3pNXATrqT18z6\nUVeBQUQsB5b32nZDj9PtwJ/Xsw+zRlRk5U+Ok3fkvTLOqVvp+MLkrLGnnpn+zuBg3tCM2pxenXNg\ndl4poTKq7K6b90DW2F/f8aHk2FN/m34bAfacmV711fymvA8U28eMTI6d8mB6veTm9rS4Yj6VNbPX\nODHNCuTENCuQE9OsQE5MswI5Mc0K5MQ0K5AT06xATkyzAjkxzQrkxDQr0JCvlW2flLdSSUfGamkv\nX5H3vDXuhfTYg9Pyanw7ph9Kjp02ZXfW2Dt3TEmO/fqjF2WNPf2h9MenPW3F0t+b9X/pNcFrP9KS\nNfYpz6XXv3aMSn8sI/FPykdMswI5Mc0K5MQ0K5AT06xATkyzAjkxzQrkxDQrUD29S2ZLekDSs5Ke\nkfT3NWIukLRb0srq54ZaY5nZH6qnwKAD+IeIeFxSK/CYpBUR8WyvuF9ExGV17Mes4Qz4iBkRmyPi\n8er0XuA5ju1dYmYDMCgleVXfy3cCj9S4+H2SngA2Af8YEc8Mxj6P6sq8BRN/m/5cdGB6Xrnf7j9K\nL806eXve2J0j05dT3LI/r2/TqPb0ueh36fMAeOc/PZ4ce+8D78oa++2fWpUc2/mVt2SNvSMjPDIa\n2KWW5NWdmJLGAj8EPhcRe3pd/DhwekTsk7QQ+BEwp49xFgOLAZomjqt3WmZDWr0dpUfQnZTfjYj/\n7X15ROyJiH3V6eXACEmTao3lpkJmr6nnU1nR3ZvkuYj49z5iph1tVCtpQbW/mt2+zOw19byU/RPg\nr4CnJK2stn0BOA0gIr5Jd4evz0rqoLs7wVXu9mXWv3q6fT0EHPddb0TcDNw80H2YNSpX/pgVyIlp\nViAnplmBnJhmBXJimhXIiWlWoCG/fOWkp45kxe88O31Zwlk/25s19suXj02OPTgl7+vcrgkZt7Mr\nrw63fXpncuzCBSv7D+ohp/71fefnlVH/+t63Jcd2npt3f590JD1+3MvpsU2JD6OPmGYFcmKaFciJ\naVYgJ6ZZgZyYZgVyYpoVyIlpViAnplmBnJhmBXJimhVoyJfkrb0iL37EzvTyqTUfTS+xA5j5YHqH\n445Rec+Je05P74jclV51CEBkVPDdMyFvGchoTr+/f/XLuVljd05Nv78nz96VNfa+hycnx+766L7k\n2I5HupLifMQ0K5AT06xAdSempLWSnqqaBrXVuFySvi5ptaQnJeUtt23WgAbrPeaFEfFKH5ddSvfq\n63OA9wLfqH6bWR/eiJeyi4A7otvDwHhJ09+A/ZoNWYORmAHcJ+mxqv9IbzOB9T3Ob6BGVzBJiyW1\nSWrr3LN/EKZlNnQNxkvZ8yJio6QpwApJqyLiwdxBIuJW4FaAljNnerV2a2h1HzEjYmP1exuwFFjQ\nK2QjMLvH+VnVNjPrQ73dvsZU3aSRNAa4GHi6V9gy4BPVp7PnArsjYnM9+zUb7up9KTsVWFo19GoG\nvhcR90j6W/h9Y6HlwEJgNXAA+Os692k27NWVmBGxBnhHje3f7HE6gGvr2Y9ZoxnytbJN+5ry4g+l\nF4aO3pT3GdSOuelFqiP25o19yrr0JSb3nJ53n4zekla/CXCkdVTW2C2vpt/f+87KW4p0ykPpf76v\nXJxX99w8Mv3xGfuz9LFP2pv27tEleWYFcmKaFciJaVYgJ6ZZgZyYZgVyYpoVyIlpViAnplmBnJhm\nBXJimhVoyJfkRV71GVMfTS/72jM77+5pPpgeu//CvH8GH9l6IH3sFyZljb1/VnpsV0t6aSDA6M3p\n9+G4p/PW3ezMKJtreT6vlHDCC+llirvmpB/fuhLvDh8xzQrkxDQrkBPTrEBOTLMCOTHNCuTENCuQ\nE9OsQANOTElnV/1Kjv7skfS5XjEXSNrdI+aG+qdsNvwNuMAgIp4H5gFIaqJ7rdilNUJ/ERGXDXQ/\nZo1osF7KXgS8FBHrBmk8s4Y2WIl5FfD9Pi57n6QnJP1U0lsHaX9mw1rdtbKSRgIfBj5f4+LHgdMj\nYp+khcCP6G7HV2ucxcBigKaJ45L3P3JnZsv009KXU+wYndEDHegcmR47dfzerLE3rJ6SHNu6Lu8+\nmbAqvX543aK8+2TPe9qTY2f/IO/P8dU56fGR+Ze+6QMZNcGRHhuJ9b2DccS8FHg8IrYeM4mIPRGx\nrzq9HBghqWaFdUTcGhHzI2J+0yljBmFaZkPXYCTm1fTxMlbSNFX9EyQtqPa3YxD2aTas1fVStmok\n9EHgMz229exbciXwWUkdwEHgqqplgpkdR729S/YDE3tt69m35Gbg5nr2YdaIXPljViAnplmBnJhm\nBXJimhXIiWlWICemWYGG/PKVR/44fVlHgH/5+PLk2K/eeWXW2DkdqA/fMTVr7OZz0kvhDqdXNAIw\n4kBHcuwpq/KWgZz0VHq52pqP5X3F/fKf/Vdy7PwbP5s19qnPpB+zPnzDz5NjbxmTVorpI6ZZgZyY\nZgVyYpoVyIlpViAnplmBnJhmBXJimhXIiWlWICemWYGcmGYFcmKaFUglLsHTcubMmPHla5NiZyzN\nWDMS2HN6em/4GQ+8mjX2hg+OT4498I6MvvDA2Lb0GtVp//GrrLFf+tq5ybGdE9LragHG/Tb98enK\n6/SeVRM8anve2AfO35ccO/bn6as6Pv/DmziwbX2/hc8+YpoVKCkxJS2RtE3S0z22nSpphaQXq98T\n+rjuNVXMi5KuGayJmw1nqUfM24BLem27Hrg/IuYA91fn/4CkU4EbgfcCC4Ab+0pgM3tNUmJGxIPA\nzl6bFwG3V6dvBy6vcdUPASsiYmdE7AJWcGyCm1kv9bzHnBoRm6vTW4Ba//k7E1jf4/yGapuZHceg\nfPhTra5e18e7khZLapPU1rln/2BMy2zIqicxt0qaDlD93lYjZiMwu8f5WdW2Y7ipkNlr6knMZcDR\nT1mvAe6uEXMvcLGkCdWHPhdX28zsOFK/Lvk+8GvgbEkbJH0a+ArwQUkvAh+oziNpvqRvA0TETuDL\nwKPVz5eqbWZ2HEmr5EXE1X1cdFGN2Dbgb3qcXwIsGdDszBrUkF++MqfEDuDAe9KXu3zlorzyszF3\ndiXHzv5xeskXwLor0kvyus6blzX2+GfTl8Y8NDGvBHL33PT7cMSuvMdy1Pb0ebeuz3ss93Wmv8vb\nd1r6uF2Jd59L8swK5MQ0K5AT06xATkyzAjkxzQrkxDQrkBPTrEBOTLMCOTHNCuTENCuQE9OsQEO+\nVjYyn1pm3ZG+RuL6T4zOGnv0lbuTYw+3TsoaO+ff0LecmzfvzpPTYw+efiRr7OYx6fFNm/Pmve+0\n9NrkEXvz6nBz/ut/xi/S63C37E0b2UdMswI5Mc0K5MQ0K5AT06xATkyzAjkxzQrUb2L20bfk3ySt\nkvSkpKWSara5krRW0lOSVkpqG8yJmw1nKUfM2zi2rcEK4JyIeDvwAvD541z/woiYFxHzBzZFs8bT\nb2LW6lsSEfdFxNFvVR+meyFnMxskg/Ee81PAT/u4LID7JD0mafEg7MusIdRVkifpX4EO4Lt9hJwX\nERslTQFWSFpVHYFrjbUYWAzQNDG9VfCB6ellWQCtv0t/Lpp9e97ds396+hKTLbvz5p3zHHr4lPRl\nHQGmvH9Tcmz77dOyxm4/Nb0EcvyavHK/w63pZXbtmc0fRz6XXh64ZUH6uEdWpj02Az5iSvokcBnw\nl9FHv/iI2Fj93gYspbtHZk3uXWL2mgElpqRLgH8GPhwRNVdQljRGUuvR03T3LXm6VqyZ/aGUr0tq\n9S25GWil++XpSknfrGJnSFpeXXUq8JCkJ4DfAD+JiHtel1thNsz0+yaqj74l3+kjdhOwsDq9BnhH\nXbMza1Cu/DErkBPTrEBOTLMCOTHNCuTENCuQE9OsQE5MswIN+eUrJz+WF7/t3enPRa1r8563Rl++\nNTl2+yN5NaejN6cvqDjtkfassbcwIzm2aXLW0HSll8py5O92ZI29ZXX6ZEavz3ssR+xJjx31oW3J\nsU3fS1vq0kdMswI5Mc0K5MQ0K5AT06xATkyzAjkxzQrkxDQrkBPTrEBOTLMCOTHNCjTkS/JynXF3\nzbXDatpwUd5qfU13TU2OPXT+4ayx1TUyOXbdGS1ZY7fsSo89sCD9/gM4+fH0ZSA3rcnrsj3tl+nL\ndO5+c9bQHJqQUQLZkl4C2aS0ZUt9xDQr0ECbCn1R0sZqhbyVkhb2cd1LJD0vabWk6wdz4mbD2UCb\nCgHcVDULmhcRy3tfKKkJuAW4FJgLXC1pbj2TNWsUA2oqlGgBsDoi1kTEYeBOYNEAxjFrOPW8x7yu\n6o+5RFKtzhAzgfU9zm+otplZPwaamN8AzgLmAZuBr9U7EUmLJbVJauvcs7/e4cyGtAElZkRsjYjO\niOgCvkXtZkEbgdk9zs+qtvU1ppsKmVUG2lRoeo+zV1C7WdCjwBxJZ0gaCVwFLBvI/swaTb8FBlVT\noQuASZI2ADcCF0iaR3dj2rXAZ6rYGcC3I2JhRHRIug64F2gClkTEM6/LrTAbZl63pkLV+eXAMV+l\nmNnxufLHrEBDvlZ2y/l5LdN3npNeuznxybyxD3781eTYCUqvxQQ48uzE5NgR70yfB8C7p21Ijv3N\n3W/LGvvgtPT7cNKj6a3bAXZkNHlsOpg1NB0T05aZBHhxVfq3gIfa02qefcQ0K5AT06xATkyzAjkx\nzQrkxDQrkBPTrEBOTLMCOTHNCuTENCuQE9OsQEO+JI/mvNK2zjPSa7PG/jhv7G2rxyfHtq7Je07c\n/5bO9LEfqLWgRN9eWtuaHDtyVtbQxEnpt7M9veqwW1f643PorLwu2yM2pS8Beto9h5Jjd+5Km7OP\nmGYFcmKaFciJaVYgJ6ZZgZyYZgVyYpoVyIlpVqCUVfKWAJcB2yLinGrbXcDZVch44NWImFfjumuB\nvUAn0BER8wdp3mbDWkqBwW3AzcAdRzdExF8cPS3pa8Du41z/woh4ZaATNGtEKctXPijpTbUukyTg\nY8CfDu60zBpbve8xzwe2RsSLfVwewH2SHpO0+HgDuXeJ2WvqrZW9Gvj+cS4/LyI2SpoCrJC0qmrr\nd4yIuBW4FaDlzJnJRZDjp+7NmS+vbh+bHLv+4vT26gCT525Njt29M70tPABj05dTPDg97/n24NT0\nZSNHb8kaOsu0h/PayO96y6jk2PEr8uay9rL0OtzOk9Pvv0jsTj/gI6akZuAjwF19TiJiY/V7G7CU\n2s2HzKyXel7KfgBYFRE1VwuWNEZS69HTwMXUbj5kZr30m5hVU6FfA2dL2iDp09VFV9HrZaykGZKO\n9iqZCjwk6QngN8BPIuKewZu62fA10KZCRMQna2z7fVOhiFgDZCxib2ZHufLHrEBOTLMCOTHNCuTE\nNCuQE9OsQE5MswIpIm+JxjeCpO3Aul6bJwGN8F8qjXA7G/k2nh4Rk/u7cpGJWYuktkb4f85GuJ2+\njf3zS1mzAjkxzQo0lBLz1hM9gTdII9xO38Z+DJn3mGaNZCgdMc0axpBITEmXSHpe0mpJ15/o+bwe\nJK2V9JSklZLaTvR8BoukJZK2SXq6x7ZTJa2Q9GL1O689WWH6uI1flLSxejxXSlqYM2bxiSmpCbgF\nuBSYC1wtae6JndXr5sKImDfMvkq4Dbik17brgfsjYg5wf3V+KLuNY28jwE3V4zkvIpbXuLxPxScm\n3cuRrI6INRFxGLgTWHSC52SJqjWedvbavAi4vTp9O3D5GzqpQdbHbazLUEjMmcD6Huc3VNuGm+QV\nBYeBqRGxuTq9he7VLoaj6yQ9Wb3UzXq5PhQSs1GcFxHvovsl+7WS3n+iJ/RGiO6vBYbjVwPfAM4C\n5gGbga/lXHkoJOZGYHaP87OqbcNKg60ouFXSdIDq97YTPJ9BFxFbI6IzIrqAb5H5eA6FxHwUmCPp\nDEkj6V4EbNkJntOgasAVBZcB11SnrwHuPoFzeV0cfeKpXEHm41nvgs+vu4jokHQdcC/QBCyJiGdO\n8LQG21RgaXfHCZqB7w2XFQWrVRYvACZJ2gDcCHwF+EG14uI6uttsDFl93MYLJM2j+2X6WuAzWWO6\n8sesPEPhpaxZw3FimhXIiWlWICemWYGcmGYFcmKaFciJaVYgJ6ZZgf4fjnaTnhDpsQoAAAAASUVO\nRK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CQLC39CYDqa7",
        "colab_type": "text"
      },
      "source": [
        "#### Random Effects matrix (Z)\n",
        "\n",
        "For simplicity, in this example $Z$ is the same for all voxels. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyZSa63dDi8Z",
        "colab_type": "code",
        "outputId": "3a03225b-3e4e-4436-8264-d3989b0d8ea0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        }
      },
      "source": [
        "import sparse\n",
        "# We need to create a block of Z for each level of each factor\n",
        "for i in np.arange(r):\n",
        "  \n",
        "  Zdata_factor = np.random.randn(n,nparams[i])\n",
        "  \n",
        "  if i==0:\n",
        "    \n",
        "    #The first factor should be block diagonal, so the factor indices are grouped\n",
        "    factorVec = np.repeat(np.arange(nlevels[i]), repeats=np.floor(n/max(nlevels[i],1)))\n",
        "    \n",
        "    if len(factorVec) < n:\n",
        "      \n",
        "      # Quick fix incase rounding leaves empty columns\n",
        "      factorVecTmp = np.zeros(n)\n",
        "      factorVecTmp[0:len(factorVec)] = factorVec\n",
        "      factorVecTmp[len(factorVec):n] = nlevels[i]-1\n",
        "      factorVec = np.int64(factorVecTmp)\n",
        "      \n",
        "    \n",
        "    # Crop the factor vector - otherwise have a few too many\n",
        "    factorVec = factorVec[0:n]\n",
        "    \n",
        "    # Give the data an intercept\n",
        "    Zdata_factor[:,0]=1\n",
        "    \n",
        "  else:\n",
        "    \n",
        "    # The factor is randomly arranged across subjects\n",
        "    factorVec = np.random.randint(0,nlevels[i],size=n) \n",
        "  \n",
        "  # Build a matrix showing where the elements of Z should be\n",
        "  indicatorMatrix_factor = np.zeros((n,nlevels[i]))\n",
        "  indicatorMatrix_factor[np.arange(n),factorVec] = 1\n",
        "  \n",
        "  # Need to repeat for each parameter the factor has \n",
        "  indicatorMatrix_factor = np.repeat(indicatorMatrix_factor, nparams[i], axis=1)\n",
        "  \n",
        "  # Enter the Z values\n",
        "  indicatorMatrix_factor[indicatorMatrix_factor==1]=Zdata_factor.reshape(Zdata_factor.shape[0]*Zdata_factor.shape[1])\n",
        "  \n",
        "  # Make sparse\n",
        "  Zfactor = scipy.sparse.csr_matrix(indicatorMatrix_factor)\n",
        "\n",
        "  # Put all the factors together\n",
        "  if i == 0:\n",
        "    Z = Zfactor\n",
        "  else:\n",
        "    Z = scipy.sparse.hstack((Z, Zfactor))\n",
        "\n",
        "\n",
        "Z2 = sparse.COO.from_scipy_sparse(Z)\n",
        "\n",
        "# Create an image of Z\n",
        "imshow(Z.toarray(), \\\n",
        "       interpolation='nearest', vmin=-5, vmax=5, aspect='auto')\n",
        "\n",
        "print(nlevels)\n",
        "print(nparams)\n"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[11  5]\n",
            "[5 3]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXmQXNd1n3+n1+mefQaYwWCwDDYu\nIADui7VYXEyFolhmuSJRsmyHkuliVSI7kmOXRTmp2EmcRK6kbLPKKblYlm3RpTIl00rIkmlLXERa\nkrlgJzZiGwwwM5h97+nu6e7pmz/6wcJ755Dz0Ot73eerQgHv4E73ne73zrn33LOQMQaKoihK4xGo\n9QQURVGU2qAGQFEUpUFRA6AoitKgqAFQFEVpUNQAKIqiNChqABRFURqUqhsAInqIiE4T0Tkieqra\n768oiqIUoGrmARBREMAZAA8CGAGwH8AvGmNOVm0SiqIoCoDq7wDuAnDOGDNojMkAeA7Ao1Weg6Io\nigIgVOX36wcwfNX1CIC7rx5ARE8CeBIAKBq5Pbxxnf0V8rTmm1COjwlk+LjVpjVfSlEUpUBekDmX\n0EHBo7K6ts4qN5nhkWljzPq1xlXbAKyJMeYZAM8AQHR7v9n0P/6d7f9bW1LsZxbPdNqum3cu8DEz\nzUwWWPTcr68oigtCCa5UI0tclu7iCjkfrf/yN0Nf/u2LbsZVWwOOAth81fUmS+aahfOdTBbM2L/4\npYUY/8HqG2FFUSpEroUrcUlWTkyYv35k1r4FaB3iPze717sGp9oGYD+AXUS0DQXF/1kAn7uWF4gs\n8GMLyjkEY1E2JtvtHKQoil/JR7k/pmWQq7PYFFe+M7cUp5Apy1eR2Vb7a83uLeqla0ZVDYAxJkdE\nvw7g+wCCAP7CGHPiWl4jvZk780MzYdt1dJZ/UZtf5jfM0CPBa3lrRVE8QmCFLwST/fwZT/aX7z0l\noxNK2HVIWHBDrXRJhwfeoOpOcGPMSwBeKvbnKcmVdstF+4e+cB3/wC89scpfbCLMZYqieJ7QxiST\nxX7cwmQLt/AFY7Fnf5LRyTvcQivCmYOX8d0p6Pp3+JfQ92vnbdcXvruDjUmueR6uKIpfCJzkyj7T\nwceFpvgirxEOgd3iOwMwfwOXJb9nV/jJrXwHELwkHAzrjaAoviTdJ5zpBdyFYAbSWgHnCr4zALmY\n8CU7RME+HiqKi/HKTEhRlKoTSDWGEg/3L9uuzRlh59MtuLdd4jsD4Az5BIDVJocFGOWr/XBa+Dnd\nASiK4pL46NoHz9IhsDNS6FrIjjryl5r5a5Wyo/GdAei+eZLJJs7Zs4WjM/yguIn/GDLtZZuWoih1\nTraNK19nQtpKD1+Ne3m34jsD0BTivj/TZLfC2VZuhSnn3S9BURTv42YlX25lbxwv1/8Gn8Poo9mi\nX993BmD8RzywNxS3fyhS0m/Lz0wx2cqZ7nJNS1EUv+DQoUZIByJJ19fAY0yOeJbLHxUWt7ORol/f\ndwag9W6uyKeHumzXTZf5N7qU1MpviqKArRCdSrbcOFfxALDtpstMNjrLfdLsDKDM+M4ALL3NA/rb\nZ+zX4Ye5kUhn+a8qFAhVFMUH5NuEMNAs17TBJb4YNKHqLuUlAzN0bGNV5/B++M4AtNw1zWQzM/bQ\nqMAp7tpZbRcOZ8o3LUVRqojbbN5qK3uJfEzIS0pw7bPaxY1aYKGyKtp3BkBayZu03crTxjQb03KI\n5wFItUMURVEkAkIIutlk1zXR4zwEPd3DX0s6d6i0spfwnQFoi3Hlnlq0+86iF3j6t99qdCiK4i26\nj3EdMhm3H8Bm2v2lZ3xnAC5f7mKyez963Hb9xv7dbExoSR0+iqIA5PQGG76y37hvnMlG0ctfy9F9\n0G91hnxnAKIjfHX/euZG2/W9d/Ie8we+ywt1J6WyEoqi1DXc/SLE1p/kyr4WbN9n75d1/iQ/PI6P\nFl/W3tsGIBdAfsIevpkXkjGcyRf/9OZN/LX6uL8/0MvdSaEz9rOC8CJ/qVSPYDiE5IN8RA2MojQ8\nghqIDSwx2fIED/kcfNee9yTlOKV6iz/L9LYBqAHpDfaTeBIOnfP9wiHzQX74k9iqBkBR6oXgBl5k\nMvje2nH6mQ6uoFMXW5msFk5qNQBOQmtb07a3ubLPabFRRalrVsf5c78qKPdy4jxjCAhVH2KTxTc8\nVwPgIJCwfyRBIVtsYRf/0qMzesisKH4gPmZ/VleFSgr5fdxFU+msXAlnHsOqoLFL8TSoAXCQdxSW\nWxY+3JCQxJFrUXePoviBpHAeyKiBspdwlpfOdApzbxWyol2iBsBB61n7R5Llrjq0DnFlv7idj8tr\ny2FFUUoglHQaAD4mdiZa/OsX/ZN1StbRcCfdx51ugb28IfWKcIIvNZFWFKW2OGv4S6ev2c0rTEYz\nxVfdLBZnhI/zTAAA0us1CqhspAfsX3xgQcgqvsC3BWHhi9GOY4riPdy4a2uh7CWMI5RcSmjNCnXO\n3KIGwEF43P7F5+L8ZslLRZsm1N+jKApHKgYnNRygNE/oIkf9IWlR2VAtISuNMyvcNHNlH5riyj56\nwwKTJYfayjYvRVH8yfo3uWKf49VqxL4Bla5mqgbAQWSXPfU3/iPepGF5M7fo6dNCg2F1ASmK52D1\ne+JClYCUUPmzyIoLM7d4Vw+oAXBywK7IMx18CPXxTODouzwTLNXr3S9eURoV56ra6WYBZGXfOsiX\n6At77UEilOFjKFt8olalUQPgIHT3nO16cY4r9uBl3l4y8qEZJkudF2K2FEXxJUvbhZ3CcvGF2NxA\njnplq8vc/dy6PlH066sBcLA0avfb97/Gx8xdz6388jFephrCAbKiKLWlacr+/Eo1/MPOUFEAK13V\nbyBlHMUwpePe5YTgfnaJGgAH4Vn7Rzz12DIbY87zmH+t/KkoVUR43CKLXD3mw3ygm7j5RmkgpQbA\nwaqjR0A+xbdcTUm+Okh1C6GhycpuDxWlYRHc6pn2yq7Q23fMMdnGNnvQyNmfDLAxUii5V1AD4CCy\nYF9FZHM8IWRlnXCjBb37JSuK8lNYMTihkoKktBeEM70FOGQeVvYSagAcpHc5InzmuAGILPDlR249\nz8YzS/rxKorXcFUMTiCwwp97Z9h40mclYVRDOWg9ZD90Sa8XMu+EGyF0nkcGZYXuZYqi1Jao45xP\nqrEvFXKUDoHTl+xlYbyr6mUa2gA4200CQGLr2qsDtz49qeVkLm3/yMMxfvfF3uariMgif8/ZvWpg\nFOVacTaFbxsS+ns8McZkl473VWpKNaOhDUAt6N0wb7ueGOF+xY/94lEme+UIzx3XQ2ZFuXacq/v0\n5/jh7sS4ENZdhxRtAIhoM4BnAfSiEJT1jDHmaSLqAvBtAAMAhgA8ZoyZIyIC8DSAhwEkAXzeGHOo\ntOn7j7lD623XG++YYGNeOXkDkzVd5ntSqdeooigfjPO5ybhM2Gwe4Q6e5Ab7Lrxr9zQbM3Om+xpm\nV11K2QHkAPyWMeYQEbUCOEhELwP4PIBXjTFfI6KnADwF4CsAPgFgl/XnbgBft/5uKAKOtPDJI71s\nTFQ4Y9j74GkmO7h/V/kmpig+Iig8I1LMv5SfI9XUd8PyprUXXF5W9hJFGwBjzBiAMevfS0R0CkA/\ngEcB3GsN+yaA11EwAI8CeNYYYwC8RUQdRNRnvU7DQI57KCfU8m4a5F/L/hO85ZjfDpwUpVy47bVR\nrLL3Cn0/tv+e0/uEktHFtwMozxkAEQ0AuBXA2wB6r1Lq4yi4iICCcRi+6sdGLJnNABDRkwCeBIBg\nZ/3V0klvsH9bkVnBjy/dsxUuC6soSmWITvOlmjPhFABolT/4Yx9x7jrK6/Yt2QAQUQuAvwPwZWPM\nYsHVX8AYY4iEzgcfgDHmGQDPAEB0y+b613rCb5iXvpVA/X8UilKPiImjIoILy9FMRmr+Eunn5Wrc\nUpIBIKIwCsr/W8aY71riiSuuHSLqAzBpyUcBbL7qxzdZsobiuhtHbNfDr2xlYzJtgrJ3dqpRlAbG\n6UoF+PkaALTv4VV6Z8/6J8InkFrb0ZsZ4WHjbiklCogAfAPAKWPMH131Xy8CeBzA16y/X7hK/utE\n9BwKh78Ljeb/B4DBd7bYrlvmBKsf5jdyYJRHAbnpbaoo9Ui+id/7He/x52Ymzt3IupT6KaXsAD4M\n4FcAHCOiI5bsd1FQ/N8hoicAXATwmPV/L6EQAnoOhTDQL5Tw3r4l73D5z93Gi8jtvW6Yyd4b6+Ev\nNsZ7FShKI+Cs2gsAi/cnmWzd9/kzMnOzLpyuUEoU0I/x/sb0AWG8AfDFYt+vXghstvvr2t7g27fh\nwzziJ3cPzyrWlYzSqIi73/EYE/ld2QcdrSnzW7keCJ/hv7dbNBO4yrQ4FP7czTyGKzzPI4OiQsmI\nDHihOkVR6oeQo/R8+ABX9pk2JnL/+sX/qFIMubj9CxX7kW5JMdlqTqP+FcWPbL5pnMl64ktMtv8k\n3/mvOK/LnGemBqDKLO20+/xjI8JXMCn49gV/z2oNWtQpinJtjM+3MtnFkXVM1v0O1wVzN1XWhaUG\noMpEZuzuHalkdH6T4O+fELpWKIpyzbSftq+m0j/HV+O5M1xpS8lbbsiOCj0ChHGVVvYSagCqjHF8\n8598cD8b88O/vovJpL4EXm41pyiVhHqczhEgfpD7x1O9Qmev6x2y4Rb+BkUqe7+hBqDKZLvsh74v\nntjHxgTu5GcAkBKqhX4GitII7Pqf/Bm58GluAGLXzzPZ8oX2iszJj6gBqDLBVns0z2qKR/x0v8GT\nvqY/zt1CitKovPcb3EXTtZGXYl5KFB8i2QioAagxG/p5M4pMnCd99bzEzwAmuadIURoCqRnS/LnK\nlngIbuC7jshRu3+/414e8TN2Skji9AhqAKpM4JLdbbP6A+7GWbyR/1xe8vY0hptSURhig/YFLgvy\nowIsby4uem5VSDRL9dpfK+VhZS+hBqCCSD2H4ahhMnOLyxcTlL3Uczj+jj2EdHE3TyDr3LDIZPPz\nPFKBZjTRTPEmeaEfQLqn/lZEt9951na9//gONiY2XLwaVwNQZ+QcixTK8oCz5RR3JzW3cmOSVAOg\n1CGf+Mhh2/U//PhWNqb9DN9NLFxXfQPj7PonhY+6LzfNUQNQZ2T32WsNRc7ypLKVIN+ZrB/gB2hJ\nlJBjrigVJN/OiygGZ3jwhOnjCxtJ4TuphbKvBWoA6ozsvH11HxYW8aFFfoA2dYD3JkZzYzwEiv8I\nLHDVZaSueVN8t2s67W7RmLD7TY3wKCOpB4HfUQNQZ6x/y67cp+/np2DRC3wHsNLNi9L5vZ+qUr/E\nB/g5VmKW73ajQh+NLOyy9Bwf0yh3vhqAOmPqw46tsXAGkA/zlZIJCj1K1QAoHiU5xN2Tkn9cKrVS\nLNIz0nXU/q7pdfyZSW3w7tZBDUCd0X7Mvppp+SSPS76cXs9k1MJ9qljRQ2BFuYLUtH1uj9MolNdt\n2jpoNzArvMEZYpPFv6cagDpj4Ua7Ijff38AH3cJ9nhHBLZRt8+7KRVH8gunOcFman8ORUBVgafva\nz2BGMApuUQNQZ3SctH+li8INRHN8ZR/g6QKKopSB2Ht8cbX1wSEmG/ynASar9CJMDUCdsbjDfsNE\nZ7hnNHQHLz8RfbeDycrdfEJRGpH0eq7ETx/ZwgcKyj7YZ+9znJvk2cimhQdwuEUNQJ0RcHQY6z7B\nb46xTl4NMXG7sFMQupUpinJtSMEUAUFnR2f5uCTskU3SE0lCSKxb1ADUGc5Y5fmd3K+47ZZhJjt/\ncmOlpqQonsJZQiW3KAQ7COXXpQJ0bpDyE1YFzZvs04YwSol0nbDfRLM38THnBoWD4WYhD6CElYWi\neBVnjS633bbzMb5Ljl+0PyPJrTyaLpDybj9vfcLrjKnb7ddNk3zTmO3ksvA0vxVWmzQTWPEmUjXQ\n1T6e9BiYFDKBpYxhN+8pKPJ0T37NMV5GDUCd4Ww52frRSTYmfYY3pM52CyuX5eK2vIpSaaRqoDTL\nXTnFKnuJyDxX7uGE/XrpJh7yKZWt8ArenZlSFCZud+VsbFlgYyaJGwBa8dfKRVGKJT5qv9dv/dfH\n2ZiDL+5hMudqHwAyjuA5Lyt7CX/NVlmTphF7JvCRzHY2JrwhyWShI7wxtnTDK4rfSfbb7+ufvLWb\nDyrjvR+dFXYOt/NQ7OVlni9gBBdWOVEDUGc4t6Qg7sYJ9vED3/iY1GCjXLNSlPJihHpW6/ZzRTv/\nEF/sSJ29KslKFzcmKx5pTK8GoM5Y3mS/2Vou8YdicZ4/AMt3caMQULeQ4lEoyw+BZ24R/P1VVvaV\nJn6ZP5M7HjnPZEMuX08NgI8RW046SGzhqw+3fkpnvHT4BC+3m9uzzGTZBD+MC83y98xHNMpIKY7I\nnBCRM8APYIPCfWd8HNuQ3Mif52OHthX9emoAFNcIuTHICkk0wSX+hOX7eQE6qVmHorgh0+luYeNn\nZQ9wQxcQivYG7pwv+vXVACiuSQlnBx3HeDON+T38Lt3QzRt4TE7xstSK4lWah/muI9VrXxVJPQNK\nMUKSoWMIvRHcogZAcQ118C12qoe7oXr+md/x0228xZ6i+AmpVIOz9IrZwJPRvLzTVQOguCYc5Sv7\nTAdfoczt5iulYIjvHjTIVKkFbdvtLpPkMaGgvlB1LRcXVvdOgYeVvYQaAMU15jTPFQhyDxCCaf70\nXN/LM5KPjRZ/eKUoxbI46MjeanYXjBDbusRky7P2KCMSCsZJncS8QskGgIiCAA4AGDXGPEJE2wA8\nB6AbwEEAv2KMyRBRFMCzAG4HMAPgM8aYoVLfX6keoaRQrnaLUEIixx+Co6e28nHlmZbSgIQX+d2T\n28Vj/iWMi+g5idRF7sb0+z1cjh3AlwCcAnDlJOIPAfyxMeY5IvozAE8A+Lr195wxZicRfdYa95ky\nvL9SJYL8CABo4q4dE+IGIDbMb7WVdeoEUoojuyPFZB2t3AB0xfm4wYn+iszJj5RkAIhoE4BPAvjv\nAP4DERGA+wF8zhryTQC/j4IBeNT6NwA8D+BPiYiMMRoM7hNWBfdmcJb7gLJt3Chke4VEs3nBf6Qo\nLqBxfjMuD/KVfToluF+0xMm/UOoO4E8A/A6AK3ujbgDzxpgrfoERAFfMbT+AYQAwxuSIaMEaP331\nCxLRkwCeBIBgZwndjpWyI7l7pIifmX3CIfAsH5dz6XtVFCeBTXy13/JKM5NJYZla4uSnFG0AiOgR\nAJPGmINEdG+5JmSMeQbAMwAQ3bJZNYSHkNw4S0Jr06Zpvupaf4R3nb/0r3yepaPUjNUxnpU+d5M3\n1UXTFn54nJzh80dA6EK2VNk4nVJe/cMAfp6IHgbQhMIZwNMAOogoZO0CNgEYtcaPAtgMYISIQgDa\nUTgMVnxCWDhjW75TqCx6jtdfGfuQdKt584FVvE/ncb7I6Pnli0x2dowv992UUCkn6UvePTwu2gAY\nY74K4KsAYO0AftsY80tE9LcAPoVCJNDjAF6wfuRF6/pN6/9fU/+/v3BmPQJA58tc2c/cx5NhzDK/\n1QJprzwGit+Y28PvxbkjwnZUgBw1roLn+T2c6RGi24rsCexlKrG/+AqA54joDwAcBvANS/4NAH9N\nROcAzAL4bAXeW6kgTVN81TX9Ie7aoTwfRzF+CAw1AEoNcIaB5lrK1wC+3Dh3OgvX8THx8eLzDMpi\nAIwxrwN43fr3IIC7hDFpAJ8ux/sptcEI95nUNtII2cG/eecrTPYnP/hEWealKPXKzF32nUjHUR45\nF8wU70jRTGDFNakNXLG3nRHK8t7HEwb+6O0HmUzX/0oj0HmCr5yyj/AKnkvDvKibcyeyuKu8Iaxq\nABTXSE2xl7cI9VGyfFcQGeMrF6m2iqK4QcpKz8WEe1HoHEa56pZmEKOThI5gtSgYoQZAcU22jd/I\nuS5+BrD9z7kBuPxvnb0qAYzwuG1FcYPbxUO1lX25ybfbXUC37uSRTkcO7yj69dUAKK6Jj/KHqfOW\naSYbfIyH3oVz6vBRyodUm5+EOIOsUCpf6tHrVZxNbo4e5Mq+FBOnBkBxzdIu/oQtjXcxGcV4CF0+\nrwZAKR/Lm2ugxIVNR8RRlG6liz8jdV0NVKlfnAkz4m08w1tCSkgbdmfPYbrI47Gz64QeeMKz3/MT\nfitP36pnDAonH7XfQHEhw93Z6AWQgyAy7XaZl5W9hBoAxTPkt/C+wZIPNzjGC4FNfkgoNpfSXYfC\nCazY74t0hYvDtQxJwRP8PfNR4cA6W1mDogZA8QzNB/gOYOlWbhQC25aZLDjMa6v4vSG4Uh8kBtwZ\nGEnZ52OOHUac74hb23jJa7eoAVA8g1RuOjzMhYHreORRppk/ZFpqon5Zf4DLph7iJUjySaEEiUey\nfN3AdrEp7nJNuHTDSqgBUDyDEfT1TR89x2RSJERQcBWZkJ4B1CtTdwjCaWGxUPmprEl0ms8issjH\nLdzGDVile2aoAVA8gxTGd/TiJibLt/JtcHiaPyirenfXLaaLZ5tLZ0O5duFsqMo7Q6nz3co6Pq4W\nDZL0EVE8w+odvG56PMQf4Hxc6E0svF6la6krtYNmudtDOkT1ghtw574RJhv/nlC5VNiwJrZW9oBa\nnxDFM5hTLUy23M0fgB03XGayqdd5av3Sdv8k/Cjep2mSG5PkVvtutOU8V6nnwHexEKKAaoEaAMUz\nBFb4yj6Q5A/d8EwHk7Wk1d+vVJaA4KIMz9sPlJcHhESwjHdzA9QAKJ4ht5fXC2pu4hE/6RXuKyUh\nX0xRykmyb+1Vu6Tso7N8EZO5kTstY0d4KLOb9ywFNQCKZ8iP8gcgK1QgzfRxbT99T2N0cFL8R17Q\nspGT/F7PCMUWK40aAMUztF7gyn5pq5AdGefb7M43+aHg/A3qFlJqT7ZdylHhO4VQSnIVVfYeVgOg\neIasUB165+2XmGxwspvJlrYWnwyjKBVF0OFSxFJGkIWW7UYhOseNRGSR/9yQy6mpAVA8Q8vHJpks\nm+dunOxcE5PRJp5EQyVkSCqKk3yLcMDrcDNuuXGcjRk+saHo98w1mw+8BgBeGMU9agAUz5B4g/cR\nmLmZH5Z1HeFGYf7G2sd7K9VDCsmUlCNdxwMLsqPFNSIKJNY+UypF2dcCNQCKZ0jt4UWtutq4AVjY\nxovGReaEw2KhOb1SH7iu4FmkspfIt/FAg7bj9l3mqrDplMpIewU1AIpnMPP86Vm8yN09WaHpRmhZ\nb+VGIjrDDf6q0BNYOoAttsRyYJHfYwmPJHQViz41imcwLXyFleHJwbj3xjNM9tZLeysxJcWjrAgZ\n4hLlrKd/021DTHbuB9tt16ntvEaRZDi8gndnpjQcsfO8mFdqIzcKb35fUPYa8t9QOLvJAUAozHeG\n6RnuLiy2UdCJQwNc6Cj05mVlL+Gv2Sp1TZjXgkM6Kxz2xYXDvn6hKcYEdx8p9YGzXSkA8LW3XA46\nttV+o93WN8zG/OjIDfy1PFBYrtyoAVBqhvMhljonSb1ZpTr/RlAIzlViYJCvBjPdUoEX/qZd++3l\nJzTJzL+kLrbarn9ycTcbI6n6fAcvS0KOs6doLw9ayIyU7yC63KgBUBoGKYkmPsYfgYU7hZwCwU4o\n/iTfZDfw4QXuP2y6YZ7Jli/wirNOvKzsJdQAKA1Dpl3IvhTqrQemeDTS8ian8dAdQC3JN3OLHJni\n6qyZl+LHwnX29f2qkIHrRtnXA2oAlIYhOsN3ANlt3HOcn+MGILvRMW5SaGCsVI3AMl+1S2dDC9eV\n7z2lPICmi/Z7JXQr3zkkh9rKN4kyowZAaRgS24TV/ig/O5DOGKIn7OcH6fX+jv/2O043DsBr8wNA\nrrWyeQCZTvvrZzys7CXUACgNQ8sQP9pb7hfaCAoRRamo3VCQ0IReqR5SRM5qkxAdJij79tN2mQkI\njYhy/LXmb6w/t58aAKVhWNzH3T0b+uaYbOrEeiYjfVLqhoXrnYrcu4rddNvv2fZ3hPDXViZyjd7W\nSsMQmOe3+8wkL0AXWVp7db+yTl1AijsG9vIe1oNneNG4lgv8/kzCfsawuKO8950aAKVh2PwKjxyZ\nuJO3l1wR/Pt5R5mKwJI+On6l45TdwCc3CsEBLfweMEVmmw8d28hkUp5BcmP1FxV6FysNw/DP8SfY\nRLlRCEuVRZvV518vcF9+ZV1AAeEcIjLPZVLSY6rXwz2BiagDwJ8D2IPCp/irAE4D+DaAARQa0zxm\njJkjIgLwNICHASQBfN4Yc6iU91eUa2FgH9+KS6sz8TBxpf7KACjVoeUiV/YLd/FaRtHz1S9dUuoO\n4GkA/2iM+RQRRQDEAfwugFeNMV8joqcAPAXgKwA+AWCX9eduAF+3/laUqjDydj+TkVC/nYQFYfMl\nuwFI9ukZgF8xEfsX3DwkhI/yqiEs5NMtizuFUNRZfuMV+/qlULQBIKJ2AD8L4PMAYIzJAMgQ0aMA\n7rWGfRPA6ygYgEcBPGuMMQDeIqIOIuozxowVPXtFuQakcg6dp7gs9GnemnJq1hHfrYlgvoUyzjOA\nyipe08lrCGGBnz1JTeGlLOVyUsoOYBuAKQB/SUQ3AzgI4EsAeq9S6uMAeq1/9wO4uuzeiCWzGQAi\nehLAkwAQ7OwsYXqKsjbJDfyhy8zxuLrIe/Yl4UqX7gAUd9AcV/YSkrJ3Go/Ww3zhsby5+HuxFAMQ\nAnAbgN8wxrxNRE+j4O75F4wxhkjaUL8/xphnADwDANEtm70boKv4jvAiV/ZSZEf7a3z/v7RNb8V6\nwZnEF07w+yK9xRuNXZzGQ6qYWwql/EYjAEaMMW9b18+jYAAmrrh2iKgPwJX99CiAzVf9/CZLpihV\nIXsXbziQu8SrN3bcx11As6ft+QJSxIZSPUyQG+TmYW7N4xN83PStdlmmQ8gG91ljl2Ip+rc0xowT\n0TARXW+MOQ3gAQAnrT+PA/ia9fcL1o+8CODXieg5FA5/F9T/r1STbIbf7mYd989OHu5lMrQ5NH5G\nw0JrCa3yz1/y5Sd5kJevuP9Dx2zXr7++r6yvX6qZ+w0A37IigAYBfAGFHIfvENETAC4CeMwa+xIK\nIaDnUAgD/UKJ760o10TkrBDaIejxFSEaI9RlD9tbHRdeS1HKzI9futl23TrJdytzt/AqpW4pyQAY\nY44AuEP4rweEsQbAF0t5P0VTyJVPAAAWzElEQVQpBalccGiZW4B77jjNZO+8eb1doD2IG458VPD7\nOaqSrutZZENmz3YV/Z6ZDke10Q4+JpAs/mZsDEeX0pCwvrERoSGMIHv7neuZzKnwpabkqxn+IAYi\nPPZ0dZHHgDtdGs5QRcWOlF3bfoaPy7bycYktxR3gBKRkQIdsdqF4ZV8L1AAoSpkIzPBwv/ioELYn\nKKCud+2KanavRh19EHnBcC/s4so+18XPeAIJ3b5dQQ2AopSJ2JiwQhQW8k0TfNzMzXajoFFGayDY\nR8koeEHZG+G2aBV6U4QTgn9/t3cTwRRFuYrETuEwTig213qc7wraB+ytBBcHBWevUhac7rvrN06w\nMaffGWCyfLg4ZSwZ82LdUOVGDYCilAmpS1U+wh/05Vt5x7HoOw7fsfYb+EBaB93ttlI9XGnnHGdD\npya28h8sUtn7DTUAilIm2s5ypbR4D98BmAV+CLzlgYu267NHN7Mxyk/JC9UVEjuEz1pIGAuktLLr\nFdQAKEqZWBG8NlLuQe9BfjB5muyVSlVFfTBS/RspcoqkbUGR3HTbEJMN/sN223XrJT6vyTvLNoWy\nowZAUcrEtm+NMNmlxzYx2cVf4KvSpna7XzozwktUKLXl7GvbmSzjaNiSaZMMjnfdSWoAFKVMjPwC\nV/ZSRyepzsyKI5lHswC8hzMpS2I1JlT0DHFZ7DKPTgov89db2ubhjmCKovyU+Md5NMnS2XVMZlqF\naCFnIpj2HK4YkXm7gy2zkx/Kk+CDM0X2gHBWHwWAdA9X7Dy1sPLoXaYoZWLmSA+ThQVdHxaSw5K7\na/H4NyZsJT/Nv49yOm3yzfxwuq0nwWSLky1MFliubB6DGgBFKRPxMb7SC2S5KpEOiyngXT9xvXHz\n7edt14dPbmNjYuuSTLYyzBW0GyQlnrjQzscV9eqloQZAUcrEykd5vwFziiuNlR6+LYhG7bIstOVk\npTh6cIftWlK8xSp7CSPkFESnuFHI7OKuKExV9j5QA6AoZSK9KDysPXz7L60u0wn7z+oh8AfTfZR/\nQovbhDBQ4QzVzWFuOSGhcJ04hworewk1AIpSJmKDPMErwqsDI9fcxmTm+hXbtRqAD2bmZsll5k03\nWmSB7zFi43yui7uECKIK+4XUAChKmcgIjWRMkD/BIb4BQNu79tWfV2rF1CN5Rw3/aDd3vQQPtzKZ\nFLnjhkw7/7kMPwKoCWoAFKVMhBNc2WedrSQBxCb5+n6J5xgpFcJZsyk7ypPuskUq+3Lj3AF0bp9l\nY+ZmuLFyixoARSkTRojkkYxC8n6hOf2CvUCZF8oYK2VC8Ez1/TMXXr6Pj3OWt5g/xxvOlOIuVAOg\nKGUiJ7QJNmGhv/AZId7b2a5SDwE8x7pDQsOZx2Zs17OjPMZXKj439mHhwDpTwuSKRA2AopSJ4AqX\nZbp5yGcmKqWZ2i/F9oNKTZm+TVjKO1bk5f7WnL0L6JKwyijBW6UGQFGKgPUbBpBvFkoPLxT3iEk9\nh0PvxZksvZEbmM6j3H205Mh1Kra5iVeQomMC6/ln1vIW/8wqXV+nnLD7LFre700NgKL4hHQfV/ZN\nl/kjnBLOGFaT9gL6NMtDVv2EFN9vBKMsKXtnQ/meA1yprgr9BqbucD8/v6AGQFF8QryHl4sMnOfx\nhIlZrgiDjkPlSseXe5mgY6PQ/cUhNubYyS1MJnV88ztqABTFJ5jDXNlnhQjA5kG+fA07ao8t7vSP\nG6TcZFvtK/4ThwbYmHKq+mCKH/jGJrgsLyQCJ/u0HLSiKJBrzecEWZYnGoPG6m/1Wm2atnDXWua8\n/cPOtfHSH5I5SQx44wxGDYCi+ITorFBT5jp+LhBICoXGWr2hcPxM+pKw3XIcppe737DzrKNpm5BD\nclyw+C5RA6AoPiHVK0QZCaWGA0IDkqyj+JjUpET5YGITXLmn99nreuRzfAzNFH/g7jyrSV0UjFAJ\nxl0NgKL4hK7jXLa0lSuc/F6+SsSQPfnM72GgpZBvs++a4ue4gg4J/XnE+kyOyCO/mVU1AIriE5rH\ns0w2/SD3OQeGeKaxCTauwnfizM0I3j3HxmT3d1ZrOjVFDYCi+ISRX+W1AgbWzTPZELqZLHLenkEq\nVaj0Ey1DQo2lPuGQvJd/ZoF5e5TUstCdC+v8/fm4RQ2AovgEOsurVo4f5at9ngUAkFM3eqQccbEk\nBtwpaKey9xsd79mdSrO380P/dW9yNT7k8vXVACiKT8h2CXXlN3GFsPEfuNKbuNNv3un6oWmK71ZW\nhO8yNs7Hzd9gHycd+s/u00NgRal7msb5w5/u5eMu38uVy/brxm3XQ8c2lm1efiPcb8+olvoBtO/g\n5wIL54s7F0ivd7dbSfZX3+2kBkBRfIIzgxUAgkK/ASO0Ehi82GO7buS0MEnhOylW2Zcb6rGXmO34\nIXfwJXjVCteUZACI6DcB/BoKxWyPAfgCgD4AzwHoBnAQwK8YYzJEFAXwLIDbAcwA+IwxZqiU91eU\nRiIfEgxAmrt2QjsSTJaf4FUxFe+zumh356XW8+97ZR13A7qlaANARP0A/j2A3caYFBF9B8BnATwM\n4I+NMc8R0Z8BeALA162/54wxO4noswD+EMBnip65ojQYccFHnBdW+6F/5slCIcexQKVrzCg/JR/n\noboU5Z9/+35eDGhxp/1a6ktcSu+IUl1AIQAxIsoCiAMYA3A/gM9Z//9NAL+PggF41Po3ADwP4E+J\niIwxGqCsKC5IbOcrveYh/ggnNwjhkB32n5XKRSiVQfysBZlYoM+x4I+PCuGvG4s35kUbAGPMKBH9\nbwCXAKQA/AAFl8+8MebK3TYCoN/6dz+AYetnc0S0gIKbaPrq1yWiJwE8CQDBTm/44RTFEwjJXCud\ngltI6EyWW9UooFqRb+IKWmraM7+Xj3P2BC5F2UuU4gLqRGFVvw3APIC/BfBQqRMyxjwD4BkAiG7Z\nrLsDRbFoP8bDO1eFoP+W+yaYbPL0+kpMqaFomuSr79Qmu3unb8cUGzN+qofJFq7nqs2p7KtBKS6g\nnwNwwRgzBQBE9F0AHwbQQUQhaxewCcCoNX4UwGYAI0QUQiEVZYa/rKIoEomtkv+XK43ESBeTRRL2\ncTmhfaXywUjRVSZi/04WXt/AB/V697ylFANwCcA9RBRHwQX0AIADAH4I4FMoRAI9DuAFa/yL1vWb\n1v+/pv5/RZGReg5LsZtSj4BAgmsqp8KXeg5jmDccz/XwUgpNF+2HlVnBmBghYskrmC777xQe5oev\noRsXmSwNfrju/KxTHlb2EqWcAbxNRM8DOAQgB+AwCq6bvwfwHBH9gSX7hvUj3wDw10R0DsAsChFD\niqJ4hO53hTOGDq4c5/fYD5SDy9XPKhAja4RomNiYkDwHe/VPaTeUk2r/1yElRQEZY34PwO85xIMA\n7hLGpgF8upT3UxSlckz8LFeqwUUhz2DBrlRzPbxKqbPiZtkRbI6k7G98+AyTHT6wk8kaFc0EVhQF\nABCZ4upg/R38QHnmTbufe3Wp+mpEcnNJMfKisl9nD5OKnOOuL2cjFgDItvnLveMGNQCKogAAeu8c\nZ7KR93ixoegee8MZsf+Ah88Amk7ZFX5Y6J+zws/R6xI1AIqiAADmXu3jwk2CW+iA3T+ebxF86Nwm\neAZncba0hyNkO07ZXXCJB5bZmOwk38G4RQ2AoigAgEy7EFEkHKw6cw+aZoRQVMEoNCqSOwmdPLoq\nfEk4cL/R8Tle5jWdSskeUAOgKAoAIHQDD33MDfNomMxOewhpbkxqQVN/tA5yTb54h/2zCEV5uY7V\nMaEQn9Aovha5GWoAFEUBAOTf5W3Cgjdwl0N+xK7Qci1CX+ISCpR5laXtQqmGWbsiXwVX7KXQcsn+\nOeYEW0t3LhT9+moAFEUBIEe5NL/Fa+eb++zNUtqaePGhMaH8gVdg2dOCDyUf8YYLK7HFReTRxeJz\nFtQAKIoCAAhv4av9lpu4bHrBfsI7fl5oMOzhDUA+6g3l7or1duNqZvg5QaCEGkJqABRFAQCsXOar\n/cVRvrpscpxfJu9KsjFGKmXRoOSF2v/dh6XevoKLacqu8CVVX0rIrRoARVEAANf9FQ+IP/fL3ACs\ndtkzf0OXhDBED6+ynWUkAimujKMz7hq5u0E6D5nbLVQDzfmrGqiiKHVE9n9xA0AH25gsMmo/6JSq\nZHoZN81wilX2EvEBHl2Vf6eDyaQw3Eq7q9QAKIoCAJj97iYmC93HjUIgYFeOoR/xM4DE1soqrkCW\nr5Zz63lsffth7jNf2lbdkg7JIW5EIZStqAVqABRFAQB0nuMK9MJe7t5xujQGHhlhYxLHNpZvYgL5\nsJC0Ns8b5kjK3uneWVnHx+TbeDx/xQvc1YD6+40URSmK8buFGPYwNwrG0V5y8BIP+ax0EJDUnSsX\n50Yh08sVOQtaFTYr9ajsJRrjt1QUZU2kVXVohq+qyZH3FTvPfequ4tdLILWRJ5/FLvN5BEb5/DMd\n3nC/eAE1AIqiAJBX0KvNQmhizKF8p7ifvdJIETNSOWiJ5mH77iF5Bw9j3bRunsmGTwjtHn2OGgBF\naUDElpNCcEwgLThzHDJptS+1nIwc4zVxJP97eMnho+8WSk0IyU/O8FQAiA5zt9byZsd7Cp/F8ET9\nKXsJNQCKolQFyW0jEXIsyDNtXNlLFTalQ+Bsa+3zEZqm+GSTW4RD5qTwS1U4NUANgKIoVSG8wBWc\nGM7p2ChIq30/lXNw9h8AgEDKnbJ3ZhG3DHGVnbo5VfTc1AAoilIVwrt5QtTyOC8/ER+2q6XV7bzY\nHAnllL1KvpnvfPq3zDDZ7E+428nZmSzZJ5xzTBZ/BqMGQFGUqhD4MU8Y60rwlXzrpy7bri+d8oY/\nPj7KV+3LW+3Kff1+Pmb6Vn64IlZLLWP2sVvUACiKUhUS+/hKHgFuAGbP2ZVjZFFwE9Wg41iyX4iI\nckQjTd8qGLQL3ChkP8Zr+KfGeR9NKqHSpxvUACiKUhUoxBVoLM4TzT6595Dt+sUXPsTGROaFMFAh\n6St2mau4le7qrrTF0hOXeJG96peCUwOgKEqViB3jZSUyHTwE8/mL99iuQ0KCmpSzIFXdlJR9dHPC\ndp1OCD70BSEBrg7zx9QAKIpSFaTQzfa9/DB09e+7bddL28o7DzpkL872sZ8/xsa8sX83/8F8Ldbo\nlUUNgKIoVUHy26ffWMdkkVX7uJbr59iYxUFeTtn1PGL21//Rj/awMa0jXNknBupvC6AGQFGUqhAf\n5Up1pZuPS8bs49KXeTnlUorNuTlATgx4I8/AechsNvAM6+Bw8d3X1AAoilIVEluEhidNXGY67CUd\nYue5j76cDVvckm/h8fwUtctMTkh2K6GyKGv3OM0/i9WYtoRUFMXjSIXljNAvd/umKdv1hQXeW6Ck\nA1nHRiQfETJ123hdoYBU9C5hj/H32ymBGgBFUaqCaRIU7TJPkho8Y0/8CvLoTvFA2f1EHHMQoodq\nUeFUIh9zfGZCKC25aHH5fqgBUBSlKoRbeMx/dlVQtI4VeewCV1PJjf45kJV2K6uCO0nqVcxrBpW3\n1Y4aAEVRqsLqBM8D6DjLFVrmfnuWbOJG/lp+6tgllammZT7/8BJ3IFW6mql/PkVFUXxNZFYoicAT\nYrGyYk/CkjKI/YRUplpCUvbtZ+xGYWmA/1yuh++s3KIGQFGUqtDEc76w3C9EATkatDTNuMvw9Srd\nR/nKfuYBHs6ZT3F1vHDd2v59twZGQg2AoihVYX4PP81tPctV0NIeu8skFRNCK6V6+i5pO2//2Syv\nSI2UVHa5SG/MzM3CDwrhnOX17rtjTQNARH8B4BEAk8aYPZasC8C3AQwAGALwmDFmjogIwNMAHgaQ\nBPB5Y8wh62ceB/CfrJf9A2PMN8v7qyiK4mVCS3w1az7Ke+/SZbtfKJASOoIVH/iCxR0udg9ldL2z\nSB4A0QmueiO8XQKWN1V2p+NmB/BXAP4UwLNXyZ4C8Kox5mtE9JR1/RUAnwCwy/pzN4CvA7jbMhi/\nB+AOFD7ag0T0ojGG53grilKXtJ3jsrlensUambZr92ybf9w9EpFO7u6Jb+SlsVdfE9KiK8yaBsAY\n809ENOAQPwrgXuvf3wTwOgoG4FEAzxpjDIC3iKiDiPqssS8bY2YBgIheBvAQgL8p+TdQFMUXzN4h\nlGs+yw2As1F8aFmIjhGMQriHt0bMzPDXF+P+K0jucpzJFsBl2OqfhjC9xpgx69/jAHqtf/cDGL5q\n3Iglez85g4ieBPCkdZkY+vJvn7b+vQ7AdJHz9QJ+nr+f5w7o/GuNzr/6bHUzqORDYGOMIaKyecyM\nMc8AeMYpJ6IDxpg7yvU+1cbP8/fz3AGdf63R+XuXYvdCE5ZrB9bfk5Z8FMDmq8ZtsmTvJ1cURVFq\nRLEG4EUAj1v/fhzAC1fJ/w0VuAfAguUq+j6AjxNRJxF1Avi4JVMURVFqhJsw0L9B4RB3HRGNoBDN\n8zUA3yGiJwBcBPCYNfwlFEJAz6EQBvoFADDGzBLRfwOw3xr3X68cCF8DzC3kM/w8fz/PHdD51xqd\nv0ehQsCOoiiK0mjUIvlMURRF8QBqABRFURoUzxsAInqIiE4T0Tkr69jTENFfENEkER2/StZFRC8T\n0Vnr785azvGDIKLNRPRDIjpJRCeI6EuW3Be/AxE1EdE7RHTUmv9/seTbiOht6z76NhFFaj3X94OI\ngkR0mIi+Z137ae5DRHSMiI4Q0QFL5ot7BwCs5NXnieg9IjpFRD/jp/lfK542AEQUBPB/UCgxsRvA\nLxLR7trOak3+CoUs56u5UjpjF4BXrWuvkgPwW8aY3QDuAfBF6zP3y++wAuB+Y8zNAG4B8JAVkfaH\nAP7YGLMTwByAJ2o4x7X4EoBTV137ae4AcJ8x5parYuf9cu8AhVpm/2iMuQHAzSh8D36a/7VhjPHs\nHwA/A+D7V11/FcBXaz0vF/MeAHD8quvTAPqsf/cBOF3rOV7D7/ICgAf9+DsAiAM4hEJdqmkAIUtu\nu6+89AeFHJlXAdwP4HsotJn1xdyt+Q0BWOeQ+eLeAdAO4AKs4Bi/zb+YP57eAeAaSkh4nPcrneFp\nrBpQtwJ4Gz76HSwXyhEUEhRfBnAewLwx5koxGi/fR38C4HcAXCkM0w3/zB0oFHv8AREdtMq6AP65\nd7YBmALwl5YL7s+JqBn+mf8143UDUHeYwjLC87G3RNQC4O8AfNkYYytU6/XfwRizaoy5BYXV9F0A\nbqjxlFxBRFfKrh+s9VxK4CPGmNtQcNt+kYh+9ur/9Pi9EwJwG4CvG2NuBbAMh7vH4/O/ZrxuAOql\nhMT7lc7wJEQURkH5f8sY811L7KvfAQCMMfMAfoiC26SDiK4kPnr1PvowgJ8noiEAz6HgBnoa/pg7\nAMAYM2r9PQng/6JggP1y74wAGDHGvG1dP4+CQfDL/K8ZrxuA/QB2WVEQEQCfRaHchN94v9IZnsNq\n6vMNAKeMMX901X/54ncgovVE1GH9O4bC+cUpFAzBp6xhnpy/MearxphNxpgBFO7114wxvwQfzB0A\niKiZiFqv/BuFki/H4ZN7xxgzDmCYiK63RA8AOAmfzL8oan0I4eJg5mEAZ1Dw4/7HWs/HxXz/BsAY\ngCwKK4onUPDjvgrgLIBXAHTVep4fMP+PoLDFfRfAEevPw375HQDsA3DYmv9xAP/Zkm8H8A4KZUr+\nFkC01nNd4/e4F8D3/DR3a55HrT8nrjyvfrl3rLneAuCAdf/8PwCdfpr/tf7RUhCKoigNitddQIqi\nKEqFUAOgKIrSoKgBUBRFaVDUACiKojQoagAURVEaFDUAiqIoDYoaAEVRlAbl/wMHP0EETx7tQwAA\nAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aRsq7mmwD4uG",
        "colab_type": "text"
      },
      "source": [
        "#### Smooth random beta\n",
        "Smooth random beta image."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YH2F-3D8Doqr",
        "colab_type": "code",
        "outputId": "e853e1ef-c3d5-4b6b-f0e4-e03bdc8cde3e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        }
      },
      "source": [
        "# Random 4D matrix (unsmoothed)\n",
        "beta_us = np.random.randn(nv*p).reshape(dimv[0],dimv[1],dimv[2],p)*20\n",
        "beta_us[3:5,3:5,3:5,3] = beta_us[3:5,3:5,3:5,3] + 100\n",
        "\n",
        "t1 = time.time()\n",
        "# Some random affine, not important for this simulation\n",
        "affine = np.diag([1, 1, 1, 1])\n",
        "beta_us_nii = nib.Nifti1Image(beta_us, affine)\n",
        "\n",
        "# Smoothed beta nifti\n",
        "beta_s_nii = nilearn.image.smooth_img(beta_us_nii, 5)\n",
        "\n",
        "# Final beta\n",
        "beta = beta_s_nii.get_fdata()\n",
        "\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "# Show unsmoothed\n",
        "imshow(beta_us[3,:,:,3].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')\n",
        "\n",
        "plt.colorbar()"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.022425413131713867\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.colorbar.Colorbar at 0x7f9e15f2a0f0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWUAAAD8CAYAAACvm7WEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XtwHVedJ/DvT0/rZcmyFVmW7dhO\nbEISggNaQwh5MAngMJkEdmayyTJUGCgMtaQWqtiagbC7pGZqqqgZHsPUTmXKLF5CVUhgNoRkmCyJ\nCXkxkBAnOI5j5yG/YsuyZUm23u/72z9uu+bGSP091m1dta6/H9ctS32PTp9+6NzW6V//jrk7REQk\nHUrmuwEiIvLv1CmLiKSIOmURkRRRpywikiLqlEVEUkSdsohIiqhTFhFJEXXKIiIpok5ZRCRFyvL5\nYTPbDOA7AEoB/G93/3pc+dLaGi9b0hhbZ+l4wHqneJmpyoB6MrxMJmAPlQ2HrIs9OWm8jin+9OV4\n/O4FAFT28HomavjndchxKAkoM7mIlwlZV8VJfvKMN1bQMh5wzEtHeZmQ48W2K1MRcl7wtpRM8JN9\nopYf85B9g4DfKy/lZcaPHOl296aANc7owx+o8Z5evoNe2DX2qLtvzmddSZl1p2xmpQD+EcAHARwB\n8LyZPezue2Zc2ZJGrPjSF2PrrTvAT4zKU/xk71tPi6BsiJ/wY0v5upbu4mUqBuLPVC/hbSkfmKRl\nDtzC99/6eyZomWPvqaZlKgb4dlf28TIn1/M2lw/SIlj5fw/SMof/0xpaZrSJt7n+dd6eqoDOoHww\nvszgCv4hUjHIe8GqY/xT5NgVNbRMyL4pG+bn8ng9b/OBL/23Q7QQ0dM7hd8+upqWK215Y1m+60pK\nPsMXmwC0u/t+dx8HcD+Am5NplohI/hxAJuBfmuQzfNEK4HDO90cAvCe/5oiIJMfhmPCA8Z0UyWtM\nOYSZbQGwBQBKlyyZ69WJiLxF2q6EmXw65Q4Aq3K+Xxktewt33wpgKwBUrlqlPKEiUjAOx9QCS0+c\nz5jy8wDWm9laM6sAcCuAh5NplohIMjJw+kqTWV8pu/ukmd0B4FFkQ+K2ufsrcT9TOgYsbo//HKjq\n4X9q9F3AP0umFvEdveLpMVqm+zIeW7f4wAgt03tRVez7E7X8jvXQal5m3f08suLE5TyyYqSNx/nV\n/AuPZRur522uOcqPVf+FtAh8kkenhESMhByL+gM8mmGkqZyWmSRlSid4e4eW89+HkaX8mDe9xH8f\nQsL8vJTvv7Ihfp4eoCU4BzCVsk6XyWtM2d0fAfBIQm0REUlc2q6EmTm/0SciMl8cwMQCG1NWpywi\nRcvh59bwhYhIqjkQMAyeKkpIJCJFK/tEH3+FMLNtZtZlZrtzlv2dmb1qZrvM7EEza4iWrzGzETPb\nGb3+KbTN6pRFpIgZpgJegb4P4MykRdsBXOrulwF4HcBXct7b5+4bo9fnQleiTllEilb2Rp/RV1Bd\n7k8D6D1j2WPufjoW81lkH6LLS0HHlDMVwBB5qG9wDd9By37H/+AYX8zr6WrjMcgTtXxAqufS+Bhk\nAFi6Oz6Wef9/DMhfuZzHkXa18bZUBGRuq/8lr2eilhZB5Sl+rCYX8WuD6qMBGf0uXUXLDLXyepbu\n5rkShgKyt5UP8W0fao7f9lJ+yFHRz49n9Qkew91xDd+m0hG+/5p2BmQh3BRw8vyGF2GyccrBV8L5\n+hSAH+V8v9bMfgegH8B/d/dnQirRjT4RKWqZsCvhZWa2I+f7rVGKiCBm9lUAkwDujRZ1Aljt7j1m\n9m4APzWzS9y9n9WlTllEitZZXCl3u3vbbNZhZp8EcCOA69yzQdHuPgZgLPr6BTPbB2ADgB0z1XOa\nOmURKVoOw9Qc3jqLZl/6CwDXuPtwzvImAL3uPmVm6wCsB7A/pE51yiJS1AKHLygzuw/AtcgOdRwB\n8DVkoy0qAWw3MwB4Noq0uBrAX5nZBLJRd59z995pKz6DOmURKVoOw3jIhIAhdbnfNs3i781Q9gEA\nD8xmPeqURaRoZR8eWViRvwXtlCt7p3DBj/piyxy7soHWM17H13XJJ2OziAIABid4SFzXP6yjZYaW\n80/ikfPi17W4nf+J1fAoX89YAw/DGmzhJ+mS9oAUoO/kqSmremgRZHg1QbOKjzbyQjVHePhYXfsA\nLdOzsZ6WqeDVYIhEtZaM8/Oi9amAqbUD+qWVT/Bzp2SMhwueeBefgLUsoMlJKWBIXCJ0pSwiRcvd\nMOW6UhYRSY2MrpRFRNIhe6NvYXVzC6u1IiJnQTf6RERSZiqhOOVCUacsIkVrrp/omwsF7ZRHG0vR\nflt8KFH96yGz5fJ1/eD8p2mZD++9kZYpH+AhQPUB2cAm6uIbvegk3+7xxXzDx2sCsqkt5euaPMJP\n5LIhWgTlg3z/9a/i21U6xts82sjbXMIj/XDoD3lYZvkgryc7yXu8+jfi38+U8e1+88M8w+DigAd8\nR5fxc6eyl7dn2Ut8dvehVh6OmpSMoi9ERNIhm5BInbKISCo4DBMJPWZdKOqURaRouUMPj4iIpIfp\n4RERkbRw6EpZRCRVdKMvbmUjwLKd8SE1o0v4DuzfwEPQNt/0Z7RM13sX0zJ1VXzCyYlq3uaey+L/\nhCrv439irdpOp/dC37V8m0p5xBKmKnl7Fr/Jw91C4vZXPHmSljnyoSW0TPkgD9eq6uFtHq/nvxYh\n945CJu9tfjY+ldzRq3lKxNYn+eyqh6/nk6Iu/y3fNz1v5xvesI9v97EraBHgvoAyhMMSS3JfKHl1\nymZ2EMAAgCkAk7Od40pEZC44gIlzMPfFB9y9O4F6REQSZgsun/LCGmwRETkLjuwTfewVwsy2mVmX\nme3OWdZoZtvN7I3o/yXRcjOzfzCzdjPbZWbvCm1zvp2yA3jMzF4wsy0zbMgWM9thZjsmRwOeyxUR\nSdBUdLUc9wr0fQCbz1j2ZQCPu/t6AI9H3wPADcjOYL0ewBYAd4euJN9O+f3u/q6oAZ83s6vPLODu\nW929zd3byhbxaWJERJLiboldKbv70wDOnJH6ZgD3RF/fA+CjOct/4FnPAmgws5aQ9eTVKbt7R/R/\nF4AHAWzKpz4RkSRlb/SV0lcemt29M/r6GIDm6OtWAIdzyh2JllGz7pTNrMbM6k5/DeBDAHbH/5SI\nSCFl5+hjLwDLTg+zRq9ph2PjuLsj+zmQl3yiL5oBPGhmp+v5obv/PO4HMuXA4Mr4z4ESHhaMxl18\nDGhwDR8qWfL6OC2z6NApWqbjhvNomVXb42NJ+9byONKQbWp9kscyZyr5YZ+s4lcPw818GuqOD/B6\nVj7OY3GbXuLH6sAttAjqAlKS1u/nJ+GpC/k+HF4ekI51SXwKy1IegoyyYd7exj38/Or8E76ymueq\naZmjV/JUoo27+L45REtw2Rt9QWPG3bMM6T1uZi3u3hkNT3RFyzsArMoptzJaRs26U3b3/QDeOduf\nFxEphDl+ou9hALcD+Hr0/0M5y+8ws/sBvAdAX84wR6yFFVUtInIWknyiz8zuA3AtskMdRwB8DdnO\n+Mdm9mlkL+5P/732CICPAGgHMAzgz0PXo05ZRIpaUhOnuvttM7x13TRlHcDnZ7MedcoiUrTcgYnM\nwnpGTp2yiBSt7PCFOmURkdRYaLkvCtopW4anV8yUB6SMPMTDo8oGeJnR86pomcM38XC31l/20TL9\nF5Kwr4DzZrCFh5dVnuSzBHspX1nfGh7u1vjaKC1jGd6ekDShIVq28yuigVa+rvqDPMRs5UP8Rvrx\nP1hOy3hJfHtqjvM0tR3X8FDJ8QYegrb4af77sHQvz/vafSmvp4RnCU3EWYTEpYaulEWkiGn4QkQk\nVTRHn4hISmSjL/LKbVFw6pRFpGidc9NBiYiknYYvRERSQtEXAYxE+ITcKM2U8Z3cfwEPE6o6MUHL\nlI7xXTRRz7Ni1R2In3VlbBmvo/NK3pa+YZ4NbCggLGzFv/GMYYMreLjbaCM/oBkefYfJKl7P4kM8\nfGyijm/7RA1f14k/5vnKQ87lpbvjQze9lO/jFc/w0MT+tTxMrX7fMC1z+Hr+e1XJEytioqZwHaWi\nL0REUsLdMKlOWUQkPTR8ISKSEhpTFhFJGXXKIiIpoThlEZGUUZxyDMsApSR6Z2gFr6dsmKeYGl3P\nQ8PG63iZhnYeNjfWyHfj6NL4MuO1/A5xyRg/uWqP8vbWBUzfeGwTD8Va8jo/Di2/PEHL7L+1iZZZ\n98MuWubgn/KMfjUdPFvayQ38sdy6N3n4Xf0b8WGQADC8Mn4i0mPv5edF65M8prBkkm/3yHIellk+\nSIug6SWeSS5TVpiICHdgUknuRUTSI4nhCzN7G4Af5SxaB+B/AmgA8BkAp68+7nT3R/JZlzplESla\nSY0pu/trADYCgJmVAugA8CCyE6J+292/kfdKIuqURaSoefI3+q4DsM/dD5klP169sAZbRETOUgZG\nX2fpVgD35Xx/h5ntMrNtZrYk3/aqUxaRouWeHVNmLwDLzGxHzmvLdPWZWQWAmwD8c7TobgAXIDu0\n0Qngm/m2WcMXIlLEDFNh0Rfd7t4WUO4GAC+6+3EAOP0/AJjZdwH8bFbNzKErZREpau5GX2fhNuQM\nXZhZbrrAjwHYnW976ZWymW0DcCOALne/NFrWiGx4yBoABwHc4u4naV0OlI3Fx0tW9PMddPT9PA1h\nSDxlCQ/pxcAq/sdEJmB26Oru+LjWxld4g8tGedrE0UYeY9vwGl9XVRff7sVPtdMyfR+4kNdzgMfQ\nHv1wwKziT/LUk8c3xccFA8D52/h2Ta3lM1UfvYbMYA6g5dfxbT5vB48dHmzhx6pxD983R6/m59eq\nf+2lZXov58OqE3xVwC8CyhBJ5r4wsxoAHwTw2ZzFf2tmG6NVHTzjvVkJuVL+PoDNZyz7MoDH3X09\ngMej70VE0sWz48rsFVSV+5C7L3X3vpxln3D3d7j7Ze5+k7t35ttk2im7+9MAzvx4vBnAPdHX9wD4\naL4NERGZC3MQfTGnZnujrznnE+EYgOaZCkZ3MbcAQEV13tEiIiLBPPxGX2rk3Vp3d2THU2Z6f6u7\nt7l7W/mikIEkEZHkJDV8USiz7ZSPn77rGP3Ps8WIiMyDhKMv5txsO+WHAdwefX07gIeSaY6ISHKy\nV8ILq1MOCYm7D8C1yD7xcgTA1wB8HcCPzezTAA4BuCVkZVMVQN/a+M+BqhP8b4myEV6mYjBgZuNq\n/pk0GjCLcoZnucTQ8vh6xmtqaR1NzxyjZTr+kM+yPFnFQ7UWneT7r+Pjb6Nlykb5sWp+upuWOfIR\nnt6zdIynEq0Y4O05uIWH8dUe5vVUdfEyfRfEh3eOB8y+Xd3Ft7ufrAcAGl+dpGWOXt9Iy7Ru76Fl\nOj60lJZJStEluXf322Z467qE2yIikri0jRkzesxaRIqWw5BZYNEX6pRFpKgtsAtldcoiUsR8TvIp\nzyl1yiJS3BbYpbI6ZREparpSjlE6BjTsiw+1YqFjADDWyHdyeTv/eOx5B6+n7hCvpz4glKj6jfiw\nr9F1PETo0J/ycLele3hbvIRv90gjPw7lg3zfjNfzdQ1u4I/f13TyEL3uy3hY4fjigHNngBZB+RBv\nz4l3831Y2R3fntEmvo8b2nlI3LEr+MztjXtokaDMikev4+fyVEAYaRIcQCajTllEJB0cgK6URUTS\nQ3HKIiJpok5ZRCQt0pfbglGnLCLFTVfKIiIp4YAnFH1hZgcBDACYAjDp7m2zna80TmE7ZQdKJuM/\ntsZ5AjPUdAZk31rLJxBt3MPDmk7dNETLZJ7kja7siQ/Xsim+Tec/zLNvtf8Zz+K14lcBYXNlPJxr\nspqf7I17eQzViY3ltEw5PwxBk3FWHw8IMds3Ssv0n88nNC0d5fun9an4+LvB1Xyi155LeLjbmp/w\nCU+nanmc2uLX+fG0Q3yaupObeYbB5CQ6fPEBd8+Nbz09X+nXzezL0fd/mc8KFlamDhGRs+UBr9lL\nfL5SdcoiUtyS65QdwGNm9kI09yhwFvOVhtKYsogUr/CHR5aZ2Y6c77e6+9Yzyrzf3TvM7DwA283s\n1besyt3NLO/biuqURaSoBT480u3ubfH1eEf0f5eZPQhgE6L5St29M6n5SjV8ISLFLWP8RZhZjZnV\nnf4awIcA7MYczFeqK2URKWr5DygAyI4VP2hmQLbf/KG7/9zMnscs5iuNo05ZRIpX/tEV2Wrc9wN4\n5zTLe5DwfKUF7ZQzFcDAyvj44bqAWYJZrDMAjJTzP0lCUli2/hOPAT11IW/P6Hnxswmz/QIApat5\nDPLyZ3kax8PX8XW1PsXrGWrm9Qys5qfY8t+O0TLHNvEY2tqOgPOiiR/zqXI+qlfVExDrbXzb938s\nPn69eQePpV/xRB8tc/gGfu4seZ0f8+Pv4XHT5X0NtEzIswbJMGWJExFJFT1mLSKSIvyPjVRRpywi\nxUtJ7kVE0iWh6IuCUacsIsVtgXXKenhERCRF6JWymW0DcCOALne/NFp2F4DPADgRFbvT3R+hdU0B\nlSdZ6k4+/lPZxz/6VjwzTMscvYqH95zawEPinEeG4bzn4lM0drXxGZ3P/1c+zfLRq3ga0boDtAiq\nH3mRlun7L7FPpQIAykb5sZpaxK8NQo7nxGKeAtQy/GBVHuN5Qrvex4/Xec/ytLonL46v58RGvm9O\nbqinZaqP8eMw2Mr3TUtA2tfSUR5aFxLWmpSFNnwRcqX8fQCbp1n+bXffGL1ohywiUnCORB6zLiTa\nKbv70wB4hmwRkTSa23zKictnTPkOM9tlZtvMjP8tJyIyD8z5K01m2ynfDeACABsBdAL45kwFzWyL\nme0wsx2TowFz+oiIJOlcuFJ29+PuPuXuGQDfRTav6Exlt7p7m7u3lS0KmERNRCRJ50KnHCVzPu1j\nyOYVFRFJlZChi7QNX4SExN0H4Fpkp0s5AuBrAK41s43IfsYcBPDZkJVlyoHBlfF3OkN2UP1BHnIz\nuIrPNrz6oRO0TE/bMlqmsp+3p/3j8Zmz6g4EhCydz0P4Gl/jsw2fuoCHjvX/8btomQyvBmNV/M72\n0pfH+boqeLhW70W8QSFtrurmYYUTtQGJ0QNmKC8biq9nUXfs2wDCQjLHlgRkx+PRnxhZylfW9NQx\nWubVL6zgK3uCFwmSsugKhnbK7n7bNIu/NwdtERFJXNquhBk9Zi0ixU2dsohISqRwzJhR7gsRKW4J\nRF+Y2Soze8LM9pjZK2b2hWj5XWbWYWY7o9dH8m2urpRFpKhZMknuJwF8yd1fjGa1fsHMtkfvfdvd\nv5HIWqBOWUSEcvdOZB+Ug7sPmNleAK1zsS4NX4hIcUv44REzWwPgcgDPRYsSTTlR0Cvl8oEMVj4x\nGFvmzc3xs/sCwFg9j5WsPcJnSN73cR6DXHEqIL5zEW/P4vb492uP8ljno1fz9Sz7HW9vwxs8lvnQ\nzQHxxc8H7JuAOOWed/D46xBNL/Fj7qW8PcNN/NeiYR9PYdn9H/gM0iXkUFQM8h6j6wp+7mz4PyO0\nzMm389+9KT6pOIYvaqZlGl/mx+EgXxUXfqNvmZntyPl+q7tvPbOQmdUCeADAF92938zuBvDX2TXh\nr5FNOfGpfJqs4QsRKW5hnXK3u8cmCDezcmQ75Hvd/SdANuVEzvvfBfCz2Tc0S8MXIlLckom+MGQf\nmtvr7t/KWZ54ygldKYtI0TIkFn1xJYBPAHjZzHZGy+4EcNtsUk7EUacsIsUroYdH3P1XyPbxZ0p8\n1iV1yiJS3BbYE33qlEWkuKlTnlmmogSDq+PDn1Y9ymcnyVTy0LDuy6pomdWP8jChjmt4PTWv8pCk\nvnXxu3p0SUBKxBeTObtCZo+uORgw63NAytJjlwSE6L3Gy4TMcm4Zvn9ObOQxXUt381SiE7V8/9R0\n8tDD4eXx+TJLJvg2VXTzthx/D09HWn+Qh/n1n8+7jMEWXqZvAy2SmIWW+0JXyiJS3NQpi4ikhCcW\nfVEw6pRFpLjpSllEJD00piwikibqlEVEUmIWWeDmW8E7ZRa2dOhGnjFs+W94KFZIKNHJDXzG65AD\nGhImVHckvs0eMOFudecoLTPewKckHm3kIVRjDXzDS/hhwNp/4eFl5TveoGXGN/EYqiESXgYA1cf4\nXZ/J6oCsfy910TKj65bSMlVd8fv51AYevlg2TItg6Ss8g97IeXyq79IRfl4Mt/CTuTJglu4kGDR8\nISKSKuqURUTSRJ2yiEiKqFMWEUmJhLLEFZI6ZREpbuqURUTSo+geszazVQB+AKAZ2c+cre7+HTNr\nBPAjAGuQzbh/i7ufjK1rylFBMou1/Jrvwd6LAkJ3eAQQWh47Tssc+hM+CeTiN3l2rbH6+NCmyj6+\n3SPLeQjf8TYeQlU6xkOW1j7Ms/WdvIiHLw628DZXN11Cy3RexS93lgZMGjvczMs07eLZ3bquWU7L\nTJsS/Qy9l8Uf95o3A7IH7uJhh6cu5OGCdYf5eTxZya/jmn7H91/5MF/XHloizEIbvgiZo28SwJfc\n/WIA7wXweTO7GMCXATzu7usBPB59LyKSHiHz86Ws06adsrt3uvuL0dcDAPYCaAVwM4B7omL3APjo\nXDVSRGTWEuqUzWyzmb1mZu1mNmcXoWc1m7WZrQFwOYDnADS7e2f01jFkhzdERFLj9BN97EXrMSsF\n8I8AbgBwMbITpl48F20O7pTNrBbAAwC+6O79ue+5+4yfN2a2xcx2mNmOiXE+TikikiTLOH0F2ASg\n3d33u/s4gPuRHS1IXFCnbGblyHbI97r7T6LFx82sJXq/BcC0yQDcfau7t7l7W3lFTRJtFhEJk9yY\nciuAwznfH4mWJY52ymZmAL4HYK+7fyvnrYcB3B59fTuAh5JvnohIfgKHL5ad/os+em2Zr/aGxClf\nCeATAF42s53RsjsBfB3Aj83s0wAOAbhlbpooIpKHsCvhbndvi3m/A8CqnO9XRssSRztld/8VZo64\nvO5sVpYpM4wsjV/liXfzehr28r08tYgHiQ5csoyWWbqHx1OWDfMclgMr4+NNx+pD4j/5mHzD6zx2\nOFPO99/4Eh7XumQvzxl58Cbenr6LeXuWPc9H2komA+p5mR+roWZ+LELOr7KANJetT8S/f+pCWgX6\n1vC4/f4LeD0hs3iXjfD9N9jKz53et/M240leJERCccrPA1hvZmuR7YxvBfCfE6n5DHqiT0SKWwKd\nsrtPmtkdAB4FUApgm7u/kn/Nv0+dsogUrwRns3b3RwA8kkxtM1OnLCJFSzOPiIikjS+sXlmdsogU\nNV0pi4ikRQoTDjEF7ZS9BJiojQ8lWvErHnIzWs/TGZaf5KP75QM83G1iMd9FfWt5CBDb7spT/MwZ\nOL+KlpmqpEVQ1c33TcdVfLsX9fCwpuqjvD2Tp/jxHA7IrFLRx8vUHuHHvCrgt6JsOGRW7IAwvvH4\n4z7axOuwDA/PW/0Yz2V74GYevtgYEG/Q8OogLWOZwj3dW3T5lEVEFjJ1yiIiaeHQjT4RkTTRjT4R\nkTRRpywikg56eEREJE08OIl9ahS0UzYARiLeJqp4CNDAGh4CtOYhHh81soKH5Szq4qFEo/U8VM3J\nZg218G2qCQgvC5nFe6iFh6BVTTtlwVs1vzBCy2TK+PE8dQEPKVx8iM+QfOiP+HaZ85jBhnaeLW2y\niq+r9yL+61VBQiG9lHco9Qd4eEHnFXy7Q0IKpwIyDJ58ey0tUzlQwJCIhdUn60pZRIqbhi9ERNLC\nAWj4QkQkRRZWn6xOWUSKm4YvRERSRNEXIiJpoSxxRIZPJll7hMd0ndrAQ9B6L6unZcoDMn0NN/F1\n1R/gbS4biw/7KhvlZ85EFQ+bC/lTrfx4QAa9IV7GJniZ3kv4/lv2Mg+tm6jjp2rjTh5+V9XL21w6\nyjMVDjfzDHkrnuETy07Uxdczuoyvp7yfn38tv+HZ8Trfx8PmJqv5OVhKMt8BgPNEconIPjwy972y\nmf0dgD8CMA5gH4A/d/dTZrYGwF4Ar0VFn3X3z8XVxc9iEZGFLBPwyt92AJe6+2UAXgfwlZz39rn7\nxugV2yED6pRFpMiZO33ly90fc/fTf448C2DlbOtSpywixcsDX8n6FID/l/P9WjP7nZk9ZWZXsR/W\njT4RKWLBuS+WmdmOnO+3uvvW3AJm9gsAy6f52a+6+0NRma8CmARwb/ReJ4DV7t5jZu8G8FMzu8Td\n+2dqiDplESluYcMT3e7eFl+NXx/3vpl9EsCNAK5zz67U3ccAjEVfv2Bm+wBsALBjpnrUKYtI8fLC\nTAdlZpsB/AWAa9x9OGd5E4Bed58ys3UA1gPYH1eXOmURKW6FmQ7qfwGoBLDdzIB/D327GsBfmdkE\nsnEen3P33riKaKdsZqsA/ABAM7JD4lvd/TtmdheAzwA4ERW9090fiavLS4Gxhvg4x5AZmzMB6QMb\nd56kZY5fuYSWKeOhpphaxNM41hyNTwd59KpFtI5KvknI8LBW1HbwS4f+1fzzujJgVvHaTh7zO17P\nGz3cxNcVEkPbX8frqTwZcP874Pd8rJGnJJ2qJG0OWg/ff8fey+tp3M1XVvcmT2t6+Hq+3Ute56lY\nE1OAPtndL5xh+QMAHjibukKulCcBfMndXzSzOgAvmNn26L1vu/s3zmaFIiKFZJmFNZ017ZTdvRPZ\nO4hw9wEz2wugda4bJiKSN0dSD4cUzFnFKUePDF4O4Llo0R1mtsvMtpnZtGMBZrbFzHaY2Y7JkaG8\nGisicjYM/MGRQjyGfTaCO2Uzq0V2bOSLUYzd3QAuALAR2Svpb073c+6+1d3b3L2trIpPvyQikih3\n/kqRoOgLMytHtkO+191/AgDufjzn/e8C+NmctFBEJB8p63QZeqVs2fiO7wHY6+7fylneklPsYwB2\nJ988EZE8nB5TnvuERIkJuVK+EsAnALxsZjujZXcCuM3MNiK72QcBfJZV5CXA1KL4EKCJGv6p1ryD\nh1md2MTD3eoP8HSGIaFhxzbxkKR12+Knol7ayPOXlA3x7R5cwdviPHIMNQHpPStP8rCmAzfx8KjG\n3XwUjaV8BcLSllb2B6QbfRtPYZmp4DtxMiDVanVX/DFd8etRWsfQcr6Paw+FjFQms49XPsl/rzLl\nhUu7U4zRF79CNi3pmWJjkkXTDlRfAAAGf0lEQVRE5l/6xowZPdEnIsXLoU5ZRCRVFtbohTplESlu\naYtDZtQpi0hxU6csIpIS7sDUwhq/KGinnKkABtbGhwCt2s4/1Sp7eaaq0XqebW64iW/++GIe1rQo\nNhFf1tiFzbHvTwWEWHVeETCD8r/xsLm+tXy7qwNC4vrW8sx2tYdoEXhAnBXLLggAwy20CFoDwrUm\nqvh+Hmrh7akZ5Ns1XhcfGjawkofnTRsbdYaqHn48JwKy7E0t4qFsA638/FryBp+BOzG6UhYRSRF1\nyiIiKeEAwuboSw11yiJSxBxwjSmLiKSDY8Hd6CvcA+giIvOhAKk7zewuM+sws53R6yM5733FzNrN\n7DUz+zCrS1fKIlLcCnej7/emxzOziwHcCuASACsA/MLMNrj7jGFSBe2Uy4aBph3xF+eDK0Lq4ZNf\nDq3g4T0N7Tx8bMU3nqNlDv+P99EyPRfHh4853yRs+H4PLTPaUkvLZMr4YR9u5n9EVZ4KONkDioRM\neFoywSsqH+D1TCzmO3rRSX5eZMr5usoHeT1dl8eH361+tJ/Wceoifswt4C/4gXW8TOk4P3foZLAA\nOt/LwynxS16Em/eERDcDuN/dxwAcMLN2AJsA/GamH9DwhYgULweQyfBXMqabHq8VwOGcMkdA5jhV\npywixS1sTHnZ6blEo9eWM6sxs1+Y2e5pXjcjcHq8EBpTFpEiFvyYdbe7t8XW5H59SEVnTI/XAWBV\nztsro2Uz0pWyiBQvB9wz9JWvmOnxHgZwq5lVmtlaAOsB/DauLl0pi0hxK8wTfX873fR47v6Kmf0Y\nwB4AkwA+Hxd5AahTFpFiV4DoC3f/RMx7fwPgb0LrUqcsIsXLPcnoioIoaKdcOpbB4v0jsWWO/EE1\nrSdTxtMZVvDwTows5UPqE7dfQctMVQakG+2PL9PwygCtY+9/XUzLrP1nfgJmeGZKnPciT604WcVj\nfgdW8TIj5/H2LH+Op9wcHeLrGm7iZWqP8nVVDPD93PXugB1NTp2R5fz3wXg4dNAs1CFpTQdW8i6j\nYT+f5TwkvWdilCVORCQtHD4V8KmVIuqURaR4KXWniEjKKHWniEg6OADXlbKISEq4ktyLiKTKQrvR\nZ17AcBEzOwEgd37jZQC6C9aAZKjNc2+htRdQm+fC+e7elE8FZvZzZLeT6Xb3zfmsKykF7ZR/b+Vm\nO1gSkLRRm+feQmsvoDZLcpSQSEQkRdQpi4ikyHx3ylvnef2zoTbPvYXWXkBtloTM65iyiIi81Xxf\nKYuISI5565TNbLOZvWZm7Wb25flqx9kws4Nm9rKZ7TSzHfPdnulEkzZ2mdnunGWNZrbdzN6I/l8S\nV0chzdDeu8ysI9rPO83sI/PZxlxmtsrMnjCzPWb2ipl9IVqe5n08U5tTu5/PZfMyfGFmpQBeB/BB\nZGd3fR7Abe6+p+CNOQtmdhBAm7unNrbTzK4GMAjgB+5+abTsbwH0uvvXow/AJe7+l/PZztNmaO9d\nAAbd/Rvz2bbpRNP+tLj7i2ZWB+AFAB8F8Emkdx/P1OZbkNL9fC6bryvlTQDa3X2/u48DuB/AzfPU\nlqLi7k8D6D1j8c0A7om+vgfZX8hUmKG9qeXune7+YvT1AIC9yE4Zn+Z9PFObJYXmq1NuBXA45/sj\nWBgniQN4zMxemG4K8hRrdvfO6OtjAJrnszGB7jCzXdHwRmqGAnKZ2RoAlwN4DgtkH5/RZmAB7Odz\njW70nZ33u/u7ANwA4PPRn94LimfHq9IecnM3gAsAbATQCeCb89uc32dmtQAeAPBFd3/LPDdp3cfT\ntDn1+/lcNF+dcgeAVTnfr4yWpZq7d0T/dwF4ENlhmIXg+Okp0KP/u+a5PbHc/bi7T3l27vfvImX7\n2czKke3c7nX3n0SLU72Pp2tz2vfzuWq+OuXnAaw3s7VmVgHgVgAPz1NbgphZTXSTBGZWA+BDAHbH\n/1RqPAzg9ujr2wE8NI9toU53bpGPIUX72cwMwPcA7HX3b+W8ldp9PFOb07yfz2Xz9vBIFH7z9wBK\nAWyLpuFOLTNbh+zVMZBNefrDNLbZzO4DcC2ymbGOA/gagJ8C+DGA1chm6bvF3VNxc22G9l6L7J/U\nDuAggM/mjNfOKzN7P4BnALwM4HSi3juRHaNN6z6eqc23IaX7+VymJ/pERFJEN/pERFJEnbKISIqo\nUxYRSRF1yiIiKaJOWUQkRdQpi4ikiDplEZEUUacsIpIi/x94S7KdCtnb0wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d_jmFhBhD6tL",
        "colab_type": "code",
        "outputId": "58aac915-3b61-4e24-ec9f-970abc08f6be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        }
      },
      "source": [
        "# Show smoothed\n",
        "imshow(beta[3,:,:,3].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.colorbar.Colorbar at 0x7f9e15ebf4a8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV8AAAD8CAYAAADQSqd1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHyVJREFUeJzt3X+MZeV5H/Dv9/6YOz92YSELeAMk\nOAQ1smiNqy1tZavFcewSF5W4UpCJamHVyjpSkGzVUmLjP+x/IqE2dmKpkaVxQcYq/oFqU6OEGhPX\nFkGtKQtCGFgnoWgds1nvsl5gZ5fZmbn3fPvHvauO8c59npl75pw7h+9ndbVz77z3Pe895973vvOe\n5zwvJcHMzKrVqrsBZmZvRO58zcxq4M7XzKwG7nzNzGrgztfMrAbufM3MauDO18ysBu58zcxq4M7X\nzKwGnUmeTPJGAJ8D0AbwXyTdOa58+4IFdS/ZE1QaX3HXYqJtiOspFFdUFImN9ePvMBbRhuLNJHZN\nSuJlp2TaE75uAK1+osxavDH2ExsrEmXKuugz8UZVZ/x7p+jG762iHTdFZZXJDNdaiR2YqGf18JET\nki5JbHFD/+qdC/rpyUFY7omnVx6SdOMk29qsLXe+JNsA/gzAuwG8COBxkg9Iem6j53Qv2YMr7/y9\nsfW22/GHo9eNP62dRD0r/fjddmZpNiyjl2fCMt2l8e+29tn4g8pEJ5VR1oestRaXmVmKy8ydiI/V\n/LHVsEz35bNhGZ5JlOnHH9YMzfXCMmu/sDD298uXxu+t5V+ID9bqhfH7a21XWAT9XfGxGiwkvuB6\n8T7+uw9+4kdxReP99OQA/+ehXwrLtff97d5Jt7VZk0w7XA/geUkvSFoF8FUAN5fTLDOzyQlAkfhX\nh0mmHS4H8ON1918E8E8na46ZWXkEYU3l/CVTtonmfDNIHgBwAAA6ey/c7s2Zmf2Muka2kUk63yMA\nrlx3/4rRYz9D0iKARQCYvfpy5680s8oIwmBK0+ZOMuf7OIBrSL6Z5AyA9wN4oJxmmZmVo4DCWx22\nPPKV1Cd5O4CHMAw1u1vSs+Oe020PsO+iU2Prne/GZ7Uv7r0WllnorIRlXl2bC8v83058EvTEahw+\nUJwd/z3XipuLTIRYKhwtMQXGRJl2os3tlUSI2CAuo0zYVjc+DpzpxvUwE8uYCFPsxdsa9Ma/Lwa9\nxHZmEmUSn/SimwjPzPQYnfjP/Fa3mqkAARiU1LmSPAxgCcNPUF/S/knqm2jOV9KDAB6cpA4zs+1U\n8sj2nZJOlFHRtp9wMzOriwCsNXDO18xsqgnCIHFLVwd8m+QToyiuiXjka2bNJSBxSgEA9pI8uO7+\n4ihSa713SDpC8lIAD5P8oaRHtto0d75m1ljDK9xSTkQn0CQdGf1/nOT9GF7lu+XO19MOZtZgxCBx\nC2shF0juPvczgPcAeGaSlnnka2aNNTzhVkoav8sA3M9hiGEHwJclfWuSCivtfBc6q/gne8cnKto3\n82pYz6/2fhKWubQdp9P6u/7FYZmH2v8wLPO/luPsVatLwa5OxI1m0jNm4nNbmTSPiYxlneV4Mq17\nJi7TXk3E+bbj/TOYj9/Omc8hB4mPRSv+o7G/K47zXd0zfluruxPZyMYnRhu2ZT4Rw9tLHIde/OZp\nz8Vvwt5sHM9fhmGc7+Sdr6QXALx14orW8cjXzBotk7e7Du58zayxyhr5bgd3vmbWWAIxmNK4Ane+\nZtZonnYwM6uYQKxm1s2qgTtfM2us4UUWnnZAr7WGa+aOjS1z9cz43wPAW2fGp6UEgL3tOP7mF/s/\nl/v95zw//6awzFOzvxiWWWnPjy+QWfA1s8pvIoInsxJwN87aiZnTcdhR57W4DItEGFQi1KxYiN/O\nnE2knUwci6ITt2d1d7ytlT3j61m5KBFqtjtucH8usY/nEsdqPn4Tzi/Ei5TumYvLlMUn3MzMKiYR\ng9R699Vz52tmjVZ45GtmVq3hCbfp7Oams1VmZiXwCTczs5oMHOdrZlYtX+E2UqCF04PZsWV+OtgV\n1nNscDqxrTNhmb/vx6sXvzqIy6wNEuFLg/HfvplsZEyEmmVWC+6+FpeZWYrDjnqvxKnPWstxo4uZ\nxOrPF8QZwvrz8YcsEyJWJGLyB4kVg9d2JcLRLhj/+35JYWSaSaSyS6wo3JmJj+dCL7EC+WwilrEk\nhaMdzMyqNUys487XzKxSArHmy4vNzKolwRdZmJlVj77IwsysakK5I1+SbQAHARyRdNMkdbnzNbNG\nK/mE20cAHAIQxKnEKu18l/o9PHry6rFlLpy5PKznid6bwzLzifReL63uDss88/K+sMzpV+NwtPZK\nEGqWWdQyE2qWyGqWWfhy5tV4Y52Xl8MyXItj6LhrfPghAKwxEWo2mwj/mk8sxNlLlImbjH6QyG5Y\nZvyxGCQyjamTSMOWUSTC8BJhlYMisbhoRfOwAktLpk7yCgD/GsAfAfgPk9Y3UedL8jCAJQADAH1J\n+ydtkJlZWYZLx6e6ub0kD667vyhp8XVl/hTAHwCIR20JZYx83ynpRAn1mJmVjNl8vifGDR5J3gTg\nuKQnSN5QRss852tmjSWUdoXb2wH8G5LvBTAL4AKS/1XSv9tqhZO2SgC+TfIJkgfOV4DkAZIHSR5c\nfSWeIzQzK9NgNPodd4tI+oSkKyRdBeD9AP7nJB0vMPnI9x2SjpC8FMDDJH8o6ZH1BUbzJosAsOfX\nLi3pzICZWUxiM3M7SDoy+v84yfsBXA/gkfHPMjOrxvCEW7mXF0v6HoDvTVrPlr8SSC6Q3H3uZwDv\nAfDMpA0yMyvPcA236FaHSUa+lwG4n+S5er4s6VvjnrC82sXTP75ibKVMLB3LVmKV1cQJzmKQiEd8\nLd5FraW4TPvs+AZlYngzKxxnUlNm0k62z8YN4tk4pSSKRABzQiYVZD8Rn9tfSJRJxPAWvbjMYDax\nYnA3KJOIUW0FMeQAwH7iA5H47A0S9ZxkvHL4Wr+aZDfDE24Nu7xY0gsA3lpiW8zMSueUkmZmFSvz\nCreyufM1s0bzAppmZhWTgLVErok6uPM1s8YaTju48zUzq1wyt0Plqu1811rA3wdxPJnIpESZzKV0\nmXC0xIKuaK3FFbXPBm3JrF5cJMLwMmUGiTL9zE5O7OVWZkXhRJluIt3hTNycIs5MiUwSLLUy+zmu\nJwpBzIUOJt5/iVSjmbC2wWx8rFZfi8u8sitxsErQyFAzM7Pp52kHM7NaeA03M7OKDaMdvHS8mVml\nfJGFmVlNPO1gZlYxRzuMsAA6r02e3SsTfpOJNSsrzWcZKw+nXndiO4nEVKkyKZlYvXZ8plntxEoC\nrZLKZD6HmeO5msgklnifRse9k1j8pXMmPqCdRCa7jMzqz51EqNnaQnURCI52MDOrmMTKlqnfLHe+\nZtZoZUw7kJzFcJWeHob95n+T9KlJ6nTna2aNVeKc7wqAX5d0mmQXwKMk/4ek72+1Qne+ZtZoZXS+\nkgTg9Ohud3SbaCJ9OidDzMxKcC7ON7plkGyTfArAcQAPS3pskra58zWzRivA8AZgL8mD624HXl+P\npIGk6wBcAeB6ktdO0q7Kpx3UHj9SzwREZyKcMuFUqZOgmdCtTAhYUCZVRyrzWVxm2qTC48rK1jZI\nZAAra6HSIJMdAHSDMLGZpfiAzizFjWmtxvVkQv5WL4y7DCYWps1kYiuDBPRzydRPSNqfq1OvkPwu\ngBsxwYrtHvmaWaOVMe1A8hKSe0Y/zwF4N4AfTtIun3Azs8YqMbfDPgD3kGxjOGi9T9KfT1KhO18z\nazSVE+3wNIC3Td6a/8+dr5k1mhPrmJlVTHJiHTOzGhADLx1vZla9MuZ8t0PY+ZK8G8BNAI5Lunb0\n2MUAvgbgKgCHAdwi6eWoLjFePZaJIMtMLGtqf2fihTOxtYmNhUVKSvOYed2Z+OZMesZMusiUVAxv\nXE0mLWemTGb/ZFYD7izHryuK4509uRZvZyluTGY16sFcvLRzu5eI4Z1LfB4qGvZNcz7fzKfnixgG\nE6/3cQDfkXQNgO+M7puZTRcN532jWx3CzlfSIwBOvu7hmwHcM/r5HgC/VXK7zMxKkby8uHJbHfxf\nJuno6OefALhso4Kja6QPAEBnz0Vb3JyZ2eZpik+4TdyqUaq1DQfukhYl7Ze0v7WwMOnmzMw2ZcdO\nO2zgGMl9ADD6/3h5TTIzK4/E8FaHrXa+DwC4bfTzbQC+WU5zzMzKMxzZTmfnmwk1+wqAGzDMd/ki\ngE8BuBPAfSQ/BOBHAG7JbIwoZ+Xc1L7KhFOVFI5WZMKggj1dJGbfozoAYDATN7g/F++czkIcdsRB\nHL6USfOYSWWYUlL6zym9GnUiSuRhzRyHoltSPSWtHJ4xraFm4cdZ0q0b/OpdJbfFzKx0dc3pRnyF\nm5k1lkAUUxrt4M7XzBptSge+7nzNrMG0g3M7mJntaFM69J3OyRAzs5KUEWpG8kqS3yX5HMlnSX5k\n0nZVO/ItgM6Z4IVmvqUy4V+JV1Z0M7FJcRG14nqKIPyGcWQX+onGZNoCZr5z4wZ1Z+J6WmtxewaJ\nTFmZELqyhhKZv1Kj7HwA0J9NfKiDlX6Z2FDRjV94JuRvMBvXs7YQl+nPhUUw6FW0ejGAoihlW30A\nH5P0JMndAJ4g+bCk57ZaoUe+ZtZcwvDbNLpF1UhHJT05+nkJwCEAl0/SNM/5mlmjlR3nS/IqDBfT\nfGySetz5mlmz5TrfvSQPrru/KGnx9YVI7gLwdQAflXRqkma58zWzBkvnbjghaf/Ymsguhh3vvZK+\nMWnL3PmaWbOVMO1AkgDuAnBI0mcnr9En3MysyQSoYHhLeDuADwD4dZJPjW7vnaRplY58WQDd00Gh\nxLdUZoHDYiYuMxhkFvrLNGjyMKhM6FJm0cEiEZI1SOybTPhSJpSqHa//mMpwlQk1yxyGTFaz1AKt\nifdgfz4uEx2v/nwiA91yIuSvP3k4JJALI+svlPMeLM/koWaSHi2lonU87WBmzTalV7i58zWzZnPn\na2ZWsXMXWUwhd75m1mhOpm5mVodycjuUzp2vmTVaGetGbgd3vmbWXIJPuAHD+Mn28uR7Qq1Mqr5M\nRXGRVDq6Ev6qUSLWNbXnEvVk0m2qk4jVTKQFzKSUTK0onCjT6mfKZFZTjuvJxDhn4nxXF6ISmX0c\nb6e1mvjMJPZxJh49FWM/U1WPmMtaVgePfM2s2TzyNTOrQeYvqxq48zWz5nKcr5lZPRztYGZWhynt\nfJ1S0sysBuHIl+TdAG4CcFzStaPHPg3gdwG8NCp2h6QHw60pGVYUtSlzvSAT8zyrmW3FZTIpEaOI\nodS0VKZM4us0MxBIpbjMZNJMpClsr8Yt6izH2+okwhgz28qki1zdnVhxORGONpgd357+fDmheu2z\nieOwUk7aziIRRpYpU5ZpnXbIjHy/CODG8zz+J5KuG93ijtfMrGrC8PLi6FaDsPOV9AiAkxW0xcys\nfErcajDJnO/tJJ8meTfJi0prkZlZiaj4Voetdr6fB3A1gOsAHAXwmY0KkjxA8iDJg/2zZ7a4OTOz\nLSpp5DsaaB4n+UwZzdpS5yvpmKSBpALAFwBcP6bsoqT9kvZ3ZsML2c3MylXetMMXcf7zX1uypc6X\n5L51d98HoJRvAjOzMmWmHLLTDmWf/8qEmn0FwA0A9pJ8EcCnANxA8joMvzMOA/hwamtEORnAMuFU\nZUUwZ7JpJaqJQnRSuyVTKBOFl4o1S9RTUplMVq5MGFnvlTitWedMIvVZIjwOiFN3reyJYxCjDHNa\niNubCf9SN26LWvE7OfPeyWQsU6/CidZcNMNekgfX3V+UtLhNLQKQ6Hwl3Xqeh+/ahraYmZUuObI9\nIWn/NjflZ/jyYjNrtim9yMKdr5k1V42hZBHndjCzZisv1OwrAP43gH9A8kWSH5qkWR75mlmjlZFP\nBtjw/NeWeeRrZlYDj3zNrNmmdM536jrfTHxukVhZV4lXlkoFmZBZKbkV/elTYVwtiwrT+SX2TXsl\nLtM5G7+w7uk4JrZ9Ot6YOvGbsL07k3MzLhKlVmzPx6+plTijlFhEGsVaOauCq5OI8+1UtLDaFJ9w\nm7rO18ysVO58zcxq4M7XzKxaRHnRDmVz52tmzeU5XzOzmrjzNTOrgTvfoSj9nVqJMLIKU0pm/mRh\nJkthsGpuK7GScjuRepGDct5pSqRVLGPVZiC5GnVC6r2TCCPLpF8sZhL7pxcWgXrjJyR7vfigZxbq\n7q8mUkq2MyslZw5oXKRKnnYwM6uDO18zs4rJ0Q5mZvXwyNfMrHqe8zUzq4M7XzOzim1uafhKVR9q\nFoQwlRYilgj/CjONIZmVK5EyKsrc1TmbqGM1bnCmvUU3jgXqzybqSWSXKzLJv1KrBWfeGPHGOnOZ\nFYXj9qxcELdnbSEsAs6OP2C9bmL14szyxamVrxNZzfpxmVYmg1pFqcSJ6Z12cDJ1M2s0Kr6l6iFv\nJPnXJJ8n+fFJ2+XO18yarYQ13Ei2AfwZgN8E8BYAt5J8yyTNcudrZs1WzgKa1wN4XtILklYBfBXA\nzZM0y52vmTVXYsphNO2wl+TBdbcDr6vpcgA/Xnf/xdFjW+ZoBzNrttzI9oSk/dvckp/hztfMGq2k\ny4uPALhy3f0rRo9tWdj5krwSwJcAXIbhd8iipM+RvBjA1wBcBeAwgFskvTyuLrWAQRDClNlRrUR2\nr/ZK/HWXyySWqCcT1hbU016JX3irH7elyIRtJULNBplwtPlEPZnMXpmsXHOJ9iTKtFcS2b0Sk3Er\nFyVe+1zivdOdvGcYZHZgIhtZ5n2cWew0swOV+HyWpaRQs8cBXEPyzRh2uu8H8DuTVJiZ8+0D+Jik\ntwD4ZwB+f3SW7+MAviPpGgDfGd03M5semZNtic5ZUh/A7QAeAnAIwH2Snp2kaeHIV9JRAEdHPy+R\nPIThRPPNAG4YFbsHwPcA/OEkjTEzK11Jg2xJDwJ4sJzaNjnnS/IqAG8D8BiAy0YdMwD8BMNpCTOz\nqTHNV7ilO1+SuwB8HcBHJZ3iuvT5kkSe/yWOQjYOAEB390WTtdbMbJNYTGfvm4rzJdnFsOO9V9I3\nRg8fI7lv9Pt9AI6f77mSFiXtl7S/PZ+42N3MrCwlzfluh7Dz5XCIexeAQ5I+u+5XDwC4bfTzbQC+\nWX7zzMwmU1Zuh7Jlph3eDuADAH5A8qnRY3cAuBPAfSQ/BOBHAG7ZniaamU1gOmcdUtEOj2LjhHTv\n2szG1AbWdo8v0z4b19NKxBp2z8R7fOZUHGOZSeOIEuaUMt++mXDOVAxvYuXd/ly8qUyZKK4byMXV\nZuoZzCZiWROx3Zn0i5l0kUU3PqgajN/Y2dU4TeZgkImrjcu0VhNx0suJMpl9nHozl2PHn3AzM9uR\n3PmamVXMqxebmVWvEXG+ZmY7kqaz93Xna2aN5pGvmVnVvHrxkNrA6oXjZ7+7rTgEpXs6kzowEWr2\napybsn02sXpsIo1j0R2fyrCYiUOBim5cZpCoJ5MuUokVfBVnZ0yFkamdWkQrLBJEbQEAikSbM5T4\n5KRWkl4e36CzRZyTU2vxTm6fjl9491S8A3svx8eqsxwWQWtQYUpJn3AzM6ueO18zs6oJPuFmZlYH\nn3AzM6vDlHa+XjrezBrr3EUW253VjORvk3yWZEEytQqyO18zay4JLOJbCZ4B8G8BPJJ9QrXTDi2h\nuGB86FZ/EDepSIRKZaR2epE4VcpEVqkgHK3IhHaVFP6VwUQoUGst0Z7EocqE6mUyjaUWQixruJHY\nViZLGJbGHzCdiQ9oJ7GdmVfiMrMvxS9q7mT8eeguxeGZmZW4S1PBpiQdAgAmQiLP8ZyvmTWaT7iZ\nmVVNyObb3kvy4Lr7i5IW1xcg+ZcA3nSe535S0qZX8nHna2bNlhv5npA09kSZpN8opT0j7nzNrNGm\nddrB0Q5m1mhVRDuQfB/JFwH8cwB/QfKh6Dke+ZpZc1WU1UzS/QDu38xzKg81a/XGp3oqZuLQmiJe\nUzC5SGS8rXYiy1omBGzQG7+tQS8TRlZOaFfmmz61mGJJYW2Z45nKjpaJ8ilp3cZMxrL22cQ+DMq0\n4qgtdM7EZWZPxsd8/nic5a93Ik5Z1joVl+Fa4oWVYHiRxXTOO3jka2bN5qxmZmbV88jXzKxqXsnC\nzKwOpeVuKJ07XzNrNk87mJlVTF5GyMysHjt15EvySgBfAnAZhlPXi5I+R/LTAH4XwEujondIenBs\nZSIULDHLopxY1kyc79oF8XdPP5FaMbV6cRALnFpVt6QY1dJSOJYklXYylZoysbGy4nwTo6lWHDYb\nlum8Ftcx+0rcmPljiRjeo0vxxn76SlhES6fDMkVFcb4AdvQJtz6Aj0l6kuRuAE+QfHj0uz+R9Mfb\n1zwzs8kwk5O7BmHnK+kogKOjn5dIHgJw+XY3zMxsYsLUXmSxqcQ6JK8C8DYAj40eup3k0yTvJnnR\nBs85QPIgyYODpcR1kGZmJSEEKr7VId35ktwF4OsAPirpFIDPA7gawHUYjow/c77nSVqUtF/S/vbu\nhRKabGa2CVJ8q0Eq2oFkF8OO915J3wAAScfW/f4LAP58W1poZjaJKY12CEe+HK4IdxeAQ5I+u+7x\nfeuKvQ/D1TvNzKbHuTnf6FaDzMj37QA+AOAHJJ8aPXYHgFtJXofhyzsM4MNhTQWB0+PzB7aX41ig\nTAhPJgVhfzYR1pZIKZkJayuCPV3WqsPKrJ6a2DdRewGgmInLDDLpIhOvPVWmlVlSOC6SkvjAthPx\ncVwN6liJX1P3TGJF4VPBhgDwVHxOpsiEkS3HKSWrHI3u5GiHR3H+6MjxMb1mZrWrb0434mWEzKy5\nhEpOuJH8TyR/OIr+up/knug57nzNrNmqmfN9GMC1kv4RgL8B8InoCe58zazRqojzlfRtSeeumf4+\ngCui5zixjpk1W/Vzvv8ewNeiQu58zay5JGCQmlfYS/LguvuLkhbXFyD5lwDedJ7nflLSN0dlPolh\nPpx7ow1W2vmyD3RfHj/T0TkTh+d0luNvskzWqUE3s+pwXE9/PlHP7Pjfp1bwLSncKrUScGZCqsLM\nZ5ltZcY3qZC+TMhapkhiW1EWv0w4X5Qxb1gmPqDtbtwdcC54IwNolbWs9alyqkmOfE9I2j++Gv3G\nuN+T/CCAmwC8S4o36pGvmTVbBdMOJG8E8AcA/qWkRCJQd75m1mQCUM0abv8ZQA/Aw8OLgvF9Sb83\n7gnufM2swQRo+69wk/Srm32OO18zay4he8Ktcu58zazZpvTyYne+ZtZs7nwBDoCZV8aHxXQSCZEy\nZVr9eIenQnQS4Wj9ubg9axeMb09/PhEi1kv8+dTNrOyYeDMmFjLFWhy+xNVEZq9+okyw8OqwUGIf\nJt7x6mbC9RLvr14mw9z4fdhK7JvV5fg4dC+IYxnby/FiB2zH22ol/sxXop5yQs2mN7GOR75m1lwC\nsFNTSpqZ7Wge+ZqZVS19eXHl3PmaWXMJUAVxvlvhztfMmq2aK9w2zZ2vmTWb53zNzComOdoBGKZ5\n7AT5fjKrtbZXy0kpmVmhN5MWsEiknRzMjW+zdvfH/h4AuvPxCrS9XlwPE/Gw/X78wleW49jRgpm3\nWCJeOJPCMZN2sp2JBU68eTqZehIx4kE1rZX4OHQSKU3XdiXqORPnr8wczUQ2RSiR4rI0HvmamVVN\n0GBQdyPOy52vmTVXdSklN82dr5k1m0PNzMyqJQDyyNfMrGKqJpn6VrjzNbNGm9YTbsyEhZS2MfIl\nAD9a99BeACcqa0A53Obtt9PaC7jN2+GXJV0ySQUkv4Xh64yckHTjJNvarEo735/bOHkwWq552rjN\n22+ntRdwm23zKox0NjOzc9z5mpnVoO7Od7Hm7W+F27z9dlp7AbfZNqnWOV8zszequke+ZmZvSLV1\nviRvJPnXJJ8n+fG62rEZJA+T/AHJp0gerLs950PybpLHST6z7rGLST5M8m9H/19UZxvX26C9nyZ5\nZLSfnyL53jrbuB7JK0l+l+RzJJ8l+ZHR49O8jzdq89Tu5zeCWqYdSLYB/A2AdwN4EcDjAG6V9Fzl\njdkEkocB7Jc0tbGRJP8FgNMAviTp2tFj/xHASUl3jr7oLpL0h3W285wN2vtpAKcl/XGdbTsfkvsA\n7JP0JMndAJ4A8FsAPojp3ccbtfkWTOl+fiOoa+R7PYDnJb0gaRXAVwHcXFNbGkXSIwBOvu7hmwHc\nM/r5Hgw/eFNhg/ZOLUlHJT05+nkJwCEAl2O69/FGbbYa1dX5Xg7gx+vuv4id8WYQgG+TfILkgbob\nswmXSTo6+vknAC6rszFJt5N8ejQtMTV/wq9H8ioAbwPwGHbIPn5dm4EdsJ+byifcNucdkv4xgN8E\n8PujP5l3FA3nmaY9xOXzAK4GcB2AowA+U29zfh7JXQC+DuCjkk6t/9207uPztHnq93OT1dX5HgFw\n5br7V4wem2qSjoz+Pw7gfgynT3aCY6N5v3Pzf8drbs9Yko5JGmi45vcXMGX7mWQXw07sXknfGD08\n1fv4fG2e9v3cdHV1vo8DuIbkm0nOAHg/gAdqaksKyYXRyQqQXADwHgDPjH/W1HgAwG2jn28D8M0a\n2xI614mNvA9TtJ9JEsBdAA5J+uy6X03tPt6ozdO8n98IarvIYhTW8qcA2gDulvRHtTQkieSvYDja\nBYapOL88jW0m+RUAN2CYyekYgE8B+O8A7gPwSxhmlbtF0lSc5NqgvTdg+KewABwG8OF186m1IvkO\nAH8F4AcAziWKvQPDOdRp3ccbtflWTOl+fiPwFW5mZjXwCTczsxq48zUzq4E7XzOzGrjzNTOrgTtf\nM7MauPM1M6uBO18zsxq48zUzq8H/A4NVMJ/j/hNPAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7FrGMBAwEP8e",
        "colab_type": "text"
      },
      "source": [
        "#### Smooth random b\n",
        "Smooth random b image."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CadvEVzeEM7F",
        "colab_type": "code",
        "outputId": "440a0d5a-0078-4946-c6b2-9071b886fd59",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        }
      },
      "source": [
        "# Random 4D matrix (unsmoothed)\n",
        "b_us = np.random.randn(nv*q).reshape(dimv[0],dimv[1],dimv[2],q)*20\n",
        "\n",
        "# Some random affine, not important for this simulation\n",
        "affine = np.diag([1, 1, 1, 1])\n",
        "b_us_nii = nib.Nifti1Image(b_us, affine)\n",
        "\n",
        "# Smoothed beta nifti\n",
        "b_s_nii = nilearn.image.smooth_img(b_us_nii, 5)\n",
        "\n",
        "# Final beta\n",
        "b = b_s_nii.get_fdata()\n",
        "\n",
        "# Show unsmoothed\n",
        "imshow(b_us[3,:,:,1].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f9e15e37208>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAH8xJREFUeJzt3Xt0VfWZN/Dvc07uFyAhECDcERQE\nBEwBFSveWsUbttVKXa2dWrGd6hrHzkytM6Ndtp3XcdQZl/PWFisvWu/jDergvVpFKhJuAoLcDJeQ\nhIQEciXXZ/7I0aEtkO8mgZTf+/2s5TIcvmx+e+9znmxOnvNsc3eIiMiJL9bTCxARke6hgi4iEggV\ndBGRQKigi4gEQgVdRCQQKugiIoFQQRcRCYQKuohIIFTQRUQCkdSVP2xmFwF4AEAcwK/d/e4j5bNz\nk7xfQSq17eqWDHodDY3cNgEg3mB0FgDae7XTWW+Ptm1Wr/RGOlvblEZno643FuHYtafxn0C2JD7r\nLRGuQSJ8CDqJP8QAgNEF5XR2f1sKnd1d24fOpuyno2jNi/A8PhDnNwwA/KZhEc5Jau8mOttcxdeA\nPv1q6Wx1VTadBQBri5Dt00pnGzaXVbp7v85yR13QzSwO4P8CuBDALgDLzWyRu398uD/TryAV//Li\nWGr7z+0ppNeyfN0oOtt3ebQna92X6+hsUx3/pLIY/8y+cNxhD+mfeXvbaDrbeiCZzgJA9mp+/2pO\n4Z+syX0O0Nm2Uv4bfayF/wbUZwMdBQC8ftd9dPa39UPp7J3vfIXODn2ZjqLq+gjP40968xsGkFTP\nH2fjnxYYddE2OrvzmZF0dvaN79DZ/3pqJp0FgOQaPpt66R46u/ziu7czua685TIVwBZ33+buzQCe\nBnBFF7YnIiJd0JWCXgBg50G/3pV4TEREesAx/6Gomc01syIzK6qtivDvLRERiaQrBb0EwJCDfj04\n8dgfcfd57l7o7oXZuV36GayIiBxBVwr6cgCjzWyEmaUAuAbAou5ZloiIRHXUl8zu3mpmNwF4DR1t\ni/Pdff2R/kyqtWJ4ciW1/fWLT6bX8pWvfkBn314zjc4CQNbrWXT25lv5toP/Lp9AZ1c/OInO3vlP\nz9LZO165is4CQO4lf/YPsMOq2d5ph9XnUov4Y9zOdwDivu88Qmc3Nw3gNwxg+oIf0tnYKXyHyb3n\nPU1n/751Dp29aDDfMfL7D6fQWQDIOaeMzrY8kU9nqx4cRmfbB9NRnJvNd409NuCL/IYBNBTw3WtJ\n+zMjbZvaZlf+sLsvBrC4m9YiIiJdoE+KiogEQgVdRCQQKugiIoFQQRcRCYQKuohIIFTQRUQCYe4R\n5ll2UXr+ED/pG7dS2bpCfp7pBSdvpLNvv8X3dAPAoNNL6ew741+is2Pe/RadnT6smM6uWDSezp45\new2dBYCY8XNS3/mUn/rYUpnOryGHH6ma06uBzh5oidbBG3uXH3N73fWv0tkHl1xAZ1Mq+MmhqdX8\nRMSskgjzcAFUTOG3HR8RYepjhOdFvJG/No0V8M+L2Cf8ZyQAoPBCvse91fk1/9eZ81a4e6cjaHWF\nLiISCBV0EZFAqKCLiARCBV1EJBAq6CIigVBBFxEJxHG940RS7xbkXLKbyrbX8zcDfn0N36oXHxbt\n9u71zfy81pPf41sRW+r57W6s4keOjp21ic6uroh2x8DWNv77/+C+++js7vV8a1hzb75FrnZlXzrb\nVNBCZwGgV4Ru3wf/cD6dzV/CH+OaYfyxiDJ2+Oo7+DZLAJj/yCw62ziEb7Ucvog/yMWX862WOa/x\nY2u/dusbdBYAHl57Fp0tHLYj0rYZukIXEQmECrqISCBU0EVEAqGCLiISCBV0EZFAqKCLiATiuLYt\nNrcmYUdZLpUdP4xrbwSAT5r53Wjem0ZnAaApg5/u17w/lc7e+cWFdPauN66ks9lj+fXGn+bOxWea\nvrafzsaMbzlr6RVhul8lf4yb8tro7N9Mf5NfA4B5xXyrXtJe/vl59q0f0NnnP5pCZ/0A3y5YtH84\nnQWA2vHNdPaUe/js0Hmf0Nnij06ls3sn8+2ev1w6k84CQK98fprkR6+cEmnbDF2hi4gEQgVdRCQQ\nKugiIoFQQRcRCYQKuohIIFTQRUQCcVzbFvPS6/DdSe9T2a/3XkFvd/Z//wOdbe8b7abYya/wNwP+\n1l+/S2fv38hP4Euu4b/vfmXQKjp775kD6CwAnJ1fQmfXPMtPwGybyLdaZq3n2xYR44/bA/EL+e0C\nyObvM4z0qXvp7MJNE/gN70+mo2dP5W9evKpsML8GAIjwktp2VW86W7x4Mp0dNL2Mzu7ezk/htDS+\n9RUAWpfl0Nncc/g1s7pU0M2sGEAtgDYArcxdqUVE5Njojiv0c929shu2IyIiXaD30EVEAtHVgu4A\nXjezFWY291ABM5trZkVmVlRfzX/sV0REounqWy4z3L3EzPoDeMPMNrr7H/1k0N3nAZgHAAWn9on2\nE0kREaF16Qrd3UsS/98D4EUAU7tjUSIiEt1RF3QzyzSz7M++BvAlAOu6a2EiIhJNV95yyQfwopl9\ntp0n3f2ItwuvrM3Gr98+l9r4b6r5Pu3WSXxTcGtNhNufA3jsxw/Q2ctf+ls6m7mD/17aOpQfL7tg\n2xl0Nq0s2ulfsYjvLbcI7eJo4Y9FrIXfbM0E/mc2sdqIL4Wzq+loYxP/nLMt/B3pY+nH5h3M+h29\nIuUL3uPXMe3Hy+jsa89Op7MVq/LpbJTjNnTkHjoLALvKBtHZymXRPgfCOOqC7u7bAJzWjWsREZEu\nUNuiiEggVNBFRAKhgi4iEggVdBGRQKigi4gE4riOz403ArlruTtu7z0zwpiAWn6MaCwzQt8bgO/c\ncSudzY1wNPeP4VunLjhzDZ39/auT6GzT4GjHItbA3zm+z6gqOpuymB9nmlzPH7e23EY6mz4g2rFo\nauWPxYHdfCui5/HjWk8bu53Otrbz6x22uJXOAsCOi/kn/iufjqOzhbP5j7Us3TaKziYVp9HZnWsG\n0lkAaOsb4XmUEe04M3SFLiISCBV0EZFAqKCLiARCBV1EJBAq6CIigVBBFxEJxHFtW2xPBhoGcG2L\n8Wp+aSn7IkwuHBftrkkVU/lJhx7hDuEnP8S31K1ez7citkzm2/qGDIt2K9jmNr71LfnXfCti6Vn8\nmmOD+OOGZn69uZn8xE4AqNzN373+azM+pLPPrZlCZ9d8PIzOxrIjtNN9OVpZSCvnX3/NTdl0dlnb\ncDobT+Jfe20j+OdQe3WUsaEAWvljkbQxI9q2CbpCFxEJhAq6iEggVNBFRAKhgi4iEggVdBGRQKig\ni4gE4ri2LXocaMrl2gB7beW/17Rk8WvI7V3PhwFUxyLciHcTP1WvopCfEDlzLn9j3eWVfCvbrtJc\nOgsAsWS+hXPh/f9BZy/77S101svT6Wx7Cr/eMufaaT8XIf7iRv7Wu1l9+Ja6vCz+uby3nm+Rq6+I\n1qqXvZM/zgfq+Nf1/t78VMR4hLbMlFQ+mzksWjtr9Ub+NZVeuDfSthm6QhcRCYQKuohIIFTQRUQC\noYIuIhIIFXQRkUCooIuIBEIFXUQkEJ32oZvZfACXAtjj7uMTj+UCeAbAcADFAK529+pOt+VAUiPX\nwHvvrb+icgDw6J6z6GyUu4MDwKC8fXS27Qt1dLa6aQCdXbpnBJ3ds7cXnU3anUJnASC5lm++/urW\nW/kND2uio5m9+T7tlpU5dDZ1Q7SPZNRO49cR2873zqcU89mSc/k1t9TwveVZJdGu8+656xd09jvP\nf5/O9hlYQ2cb1vLnujmV728/ZdoWOgsAlb35scpJcb5/n8WcuQUALvqTx24D8Ja7jwbwVuLXIiLS\ngzot6O7+LoCqP3n4CgCPJr5+FMDsbl6XiIhEdLTvoee7e2ni6zIA+YcLmtlcMysys6K2+mgfuxcR\nEV6Xfyjq7g7gsANP3H2euxe6e2E8k591IiIi0RxtQS83s4EAkPj/nu5bkoiIHI2jLeiLAFyX+Po6\nAAu7ZzkiInK0mLbFpwDMBJBnZrsA3AngbgDPmtn1ALYDuJr5y+IZrcidwl3Mf/e166kcAFhmK539\ntzOeo7MA8HdLqF0DAMT38iNx2wbxa26r4d+qipXwLVkWsWsqxt9YHS1j+J+XrJrBt6gWPsG3Q468\naymd3bPwFDoLAO2lfHto8ii+nbWqN9+2WNCH3+6BN/nn0N6p/HhZALhnx8V0dsWc++ns7A3X0Nna\n9D50tq0X/0TeXce3IQLAWaduprPrK/jWZVanBd3d5xzmt87v5rWIiEgX6JOiIiKBUEEXEQmECrqI\nSCBU0EVEAqGCLiISiGgj5rqotSkJ5VvzqGxaPn+37eE/59uQHhgQrTknqYJvRZxwBj+ZLSuZnzDY\n0MpPRdz58mg6W1F42A/4HlL7SH7C4Nzx79PZyc//LZ1N38dPfNz65CQ621YabfLkz859ns4urODX\nsXrbGDr7d6Nep7Nv3ngqnX3vydPpLACcc+Ymfh0Nh50S8mfmDF5OZx9vn0ZnG5/h2wWzr+VfpwDw\n/scn0dmsTdGecwxdoYuIBEIFXUQkECroIiKBUEEXEQmECrqISCBU0EVEAmEd96c4PvLG5vllj15G\nZVfPn0Bvt2oaPx0ua2O0VqH6CQf48D6+xXHoq/yowwvufo/OPv7ieXS2//TSzkMHKd+XTWdbGvlj\nEUVqMX+z46a8COMho66jIk5n+67j11FyKT+FE+18Cydi/Os8dWe010jrSXw7a/rKDDrb3Idfc8aE\nTu9R/7lRuZV0dmMF32YJAL6cn87YNIFvzf50zj+tcPfCznK6QhcRCYQKuohIIFTQRUQCoYIuIhII\nFXQRkUCooIuIBEIFXUQkEMd1fG5tUxre3sSNB825bC+93WuHbKCzz1TOoLMAkN2L77E9sIPvkS6f\nyvcxv1B8Gp390iX8yNHVewfTWQAY0Y8/J9vKuTHJAJC0nr8j/Vtz76Gz397M3zW+eOkQOgsA8dP2\n09mmnb3obH4+v93yUv5O9xeM20hnJ07dRWcB4IXd/Hjgm+a+QGfvWMt9ZgUA9pXyx3jd6lw6m1Ea\n7XM6SZdV0NnJOXz2UzKnK3QRkUCooIuIBEIFXUQkECroIiKBUEEXEQmECrqISCA6bVs0s/kALgWw\nx93HJx77CYAbAHzWd3O7uy/u9G9rNaCaG825z/hWtoWt/Kjd7LFVdBYAGlf0pbMtI/lRu7PGraez\nVc38yNFX3+x0wubnxp2xjc4CwJqtfGtfLJkfGZtey6/hhm1X0dnt7/Pr/fKsIn4RAF7+YAqdzb5q\nD53du6o/nbVBzXT2zY/G0dl+hRFOCICkGD8KekX9cDrbuIsf15xRxl+btkysp7N1XJf151o/5Vsi\n7zp5EZ19iswxR2EBgIsO8fi/u/ukxH+dF3MRETmmOi3o7v4ugGiXtSIictx15T30m8zsIzObb2Y5\n3bYiERE5Kkdb0B8CMArAJAClAO47XNDM5ppZkZkVtdXx712JiEg0R1XQ3b3c3dvcvR3AwwCmHiE7\nz90L3b0wnsX/oFNERKI5qoJuZgMP+uWVANZ1z3JERORoMW2LTwGYCSDPzHYBuBPATDObBMABFAO4\nkfrbYg5P59rZRg3i78z9aTnfWviDSW/QWQC4r+5COut1/J3uV93DT6grn823p8VH8G9rrdk0lM4C\nQNZmfv/a+WGSiJ/PT3Fs/OkgOtv0jRY6+8lNY+ksAOT8I3+X+cqP+FbE7B38GvYP5dsFM3rxLbXP\nv3wWvwgAA6eV0tkXFvHTTqect4nOrn1nNJ3tn8O3Ze5/awCdBYBZ13xIZ/954xURtsy1OXda0N19\nziEefiTCSkRE5DjQJ0VFRAKhgi4iEggVdBGRQKigi4gEQgVdRCQQKugiIoHotG2xW7UZYnVcg/KW\nrXz/Z0Yx3x/9r/svobMAkFzNN1SnHTA6m/397XQ2y/ntnp7LNzKXN/F3SgeAvMI6OvvGr86gs73S\nmuhseSH/mYNYSiOdLb4s2qeYm3em0dlBp5XT2ZYN+XS2vYF/+aa8z6933LV8/zcArC3lPxtg/FRl\nrCqOMq6Z3275Gv4Y58ys6Dx0kC21/ehs1WZ+1C5LV+giIoFQQRcRCYQKuohIIFTQRUQCoYIuIhII\nFXQRkUAc17bF5HpgwAdc9pwfL6O3+3TjmXQ2awDfegcA2e/wrX1ls/j2u60f8qNrkxr4tsWd7fx2\nY/x0WQBA8pn8rWX3TeU3fuA9vu3tw5vvp7MT3/wBnW0p4EcUA0BSCt9/t6eKfw71b3A6O3Z0CZ3d\n0DqYzkZpQwQArM+mo5ml/P61p/KtlkOn76Kz0/oW09mnfhdtlHDD8BQ6WzCOb2ctJnO6QhcRCYQK\nuohIIFTQRUQCoYIuIhIIFXQRkUCooIuIBMLc+Tairuo3rq9/9TezqOzrm/i7sI8bzN91/Oy+m+ks\nAGxvzKOzb794Op2dfvlHdHbpKxPpbPs4vi2zpSKdzgJAvIH//p+xm2+1bO7Dr6FlTAOdTV2bQWfT\nzqzkFwGgfiX/vGgefoDO9u9XQ2era/n9e3Lqr+nsNx+5hc4CQMF7/FTLLV/nxyIOfJd/vtUO5rND\nFvPnevsV/HkGgBlXrqKzceNr768KH1/h7oWd5XSFLiISCBV0EZFAqKCLiARCBV1EJBAq6CIigVBB\nFxEJRKfTFs1sCIDHAOQDcADz3P0BM8sF8AyA4egYBna1u1cfaVt1zalYUjKCWljqOr6lru02fmjk\nQ7ddQGcBwJPa6WxyFt+GtOTNCXS2Ob+Vzman8lMOmzP4yXAAkPsh//1//6W1dDa5iJ/WF2VAZIw/\nbGht428GDgBNEc5JLMY/L+p/15/OTriCv5nzd+/lWxHbBkZrZY7X8JMqrY1/zjX14ltfZ127lM4u\nBj+dtSU72rH4/SuT6Wx7cpRtP06lmFdoK4Afuvs4ANMB/MDMxgG4DcBb7j4awFuJX4uISA/ptKC7\ne6m7r0x8XQtgA4ACAFcAeDQRexTA7GO1SBER6Vyk99DNbDiAyQCWAch3988+olmGjrdkRESkh9AF\n3cyyADwP4BZ3/6PPJ3vH/IBDviFkZnPNrMjMitpq+I9ti4hINFRBN7NkdBTzJ9z9hcTD5WY2MPH7\nAwHsOdSfdfd57l7o7oXxXvzsCRERiabTgm5mBuARABvc/eAbOi4CcF3i6+sALOz+5YmICIvp9zsL\nwDcBrDWz1YnHbgdwN4Bnzex6ANsBXH1sligiIoxOC7q7LwFwuIbQ86P8Ze3thoaGVC47lh85Gn88\nQk9wSROdBYDBefvo7P4B/F3Ka2r5Pvvknfx2z5vGjwe+cPw6OgsA9z9+LZ2tOMCPSW09jR+/2l7N\nPX8AoGlKPZ1Nbov2Gbu0Mv6zDymT+J78pl78ud700hg6m17L9zyf/VdFdBYAXkOnU13/V2/+9Vc/\nmD/XS/5lGp0dfTP/GlmzYzCdBYDkdL4nf0zeXjq7lczpk6IiIoFQQRcRCYQKuohIIFTQRUQCoYIu\nIhIIFXQRkUDwvVfdwGKOlBSuxfC6Ccvo7b648zQ66+38SE4AKPtgIJ2Nn8rfsb29IcKhT+dbztbv\n49f72qKp/BoA+Pf49jsryaSzsYH8SIjMAv4YXzJsPZ196bkZdBYATpu1kc7W3MDfOX7Mb5bT2fW3\n8COYt97AP+/fLD6ZzgLA6efyx2Jy7x109vEVX6az+Tdvo7O75p9EZ+PRDgUyNvNtp+tO48dGs3SF\nLiISCBV0EZFAqKCLiARCBV1EJBAq6CIigVBBFxEJxHFtW/SmOFq2ca06j8e/QG+3riJCi1x6hFvB\nA2gZwU99bK7lW5aSsvj717c18d93t24cRGdTo3Vwok8WPxWxPIs/FrEd/Pmrj7DmSeO209mFU/gW\nQAAYkMa3T374o+F0dsOyKXQ26fv8ZL/c3/Hno2pGtCfG6sVj6ewHA/kJkUln1NHZT14dTWfjs6vp\nbGtJLzoLAFUT+WOXuav7r6d1hS4iEggVdBGRQKigi4gEQgVdRCQQKugiIoFQQRcRCcRxbVtMTm9B\n/sRyKltSkktvN7WUvyFxeoQb9gLAhP676ezSpePo7MfXzKOzU39+M53N2cK3spXN5W+iDAAXF3xM\nZ59YfQ6d7buOnyZZf/V+OnvnmsvpbGtLnM4CwMKVk+lsn1X883PfRL6t9uNzHqGzYw7cSGfTN/M3\nZwaAnLPL6Gzl/ix+w1v4dlZr4zfbUM+3cGZFmO4JAPHXc/hsM/+8Z+kKXUQkECroIiKBUEEXEQmE\nCrqISCBU0EVEAqGCLiISCBV0EZFAdNqHbmZDADwGIB+AA5jn7g+Y2U8A3ACgIhG93d0XH2lbLU3J\n2L2lH7WwjBK+L9gjtBC3LOX72wFgWTbfV5rczI/OvL/qFDpbP5jvV02p43uesSqFzwJ46e2ZdLb1\nC0109mdfX0Bnb1r5DTrbN5vvs8/4abQxqTX/yI9gHXQy3zu/atMwOjvm5e/RWTvAv0iac9rpLADs\n3tmXDyfx2045iT9//XL5Y1y8NZ/Oxj5Ip7MAUDOa37+RE0v4DT/MxZgPFrUC+KG7rzSzbAArzOyN\nxO/9u7vfy69KRESOlU4LuruXAihNfF1rZhsAFBzrhYmISDSR3kM3s+EAJgNYlnjoJjP7yMzmm9kh\n35sws7lmVmRmRW11/B1IREQkGrqgm1kWgOcB3OLuNQAeAjAKwCR0XMHfd6g/5+7z3L3Q3QvjWRHm\nOIiISCRUQTezZHQU8yfc/QUAcPdyd29z93Z0vGU/9dgtU0REOtNpQTczA/AIgA3ufv9Bjw88KHYl\ngHXdvzwREWExXS5nAfgmgLVmtjrx2O0A5pjZJHS0MhYD4OdzEpIjTHatHc63CvU9eW+kdZySy437\nBaKNz52/6AI6m7WLjqL2cn48cOPeaC1ZBefxY1L3bxxEZ9ceGEJnR/6I37/k/9dIZzdczLeyAcCI\ntIrOQwkb3+DvSJ87tZLO1tRm0Flkt9DR1qZoo4RPWsC//rbdwG83rzf/M7fyd/g+jf/zraf47ICL\n6CwA/NXIVXS2d5x/fv6OzDFdLksAHKrB+og95yIicnzpk6IiIoFQQRcRCYQKuohIIFTQRUQCoYIu\nIhIIpm2x28RT2tB7CDcV7ezpW+ntDkmrorO/WHI+nQWAeIxvycrcxX9/bJ3BT4fDLn4SYEEOv90d\nG7L5NQAorh7Mh9P5CZEb6gd2HkrYcj2fTXqNn36Z0kxHAQCb1vKtlvEs/lhkvsBPLvzNP/8nnb1l\n49fpbNqD/IRRAPj0cn7C58iH+Smce6YMoLONY1rp7E8fnUNnW7L5cwcAOwfxx+7NffzEVeAtKqUr\ndBGRQKigi4gEQgVdRCQQKugiIoFQQRcRCYQKuohIII5r22Jbcwz7dnMteL8tm0RvN57Ftyyllkfb\n5arK/nT2X//6MTp7xy+/RWdrRvGtkzXlfNtb6rgaOgsA0wu209mlr06ks/tb+KmPQ6fzoyevLVjW\neSjhnnVforMAkPUHvpX0Z99bQGdve+zbdPaGNd+ks/5BHzpbflW0Hs6Mzfx14QX/uYTO/nL5OXS2\nVy4/nrW+vjedPf+La+gsACzZOZLO5mTy0xZZukIXEQmECrqISCBU0EVEAqGCLiISCBV0EZFAqKCL\niARCBV1EJBDmHm08ZFekDxziw6+/lcoOu7CY3u7GdfwoU+sTrce2vZ4fDWpt/LhWT2+js0l7+DX0\n3kxHUTWJ728HgFPG76SzKTF+/yoaM+lsWSXfQxzbnUZn377m3+gsAFx29z/Q2YaB/GssqYF/Ds26\n6g909tXtY+ls3d4MOgsAz53/Czr7852X0NnGm/Po7JYfpdLZ+Bb+cw9N/fnnMQCMGb2bzg7O3Edn\nF0xbsMLdCzvL6QpdRCQQKugiIoFQQRcRCYQKuohIIFTQRUQCoYIuIhKI49q2aGYVAA41gzUPQOVx\nW8jxFfK+Adq/E53278QwzN37dRY6rgX9sIswK2J6LE9EIe8boP070Wn/wqK3XEREAqGCLiISiL+U\ngj6vpxdwDIW8b4D270Sn/QvIX8R76CIi0nV/KVfoIiLSRT1a0M3sIjP7xMy2mNltPbmWY8HMis1s\nrZmtNrOinl5PV5nZfDPbY2brDnos18zeMLPNif/n9OQau+Iw+/cTMytJnMPVZjarJ9d4tMxsiJm9\nbWYfm9l6M/ubxONBnL8j7F8Q54/VY2+5mFkcwCYAFwLYBWA5gDnu/nGPLOgYMLNiAIXuHkIfLMzs\niwDqADzm7uMTj90DoMrd7058U85x9x/15DqP1mH27ycA6tz93p5cW1eZ2UAAA919pZllA1gBYDaA\nbyOA83eE/bsaAZw/Vk9eoU8FsMXdt7l7M4CnAVzRg+uRTrj7uwCq/uThKwA8mvj6UXS8iE5Ih9m/\nILh7qbuvTHxdC2ADgAIEcv6OsH//X+nJgl4A4OA7JuxCeCfAAbxuZivMbG5PL+YYyXf30sTXZQDy\ne3Ixx8hNZvZR4i2ZE/ItiYOZ2XAAkwEsQ4Dn70/2Dwjs/B2Jfih6bM1w9ykALgbwg8Q/6YPlHe/f\nhdY29RCAUQAmASgFcF/PLqdrzCwLwPMAbnH3moN/L4Tzd4j9C+r8daYnC3oJgIPvHTc48Vgw3L0k\n8f89AF5Ex9tMoSlPvH/52fuYe3p4Pd3K3cvdvc3d2wE8jBP4HJpZMjqK3RPu/kLi4WDO36H2L6Tz\nx+jJgr4cwGgzG2FmKQCuAbCoB9fTrcwsM/HDGZhZJoAvAVh35D91QloE4LrE19cBWNiDa+l2nxW7\nhCtxgp5DMzMAjwDY4O73H/RbQZy/w+1fKOeP1aMfLEq0EP0HgDiA+e7+8x5bTDczs5HouCoHgCQA\nT57o+2dmTwGYiY4JduUA7gTwEoBnAQxFxyTNq939hPzB4mH2byY6/rnuAIoB3HjQe84nDDObAeA9\nAGsBfHZ38NvR8T7zCX/+jrB/cxDA+WPpk6IiIoHQD0VFRAKhgi4iEggVdBGRQKigi4gEQgVdRCQQ\nKugiIoFQQRcRCYQKuohIIP4H5Nm0/Xk4uEAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCNew36kESqR",
        "colab_type": "code",
        "outputId": "31e6ff48-f6c4-4e7a-9f16-6d77a00872fb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 286
        }
      },
      "source": [
        "# Show smoothed\n",
        "imshow(b[3,:,:,1].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f9e15d8d208>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGXZJREFUeJzt3V2IpOd55vHrrre6e7p7PjSy5NlZ\nWYkco5PgZeVlVlk2ZtESErxLQPaJiA6CAiHyQQwx+CBGJ/bJggmxsz4IhnEsIoPjxGB7LVizG2MW\nvDkxHgtjK1Y+FFlypIw0kkfWTH9XvXXvQZeWWe/M9H11dXVrnvx/INRd/fTbz/tRd79TfdX9RGYK\nAHDrGxz1BAAAB4OCDgCNoKADQCMo6ADQCAo6ADSCgg4AjaCgA0AjKOgA0AgKOgA0YjjLN0fE+yR9\nWlIn6U8y8xM3G9+dXMmFt99W2naOuvI8BtvloRqM62MlKaM+drJgbNcYq4Hxbt5JfcJhHouYeOOr\nsn6qlcP6sRgM6xMednPaOUnjvn7fNBnVxw526ue62ykP1WDLOxaxMyqPzd646IzLPjrj3nRYL3u5\nYFyckiYL9XmkMeWNn7z4Wmbeude4fRf0iOgk/bGkX5X0oqTvRMSTmfnDG33Pwttv0z1/8MHS9rde\nXi3P5fhz9YO+/JrX6qA3Cu/G2foTbOtsXx6by/WxsVk/FguXvX+gLV41frsZtm+rn5PR2+vF4+Qd\n6+Wxdxyvj3W9ula/lq++fKI8duWF+tP31I/qRfrkP3jHYvDjV8pjJz+5XB6bff26747Xj5vO3FEe\nOvoXp+rblbR5ZrE8dvtE/fn31OMfeaEybpaXXO6X9GxmPpeZO5L+XNKDM2wPADCDWQr6XZL+8ZrP\nX5w+BgA4AnP/o2hEPBoRFyLiQv/Gxrx/HAD8szVLQX9J0t3XfP6O6WP/j8w8n5nnMvNcd2plhh8H\nALiZWQr6dyTdGxHvjIhFSb8h6cmDmRYAwLXvlEtmjiPiQ5L+p3Zji49n5l/f7HtC0sKw9pfrzcX6\nX+XHK/Vkx6geOJAkZVdPdqRzNJ1k2MiIIjpjjYij5EU4nUiWdVthTMKJC26MnBypZzw2om/zSk86\n4S530ZtJfbyTXHHm4SzUM+jrBzmMsZI0GNfn0e0c/OJCM+XQM/Prkr5+QHMBAMyAd4oCQCMo6ADQ\nCAo6ADSCgg4AjaCgA0AjKOgA0IiZYouuQaQWizn0wUq9zebOqfpuuNlrJ7876eqDrby40Uq42zJa\nqhpthyUpjGPhZNatjPS4vuHtrXrnuyvGFFzbW/WM+2DHaJ9rdKId9Ma1OTbD8GOnJe7BZ69txhzC\nyNhL0sDIlncLB38suEMHgEZQ0AGgERR0AGgEBR0AGkFBB4BGUNABoBGHGlt0DBfqbTZ3Thhje28V\n727biBcaKaSFNWO7RlRvuFWfw8BYCd7lxBat1evX6/cgvepxwY1t77qIgdHa1Ygidlb74/JQa6wm\n8+rhK8XQKDld/ZzEsWPlsblYvy6yM+95jeveqRdV3KEDQCMo6ADQCAo6ADSCgg4AjaCgA0AjKOgA\n0IhDjS1OMrS5U4sMTYwV2+VEyIZmVsiILXabxlgjMtgZUURrJXF3cXfraqkfi7RuK+qDo6/PoT/m\n3dvkgrFyvJHhtDpaGlOeGKnMXPAinINjS/WxA+NYOFHEk6vlsf2p5fLY8apXIvvF+kmZdGbn1wLu\n0AGgERR0AGgEBR0AGkFBB4BGUNABoBEUdABoxOHGFvuB1t8oRpHGRjxty1hY1+jsJ3mRwYW1+tjh\nhrGY7Ki+XWdR28nQOxZOTC7qDTCtxY7TOBbpRPW8pJ4mRvzOuW2aGLHafrE+h9FKfRLjk/UYoiTF\n6Lb6WGOx6vFqfZHv8Qlj7HL9ZPdL3nPEOSfu869ipoIeEc9LuiqplzTOzHMHMSkAgO8g7tD/Y2a+\ndgDbAQDMgNfQAaARsxb0lPSXEfHdiHj0egMi4tGIuBARF/qr6zP+OADAjcz6kst7M/OliHi7pG9E\nxN9k5reuHZCZ5yWdl6Sld75jDosuAQCkGe/QM/Ol6f8vSfqqpPsPYlIAAN++C3pErEbEiTc/lvRr\nkp4+qIkBADyzvORyRtJXI+LN7fxZZv6Pm37HONS9VsuLOquUD4xs8nDDy34ON5yx9VeUhtvzefXJ\nybaO691JJXmZ3Ek9FqxJfRF2a2waV7ebQ3duhbIzzrWxf/1yfbs7J+vnbvNO4+RJ6pfqB885zlZ2\nftlpW1ufg9EFWpJkdEqeSyRl3wU9M5+T9K8PcC4AgBkQWwSARlDQAaARFHQAaAQFHQAaQUEHgEYc\navvc6KXFnx58y0hnpXSnVavkxSediJMTL3SieqMVI7a4Ut+uJE2MrqqTBaONr7UivdMeuL5dDeb3\nJuYYGyvd98ZFZAzt6wvda+u09xzdWa0faOdaHi/P5zni1Avr+S/tNkMpb9zcdgF36ADQCAo6ADSC\ngg4AjaCgA0AjKOgA0AgKOgA04nBji1mPATld2ebVrU/yVrpPYyX4QV/fbm80vxuv1seOVr2o3mSp\nPj6N1eutboRzugWx4oKSYsc419v1sd1OfQ7OdmXE75y4oCTl8frY3oi+jo1uks51EcZzz70uiC0C\nAA4EBR0AGkFBB4BGUNABoBEUdABoBAUdABpxqLHFyVDaOV3L9Tjd+nJxfp3yBlv1bNFwrf77sduq\nz8GJTvbHjG6ERgxR8o6zFUW02t/Vh2piDDY6IkreYuPDNWOssSi52zm0ylngW/IWGx+dMK5P41pO\no1um1/2yPNTnXPdF3KEDQCMo6ADQCAo6ADSCgg4AjaCgA0AjKOgA0AgKOgA0Ys8cekQ8LunXJV3K\nzHdPH7td0l9IukfS85IeyszX99zW4kSDn18vTWyhq/f77IyxEyebLGlzvd7vc2ex3pt3eLX+u3Qw\nmkOfzX1s11kAPaK+bSdDbN2CGJvtnFa0kjrj/QkLa/XtLqzVJ221YLZaTJvXhZFb742WuLns9Lmt\nD02jJa6TWX8rqDw9/lTS+37msY9K+mZm3ivpm9PPAQBHaM+CnpnfknT5Zx5+UNIT04+fkPT+A54X\nAMC039fQz2TmxenHL0s6c6OBEfFoRFyIiAv9ldrLLQAA38x/FM3M1E1erczM85l5LjPPdSeN9dEA\nAJb9FvRXIuKsJE3/f+ngpgQA2I/9FvQnJT0y/fgRSV87mOkAAParElv8oqQHJN0RES9K+pikT0j6\nUkT8tqQXJD1U+WGrizv6t3f/uDSxSdbjQlt9vQvw69sr5bGS9JoRqVsb1X8/ToxV461Vyo2YVWe2\nX+02jTkbkUHjVFttla2V4M1j4Yy3zp+RDY2J03a4fpAnZlNtpw1zHqsfjMFK/SAbu6eJ8TzNHfOe\n14lEmhHqij1PXWY+fIMv/coBzwUAMAPeKQoAjaCgA0AjKOgA0AgKOgA0goIOAI0wA0qzWe229Uun\nflQae3lcf1fpjzdvL4+9OjKWKJeUTqbO6fjmJJaMsc5K8G5Ur9sx5jHytl3VL9UPRu+sXm/e2lix\nTONZ1huXZ29cGONlY+zx+hx2xxudUU/UL4zl5foFF8YJ2d6ut54cyWhTKSm3uvpgp31pEXfoANAI\nCjoANIKCDgCNoKADQCMo6ADQCAo6ADTiUGOLA6VWBtulsZdVjy1uGivgvrHlxRa3No3s27ax8LPR\nFdHqwGd09nNiiJI03KhHw4abxoaNCGBfX7Nb4xUj4mhsV/Jii073wuzmE8scr9YnvHPay9N1t9ee\n05J0+lR91bJTx7bKY8eT+nPvSlc/2Vd67553bHVnPPhui9yhA0AjKOgA0AgKOgA0goIOAI2goANA\nIyjoANAICjoANOJQc+jbOdSPtu8sjb24daq83X9ar499Y225PFaS+qv1jHu3buTQrVa0c8qsm+1z\nnZa43XY99zwwsvPO2LRuV7xMcBpdUp1NO5n13riURyfr5yNu896g4GTL/+XxK+WxK8P6PDbG9VD+\nTl8/eetD44KTNB4YJ/DgY+jcoQNAKyjoANAICjoANIKCDgCNoKADQCMo6ADQiD0zNhHxuKRfl3Qp\nM989fezjkn5H0qvTYY9l5tf32tb6eFEXLv9caWJrO/UWl6+v1/NbO+vOUvDSYKv+O6/brueQnLFO\nvNCKLRotYCVZbW7nxpjDwDhuaUY4cw4rtkvSxDnGc5rDPOJ0b9qZ1CODk3G9BqyP6s/rje362PHI\nTHaPjXvk/mja5/6ppPdd5/E/ysz7pv/tWcwBAPO1Z0HPzG9JunwIcwEAzGCW19A/FBHfj4jHI+L0\ngc0IALAv+y3on5H0Lkn3Sboo6ZM3GhgRj0bEhYi4MHrDWZcMAODYV0HPzFcys8/MiaTPSrr/JmPP\nZ+a5zDy3cMrrowIAqNtXQY+Is9d8+gFJTx/MdAAA+1WJLX5R0gOS7oiIFyV9TNIDEXGfdkNkz0v6\nYOWH7YyHeu7S20oTmxireE9Gxu8la1VuzS0alkZiKYyxTodBp7OfJE3qjSfVL9UnPTEa2jlzcMZ6\nnRndrpZG50mjo2U4F9GgvoPbUY8LStKrxvPv9aXV8tgwcrW9ERecbNUv/Nj2Loxuq35OBuODjy3u\nuWeZ+fB1Hv7cgc8EADAT3ikKAI2goANAIyjoANAICjoANIKCDgCNoKADQCPMJPJschwa/fTYYf7I\n/5+bK3dy3U7u2cjYOjntgdGS081eO31VnW1HGsdioT6H3olTm5HgwXZ97IIxtjPGDjfqx224Ud/u\n4k+9C6Nfqh/orHfPtd6rMZxXy193u0b7Y+e9DFXcoQNAIyjoANAICjoANIKCDgCNoKADQCMo6ADQ\niEONLSpDg63a7xAnsmQtX2/GkJx5TBaMeczpyIexbPxgxzsYafTx9SKRRqvd+oLtVmzRuYQkaeiM\nXzO2u1nPsg0365Nwrgs7TmfETq3NGlnE8XL9ghut1MeOzZS1E6v1Y8N74w4dABpBQQeARlDQAaAR\nFHQAaAQFHQAaQUEHgEYccmxRilEt1jOwklDzarVmxieNSTsRx7QycnXZeb/PvThb/cA5kcH+mLES\nvBNbNKN6zuruzrYXNuqDF94YlccO13bKY2PDaPkoKTaN8RPjYCzVM6r96dXy2J23LdfHnjTaQ0oa\nHzMiuN3B1y3u0AGgERR0AGgEBR0AGkFBB4BGUNABoBEUdABoxJ6xxYi4W9LnJZ3R7hKo5zPz0xFx\nu6S/kHSPpOclPZSZr990W5P6IrhO1CuMhZHdeJrVbdHoBOgsJjtxFtbt5heHnBjd7wbOgtnOsXC2\naxyLsPKp3nU0GBtjt+sb7tbrscXBlfoq0blurCgtabKxWR/c11c8j5V6vHBgRHCHi/W0tnlZqNuu\nz8NZMLuq8tPHkj6Smb8o6d9J+t2I+EVJH5X0zcy8V9I3p58DAI7IngU9My9m5lPTj69KekbSXZIe\nlPTEdNgTkt4/r0kCAPbmLUMQcY+k90j6tqQzmXlx+qWXtfuSDADgiJQLekQcl/RlSR/OzCvXfi0z\nUzd4VTgiHo2ICxFxoV9fn2myAIAbKxX0iFjQbjH/QmZ+ZfrwKxFxdvr1s5IuXe97M/N8Zp7LzHPd\nar3fAgDAs2dBj4iQ9DlJz2Tmp6750pOSHpl+/Iikrx389AAAVZX8zi9L+k1JP4iI700fe0zSJyR9\nKSJ+W9ILkh6azxQBABV7FvTM/CvduBfqrzg/LHpp8Y1asDPqcVVvrLuiuZFD7ZeMVcrH9bEjI6g9\nWTJ20A3ZGn9Cn8yr5e+8OiW714XzPomJk4c351GUw3roOY4ZfYclxXBOXbiN9rmTFadXcn3ocMso\nLpLSeB9BOG/AKOKdogDQCAo6ADSCgg4AjaCgA0AjKOgA0AgKOgA0Yk55o+sbjKTjL9ViPVY8bV5R\nNnkrczvtcwcjo+WvMbZfqf+OdtrySuZhNgbPK4rotFV2WtxK0sBJsznR18X6+Rscr19wuWSc7MlK\nfaykNFrXprF/vTHnycJ8LiKnnbEkdUbMcTB2s7KFbR74FgEAR4KCDgCNoKADQCMo6ADQCAo6ADSC\ngg4AjTjU2GK3PdHJZ2urFk2W6lMbr9TjTf2y9zvMiZEZC5pb3foGRmfGsRNxNGKWkjRZqI/NgdFh\ncF6xReMYd1veJAYjczJFzvWZQ+OEqD52MvSORb9odA5dMa7l5frYNCrZYFS/NhfW6tuVpMW1+kXX\nmZHICu7QAaARFHQAaAQFHQAaQUEHgEZQ0AGgERR0AGjEocYWY3tH3XP/VBo7PHmivN3BHfWx292x\n8ljJjPYZa752RuzNWUzWWjDbiIVJUm+sw2t1v3NuK4xjHEYHxeGmMQdJ3bYxEWNobxy38TGjE6gR\nRXTjrL0xj536U1WjE87i6MZBntTn614Xw416hHowMjpg/vfiNutbBAC8lVHQAaARFHQAaAQFHQAa\nQUEHgEZQ0AGgERR0AGjEnjn0iLhb0uclndFuovZ8Zn46Ij4u6XckvTod+lhmfv1m28pxr/7yT0sT\n67p6RjNuP14em52XvR4vOVnf+nbDyVM7rXaNfPvAfBdCGrFZZ6V7a7tOS9yd+tjBtjEHSZ0xfmDk\n4Z1j4VybvfH2C6cdriRNjPcnjFfqF/74hHGyT9Yv/OFi/c0aboPbrYnR/tioAVWVp/RY0kcy86mI\nOCHpuxHxjenX/igz//DgpwUAcO1Z0DPzoqSL04+vRsQzku6a98QAAB7rNfSIuEfSeyR9e/rQhyLi\n+xHxeEScvsH3PBoRFyLiwkjmv2sBAGXlgh4RxyV9WdKHM/OKpM9Iepek+7R7B//J631fZp7PzHOZ\neW5BxottAABLqaBHxIJ2i/kXMvMrkpSZr2Rmn5kTSZ+VdP/8pgkA2MueBT0iQtLnJD2TmZ+65vGz\n1wz7gKSnD356AICqSsrllyX9pqQfRMT3po89JunhiLhPu1HG5yV9cK8NxWCgwbHayy6xXM9Z9av1\nFc1Hq170frxcH5vGpp0om9MS12ovO4fY1P+dhnEsnKieFagzBjstiiUpJvM5eM5xmxhtbp0Wt277\nXCeua51A51ru6oNPHK/3xD1zfK0+CUlnlq/W5zHcKo/94+K4Ssrlr3T903DTzDkA4HDxTlEAaAQF\nHQAaQUEHgEZQ0AGgERR0AGiE2W9v1p/WaXDn20pD+7fVlwffOWnEFlfMTnLO6vVOks2IyQ2M2KLV\nHc5M3jnxtP6YsWJ7/fSZUcv6ueu2zOvCiVr2xrEYGvFCo9uiE7/t5/mGbuP8dVv1+83xZv3inJyo\nb/fs8pXyWEn696eeLY+9d+nl8thqbJE7dABoBAUdABpBQQeARlDQAaARFHQAaAQFHQAacaixxRx2\n6m8/WRo7OlXvtrhzvP57ye0k53SH63bqmazhthFbNBZ+DitaWB8reQtsO7HFfqUetoyJEy+sXxfd\nthlbrDfsU6Sx0LgR4XTiheNV43wY506Soq/v32BOC3cP1uo50s1T9YM8MFuS3r3wk/LYf7WwYW27\ngjt0AGgEBR0AGkFBB4BGUNABoBEUdABoBAUdABpBQQeARhxu+9xBaLJSy4D2y/XfNU6L2xx4eWOn\n9amTF++MHHo3MjLERk7by3SbK9IvGfnd5Xp/4DTm3I+MsU6bZJntc42xznatdsbG+XDeFyBJYRzn\n6Ofz3gAnsz4e1Q/c2LnoJa0aEzndrVjbruAOHQAaQUEHgEZQ0AGgERR0AGgEBR0AGkFBB4BGRBqr\nz8/8wyJelfTCdb50h6TXDm0ih6vlfZPYv1sd+3dr+PnMvHOvQYda0G84iYgLmXnuqOcxDy3vm8T+\n3erYv7bwkgsANIKCDgCNeKsU9PNHPYE5annfJPbvVsf+NeQt8Ro6AGB2b5U7dADAjI60oEfE+yLi\nbyPi2Yj46FHOZR4i4vmI+EFEfC8iLhz1fGYVEY9HxKWIePqax26PiG9ExN9P/3/6KOc4ixvs38cj\n4qXpOfxeRPzno5zjfkXE3RHxvyLihxHx1xHxe9PHmzh/N9m/Js5f1ZG95BIRnaS/k/Srkl6U9B1J\nD2fmD49kQnMQEc9LOpeZLeRgFRH/QdKapM9n5runj/2BpMuZ+YnpL+XTmfn7RznP/brB/n1c0lpm\n/uFRzm1WEXFW0tnMfCoiTkj6rqT3S/otNXD+brJ/D6mB81d1lHfo90t6NjOfy8wdSX8u6cEjnA/2\nkJnfknT5Zx5+UNIT04+f0O6T6JZ0g/1rQmZezMynph9flfSMpLvUyPm7yf79s3KUBf0uSf94zecv\nqr0TkJL+MiK+GxGPHvVk5uRMZl6cfvyypDNHOZk5+VBEfH/6kswt+ZLEtSLiHknvkfRtNXj+fmb/\npMbO383wR9H5em9m/htJ/0nS707/Sd+s3H39rrXY1GckvUvSfZIuSvrk0U5nNhFxXNKXJX04M69c\n+7UWzt919q+p87eXoyzoL0m6+5rP3zF9rBmZ+dL0/5ckfVW7LzO15pXp65dvvo556Yjnc6Ay85XM\n7DNzIumzuoXPYUQsaLfYfSEzvzJ9uJnzd739a+n8VRxlQf+OpHsj4p0RsSjpNyQ9eYTzOVARsTr9\n44wiYlXSr0l6+ubfdUt6UtIj048fkfS1I5zLgXuz2E19QLfoOYyIkPQ5Sc9k5qeu+VIT5+9G+9fK\n+as60jcWTSNE/1VSJ+nxzPwvRzaZAxYRv6Ddu3JpdzHuP7vV9y8ivijpAe12sHtF0sck/TdJX5L0\nc9rtpPlQZt6Sf1i8wf49oN1/rqek5yV98JrXnG8ZEfFeSf9b0g8kvbkK9GPafZ35lj9/N9m/h9XA\n+avinaIA0Aj+KAoAjaCgA0AjKOgA0AgKOgA0goIOAI2goANAIyjoANAICjoANOL/AO/lTlq36PjM\nAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J5n048xZEW-d",
        "colab_type": "text"
      },
      "source": [
        " #### Y vector (New response)\n",
        " \n",
        "Generate response for the whole field."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7jadlKZcEUxx",
        "colab_type": "code",
        "outputId": "bf522988-334f-4034-b111-a1169603fa82",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "\n",
        "# Reshape X\n",
        "X = X.reshape(1, X.shape[0], X.shape[1])\n",
        "\n",
        "# Reshape beta\n",
        "beta = beta.reshape(beta.shape[0]*beta.shape[1]*beta.shape[2],beta.shape[3],1)\n",
        "beta_True = beta\n",
        "\n",
        "# Reshape Z (note: This step is slow because of the sparse to dense conversion;\n",
        "# it could probably be made quicker but this is only for one simulation at current)\n",
        "Ztmp = Z.toarray().reshape(1, Z.shape[0], Z.shape[1])\n",
        "\n",
        "# Reshape b\n",
        "b = b.reshape(b.shape[0]*b.shape[1]*b.shape[2],b.shape[3],1)\n",
        "\n",
        "print(X.shape)\n",
        "print(Ztmp.shape)\n",
        "print(beta.shape)\n",
        "print(b.shape)\n",
        "\n",
        "# Generate Y\n",
        "Y = np.matmul(X,beta)+np.matmul(Ztmp,b) + np.random.randn(n,1)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 1000, 16)\n",
            "(1, 1000, 70)\n",
            "(27000, 16, 1)\n",
            "(27000, 70, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7V1oJXx2EcEX",
        "colab_type": "text"
      },
      "source": [
        "Check Y looks reasonable."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nk8lqtsbEZ1J",
        "colab_type": "code",
        "outputId": "7a02f31d-550a-4564-fd66-6067c72daefe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        }
      },
      "source": [
        "print(Y.shape)\n",
        "\n",
        "Y_imageformat = Y.reshape((dimv[0],dimv[1],dimv[2],n))\n",
        "\n",
        "imshow(Y_imageformat[3,:,:,1].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(27000, 1000, 1)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f9e15d66438>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGpBJREFUeJzt3VuMXXd1x/Hf2ucyd99ycRyTNiTk\noYi2obKiqqAqFQKlVaXAS0QeUCohzAOpispDUR5KpKpSVHEpDxWSaSKCxFUCSh6ilgi1SnkoipNG\nSSClpJFTbBw7tmN7PDPntvfqw5xQE9metWbmzOA/34+EMh4v7/nvy6zZnPmdtc3dBQC4+lXbvQAA\nwOagoQNAIWjoAFAIGjoAFIKGDgCFoKEDQCFo6ABQCBo6ABSChg4AhWhv5B+b2V2SPi+pJekf3f2h\nK9W3Fua8fc3u2MZrC6+jGoZLU7WSVI3i76S1zJtum8R2E7Vqmnht9l3CmfIqfq/g3Va4djQVvy7q\nqXCp1Ekei8zJruPHwkaJNaTOR2KzyWPR6cQXPduOfwN2EwejUfy66DWdcO3yMF4rST6IX8uZc90/\nfvSUu1+3Vt26G7qZtST9g6T3Sjoq6Skze8zdf3zZL3bNbt3w138e2n7r9fiBnD0eP5lzxxMNT9LU\n2Tpc216J11b9eG1reRCuteV+vLYX366k1A8An50O1w5u3BmuPXtbvEufe1u4VPXe3LGwVvw68nPd\ncG33TLzzVv34dT+aj5+74d7cXc/+G8+Ea2+/5li49qbp+HYzTfrFCzeEa587fmO4VpL6R+fDtVOn\n4uf6J3/zl69E6jbykssdkl5y95fdfSDp65Lu3sD2AAAbsJGGvl/Szy7689Hx5wAA22DivxQ1s4Nm\ndtjMDtcXlib95QDg19ZGGvoxSTdd9Oe3jD/3S9z9kLsfcPcDrfm5DXw5AMCVbKShPyXpNjN7q5l1\nJX1Q0mObsywAQNa6Uy7uPjKz+yX9i1Zji4+4+4+u+I8ak63EYj2dpfhv8Lvn47/BnzofT5dIUmc5\nkyOLa6bi8SZV8WRH1U6kJDrJ0z+KHzufjqcOmm58zXUnfl00nXgSpermrosqEVsctuO1ibCGLJNQ\nTVxuVuVii7XHz8nZ4Uy4trJd4drFUTxVdXQxvt2VxUz2VepeiB+L9nJq07FtbuQfu/vjkh7fpLUA\nADaAd4oCQCFo6ABQCBo6ABSChg4AhaChA0AhaOgAUIgNxRazrJa6Z2KB2OnX4tud/3k8Kz5z7EJ8\nw5JUxzO59Y54ZnW4Mz6Bz+fjp6kaxmvby/E1rG47ntVuEnn44VxifO5suFTNbDyoPTedmzBomRx6\nL35OMpn8jCaRhc9a7sevo2NL8cmaJ1YWwrVnV+L59tNn4hMR2ydz3yPTpxPvnzm7+eeEO3QAKAQN\nHQAKQUMHgELQ0AGgEDR0ACgEDR0ACrGlscVqKM2ciNXOvxqPyM2+cj6+iFcTeUhJ1k3EC2fWfCj3\nLzSJp9cP5hJPjW/ite1EXFCSWv14DLBpx/evvzMRcYwn2VTNx6OIu2ZX4huWlAmc9Xvxmbij6cS3\nZBM/xhney10XFxbjo2t7/fixSDyTXMNEBLd1Or6GTAxRkqZez4zyzj2wPoI7dAAoBA0dAApBQweA\nQtDQAaAQNHQAKAQNHQAKseWxxWgccebVXni7dvpsuLY+l4g4Sqp2xafDeSsecaoTU/VGs/HtJh7A\nrtFMLpJVjeJrbhLJt/6e+DqGOxMTFOfj19Cemdwj2EeJeOj56Xisb9hJTPez+HFr9RO1g1xssbmQ\niJ1OJbKIidvNKrF/naXEsYhfQpJWJ8puJ+7QAaAQNHQAKAQNHQAKQUMHgELQ0AGgEDR0ACjE1sYW\nR42mT/ZDte1z8byQJ8ayVfNz4VpJ0q4d4dLhfHyK2zARGRxNJ2KLicSZbf6wt1+oE+m7we74+Wt2\nxB8IvjAdu9YkabY9CNdK0qCOH+huO75mteLHwkaJqN75eG07N3gyFdcdxZ/lrNF8/Fh44ta06SQe\n8D2Xi/Z6Iko6TExRjdpQQzezI5IWJdWSRu5+YDMWBQDI24w79D9y91ObsB0AwAbwGjoAFGKjDd0l\nfc/Mnjazg5cqMLODZnbYzA4Phksb/HIAgMvZ6Esu73b3Y2Z2vaQnzOy/3P3Jiwvc/ZCkQ5K0Y2F/\n5sldAICEDd2hu/ux8X9PSvqOpDs2Y1EAgLx1N3QzmzOzhTc+lvQ+SS9s1sIAADkbecllr6Tv2Gru\nsi3pq+7+z1f6BzZs1Dm5GNq4jRJzKGfi40k9kSuXpOF18+Ha/p7E091n42to4ptN5XGVfGh8Zh2j\nuUTWd1c8EN+ZjefFO1V8u71RYuck9er4t84okVlXM5mRuFNnE0+jT9RKufc+9PbEL9A6lVmPn+tR\nO75/g93xNUi59wZY4lxHrbuhu/vLkn53E9cCANgAYosAUAgaOgAUgoYOAIWgoQNAIWjoAFCILR2f\nq9FIeu1MrHZ6KrxZ37UQrh1ekxuf27s+vo7+QjyG1HQy8aZwaao2E0PM1o9mEqNPZ+PjZdvt+A4u\nDxPjjJvcvc1gFP/W6fUTB26YiS3GN9tJTN3oLubmKtdTifhdIhFZT8eLW3viB2P3zvjBmO0Mw7WS\n1GnF49atxDfrkWAdd+gAUAgaOgAUgoYOAIWgoQNAIWjoAFAIGjoAFGJLY4te16pffz1U27pmT3y7\n3fhuDHfksnqD+fjPvCbxpHtLDJOsBvH4VpXYbt3NTXurE/uXuVWwKr5/TWJC3eJyfApnZruSVNeJ\nqYH9+DhCGyYO3IQeF+PJrpCJLQ7jw0tV747HWd+293S49nd2HQvX3jLzWrhWkm5onwvXLlQr4don\ngnXcoQNAIWjoAFAIGjoAFIKGDgCFoKEDQCFo6ABQiK2dtphg3XhGbjQfn4g42JF4oq2kYeJhzt6K\nx7eqYTxz1o6nm1LbreKHTZI0mklMiExMDWxG8fuKYT9+yWa2q0xcUEo9zDkTL8zEWTMPZ05dx5b8\nHklMGR3sjh+MhT3xqYi/tfPVcO0fLLwUrv3tqePhWkm6sRU/dvNVPFYbxR06ABSChg4AhaChA0Ah\naOgAUAgaOgAUgoYOAIWgoQNAIdYM9ZrZI5L+VNJJd3/H+HN7JH1D0s1afSD1Pe6+5lxca7fU2hUb\ni9tcsytUJ0m96+KZ9d7u3JjUTMZWiYelt1PjWuPZ3cSDxFOZZ0mqEg9Abw3itU0vnt1tEnnxTBa+\nGuSuiwxPbNrqeHFmzO1oLr7dOvn+hOGOxDp2xC+6XTO9cO3uznK4NjO2tjOpGcWS+p74hgqKfHd8\nSdJdb/rcJyV9391vk/T98Z8BANtozYbu7k9KOvOmT98t6dHxx49Kev8mrwsAkLTet/7vdfc33hP7\nqqS9lys0s4OSDkrSdJV4XAkAIGXDvxR1d9cVXuR190PufsDdD3QnMLsAALBqvQ39hJntk6Txf09u\n3pIAAOux3ob+mKT7xh/fJ+m7m7McAMB6RWKLX5N0p6RrzeyopE9JekjSN83sw5JekXRP6Kt1OvJ9\n14dK+zfMheokafnaeOytd00unjaam0xksOkmxst24rWZUbuT1Ool4neL8fuKTAQwE7OsRrnrwqv4\nddF0UptOLCJe2iR+W2bZScKdzHzgeG1vFF/0q/14dvKnnRvCtWebxNzhpF7qwnglVLXmEXP3ey/z\nV+9JrAYAMGG8UxQACkFDB4BC0NABoBA0dAAoBA0dAAqx3rf+r4u3Kw2vjcWA+nvikZ7BzsRTx3fm\npqfVC4mRhIlI1nBH/GdpayVe215OTBjsh0tX6xOHIhMZ7JxPRAYTpy+z3uxQvToRO/XEdZG6xcqk\nBSd07iSpvZSY5Ph6vOWcml4I1/6n3hKuPbESjzi2UxeRdG4wE669MIhPiZX+I1TFHToAFIKGDgCF\noKEDQCFo6ABQCBo6ABSChg4Ahdja2KJJ9XRsMuJoKhGFSqR/mqnESERJ1Vw8w9WZGoVrLZHUGw0T\n0ySX46e0dS53+jvnMvHJ+HbbiQdKZx9sHdVkEmSS6sSzWjLb9lZiumcmiph4+HR3MZfhtHPx2k4i\n4rjSjx/kUxfiMedTC/E4ZDbO6iuJ77+lzb+f5g4dAApBQweAQtDQAaAQNHQAKAQNHQAKQUMHgELQ\n0AGgEFuaQ5dyT22PykwnTS8gUT4zFc+sz03Fw9dVYgfPzsazu4s2F66VpGoQz/qmxvgO4vtXxaP+\nqSfdZ8bhStJwIb7m0e7EohM5dK/iO9g9F9+/zlIufN09Hw/E+6nEOhbj779YuRCvHSRGV2dveVuJ\n6769ktt2BHfoAFAIGjoAFIKGDgCFoKEDQCFo6ABQCBo6ABRizdyTmT0i6U8lnXT3d4w/96Ckj0h6\nbVz2gLs/vuZXc6kaxsbXtvvx6FRrJREVSsSKJGk4E4+G9abisb5MbHG2E69tErHMlZmpcK0kNd34\nsfDWBPKpSU0nvobRbG7bmSjiwnUXwrWtTETV58O19WvxGb5VnYwtnktEcIPf/5LU6iXWPEpEalfi\n97FNfLOr60iMgm4lelz46wdqviTprkt8/nPufvv4f2s3cwDARK3Z0N39SUlntmAtAIAN2Mhr6Peb\n2XNm9oiZ7d60FQEA1mW9Df0Lkm6VdLuk45I+c7lCMztoZofN7PBwuLTOLwcAWMu6Grq7n3D32t0b\nSV+UdMcVag+5+wF3P9Dp5GaHAADi1tXQzWzfRX/8gKQXNmc5AID1isQWvybpTknXmtlRSZ+SdKeZ\n3a7VZ2IfkfTRyBezxtU5H8v1ZGJv04m4YDZOZ4mRfb1mJlx7JjMiMp5OS01m7HQTUwAl9abi287E\nvbwdPydNYv+aeOpNdWLfJKmaiR+7HdP9+HYT+7c4Hb/eMuejSX+PxNds/fhE0vZS/H5z6nx82mKm\nB9SJa0iSLD54Uq3h5scW1+xW7n7vJT798KavBACwIbxTFAAKQUMHgELQ0AGgEDR0ACgEDR0ACkFD\nB4BCJJ6LvnE2qtU+tRiqrfrxnG81jM8+bfdz8zDbidG81SB+OFcUX3NmMtr8TCLzXOVysN6Jjz6t\np+L3CnU3cYyH8drEJGFZfNckSU0vnns+tzIdrs3k0Ot+fA3dxKn2+GZX1zGdaCOJzLq3J3O/WY0y\nB2Nymfwq9zaQ2DY3f5MAgO1AQweAQtDQAaAQNHQAKAQNHQAKQUMHgEJsaWxRdS0/dz5UWg3iYzan\nRokniffjI0dX6+PzM63OjPCM166047E3T8SsmiYXyUpJ3Cr4hG4rqvglpM5i8lhU8fjrUm8hXOuJ\nKGl7MX4NtZfDpWmj2fg6mm4izpqIvg7mJxOTzcrGXzcbd+gAUAgaOgAUgoYOAIWgoQNAIWjoAFAI\nGjoAFGJrY4syqT2BL5mIOLYu5H6GTSWeSJ+JWY3mEjGr6XhErpeJIianLVbL8XhaqxdfR2sQX0Or\nn3jCfCJC1l6J10pSNxFzbDqJOGvi8qwSx61zYXJTAOvp+KKHiUmOo5nE91Mijdx0JhhbTBy7qt78\nr88dOgAUgoYOAIWgoQNAIWjoAFAIGjoAFIKGDgCFWDNDaGY3SfqypL2SXNIhd/+8me2R9A1JN0s6\nIuked3/9ihvrtOX7rg0trGll4nfxn0tNJ/czrEk8qNbqeDQsEyPrvp55iPLkkqjtpfg6OrFnga/W\nLsWPRWuQiC0mYmHZCJk1iUxkLh0aX0NmuxNagyQ1iUsuE0VMPPtdo7lMvDi+3cy+SZJsghNMAyJH\ndyTpE+7+dkm/L+ljZvZ2SZ+U9H13v03S98d/BgBskzUbursfd/dnxh8vSnpR0n5Jd0t6dFz2qKT3\nT2qRAIC1pV5/MLObJb1T0g8l7XX34+O/elWrL8kAALZJuKGb2bykb0n6uLv/0mOH3N11mVfpzOyg\nmR02s8OD0QQfmwIAv+ZCDd3MOlpt5l9x92+PP33CzPaN/36fpJOX+rfufsjdD7j7gW478VsOAEDK\nmg3dzEzSw5JedPfPXvRXj0m6b/zxfZK+u/nLAwBERUI575L0IUnPm9mz4889IOkhSd80sw9LekXS\nPZNZIgAgYs2G7u4/kHS5cOV7Ml+snm5p8W2xJ6AnHl4/UV4lxqQmMquZjHT3XLy2vRxfbyrHLKnV\nSzyRPjGOtjWczEhcaxIjYxNrkKRWL76QzLbN47W5azNR2829V6NO1DeJ8blNdzLZ8uFC/BiP5hIX\nnCSfTtR3ctuO4J2iAFAIGjoAFIKGDgCFoKEDQCFo6ABQCBo6ABRicrNWL6Geks7dksgtTUIyKZR5\nAnrmKezVKDEyNhF7y6w3EwGUclHLywZdL6FOPIU9E+urRontJsfnZiKDmXyoJw6cJ0ZMZ2qbxPlY\n3XaiNtFxmk68djQTP8bDnfGT3dndjy9C0q4d8fEmC1Pxbf9vsI47dAAoBA0dAApBQweAQtDQAaAQ\nNHQAKAQNHQAKsaWxxaYjLe+LZeU886MmEQuzRJRNklr9zJPu47WtXry2GoZLU3HB7BPNM/G0zDoy\nUdLM+Wgvx6+LbnwJkiRLjAP1VuLAZc5fIl6YmbZYJ+KCklRPZaYixmtH0/E1jBITFNu74vni/dee\njS9C0i0Lp8O1e7pL4dp/C9Zxhw4AhaChA0AhaOgAUAgaOgAUgoYOAIWgoQNAIbY0tqjK1cwHJ521\nElHEKl7b1MlJciuZyFniYbmJGFlGM5V5AG7uwchN5gG4iXNi/fhx674er50+lYiRJiZlSrlJjp6I\nAaYe5pypTa0hXislo4hz8e1moojNQnzM6J7ERMT9c4kntEvaNx2v39OOxxajuEMHgELQ0AGgEDR0\nACgEDR0ACkFDB4BC0NABoBA0dAAoxJqJUzO7SdKXJe2V5JIOufvnzexBSR+R9Nq49AF3f3zNrxiN\nMidi2pknpSsXvU7JjPytZ+MLqbuJPO6ueB53x55cDvb6hQvh2ipxoH9+fke4dskXwrXdc/H3EFiT\nvDAy5dWExtxOxZdQdxNrSM4SrjNjbmcS2028p8I68fdIWGLcdq/OhfLPDONB+5XsnOKAyGpHkj7h\n7s+Y2YKkp83sifHffc7dP73pqwIApK3Z0N39uKTj448XzexFSfsnvTAAQE7qNXQzu1nSOyX9cPyp\n+83sOTN7xMx2X+bfHDSzw2Z2uF7c/Le6AgBWhRu6mc1L+pakj7v7eUlfkHSrpNu1egf/mUv9O3c/\n5O4H3P1AayExyAEAkBJq6GbW0Woz/4q7f1uS3P2Eu9fu3kj6oqQ7JrdMAMBa1mzoZmaSHpb0ort/\n9qLP77uo7AOSXtj85QEAoiIpl3dJ+pCk583s2fHnHpB0r5ndrtUA1xFJH11zS43JeplHx8ckUkiy\neKpPklQN43EvC04GliRPpKGamXgka3bnSrj2tmteW7voIrfOnwrXjpr4r2dGibzny935cG0mWlhl\nr4tRIkqa2W7iGmqaCY1gTo7PzYzm9XYiipg4cM1yfNGnLX4NLS4nMpmSftqJX0itKnNlxERSLj/Q\npVPha2fOAQBbhneKAkAhaOgAUAgaOgAUgoYOAIWgoQNAIZIBpQ1yqdWPRa0y8cLoNiWpSj/dPV6b\nmbY4mo3XWp2ITiaSbO1kbGq6GoZrlz0+sq9ORBxTx2LzU2HrUtWJ/GTm+kxsNjUJdDoZh8zEQxMx\n4Ext53xiBy3e9ppWbgpnL7EMn0DqlDt0ACgEDR0ACkFDB4BC0NABoBA0dAAoBA0dAAqx5bHFaMSp\nNUhEluLPLlZ7KRdDysQWMw/izRjNxn/u9lbiccHTvdwDR462LvlQqktaquPrOLMcf3KwJSKqGZmJ\ngZLUZOKTiQmKrX78+qyGmTGj8Wuonsod48zkUI8nX9WODw5NfV+3e4ljnDh3UjKKmKj9n2Add+gA\nUAgaOgAUgoYOAIWgoQNAIWjoAFAIGjoAFIKGDgCF2NocuiXGeCZGn2ZG4mayrVJy9GmitElkfVuJ\n/Rv0WuHasyvx/LckTbXiofyVUTzYncnOV6NEeDdRmn0PQWY0byZbbonrLTHNWE07vt3WIPdejSZz\n7BLHrbMYX8fMmXhgvHs+fh1X/WQQPcGrzX9PBXfoAFAIGjoAFIKGDgCFoKEDQCFo6ABQCBo6ABTC\n3HMRpQ19MbPXJL1yib+6VtKpLVvI1ip53yT272rH/l0dftPdr1uraEsb+mUXYXbY3Q9s9zomoeR9\nk9i/qx37VxZecgGAQtDQAaAQvyoN/dB2L2CCSt43if272rF/BfmVeA0dALBxvyp36ACADdrWhm5m\nd5nZT8zsJTP75HauZRLM7IiZPW9mz5rZ4e1ez0aZ2SNmdtLMXrjoc3vM7Akz++n4v7u3c40bcZn9\ne9DMjo3P4bNm9ifbucb1MrObzOxfzezHZvYjM/uL8eeLOH9X2L8izl/Utr3kYmYtSf8t6b2Sjkp6\nStK97v7jbVnQBJjZEUkH3L2EHKzM7A8lXZD0ZXd/x/hzfyfpjLs/NP6hvNvd/2o717lel9m/ByVd\ncPdPb+faNsrM9kna5+7PmNmCpKclvV/Sn6mA83eF/btHBZy/qO28Q79D0kvu/rK7DyR9XdLd27ge\nrMHdn5R05k2fvlvSo+OPH9XqN9FV6TL7VwR3P+7uz4w/XpT0oqT9KuT8XWH/fq1sZ0PfL+lnF/35\nqMo7AS7pe2b2tJkd3O7FTMhedz8+/vhVSXu3czETcr+ZPTd+SeaqfEniYmZ2s6R3SvqhCjx/b9o/\nqbDzdyX8UnSy3u3uvyfpjyV9bPx/6Yvlq6/flRab+oKkWyXdLum4pM9s73I2xszmJX1L0sfd/fzF\nf1fC+bvE/hV1/taynQ39mKSbLvrzW8afK4a7Hxv/96Sk72j1ZabSnBi/fvnG65gnt3k9m8rdT7h7\n7e6NpC/qKj6HZtbRarP7irt/e/zpYs7fpfavpPMXsZ0N/SlJt5nZW82sK+mDkh7bxvVsKjObG/9y\nRmY2J+l9kl648r+6Kj0m6b7xx/dJ+u42rmXTvdHsxj6gq/QcmplJeljSi+7+2Yv+qojzd7n9K+X8\nRW3rG4vGEaK/l9SS9Ii7/+22LWaTmdktWr0rl1Yfxv3Vq33/zOxrku7U6gS7E5I+JemfJH1T0m9o\ndZLmPe5+Vf5i8TL7d6dW/++6Szoi6aMXveZ81TCzd0v6d0nP6/8f1fyAVl9nvurP3xX2714VcP6i\neKcoABSCX4oCQCFo6ABQCBo6ABSChg4AhaChA0AhaOgAUAgaOgAUgoYOAIX4P14UlY/EJqpTAAAA\nAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0N-oRTM1EhCb",
        "colab_type": "text"
      },
      "source": [
        "#### Transpose products\n",
        "\n",
        "Calculate X'Y, X'Z, X'Y, Y'Y, Y'Z, Y'X Z'Z, Z'X and Z'Y."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "px75KQpHEe7u",
        "colab_type": "code",
        "outputId": "f7a72d32-022a-4b5a-dcb1-467d5549d60d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        }
      },
      "source": [
        "# X'Z\\Z'X\n",
        "XtZ = np.matmul(X.transpose(0,2,1),Ztmp)\n",
        "ZtX = XtZ.transpose(0,2,1)\n",
        "\n",
        "# Z'Y\\Y'Z\n",
        "YtZ = np.matmul(Y.transpose(0,2,1),Ztmp)\n",
        "ZtY = YtZ.transpose(0,2,1)\n",
        "\n",
        "# Y'X/X'Y\n",
        "YtX = np.matmul(Y.transpose(0,2,1),X)\n",
        "XtY = YtX.transpose(0,2,1)\n",
        "\n",
        "# YtY\n",
        "YtY = np.matmul(Y.transpose(0,2,1),Y)\n",
        "\n",
        "# ZtZ\n",
        "ZtZ = np.matmul(Ztmp.transpose(0,2,1),Ztmp)\n",
        "\n",
        "# X'X\n",
        "XtX = np.matmul(X.transpose(0,2,1),X)\n",
        "\n",
        "\n",
        "print(XtZ.shape)\n",
        "print(ZtX.shape)\n",
        "\n",
        "print(XtY.shape)\n",
        "print(YtX.shape)\n",
        "\n",
        "print(YtZ.shape)\n",
        "print(ZtY.shape)\n",
        "\n",
        "print(XtX.shape)\n",
        "\n",
        "print(YtY.shape)\n",
        "\n",
        "print(ZtZ.shape)"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 16, 70)\n",
            "(1, 70, 16)\n",
            "(27000, 16, 1)\n",
            "(27000, 1, 16)\n",
            "(27000, 1, 70)\n",
            "(27000, 70, 1)\n",
            "(1, 16, 16)\n",
            "(27000, 1, 1)\n",
            "(1, 70, 70)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CUHZxjFvEmih",
        "colab_type": "text"
      },
      "source": [
        "### Demonstration: Time taken just looping\n",
        "\n",
        "This is a demonstration showing how long PLS takes when doing each voxel seperately."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TokOi5r2Ej2T",
        "colab_type": "code",
        "outputId": "28503fa6-eca0-4b4c-ad9c-32fd59d46526",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# Initialize empty estimates\n",
        "beta_est = np.zeros(beta.shape)\n",
        "print(beta.shape)\n",
        "\n",
        "demo = False\n",
        "if demo:\n",
        "  \n",
        "  # Initialize temporary X'X, X'Z, Z'X and Z'Z\n",
        "  XtZtmp = XtZ[0,:,:]\n",
        "  ZtXtmp = ZtX[0,:,:]\n",
        "  ZtZtmp = ZtZ[0,:,:]\n",
        "  XtXtmp = XtX[0,:,:]\n",
        "\n",
        "  print(type(ZtZ))\n",
        "\n",
        "  t1 = time.time()\n",
        "  for i in np.arange(beta.shape[0]):\n",
        "\n",
        "    print(i)\n",
        "\n",
        "    XtYtmp = XtY[i,:,:]\n",
        "    ZtYtmp = ZtY[i,:,:] \n",
        "    YtYtmp = YtY[i,:,:] \n",
        "    YtZtmp = YtZ[i,:,:]\n",
        "    YtXtmp = YtX[i,:,:]\n",
        "\n",
        "    param_est,bvals = FS(XtXtmp, XtYtmp, ZtXtmp, ZtYtmp, ZtZtmp, XtZtmp, YtZtmp, YtYtmp, YtXtmp, nlevels, nparams, 1e-6, n)\n",
        "\n",
        "    # Get current beta\n",
        "    beta_est[i,:,:] = param_est[0:beta.shape[1]]\n",
        "\n",
        "  t2 = time.time()\n",
        "  print(\"Time taken in seconds for this example:\")\n",
        "  print(t2-t1)\n",
        "  print(\"Estimated time taken for this example on a nifti of size (100x100x100), in hours:\")\n",
        "  print(100*100*100*(t2-t1)/(nv*60*60))"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(27000, 16, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SJnJ524uFbV1",
        "colab_type": "code",
        "outputId": "ae555b0e-a7b5-40bb-84a7-2e5a9b04e147",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        }
      },
      "source": [
        "print(\"True beta (3)\")\n",
        "beta_map=beta.reshape(dimv[0],dimv[1],dimv[2],beta.shape[1])\n",
        "beta_est_map=beta_est.reshape(dimv[0],dimv[1],dimv[2],beta.shape[1])\n",
        "\n",
        "\n",
        "# Show true beta, 3rd x-slice, 3rd parameter\n",
        "imshow(beta_map[3,:,:,3].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "True beta (3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.colorbar.Colorbar at 0x7f9e15cf21d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV8AAAD8CAYAAADQSqd1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHyVJREFUeJzt3X+MZeV5H/Dv9/6YOz92YSELeAMk\nOAQ1smiNqy1tZavFcewSF5W4UpCJamHVyjpSkGzVUmLjP+x/IqE2dmKpkaVxQcYq/oFqU6OEGhPX\nFkGtKQtCGFgnoWgds1nvsl5gZ5fZmbn3fPvHvauO8c59npl75pw7h+9ndbVz77z3Pe895973vvOe\n5zwvJcHMzKrVqrsBZmZvRO58zcxq4M7XzKwG7nzNzGrgztfMrAbufM3MauDO18ysBu58zcxq4M7X\nzKwGnUmeTPJGAJ8D0AbwXyTdOa58+4IFdS/ZE1QaX3HXYqJtiOspFFdUFImN9ePvMBbRhuLNJHZN\nSuJlp2TaE75uAK1+osxavDH2ExsrEmXKuugz8UZVZ/x7p+jG762iHTdFZZXJDNdaiR2YqGf18JET\nki5JbHFD/+qdC/rpyUFY7omnVx6SdOMk29qsLXe+JNsA/gzAuwG8COBxkg9Iem6j53Qv2YMr7/y9\nsfW22/GHo9eNP62dRD0r/fjddmZpNiyjl2fCMt2l8e+29tn4g8pEJ5VR1oestRaXmVmKy8ydiI/V\n/LHVsEz35bNhGZ5JlOnHH9YMzfXCMmu/sDD298uXxu+t5V+ID9bqhfH7a21XWAT9XfGxGiwkvuB6\n8T7+uw9+4kdxReP99OQA/+ehXwrLtff97d5Jt7VZk0w7XA/geUkvSFoF8FUAN5fTLDOzyQlAkfhX\nh0mmHS4H8ON1918E8E8na46ZWXkEYU3l/CVTtonmfDNIHgBwAAA6ey/c7s2Zmf2Muka2kUk63yMA\nrlx3/4rRYz9D0iKARQCYvfpy5680s8oIwmBK0+ZOMuf7OIBrSL6Z5AyA9wN4oJxmmZmVo4DCWx22\nPPKV1Cd5O4CHMAw1u1vSs+Oe020PsO+iU2Prne/GZ7Uv7r0WllnorIRlXl2bC8v83058EvTEahw+\nUJwd/z3XipuLTIRYKhwtMQXGRJl2os3tlUSI2CAuo0zYVjc+DpzpxvUwE8uYCFPsxdsa9Ma/Lwa9\nxHZmEmUSn/SimwjPzPQYnfjP/Fa3mqkAARiU1LmSPAxgCcNPUF/S/knqm2jOV9KDAB6cpA4zs+1U\n8sj2nZJOlFHRtp9wMzOriwCsNXDO18xsqgnCIHFLVwd8m+QToyiuiXjka2bNJSBxSgEA9pI8uO7+\n4ihSa713SDpC8lIAD5P8oaRHtto0d75m1ljDK9xSTkQn0CQdGf1/nOT9GF7lu+XO19MOZtZgxCBx\nC2shF0juPvczgPcAeGaSlnnka2aNNTzhVkoav8sA3M9hiGEHwJclfWuSCivtfBc6q/gne8cnKto3\n82pYz6/2fhKWubQdp9P6u/7FYZmH2v8wLPO/luPsVatLwa5OxI1m0jNm4nNbmTSPiYxlneV4Mq17\nJi7TXk3E+bbj/TOYj9/Omc8hB4mPRSv+o7G/K47zXd0zfluruxPZyMYnRhu2ZT4Rw9tLHIde/OZp\nz8Vvwt5sHM9fhmGc7+Sdr6QXALx14orW8cjXzBotk7e7Du58zayxyhr5bgd3vmbWWAIxmNK4Ane+\nZtZonnYwM6uYQKxm1s2qgTtfM2us4UUWnnZAr7WGa+aOjS1z9cz43wPAW2fGp6UEgL3tOP7mF/s/\nl/v95zw//6awzFOzvxiWWWnPjy+QWfA1s8pvIoInsxJwN87aiZnTcdhR57W4DItEGFQi1KxYiN/O\nnE2knUwci6ITt2d1d7ytlT3j61m5KBFqtjtucH8usY/nEsdqPn4Tzi/Ei5TumYvLlMUn3MzMKiYR\ng9R699Vz52tmjVZ45GtmVq3hCbfp7Oams1VmZiXwCTczs5oMHOdrZlYtX+E2UqCF04PZsWV+OtgV\n1nNscDqxrTNhmb/vx6sXvzqIy6wNEuFLg/HfvplsZEyEmmVWC+6+FpeZWYrDjnqvxKnPWstxo4uZ\nxOrPF8QZwvrz8YcsEyJWJGLyB4kVg9d2JcLRLhj/+35JYWSaSaSyS6wo3JmJj+dCL7EC+WwilrEk\nhaMdzMyqNUys487XzKxSArHmy4vNzKolwRdZmJlVj77IwsysakK5I1+SbQAHARyRdNMkdbnzNbNG\nK/mE20cAHAIQxKnEKu18l/o9PHry6rFlLpy5PKznid6bwzLzifReL63uDss88/K+sMzpV+NwtPZK\nEGqWWdQyE2qWyGqWWfhy5tV4Y52Xl8MyXItj6LhrfPghAKwxEWo2mwj/mk8sxNlLlImbjH6QyG5Y\nZvyxGCQyjamTSMOWUSTC8BJhlYMisbhoRfOwAktLpk7yCgD/GsAfAfgPk9Y3UedL8jCAJQADAH1J\n+ydtkJlZWYZLx6e6ub0kD667vyhp8XVl/hTAHwCIR20JZYx83ynpRAn1mJmVjNl8vifGDR5J3gTg\nuKQnSN5QRss852tmjSWUdoXb2wH8G5LvBTAL4AKS/1XSv9tqhZO2SgC+TfIJkgfOV4DkAZIHSR5c\nfSWeIzQzK9NgNPodd4tI+oSkKyRdBeD9AP7nJB0vMPnI9x2SjpC8FMDDJH8o6ZH1BUbzJosAsOfX\nLi3pzICZWUxiM3M7SDoy+v84yfsBXA/gkfHPMjOrxvCEW7mXF0v6HoDvTVrPlr8SSC6Q3H3uZwDv\nAfDMpA0yMyvPcA236FaHSUa+lwG4n+S5er4s6VvjnrC82sXTP75ibKVMLB3LVmKV1cQJzmKQiEd8\nLd5FraW4TPvs+AZlYngzKxxnUlNm0k62z8YN4tk4pSSKRABzQiYVZD8Rn9tfSJRJxPAWvbjMYDax\nYnA3KJOIUW0FMeQAwH7iA5H47A0S9ZxkvHL4Wr+aZDfDE24Nu7xY0gsA3lpiW8zMSueUkmZmFSvz\nCreyufM1s0bzAppmZhWTgLVErok6uPM1s8YaTju48zUzq1wyt0Plqu1811rA3wdxPJnIpESZzKV0\nmXC0xIKuaK3FFbXPBm3JrF5cJMLwMmUGiTL9zE5O7OVWZkXhRJluIt3hTNycIs5MiUwSLLUy+zmu\nJwpBzIUOJt5/iVSjmbC2wWx8rFZfi8u8sitxsErQyFAzM7Pp52kHM7NaeA03M7OKDaMdvHS8mVml\nfJGFmVlNPO1gZlYxRzuMsAA6r02e3SsTfpOJNSsrzWcZKw+nXndiO4nEVKkyKZlYvXZ8plntxEoC\nrZLKZD6HmeO5msgklnifRse9k1j8pXMmPqCdRCa7jMzqz51EqNnaQnURCI52MDOrmMTKlqnfLHe+\nZtZoZUw7kJzFcJWeHob95n+T9KlJ6nTna2aNVeKc7wqAX5d0mmQXwKMk/4ek72+1Qne+ZtZoZXS+\nkgTg9Ohud3SbaCJ9OidDzMxKcC7ON7plkGyTfArAcQAPS3pskra58zWzRivA8AZgL8mD624HXl+P\npIGk6wBcAeB6ktdO0q7Kpx3UHj9SzwREZyKcMuFUqZOgmdCtTAhYUCZVRyrzWVxm2qTC48rK1jZI\nZAAra6HSIJMdAHSDMLGZpfiAzizFjWmtxvVkQv5WL4y7DCYWps1kYiuDBPRzydRPSNqfq1OvkPwu\ngBsxwYrtHvmaWaOVMe1A8hKSe0Y/zwF4N4AfTtIun3Azs8YqMbfDPgD3kGxjOGi9T9KfT1KhO18z\nazSVE+3wNIC3Td6a/8+dr5k1mhPrmJlVTHJiHTOzGhADLx1vZla9MuZ8t0PY+ZK8G8BNAI5Lunb0\n2MUAvgbgKgCHAdwi6eWoLjFePZaJIMtMLGtqf2fihTOxtYmNhUVKSvOYed2Z+OZMesZMusiUVAxv\nXE0mLWemTGb/ZFYD7izHryuK4509uRZvZyluTGY16sFcvLRzu5eI4Z1LfB4qGvZNcz7fzKfnixgG\nE6/3cQDfkXQNgO+M7puZTRcN532jWx3CzlfSIwBOvu7hmwHcM/r5HgC/VXK7zMxKkby8uHJbHfxf\nJuno6OefALhso4Kja6QPAEBnz0Vb3JyZ2eZpik+4TdyqUaq1DQfukhYl7Ze0v7WwMOnmzMw2ZcdO\nO2zgGMl9ADD6/3h5TTIzK4/E8FaHrXa+DwC4bfTzbQC+WU5zzMzKMxzZTmfnmwk1+wqAGzDMd/ki\ngE8BuBPAfSQ/BOBHAG7JbIwoZ+Xc1L7KhFOVFI5WZMKggj1dJGbfozoAYDATN7g/F++czkIcdsRB\nHL6USfOYSWWYUlL6zym9GnUiSuRhzRyHoltSPSWtHJ4xraFm4cdZ0q0b/OpdJbfFzKx0dc3pRnyF\nm5k1lkAUUxrt4M7XzBptSge+7nzNrMG0g3M7mJntaFM69J3OyRAzs5KUEWpG8kqS3yX5HMlnSX5k\n0nZVO/ItgM6Z4IVmvqUy4V+JV1Z0M7FJcRG14nqKIPyGcWQX+onGZNoCZr5z4wZ1Z+J6WmtxewaJ\nTFmZELqyhhKZv1Kj7HwA0J9NfKiDlX6Z2FDRjV94JuRvMBvXs7YQl+nPhUUw6FW0ejGAoihlW30A\nH5P0JMndAJ4g+bCk57ZaoUe+ZtZcwvDbNLpF1UhHJT05+nkJwCEAl0/SNM/5mlmjlR3nS/IqDBfT\nfGySetz5mlmz5TrfvSQPrru/KGnx9YVI7gLwdQAflXRqkma58zWzBkvnbjghaf/Ymsguhh3vvZK+\nMWnL3PmaWbOVMO1AkgDuAnBI0mcnr9En3MysyQSoYHhLeDuADwD4dZJPjW7vnaRplY58WQDd00Gh\nxLdUZoHDYiYuMxhkFvrLNGjyMKhM6FJm0cEiEZI1SOybTPhSJpSqHa//mMpwlQk1yxyGTFaz1AKt\nifdgfz4uEx2v/nwiA91yIuSvP3k4JJALI+svlPMeLM/koWaSHi2lonU87WBmzTalV7i58zWzZnPn\na2ZWsXMXWUwhd75m1mhOpm5mVodycjuUzp2vmTVaGetGbgd3vmbWXIJPuAHD+Mn28uR7Qq1Mqr5M\nRXGRVDq6Ev6qUSLWNbXnEvVk0m2qk4jVTKQFzKSUTK0onCjT6mfKZFZTjuvJxDhn4nxXF6ISmX0c\nb6e1mvjMJPZxJh49FWM/U1WPmMtaVgePfM2s2TzyNTOrQeYvqxq48zWz5nKcr5lZPRztYGZWhynt\nfJ1S0sysBuHIl+TdAG4CcFzStaPHPg3gdwG8NCp2h6QHw60pGVYUtSlzvSAT8zyrmW3FZTIpEaOI\nodS0VKZM4us0MxBIpbjMZNJMpClsr8Yt6izH2+okwhgz28qki1zdnVhxORGONpgd357+fDmheu2z\nieOwUk7aziIRRpYpU5ZpnXbIjHy/CODG8zz+J5KuG93ijtfMrGrC8PLi6FaDsPOV9AiAkxW0xcys\nfErcajDJnO/tJJ8meTfJi0prkZlZiaj4Voetdr6fB3A1gOsAHAXwmY0KkjxA8iDJg/2zZ7a4OTOz\nLSpp5DsaaB4n+UwZzdpS5yvpmKSBpALAFwBcP6bsoqT9kvZ3ZsML2c3MylXetMMXcf7zX1uypc6X\n5L51d98HoJRvAjOzMmWmHLLTDmWf/8qEmn0FwA0A9pJ8EcCnANxA8joMvzMOA/hwamtEORnAMuFU\nZUUwZ7JpJaqJQnRSuyVTKBOFl4o1S9RTUplMVq5MGFnvlTitWedMIvVZIjwOiFN3reyJYxCjDHNa\niNubCf9SN26LWvE7OfPeyWQsU6/CidZcNMNekgfX3V+UtLhNLQKQ6Hwl3Xqeh+/ahraYmZUuObI9\nIWn/NjflZ/jyYjNrtim9yMKdr5k1V42hZBHndjCzZisv1OwrAP43gH9A8kWSH5qkWR75mlmjlZFP\nBtjw/NeWeeRrZlYDj3zNrNmmdM536jrfTHxukVhZV4lXlkoFmZBZKbkV/elTYVwtiwrT+SX2TXsl\nLtM5G7+w7uk4JrZ9Ot6YOvGbsL07k3MzLhKlVmzPx6+plTijlFhEGsVaOauCq5OI8+1UtLDaFJ9w\nm7rO18ysVO58zcxq4M7XzKxaRHnRDmVz52tmzeU5XzOzmrjzNTOrgTvfoSj9nVqJMLIKU0pm/mRh\nJkthsGpuK7GScjuRepGDct5pSqRVLGPVZiC5GnVC6r2TCCPLpF8sZhL7pxcWgXrjJyR7vfigZxbq\n7q8mUkq2MyslZw5oXKRKnnYwM6uDO18zs4rJ0Q5mZvXwyNfMrHqe8zUzq4M7XzOzim1uafhKVR9q\nFoQwlRYilgj/CjONIZmVK5EyKsrc1TmbqGM1bnCmvUU3jgXqzybqSWSXKzLJv1KrBWfeGPHGOnOZ\nFYXj9qxcELdnbSEsAs6OP2C9bmL14szyxamVrxNZzfpxmVYmg1pFqcSJ6Z12cDJ1M2s0Kr6l6iFv\nJPnXJJ8n+fFJ2+XO18yarYQ13Ei2AfwZgN8E8BYAt5J8yyTNcudrZs1WzgKa1wN4XtILklYBfBXA\nzZM0y52vmTVXYsphNO2wl+TBdbcDr6vpcgA/Xnf/xdFjW+ZoBzNrttzI9oSk/dvckp/hztfMGq2k\ny4uPALhy3f0rRo9tWdj5krwSwJcAXIbhd8iipM+RvBjA1wBcBeAwgFskvTyuLrWAQRDClNlRrUR2\nr/ZK/HWXyySWqCcT1hbU016JX3irH7elyIRtJULNBplwtPlEPZnMXpmsXHOJ9iTKtFcS2b0Sk3Er\nFyVe+1zivdOdvGcYZHZgIhtZ5n2cWew0swOV+HyWpaRQs8cBXEPyzRh2uu8H8DuTVJiZ8+0D+Jik\ntwD4ZwB+f3SW7+MAviPpGgDfGd03M5semZNtic5ZUh/A7QAeAnAIwH2Snp2kaeHIV9JRAEdHPy+R\nPIThRPPNAG4YFbsHwPcA/OEkjTEzK11Jg2xJDwJ4sJzaNjnnS/IqAG8D8BiAy0YdMwD8BMNpCTOz\nqTHNV7ilO1+SuwB8HcBHJZ3iuvT5kkSe/yWOQjYOAEB390WTtdbMbJNYTGfvm4rzJdnFsOO9V9I3\nRg8fI7lv9Pt9AI6f77mSFiXtl7S/PZ+42N3MrCwlzfluh7Dz5XCIexeAQ5I+u+5XDwC4bfTzbQC+\nWX7zzMwmU1Zuh7Jlph3eDuADAH5A8qnRY3cAuBPAfSQ/BOBHAG7ZniaamU1gOmcdUtEOj2LjhHTv\n2szG1AbWdo8v0z4b19NKxBp2z8R7fOZUHGOZSeOIEuaUMt++mXDOVAxvYuXd/ly8qUyZKK4byMXV\nZuoZzCZiWROx3Zn0i5l0kUU3PqgajN/Y2dU4TeZgkImrjcu0VhNx0suJMpl9nHozl2PHn3AzM9uR\n3PmamVXMqxebmVWvEXG+ZmY7kqaz93Xna2aN5pGvmVnVvHrxkNrA6oXjZ7+7rTgEpXs6kzowEWr2\napybsn02sXpsIo1j0R2fyrCYiUOBim5cZpCoJ5MuUokVfBVnZ0yFkamdWkQrLBJEbQEAikSbM5T4\n5KRWkl4e36CzRZyTU2vxTm6fjl9491S8A3svx8eqsxwWQWtQYUpJn3AzM6ueO18zs6oJPuFmZlYH\nn3AzM6vDlHa+XjrezBrr3EUW253VjORvk3yWZEEytQqyO18zay4JLOJbCZ4B8G8BPJJ9QrXTDi2h\nuGB86FZ/EDepSIRKZaR2epE4VcpEVqkgHK3IhHaVFP6VwUQoUGst0Z7EocqE6mUyjaUWQixruJHY\nViZLGJbGHzCdiQ9oJ7GdmVfiMrMvxS9q7mT8eeguxeGZmZW4S1PBpiQdAgAmQiLP8ZyvmTWaT7iZ\nmVVNyObb3kvy4Lr7i5IW1xcg+ZcA3nSe535S0qZX8nHna2bNlhv5npA09kSZpN8opT0j7nzNrNGm\nddrB0Q5m1mhVRDuQfB/JFwH8cwB/QfKh6Dke+ZpZc1WU1UzS/QDu38xzKg81a/XGp3oqZuLQmiJe\nUzC5SGS8rXYiy1omBGzQG7+tQS8TRlZOaFfmmz61mGJJYW2Z45nKjpaJ8ilp3cZMxrL22cQ+DMq0\n4qgtdM7EZWZPxsd8/nic5a93Ik5Z1joVl+Fa4oWVYHiRxXTOO3jka2bN5qxmZmbV88jXzKxqXsnC\nzKwOpeVuKJ07XzNrNk87mJlVTF5GyMysHjt15EvySgBfAnAZhlPXi5I+R/LTAH4XwEujondIenBs\nZSIULDHLopxY1kyc79oF8XdPP5FaMbV6cRALnFpVt6QY1dJSOJYklXYylZoysbGy4nwTo6lWHDYb\nlum8Ftcx+0rcmPljiRjeo0vxxn76SlhES6fDMkVFcb4AdvQJtz6Aj0l6kuRuAE+QfHj0uz+R9Mfb\n1zwzs8kwk5O7BmHnK+kogKOjn5dIHgJw+XY3zMxsYsLUXmSxqcQ6JK8C8DYAj40eup3k0yTvJnnR\nBs85QPIgyYODpcR1kGZmJSEEKr7VId35ktwF4OsAPirpFIDPA7gawHUYjow/c77nSVqUtF/S/vbu\nhRKabGa2CVJ8q0Eq2oFkF8OO915J3wAAScfW/f4LAP58W1poZjaJKY12CEe+HK4IdxeAQ5I+u+7x\nfeuKvQ/D1TvNzKbHuTnf6FaDzMj37QA+AOAHJJ8aPXYHgFtJXofhyzsM4MNhTQWB0+PzB7aX41ig\nTAhPJgVhfzYR1pZIKZkJayuCPV3WqsPKrJ6a2DdRewGgmInLDDLpIhOvPVWmlVlSOC6SkvjAthPx\ncVwN6liJX1P3TGJF4VPBhgDwVHxOpsiEkS3HKSWrHI3u5GiHR3H+6MjxMb1mZrWrb0434mWEzKy5\nhEpOuJH8TyR/OIr+up/knug57nzNrNmqmfN9GMC1kv4RgL8B8InoCe58zazRqojzlfRtSeeumf4+\ngCui5zixjpk1W/Vzvv8ewNeiQu58zay5JGCQmlfYS/LguvuLkhbXFyD5lwDedJ7nflLSN0dlPolh\nPpx7ow1W2vmyD3RfHj/T0TkTh+d0luNvskzWqUE3s+pwXE9/PlHP7Pjfp1bwLSncKrUScGZCqsLM\nZ5ltZcY3qZC+TMhapkhiW1EWv0w4X5Qxb1gmPqDtbtwdcC54IwNolbWs9alyqkmOfE9I2j++Gv3G\nuN+T/CCAmwC8S4o36pGvmTVbBdMOJG8E8AcA/qWkRCJQd75m1mQCUM0abv8ZQA/Aw8OLgvF9Sb83\n7gnufM2swQRo+69wk/Srm32OO18zay4he8Ktcu58zazZpvTyYne+ZtZs7nwBDoCZV8aHxXQSCZEy\nZVr9eIenQnQS4Wj9ubg9axeMb09/PhEi1kv8+dTNrOyYeDMmFjLFWhy+xNVEZq9+okyw8OqwUGIf\nJt7x6mbC9RLvr14mw9z4fdhK7JvV5fg4dC+IYxnby/FiB2zH22ol/sxXop5yQs2mN7GOR75m1lwC\nsFNTSpqZ7Wge+ZqZVS19eXHl3PmaWXMJUAVxvlvhztfMmq2aK9w2zZ2vmTWb53zNzComOdoBGKZ5\n7AT5fjKrtbZXy0kpmVmhN5MWsEiknRzMjW+zdvfH/h4AuvPxCrS9XlwPE/Gw/X78wleW49jRgpm3\nWCJeOJPCMZN2sp2JBU68eTqZehIx4kE1rZX4OHQSKU3XdiXqORPnr8wczUQ2RSiR4rI0HvmamVVN\n0GBQdyPOy52vmTVXdSklN82dr5k1m0PNzMyqJQDyyNfMrGKqJpn6VrjzNbNGm9YTbsyEhZS2MfIl\nAD9a99BeACcqa0A53Obtt9PaC7jN2+GXJV0ySQUkv4Xh64yckHTjJNvarEo735/bOHkwWq552rjN\n22+ntRdwm23zKox0NjOzc9z5mpnVoO7Od7Hm7W+F27z9dlp7AbfZNqnWOV8zszequke+ZmZvSLV1\nviRvJPnXJJ8n+fG62rEZJA+T/AHJp0gerLs950PybpLHST6z7rGLST5M8m9H/19UZxvX26C9nyZ5\nZLSfnyL53jrbuB7JK0l+l+RzJJ8l+ZHR49O8jzdq89Tu5zeCWqYdSLYB/A2AdwN4EcDjAG6V9Fzl\njdkEkocB7Jc0tbGRJP8FgNMAviTp2tFj/xHASUl3jr7oLpL0h3W285wN2vtpAKcl/XGdbTsfkvsA\n7JP0JMndAJ4A8FsAPojp3ccbtfkWTOl+fiOoa+R7PYDnJb0gaRXAVwHcXFNbGkXSIwBOvu7hmwHc\nM/r5Hgw/eFNhg/ZOLUlHJT05+nkJwCEAl2O69/FGbbYa1dX5Xg7gx+vuv4id8WYQgG+TfILkgbob\nswmXSTo6+vknAC6rszFJt5N8ejQtMTV/wq9H8ioAbwPwGHbIPn5dm4EdsJ+byifcNucdkv4xgN8E\n8PujP5l3FA3nmaY9xOXzAK4GcB2AowA+U29zfh7JXQC+DuCjkk6t/9207uPztHnq93OT1dX5HgFw\n5br7V4wem2qSjoz+Pw7gfgynT3aCY6N5v3Pzf8drbs9Yko5JGmi45vcXMGX7mWQXw07sXknfGD08\n1fv4fG2e9v3cdHV1vo8DuIbkm0nOAHg/gAdqaksKyYXRyQqQXADwHgDPjH/W1HgAwG2jn28D8M0a\n2xI614mNvA9TtJ9JEsBdAA5J+uy6X03tPt6ozdO8n98IarvIYhTW8qcA2gDulvRHtTQkieSvYDja\nBYapOL88jW0m+RUAN2CYyekYgE8B+O8A7gPwSxhmlbtF0lSc5NqgvTdg+KewABwG8OF186m1IvkO\nAH8F4AcAziWKvQPDOdRp3ccbtflWTOl+fiPwFW5mZjXwCTczsxq48zUzq4E7XzOzGrjzNTOrgTtf\nM7MauPM1M6uBO18zsxq48zUzq8H/A4NVMJ/j/hNPAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iIodupfkRONO",
        "colab_type": "code",
        "outputId": "3e38729f-f37a-4af8-9359-964988f29515",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 304
        }
      },
      "source": [
        "print(\"Estimated beta (3)\")\n",
        "\n",
        "# Show estimated beta, 3rd x-slice, 3rd parameter\n",
        "imshow(beta_est_map[3,:,:,3].reshape(dimv[0],dimv[1]), vmin=-3, vmax=3, \\\n",
        "                    interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()\n"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Estimated beta (3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.colorbar.Colorbar at 0x7f9e15c102b0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV8AAAD8CAYAAADQSqd1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEXdJREFUeJzt3X+oX/V9x/HXy9S1IxGaYJdmMc7O\nuoJIF0dwjJbNrj+WlkLqYNIMOmVlaWGCgmO1FlbHKMja2hVWOm5naAr2h6CZ4qQ2LUImrNZEnCbG\ntiIRk6VJU7dpHJ3m3tf+OCfsu/Te+/3ee873fM49eT7kcL/f8z0/3h70fd/3fT6f83USAQC6dV7p\nAADgXETyBYACSL4AUADJFwAKIPkCQAEkXwAogOQLAGPYfoPtH9j+N9sHbf9142MyzhcAFmfbklYn\nOWX7fEmPSLoxyfeXe8zXtRYdAAxUqir1VP32/HppVLl2mnxXrVmd161b1+UpAaxQr75w5GSSNzU5\nxh+8a3V+9uLs2O32P/k/ByX9fGTVTJKZ0W1sr5K0X9JbJX0pyaNNYmuUfG1vlfRFSask/WOS2xc9\n2bp1+tWbb2pySgDniMM3/cXzTY/xsxdn9YOHLh673aoNP/55ki2LbZNkVtJm22+UtNv2FUkOLDe2\nZd9wq38LfEnS+yVdLmm77cuXezwAaFskzU3wz5KOmfynpIclbW0SW5PRDldJejbJc0lelfRNSdua\nBAMAbYqi1zI7dhnH9pvqile2f1nSeyU90yS2Jm2HjZJeGHl/RNJvn72R7R2SdkjSqrVrG5wOAJZu\nqZXtAjZI2lX/xX+epLuTPNDkgFO/4VY3rWck6fUXb2JcG4DORNFsC8Npkzwp6crmEf2fJsn3qKRN\nI+8vqtcBQG/MNRsRNjVNku9jki6z/RZVSffDkv64lagAoAWRNDu05JvktO0bJD2kaqjZziQHW4sM\nAFowxMpXSR6U9GBLsQBAqyLptZ4+QoHpxQAGK8rw2g4A0HuRZvuZe0m+AIarmuHWTyRfAANmzcql\ng5gXyRfAYFU33Ei+ANCpapwvyRcAOjdH5QsA3aLyBYACImu2p98TTPIFMGi0HQCgY5H1alaVDmNe\nJF8Ag1VNsqDtAACd44YbAHQssWZD5QsAnZuj8gWAblU33PqZ5voZFQC0gBtuAFDILON8AaBbzHAD\ngELmGO0AAN2qHqxD8gWATkXWa0wvBoBuJertJIt+RgUArbDmJljGHsXeZPth20/bPmj7xqaRUfkC\nGKyotcr3tKSbkzxu+wJJ+23vSfL0cg9I8gUwaG3ccEtyTNKx+vXLtg9J2iiJ5AsAZ4vc+sPUbV8i\n6UpJjzY5TqPka/uwpJclzUo6nWRLk+MBQJuqr46fKM1daHvfyPuZJDNnb2R7jaR7JN2U5KUmsbVR\n+b4ryckWjgMALfOkz/M9Oa54tH2+qsR7V5J7m0ZG2wHAYEXtzHCzbUl3SjqU5I7GB1TzoWaR9B3b\n+23vmG8D2zts77O9b/bUKw1PBwBLM1tXv4stE3iHpI9I+n3bT9TLB5rE1bTyfWeSo7Z/RdIe288k\n2Tu6Qd03mZGk11+8KQ3PBwATS9xK5ZvkEandp7I3Sr5JjtY/T9jeLekqSXsX3wsAulHdcOvn9OJl\n/0qwvboebCzbqyW9T9KBtgIDgOaq73Abt5TQpPJdL2l31YfW6yR9Pcm3W4kKAFpQ3XAb2MPUkzwn\n6TdbjAUAWscjJQGgY9OY4dYWki+AQeMLNAGgY4n02hzJFwA6VbUdSL4A0LkJZ7B1juQLYLAGOdQM\nAPqPtgMAFDHJd7SVQPIFMFjVaId+PtuB5AtgsJhkAQCF0HYAgI4x2gEACmG0AwB0LLFOk3wBoHu0\nHQCgY/R8AaAQki8AdIxxvgBQCON8AaBjiXSah6kDQPdoOwBAx+j5AkAhIfkCQPf6esOtn51oAGhB\nUvV8xy2TsL3T9gnbB9qIjeQLYMCs2bnzxi4T+qqkrW1FRtsBwKC11fNNstf2Ja0cTBNUvvOV2rbX\n2d5j+8f1z7VtBQQAbTnzbIcJ2g4X2t43suyYdmyT1Ntf1S+W2rdI+l6SyyR9r34PAP2Squ87bpF0\nMsmWkWVm2qGNTb5J9kp68azV2yTtql/vkvShluMCgFbMyWOXEpbb812f5Fj9+ieS1i+0YV2+75Ck\nVWvpTgDoTuobbn3UOKokUdVaWejzmTOl/Ko1q5ueDgCWZMK2w1i2vyHpXyW9zfYR2x9tEtdyK9/j\ntjckOWZ7g6QTTYIAgGlpcbTD9lYOVFtu5Xu/pOvq19dJuq+dcACgPVVl67FLCWMr37rUvlrVUIwj\nkj4t6XZJd9dl9/OSrp1mkACwXCv2wTqLlNrvbjkWAGjdpD3drjHDDcBgRdZcT0c7kHwBDFpPC1+S\nL4ABC8/zBYAyelr6knwBDBqVLwB0LJLm5ki+ANCtSKLyBYDuMc4XAEog+QJA18o9u2Ecki+AYaPy\nBYCORQqjHQCgBJIvAHSPtgMAFEDyBYCOMckCAMpgkgUAlMBoBwDonql8AaBjETfcAKB75oYbABRB\n5QsABcyVDmB+JF8Aw9Xjcb79/EJ7AGiJM36Z6Dj2Vts/tP2s7VuaxkXyBTBsmWAZw/YqSV+S9H5J\nl0vabvvyJmGRfAFgvKskPZvkuSSvSvqmpG1NDkjyBTBoE7YdLrS9b2TZcdZhNkp6YeT9kXrdso29\n4WZ7p6QPSjqR5Ip63W2S/kzST+vNbk3yYJNAAKB10aTTi08m2TLlaP6fSSrfr0raOs/6LyTZXC8k\nXgD91ELPV9JRSZtG3l9Ur1u2sck3yV5JLzY5CQCU0tJoh8ckXWb7LbZ/SdKHJd3fJK4mPd8bbD9p\ne6fttQttZHvHmT7K7KlXGpwOAJahhco3yWlJN0h6SNIhSXcnOdgkrOUm3y9LulTSZknHJH1+oQ2T\nzCTZkmTLqjWrl3k6AFimdtoOSvJgkt9IcmmSzzQNa1kz3JIcP/Pa9lckPdA0EABo21ImUXRtWZWv\n7Q0jb6+RdKCdcACgZXMevxQwyVCzb0i6WtU4uCOSPi3patubVRXshyV9bIoxAsCy9bXyHZt8k2yf\nZ/WdU4gFANq3UpMvAKxYPe75knwBDBvJFwC6554+TJ0H6wBAAVS+AIaNtgMAdIwbbgBQCMkXAAog\n+QJAt6z+jnYg+QIYLnq+AFAIyRcACiD5AkD3aDsAQAkkXwDoWBjtAABlUPkCQPfo+QJACSRfAOjY\nEr4avmskXwCDZdF2AIAiSL4AUALJFwAK6Gny5TvcAAxX/VSzcUtTtv/I9kHbc7a3TLIPyRfAsGWC\npbkDkv5Q0t5Jd6DtAGDQuphenOSQJNmeeB+SL4BBm7CtcKHtfSPvZ5LMTCeiytjka3uTpK9JWq+q\nQJ9J8kXb6yR9S9Ilkg5LujbJf0wvVABYosnbCieTLNqrtf1dSW+e56NPJblvqaFNUvmelnRzksdt\nXyBpv+09kq6X9L0kt9u+RdItkj6x1AAAYKpaGu2Q5D3tHKky9oZbkmNJHq9fvyzpkKSNkrZJ2lVv\ntkvSh9oMDACaOjPDbdqjHZZjSaMdbF8i6UpJj0pan+RY/dFPVLUl5ttnh+19tvfNnnqlQagAsHSe\ny9il8Tnsa2wfkfQ7kv7Z9kPj9pn4hpvtNZLukXRTkpdG7+oliT3/74+6aT0jSa+/eFNPhzsDGKSO\nHqyTZLek3UvZZ6LK1/b5qhLvXUnurVcft72h/nyDpBNLOTEAdGHFth1clbh3SjqU5I6Rj+6XdF39\n+jpJS77bBwBT180kiyWbpO3wDkkfkfSU7SfqdbdKul3S3bY/Kul5SddOJ0QAWL4V+1SzJI+oumk4\nn3e3Gw4AtGylJl8AWLH49mIA6B7fZAEApaSf2ZfkC2DQqHwBoGt8ezEAlMENNwAogOQLAF2LuOEG\nACVwww0ASiD5AkC3mGQBACWknYelTwPJF8Cw9TP3knwBDBttBwDoWiTRdgCAAvqZe0m+AIaNtgMA\nFMBoBwDoGk81A4DuVZMs+pl9Sb4Aho2nmgFA96h8AaBrPe75nlc6AACYnurZDuOWpmx/1vYztp+0\nvdv2G8ftQ/IFMGzJ+KW5PZKuSPJ2ST+S9MlxO5B8AQxXqq8RGrc0Pk3ynSSn67ffl3TRuH3o+QIY\ntu5vuP2ppG+N22hs5Wt7k+2HbT9t+6DtG+v1t9k+avuJevlAC0EDQLsywSJdaHvfyLLj7MPY/q7t\nA/Ms20a2+ZSk05LuGhfWJJXvaUk3J3nc9gWS9tveU3/2hSSfm+AYAFCE5ybqK5xMsmWxDZK8Z9Hz\n2NdL+qCkdyfjy+2xyTfJMUnH6tcv2z4kaeO4/QCguKiTSRa2t0r6S0m/l+S/J9lnSTfcbF8i6UpJ\nj9arbqiHVuy0vXaBfXacKeVnT72ylNMBQCNW5IxfWvD3ki6QtKduw/7DuB0mTr6210i6R9JNSV6S\n9GVJl0rarKoy/vx8+yWZSbIlyZZVa1ZPejoAaEcHQ82SvDXJpiSb6+Xj4/aZaLSD7fNVJd67ktxb\nn+z4yOdfkfTAMuMGgOnp6fTiSUY7WNKdkg4luWNk/YaRza6RdKD98ACggTM933FLAZNUvu+Q9BFJ\nT9l+ol53q6Tttjer+tc7LOljU4kQABqYcLRD5yYZ7fCIqsdinu3B9sMBgDa1Nn24dcxwAzBcEckX\nAIroZ9eB5Atg2HiYOgCUQPIFgI4l0mw/+w4kXwDDRuULAAWQfAGgY5HUwne0TQPJF8CARQo9XwDo\nVsQNNwAogp4vABRA8gWArvFgHQDoXiSt1EdKAsCKRuULAF1jejEAdC9SGOcLAAUwww0ACqDnCwAd\nSxjtAABFUPkCQNeizM6WDmJeJF8Aw8UjJQGgEIaaAUC3IikdVL62/0bSNlVfVH9C0vVJ/n2xfc6b\nelQAUErqh6mPW5r7bJK3J9ks6QFJfzVuBypfAIPWxQ23JC+NvF2tquhelNPhMAzbP5X0/MiqCyWd\n7CyAdhDz9K20eCVinoZfS/KmJgew/W1V/57jvEHSz0fezySZWeK5PiPpTyT9l6R3Jfnpott3mXx/\n4eT2viRbigWwDMQ8fSstXomYzwW2vyvpzfN89Kkk941s90lJb0jy6cWOR9sBACaQ5D0TbnqXpAcl\nLZp8ueEGAA3Zvmzk7TZJz4zbp3Tlu6SeSk8Q8/SttHglYj7X3W77baqGmj0v6ePjdija8wWAcxVt\nBwAogOQLAAUUS762t9r+oe1nbd9SKo6lsH3Y9lO2n7C9r3Q887G90/YJ2wdG1q2zvcf2j+ufa0vG\nOGqBeG+zfbS+zk/Y/kDJGEfZ3mT7YdtP2z5o+8Z6fZ+v8UIx9/Y6nwuK9Hxtr5L0I0nvlXRE0mOS\ntid5uvNglsD2YUlbkvR2YLrt35V0StLXklxRr/tbSS8mub3+Rbc2ySdKxnnGAvHeJulUks+VjG0+\ntjdI2pDkcdsXSNov6UOSrld/r/FCMV+rnl7nc0GpyvcqSc8meS7Jq5K+qWp4BhpKslfSi2et3iZp\nV/16l6r/8XphgXh7K8mxJI/Xr1+WdEjSRvX7Gi8UMwoqlXw3Snph5P0RrYz/GCLpO7b3295ROpgl\nWJ/kWP36J5LWlwxmQjfYfrJuS/TmT/hRti+RdKWkR7VCrvFZMUsr4DoPFTfcluadSX5L0vsl/Xn9\nJ/OKkqrP1PfxhV+WdKmkzZKOSfp82XB+ke01ku6RdNNZD1Xp7TWeJ+beX+chK5V8j0raNPL+onpd\nryU5Wv88IWm3qvbJSnC87vud6f+dKBzPopIcTzKbZE7SV9Sz62z7fFVJ7K4k99are32N54u579d5\n6Eol38ckXWb7LbZ/SdKHJd1fKJaJ2F5d36yQ7dWS3ifpwOJ79cb9kq6rX18n6b5Fti3uTBKrXaMe\nXWfblnSnpENJ7hj5qLfXeKGY+3ydzwXFZrjVw1r+TtIqSTuTfKZIIBOy/euqql2pmpb99T7GbPsb\nkq5W9Ri946oe7vFPku6WdLGqqY/XJunFTa4F4r1a1Z/CkXRY0sdG+qlF2X6npH+R9JSqqaSSdKuq\nHmpfr/FCMW9XT6/zuYDpxQBQADfcAKAAki8AFEDyBYACSL4AUADJFwAKIPkCQAEkXwAo4H8BZIeh\nfScW0rQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wUapPX6JdyPG",
        "colab_type": "text"
      },
      "source": [
        "**Note:** Result is roughly similar... but lots of local minima disrupting the image."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y3pBWlFqeRZx",
        "colab_type": "text"
      },
      "source": [
        "### Scaled up Helper Functions\n",
        "\n",
        "This section contains the Nifti-equivalent functions to those given *helper functions* for an individual voxel."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RlYTkC9Qgg3l",
        "colab_type": "text"
      },
      "source": [
        "#### Initial beta (broadcasted)\n",
        "\n",
        "The below function returns the OLS estimator for several voxels at once, where the OLS estimator for a given voxel, $\\hat{\\beta}$, is given by:\n",
        "\n",
        "$$\\hat{\\beta}_{OLS}=(X'X)^{-1}X'Y$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `XtX`: The design matrices transposed and multiplied by themselves ($X'X$ in the above notation)\n",
        " - `XtY`: The design matrices transposed and multiplied by the response vectors ($X'Y$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `beta`: The OLS estimates of $\\beta$ ($\\hat{\\beta}_{OLS}$ in the above notation).\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9QDdV-iTggJz",
        "colab_type": "code",
        "outputId": "1485e933-6ab0-466f-bc4d-988567163c76",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "def initBeta_broadcasted(XtX, XtY):\n",
        "  \n",
        "  # Get the beta estimator\n",
        "  beta = np.linalg.solve(XtX,XtY)\n",
        "  \n",
        "  # Return the result\n",
        "  return(beta)\n",
        "\n",
        "print(initBeta(XtX,XtY))\n",
        "print(np.linalg.inv(XtX) @ XtY)"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[ 4.91795182]\n",
            "  [-0.25607615]\n",
            "  [ 2.77596286]\n",
            "  ...\n",
            "  [-3.36219379]\n",
            "  [-2.96547974]\n",
            "  [ 3.39332544]]\n",
            "\n",
            " [[ 3.71803234]\n",
            "  [-0.01798858]\n",
            "  [ 3.00100956]\n",
            "  ...\n",
            "  [-3.31809963]\n",
            "  [-2.18706795]\n",
            "  [ 3.21701699]]\n",
            "\n",
            " [[ 1.96631735]\n",
            "  [ 0.22556045]\n",
            "  [ 3.19126916]\n",
            "  ...\n",
            "  [-3.05289266]\n",
            "  [-0.85186301]\n",
            "  [ 2.8118537 ]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[ 1.94796995]\n",
            "  [-1.38674741]\n",
            "  [-3.51578693]\n",
            "  ...\n",
            "  [ 2.39875941]\n",
            "  [-3.81006659]\n",
            "  [ 3.97951269]]\n",
            "\n",
            " [[ 2.51012841]\n",
            "  [-2.43568087]\n",
            "  [-3.70279756]\n",
            "  ...\n",
            "  [ 2.18015105]\n",
            "  [-4.09124499]\n",
            "  [ 2.97008268]]\n",
            "\n",
            " [[ 2.88400237]\n",
            "  [-2.99876638]\n",
            "  [-3.81711186]\n",
            "  ...\n",
            "  [ 1.84491224]\n",
            "  [-4.24256379]\n",
            "  [ 2.2645892 ]]]\n",
            "[[[ 4.91795182]\n",
            "  [-0.25607615]\n",
            "  [ 2.77596286]\n",
            "  ...\n",
            "  [-3.36219379]\n",
            "  [-2.96547974]\n",
            "  [ 3.39332544]]\n",
            "\n",
            " [[ 3.71803234]\n",
            "  [-0.01798858]\n",
            "  [ 3.00100956]\n",
            "  ...\n",
            "  [-3.31809963]\n",
            "  [-2.18706795]\n",
            "  [ 3.21701699]]\n",
            "\n",
            " [[ 1.96631735]\n",
            "  [ 0.22556045]\n",
            "  [ 3.19126916]\n",
            "  ...\n",
            "  [-3.05289266]\n",
            "  [-0.85186301]\n",
            "  [ 2.8118537 ]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[ 1.94796995]\n",
            "  [-1.38674741]\n",
            "  [-3.51578693]\n",
            "  ...\n",
            "  [ 2.39875941]\n",
            "  [-3.81006659]\n",
            "  [ 3.97951269]]\n",
            "\n",
            " [[ 2.51012841]\n",
            "  [-2.43568087]\n",
            "  [-3.70279756]\n",
            "  ...\n",
            "  [ 2.18015105]\n",
            "  [-4.09124499]\n",
            "  [ 2.97008268]]\n",
            "\n",
            " [[ 2.88400237]\n",
            "  [-2.99876638]\n",
            "  [-3.81711186]\n",
            "  ...\n",
            "  [ 1.84491224]\n",
            "  [-4.24256379]\n",
            "  [ 2.2645892 ]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6FZyyQAEh3mp",
        "colab_type": "text"
      },
      "source": [
        "### Sum of square residuals (broadcasted)\n",
        "\n",
        "The function below calculates the sum of the square residuals, $e^Te$, using the below formula:\n",
        "\n",
        "$$e^Te = (Y-X\\beta)^T(Y-X\\beta)$$ \n",
        "$$=Y^TY - 2Y^TX\\beta + \\beta^T X^TX \\beta$$\n",
        "\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `YtX`: $Y$ transpose multiplied by $X$ ($Y^TX$ in the above notation).\n",
        " - `YtY`: $Y$ transpose multiplied by $Y$ ($Y^TY$ in the above notation).\n",
        " - `XtX`: $X$ transpose multiplied by $X$ ($X^TX$ in the above notation).\n",
        " - `beta`: An estimate of the parameter vector ($\\beta$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "\n",
        " - `ete`: The sum of square residuals ($e^Te$ in the above notation)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vFycwRkYeklg",
        "colab_type": "code",
        "outputId": "ebb94358-a094-44bc-e9ca-c8cf8290a539",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246
        }
      },
      "source": [
        "def ssr_broadcasted(YtX, YtY, XtX, beta):\n",
        "  \n",
        "  # Return the sum of squared residuals\n",
        "  return(YtY - 2*YtX @ beta + beta.transpose((0,2,1)) @ XtX @ beta)\n",
        "\n",
        "print(ssr_broadcasted(YtX,YtY,XtX,initBeta(XtX,XtY)))"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[56534.75774914]]\n",
            "\n",
            " [[44888.06370721]]\n",
            "\n",
            " [[32890.74970878]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[34124.31298716]]\n",
            "\n",
            " [[47911.54109047]]\n",
            "\n",
            " [[60290.02204439]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MOvNqJPojgGp",
        "colab_type": "text"
      },
      "source": [
        "#### Initial Sigma (Broadcasted)\n",
        "\n",
        "The function below returns an initial estimate for the Fixed Effects Variance, $\\sigma^2$. The estimator used is based on the suggested OLS estimator in Demidenko (2012) and is given by:\n",
        "\n",
        "$$\\hat{\\sigma}^2_{OLS}=\\frac{1}{n}(Y-X\\beta)^T(Y-X\\beta)$$\n",
        "$$=\\frac{1}{n}e^Te$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `ete`: The sum of square residuals ($e^Te$ in the above notation).\n",
        " - `n`: The total number of observations ($n$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `sigma2`: The OLS estimate of $\\sigma^2$ ($\\hat{\\sigma}^2_{OLS}$ in the above notation)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kY1ILWTLjnKy",
        "colab_type": "code",
        "outputId": "e5aaa875-7fd0-433e-971d-ac09e892f898",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "def initSigma2_broadcasted(ete, n):\n",
        "\n",
        "  # Return the OLS estimate of sigma\n",
        "  return(1/n*ete[:,0,0])\n",
        "\n",
        "print(initSigma2_broadcasted(ssr_broadcasted(YtX,YtY,XtX,initBeta(XtX,XtY)),n))"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[56.53475775 44.88806371 32.89074971 ... 34.12431299 47.91154109\n",
            " 60.29002204]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AAURtA0rlZVL",
        "colab_type": "text"
      },
      "source": [
        "### Kron (broadcasted)\n",
        "\n",
        "Taken from https://stackoverflow.com/questions/57259557/kronecker-product-of-matrix-array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H5saAin0lY69",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def kron_broadcasted(A,B):\n",
        "  i,j,k = A.shape\n",
        "  i,l,m = B.shape\n",
        "  return(np.einsum(\"ijk,ilm->ijlkm\",A,B).reshape(i,j*l,k*m))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w7gpsuCTnUee",
        "colab_type": "text"
      },
      "source": [
        "#### Matrix to Vector (broadcasted)\n",
        "\n",
        "This function takes in a stack of matrices and returns the corresponding vectorized forms of those matrices."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IziwYuRbnUuG",
        "colab_type": "code",
        "outputId": "82fad9b6-9f31-4dba-de8b-839db34ff812",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 967
        }
      },
      "source": [
        "def mat2vec_broadcasted(matrix):\n",
        "  \n",
        "  #Return vectorised matrix\n",
        "  return(matrix.transpose(0,2,1).reshape(matrix.shape[0],matrix.shape[1]*matrix.shape[2],1))\n",
        "\n",
        "# Example:\n",
        "matrix = np.random.randn(4,3,3)\n",
        "print(matrix)\n",
        "print(mat2vec_broadcasted(matrix))"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[ 1.54154642  0.55444553  0.01323832]\n",
            "  [ 0.6083838   0.99857481  0.75309664]\n",
            "  [ 1.9640216  -1.82181275  0.70789548]]\n",
            "\n",
            " [[-1.2714879  -0.30505595 -0.8450612 ]\n",
            "  [ 2.18875803  0.70191168 -1.38517448]\n",
            "  [-0.37531452  0.97008818 -0.85685369]]\n",
            "\n",
            " [[-1.83613454 -0.74693756  0.54683516]\n",
            "  [-0.02560497 -0.52978332 -0.13097986]\n",
            "  [-1.72285502  0.21697901  2.33511694]]\n",
            "\n",
            " [[-0.72449532  1.97059396  0.34013728]\n",
            "  [ 1.66597969  2.55326419  0.3588915 ]\n",
            "  [ 1.32161462  1.9372769  -0.42979633]]]\n",
            "[[[ 1.54154642]\n",
            "  [ 0.6083838 ]\n",
            "  [ 1.9640216 ]\n",
            "  [ 0.55444553]\n",
            "  [ 0.99857481]\n",
            "  [-1.82181275]\n",
            "  [ 0.01323832]\n",
            "  [ 0.75309664]\n",
            "  [ 0.70789548]]\n",
            "\n",
            " [[-1.2714879 ]\n",
            "  [ 2.18875803]\n",
            "  [-0.37531452]\n",
            "  [-0.30505595]\n",
            "  [ 0.70191168]\n",
            "  [ 0.97008818]\n",
            "  [-0.8450612 ]\n",
            "  [-1.38517448]\n",
            "  [-0.85685369]]\n",
            "\n",
            " [[-1.83613454]\n",
            "  [-0.02560497]\n",
            "  [-1.72285502]\n",
            "  [-0.74693756]\n",
            "  [-0.52978332]\n",
            "  [ 0.21697901]\n",
            "  [ 0.54683516]\n",
            "  [-0.13097986]\n",
            "  [ 2.33511694]]\n",
            "\n",
            " [[-0.72449532]\n",
            "  [ 1.66597969]\n",
            "  [ 1.32161462]\n",
            "  [ 1.97059396]\n",
            "  [ 2.55326419]\n",
            "  [ 1.9372769 ]\n",
            "  [ 0.34013728]\n",
            "  [ 0.3588915 ]\n",
            "  [-0.42979633]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IdB9CszhoA1R",
        "colab_type": "text"
      },
      "source": [
        "#### Vector to matrix (broadcasted)\n",
        "\n",
        "This function takes in a stack of vectors and returns the corresponding matrix forms of those vectors."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wud5dy0CoBFR",
        "colab_type": "code",
        "outputId": "9e76aa46-6bcd-465f-a847-645ba028cb09",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 545
        }
      },
      "source": [
        "def vec2mat_broadcasted(vec):\n",
        "  \n",
        "  # Return matrix\n",
        "  return(vec.reshape(vec.shape[0], np.int64(np.sqrt(vec.shape[1])),np.int64(np.sqrt(vec.shape[1]))).transpose(0,2,1))\n",
        "\n",
        "# Example\n",
        "vec = np.random.randn(4,4,1)\n",
        "mat = vec2mat_broadcasted(vec)\n",
        "print(vec)\n",
        "print(mat)"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[[ 0.00950134]\n",
            "  [-0.39246823]\n",
            "  [ 0.64527246]\n",
            "  [-0.67533304]]\n",
            "\n",
            " [[-0.68394147]\n",
            "  [-0.21642699]\n",
            "  [ 0.09936041]\n",
            "  [ 0.13129012]]\n",
            "\n",
            " [[ 0.23868394]\n",
            "  [ 0.21281357]\n",
            "  [-1.7283013 ]\n",
            "  [-0.65900032]]\n",
            "\n",
            " [[-1.26149455]\n",
            "  [ 1.49131841]\n",
            "  [-0.77136993]\n",
            "  [-0.69883302]]]\n",
            "[[[ 0.00950134  0.64527246]\n",
            "  [-0.39246823 -0.67533304]]\n",
            "\n",
            " [[-0.68394147  0.09936041]\n",
            "  [-0.21642699  0.13129012]]\n",
            "\n",
            " [[ 0.23868394 -1.7283013 ]\n",
            "  [ 0.21281357 -0.65900032]]\n",
            "\n",
            " [[-1.26149455 -0.77136993]\n",
            "  [ 1.49131841 -0.69883302]]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j75PTwshvngn",
        "colab_type": "text"
      },
      "source": [
        "#### Matrix to half vector (broadcasted)\n",
        "\n",
        "This function takes in a stack of matrices and returns the corresponding vector forms of the lower halves of those matrices."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E31shCYgvnx8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def mat2vech_broadcasted(matrix):\n",
        "  \n",
        "  # Number of voxels, nv\n",
        "  nv = matrix.shape[0]\n",
        "  \n",
        "  # Get lower triangular indices\n",
        "  rowinds, colinds = np.tril_indices(matrix.shape[1]) \n",
        "  \n",
        "  # Number of covariance parameters, nc\n",
        "  nc = len(rowinds)\n",
        "  \n",
        "  # They're in the wrong order so we need to order them\n",
        "  # To do this we first hash them\n",
        "  indhash = colinds*matrix.shape[1]+rowinds\n",
        "  \n",
        "  # Sort permutation\n",
        "  perm=np.argsort(indhash)\n",
        "  \n",
        "  # Return vectorised half-matrix\n",
        "  return(matrix[:,rowinds[perm],colinds[perm]].reshape((nv,nc,1)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LWzmL4Bp1QiX",
        "colab_type": "text"
      },
      "source": [
        "#### Half vector to matrix (broadcasted)\n",
        "\n",
        "This function takes in a stack of vectors and returns the corresponding matrix forms treating the elements of the vectors as the elements of lower halves of those matrices."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IV2UNK8L1Qxz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def vech2mat_broadcasted(vech):\n",
        "  \n",
        "  # Number of voxels\n",
        "  nv = vech.shape[0]\n",
        "  \n",
        "  # dimension of matrix\n",
        "  n = np.int64((-1+np.sqrt(1+8*vech.shape[1]))/2)\n",
        "  matrix = np.zeros((nv,n,n))\n",
        "  \n",
        "  # Get lower triangular indices\n",
        "  rowinds, colinds = np.tril_indices(n)\n",
        "  \n",
        "  # They're in the wrong order so we need to order them\n",
        "  # To do this we first hash them\n",
        "  indhash = colinds*n+rowinds\n",
        "  \n",
        "  # Sort permutation\n",
        "  perm=np.argsort(indhash)\n",
        "  \n",
        "  # Assign values to lower half\n",
        "  matrix[:,rowinds[perm],colinds[perm]] = vech.reshape(vech.shape[0],vech.shape[1])\n",
        "  \n",
        "  # Assign values to upper half\n",
        "  matrix[:,colinds[perm],rowinds[perm]] = vech.reshape(vech.shape[0],vech.shape[1])\n",
        "  \n",
        "  # Return vectorised half-matrix\n",
        "  return(matrix)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vXZTnBHRkOm7",
        "colab_type": "text"
      },
      "source": [
        "#### Initial D_k (broadcasted)\n",
        "\n",
        "The function below returns an initial estimate for the Random Effects Variance matrix for the $k^{th}$ grouping factor, $D_k$. The estimator used is an adaption of the suggested estimator in Demidenko (2012) and is given by:\n",
        "\n",
        "$$vec(\\hat{D}_{k})=\\bigg[\\sum_{j=1}^{l_k}(Z_{(k,j)}^TZ_{(k,j)}) \\otimes (Z_{(k,j)}^TZ_{(k,j)})\\bigg]^{-1}vec\\bigg(\\sum_{j=1}^{l_k}[\\hat{\\sigma}^{-2}_{OLS}Z_{(k,j)}^Tee^TZ_{(k,j)} - Z_{(k,j)}^TZ_{(k,j)}]\\bigg)$$\n",
        "\n",
        "Or:\n",
        "\n",
        "$$\\hat{D}_{k}=matrix\\bigg(\\bigg[\\sum_{j=1}^{l_k}(Z_{(k,j)}^TZ_{(k,j)}) \\otimes (Z_{(k,j)}^TZ_{(k,j)})\\bigg]^{-1}vec\\bigg(\\sum_{j=1}^{l_k}[\\hat{\\sigma}^{-2}_{OLS}Z_{(k,j)}^Tee^TZ_{(k,j)} - Z_{(k,j)}^TZ_{(k,j)}]\\bigg)\\bigg)$$\n",
        "\n",
        "----\n",
        "\n",
        "This function takes the following inputs:\n",
        "\n",
        "----\n",
        "\n",
        " - `k`: The grouping factor we wish to estimate $D$ for ($k$ in the above notation)\n",
        " - `lk`: The number of levels belonging to grouping factor $k$ ($l_k$ in the above notation).\n",
        " - `ZtZ`: The $Z$ matrix transposed and then multiplied by itself ($Z^TZ$ in the above notation).\n",
        " - `Zte`: The $Z$ matrix transposed and then multiplied by the OLS residuals ($Z^Te=Z^T(Y-X\\beta)$ in the above notation).\n",
        " - `sigma2`: The OLS estimate of $\\sigma^2$ ($\\hat{\\sigma}^2_{OLS}$ in the above notation).\n",
        " \n",
        "----\n",
        "\n",
        "It returns as outputs:\n",
        "\n",
        "----\n",
        "\n",
        "- `Dkest`: The inital estimate of $D_k$ ($\\hat{D}_k$ in the above notation).\n",
        "\n",
        "\n",
        "###CHECK DERIVATION"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JSNOV2V5kO36",
        "colab_type": "code",
        "outputId": "4929a47a-4366-426d-aa64-47e80fc6c011",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246
        }
      },
      "source": [
        "def initDk_broadcasted(k, lk, ZtZ, Zte, sigma2):\n",
        "  \n",
        "  # Initalize D to zeros\n",
        "  invSig2ZteetZminusZtZ = np.zeros((Zte.shape[0],nparams[k],nparams[k]))\n",
        "  \n",
        "  # For each level j we need to add a term\n",
        "  for j in np.arange(nlevels[k]):\n",
        "    \n",
        "    Ikj = faclev_indices(k, j, nlevels, nparams)\n",
        "\n",
        "    # Work out Z_(k, j)'Z_(k, j)\n",
        "    ZkjtZkj = ZtZ[np.ix_(np.arange(ZtZ.shape[0]),Ikj,Ikj)]\n",
        "    \n",
        "    # Work out Z_(k,j)'e\n",
        "    Zkjte = Zte[:, Ikj,:]\n",
        "    \n",
        "    if j==0:\n",
        "      \n",
        "      # Add first Z_(k,j)'Z_(k,j) kron Z_(k,j)'Z_(k,j)\n",
        "      ZtZkronZtZ = kron_broadcasted(ZkjtZkj,ZkjtZkj.transpose(0,2,1))\n",
        "      \n",
        "      # Add first \\sigma^{-2}Z'ee'Z - Z_(k,j)'Z_(k,j)\n",
        "      invSig2ZteetZminusZtZ = np.einsum('i,ijk->ijk',1/sigma2,(Zkjte @ Zkjte.transpose(0,2,1))) - ZkjtZkj\n",
        "      \n",
        "    else:\n",
        "      \n",
        "      # Add next Z_(k,j)'Z_(k,j) kron Z_(k,j)'Z_(k,j)\n",
        "      ZtZkronZtZ = ZtZkronZtZ + kron_broadcasted(ZkjtZkj,ZkjtZkj.transpose(0,2,1))\n",
        "      \n",
        "      # Add next \\sigma^{-2}Z'ee'Z - Z_(k,j)'Z_(k,j)\n",
        "      invSig2ZteetZminusZtZ = invSig2ZteetZminusZtZ + np.einsum('i,ijk->ijk',1/sigma2,(Zkjte @ Zkjte.transpose(0,2,1))) - ZkjtZkj\n",
        "      \n",
        "  # Work out the final term.\n",
        "  Dkest = vec2mat_broadcasted(np.linalg.inv(ZtZkronZtZ) @ mat2vec_broadcasted(invSig2ZteetZminusZtZ)) \n",
        "  \n",
        "  \n",
        "  return(Dkest)\n",
        "\n",
        "Zte = ZtY-ZtX @ np.random.randn(ZtY.shape[0],ZtX.shape[2],1)\n",
        "\n",
        "t1 = time.time()\n",
        "initDk_broadcasted(0, nlevels[0], ZtZ, Zte , np.ones(Zte.shape[0]))\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)\n",
        "\n",
        "t1 = time.time()\n",
        "initDk_broadcasted(0, nlevels[0], ZtZ, Zte , np.ones(Zte.shape[0]))\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)\n",
        "\n",
        "print(initDk_broadcasted(0, nlevels[0], ZtZ, Zte , np.ones(Zte.shape[0]))[8,:,:])\n",
        "\n",
        "t1 = time.time()\n",
        "print(initDk(0, nlevels[0], ZtZ[0,:,:], Zte[8,:,:], 1))\n",
        "t2 = time.time()\n",
        "\n",
        "print(t2-t1)"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.41564035415649414\n",
            "0.1696946620941162\n",
            "[[ 5.36384127 -0.64222882 -3.34108776 -0.38947429 -0.39417497]\n",
            " [-0.64222882  4.12494155 -0.05795925 -1.36253347  0.40706079]\n",
            " [-3.34108776 -0.05795925  5.35791004  0.57703582 -2.33859488]\n",
            " [-0.38947429 -1.36253347  0.57703582  1.35720552 -0.35707717]\n",
            " [-0.39417497  0.40706079 -2.33859488 -0.35707717  5.31388962]]\n",
            "[[ 5.36384127 -0.64222882 -3.34108776 -0.38947429 -0.39417497]\n",
            " [-0.64222882  4.12494155 -0.05795925 -1.36253347  0.40706079]\n",
            " [-3.34108776 -0.05795925  5.35791004  0.57703582 -2.33859488]\n",
            " [-0.38947429 -1.36253347  0.57703582  1.35720552 -0.35707717]\n",
            " [-0.39417497  0.40706079 -2.33859488 -0.35707717  5.31388962]]\n",
            "0.0030145645141601562\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6tddRYrhtNWG",
        "colab_type": "text"
      },
      "source": [
        "#### Make D non-negative definite (Broadcasted)\n",
        "\n",
        "This function projects a stack of matrices into the space of non-negative definite matrices."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0en3ealntNes",
        "colab_type": "code",
        "outputId": "f44d5c36-b9cf-46b8-caa8-5ef405fd5b7f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        }
      },
      "source": [
        "def makeDnnd_broadcasted(D):\n",
        "  \n",
        "  # Check if we have negative eigenvalues\n",
        "  if not np.all(np.linalg.eigvals(D)>0):\n",
        "  \n",
        "    # If we have negative eigenvalues\n",
        "    eigvals,eigvecs = np.linalg.eigh(D)\n",
        "    \n",
        "    # Work out elementwise max of lambda and 0\n",
        "    lamplus = np.zeros((eigvals.shape[0], eigvals.shape[1], eigvals.shape[1]))\n",
        "    diag = np.arange(eigvals.shape[1])\n",
        "    lamplus[:, diag, diag] = np.maximum(eigvals,0)\n",
        "    \n",
        "    # Work out D+\n",
        "    D_nnd = eigvecs @ lamplus @ np.linalg.inv(eigvecs)\n",
        "    \n",
        "  else:\n",
        "    \n",
        "    # D is already non-negative in this case\n",
        "    D_nnd = D\n",
        "    \n",
        "  return(D_nnd)\n",
        "  \n",
        "D = np.random.randn(100000,3,3)\n",
        "t1 = time.time()\n",
        "print(makeDnnd_broadcasted(D)[10,:,:])\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "t1 = time.time()\n",
        "print(makeDnnd(D[10,:,:]))\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "\n",
        "### COULD BE IMPROVED SPEED WISE"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 0.05000751  0.06102691 -0.07996357]\n",
            " [ 0.06102691  0.07447449 -0.09758393]\n",
            " [-0.07996357 -0.09758393  0.12786422]]\n",
            "1.0307631492614746\n",
            "[[ 0.05000751  0.06102691 -0.07996357]\n",
            " [ 0.06102691  0.07447449 -0.09758393]\n",
            " [-0.07996357 -0.09758393  0.12786422]]\n",
            "0.001069784164428711\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i2RBNITFOdJs",
        "colab_type": "text"
      },
      "source": [
        "### Get D from dict (broadcasted)\n",
        "\n",
        "This function takes in a dictionary, `Ddict`, in which entry `k` is a stack of the $k^{th}$ diagonal block for every voxel."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mcziKRJvOdUJ",
        "colab_type": "code",
        "outputId": "12f4c4a4-5266-442d-b131-bc7608a12078",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "def getDfromDict_broadcasted(Ddict, nparams, nlevels):\n",
        "  \n",
        "  # Get number of voxels\n",
        "  nv = Ddict[0].shape[0]\n",
        "  \n",
        "  # Work out indices (there is one block of D per level)\n",
        "  inds = np.zeros(np.sum(nlevels)+1)\n",
        "  counter = 0\n",
        "  for k in np.arange(len(nparams)):\n",
        "    for j in np.arange(nlevels[k]):\n",
        "      inds[counter] = np.concatenate((np.array([0]), np.cumsum(nlevels*nparams)))[k] + nparams[k]*j\n",
        "      counter = counter + 1\n",
        "      \n",
        "  \n",
        "  # Last index will be missing so add it\n",
        "  inds[len(inds)-1]=inds[len(inds)-2]+nparams[-1]\n",
        "  \n",
        "  # Make sure indices are ints\n",
        "  inds = np.int64(inds)\n",
        "  \n",
        "  # Initial D\n",
        "  D = np.zeros((nv,np.sum(nparams*nlevels),np.sum(nparams*nlevels)))\n",
        "\n",
        "  counter = 0\n",
        "  for k in np.arange(len(nparams)):\n",
        "    for j in np.arange(nlevels[k]):\n",
        "\n",
        "      D[:, inds[counter]:inds[counter+1], inds[counter]:inds[counter+1]] = Ddict[k]\n",
        "      counter = counter + 1\n",
        "  \n",
        "  return(D)\n",
        "  \n",
        "# Example dict\n",
        "nparams_tmp = np.array([10,2,3])\n",
        "nlevels_tmp = np.array([30,22,4])\n",
        "\n",
        "Ddict_tmp = dict()\n",
        "for i in np.arange(len(nparams_tmp)):\n",
        "  \n",
        "  Ddict_tmp[i] = np.random.randn(2000,nparams_tmp[i],nparams_tmp[i])\n",
        "\n",
        "t1 = time.time()\n",
        "D = getDfromDict_broadcasted(Ddict_tmp, nparams_tmp, nlevels_tmp)\n",
        "t2 = time.time()\n",
        "print(t2-t1)"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3.0461831092834473\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kn997hc1hDGK",
        "colab_type": "text"
      },
      "source": [
        "#### Force Symmetric (Broadcasted)\n",
        "\n",
        "This function forces a stack of matrices to be symmetric."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BphjFMdxhDWt",
        "colab_type": "code",
        "outputId": "2a8d276a-632e-4817-da03-23b5b5737476",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "# Developer NOTE: DO NOT USE NUMBA ON THIS FUNCTION - NUMBA BUGGERS IT UP FOR UNKNOWN REASONS\n",
        "def forceSym_broadcasted(x):\n",
        "  \n",
        "  # Force it to be symmetric\n",
        "  return((x+x.transpose((0,2,1)))/2)\n",
        "\n",
        "  \n",
        "print(ZtZ.shape)\n",
        "t1 = time.time()\n",
        "ZtZ = forceSym_broadcasted(forceSym_broadcasted(ZtZ))\n",
        "print(time.time()-t1)\n",
        "\n",
        "print(ZtZ.shape)\n",
        "t1 = time.time()\n",
        "ZtZ = forceSym_broadcasted(forceSym_broadcasted(ZtZ))\n",
        "print(time.time()-t1)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 70, 70)\n",
            "0.00017309188842773438\n",
            "(1, 70, 70)\n",
            "0.00013685226440429688\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T01OEeP9GT65",
        "colab_type": "text"
      },
      "source": [
        "#### Derivative of $\\frac{\\delta l}{\\delta \\beta}$ (broadcasted)\n",
        "\n",
        "This function returns the derivative of $l$ with respect to $\\beta$ for all voxels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mwCgeYh0GUHO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_dldB_broadcasted(sigma2, Xte, XtZ, DinvIplusZtZD, Zte):\n",
        "  \n",
        "  # Work out the derivative (Note: we leave everything as 3D for ease of future computation)\n",
        "  deriv = np.einsum('i,ijk->ijk',1/sigma2, (Xte - (XtZ @ DinvIplusZtZD @ Zte)))\n",
        "                    \n",
        "  # Return the derivative\n",
        "  return(deriv)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q2wffHXtIgd2",
        "colab_type": "text"
      },
      "source": [
        "#### Derivative of $\\frac{\\delta l}{\\delta \\sigma^2}$ (broadcasted)\n",
        "\n",
        "This function returns the derivative of $l$ with respect to $\\sigma^2$ for all voxels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4WUjR0_oIjJv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_dldsigma2_broadcasted(n, ete, Zte, sigma2, DinvIplusZtZD):\n",
        "  \n",
        "  # Get e'(I+ZDZ')^(-1)e=e'e-e'ZD(I+Z'ZD)^(-1)Z'e\n",
        "  etinvIplusZtDZe = ete - forceSym_broadcasted(Zte.transpose((0,2,1)) @ DinvIplusZtZD @ Zte)\n",
        "  \n",
        "  # Get the derivative\n",
        "  deriv = -n/(2*sigma2) + np.einsum('i,ijk->ijk',1/(2*(sigma2**2)), etinvIplusZtDZe).reshape(sigma2.shape[0])\n",
        "  \n",
        "  return(deriv)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JOn3facWIgoz",
        "colab_type": "text"
      },
      "source": [
        "#### Derivative of $\\frac{\\delta l}{\\delta D_k}$ (broadcasted)\n",
        "\n",
        "This function returns the derivative of $l$ with respect to $D_k$ for all voxels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bWRg9WRpMwwc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_dldDk_broadcasted(k, nlevels, nparams, ZtZ, Zte, sigma2, DinvIplusZtZD):\n",
        "\n",
        "  # Number of voxels\n",
        "  nv = Zte.shape[0]\n",
        "  \n",
        "  # Initalize the derivative to zeros\n",
        "  dldDk = np.zeros((nv, nparams[k],nparams[k]))\n",
        "  \n",
        "  # For each level j we need to add a term\n",
        "  for j in np.arange(nlevels[k]):\n",
        "\n",
        "    # Get the indices for the kth factor jth level\n",
        "    Ikj = faclev_indices(k, j, nlevels, nparams)\n",
        "    \n",
        "    # Get (the kj^th columns of Z)^T multiplied by Z\n",
        "    Z_kjtZ = ZtZ[:,Ikj,:]\n",
        "    Z_kjte = Zte[:,Ikj,:]\n",
        "    \n",
        "    # Get the first term of the derivative\n",
        "    Z_kjtVinve = Z_kjte - (Z_kjtZ @ DinvIplusZtZD @ Zte)\n",
        "    firstterm = np.einsum('i,ijk->ijk',1/sigma2,forceSym_broadcasted(Z_kjtVinve @ Z_kjtVinve.transpose((0,2,1))))\n",
        "    \n",
        "    # Get (the kj^th columns of Z)^T multiplied by (the kj^th columns of Z)\n",
        "    Z_kjtZ_kj = ZtZ[np.ix_(np.arange(ZtZ.shape[0]),Ikj,Ikj)]\n",
        "    secondterm = forceSym_broadcasted(Z_kjtZ_kj) - forceSym_broadcasted(Z_kjtZ @ DinvIplusZtZD @ Z_kjtZ.transpose((0,2,1)))\n",
        "    \n",
        "    if j == 0:\n",
        "      \n",
        "      # Start a running sum over j\n",
        "      dldDk = firstterm - secondterm\n",
        "      \n",
        "    else:\n",
        "    \n",
        "      # Add these to the running sum\n",
        "      dldDk = dldDk + firstterm - secondterm\n",
        "    \n",
        "  # Halve the sum (the coefficient of a half was not included in the above)\n",
        "  dldDk = forceSym_broadcasted(dldDk/2)\n",
        "\n",
        "  # Store it in the dictionary\n",
        "  return(dldDk)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zh7nHo0KRHk-",
        "colab_type": "text"
      },
      "source": [
        "#### Covariance of derivative of $\\frac{\\delta l}{\\delta \\beta}$ (broadcasted)\n",
        "\n",
        "This function returns the covariance of the derivative of $l$ with respect to $\\beta$ for all voxels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BWIE2Oy4RHxD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_covdldbeta_broadcasted(XtZ, XtX, ZtZ, DinvIplusZtZD, sigma2):\n",
        "  \n",
        "  # Get the covariance of the derivative\n",
        "  covderiv = np.einsum('i,ijk->ijk',1/sigma2,(XtX - forceSym_broadcasted(XtZ @ DinvIplusZtZD @ XtZ.transpose((0,2,1)))))\n",
        "  \n",
        "  # Return the covariance of the derivative\n",
        "  return(covderiv)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y-8VOZR5ULKX",
        "colab_type": "text"
      },
      "source": [
        "#### Covariance of derivative of $\\frac{\\delta l}{\\delta \\sigma^2}$ and $\\frac{\\delta l}{\\delta D_k}$ (broadcasted)\n",
        "\n",
        "This function returns the covariance of the derivative of $l$ with respect to $\\sigma^2$ and $D_k$ for all voxels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dW40iTC-ULV5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_covdldDkdsigma2_broadcasted(k, sigma2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict):\n",
        "  \n",
        "  # Number of voxels\n",
        "  nv = DinvIplusZtZD.shape[0]\n",
        "  \n",
        "  # Sum of R_(k, j) over j\n",
        "  RkSum = np.zeros((nv,nparams[k],nparams[k]))\n",
        "\n",
        "  for j in np.arange(nlevels[k]):\n",
        "\n",
        "    # Get the indices for the kth factor jth level\n",
        "    Ikj = faclev_indices(k, j, nlevels, nparams)\n",
        "\n",
        "    # Work out R_(k, j)\n",
        "    Rkj = ZtZ[np.ix_(np.arange(ZtZ.shape[0]),Ikj,Ikj)] - forceSym_broadcasted(ZtZ[:,Ikj,:] @ DinvIplusZtZD @ ZtZ[:,:,Ikj])\n",
        "    \n",
        "    # Add together\n",
        "    RkSum = RkSum + Rkj\n",
        "\n",
        "  # Multiply by duplication matrices and save\n",
        "  covdldDdldsigma2 = np.einsum('i,ijk->ijk', 1/(2*sigma2), invDupMatdict[k] @ mat2vec_broadcasted(RkSum))\n",
        "  \n",
        "  return(covdldDdldsigma2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6d1GYRfPqLs3",
        "colab_type": "text"
      },
      "source": [
        "#### Covariance of derivative of $\\frac{\\delta l}{\\delta D_{k_1}}$ and $\\frac{\\delta l}{\\delta D_{k_2}}$ (broadcasted)\n",
        "\n",
        "This function returns the covariance of the derivative of $l$ with respect to $D_{k_1}$ and $D_{k_2}$ for all voxels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QhknnB8aqL44",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_covdldDk1Dk2_broadcasted(k1, k2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict):\n",
        "  \n",
        "  # Sum of R_(k1, k2, i, j) kron R_(k1, k2, i, j) over i and j \n",
        "  for i in np.arange(nlevels[k1]):\n",
        "\n",
        "    for j in np.arange(nlevels[k2]):\n",
        "      \n",
        "      # Get the indices for the k1th factor jth level\n",
        "      Ik1i = faclev_indices(k1, i, nlevels, nparams)\n",
        "      Ik2j = faclev_indices(k2, j, nlevels, nparams)\n",
        "      \n",
        "      # Work out R_(k1, k2, i, j)\n",
        "      Rk1k2ij = ZtZ[np.ix_(np.arange(ZtZ.shape[0]),Ik1i,Ik2j)] - (ZtZ[:,Ik1i,:] @ DinvIplusZtZD @ ZtZ[:,:,Ik2j])\n",
        "      \n",
        "      # Work out Rk1k2ij kron Rk1k2ij\n",
        "      RkRt = kron_broadcasted(Rk1k2ij,Rk1k2ij)\n",
        "      \n",
        "      # Add together\n",
        "      if (i == 0) and (j == 0):\n",
        "      \n",
        "        RkRtSum = RkRt\n",
        "      \n",
        "      else:\n",
        "        \n",
        "        RkRtSum = RkRtSum + RkRt\n",
        "    \n",
        "  # Multiply by duplication matrices and save\n",
        "  covdldDk1dldk2 = 1/2 * invDupMatdict[k1] @ RkRtSum @ invDupMatdict[k2].transpose()\n",
        "  \n",
        "  # Return the result\n",
        "  return(covdldDk1dldk2)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "idaEKRat3dsM",
        "colab_type": "text"
      },
      "source": [
        "#### Log likelihood (broadcasted)\n",
        "\n",
        "This function returns the loglikelihood for all voxels."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m6OSs_pj3d55",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def llh_broadcasted(n, ZtZ, Zte, ete, sigma2, DinvIplusZtZD,D):\n",
        "  \n",
        "  # Work out -1/2(nln(sigma^2) + ln|I+Z'ZD|)\n",
        "  firstterm = -0.5*(n*np.log(sigma2) + np.log(np.linalg.det(np.eye(ZtZ.shape[1]) + ZtZ @ D)))\n",
        "                    \n",
        "  # Work out sigma^(-2)*(e'e - e'ZD(I+Z'ZD)^(-1)Z'e)\n",
        "  secondterm = -0.5*np.einsum('i,ijk->ijk',(1/sigma2),(ete - forceSym_broadcasted(Zte.transpose((0,2,1)) @ DinvIplusZtZD @ Zte)))\n",
        "  \n",
        "  # Work out the log likelihood\n",
        "  llh = firstterm + secondterm\n",
        "  \n",
        "  # Return result\n",
        "  return(llh)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x72ZuXyheljG",
        "colab_type": "text"
      },
      "source": [
        "### Broadcasted FS\n",
        "\n",
        "The below function performs FS for a NIFTI."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qJjgNbXjeszs",
        "colab_type": "code",
        "outputId": "d8dcd570-54f6-42d0-b618-3c199aed2504",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 614
        }
      },
      "source": [
        "def FS_broadcasted(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nparams, tol,n):\n",
        "  \n",
        "  #-------------------------------------------------------------------------------\n",
        "  # Testing - one voxel\n",
        "  \n",
        "  # one voxel index\n",
        "  ovi = 10\n",
        "  \n",
        "  XtX_ov = XtX[0,:,:] \n",
        "  XtY_ov = XtY[ovi,:,:]\n",
        "  ZtX_ov = ZtX[0,:,:]\n",
        "  ZtY_ov = ZtY[ovi,:,:] \n",
        "  ZtZ_ov = ZtZ[0,:,:] \n",
        "  XtZ_ov = XtZ[0,:,:] \n",
        "  YtZ_ov = YtZ[ovi,:,:] \n",
        "  YtY_ov = YtY[ovi,:,:] \n",
        "  YtX_ov = YtX[ovi,:,:]\n",
        "  \n",
        "  \n",
        "  t1_total = time.time()\n",
        "  t1 = time.time()\n",
        "  \n",
        "  # Useful scalars\n",
        "  # ------------------------------------------------------------------------------\n",
        "\n",
        "  # Number of factors, r\n",
        "  r = len(nlevels)\n",
        "\n",
        "  # Number of random effects, q\n",
        "  q = np.sum(np.dot(nparams,nlevels))\n",
        "\n",
        "  # Number of fixed effects, p\n",
        "  p = XtX.shape[1]\n",
        "\n",
        "  # Number of voxels, nv\n",
        "  nv = XtY.shape[0]\n",
        "  \n",
        "  # Initial estimates\n",
        "  # ------------------------------------------------------------------------------\n",
        "\n",
        "  # Inital beta\n",
        "  beta = initBeta_broadcasted(XtX, XtY)\n",
        "  beta_ov = initBeta(XtX_ov, XtY_ov)\n",
        "  #print('beta', beta[10,:,:], beta_ov)\n",
        "  \n",
        "  # Work out e'e\n",
        "  ete = ssr_broadcasted(YtX, YtY, XtX, beta)\n",
        "  ete_ov = ssr(YtX_ov, YtY_ov, XtX_ov, beta_ov)\n",
        "  #print('ete', ete[10], ete_ov)\n",
        "  \n",
        "  # Initial sigma2\n",
        "  sigma2 = initSigma2_broadcasted(ete, n)\n",
        "  sigma2_ov = initSigma2(ete_ov, n)\n",
        "  #print('sigma2', sigma2[10], sigma2_ov)\n",
        "\n",
        "  Zte = ZtY - (ZtX @ beta) \n",
        "  Zte_ov = ZtY_ov - (ZtX_ov @ beta_ov) \n",
        "\n",
        "  # Inital D\n",
        "  # Dictionary version\n",
        "  Ddict = dict()\n",
        "  for k in np.arange(len(nparams)):\n",
        "\n",
        "    Ddict[k] = makeDnnd_broadcasted(initDk_broadcasted(k, nlevels[k], ZtZ, Zte, sigma2))\n",
        "  \n",
        "  Ddict_ov = dict()\n",
        "  for k in np.arange(len(nparams)):\n",
        "\n",
        "    Ddict_ov[k] = makeDnnd(initDk(k, nlevels[k], ZtZ_ov, Zte_ov, sigma2_ov))\n",
        "    #print('D' + str(k), Ddict[k][10,:,:], Ddict_ov[k])\n",
        "  \n",
        "  # Full version of D\n",
        "  D = getDfromDict_broadcasted(Ddict, nparams, nlevels)\n",
        "    \n",
        "  # Matrix version\n",
        "  D_ov = np.array([])\n",
        "  for i in np.arange(len(nparams)):\n",
        "\n",
        "    for j in np.arange(nlevels[i]):\n",
        "\n",
        "      if i == 0 and j == 0:\n",
        "\n",
        "        D_ov = Ddict_ov[i]\n",
        "\n",
        "      else:\n",
        "\n",
        "        D_ov = scipy.linalg.block_diag(D_ov, Ddict_ov[i])\n",
        "  \n",
        "  #print('D', D[10,:,:], D_ov)\n",
        "  \n",
        "  # Duplication matrices\n",
        "  # ------------------------------------------------------------------------------\n",
        "  invDupMatdict = dict()\n",
        "  for i in np.arange(len(nparams)):\n",
        "\n",
        "    invDupMatdict[i] = np.asarray(invDupMat(nparams[i]).todense())\n",
        "    \n",
        "    #print('invDupMat ' + str(i), invDupMatdict[i])\n",
        "  \n",
        "  # Index variables\n",
        "  # ------------------------------------------------------------------------------\n",
        "  # Work out the total number of paramateres\n",
        "  tnp = np.int32(p + 1 + np.sum(nparams*(nparams+1)/2))\n",
        "\n",
        "  # Indices for submatrics corresponding to Dks\n",
        "  FishIndsDk = np.int32(np.cumsum(nparams*(nparams+1)/2) + p + 1)\n",
        "  FishIndsDk = np.insert(FishIndsDk,0,p+1)\n",
        "\n",
        "  Zte = ZtY - (ZtX @ beta)\n",
        "  Zte_ov = ZtY_ov - (ZtX_ov @ beta_ov)\n",
        "  \n",
        "  # Inverse of (I+Z'ZD) multiplied by D\n",
        "  IplusZtZD = np.eye(q) + ZtZ @ D\n",
        "  DinvIplusZtZD =  forceSym_broadcasted(D @ np.linalg.inv(IplusZtZD)) # Currently using numpy... might be able to improve on this inversion\n",
        "  \n",
        "  \n",
        "  # Inverse of (I+Z'ZD) multiplied by D\n",
        "  IplusZtZD_ov = np.eye(q) + ZtZ_ov @ D_ov\n",
        "  DinvIplusZtZD_ov = forceSym(D_ov @ scipy.sparse.linalg.inv(scipy.sparse.csc_matrix(IplusZtZD_ov)))\n",
        "  \n",
        "  #print('D(I + ZtZ)^(-1)', DinvIplusZtZD[10,:,:], DinvIplusZtZD_ov)\n",
        "\n",
        "  \n",
        "  # Step size lambda\n",
        "  lam = np.ones(XtY.shape[0])\n",
        "  lam_ov = 1\n",
        "\n",
        "  # Initial log likelihoods\n",
        "  llhprev = -10*np.ones(XtY.shape[0])\n",
        "  llhcurr = 10*np.ones(XtY.shape[0])\n",
        "  \n",
        "  t2 = time.time()\n",
        "  print('Setup time: ', t2-t1)\n",
        "  \n",
        "  nit=0\n",
        "  examplellh=np.zeros(100)\n",
        "  while np.any(np.abs(llhprev-llhcurr)>tol):\n",
        "    \n",
        "    t1 = time.time()\n",
        "    # Change current likelihood to previous\n",
        "    llhprev = llhcurr\n",
        "\n",
        "    # Matrices needed later by many calculations:\n",
        "    # ----------------------------------------------------------------------------\n",
        "    # X transpose e and Z transpose e\n",
        "    Xte = XtY - (XtX @ beta)\n",
        "    Zte = ZtY - (ZtX @ beta)\n",
        "    \n",
        "    Xte_ov = XtY_ov - (XtX_ov @ beta_ov)\n",
        "    Zte_ov = ZtY_ov - (ZtX_ov @ beta_ov)\n",
        "    \n",
        "    # Inverse of (I+Z'ZD) multiplied by D\n",
        "    IplusZtZD = np.eye(q) + (ZtZ @ D)\n",
        "    DinvIplusZtZD = forceSym_broadcasted(D @ np.linalg.inv(IplusZtZD)) # Currently using numpy... might be able to improve on this inversion\n",
        "    \n",
        "    IplusZtZD_ov = np.eye(q) + (ZtZ_ov @ D_ov)\n",
        "    DinvIplusZtZD_ov = forceSym(D_ov @ scipy.sparse.linalg.inv(scipy.sparse.csc_matrix(IplusZtZD_ov)))\n",
        "    #print('D(I + ZtZ)^(-1)', DinvIplusZtZD[10,:,:], DinvIplusZtZD_ov)\n",
        "\n",
        "    # Sum of squared residuals\n",
        "    ete = ssr_broadcasted(YtX, YtY, XtX, beta)\n",
        "    ete_ov = ssr(YtX_ov, YtY_ov, XtX_ov, beta_ov)\n",
        "    #print('ete', ete[10,:,:], ete_ov)\n",
        "    \n",
        "    # Derivatives\n",
        "    # ----------------------------------------------------------------------------\n",
        "\n",
        "    # Derivative wrt beta\n",
        "    dldB = get_dldB_broadcasted(sigma2, Xte, XtZ, DinvIplusZtZD, Zte)\n",
        "    dldB_ov = get_dldB(sigma2_ov, Xte_ov, XtZ_ov, DinvIplusZtZD_ov, Zte_ov)\n",
        "    #print('dldB', dldB[10,:,:], dldB_ov)\n",
        "    \n",
        "    # Derivative wrt sigma^2\n",
        "    dldsigma2 = get_dldsigma2_broadcasted(n, ete, Zte, sigma2, DinvIplusZtZD) \n",
        "    dldsigma2_ov = get_dldsigma2(n, ete_ov, Zte_ov, sigma2_ov, DinvIplusZtZD_ov) \n",
        "    #print('dldsigma2', dldsigma2[10], dldsigma2_ov)\n",
        "    \n",
        "    # For each factor, factor k, work out dl/dD_k\n",
        "    dldDdict = dict()\n",
        "    for k in np.arange(len(nparams)):\n",
        "      # Store it in the dictionary\n",
        "      dldDdict[k] = get_dldDk_broadcasted(k, nlevels, nparams, ZtZ, Zte, sigma2, DinvIplusZtZD)\n",
        "\n",
        "    dldDdict_ov = dict()\n",
        "    for k in np.arange(len(nparams)):\n",
        "      # Store it in the dictionary\n",
        "      dldDdict_ov[k] = get_dldDk(k, nlevels, nparams, ZtZ_ov, Zte_ov, sigma2_ov, DinvIplusZtZD_ov)\n",
        "      #print('dldDdict '+str(k),dldDdict[k][10,:,:],dldDdict_ov[k])\n",
        "    \n",
        "    # Covariances\n",
        "    # ----------------------------------------------------------------------------\n",
        "\n",
        "    # Construct the Fisher Information matrix\n",
        "    # ----------------------------------------------------------------------------\n",
        "    FisherInfoMat = np.zeros((nv,tnp,tnp))\n",
        "    FisherInfoMat_ov = np.zeros((tnp,tnp))\n",
        "    \n",
        "    # Covariance of dl/dsigma2\n",
        "    covdldsigma2 = n/(2*(sigma2**2))\n",
        "    covdldsigma2_ov = n/(2*(sigma2_ov**2))\n",
        "    #print('covdldsigma2', covdldsigma2[10],covdldsigma2_ov)\n",
        "    \n",
        "    # Add dl/dsigma2 covariance\n",
        "    FisherInfoMat[:,p,p] = covdldsigma2\n",
        "    FisherInfoMat_ov[p,p] = covdldsigma2_ov\n",
        "    \n",
        "    # Add dl/dbeta covariance\n",
        "    covdldB = get_covdldbeta_broadcasted(XtZ, XtX, ZtZ, DinvIplusZtZD, sigma2)\n",
        "    covdldB_ov = get_covdldbeta(XtZ_ov, XtX_ov, ZtZ_ov, DinvIplusZtZD_ov, sigma2_ov)\n",
        "    #print('covdldB', covdldB[10,:,:], covdldB_ov)\n",
        "    \n",
        "    FisherInfoMat[np.ix_(np.arange(nv), np.arange(p),np.arange(p))] = covdldB\n",
        "    FisherInfoMat_ov[np.ix_(np.arange(p),np.arange(p))] = covdldB_ov\n",
        "    #print('FImat check', FisherInfoMat[10,:,:], FisherInfoMat_ov)\n",
        "    \n",
        "    # Add dl/dsigma2 dl/dD covariance\n",
        "    for k in np.arange(len(nparams)):\n",
        "\n",
        "      # Get covariance of dldsigma and dldD      \n",
        "      covdldsigmadD = get_covdldDkdsigma2_broadcasted(k, sigma2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict).reshape(nv,FishIndsDk[k+1]-FishIndsDk[k])\n",
        "      covdldsigmadD_ov =  get_covdldDkdsigma2(k, sigma2_ov, nlevels, nparams, ZtZ_ov, DinvIplusZtZD_ov, invDupMatdict).reshape(FishIndsDk[k+1]-FishIndsDk[k])\n",
        "      \n",
        "      #print('cov(dl/dsigma,dl/dD) ' + str(k), covdldsigmadD[10,:],covdldsigmadD_ov)\n",
        "      \n",
        "      # Assign to the relevant block\n",
        "      FisherInfoMat[:,p, FishIndsDk[k]:FishIndsDk[k+1]] = covdldsigmadD\n",
        "      FisherInfoMat[:,FishIndsDk[k]:FishIndsDk[k+1],p:(p+1)] = FisherInfoMat[:,p:(p+1), FishIndsDk[k]:FishIndsDk[k+1]].transpose((0,2,1))\n",
        "    \n",
        "      FisherInfoMat_ov[p, FishIndsDk[k]:FishIndsDk[k+1]] = covdldsigmadD_ov\n",
        "      FisherInfoMat_ov[FishIndsDk[k]:FishIndsDk[k+1],p] = FisherInfoMat_ov[p, FishIndsDk[k]:FishIndsDk[k+1]].transpose()\n",
        "\n",
        "      #print('Fisher Info Mat check ' + str(k), FisherInfoMat[10,:,:], FisherInfoMat_ov)\n",
        "      #print(np.sum(np.sum(np.abs(FisherInfoMat[10,:,:]- FisherInfoMat_ov))))\n",
        "   \n",
        "    # Add dl/dD covariance\n",
        "    for k1 in np.arange(len(nparams)):\n",
        "\n",
        "      for k2 in np.arange(k1+1):\n",
        "\n",
        "        IndsDk1 = np.arange(FishIndsDk[k1],FishIndsDk[k1+1])\n",
        "        IndsDk2 = np.arange(FishIndsDk[k2],FishIndsDk[k2+1])\n",
        "\n",
        "        # Get covariance between D_k1 and D_k2 \n",
        "        covdldDk1dDk2 = get_covdldDk1Dk2_broadcasted(k1, k2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict)\n",
        "        \n",
        "        # Get covariance between D_k1 and D_k2 \n",
        "        covdldDk1dDk2_ov = get_covdldDk1Dk2(k1, k2, nlevels, nparams, ZtZ_ov, DinvIplusZtZD_ov, invDupMatdict)\n",
        "        \n",
        "        #print('cov(dldDk, dldDk) ' + str(k1) + ' ' + str(k2))\n",
        "        #print(covdldDk1dDk2[10,:,:])\n",
        "        #print(covdldDk1dDk2_ov)\n",
        "        #print(np.sum(np.sum(np.abs(covdldDk1dDk2[10,:,:]-covdldDk1dDk2_ov))))\n",
        "        \n",
        "        # Add to FImat\n",
        "        FisherInfoMat[np.ix_(np.arange(nv), IndsDk1, IndsDk2)] = covdldDk1dDk2\n",
        "        FisherInfoMat[np.ix_(np.arange(nv), IndsDk2, IndsDk1)] = FisherInfoMat[np.ix_(np.arange(nv), IndsDk1, IndsDk2)].transpose((0,2,1))\n",
        "        \n",
        "        # Add to FImat \n",
        "        FisherInfoMat_ov[np.ix_(IndsDk1, IndsDk2)] = covdldDk1dDk2_ov\n",
        "        FisherInfoMat_ov[np.ix_(IndsDk2, IndsDk1)] = FisherInfoMat_ov[np.ix_(IndsDk1, IndsDk2)].transpose()\n",
        "\n",
        "        #print('Fisher Info Mat check ' + str(k1) + ' ' + str(k2))\n",
        "        #print(np.sum(np.sum(np.abs(FisherInfoMat[10,:,:]- FisherInfoMat_ov))))\n",
        "       \n",
        "    # Derivative and parameters\n",
        "    # ----------------------------------------------------------------------------\n",
        "\n",
        "    # Concatenate paramaters and derivatives together\n",
        "    # ----------------------------------------------------------------------------\n",
        "    paramVector = np.concatenate((beta, sigma2.reshape(nv,1,1)),axis=1)\n",
        "    derivVector = np.concatenate((dldB, dldsigma2.reshape(nv,1,1)),axis=1)\n",
        "    \n",
        "    \n",
        "    paramVector_ov = np.concatenate((beta_ov, np.array([[sigma2_ov]])))\n",
        "    derivVector_ov = np.concatenate((dldB_ov, dldsigma2_ov))\n",
        "    \n",
        "    #print('paramvec check', np.sum(np.sum(np.abs(paramVector_ov-paramVector[10,:,:]))))\n",
        "    #print('derivvec check', np.sum(np.sum(np.abs(derivVector_ov-derivVector[10,:,:]))))\n",
        "\n",
        "\n",
        "    for k in np.arange(len(nparams)):\n",
        "\n",
        "      paramVector = np.concatenate((paramVector, mat2vech_broadcasted(Ddict[k])),axis=1)\n",
        "      derivVector = np.concatenate((derivVector, mat2vech_broadcasted(dldDdict[k])),axis=1)\n",
        "      \n",
        "      paramVector_ov = np.concatenate((paramVector_ov, mat2vech(Ddict_ov[k])))\n",
        "      derivVector_ov = np.concatenate((derivVector_ov, mat2vech(dldDdict_ov[k])))\n",
        "    \n",
        "      #print('paramvec check', np.sum(np.sum(np.abs(paramVector_ov-paramVector[10,:,:]))))\n",
        "      #print('derivvec check', np.sum(np.sum(np.abs(derivVector_ov-derivVector[10,:,:]))))\n",
        "      \n",
        "    \n",
        "    # Update step\n",
        "    # ----------------------------------------------------------------------------\n",
        "    FisherInfoMat = forceSym_broadcasted(FisherInfoMat)\n",
        "    paramVector = paramVector + np.einsum('i,ijk->ijk',lam,(np.linalg.inv(FisherInfoMat) @ derivVector))\n",
        "    \n",
        "    \n",
        "    FisherInfoMat_ov = forceSym(FisherInfoMat_ov)\n",
        "    paramVector_ov = paramVector_ov + lam_ov*(np.linalg.inv(FisherInfoMat_ov) @ derivVector_ov)\n",
        "    \n",
        "    #print('Final FI check', np.sum(np.sum(np.abs(FisherInfoMat[10,:,:]-FisherInfoMat_ov))))\n",
        "    #print('Update check', np.sum(np.sum(np.abs(paramVector[10,:,:]-paramVector_ov))))\n",
        "\n",
        "    # Get the new parameters\n",
        "    beta = paramVector[:,0:p,:]\n",
        "    sigma2 = paramVector[:,p:(p+1)][:,0,0]\n",
        "    \n",
        "    \n",
        "    beta_ov = paramVector_ov[0:p]\n",
        "    sigma2_ov = paramVector_ov[p:(p+1)][0,0]\n",
        "\n",
        "    #print('beta check', np.sum(np.sum(np.abs(beta_ov - beta[10,:,:]))))\n",
        "    #print('sigma2 check', np.sum(np.sum(np.abs(sigma2_ov - sigma2[10]))))\n",
        "    \n",
        "    # D as a dictionary\n",
        "    for k in np.arange(len(nparams)):\n",
        "\n",
        "      Ddict[k] = makeDnnd_broadcasted(vech2mat_broadcasted(paramVector[:,FishIndsDk[k]:FishIndsDk[k+1],:]))\n",
        "      \n",
        "\n",
        "    for k in np.arange(len(nparams)):\n",
        "\n",
        "      Ddict_ov[k] = makeDnnd(vech2mat(paramVector_ov[FishIndsDk[k]:FishIndsDk[k+1]]))\n",
        "      \n",
        "      #print('Ddict check ' + str(k), Ddict[k][10,:,:], Ddict_ov[k])\n",
        "      \n",
        "      \n",
        "    # Full version of D\n",
        "    D = getDfromDict_broadcasted(Ddict, nparams, nlevels)\n",
        "    \n",
        "    for i in np.arange(len(nparams)):\n",
        "\n",
        "      for j in np.arange(nlevels[i]):\n",
        "\n",
        "\n",
        "        if i == 0 and j == 0:\n",
        "\n",
        "          D_ov = Ddict_ov[i]\n",
        "\n",
        "        else:\n",
        "\n",
        "          D_ov = scipy.linalg.block_diag(D_ov, Ddict_ov[i])\n",
        "    \n",
        "    #print('Dcheck', np.sum(np.sum(np.abs(D[10,:,:]-D_ov))))\n",
        "    \n",
        "    # Update the step size\n",
        "    llhcurr = llh_broadcasted(n, ZtZ, Zte, ete, sigma2, DinvIplusZtZD,D)[:,0,0]\n",
        "    llhcurr_ov = llh(n, ZtZ_ov, Zte_ov, ete_ov, sigma2_ov, DinvIplusZtZD_ov,D_ov)\n",
        "    #print('llh check',llhcurr[10]-llhcurr_ov)\n",
        "    \n",
        "    lam[llhprev>llhcurr] = lam[llhprev>llhcurr]/2\n",
        "    \n",
        "    examplellh[nit] = llhcurr[10]\n",
        "    if nit==99:\n",
        "      break\n",
        "    \n",
        "    print('sumlam:', np.sum(lam))\n",
        "\n",
        "    t2 = time.time()\n",
        "    nit = nit + 1\n",
        "    print('Iteration num: ', nit)\n",
        "    print('Iteration time: ', t2-t1)\n",
        "    print('Num converged:', nv-np.sum(np.abs(llhprev-llhcurr)>tol))\n",
        "    \n",
        "  bvals = DinvIplusZtZD @ Zte\n",
        "  \n",
        "  print('Total time taken: ', time.time()-t1_total)\n",
        "  print('Estimated NIFTI time (hours): ', 100*100*100/(nv*60*60)*(time.time()-t1_total))\n",
        "  \n",
        "  return(paramVector, bvals, examplellh)\n",
        "\n",
        "t1 = time.time()\n",
        "paramVec, bvals, examplellh = FS_broadcasted(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nparams, 1e-6,n)\n",
        "t2 = time.time()\n",
        "print(t2-t1)\n",
        "\n",
        "print(\"Predicted time on nifti of size 100x100x100 (in hours): \", 100*100*100*(t2-t1)/(nv*60*60))\n",
        "\n"
      ],
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Setup time:  14.18008303642273\n",
            "sumlam: 13500.0\n",
            "Iteration num:  1\n",
            "Iteration time:  124.39200711250305\n",
            "Num converged: -26000\n",
            "sumlam: 12679.5\n",
            "Iteration num:  2\n",
            "Iteration time:  126.12909817695618\n",
            "Num converged: -26000\n",
            "sumlam: 11423.625\n",
            "Iteration num:  3\n",
            "Iteration time:  125.46556496620178\n",
            "Num converged: -26000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-76-2cfd15d7b6b9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    372\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    373\u001b[0m \u001b[0mt1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 374\u001b[0;31m \u001b[0mparamVec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbvals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexamplellh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFS_broadcasted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXtX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mXtY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mZtX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mZtY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mZtZ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mXtZ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mYtZ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mYtY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mYtX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnlevels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1e-6\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    375\u001b[0m \u001b[0mt2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    376\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt2\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mt1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-76-2cfd15d7b6b9>\u001b[0m in \u001b[0;36mFS_broadcasted\u001b[0;34m(XtX, XtY, ZtX, ZtY, ZtZ, XtZ, YtZ, YtY, YtX, nlevels, nparams, tol, n)\u001b[0m\n\u001b[1;32m    242\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m         \u001b[0;31m# Get covariance between D_k1 and D_k2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 244\u001b[0;31m         \u001b[0mcovdldDk1dDk2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_covdldDk1Dk2_broadcasted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnlevels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mZtZ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDinvIplusZtZD\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minvDupMatdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    245\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    246\u001b[0m         \u001b[0;31m# Get covariance between D_k1 and D_k2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-74-25edd84c09b9>\u001b[0m in \u001b[0;36mget_covdldDk1Dk2_broadcasted\u001b[0;34m(k1, k2, nlevels, nparams, ZtZ, DinvIplusZtZD, invDupMatdict)\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m       \u001b[0;31m# Work out R_(k1, k2, i, j)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m       \u001b[0mRk1k2ij\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mZtZ\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mix_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mZtZ\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mIk1i\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mIk2j\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mZtZ\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mIk1i\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m@\u001b[0m \u001b[0mDinvIplusZtZD\u001b[0m \u001b[0;34m@\u001b[0m \u001b[0mZtZ\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mIk2j\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m       \u001b[0;31m# Work out Rk1k2ij kron Rk1k2ij\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "axQkNrbiKtLw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "beta_est_map=beta_True.reshape(dimv[0],dimv[1],dimv[2],beta.shape[1])\n",
        "\n",
        "\n",
        "# Show true beta, 3rd x-slice, 3rd parameter\n",
        "imshow(beta_est_map[3,:,:,3].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gmn9Qm8XTshL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "beta_est_map=paramVec[:,0:p,:].reshape(dimv[0],dimv[1],dimv[2],beta.shape[1])\n",
        "\n",
        "\n",
        "# Show true beta, 3rd x-slice, 3rd parameter\n",
        "imshow(beta_est_map[3,:,:,3].reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aYYN5DWIVtMb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "beta_est_map=paramVec[:,0:p,:].reshape(dimv[0],dimv[1],dimv[2],beta.shape[1])\n",
        "beta_True_map=beta_True.reshape(dimv[0],dimv[1],dimv[2],beta.shape[1])\n",
        "\n",
        "\n",
        "# Show true beta, 3rd x-slice, 3rd parameter\n",
        "imshow((beta_est_map[3,:,:,3]-beta_True_map[3,:,:,3]).reshape(dimv[0],dimv[1]), \\\n",
        "                    interpolation='nearest', aspect='auto')\n",
        "plt.colorbar()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PgF9XbZVeRy0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}