diff --git a/notebooks/Kaggle_preparedata.ipynb b/notebooks/Kaggle_preparedata.ipynb new file mode 100644 index 0000000..9ebda4f --- /dev/null +++ b/notebooks/Kaggle_preparedata.ipynb @@ -0,0 +1,233 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np \n", + "import pandas as pd\n", + "from scipy.io import loadmat\n", + "from keras.utils.np_utils import to_categorical" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "path_to_train = '/media/sf_VBox_Shared/timeseries/KaggleEEG/train_1/'" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def mat_to_data(path, max_timesteps=None):\n", + " try :\n", + " mat = loadmat(path)\n", + " names = mat['dataStruct'].dtype.names\n", + " ndata = {n: mat['dataStruct'][n][0, 0] for n in names}\n", + " if max_timesteps is None:\n", + " return ndata['data']\n", + " else:\n", + " return ndata['data'][:max_timesteps,:]\n", + " except :\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "fnames = os.listdir(path_to_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "fnames_split = [fname.split('_') for fname in fnames if fname[-4:]=='.mat']\n", + "metadata = pd.DataFrame([{'patient':int(f[0]), 'segment':int(f[1]), 'class':int(f[2][0])} for f in fnames_split])" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "nr_val = 10*6" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "metadata_0 = metadata[metadata['class']==0]\n", + "metadata_1 = metadata[metadata['class']==1]\n", + "\n", + "nr_segments_0 = metadata_0['segment']" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "np.random.seed(123)\n", + "ind_0 = metadata_0.sort_values('segment').index\n", + "ind_1 = metadata_1.sort_values('segment').index\n", + "val_ind = np.random.permutation(ind_0[-nr_val:].append( ind_1[-nr_val:]))\n", + "train_ind = np.random.permutation(ind_0[:-nr_val].append(ind_1[:-nr_val]))[:100]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def read_data(filenames, path_to_train, max_timesteps=None):\n", + " mfiles = [mat_to_data(os.path.join(path_to_train, fname), max_timesteps) for fname in filenames]\n", + " data = [m for m in mfiles if m is not None]\n", + " return np.array(data)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_val = read_data(pd.Series(fnames)[val_ind], path_to_train)\n", + "y_val = metadata.loc[val_ind]['class']\n", + "y_val_binary = to_categorical(np.array(y_val), 2)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "outdatapath = '/media/sf_VBox_Shared/timeseries/KaggleEEG/train_1/prepared/'\n", + "np.save(outdatapath+'X_val', X_val)\n", + "np.save(outdatapath+'y_val_binary', y_val_binary)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "X_train = read_data(pd.Series(fnames)[train_ind], path_to_train, max_timesteps=2400)\n", + "y_train = metadata.loc[train_ind]['class']\n", + "y_train_binary = to_categorical(np.array(y_train), 2)\n", + "np.save(outdatapath+'X_train', X_train)\n", + "np.save(outdatapath+'y_train_binary', y_train_binary)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(100, 24000, 16)" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "X_train.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "np.save(outdatapath+'y_val_binary', y_val_binary)\n", + "np.save(outdatapath+'y_train_binary', y_train_binary)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/deeplearning_kaggle.ipynb b/notebooks/deeplearning_kaggle.ipynb new file mode 100644 index 0000000..b7bbade --- /dev/null +++ b/notebooks/deeplearning_kaggle.ipynb @@ -0,0 +1,874 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using Theano backend.\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "from keras.utils.np_utils import to_categorical\n", + "import keras" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Populating the interactive namespace from numpy and matplotlib\n" + ] + } + ], + "source": [ + "%pylab inline\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "datapath = '/media/sf_VBox_Shared/timeseries/KaggleEEG/train_1/prepared/'\n", + "resultpath = 'datapath'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "ext = '.npy'\n", + "X_train = np.load(datapath+'X_train'+ext)\n", + "y_train_binary = np.load(datapath+'y_train_binary'+ext)\n", + "X_val = np.load(datapath+'X_val'+ext)\n", + "y_val_binary = np.load(datapath+'y_val_binary'+ext)\n", + "#X_test = np.load(datapath+'X_test'+ext)\n", + "#y_test_binary = np.load(datapath+'y_test_binary'+ext)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2\n" + ] + } + ], + "source": [ + "num_classes = y_train_binary.shape[1]\n", + "print(num_classes)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generate models" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from keras.models import Sequential\n", + "from keras.layers import Dense, Activation, Convolution1D, Flatten, MaxPooling1D\n", + "from keras.optimizers import Adam" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "import numpy as np\n", + "sys.path.insert(0, os.path.abspath('..'))\n", + "\n", + "from mcfly import modelgen, find_architecture" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "models = modelgen.generate_models(X_train.shape,\n", + " num_classes,\n", + " number_of_models = 10\n", + " #model_type = 'CNN',\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'fc_hidden_nodes': 303, 'regularization_rate': 0.008797457773821373, 'filters': array([94, 90, 38, 31, 10, 19, 73]), 'learning_rate': 0.010806047643803933}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_1 (BatchNormaliz(None, 2400, 16) 32 batchnormalization_input_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_1 (Convolution1D) (None, 2400, 94) 4606 batchnormalization_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_2 (BatchNormaliz(None, 2400, 94) 188 convolution1d_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_1 (Activation) (None, 2400, 94) 0 batchnormalization_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_2 (Convolution1D) (None, 2400, 90) 25470 activation_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_3 (BatchNormaliz(None, 2400, 90) 180 convolution1d_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_2 (Activation) (None, 2400, 90) 0 batchnormalization_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_3 (Convolution1D) (None, 2400, 38) 10298 activation_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_4 (BatchNormaliz(None, 2400, 38) 76 convolution1d_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_3 (Activation) (None, 2400, 38) 0 batchnormalization_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_4 (Convolution1D) (None, 2400, 31) 3565 activation_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_5 (BatchNormaliz(None, 2400, 31) 62 convolution1d_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_4 (Activation) (None, 2400, 31) 0 batchnormalization_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_5 (Convolution1D) (None, 2400, 10) 940 activation_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_6 (BatchNormaliz(None, 2400, 10) 20 convolution1d_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_5 (Activation) (None, 2400, 10) 0 batchnormalization_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_6 (Convolution1D) (None, 2400, 19) 589 activation_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_7 (BatchNormaliz(None, 2400, 19) 38 convolution1d_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_6 (Activation) (None, 2400, 19) 0 batchnormalization_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_7 (Convolution1D) (None, 2400, 73) 4234 activation_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_8 (BatchNormaliz(None, 2400, 73) 146 convolution1d_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_7 (Activation) (None, 2400, 73) 0 batchnormalization_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "flatten_1 (Flatten) (None, 175200) 0 activation_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_1 (Dense) (None, 303) 53085903 flatten_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_8 (Activation) (None, 303) 0 dense_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_2 (Dense) (None, 2) 608 activation_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_9 (BatchNormaliz(None, 2) 4 dense_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_9 (Activation) (None, 2) 0 batchnormalization_9[0][0] \n", + "====================================================================================================\n", + "Total params: 53136959\n", + "____________________________________________________________________________________________________\n", + "{'fc_hidden_nodes': 1609, 'regularization_rate': 0.06158512598386167, 'filters': array([42, 81, 22, 70, 92, 55, 67]), 'learning_rate': 0.054001824387223424}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_10 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_8 (Convolution1D) (None, 2400, 42) 2058 batchnormalization_10[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_11 (BatchNormali(None, 2400, 42) 84 convolution1d_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_10 (Activation) (None, 2400, 42) 0 batchnormalization_11[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_9 (Convolution1D) (None, 2400, 81) 10287 activation_10[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_12 (BatchNormali(None, 2400, 81) 162 convolution1d_9[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_11 (Activation) (None, 2400, 81) 0 batchnormalization_12[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_10 (Convolution1D) (None, 2400, 22) 5368 activation_11[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_13 (BatchNormali(None, 2400, 22) 44 convolution1d_10[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_12 (Activation) (None, 2400, 22) 0 batchnormalization_13[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_11 (Convolution1D) (None, 2400, 70) 4690 activation_12[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_14 (BatchNormali(None, 2400, 70) 140 convolution1d_11[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_13 (Activation) (None, 2400, 70) 0 batchnormalization_14[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_12 (Convolution1D) (None, 2400, 92) 19412 activation_13[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_15 (BatchNormali(None, 2400, 92) 184 convolution1d_12[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_14 (Activation) (None, 2400, 92) 0 batchnormalization_15[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_13 (Convolution1D) (None, 2400, 55) 15235 activation_14[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_16 (BatchNormali(None, 2400, 55) 110 convolution1d_13[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_15 (Activation) (None, 2400, 55) 0 batchnormalization_16[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_14 (Convolution1D) (None, 2400, 67) 11122 activation_15[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_17 (BatchNormali(None, 2400, 67) 134 convolution1d_14[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_16 (Activation) (None, 2400, 67) 0 batchnormalization_17[0][0] \n", + "____________________________________________________________________________________________________\n", + "flatten_2 (Flatten) (None, 160800) 0 activation_16[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_3 (Dense) (None, 1609) 258728809 flatten_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_17 (Activation) (None, 1609) 0 dense_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_4 (Dense) (None, 2) 3220 activation_17[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_18 (BatchNormali(None, 2) 4 dense_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_18 (Activation) (None, 2) 0 batchnormalization_18[0][0] \n", + "====================================================================================================\n", + "Total params: 258801095\n", + "____________________________________________________________________________________________________\n", + "{'regularization_rate': 0.06894068294109973, 'lstm_dims': array([17, 34, 21, 71, 58]), 'filters': array([78, 17, 41, 25]), 'learning_rate': 0.03925895453985795}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_19 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_1 (Reshape) (None, 1, 2400, 16) 0 batchnormalization_19[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_1 (Convolution2D) (None, 78, 2400, 16)312 reshape_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_20 (BatchNormali(None, 78, 2400, 16)32 convolution2d_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_19 (Activation) (None, 78, 2400, 16)0 batchnormalization_20[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_2 (Convolution2D) (None, 17, 2400, 16)3995 activation_19[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_21 (BatchNormali(None, 17, 2400, 16)32 convolution2d_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_20 (Activation) (None, 17, 2400, 16)0 batchnormalization_21[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_3 (Convolution2D) (None, 41, 2400, 16)2132 activation_20[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_22 (BatchNormali(None, 41, 2400, 16)32 convolution2d_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_21 (Activation) (None, 41, 2400, 16)0 batchnormalization_22[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_4 (Convolution2D) (None, 25, 2400, 16)3100 activation_21[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_23 (BatchNormali(None, 25, 2400, 16)32 convolution2d_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_22 (Activation) (None, 25, 2400, 16)0 batchnormalization_23[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_2 (Reshape) (None, 2400, 400) 0 activation_22[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_1 (LSTM) (None, 2400, 17) 28424 reshape_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_2 (LSTM) (None, 2400, 34) 7072 lstm_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_3 (LSTM) (None, 2400, 21) 4704 lstm_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_4 (LSTM) (None, 2400, 71) 26412 lstm_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_5 (LSTM) (None, 2400, 58) 30160 lstm_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "dropout_1 (Dropout) (None, 2400, 58) 0 lstm_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "timedistributed_1 (TimeDistributed)(None, 2400, 2) 118 dropout_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_23 (Activation) (None, 2400, 2) 0 timedistributed_1[0][0] \n", + "____________________________________________________________________________________________________\n", + "lambda_1 (Lambda) (None, 2) 0 activation_23[0][0] \n", + "====================================================================================================\n", + "Total params: 106589\n", + "____________________________________________________________________________________________________\n", + "{'fc_hidden_nodes': 373, 'regularization_rate': 0.007988074027389422, 'filters': array([87]), 'learning_rate': 0.0193280166086941}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_24 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_15 (Convolution1D) (None, 2400, 87) 4263 batchnormalization_24[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_25 (BatchNormali(None, 2400, 87) 174 convolution1d_15[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_24 (Activation) (None, 2400, 87) 0 batchnormalization_25[0][0] \n", + "____________________________________________________________________________________________________\n", + "flatten_3 (Flatten) (None, 208800) 0 activation_24[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_6 (Dense) (None, 373) 77882773 flatten_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_25 (Activation) (None, 373) 0 dense_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_7 (Dense) (None, 2) 748 activation_25[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_26 (BatchNormali(None, 2) 4 dense_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_26 (Activation) (None, 2) 0 batchnormalization_26[0][0] \n", + "====================================================================================================\n", + "Total params: 77887994\n", + "____________________________________________________________________________________________________\n", + "{'fc_hidden_nodes': 418, 'regularization_rate': 0.0010235299274346152, 'filters': array([37, 23, 49]), 'learning_rate': 0.00044749206607106715}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_27 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_16 (Convolution1D) (None, 2400, 37) 1813 batchnormalization_27[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_28 (BatchNormali(None, 2400, 37) 74 convolution1d_16[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_27 (Activation) (None, 2400, 37) 0 batchnormalization_28[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_17 (Convolution1D) (None, 2400, 23) 2576 activation_27[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_29 (BatchNormali(None, 2400, 23) 46 convolution1d_17[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_28 (Activation) (None, 2400, 23) 0 batchnormalization_29[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_18 (Convolution1D) (None, 2400, 49) 3430 activation_28[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_30 (BatchNormali(None, 2400, 49) 98 convolution1d_18[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_29 (Activation) (None, 2400, 49) 0 batchnormalization_30[0][0] \n", + "____________________________________________________________________________________________________\n", + "flatten_4 (Flatten) (None, 117600) 0 activation_29[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_8 (Dense) (None, 418) 49157218 flatten_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_30 (Activation) (None, 418) 0 dense_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_9 (Dense) (None, 2) 838 activation_30[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_31 (BatchNormali(None, 2) 4 dense_9[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_31 (Activation) (None, 2) 0 batchnormalization_31[0][0] \n", + "====================================================================================================\n", + "Total params: 49166129\n", + "____________________________________________________________________________________________________\n", + "{'regularization_rate': 0.08583745304185751, 'lstm_dims': array([36, 15, 89]), 'filters': array([38, 63, 97, 23, 45, 39, 37, 80]), 'learning_rate': 0.006500795210253031}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_32 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_3 (Reshape) (None, 1, 2400, 16) 0 batchnormalization_32[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_5 (Convolution2D) (None, 38, 2400, 16)152 reshape_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_33 (BatchNormali(None, 38, 2400, 16)32 convolution2d_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_32 (Activation) (None, 38, 2400, 16)0 batchnormalization_33[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_6 (Convolution2D) (None, 63, 2400, 16)7245 activation_32[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_34 (BatchNormali(None, 63, 2400, 16)32 convolution2d_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_33 (Activation) (None, 63, 2400, 16)0 batchnormalization_34[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_7 (Convolution2D) (None, 97, 2400, 16)18430 activation_33[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_35 (BatchNormali(None, 97, 2400, 16)32 convolution2d_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_34 (Activation) (None, 97, 2400, 16)0 batchnormalization_35[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_8 (Convolution2D) (None, 23, 2400, 16)6716 activation_34[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_36 (BatchNormali(None, 23, 2400, 16)32 convolution2d_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_35 (Activation) (None, 23, 2400, 16)0 batchnormalization_36[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_9 (Convolution2D) (None, 45, 2400, 16)3150 activation_35[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_37 (BatchNormali(None, 45, 2400, 16)32 convolution2d_9[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_36 (Activation) (None, 45, 2400, 16)0 batchnormalization_37[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_10 (Convolution2D) (None, 39, 2400, 16)5304 activation_36[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_38 (BatchNormali(None, 39, 2400, 16)32 convolution2d_10[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_37 (Activation) (None, 39, 2400, 16)0 batchnormalization_38[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_11 (Convolution2D) (None, 37, 2400, 16)4366 activation_37[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_39 (BatchNormali(None, 37, 2400, 16)32 convolution2d_11[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_38 (Activation) (None, 37, 2400, 16)0 batchnormalization_39[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_12 (Convolution2D) (None, 80, 2400, 16)8960 activation_38[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_40 (BatchNormali(None, 80, 2400, 16)32 convolution2d_12[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_39 (Activation) (None, 80, 2400, 16)0 batchnormalization_40[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_4 (Reshape) (None, 2400, 1280) 0 activation_39[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_6 (LSTM) (None, 2400, 36) 189648 reshape_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_7 (LSTM) (None, 2400, 15) 3120 lstm_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_8 (LSTM) (None, 2400, 89) 37380 lstm_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "dropout_2 (Dropout) (None, 2400, 89) 0 lstm_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "timedistributed_2 (TimeDistributed)(None, 2400, 2) 180 dropout_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_40 (Activation) (None, 2400, 2) 0 timedistributed_2[0][0] \n", + "____________________________________________________________________________________________________\n", + "lambda_2 (Lambda) (None, 2) 0 activation_40[0][0] \n", + "====================================================================================================\n", + "Total params: 284939\n", + "____________________________________________________________________________________________________\n", + "{'regularization_rate': 0.08552873197510165, 'lstm_dims': array([74, 60, 24]), 'filters': array([28, 60]), 'learning_rate': 0.0003099866487143127}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_41 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_5 (Reshape) (None, 1, 2400, 16) 0 batchnormalization_41[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_13 (Convolution2D) (None, 28, 2400, 16)112 reshape_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_42 (BatchNormali(None, 28, 2400, 16)32 convolution2d_13[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_41 (Activation) (None, 28, 2400, 16)0 batchnormalization_42[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_14 (Convolution2D) (None, 60, 2400, 16)5100 activation_41[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_43 (BatchNormali(None, 60, 2400, 16)32 convolution2d_14[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_42 (Activation) (None, 60, 2400, 16)0 batchnormalization_43[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_6 (Reshape) (None, 2400, 960) 0 activation_42[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_9 (LSTM) (None, 2400, 74) 306360 reshape_6[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_10 (LSTM) (None, 2400, 60) 32400 lstm_9[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_11 (LSTM) (None, 2400, 24) 8160 lstm_10[0][0] \n", + "____________________________________________________________________________________________________\n", + "dropout_3 (Dropout) (None, 2400, 24) 0 lstm_11[0][0] \n", + "____________________________________________________________________________________________________\n", + "timedistributed_3 (TimeDistributed)(None, 2400, 2) 50 dropout_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_43 (Activation) (None, 2400, 2) 0 timedistributed_3[0][0] \n", + "____________________________________________________________________________________________________\n", + "lambda_3 (Lambda) (None, 2) 0 activation_43[0][0] \n", + "====================================================================================================\n", + "Total params: 352278\n", + "____________________________________________________________________________________________________\n", + "{'regularization_rate': 0.07092037383309183, 'lstm_dims': array([68, 35, 72, 58, 37]), 'filters': array([77, 56, 20, 12, 19, 62, 99, 28]), 'learning_rate': 0.00026731724523737597}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_44 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_7 (Reshape) (None, 1, 2400, 16) 0 batchnormalization_44[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_15 (Convolution2D) (None, 77, 2400, 16)308 reshape_7[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_45 (BatchNormali(None, 77, 2400, 16)32 convolution2d_15[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_44 (Activation) (None, 77, 2400, 16)0 batchnormalization_45[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_16 (Convolution2D) (None, 56, 2400, 16)12992 activation_44[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_46 (BatchNormali(None, 56, 2400, 16)32 convolution2d_16[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_45 (Activation) (None, 56, 2400, 16)0 batchnormalization_46[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_17 (Convolution2D) (None, 20, 2400, 16)3380 activation_45[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_47 (BatchNormali(None, 20, 2400, 16)32 convolution2d_17[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_46 (Activation) (None, 20, 2400, 16)0 batchnormalization_47[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_18 (Convolution2D) (None, 12, 2400, 16)732 activation_46[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_48 (BatchNormali(None, 12, 2400, 16)32 convolution2d_18[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_47 (Activation) (None, 12, 2400, 16)0 batchnormalization_48[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_19 (Convolution2D) (None, 19, 2400, 16)703 activation_47[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_49 (BatchNormali(None, 19, 2400, 16)32 convolution2d_19[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_48 (Activation) (None, 19, 2400, 16)0 batchnormalization_49[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_20 (Convolution2D) (None, 62, 2400, 16)3596 activation_48[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_50 (BatchNormali(None, 62, 2400, 16)32 convolution2d_20[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_49 (Activation) (None, 62, 2400, 16)0 batchnormalization_50[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_21 (Convolution2D) (None, 99, 2400, 16)18513 activation_49[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_51 (BatchNormali(None, 99, 2400, 16)32 convolution2d_21[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_50 (Activation) (None, 99, 2400, 16)0 batchnormalization_51[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_22 (Convolution2D) (None, 28, 2400, 16)8344 activation_50[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_52 (BatchNormali(None, 28, 2400, 16)32 convolution2d_22[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_51 (Activation) (None, 28, 2400, 16)0 batchnormalization_52[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_8 (Reshape) (None, 2400, 448) 0 activation_51[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_12 (LSTM) (None, 2400, 68) 140624 reshape_8[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_13 (LSTM) (None, 2400, 35) 14560 lstm_12[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_14 (LSTM) (None, 2400, 72) 31104 lstm_13[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_15 (LSTM) (None, 2400, 58) 30392 lstm_14[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_16 (LSTM) (None, 2400, 37) 14208 lstm_15[0][0] \n", + "____________________________________________________________________________________________________\n", + "dropout_4 (Dropout) (None, 2400, 37) 0 lstm_16[0][0] \n", + "____________________________________________________________________________________________________\n", + "timedistributed_4 (TimeDistributed)(None, 2400, 2) 76 dropout_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_52 (Activation) (None, 2400, 2) 0 timedistributed_4[0][0] \n", + "____________________________________________________________________________________________________\n", + "lambda_4 (Lambda) (None, 2) 0 activation_52[0][0] \n", + "====================================================================================================\n", + "Total params: 279820\n", + "____________________________________________________________________________________________________\n", + "{'fc_hidden_nodes': 643, 'regularization_rate': 0.00023735626231246638, 'filters': array([ 21, 100, 66, 87, 95]), 'learning_rate': 0.004378685617119499}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_53 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_9[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_19 (Convolution1D) (None, 2400, 21) 1029 batchnormalization_53[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_54 (BatchNormali(None, 2400, 21) 42 convolution1d_19[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_53 (Activation) (None, 2400, 21) 0 batchnormalization_54[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_20 (Convolution1D) (None, 2400, 100) 6400 activation_53[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_55 (BatchNormali(None, 2400, 100) 200 convolution1d_20[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_54 (Activation) (None, 2400, 100) 0 batchnormalization_55[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_21 (Convolution1D) (None, 2400, 66) 19866 activation_54[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_56 (BatchNormali(None, 2400, 66) 132 convolution1d_21[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_55 (Activation) (None, 2400, 66) 0 batchnormalization_56[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_22 (Convolution1D) (None, 2400, 87) 17313 activation_55[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_57 (BatchNormali(None, 2400, 87) 174 convolution1d_22[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_56 (Activation) (None, 2400, 87) 0 batchnormalization_57[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution1d_23 (Convolution1D) (None, 2400, 95) 24890 activation_56[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_58 (BatchNormali(None, 2400, 95) 190 convolution1d_23[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_57 (Activation) (None, 2400, 95) 0 batchnormalization_58[0][0] \n", + "____________________________________________________________________________________________________\n", + "flatten_5 (Flatten) (None, 228000) 0 activation_57[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_13 (Dense) (None, 643) 146604643 flatten_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_58 (Activation) (None, 643) 0 dense_13[0][0] \n", + "____________________________________________________________________________________________________\n", + "dense_14 (Dense) (None, 2) 1288 activation_58[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_59 (BatchNormali(None, 2) 4 dense_14[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_59 (Activation) (None, 2) 0 batchnormalization_59[0][0] \n", + "====================================================================================================\n", + "Total params: 146676203\n", + "____________________________________________________________________________________________________\n", + "{'regularization_rate': 0.002375498624282884, 'lstm_dims': array([46, 30]), 'filters': array([10, 28]), 'learning_rate': 0.014954324531148583}\n", + "____________________________________________________________________________________________________\n", + "Layer (type) Output Shape Param # Connected to \n", + "====================================================================================================\n", + "batchnormalization_60 (BatchNormali(None, 2400, 16) 32 batchnormalization_input_10[0][0]\n", + "____________________________________________________________________________________________________\n", + "reshape_9 (Reshape) (None, 1, 2400, 16) 0 batchnormalization_60[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_23 (Convolution2D) (None, 10, 2400, 16)40 reshape_9[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_61 (BatchNormali(None, 10, 2400, 16)32 convolution2d_23[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_60 (Activation) (None, 10, 2400, 16)0 batchnormalization_61[0][0] \n", + "____________________________________________________________________________________________________\n", + "convolution2d_24 (Convolution2D) (None, 28, 2400, 16)868 activation_60[0][0] \n", + "____________________________________________________________________________________________________\n", + "batchnormalization_62 (BatchNormali(None, 28, 2400, 16)32 convolution2d_24[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_61 (Activation) (None, 28, 2400, 16)0 batchnormalization_62[0][0] \n", + "____________________________________________________________________________________________________\n", + "reshape_10 (Reshape) (None, 2400, 448) 0 activation_61[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_17 (LSTM) (None, 2400, 46) 91080 reshape_10[0][0] \n", + "____________________________________________________________________________________________________\n", + "lstm_18 (LSTM) (None, 2400, 30) 9240 lstm_17[0][0] \n", + "____________________________________________________________________________________________________\n", + "dropout_5 (Dropout) (None, 2400, 30) 0 lstm_18[0][0] \n", + "____________________________________________________________________________________________________\n", + "timedistributed_5 (TimeDistributed)(None, 2400, 2) 62 dropout_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "activation_62 (Activation) (None, 2400, 2) 0 timedistributed_5[0][0] \n", + "____________________________________________________________________________________________________\n", + "lambda_5 (Lambda) (None, 2) 0 activation_62[0][0] \n", + "====================================================================================================\n", + "Total params: 101386\n", + "____________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "for model, params, model_types in models:\n", + " print(params)\n", + " model.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare models" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training model 0 CNN\n", + "Train on 100 samples, validate on 120 samples\n", + "Epoch 1/5\n", + " 80/100 [=======================>......] - ETA: 3s - loss: 96.3190 - acc: 0.5625" + ] + }, + { + "ename": "ValueError", + "evalue": "Shape mismatch: x has 17520000 cols (and 20 rows) but y has 175200 rows (and 303 cols)\nApply node that caused the error: Dot22(Reshape{2}.0, dense_1_W)\nToposort index: 331\nInputs types: [TensorType(float32, matrix), TensorType(float32, matrix)]\nInputs shapes: [(20, 17520000), (175200, 303)]\nInputs strides: [(70080000, 4), (1212, 4)]\nInputs values: ['not shown', 'not shown']\nOutputs clients: [[Elemwise{Composite{(i0 * (Abs((i1 + i2)) + i1 + i2))}}[(0, 1)](TensorConstant{(1, 1) of 0.5}, Dot22.0, InplaceDimShuffle{x,0}.0)]]\n\nHINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag 'optimizer=fast_compile'. If that does not work, Theano optimizations can be disabled with 'optimizer=None'.\nHINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/theano/compile/function_module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 858\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 859\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 860\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Shape mismatch: x has 17520000 cols (and 20 rows) but y has 175200 rows (and 303 cols)", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0moutputfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresultpath\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m 'modelcomparison.json')\n\u001b[0m", + "\u001b[0;32m/home/dafne/timeseries/mcfly/mcfly/find_architecture.py\u001b[0m in \u001b[0;36mtrain_models_on_samples\u001b[0;34m(X_train, y_train, X_val, y_val, models, nr_epochs, subset_size, verbose, outputfile)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[0;31m# see comment on subsize_set\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 71\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 72\u001b[0;31m verbose=verbose)\n\u001b[0m\u001b[1;32m 73\u001b[0m \u001b[0mhistories\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[0mval_accuracies\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, **kwargs)\u001b[0m\n\u001b[1;32m 407\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 408\u001b[0m \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 409\u001b[0;31m sample_weight=sample_weight)\n\u001b[0m\u001b[1;32m 410\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 411\u001b[0m def evaluate(self, x, y, batch_size=32, verbose=1,\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight)\u001b[0m\n\u001b[1;32m 1050\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1051\u001b[0m \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1052\u001b[0;31m callback_metrics=callback_metrics)\n\u001b[0m\u001b[1;32m 1053\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1054\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, nb_epoch, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics)\u001b[0m\n\u001b[1;32m 803\u001b[0m val_outs = self._test_loop(val_f, val_ins,\n\u001b[1;32m 804\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 805\u001b[0;31m verbose=0)\n\u001b[0m\u001b[1;32m 806\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_outs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 807\u001b[0m \u001b[0mval_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mval_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_test_loop\u001b[0;34m(self, f, ins, batch_size, verbose)\u001b[0m\n\u001b[1;32m 888\u001b[0m \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mslice_X\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 889\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 890\u001b[0;31m \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 891\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 892\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mbatch_index\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/keras/backend/theano_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 516\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 517\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 518\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 520\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/theano/compile/function_module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 869\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnodes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mposition_of_error\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 870\u001b[0m \u001b[0mthunk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mthunk\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 871\u001b[0;31m storage_map=getattr(self.fn, 'storage_map', None))\n\u001b[0m\u001b[1;32m 872\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 873\u001b[0m \u001b[0;31m# old-style linkers raise their own exceptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/theano/gof/link.py\u001b[0m in \u001b[0;36mraise_with_op\u001b[0;34m(node, thunk, exc_info, storage_map)\u001b[0m\n\u001b[1;32m 312\u001b[0m \u001b[0;31m# extra long error message in that case.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 314\u001b[0;31m \u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_trace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 316\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/six.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m 683\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 684\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 685\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 686\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/dafne/anaconda2/envs/mcfly/lib/python3.5/site-packages/theano/compile/function_module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 857\u001b[0m \u001b[0mt0_fn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 858\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 859\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 860\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'position_of_error'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Shape mismatch: x has 17520000 cols (and 20 rows) but y has 175200 rows (and 303 cols)\nApply node that caused the error: Dot22(Reshape{2}.0, dense_1_W)\nToposort index: 331\nInputs types: [TensorType(float32, matrix), TensorType(float32, matrix)]\nInputs shapes: [(20, 17520000), (175200, 303)]\nInputs strides: [(70080000, 4), (1212, 4)]\nInputs values: ['not shown', 'not shown']\nOutputs clients: [[Elemwise{Composite{(i0 * (Abs((i1 + i2)) + i1 + i2))}}[(0, 1)](TensorConstant{(1, 1) of 0.5}, Dot22.0, InplaceDimShuffle{x,0}.0)]]\n\nHINT: Re-running with most Theano optimization disabled could give you a back-trace of when this node was created. This can be done with by setting the Theano flag 'optimizer=fast_compile'. If that does not work, Theano optimizations can be disabled with 'optimizer=None'.\nHINT: Use the Theano flag 'exception_verbosity=high' for a debugprint and storage map footprint of this apply node." + ] + } + ], + "source": [ + "histories, val_accuracies, val_losses = find_architecture.train_models_on_samples(X_train, y_train_binary,\n", + " X_val, y_val_binary,\n", + " models,nr_epochs=5,\n", + " subset_size=500,\n", + " verbose=True, \n", + " outputfile=resultpath+\\\n", + " 'modelcomparison.json')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "for i in range(len(models)):\n", + " name = str(models[i][1])\n", + " find_architecture.plotTrainingProcess(histories[i], name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "results = pd.DataFrame({'model':[str(params) for model, params, model_types in models],\n", + " 'train_acc': [history.history['acc'][-1] for history in histories],\n", + " 'train_loss': [history.history['loss'][-1] for history in histories],\n", + " 'val_acc': [history.history['val_acc'][-1] for history in histories],\n", + " 'val_loss': [history.history['val_loss'][-1] for history in histories]\n", + " })\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "results.to_csv(datapath+'results.csv')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Train the best model for real" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "best_model_index = np.argmax(val_accuracies)\n", + "#best_model_index = 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "best_model, best_params, best_model_types = models[best_model_index]\n", + "print(best_model_index, best_model_types, best_params)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We make a copy of the model, to start training from fresh." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "best_model_copy = modelgen.generate_CNN_model(X_train.shape, num_classes, best_params['filters'], best_params['fc_hidden_nodes'],\n", + " best_params['learning_rate'], best_params['regularization_rate'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}