train_neural_networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import absl.logging\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from sklearn.utils import class_weight\n",
    "from project_scripts.widgets import explore_dataset_widget, plot_data, dataset_movie_widget\n",
    "from project_scripts.data_loading import load_dataset, slices_to_textures, dataset_to_embeddings\n",
    "from project_scripts.neural_networks import B_frame_CNN, texture_CNN, ensemble_MLP\n",
    "\n",
    "# suppress 'WARNING:absl:Found untraced functions such as _jit_compiled_convolution_op ...' errors\n",
    "# these errors do not affect training or inference and even appear in the Tensorflow offficial tutorials\n",
    "tf.get_logger().setLevel('ERROR')\n",
    "absl.logging.set_verbosity('ERROR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb869f504091441cb018af4e086a13b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=0, description='slice_idx', max=6), IntSlider(value=0, description='fram…"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "explore_dataset_widget(plot_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Dataset\n",
    "\n",
    "<img src='manuscript/figure_1.png' width=\"600\"/>\n",
    "\n",
    "Figure 1. Caption"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LOADING: directory=(training_data) | label=([0, 1]) | total_bframes=(0)\n",
      "LOADING: directory=(training_data\\cancer) | label=([0, 1]) | total_bframes=(0)\n",
      "LOADING: directory=(training_data\\cancer\\3-21-2017-s1) | label=([1, 0]) | total_bframes=(52)\n",
      "LOADING: directory=(training_data\\cancer\\3-21-2017-s2) | label=([1, 0]) | total_bframes=(26)\n",
      "LOADING: directory=(training_data\\cancer\\6-24-2019-s3) | label=([1, 0]) | total_bframes=(137)\n",
      "LOADING: directory=(training_data\\cancer\\9-11-2018-s2) | label=([1, 0]) | total_bframes=(146)\n",
      "LOADING: directory=(training_data\\cancer\\9-28-2020-Tumor) | label=([1, 0]) | total_bframes=(31)\n",
      "LOADING: directory=(training_data\\non_cancer) | label=([0, 1]) | total_bframes=(0)\n",
      "LOADING: directory=(training_data\\non_cancer\\10-5-2020-C1-NormalWhiteMatter) | label=([0, 1]) | total_bframes=(42)\n",
      "LOADING: directory=(training_data\\non_cancer\\4-24-2018-s2) | label=([0, 1]) | total_bframes=(156)\n",
      "LOADING: directory=(training_data\\non_cancer\\8-24-2017-C2-s1) | label=([0, 1]) | total_bframes=(243)\n",
      "183/183 [==============================] - 1s 3ms/step\n",
      "183/183 [==============================] - 0s 1ms/step\n"
     ]
    }
   ],
   "source": [
    "X_train, Y_train, max_val, min_val = load_dataset('training_data')\n",
    "X_train_textures = slices_to_textures(X_train)\n",
    "X_train_embeddings = dataset_to_embeddings(X_train, X_train_textures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAINING DATA\n",
      "(num_slices, slice_height, slice_width, slice_channels): (5831, 200, 100)\n",
      "(num_textures, texture_height, texture_width, texture_channels): (5831, 100, 100, 1)\n",
      "(num_embeddings, len_embedding): (5831, 128)\n",
      "(num_labels, #_classes): (5831, 2) \n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c93fd599cb245608451e597acdd68f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Play(value=0, description='idx', interval=500, max=5830), IntSlider(value=0, description…"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('TRAINING DATA')\n",
    "print(f'(num_slices, slice_height, slice_width, slice_channels): {X_train.shape}')\n",
    "print(f'(num_textures, texture_height, texture_width, texture_channels): {X_train_textures.shape}')\n",
    "print(f'(num_embeddings, len_embedding): {X_train_embeddings.shape}')\n",
    "print(f'(num_labels, #_classes): {Y_train.shape} \\n')\n",
    "\n",
    "dataset_movie_widget(X_train_textures, Y_train) # the widget will render when this cell is run"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Network Training\n",
    "- b_frame_CNN and texture_CNN need can be trained independently of one another\n",
    "- In order to train ensemble_MLP, a pre-trained b_frame_CNN and texture_CNN is needed to convert the slices and textures into embeddings respectively. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_2 (InputLayer)        [(None, 200, 100, 1)]     0         \n",
      "                                                                 \n",
      " conv2d_4 (Conv2D)           (None, 100, 100, 32)      320       \n",
      "                                                                 \n",
      " conv2d_5 (Conv2D)           (None, 50, 50, 64)        18496     \n",
      "                                                                 \n",
      " max_pooling2d_2 (MaxPooling  (None, 24, 24, 64)       0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " gradmaps (Conv2D)           (None, 12, 12, 128)       73856     \n",
      "                                                                 \n",
      " max_pooling2d_3 (MaxPooling  (None, 5, 5, 128)        0         \n",
      " 2D)                                                             \n",
      "                                                                 \n",
      " conv2d_6 (Conv2D)           (None, 3, 3, 256)         295168    \n",
      "                                                                 \n",
      " conv2d_7 (Conv2D)           (None, 2, 2, 256)         590080    \n",
      "                                                                 \n",
      " flatten_1 (Flatten)         (None, 1024)              0         \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 1024)              0         \n",
      "                                                                 \n",
      " embedding (Dense)           (None, 64)                65600     \n",
      "                                                                 \n",
      " dropout_3 (Dropout)         (None, 64)                0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 2)                 130       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,043,650\n",
      "Trainable params: 1,043,650\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "my_bframe_CNN = B_frame_CNN(3, 'relu', 'same')\n",
    "my_bframe_CNN.model().summary()\n",
    "my_bframe_CNN.compile(\n",
    "    optimizer=tf.keras.optimizers.SGD(learning_rate=0.001), \n",
    "    loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1), \n",
    "    metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "583/583 [==============================] - 6s 9ms/step - loss: 0.6935 - accuracy: 0.5326 - val_loss: 0.6898 - val_accuracy: 0.5476\n",
      "Epoch 2/30\n",
      "583/583 [==============================] - 5s 8ms/step - loss: 0.6870 - accuracy: 0.5697 - val_loss: 0.6876 - val_accuracy: 0.5638\n",
      "Epoch 3/30\n",
      "583/583 [==============================] - 5s 8ms/step - loss: 0.6839 - accuracy: 0.5894 - val_loss: 0.6794 - val_accuracy: 0.6195\n",
      "Epoch 4/30\n",
      "583/583 [==============================] - 5s 8ms/step - loss: 0.6754 - accuracy: 0.6233 - val_loss: 0.6667 - val_accuracy: 0.6607\n",
      "Epoch 5/30\n",
      "583/583 [==============================] - 5s 8ms/step - loss: 0.6626 - accuracy: 0.6565 - val_loss: 0.6466 - val_accuracy: 0.6718\n",
      "Epoch 6/30\n",
      "583/583 [==============================] - 5s 9ms/step - loss: 0.6253 - accuracy: 0.7159 - val_loss: 0.5865 - val_accuracy: 0.7858\n",
      "Epoch 7/30\n",
      "583/583 [==============================] - 4s 6ms/step - loss: 0.5448 - accuracy: 0.7927 - val_loss: 0.5320 - val_accuracy: 0.7823\n",
      "Epoch 8/30\n",
      "583/583 [==============================] - 5s 8ms/step - loss: 0.4677 - accuracy: 0.8426 - val_loss: 0.4181 - val_accuracy: 0.8775\n",
      "Epoch 9/30\n",
      "583/583 [==============================] - 5s 8ms/step - loss: 0.4267 - accuracy: 0.8666 - val_loss: 0.3756 - val_accuracy: 0.9246\n",
      "Epoch 10/30\n",
      "583/583 [==============================] - 4s 6ms/step - loss: 0.4123 - accuracy: 0.8801 - val_loss: 0.3943 - val_accuracy: 0.8843\n",
      "Epoch 11/30\n",
      "583/583 [==============================] - 4s 6ms/step - loss: 0.4075 - accuracy: 0.8769 - val_loss: 0.3559 - val_accuracy: 0.9237\n",
      "Epoch 12/30\n",
      "583/583 [==============================] - 5s 8ms/step - loss: 0.3868 - accuracy: 0.8947 - val_loss: 0.3366 - val_accuracy: 0.9323\n",
      "Epoch 13/30\n",
      "583/583 [==============================] - 5s 9ms/step - loss: 0.3866 - accuracy: 0.8979 - val_loss: 0.3362 - val_accuracy: 0.9340\n",
      "Epoch 14/30\n",
      "583/583 [==============================] - 4s 6ms/step - loss: 0.3736 - accuracy: 0.9076 - val_loss: 0.4031 - val_accuracy: 0.8783\n",
      "Epoch 15/30\n",
      "583/583 [==============================] - 4s 7ms/step - loss: 0.3687 - accuracy: 0.9076 - val_loss: 0.3979 - val_accuracy: 0.8775\n",
      "Epoch 16/30\n",
      "583/583 [==============================] - 4s 7ms/step - loss: 0.3651 - accuracy: 0.9164 - val_loss: 0.3379 - val_accuracy: 0.9323\n",
      "Epoch 17/30\n",
      "583/583 [==============================] - 4s 7ms/step - loss: 0.3589 - accuracy: 0.9179 - val_loss: 0.3937 - val_accuracy: 0.8800\n",
      "Epoch 18/30\n",
      "583/583 [==============================] - 4s 7ms/step - loss: 0.3668 - accuracy: 0.9140 - val_loss: 0.3405 - val_accuracy: 0.9314\n",
      "Epoch 19/30\n",
      "583/583 [==============================] - 5s 9ms/step - loss: 0.3707 - accuracy: 0.9097 - val_loss: 0.3295 - val_accuracy: 0.9374\n",
      "Epoch 20/30\n",
      "583/583 [==============================] - 5s 9ms/step - loss: 0.3679 - accuracy: 0.9089 - val_loss: 0.3337 - val_accuracy: 0.9460\n",
      "Epoch 21/30\n",
      "583/583 [==============================] - 6s 10ms/step - loss: 0.3589 - accuracy: 0.9179 - val_loss: 0.3252 - val_accuracy: 0.9400\n",
      "Epoch 22/30\n",
      "583/583 [==============================] - 7s 13ms/step - loss: 0.3515 - accuracy: 0.9205 - val_loss: 0.3186 - val_accuracy: 0.9503\n",
      "Epoch 23/30\n",
      "583/583 [==============================] - 7s 11ms/step - loss: 0.3604 - accuracy: 0.9164 - val_loss: 0.3478 - val_accuracy: 0.9306\n",
      "Epoch 24/30\n",
      "583/583 [==============================] - 7s 11ms/step - loss: 0.3556 - accuracy: 0.9187 - val_loss: 0.3531 - val_accuracy: 0.9195\n",
      "Epoch 25/30\n",
      "583/583 [==============================] - 6s 11ms/step - loss: 0.3563 - accuracy: 0.9187 - val_loss: 0.3667 - val_accuracy: 0.9126\n",
      "Epoch 26/30\n",
      "583/583 [==============================] - 7s 11ms/step - loss: 0.3587 - accuracy: 0.9202 - val_loss: 0.3598 - val_accuracy: 0.9143\n",
      "Epoch 27/30\n",
      "583/583 [==============================] - 7s 11ms/step - loss: 0.3603 - accuracy: 0.9172 - val_loss: 0.3338 - val_accuracy: 0.9297\n",
      "Epoch 28/30\n",
      "583/583 [==============================] - 7s 11ms/step - loss: 0.3556 - accuracy: 0.9207 - val_loss: 0.3318 - val_accuracy: 0.9392\n",
      "Epoch 29/30\n",
      "583/583 [==============================] - 7s 12ms/step - loss: 0.3522 - accuracy: 0.9232 - val_loss: 0.3156 - val_accuracy: 0.9494\n",
      "Epoch 30/30\n",
      "583/583 [==============================] - 7s 11ms/step - loss: 0.3617 - accuracy: 0.9153 - val_loss: 0.3886 - val_accuracy: 0.8877\n"
     ]
    }
   ],
   "source": [
    "checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath = 'saved_models\\\\models_history\\\\bframe_cnn\\\\epoch_{epoch:02d}-val_acc_{val_accuracy:.2f}.tf',\n",
    "    monitor = 'val_accuracy',\n",
    "    save_best_only = True,\n",
    "    mode = 'max'\n",
    ")\n",
    "early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10) # stops training after 'patience' epochs of no improvement\n",
    "log_csv = tf.keras.callbacks.CSVLogger('saved_models\\\\models_history\\\\logs\\\\bframe_cnn_log.csv', separator=',', append=False) # save training and validation curves\n",
    "\n",
    "y_integers = np.argmax(Y_train, axis=1)\n",
    "class_weights = class_weight.compute_class_weight(\n",
    "                                        class_weight = 'balanced',\n",
    "                                        classes = np.unique(y_integers),\n",
    "                                        y = y_integers                                                  \n",
    "                                    )\n",
    "\n",
    "history = my_bframe_CNN.fit(\n",
    "    np.expand_dims(X_train, axis=3), \n",
    "    Y_train, \n",
    "    batch_size = 8,\n",
    "    shuffle = True,\n",
    "    epochs = 30, \n",
    "    validation_split = 0.2,\n",
    "    class_weight=dict(enumerate(class_weights)), \n",
    "    callbacks = [early_stop, log_csv, checkpoint]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_6\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_7 (InputLayer)        [(None, 100, 100, 1)]     0         \n",
      "                                                                 \n",
      " conv2d_20 (Conv2D)          (None, 50, 50, 32)        320       \n",
      "                                                                 \n",
      " max_pooling2d_12 (MaxPoolin  (None, 24, 24, 32)       0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " gradmaps (Conv2D)           (None, 12, 12, 32)        9248      \n",
      "                                                                 \n",
      " max_pooling2d_13 (MaxPoolin  (None, 5, 5, 32)         0         \n",
      " g2D)                                                            \n",
      "                                                                 \n",
      " conv2d_21 (Conv2D)          (None, 3, 3, 32)          9248      \n",
      "                                                                 \n",
      " conv2d_22 (Conv2D)          (None, 2, 2, 16)          4624      \n",
      "                                                                 \n",
      " embedding (Flatten)         (None, 64)                0         \n",
      "                                                                 \n",
      " dropout_8 (Dropout)         (None, 64)                0         \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 2)                 130       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 23,570\n",
      "Trainable params: 23,570\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "my_texture_CNN = texture_CNN(3, 'relu', 'same')\n",
    "my_texture_CNN.model().summary() \n",
    "my_texture_CNN.compile(\n",
    "    optimizer=tf.keras.optimizers.SGD(learning_rate=0.01), \n",
    "    loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.1), \n",
    "    metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "583/583 [==============================] - 4s 7ms/step - loss: 0.6885 - accuracy: 0.5791 - val_loss: 0.6807 - val_accuracy: 0.6984\n",
      "Epoch 2/30\n",
      "583/583 [==============================] - 3s 6ms/step - loss: 0.6594 - accuracy: 0.6711 - val_loss: 0.6113 - val_accuracy: 0.8860\n",
      "Epoch 3/30\n",
      "583/583 [==============================] - 3s 6ms/step - loss: 0.4191 - accuracy: 0.9307 - val_loss: 0.2847 - val_accuracy: 0.9632\n",
      "Epoch 4/30\n",
      "583/583 [==============================] - 3s 4ms/step - loss: 0.2894 - accuracy: 0.9595 - val_loss: 0.2766 - val_accuracy: 0.9632\n",
      "Epoch 5/30\n",
      "583/583 [==============================] - 3s 6ms/step - loss: 0.2834 - accuracy: 0.9620 - val_loss: 0.2816 - val_accuracy: 0.9649\n",
      "Epoch 6/30\n",
      "583/583 [==============================] - 3s 4ms/step - loss: 0.2777 - accuracy: 0.9631 - val_loss: 0.2759 - val_accuracy: 0.9640\n",
      "Epoch 7/30\n",
      "583/583 [==============================] - 3s 5ms/step - loss: 0.2765 - accuracy: 0.9631 - val_loss: 0.2668 - val_accuracy: 0.9700\n",
      "Epoch 8/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2740 - accuracy: 0.9659 - val_loss: 0.2661 - val_accuracy: 0.9692\n",
      "Epoch 9/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2720 - accuracy: 0.9683 - val_loss: 0.2647 - val_accuracy: 0.9683\n",
      "Epoch 10/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2717 - accuracy: 0.9674 - val_loss: 0.2654 - val_accuracy: 0.9666\n",
      "Epoch 11/30\n",
      "583/583 [==============================] - 3s 5ms/step - loss: 0.2695 - accuracy: 0.9676 - val_loss: 0.2663 - val_accuracy: 0.9709\n",
      "Epoch 12/30\n",
      "583/583 [==============================] - 3s 4ms/step - loss: 0.2663 - accuracy: 0.9700 - val_loss: 0.2590 - val_accuracy: 0.9700\n",
      "Epoch 13/30\n",
      "583/583 [==============================] - 3s 6ms/step - loss: 0.2660 - accuracy: 0.9706 - val_loss: 0.2567 - val_accuracy: 0.9734\n",
      "Epoch 14/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2649 - accuracy: 0.9689 - val_loss: 0.2577 - val_accuracy: 0.9717\n",
      "Epoch 15/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2638 - accuracy: 0.9704 - val_loss: 0.2537 - val_accuracy: 0.9709\n",
      "Epoch 16/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2618 - accuracy: 0.9726 - val_loss: 0.2527 - val_accuracy: 0.9734\n",
      "Epoch 17/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2605 - accuracy: 0.9713 - val_loss: 0.2529 - val_accuracy: 0.9734\n",
      "Epoch 18/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2601 - accuracy: 0.9730 - val_loss: 0.2517 - val_accuracy: 0.9726\n",
      "Epoch 19/30\n",
      "583/583 [==============================] - 3s 5ms/step - loss: 0.2586 - accuracy: 0.9745 - val_loss: 0.2507 - val_accuracy: 0.9777\n",
      "Epoch 20/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2579 - accuracy: 0.9738 - val_loss: 0.2626 - val_accuracy: 0.9743\n",
      "Epoch 21/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2560 - accuracy: 0.9728 - val_loss: 0.2545 - val_accuracy: 0.9777\n",
      "Epoch 22/30\n",
      "583/583 [==============================] - 3s 6ms/step - loss: 0.2546 - accuracy: 0.9760 - val_loss: 0.2500 - val_accuracy: 0.9820\n",
      "Epoch 23/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2544 - accuracy: 0.9734 - val_loss: 0.2486 - val_accuracy: 0.9777\n",
      "Epoch 24/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2532 - accuracy: 0.9762 - val_loss: 0.2466 - val_accuracy: 0.9803\n",
      "Epoch 25/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2517 - accuracy: 0.9768 - val_loss: 0.2476 - val_accuracy: 0.9786\n",
      "Epoch 26/30\n",
      "583/583 [==============================] - 3s 4ms/step - loss: 0.2511 - accuracy: 0.9747 - val_loss: 0.2477 - val_accuracy: 0.9786\n",
      "Epoch 27/30\n",
      "583/583 [==============================] - 3s 5ms/step - loss: 0.2498 - accuracy: 0.9775 - val_loss: 0.2474 - val_accuracy: 0.9786\n",
      "Epoch 28/30\n",
      "583/583 [==============================] - 3s 5ms/step - loss: 0.2485 - accuracy: 0.9764 - val_loss: 0.2554 - val_accuracy: 0.9760\n",
      "Epoch 29/30\n",
      "583/583 [==============================] - 2s 4ms/step - loss: 0.2470 - accuracy: 0.9777 - val_loss: 0.2490 - val_accuracy: 0.9794\n",
      "Epoch 30/30\n",
      "583/583 [==============================] - 3s 4ms/step - loss: 0.2464 - accuracy: 0.9792 - val_loss: 0.2475 - val_accuracy: 0.9803\n"
     ]
    }
   ],
   "source": [
    "checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath = 'saved_models\\\\models_history\\\\texture_cnn\\\\epoch_{epoch:02d}-val_acc_{val_accuracy:.2f}.tf',\n",
    "    monitor = 'val_accuracy',\n",
    "    save_best_only = True, # only save model if val_accuracy improves\n",
    "    mode = 'max' # higher val_accuracy is better\n",
    ")\n",
    "early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10) # stops training after 'patience' epochs of no improvement\n",
    "log_csv = tf.keras.callbacks.CSVLogger('saved_models\\\\models_history\\\\logs\\\\texture_cnn_log.csv', separator=',', append=False) # save training and validation curves\n",
    "\n",
    "history = my_texture_CNN.fit(\n",
    "    X_train_textures, \n",
    "    Y_train, \n",
    "    batch_size = 8,\n",
    "    shuffle = True,\n",
    "    epochs = 30, \n",
    "    validation_split = 0.2,\n",
    "    class_weight=dict(enumerate(class_weights)), \n",
    "    callbacks = [early_stop, log_csv, checkpoint]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_9\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_6 (InputLayer)        [(None, 1, 128)]          0         \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 1, 64)             8256      \n",
      "                                                                 \n",
      " dropout_7 (Dropout)         (None, 1, 64)             0         \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 1, 2)              130       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 8,386\n",
      "Trainable params: 8,386\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "my_ensemble_MLP = ensemble_MLP()\n",
    "my_ensemble_MLP.model().summary()\n",
    "my_ensemble_MLP.compile(\n",
    "    optimizer=tf.keras.optimizers.SGD(learning_rate=0.01), \n",
    "    loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0), \n",
    "    metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "583/583 [==============================] - 2s 3ms/step - loss: 0.1294 - accuracy: 0.9762 - val_loss: 0.0730 - val_accuracy: 0.9820\n",
      "Epoch 2/30\n",
      "583/583 [==============================] - 2s 3ms/step - loss: 0.0647 - accuracy: 0.9798 - val_loss: 0.0569 - val_accuracy: 0.9837\n",
      "Epoch 3/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0555 - accuracy: 0.9813 - val_loss: 0.0547 - val_accuracy: 0.9829\n",
      "Epoch 4/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0525 - accuracy: 0.9813 - val_loss: 0.0555 - val_accuracy: 0.9829\n",
      "Epoch 5/30\n",
      "583/583 [==============================] - 2s 3ms/step - loss: 0.0501 - accuracy: 0.9822 - val_loss: 0.0528 - val_accuracy: 0.9854\n",
      "Epoch 6/30\n",
      "583/583 [==============================] - 2s 3ms/step - loss: 0.0460 - accuracy: 0.9846 - val_loss: 0.0517 - val_accuracy: 0.9829\n",
      "Epoch 7/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0457 - accuracy: 0.9835 - val_loss: 0.0495 - val_accuracy: 0.9854\n",
      "Epoch 8/30\n",
      "583/583 [==============================] - 2s 3ms/step - loss: 0.0454 - accuracy: 0.9826 - val_loss: 0.0479 - val_accuracy: 0.9871\n",
      "Epoch 9/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0447 - accuracy: 0.9850 - val_loss: 0.0467 - val_accuracy: 0.9863\n",
      "Epoch 10/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0438 - accuracy: 0.9841 - val_loss: 0.0434 - val_accuracy: 0.9871\n",
      "Epoch 11/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0431 - accuracy: 0.9848 - val_loss: 0.0459 - val_accuracy: 0.9871\n",
      "Epoch 12/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0409 - accuracy: 0.9846 - val_loss: 0.0424 - val_accuracy: 0.9863\n",
      "Epoch 13/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0409 - accuracy: 0.9854 - val_loss: 0.0455 - val_accuracy: 0.9863\n",
      "Epoch 14/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0401 - accuracy: 0.9846 - val_loss: 0.0457 - val_accuracy: 0.9863\n",
      "Epoch 15/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0391 - accuracy: 0.9854 - val_loss: 0.0429 - val_accuracy: 0.9871\n",
      "Epoch 16/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0382 - accuracy: 0.9850 - val_loss: 0.0440 - val_accuracy: 0.9863\n",
      "Epoch 17/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0371 - accuracy: 0.9863 - val_loss: 0.0425 - val_accuracy: 0.9863\n",
      "Epoch 18/30\n",
      "583/583 [==============================] - 1s 2ms/step - loss: 0.0382 - accuracy: 0.9863 - val_loss: 0.0419 - val_accuracy: 0.9871\n"
     ]
    }
   ],
   "source": [
    "checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "    filepath = 'saved_models\\\\models_history\\\\ensemble_mlp\\\\epoch_{epoch:02d}-val_acc_{val_accuracy:.2f}.tf',\n",
    "    monitor = 'val_accuracy',\n",
    "    save_best_only = True,\n",
    "    mode = 'max'\n",
    ")\n",
    "early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10)\n",
    "log_csv = tf.keras.callbacks.CSVLogger('saved_models\\\\models_history\\\\logs\\\\ensemble_mlp_log.csv', separator=',', append=False)\n",
    "\n",
    "history = my_ensemble_MLP.fit(\n",
    "    X_train_embeddings,\n",
    "    Y_train, \n",
    "    batch_size = 8,\n",
    "    shuffle = True,\n",
    "    epochs = 30, \n",
    "    validation_split = 0.2,\n",
    "    callbacks = [early_stop, log_csv, checkpoint]\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.11 ('tf_gpu')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b631b1a244feb9a118b2d050b724f81aaa8786bc0e8199c429b30da09a8722a1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}