MNIST_final_solution.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "A9PEb2mLkde2"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras import optimizers\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "eEkKbcEwkkPn"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.datasets import mnist\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "x_train, x_test = np.array(x_train, np.float32), np.array(x_test, np.float32)\n",
    "\n",
    "x_train, x_test = x_train / 255-0.5, x_test / 255-0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RSuRlnsUkol3"
   },
   "outputs": [],
   "source": [
    "y_train=tf.keras.utils.to_categorical(y_train,10)\n",
    "y_test=tf.keras.utils.to_categorical(y_test,10)\n",
    "x_train=np.expand_dims(x_train,axis=-1)\n",
    "x_test=np.expand_dims(x_test,axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l3PntfIGj-Ll"
   },
   "outputs": [],
   "source": [
    "def squeeze_excite_block2D(filters,input):                       # squeeze and exite is a good thing\n",
    "    se = tf.keras.layers.GlobalAveragePooling2D()(input)\n",
    "    se = tf.keras.layers.Reshape((1, filters))(se) \n",
    "    se = tf.keras.layers.Dense(filters//32, activation='relu')(se)\n",
    "    se = tf.keras.layers.Dense(filters, activation='sigmoid')(se)\n",
    "    se = tf.keras.layers.multiply([input, se])\n",
    "    return se\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "datagen = tf.keras.preprocessing.image.ImageDataGenerator(rotation_range=10, width_shift_range=0.1, shear_range=10,\n",
    "                        height_shift_range=0.1, zoom_range=0.2)\n",
    "datagen.fit(x_train)\n",
    "datagen2 = tf.keras.preprocessing.image.ImageDataGenerator()\n",
    "datagen2.fit(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_model():\n",
    "        s = tf.keras.Input(shape=x_train.shape[1:]) \n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(s)\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = squeeze_excite_block2D(128,x)\n",
    "\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = squeeze_excite_block2D(128,x)\n",
    "        x = tf.keras.layers.AveragePooling2D(2)(x)\n",
    "\n",
    "\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.Conv2D(128,(3,3),activation='relu',padding='same')(x)\n",
    "        x = tf.keras.layers.BatchNormalization()(x)\n",
    "        x = squeeze_excite_block2D(128,x)\n",
    "        x = tf.keras.layers.AveragePooling2D(2)(x)\n",
    "\n",
    "\n",
    "        x = tf.keras.layers.concatenate([tf.keras.layers.GlobalMaxPooling2D()(x),\n",
    "                                         tf.keras.layers.GlobalAveragePooling2D()(x)])\n",
    "\n",
    "        x = tf.keras.layers.Dense(10,activation='softmax',use_bias=False,\n",
    "                                  kernel_regularizer=tf.keras.regularizers.l1(0.00025))(x) # this make stacking better\n",
    "        return tf.keras.Model(inputs=s, outputs=x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ZhFRi8UHkxgH",
    "outputId": "955c132a-79f9-4925-931f-8558490ca977",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 acc: 0.9973\n",
      "1 acc: 0.9977\n",
      "2 acc: 0.9973\n",
      "3 acc: 0.9971\n",
      "4 acc: 0.9976\n",
      "5 acc: 0.9977\n",
      "6 acc: 0.9978\n",
      "7 acc: 0.9974\n",
      "8 acc: 0.9978\n",
      "9 acc: 0.9981\n",
      "10 acc: 0.9974\n",
      "11 acc: 0.9971\n",
      "12 acc: 0.9975\n",
      "13 acc: 0.9974\n",
      "14 acc: 0.9976\n",
      "15 acc: 0.9973\n",
      "16 acc: 0.9974\n",
      "17 acc: 0.9976\n",
      "18 acc: 0.9969\n",
      "19 acc: 0.9975\n"
     ]
    }
   ],
   "source": [
    "batch_size=32\n",
    "supermodel=[]\n",
    "for i in range(20):\n",
    "        np.random.seed(i)\n",
    "        model=make_model()                \n",
    "        model.compile(optimizer=optimizers.Adam(lr=0.001), loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "        model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size,shuffle=True),\n",
    "                    steps_per_epoch=len(x_train) / batch_size, epochs=13,verbose=0)\n",
    "        model.compile(optimizer=optimizers.Adam(lr=0.0001), loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "        model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size,shuffle=True),\n",
    "                    steps_per_epoch=len(x_train) / batch_size, epochs=3,verbose=0)\n",
    "        model.compile(optimizer=optimizers.Adam(lr=0.00001), loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "        model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size,shuffle=True),\n",
    "                    steps_per_epoch=len(x_train) / batch_size, epochs=3,verbose=0)\n",
    "        model.fit(x_train, y_train, batch_size=batch_size,shuffle=True, epochs=1,verbose=0)\n",
    "        supermodel.append(model)\n",
    "        print(i,'acc:',accuracy_score(np.argmax(y_test,axis=1),np.argmax(model.predict(x_test),axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "P=np.asarray([a.predict(x_test) for a in supermodel])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9984"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(np.argmax(y_test,axis=1),np.argmax(np.mean(P,axis=0),axis=1)) # 20 models stack accurasy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# after 15 run error was 0.16%,0.16%,0.18%,0.16%,0.17%,0.16%,0.18%,0.18%,0.17%,0.18%,0.16%,0.17%,0.18%,0.17%,0.17%"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": " Mnist_example_jau_kitas_CNN_2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}