cifar2-noquant.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX 1070 (CNMeM is enabled with initial size: 40.0% of memory, cuDNN 5005)\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cuda_convnet (faster)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "# os.environ['THEANO_FLAGS']='contexts=dev0->cuda0;dev1->cuda1'\n",
    "os.environ['THEANO_FLAGS']='device=gpu0'\n",
    "import time\n",
    "import matplotlib\n",
    "import lasagne\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import pylab as P\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import cPickle as pickle\n",
    "from helpers.DiscreteLayer import DiscreteLayer\n",
    "try:\n",
    "    from lasagne.layers.dnn import Conv2DDNNLayer as conv\n",
    "    from lasagne.layers.dnn import MaxPool2DDNNLayer as pool\n",
    "    print('Using cuda_convnet (faster)')\n",
    "except ImportError:\n",
    "    from lasagne.layers import Conv2DLayer as conv\n",
    "    from lasagne.layers import MaxPool2DLayer as pool\n",
    "from keras.datasets import cifar10\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Constants\n",
    "NUM_EPOCHS = 1500\n",
    "BATCH_SIZE = 256\n",
    "LEARNING_RATE = 0.001\n",
    "DIM = 32\n",
    "CHANNEL = 3\n",
    "NUM_CLASSES = 10\n",
    "\n",
    "# Disc. Layer Settings\n",
    "DISC = False\n",
    "QUANT = np.array([0.125, 0.125, 0.125, 0.125, 0.125, 0.125], dtype='float32')\n",
    "VARIANCE_DEVIDER = 16.0\n",
    "\n",
    "# Test Specs\n",
    "TEST_NAME = 'cifar10-nodisc'\n",
    "\n",
    "# Additional Settings\n",
    "lasagne.random.set_rng(np.random.RandomState(12345))  # Set random state so we can investigate results\n",
    "np.random.seed(1234)\n",
    "#theano.config.exception_verbosity = 'high'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('X_train shape:', (50000, 3, 32, 32))\n",
      "(50000, 'train samples')\n",
      "(10000, 'test samples')\n"
     ]
    }
   ],
   "source": [
    "def load_data():\n",
    "    (X_train, y_train), (X_test, y_test) = cifar10.load_data()\n",
    "    X_train = X_train.astype('float32')\n",
    "    X_test = X_test.astype('float32')\n",
    "    X_train /= 255\n",
    "    X_test /= 255\n",
    "    \n",
    "    X_train = np.transpose(X_train, (0, 3, 2, 1))\n",
    "    X_test = np.transpose(X_test, (0, 3, 2, 1))\n",
    "    \n",
    "    # convert class vectors to binary class matrices\n",
    "    y_train = np.squeeze(y_train)\n",
    "    y_test = np.squeeze(y_test)\n",
    "    \n",
    "    print('X_train shape:', X_train.shape)\n",
    "    print(X_train.shape[0], 'train samples')\n",
    "    print(X_test.shape[0], 'test samples')\n",
    "\n",
    "    return dict(\n",
    "        X_train=X_train,\n",
    "        y_train=y_train,\n",
    "        X_test=X_test,\n",
    "        y_test=y_test,\n",
    "        num_examples_train=X_train.shape[0],\n",
    "        num_examples_test=X_test.shape[0],\n",
    "        input_height=X_train.shape[2],\n",
    "        input_width=X_train.shape[3],\n",
    "        input_channel=X_train.shape[1],\n",
    "        output_dim=10,)\n",
    "data = load_data()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Disc. Layer\n",
      "Transformer network output shape:  (None, 3, 32, 32)\n"
     ]
    }
   ],
   "source": [
    "def build_model(input_width, input_height, channel, output_dim,\n",
    "                batch_size=BATCH_SIZE, withdisc=True):\n",
    "    ini = lasagne.init.HeUniform()\n",
    "    l_in = lasagne.layers.InputLayer(shape=(None, channel, input_width, input_height),)\n",
    "    #l_in = lasagne.layers.DropoutLayer(l_in, p=0.1)\n",
    "\n",
    "    # Localization network\n",
    "    b = np.zeros((2, 3), dtype=theano.config.floatX)\n",
    "    b[0, 0] = 1\n",
    "    b[1, 1] = 1\n",
    "    b = b.flatten()\n",
    "    loc_l1 = pool(l_in, pool_size=(2, 2))\n",
    "    loc_l2 = conv(\n",
    "        loc_l1, num_filters=20, filter_size=(5, 5), W=ini)\n",
    "    loc_l3 = pool(loc_l2, pool_size=(2, 2))\n",
    "    loc_l4 = conv(loc_l3, num_filters=20, filter_size=(5, 5), W=ini)\n",
    "    loc_l5 = lasagne.layers.DenseLayer(\n",
    "        loc_l4, num_units=50, W=lasagne.init.HeUniform('relu'))\n",
    "    loc_out = lasagne.layers.DenseLayer(\n",
    "        loc_l5, num_units=6, b=b, W=lasagne.init.Constant(0.0), \n",
    "        nonlinearity=lasagne.nonlinearities.identity, name='param_regressor')\n",
    "    \n",
    "    if withdisc:\n",
    "        sharedBins = theano.shared(None, name='sharedBins')\n",
    "        l_dis = DiscreteLayer(loc_out, sharedBins=sharedBins, name='disclayer')\n",
    "        print(\"Using Discret. Layer\")\n",
    "    else:\n",
    "        l_dis = loc_out\n",
    "        print(\"No Disc. Layer\")\n",
    "    \n",
    "    # Transformer network\n",
    "    l_trans1 = lasagne.layers.TransformerLayer(l_in, l_dis, downsample_factor=1.0)\n",
    "    print \"Transformer network output shape: \", l_trans1.output_shape\n",
    "    \n",
    "    # Classification network\n",
    "    class_l1 = conv(\n",
    "        l_trans1,\n",
    "        num_filters=32,\n",
    "        filter_size=(3, 3),\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "    class_l2 = pool(class_l1, pool_size=(2, 2))\n",
    "    class_l3 = conv(\n",
    "        class_l2,\n",
    "        num_filters=32,\n",
    "        filter_size=(3, 3),\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "    class_l4 = pool(class_l3, pool_size=(2, 2))\n",
    "    class_l5 = lasagne.layers.DenseLayer(\n",
    "        class_l4,\n",
    "        num_units=256,\n",
    "        nonlinearity=lasagne.nonlinearities.rectify,\n",
    "        W=ini,\n",
    "    )\n",
    "\n",
    "    l_out = lasagne.layers.DenseLayer(\n",
    "        class_l5,\n",
    "        num_units=output_dim,\n",
    "        nonlinearity=lasagne.nonlinearities.softmax,\n",
    "        W=ini,\n",
    "    )\n",
    "    if withdisc:\n",
    "        return l_out, l_trans1, sharedBins\n",
    "    else:\n",
    "        return l_out, l_trans1\n",
    "\n",
    "if DISC:\n",
    "    model, l_transform, sharedBins = build_model(DIM, DIM, CHANNEL, NUM_CLASSES, withdisc=DISC)\n",
    "else:\n",
    "    model, l_transform = build_model(DIM, DIM, CHANNEL, NUM_CLASSES, withdisc=DISC)\n",
    "\n",
    "model_params = lasagne.layers.get_all_params(model, trainable=True)\n",
    "# params = lasagne.layers.get_all_params(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Input: dist, dist.shape = (-1, num_units)\n",
    "Find quantization bins of a given dist history\n",
    "Returns a list of (x, num_units), where x's length is a random variable\n",
    "\"\"\"\n",
    "def find_quantization_bins(dist, sharedBins):\n",
    "    # Quantizer function\n",
    "    def Q(x, y):\n",
    "        return y * np.floor((x/y) + .5)\n",
    "    \n",
    "    shape = dist.shape\n",
    "    init_Q = QUANT\n",
    "    final_Q = []\n",
    "    \n",
    "    # Theta iterator\n",
    "    for i in range(shape[1]):\n",
    "        theta_i = dist[:, i]\n",
    "        \n",
    "        # Whats is the error threshold for this distribution\n",
    "        Q_eps = np.var(theta_i) / VARIANCE_DEVIDER\n",
    "        \n",
    "        # Batch Iterator\n",
    "        final_Q_i = []\n",
    "        for j in range(shape[0]):\n",
    "            theta = theta_i[j]\n",
    "            \n",
    "            # Quantized theta = Quantization bins\n",
    "            q = init_Q[i]\n",
    "            x_i = theta\n",
    "            x_o = Q(x_i, q)\n",
    "            \n",
    "            # Optimize x_o\n",
    "\n",
    "            while(np.abs(x_o - x_i) > Q_eps):\n",
    "                q = q / 2\n",
    "                x_o = Q(x_i, q)\n",
    "            \n",
    "            # End of optimisation\n",
    "            final_Q_i.append(x_o)\n",
    "        \n",
    "        # Append to outer list\n",
    "        uniques = np.unique(np.array(final_Q_i))\n",
    "        final_Q.append(uniques.astype(theano.config.floatX))\n",
    "        \n",
    "    # Report\n",
    "    print \"New Bin Sizes: [\" + \", \".join([str(final_Q[x].shape[0]) for x in range(shape[1])] ) + \"]\"\n",
    "    sharedBins.set_value(final_Q)\n",
    "    return final_Q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def build_functions():\n",
    "    X = T.tensor4(dtype=theano.config.floatX)\n",
    "    y = T.ivector()\n",
    "\n",
    "    ## Layer History\n",
    "    if DISC:\n",
    "        l_disc = next(l for l in lasagne.layers.get_all_layers(model) if l.name is 'disclayer')\n",
    "        l_paramreg = next(l for l in lasagne.layers.get_all_layers(model) if l.name is 'param_regressor')\n",
    "        l_disc_output, l_paramreg_output = lasagne.layers.get_output([l_disc, l_paramreg], X, deterministic=False)\n",
    "    ## Layer History\n",
    "\n",
    "    # training output\n",
    "    output_train = lasagne.layers.get_output(model, X, deterministic=False)\n",
    "\n",
    "    # evaluation output. Also includes output of transform for plotting\n",
    "    output_eval, transform_eval = lasagne.layers.get_output([model, l_transform], X, deterministic=True)\n",
    "\n",
    "    sh_lr = theano.shared(lasagne.utils.floatX(LEARNING_RATE))\n",
    "    cost = T.mean(T.nnet.categorical_crossentropy(output_train, y))\n",
    "    updates = lasagne.updates.adam(cost, model_params, learning_rate=sh_lr)\n",
    "\n",
    "    if DISC:\n",
    "        train = theano.function([X, y], [cost, output_train, l_disc_output, l_paramreg_output], updates=updates)\n",
    "    else:\n",
    "        train = theano.function([X, y], [cost, output_train], updates=updates)\n",
    "    eval = theano.function([X], [output_eval, transform_eval])\n",
    "    \n",
    "    return train, eval, sh_lr\n",
    "\n",
    "train, eval, sh_lr = build_functions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_epoch(X, y):\n",
    "    # History Keeping\n",
    "    param_output = []\n",
    "    disc_output = []\n",
    "    # History\n",
    "    num_samples = X.shape[0]\n",
    "    num_batches = int(np.ceil(num_samples / float(BATCH_SIZE)))\n",
    "    costs = []\n",
    "    correct = 0\n",
    "    for i in range(num_batches):\n",
    "        idx = range(i*BATCH_SIZE, np.minimum((i+1)*BATCH_SIZE, num_samples))\n",
    "        X_batch = X[idx]\n",
    "        y_batch = y[idx]\n",
    "        if DISC:\n",
    "            cost, output_train, l_disc_output, l_paramreg_output = train(X_batch, y_batch)\n",
    "            param_output = np.append(param_output, l_paramreg_output)\n",
    "            disc_output = np.append(disc_output, l_disc_output)\n",
    "        else:\n",
    "            cost, output_train = train(X_batch, y_batch)\n",
    "        costs += [cost]\n",
    "        preds = np.argmax(output_train, axis=-1)\n",
    "        correct += np.sum(y_batch == preds)\n",
    "    \n",
    "    return np.mean(costs), correct / float(num_samples), param_output, disc_output\n",
    "\n",
    "\n",
    "def eval_epoch(X, y):\n",
    "    output_eval, transform_eval = eval(X)\n",
    "    preds = np.argmax(output_eval, axis=-1)\n",
    "    acc = np.mean(preds == y)\n",
    "    return acc, transform_eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0: T.cost 1.620609, Train acc 0.420780, test acc 0.513700, took 4.33 sec.\n",
      "Epoch 1: T.cost 1.285203, Train acc 0.543660, test acc 0.568900, took 4.35 sec.\n",
      "Epoch 2: T.cost 1.156990, Train acc 0.592080, test acc 0.598700, took 4.16 sec.\n",
      "Epoch 3: T.cost 1.073900, Train acc 0.622920, test acc 0.611000, took 4.18 sec.\n",
      "Epoch 4: T.cost 1.003908, Train acc 0.649180, test acc 0.630300, took 4.22 sec.\n",
      "Epoch 5: T.cost 0.945319, Train acc 0.673120, test acc 0.636000, took 4.26 sec.\n",
      "Epoch 6: T.cost 0.892985, Train acc 0.689540, test acc 0.647700, took 4.21 sec.\n",
      "Epoch 7: T.cost 0.846352, Train acc 0.707240, test acc 0.662200, took 4.2 sec.\n",
      "Epoch 8: T.cost 0.801460, Train acc 0.722420, test acc 0.668000, took 4.2 sec.\n",
      "Epoch 9: T.cost 0.766847, Train acc 0.735980, test acc 0.672800, took 4.2 sec.\n",
      "Epoch 10: T.cost 0.734637, Train acc 0.747540, test acc 0.678200, took 4.2 sec.\n",
      "Epoch 11: T.cost 0.694783, Train acc 0.762780, test acc 0.676400, took 4.19 sec.\n",
      "Epoch 12: T.cost 0.673634, Train acc 0.769540, test acc 0.665200, took 4.2 sec.\n",
      "Epoch 13: T.cost 0.642726, Train acc 0.779060, test acc 0.663600, took 4.19 sec.\n",
      "Epoch 14: T.cost 0.612959, Train acc 0.789400, test acc 0.661200, took 4.21 sec.\n",
      "Epoch 15: T.cost 0.584202, Train acc 0.799600, test acc 0.670000, took 4.2 sec.\n",
      "Epoch 16: T.cost 0.559770, Train acc 0.809120, test acc 0.661100, took 4.2 sec.\n",
      "Epoch 17: T.cost 0.544589, Train acc 0.813340, test acc 0.663800, took 4.18 sec.\n",
      "Epoch 18: T.cost 0.523472, Train acc 0.820740, test acc 0.667700, took 4.22 sec.\n",
      "New LR: 0.000990000047022\n",
      "Epoch 19: T.cost 0.499944, Train acc 0.826700, test acc 0.664000, took 4.22 sec.\n",
      "Epoch 20: T.cost 0.470460, Train acc 0.836580, test acc 0.660400, took 4.19 sec.\n",
      "Epoch 21: T.cost 0.442180, Train acc 0.847000, test acc 0.658100, took 4.15 sec.\n",
      "Epoch 22: T.cost 0.424098, Train acc 0.852720, test acc 0.661200, took 4.21 sec.\n",
      "Epoch 23: T.cost 0.405313, Train acc 0.859120, test acc 0.655700, took 4.19 sec.\n",
      "Epoch 24: T.cost 0.395486, Train acc 0.862680, test acc 0.645900, took 4.19 sec.\n",
      "Epoch 25: T.cost 0.389796, Train acc 0.864160, test acc 0.650600, took 4.31 sec.\n",
      "Epoch 26: T.cost 0.368855, Train acc 0.871200, test acc 0.657700, took 4.22 sec.\n",
      "Epoch 27: T.cost 0.350202, Train acc 0.878700, test acc 0.660400, took 4.22 sec.\n",
      "Epoch 28: T.cost 0.331989, Train acc 0.882840, test acc 0.667200, took 4.18 sec.\n",
      "Epoch 29: T.cost 0.303051, Train acc 0.895860, test acc 0.664800, took 4.21 sec.\n",
      "Epoch 30: T.cost 0.277236, Train acc 0.906340, test acc 0.661600, took 4.21 sec.\n",
      "Epoch 31: T.cost 0.271154, Train acc 0.905760, test acc 0.667100, took 4.21 sec.\n",
      "Epoch 32: T.cost 0.269432, Train acc 0.906560, test acc 0.674900, took 4.19 sec.\n",
      "Epoch 33: T.cost 0.273778, Train acc 0.902860, test acc 0.673600, took 4.19 sec.\n",
      "Epoch 34: T.cost 0.259719, Train acc 0.907760, test acc 0.674200, took 4.21 sec.\n",
      "Epoch 35: T.cost 0.223983, Train acc 0.923020, test acc 0.667000, took 4.19 sec.\n",
      "Epoch 36: T.cost 0.197278, Train acc 0.934180, test acc 0.664200, took 4.19 sec.\n",
      "Epoch 37: T.cost 0.182457, Train acc 0.938680, test acc 0.661100, took 4.19 sec.\n",
      "Epoch 38: T.cost 0.179313, Train acc 0.939360, test acc 0.667900, took 4.2 sec.\n",
      "New LR: 0.00098010008689\n",
      "Epoch 39: T.cost 0.182380, Train acc 0.936300, test acc 0.673300, took 4.21 sec.\n",
      "Epoch 40: T.cost 0.184556, Train acc 0.934360, test acc 0.668000, took 4.19 sec.\n",
      "Epoch 41: T.cost 0.183690, Train acc 0.934840, test acc 0.666200, took 4.19 sec.\n",
      "Epoch 42: T.cost 0.183760, Train acc 0.933860, test acc 0.668600, took 4.2 sec.\n",
      "Epoch 43: T.cost 0.176128, Train acc 0.936960, test acc 0.657000, took 4.19 sec.\n",
      "Epoch 44: T.cost 0.159651, Train acc 0.943920, test acc 0.651600, took 4.18 sec.\n",
      "Epoch 45: T.cost 0.158333, Train acc 0.942500, test acc 0.665400, took 4.24 sec.\n",
      "Epoch 46: T.cost 0.146357, Train acc 0.948340, test acc 0.662000, took 4.22 sec.\n",
      "Epoch 47: T.cost 0.133339, Train acc 0.953280, test acc 0.664800, took 4.2 sec.\n",
      "Epoch 48: T.cost 0.110999, Train acc 0.962680, test acc 0.655000, took 4.19 sec.\n",
      "Epoch 49: T.cost 0.102676, Train acc 0.965800, test acc 0.653100, took 4.21 sec.\n",
      "Epoch 50: T.cost 0.095328, Train acc 0.969320, test acc 0.665100, took 4.21 sec.\n",
      "Epoch 51: T.cost 0.096248, Train acc 0.967980, test acc 0.667900, took 4.22 sec.\n",
      "Epoch 52: T.cost 0.099006, Train acc 0.966080, test acc 0.670400, took 4.23 sec.\n",
      "Epoch 53: T.cost 0.103682, Train acc 0.963260, test acc 0.661800, took 4.21 sec.\n",
      "Epoch 54: T.cost 0.105168, Train acc 0.963340, test acc 0.660800, took 4.21 sec.\n",
      "Epoch 55: T.cost 0.093636, Train acc 0.967360, test acc 0.653500, took 4.21 sec.\n",
      "Epoch 56: T.cost 0.090045, Train acc 0.968940, test acc 0.654400, took 4.17 sec.\n",
      "Epoch 57: T.cost 0.089248, Train acc 0.968360, test acc 0.670000, took 4.15 sec.\n",
      "Epoch 58: T.cost 0.075167, Train acc 0.975000, test acc 0.666400, took 4.21 sec.\n",
      "New LR: 0.000970299127512\n",
      "Epoch 59: T.cost 0.067690, Train acc 0.978000, test acc 0.668900, took 4.19 sec.\n",
      "Epoch 60: T.cost 0.060380, Train acc 0.981300, test acc 0.671200, took 4.21 sec.\n",
      "Epoch 61: T.cost 0.057647, Train acc 0.981860, test acc 0.667900, took 4.2 sec.\n",
      "Epoch 62: T.cost 0.054743, Train acc 0.982280, test acc 0.670800, took 4.19 sec.\n",
      "Epoch 63: T.cost 0.048588, Train acc 0.984360, test acc 0.671400, took 4.21 sec.\n",
      "Epoch 64: T.cost 0.049414, Train acc 0.984760, test acc 0.667600, took 4.16 sec.\n",
      "Epoch 65: T.cost 0.052908, Train acc 0.982480, test acc 0.664400, took 4.2 sec.\n",
      "Epoch 66: T.cost 0.053963, Train acc 0.982200, test acc 0.669500, took 4.21 sec.\n",
      "Epoch 67: T.cost 0.054332, Train acc 0.982780, test acc 0.667000, took 4.21 sec.\n",
      "Epoch 68: T.cost 0.052512, Train acc 0.983280, test acc 0.661600, took 4.22 sec.\n",
      "Epoch 69: T.cost 0.051681, Train acc 0.982220, test acc 0.661700, took 4.21 sec.\n",
      "Epoch 70: T.cost 0.052904, Train acc 0.982160, test acc 0.662000, took 4.21 sec.\n",
      "Epoch 71: T.cost 0.051585, Train acc 0.982180, test acc 0.658900, took 4.21 sec.\n",
      "Epoch 72: T.cost 0.047012, Train acc 0.984540, test acc 0.664400, took 4.23 sec.\n",
      "Epoch 73: T.cost 0.047567, Train acc 0.983840, test acc 0.660900, took 4.22 sec.\n",
      "Epoch 74: T.cost 0.042793, Train acc 0.985920, test acc 0.655800, took 4.21 sec.\n",
      "Epoch 75: T.cost 0.044084, Train acc 0.985820, test acc 0.664600, took 4.18 sec.\n",
      "Epoch 76: T.cost 0.041187, Train acc 0.986360, test acc 0.660400, took 4.21 sec.\n",
      "Epoch 77: T.cost 0.043469, Train acc 0.985500, test acc 0.670400, took 4.21 sec.\n",
      "Epoch 78: T.cost 0.043142, Train acc 0.985020, test acc 0.669600, took 4.21 sec.\n",
      "New LR: 0.000960596131627\n",
      "Epoch 79: T.cost 0.039951, Train acc 0.986580, test acc 0.669200, took 4.21 sec.\n",
      "Epoch 80: T.cost 0.039246, Train acc 0.987220, test acc 0.669200, took 4.2 sec.\n",
      "Epoch 81: T.cost 0.032924, Train acc 0.989360, test acc 0.668600, took 4.21 sec.\n",
      "Epoch 82: T.cost 0.032876, Train acc 0.989080, test acc 0.667200, took 4.2 sec.\n",
      "Epoch 83: T.cost 0.035341, Train acc 0.988200, test acc 0.666900, took 4.17 sec.\n",
      "Epoch 84: T.cost 0.031052, Train acc 0.989960, test acc 0.669900, took 4.15 sec.\n",
      "Epoch 85: T.cost 0.029144, Train acc 0.990820, test acc 0.667700, took 4.2 sec.\n",
      "Epoch 86: T.cost 0.031127, Train acc 0.989520, test acc 0.666900, took 4.2 sec.\n",
      "Epoch 87: T.cost 0.028842, Train acc 0.990440, test acc 0.667400, took 4.17 sec.\n",
      "Epoch 88: T.cost 0.028011, Train acc 0.990660, test acc 0.663700, took 4.19 sec.\n",
      "Epoch 89: T.cost 0.031690, Train acc 0.989260, test acc 0.669100, took 4.19 sec.\n",
      "Epoch 90: T.cost 0.032481, Train acc 0.989420, test acc 0.665600, took 4.2 sec.\n",
      "Epoch 91: T.cost 0.026931, Train acc 0.991100, test acc 0.667100, took 4.19 sec.\n",
      "Epoch 92: T.cost 0.030630, Train acc 0.989700, test acc 0.673400, took 4.17 sec.\n",
      "Epoch 93: T.cost 0.035810, Train acc 0.988800, test acc 0.667500, took 4.21 sec.\n",
      "Epoch 94: T.cost 0.034444, Train acc 0.988540, test acc 0.661800, took 4.17 sec.\n",
      "Epoch 95: T.cost 0.029294, Train acc 0.990200, test acc 0.668900, took 4.18 sec.\n",
      "Epoch 96: T.cost 0.030341, Train acc 0.990420, test acc 0.664100, took 4.21 sec.\n",
      "Epoch 97: T.cost 0.029388, Train acc 0.990640, test acc 0.671500, took 4.21 sec.\n",
      "Epoch 98: T.cost 0.023136, Train acc 0.992540, test acc 0.665100, took 4.19 sec.\n",
      "New LR: 0.000950990177225\n",
      "Epoch 99: T.cost 0.022994, Train acc 0.991940, test acc 0.663700, took 4.22 sec.\n",
      "Epoch 100: T.cost 0.026238, Train acc 0.991640, test acc 0.669100, took 4.19 sec.\n",
      "Epoch 101: T.cost 0.020433, Train acc 0.993760, test acc 0.667900, took 4.21 sec.\n",
      "Epoch 102: T.cost 0.022881, Train acc 0.992400, test acc 0.670100, took 4.21 sec.\n",
      "Epoch 103: T.cost 0.023629, Train acc 0.991840, test acc 0.661800, took 4.21 sec.\n",
      "Epoch 104: T.cost 0.025734, Train acc 0.991600, test acc 0.668400, took 4.19 sec.\n",
      "Epoch 105: T.cost 0.024572, Train acc 0.991100, test acc 0.667000, took 4.19 sec.\n",
      "Epoch 106: T.cost 0.016202, Train acc 0.995020, test acc 0.662100, took 4.22 sec.\n",
      "Epoch 107: T.cost 0.021960, Train acc 0.992940, test acc 0.672000, took 4.21 sec.\n",
      "Epoch 108: T.cost 0.024422, Train acc 0.991600, test acc 0.669300, took 4.18 sec.\n",
      "Epoch 109: T.cost 0.028303, Train acc 0.990640, test acc 0.672300, took 4.18 sec.\n",
      "Epoch 110: T.cost 0.027705, Train acc 0.990720, test acc 0.664900, took 4.17 sec.\n",
      "Epoch 111: T.cost 0.018551, Train acc 0.993660, test acc 0.667700, took 4.21 sec.\n",
      "Epoch 112: T.cost 0.017384, Train acc 0.994420, test acc 0.668500, took 4.23 sec.\n",
      "Epoch 113: T.cost 0.020736, Train acc 0.993360, test acc 0.671100, took 4.16 sec.\n",
      "Epoch 114: T.cost 0.023161, Train acc 0.992280, test acc 0.667800, took 4.18 sec.\n",
      "Epoch 115: T.cost 0.027623, Train acc 0.990340, test acc 0.659600, took 4.2 sec.\n",
      "Epoch 116: T.cost 0.022695, Train acc 0.992700, test acc 0.665700, took 4.19 sec.\n",
      "Epoch 117: T.cost 0.023253, Train acc 0.992120, test acc 0.662800, took 4.2 sec.\n",
      "Epoch 118: T.cost 0.028512, Train acc 0.990240, test acc 0.663000, took 4.31 sec.\n",
      "New LR: 0.000941480284673\n",
      "Epoch 119: T.cost 0.021187, Train acc 0.993240, test acc 0.667900, took 4.31 sec.\n",
      "Epoch 120: T.cost 0.018349, Train acc 0.993900, test acc 0.665600, took 4.25 sec.\n",
      "Epoch 121: T.cost 0.020752, Train acc 0.993160, test acc 0.664100, took 4.23 sec.\n",
      "Epoch 122: T.cost 0.019170, Train acc 0.994180, test acc 0.666500, took 4.18 sec.\n",
      "Epoch 123: T.cost 0.021341, Train acc 0.993580, test acc 0.662600, took 4.15 sec.\n",
      "Epoch 124: T.cost 0.018153, Train acc 0.994720, test acc 0.669700, took 4.21 sec.\n",
      "Epoch 125: T.cost 0.017660, Train acc 0.994280, test acc 0.672300, took 4.21 sec.\n",
      "Epoch 126: T.cost 0.012796, Train acc 0.996060, test acc 0.667400, took 4.21 sec.\n",
      "Epoch 127: T.cost 0.025126, Train acc 0.991740, test acc 0.669600, took 4.22 sec.\n",
      "Epoch 128: T.cost 0.018966, Train acc 0.993520, test acc 0.668100, took 4.21 sec.\n",
      "Epoch 129: T.cost 0.014636, Train acc 0.995240, test acc 0.670200, took 4.22 sec.\n",
      "Epoch 130: T.cost 0.017616, Train acc 0.994120, test acc 0.668400, took 4.21 sec.\n",
      "Epoch 131: T.cost 0.023351, Train acc 0.992300, test acc 0.664600, took 4.22 sec.\n",
      "Epoch 132: T.cost 0.019298, Train acc 0.993720, test acc 0.668000, took 4.21 sec.\n",
      "Epoch 133: T.cost 0.015839, Train acc 0.995040, test acc 0.667000, took 4.21 sec.\n",
      "Epoch 134: T.cost 0.013449, Train acc 0.995820, test acc 0.672100, took 4.21 sec.\n",
      "Epoch 135: T.cost 0.012939, Train acc 0.995600, test acc 0.673000, took 4.21 sec.\n",
      "Epoch 136: T.cost 0.019204, Train acc 0.993880, test acc 0.667100, took 4.2 sec.\n",
      "Epoch 137: T.cost 0.020242, Train acc 0.993420, test acc 0.671200, took 4.23 sec.\n",
      "Epoch 138: T.cost 0.025237, Train acc 0.992420, test acc 0.670000, took 4.21 sec.\n",
      "New LR: 0.000932065474335\n",
      "Epoch 139: T.cost 0.031108, Train acc 0.989900, test acc 0.667000, took 4.23 sec.\n",
      "Epoch 140: T.cost 0.025176, Train acc 0.992080, test acc 0.668600, took 4.22 sec.\n",
      "Epoch 141: T.cost 0.015569, Train acc 0.995060, test acc 0.671300, took 4.21 sec.\n",
      "Epoch 142: T.cost 0.013589, Train acc 0.995320, test acc 0.673800, took 4.23 sec.\n",
      "Epoch 143: T.cost 0.014413, Train acc 0.995260, test acc 0.667300, took 4.21 sec.\n",
      "Epoch 144: T.cost 0.013873, Train acc 0.995660, test acc 0.671600, took 4.22 sec.\n",
      "Epoch 145: T.cost 0.016450, Train acc 0.994840, test acc 0.667000, took 4.21 sec.\n",
      "Epoch 146: T.cost 0.014775, Train acc 0.995100, test acc 0.670600, took 4.22 sec.\n",
      "Epoch 147: T.cost 0.021238, Train acc 0.992980, test acc 0.664400, took 4.18 sec.\n",
      "Epoch 148: T.cost 0.014090, Train acc 0.995420, test acc 0.669100, took 4.2 sec.\n",
      "Epoch 149: T.cost 0.013068, Train acc 0.995580, test acc 0.670800, took 4.2 sec.\n",
      "Epoch 150: T.cost 0.017564, Train acc 0.994520, test acc 0.664000, took 4.19 sec.\n",
      "Epoch 151: T.cost 0.023918, Train acc 0.992420, test acc 0.668900, took 4.23 sec.\n",
      "Epoch 152: T.cost 0.010228, Train acc 0.996760, test acc 0.668700, took 4.23 sec.\n",
      "Epoch 153: T.cost 0.013650, Train acc 0.995860, test acc 0.667800, took 4.21 sec.\n",
      "Epoch 154: T.cost 0.033550, Train acc 0.990200, test acc 0.667700, took 4.21 sec.\n",
      "Epoch 155: T.cost 0.025830, Train acc 0.992200, test acc 0.662600, took 4.21 sec.\n",
      "Epoch 156: T.cost 0.023344, Train acc 0.992720, test acc 0.671200, took 4.19 sec.\n",
      "Epoch 157: T.cost 0.016097, Train acc 0.994620, test acc 0.668900, took 4.17 sec.\n",
      "Epoch 158: T.cost 0.012182, Train acc 0.995980, test acc 0.667400, took 4.17 sec.\n",
      "New LR: 0.000922744824202\n",
      "Epoch 159: T.cost 0.012652, Train acc 0.996220, test acc 0.667700, took 4.21 sec.\n",
      "Epoch 160: T.cost 0.012222, Train acc 0.996000, test acc 0.665400, took 4.2 sec.\n",
      "Epoch 161: T.cost 0.013923, Train acc 0.995740, test acc 0.663500, took 4.21 sec.\n",
      "Epoch 162: T.cost 0.011400, Train acc 0.996020, test acc 0.670000, took 4.21 sec.\n",
      "Epoch 163: T.cost 0.014221, Train acc 0.995720, test acc 0.668000, took 4.16 sec.\n",
      "Epoch 164: T.cost 0.011143, Train acc 0.996200, test acc 0.668800, took 4.19 sec.\n",
      "Epoch 165: T.cost 0.009013, Train acc 0.997120, test acc 0.669200, took 4.15 sec.\n",
      "Epoch 166: T.cost 0.017052, Train acc 0.994720, test acc 0.668400, took 4.18 sec.\n",
      "Epoch 167: T.cost 0.022851, Train acc 0.993020, test acc 0.668200, took 4.18 sec.\n",
      "Epoch 168: T.cost 0.015416, Train acc 0.995020, test acc 0.669100, took 4.2 sec.\n",
      "Epoch 169: T.cost 0.017720, Train acc 0.994580, test acc 0.668000, took 4.16 sec.\n",
      "Epoch 170: T.cost 0.013994, Train acc 0.995700, test acc 0.663200, took 4.17 sec.\n",
      "Epoch 171: T.cost 0.012750, Train acc 0.995760, test acc 0.665800, took 4.2 sec.\n",
      "Epoch 172: T.cost 0.010159, Train acc 0.996660, test acc 0.668700, took 4.21 sec.\n",
      "Epoch 173: T.cost 0.017437, Train acc 0.994380, test acc 0.673000, took 4.21 sec.\n",
      "Epoch 174: T.cost 0.022713, Train acc 0.992860, test acc 0.671300, took 4.22 sec.\n",
      "Epoch 175: T.cost 0.014792, Train acc 0.995520, test acc 0.667500, took 4.21 sec.\n",
      "Epoch 176: T.cost 0.013977, Train acc 0.995380, test acc 0.671800, took 4.22 sec.\n",
      "Epoch 177: T.cost 0.016225, Train acc 0.995060, test acc 0.669500, took 4.2 sec.\n",
      "Epoch 178: T.cost 0.014024, Train acc 0.995380, test acc 0.666000, took 4.2 sec.\n",
      "New LR: 0.000913517354638\n",
      "Epoch 179: T.cost 0.017858, Train acc 0.994120, test acc 0.668000, took 4.21 sec.\n",
      "Epoch 180: T.cost 0.011462, Train acc 0.996220, test acc 0.668400, took 4.21 sec.\n",
      "Epoch 181: T.cost 0.007063, Train acc 0.997800, test acc 0.669700, took 4.21 sec.\n",
      "Epoch 182: T.cost 0.010484, Train acc 0.996780, test acc 0.662900, took 4.21 sec.\n",
      "Epoch 183: T.cost 0.011207, Train acc 0.996460, test acc 0.666800, took 4.21 sec.\n",
      "Epoch 184: T.cost 0.019032, Train acc 0.994060, test acc 0.671400, took 4.18 sec.\n",
      "Epoch 185: T.cost 0.022923, Train acc 0.992740, test acc 0.665200, took 4.19 sec.\n",
      "Epoch 186: T.cost 0.015105, Train acc 0.995140, test acc 0.672900, took 4.2 sec.\n",
      "Epoch 187: T.cost 0.012930, Train acc 0.995820, test acc 0.672300, took 4.21 sec.\n",
      "Epoch 188: T.cost 0.016601, Train acc 0.994940, test acc 0.669300, took 4.21 sec.\n",
      "Epoch 189: T.cost 0.012327, Train acc 0.995860, test acc 0.674200, took 4.18 sec.\n",
      "Epoch 190: T.cost 0.012885, Train acc 0.995940, test acc 0.661400, took 4.21 sec.\n",
      "Epoch 191: T.cost 0.015942, Train acc 0.995020, test acc 0.668400, took 4.22 sec.\n",
      "Epoch 192: T.cost 0.015896, Train acc 0.994800, test acc 0.667700, took 4.19 sec.\n",
      "Epoch 193: T.cost 0.014222, Train acc 0.995400, test acc 0.670700, took 4.18 sec.\n",
      "Epoch 194: T.cost 0.010957, Train acc 0.996340, test acc 0.670400, took 4.22 sec.\n",
      "Epoch 195: T.cost 0.008898, Train acc 0.997040, test acc 0.668800, took 4.2 sec.\n",
      "Epoch 196: T.cost 0.011671, Train acc 0.996020, test acc 0.664900, took 4.22 sec.\n",
      "Epoch 197: T.cost 0.014787, Train acc 0.995080, test acc 0.661900, took 4.21 sec.\n",
      "Epoch 198: T.cost 0.015562, Train acc 0.994760, test acc 0.665100, took 4.2 sec.\n",
      "New LR: 0.000904382201261\n",
      "Epoch 199: T.cost 0.013267, Train acc 0.995820, test acc 0.666100, took 4.23 sec.\n",
      "Epoch 200: T.cost 0.021789, Train acc 0.993720, test acc 0.668000, took 4.22 sec.\n",
      "Epoch 201: T.cost 0.018922, Train acc 0.994300, test acc 0.670900, took 4.22 sec.\n",
      "Epoch 202: T.cost 0.009350, Train acc 0.997100, test acc 0.669700, took 4.22 sec.\n",
      "Epoch 203: T.cost 0.010662, Train acc 0.996540, test acc 0.670400, took 4.22 sec.\n",
      "Epoch 204: T.cost 0.008311, Train acc 0.997640, test acc 0.671300, took 4.21 sec.\n",
      "Epoch 205: T.cost 0.010161, Train acc 0.997040, test acc 0.673600, took 4.16 sec.\n",
      "Epoch 206: T.cost 0.010353, Train acc 0.996580, test acc 0.672000, took 4.17 sec.\n",
      "Epoch 207: T.cost 0.011295, Train acc 0.996580, test acc 0.675000, took 4.15 sec.\n",
      "Epoch 208: T.cost 0.013761, Train acc 0.995260, test acc 0.671600, took 4.17 sec.\n",
      "Epoch 209: T.cost 0.010522, Train acc 0.996780, test acc 0.674100, took 4.17 sec.\n",
      "Epoch 210: T.cost 0.013251, Train acc 0.995960, test acc 0.670000, took 4.16 sec.\n",
      "Epoch 211: T.cost 0.016063, Train acc 0.994740, test acc 0.669400, took 4.13 sec.\n",
      "Epoch 212: T.cost 0.022700, Train acc 0.992780, test acc 0.671400, took 4.17 sec.\n",
      "Epoch 213: T.cost 0.017290, Train acc 0.994680, test acc 0.675400, took 4.16 sec.\n",
      "Epoch 214: T.cost 0.018378, Train acc 0.994580, test acc 0.672100, took 4.17 sec.\n",
      "Epoch 215: T.cost 0.012017, Train acc 0.995880, test acc 0.669200, took 4.16 sec.\n",
      "Epoch 216: T.cost 0.012504, Train acc 0.995980, test acc 0.673000, took 4.15 sec.\n",
      "Epoch 217: T.cost 0.006907, Train acc 0.997780, test acc 0.679900, took 4.15 sec.\n",
      "Epoch 218: T.cost 0.014918, Train acc 0.995400, test acc 0.664500, took 4.17 sec.\n",
      "New LR: 0.000895338384435\n",
      "Epoch 219: T.cost 0.009802, Train acc 0.996740, test acc 0.670000, took 4.16 sec.\n",
      "Epoch 220: T.cost 0.010778, Train acc 0.996860, test acc 0.670600, took 4.18 sec.\n",
      "Epoch 221: T.cost 0.013768, Train acc 0.995980, test acc 0.673000, took 4.17 sec.\n",
      "Epoch 222: T.cost 0.020187, Train acc 0.993280, test acc 0.669000, took 4.13 sec.\n",
      "Epoch 223: T.cost 0.013052, Train acc 0.996080, test acc 0.676100, took 4.15 sec.\n",
      "Epoch 224: T.cost 0.011487, Train acc 0.996460, test acc 0.675000, took 4.17 sec.\n",
      "Epoch 225: T.cost 0.007152, Train acc 0.997980, test acc 0.673000, took 4.17 sec.\n",
      "Epoch 226: T.cost 0.005475, Train acc 0.998180, test acc 0.674600, took 4.22 sec.\n",
      "Epoch 227: T.cost 0.008162, Train acc 0.997240, test acc 0.668400, took 4.16 sec.\n",
      "Epoch 228: T.cost 0.009423, Train acc 0.997020, test acc 0.667100, took 4.17 sec.\n",
      "Epoch 229: T.cost 0.016620, Train acc 0.994760, test acc 0.670100, took 4.17 sec.\n",
      "Epoch 230: T.cost 0.023319, Train acc 0.992980, test acc 0.667400, took 4.17 sec.\n",
      "Epoch 231: T.cost 0.017683, Train acc 0.994760, test acc 0.674600, took 4.17 sec.\n",
      "Epoch 232: T.cost 0.009722, Train acc 0.996620, test acc 0.674500, took 4.17 sec.\n",
      "Epoch 233: T.cost 0.007561, Train acc 0.997620, test acc 0.674100, took 4.16 sec.\n",
      "Epoch 234: T.cost 0.004767, Train acc 0.998320, test acc 0.678900, took 4.18 sec.\n",
      "Epoch 235: T.cost 0.004343, Train acc 0.998500, test acc 0.677800, took 4.17 sec.\n",
      "Epoch 236: T.cost 0.014594, Train acc 0.995480, test acc 0.671800, took 4.11 sec.\n",
      "Epoch 237: T.cost 0.014825, Train acc 0.995080, test acc 0.678000, took 4.16 sec.\n",
      "Epoch 238: T.cost 0.009356, Train acc 0.996880, test acc 0.675400, took 4.17 sec.\n",
      "New LR: 0.00088638498215\n",
      "Epoch 239: T.cost 0.007491, Train acc 0.997660, test acc 0.670000, took 4.18 sec.\n",
      "Epoch 240: T.cost 0.008999, Train acc 0.996920, test acc 0.674900, took 4.19 sec.\n",
      "Epoch 241: T.cost 0.010945, Train acc 0.996720, test acc 0.665500, took 4.14 sec.\n",
      "Epoch 242: T.cost 0.019304, Train acc 0.994140, test acc 0.670600, took 4.17 sec.\n",
      "Epoch 243: T.cost 0.015431, Train acc 0.995020, test acc 0.675900, took 4.16 sec.\n",
      "Epoch 244: T.cost 0.009191, Train acc 0.996900, test acc 0.672000, took 4.16 sec.\n",
      "Epoch 245: T.cost 0.016481, Train acc 0.994720, test acc 0.666100, took 4.16 sec.\n",
      "Epoch 246: T.cost 0.010055, Train acc 0.996900, test acc 0.675900, took 4.2 sec.\n",
      "Epoch 247: T.cost 0.013039, Train acc 0.995900, test acc 0.673700, took 4.16 sec.\n",
      "Epoch 248: T.cost 0.010673, Train acc 0.996140, test acc 0.680100, took 4.28 sec.\n",
      "Epoch 249: T.cost 0.007774, Train acc 0.997480, test acc 0.682100, took 4.27 sec.\n",
      "Epoch 250: T.cost 0.005292, Train acc 0.998240, test acc 0.677000, took 4.16 sec.\n",
      "Epoch 251: T.cost 0.004403, Train acc 0.998760, test acc 0.683600, took 4.18 sec.\n",
      "Epoch 252: T.cost 0.006307, Train acc 0.998020, test acc 0.669900, took 4.23 sec.\n",
      "Epoch 253: T.cost 0.012339, Train acc 0.996240, test acc 0.664000, took 4.23 sec.\n",
      "Epoch 254: T.cost 0.026986, Train acc 0.992620, test acc 0.673200, took 4.19 sec.\n",
      "Epoch 255: T.cost 0.018548, Train acc 0.994300, test acc 0.674600, took 4.17 sec.\n",
      "Epoch 256: T.cost 0.018381, Train acc 0.994760, test acc 0.668000, took 4.16 sec.\n",
      "Epoch 257: T.cost 0.007866, Train acc 0.997300, test acc 0.678500, took 4.17 sec.\n",
      "Epoch 258: T.cost 0.012315, Train acc 0.996360, test acc 0.673300, took 4.18 sec.\n",
      "New LR: 0.000877521130024\n",
      "Epoch 259: T.cost 0.010510, Train acc 0.996940, test acc 0.681400, took 4.17 sec.\n",
      "Epoch 260: T.cost 0.006693, Train acc 0.997700, test acc 0.679500, took 4.14 sec.\n",
      "Epoch 261: T.cost 0.005929, Train acc 0.998180, test acc 0.678100, took 4.14 sec.\n",
      "Epoch 262: T.cost 0.009977, Train acc 0.996840, test acc 0.676700, took 4.2 sec.\n",
      "Epoch 263: T.cost 0.007228, Train acc 0.997680, test acc 0.680700, took 4.2 sec.\n",
      "Epoch 264: T.cost 0.007451, Train acc 0.997880, test acc 0.678300, took 4.16 sec.\n",
      "Epoch 265: T.cost 0.012479, Train acc 0.996180, test acc 0.673400, took 4.15 sec.\n",
      "Epoch 266: T.cost 0.014733, Train acc 0.995700, test acc 0.670900, took 4.22 sec.\n",
      "Epoch 267: T.cost 0.019873, Train acc 0.994280, test acc 0.673800, took 4.19 sec.\n",
      "Epoch 268: T.cost 0.011944, Train acc 0.996020, test acc 0.675800, took 4.26 sec.\n",
      "Epoch 269: T.cost 0.007833, Train acc 0.997760, test acc 0.674100, took 4.27 sec.\n",
      "Epoch 270: T.cost 0.005883, Train acc 0.998140, test acc 0.675000, took 4.22 sec.\n",
      "Epoch 271: T.cost 0.003638, Train acc 0.998720, test acc 0.674400, took 4.31 sec.\n",
      "Epoch 272: T.cost 0.007238, Train acc 0.997580, test acc 0.672500, took 4.25 sec.\n",
      "Epoch 273: T.cost 0.010112, Train acc 0.997000, test acc 0.670300, took 4.27 sec.\n",
      "Epoch 274: T.cost 0.017452, Train acc 0.994660, test acc 0.676500, took 4.3 sec.\n",
      "Epoch 275: T.cost 0.005461, Train acc 0.998020, test acc 0.677000, took 4.16 sec.\n",
      "Epoch 276: T.cost 0.011668, Train acc 0.996300, test acc 0.672800, took 4.2 sec.\n",
      "Epoch 277: T.cost 0.013707, Train acc 0.995700, test acc 0.675000, took 4.29 sec.\n",
      "Epoch 278: T.cost 0.013784, Train acc 0.995900, test acc 0.673300, took 4.3 sec.\n",
      "New LR: 0.000868745906046\n",
      "Epoch 279: T.cost 0.012806, Train acc 0.995960, test acc 0.669500, took 4.36 sec.\n",
      "Epoch 280: T.cost 0.010886, Train acc 0.996660, test acc 0.677200, took 4.35 sec.\n",
      "Epoch 281: T.cost 0.007840, Train acc 0.997600, test acc 0.673500, took 4.26 sec.\n",
      "Epoch 282: T.cost 0.005673, Train acc 0.998420, test acc 0.673600, took 4.23 sec.\n",
      "Epoch 283: T.cost 0.006973, Train acc 0.997940, test acc 0.672300, took 4.21 sec.\n",
      "Epoch 284: T.cost 0.006527, Train acc 0.998020, test acc 0.677600, took 4.23 sec.\n",
      "Epoch 285: T.cost 0.011027, Train acc 0.996480, test acc 0.672400, took 4.39 sec.\n",
      "Epoch 286: T.cost 0.010055, Train acc 0.996740, test acc 0.672400, took 4.3 sec.\n",
      "Epoch 287: T.cost 0.015970, Train acc 0.995320, test acc 0.670100, took 4.3 sec.\n",
      "Epoch 288: T.cost 0.012392, Train acc 0.996160, test acc 0.677100, took 4.29 sec.\n",
      "Epoch 289: T.cost 0.007329, Train acc 0.997640, test acc 0.675200, took 4.29 sec.\n",
      "Epoch 290: T.cost 0.007968, Train acc 0.997500, test acc 0.681100, took 4.3 sec.\n",
      "Epoch 291: T.cost 0.008640, Train acc 0.997220, test acc 0.674400, took 4.29 sec.\n",
      "Epoch 292: T.cost 0.015413, Train acc 0.995780, test acc 0.671500, took 4.29 sec.\n",
      "Epoch 293: T.cost 0.011219, Train acc 0.996360, test acc 0.675000, took 4.28 sec.\n",
      "Epoch 294: T.cost 0.006814, Train acc 0.997780, test acc 0.674300, took 4.28 sec.\n",
      "Epoch 295: T.cost 0.007500, Train acc 0.997860, test acc 0.677000, took 4.28 sec.\n",
      "Epoch 296: T.cost 0.004835, Train acc 0.998280, test acc 0.674600, took 4.31 sec.\n",
      "Epoch 297: T.cost 0.008453, Train acc 0.997280, test acc 0.672600, took 4.27 sec.\n",
      "Epoch 298: T.cost 0.006734, Train acc 0.998000, test acc 0.675400, took 4.27 sec.\n",
      "New LR: 0.000860058445833\n",
      "Epoch 299: T.cost 0.012391, Train acc 0.996400, test acc 0.676100, took 4.29 sec.\n",
      "Epoch 300: T.cost 0.013282, Train acc 0.996300, test acc 0.682100, took 4.24 sec.\n",
      "Epoch 301: T.cost 0.004931, Train acc 0.998380, test acc 0.682800, took 4.37 sec.\n",
      "Epoch 302: T.cost 0.014586, Train acc 0.995500, test acc 0.674800, took 4.32 sec.\n",
      "Epoch 303: T.cost 0.014796, Train acc 0.995940, test acc 0.676400, took 4.35 sec.\n",
      "Epoch 304: T.cost 0.007680, Train acc 0.997400, test acc 0.674100, took 4.31 sec.\n",
      "Epoch 305: T.cost 0.008058, Train acc 0.997540, test acc 0.673800, took 4.22 sec.\n",
      "Epoch 306: T.cost 0.009850, Train acc 0.996700, test acc 0.676600, took 4.34 sec.\n",
      "Epoch 307: T.cost 0.011459, Train acc 0.996460, test acc 0.676300, took 4.27 sec.\n",
      "Epoch 308: T.cost 0.007225, Train acc 0.997760, test acc 0.676300, took 4.29 sec.\n",
      "Epoch 309: T.cost 0.006003, Train acc 0.998100, test acc 0.677900, took 4.32 sec.\n",
      "Epoch 310: T.cost 0.013893, Train acc 0.995900, test acc 0.668400, took 4.35 sec.\n",
      "Epoch 311: T.cost 0.012481, Train acc 0.996500, test acc 0.671800, took 4.25 sec.\n",
      "Epoch 312: T.cost 0.007128, Train acc 0.997840, test acc 0.671300, took 4.19 sec.\n",
      "Epoch 313: T.cost 0.008513, Train acc 0.997540, test acc 0.674500, took 4.18 sec.\n",
      "Epoch 314: T.cost 0.008082, Train acc 0.997580, test acc 0.675500, took 4.19 sec.\n",
      "Epoch 315: T.cost 0.011645, Train acc 0.996280, test acc 0.674100, took 4.21 sec.\n",
      "Epoch 316: T.cost 0.013864, Train acc 0.996240, test acc 0.670900, took 4.24 sec.\n",
      "Epoch 317: T.cost 0.007948, Train acc 0.997820, test acc 0.677100, took 4.28 sec.\n",
      "Epoch 318: T.cost 0.010326, Train acc 0.996900, test acc 0.671200, took 4.34 sec.\n",
      "New LR: 0.000851457885001\n",
      "Epoch 319: T.cost 0.008839, Train acc 0.997220, test acc 0.672500, took 4.32 sec.\n",
      "Epoch 320: T.cost 0.010501, Train acc 0.996660, test acc 0.666500, took 4.2 sec.\n",
      "Epoch 321: T.cost 0.006216, Train acc 0.998000, test acc 0.672700, took 4.21 sec.\n",
      "Epoch 322: T.cost 0.007224, Train acc 0.997600, test acc 0.672400, took 4.2 sec.\n",
      "Epoch 323: T.cost 0.010499, Train acc 0.996700, test acc 0.675200, took 4.3 sec.\n",
      "Epoch 324: T.cost 0.006808, Train acc 0.997800, test acc 0.674900, took 4.41 sec.\n",
      "Epoch 325: T.cost 0.003828, Train acc 0.998520, test acc 0.682400, took 4.57 sec.\n",
      "Epoch 326: T.cost 0.008590, Train acc 0.997440, test acc 0.678100, took 4.5 sec.\n",
      "Epoch 327: T.cost 0.008281, Train acc 0.997660, test acc 0.675300, took 4.54 sec.\n",
      "Epoch 328: T.cost 0.007663, Train acc 0.997580, test acc 0.679100, took 4.43 sec.\n",
      "Epoch 329: T.cost 0.010943, Train acc 0.996960, test acc 0.666500, took 4.46 sec.\n",
      "Epoch 330: T.cost 0.010302, Train acc 0.996900, test acc 0.671600, took 4.51 sec.\n",
      "Epoch 331: T.cost 0.010335, Train acc 0.996660, test acc 0.670300, took 4.39 sec.\n",
      "Epoch 332: T.cost 0.009179, Train acc 0.997340, test acc 0.671300, took 4.39 sec.\n",
      "Epoch 333: T.cost 0.010813, Train acc 0.996700, test acc 0.671300, took 4.39 sec.\n",
      "Epoch 334: T.cost 0.011260, Train acc 0.996800, test acc 0.673600, took 4.39 sec.\n",
      "Epoch 335: T.cost 0.005905, Train acc 0.998080, test acc 0.674600, took 4.3 sec.\n",
      "Epoch 336: T.cost 0.003343, Train acc 0.999020, test acc 0.674500, took 4.19 sec.\n",
      "Epoch 337: T.cost 0.004574, Train acc 0.998440, test acc 0.675600, took 4.11 sec.\n",
      "Epoch 338: T.cost 0.003827, Train acc 0.998800, test acc 0.674200, took 4.24 sec.\n",
      "New LR: 0.000842943301541\n",
      "Epoch 339: T.cost 0.005814, Train acc 0.998120, test acc 0.670400, took 4.22 sec.\n",
      "Epoch 340: T.cost 0.008903, Train acc 0.997480, test acc 0.665400, took 4.16 sec.\n",
      "Epoch 341: T.cost 0.014649, Train acc 0.996180, test acc 0.669300, took 4.16 sec.\n",
      "Epoch 342: T.cost 0.023212, Train acc 0.993720, test acc 0.670500, took 4.25 sec.\n",
      "Epoch 343: T.cost 0.020984, Train acc 0.994360, test acc 0.671200, took 4.36 sec.\n",
      "Epoch 344: T.cost 0.008228, Train acc 0.997680, test acc 0.674200, took 4.27 sec.\n",
      "Epoch 345: T.cost 0.005705, Train acc 0.998300, test acc 0.673700, took 4.28 sec.\n",
      "Epoch 346: T.cost 0.002357, Train acc 0.999240, test acc 0.673300, took 4.34 sec.\n",
      "Epoch 347: T.cost 0.001399, Train acc 0.999580, test acc 0.675100, took 4.32 sec.\n",
      "Epoch 348: T.cost 0.002692, Train acc 0.999220, test acc 0.673200, took 4.33 sec.\n",
      "Epoch 349: T.cost 0.006794, Train acc 0.997880, test acc 0.667900, took 4.33 sec.\n",
      "Epoch 350: T.cost 0.014302, Train acc 0.995780, test acc 0.673900, took 4.36 sec.\n",
      "Epoch 351: T.cost 0.011151, Train acc 0.996820, test acc 0.673100, took 4.32 sec.\n",
      "Epoch 352: T.cost 0.012129, Train acc 0.996580, test acc 0.673100, took 4.31 sec.\n",
      "Epoch 353: T.cost 0.011771, Train acc 0.996720, test acc 0.678000, took 4.3 sec.\n",
      "Epoch 354: T.cost 0.010113, Train acc 0.996880, test acc 0.678400, took 4.3 sec.\n",
      "Epoch 355: T.cost 0.007098, Train acc 0.998040, test acc 0.678000, took 4.29 sec.\n",
      "Epoch 356: T.cost 0.005650, Train acc 0.998120, test acc 0.677200, took 4.32 sec.\n",
      "Epoch 357: T.cost 0.008236, Train acc 0.997520, test acc 0.680000, took 4.35 sec.\n",
      "Epoch 358: T.cost 0.009875, Train acc 0.997340, test acc 0.678700, took 4.35 sec.\n",
      "New LR: 0.000834513888694\n",
      "Epoch 359: T.cost 0.010228, Train acc 0.996980, test acc 0.672300, took 4.31 sec.\n",
      "Epoch 360: T.cost 0.009003, Train acc 0.997480, test acc 0.676000, took 4.32 sec.\n",
      "Epoch 361: T.cost 0.005762, Train acc 0.998140, test acc 0.671900, took 4.31 sec.\n",
      "Epoch 362: T.cost 0.009522, Train acc 0.997100, test acc 0.676700, took 4.3 sec.\n",
      "Epoch 363: T.cost 0.005958, Train acc 0.998240, test acc 0.678900, took 4.3 sec.\n",
      "Epoch 364: T.cost 0.005380, Train acc 0.998160, test acc 0.672700, took 4.32 sec.\n",
      "Epoch 365: T.cost 0.005377, Train acc 0.998340, test acc 0.673900, took 4.32 sec.\n",
      "Epoch 366: T.cost 0.004213, Train acc 0.998500, test acc 0.673500, took 4.3 sec.\n",
      "Epoch 367: T.cost 0.004281, Train acc 0.998480, test acc 0.672100, took 4.32 sec.\n",
      "Epoch 368: T.cost 0.004927, Train acc 0.998380, test acc 0.673200, took 4.31 sec.\n",
      "Epoch 369: T.cost 0.026206, Train acc 0.993500, test acc 0.671900, took 4.29 sec.\n",
      "Epoch 370: T.cost 0.015273, Train acc 0.995640, test acc 0.670800, took 4.33 sec.\n",
      "Epoch 371: T.cost 0.010383, Train acc 0.996880, test acc 0.677500, took 4.29 sec.\n",
      "Epoch 372: T.cost 0.006811, Train acc 0.997900, test acc 0.671000, took 4.29 sec.\n",
      "Epoch 373: T.cost 0.004585, Train acc 0.998520, test acc 0.674600, took 4.3 sec.\n",
      "Epoch 374: T.cost 0.002489, Train acc 0.998980, test acc 0.676700, took 4.32 sec.\n",
      "Epoch 375: T.cost 0.004138, Train acc 0.998540, test acc 0.677100, took 4.34 sec.\n",
      "Epoch 376: T.cost 0.007071, Train acc 0.997760, test acc 0.674700, took 4.39 sec.\n",
      "Epoch 377: T.cost 0.007396, Train acc 0.997680, test acc 0.672100, took 4.42 sec.\n",
      "Epoch 378: T.cost 0.012113, Train acc 0.996680, test acc 0.673800, took 4.34 sec.\n",
      "New LR: 0.000826168724452\n",
      "Epoch 379: T.cost 0.012042, Train acc 0.996440, test acc 0.669600, took 4.35 sec.\n",
      "Epoch 380: T.cost 0.007261, Train acc 0.997620, test acc 0.673400, took 4.32 sec.\n",
      "Epoch 381: T.cost 0.006242, Train acc 0.998160, test acc 0.674700, took 4.32 sec.\n",
      "Epoch 382: T.cost 0.006019, Train acc 0.998240, test acc 0.677400, took 4.3 sec.\n",
      "Epoch 383: T.cost 0.004651, Train acc 0.998660, test acc 0.675300, took 4.39 sec.\n",
      "Epoch 384: T.cost 0.007807, Train acc 0.997640, test acc 0.670800, took 4.4 sec.\n",
      "Epoch 385: T.cost 0.011312, Train acc 0.996720, test acc 0.667000, took 4.36 sec.\n",
      "Epoch 386: T.cost 0.006614, Train acc 0.997880, test acc 0.673400, took 4.3 sec.\n",
      "Epoch 387: T.cost 0.009884, Train acc 0.997200, test acc 0.671400, took 4.21 sec.\n",
      "Epoch 388: T.cost 0.007426, Train acc 0.997860, test acc 0.671000, took 4.19 sec.\n",
      "Epoch 389: T.cost 0.006598, Train acc 0.997900, test acc 0.668500, took 4.19 sec.\n",
      "Epoch 390: T.cost 0.007914, Train acc 0.997620, test acc 0.677200, took 4.2 sec.\n",
      "Epoch 391: T.cost 0.006976, Train acc 0.997840, test acc 0.673600, took 4.22 sec.\n",
      "Epoch 392: T.cost 0.010231, Train acc 0.996860, test acc 0.673800, took 4.2 sec.\n",
      "Epoch 393: T.cost 0.013003, Train acc 0.996360, test acc 0.671200, took 4.22 sec.\n",
      "Epoch 394: T.cost 0.013349, Train acc 0.995960, test acc 0.671500, took 4.21 sec.\n",
      "Epoch 395: T.cost 0.006434, Train acc 0.998180, test acc 0.676800, took 4.19 sec.\n",
      "Epoch 396: T.cost 0.005111, Train acc 0.998120, test acc 0.675800, took 4.24 sec.\n",
      "Epoch 397: T.cost 0.005195, Train acc 0.998340, test acc 0.672000, took 4.19 sec.\n",
      "Epoch 398: T.cost 0.005145, Train acc 0.998400, test acc 0.671400, took 4.18 sec.\n",
      "New LR: 0.000817907059682\n",
      "Epoch 399: T.cost 0.005651, Train acc 0.998460, test acc 0.673300, took 4.33 sec.\n",
      "Epoch 400: T.cost 0.005914, Train acc 0.998140, test acc 0.673700, took 4.22 sec.\n",
      "Epoch 401: T.cost 0.004860, Train acc 0.998680, test acc 0.676100, took 4.2 sec.\n",
      "Epoch 402: T.cost 0.004298, Train acc 0.998560, test acc 0.668600, took 4.22 sec.\n",
      "Epoch 403: T.cost 0.006009, Train acc 0.998380, test acc 0.668500, took 4.27 sec.\n",
      "Epoch 404: T.cost 0.009735, Train acc 0.997420, test acc 0.667800, took 4.21 sec.\n",
      "Epoch 405: T.cost 0.013556, Train acc 0.995840, test acc 0.674100, took 4.19 sec.\n",
      "Epoch 406: T.cost 0.005975, Train acc 0.998200, test acc 0.672200, took 4.18 sec.\n",
      "Epoch 407: T.cost 0.006831, Train acc 0.997840, test acc 0.670000, took 4.2 sec.\n",
      "Epoch 408: T.cost 0.013542, Train acc 0.996160, test acc 0.674200, took 4.22 sec.\n",
      "Epoch 409: T.cost 0.008757, Train acc 0.997500, test acc 0.666900, took 4.23 sec.\n",
      "Epoch 410: T.cost 0.002623, Train acc 0.999020, test acc 0.673400, took 4.22 sec.\n",
      "Epoch 411: T.cost 0.002466, Train acc 0.999400, test acc 0.673700, took 4.2 sec.\n",
      "Epoch 412: T.cost 0.008932, Train acc 0.997540, test acc 0.674200, took 4.25 sec.\n",
      "Epoch 413: T.cost 0.011890, Train acc 0.996760, test acc 0.677600, took 4.19 sec.\n",
      "Epoch 414: T.cost 0.005506, Train acc 0.998220, test acc 0.672000, took 4.4 sec.\n",
      "Epoch 415: T.cost 0.013976, Train acc 0.995760, test acc 0.670200, took 4.22 sec.\n",
      "Epoch 416: T.cost 0.013915, Train acc 0.996360, test acc 0.672500, took 4.24 sec.\n",
      "Epoch 417: T.cost 0.008036, Train acc 0.997900, test acc 0.674800, took 4.18 sec.\n",
      "Epoch 418: T.cost 0.003960, Train acc 0.998840, test acc 0.672800, took 4.24 sec.\n",
      "New LR: 0.000809727972373\n",
      "Epoch 419: T.cost 0.005552, Train acc 0.998400, test acc 0.669900, took 4.29 sec.\n",
      "Epoch 420: T.cost 0.008895, Train acc 0.997800, test acc 0.681300, took 4.29 sec.\n",
      "Epoch 421: T.cost 0.003872, Train acc 0.998940, test acc 0.680100, took 4.29 sec.\n",
      "Epoch 422: T.cost 0.003164, Train acc 0.999080, test acc 0.676000, took 4.24 sec.\n",
      "Epoch 423: T.cost 0.005173, Train acc 0.998360, test acc 0.675500, took 4.23 sec.\n",
      "Epoch 424: T.cost 0.007161, Train acc 0.997700, test acc 0.672700, took 4.25 sec.\n",
      "Epoch 425: T.cost 0.004968, Train acc 0.998420, test acc 0.671500, took 4.19 sec.\n",
      "Epoch 426: T.cost 0.011485, Train acc 0.996780, test acc 0.672600, took 4.17 sec.\n",
      "Epoch 427: T.cost 0.009468, Train acc 0.997300, test acc 0.676300, took 4.18 sec.\n",
      "Epoch 428: T.cost 0.010053, Train acc 0.997120, test acc 0.674000, took 4.18 sec.\n",
      "Epoch 429: T.cost 0.005848, Train acc 0.998020, test acc 0.677800, took 4.18 sec.\n",
      "Epoch 430: T.cost 0.003332, Train acc 0.998960, test acc 0.680000, took 4.19 sec.\n",
      "Epoch 431: T.cost 0.007819, Train acc 0.997560, test acc 0.676500, took 4.19 sec.\n",
      "Epoch 432: T.cost 0.008835, Train acc 0.997480, test acc 0.674700, took 4.18 sec.\n",
      "Epoch 433: T.cost 0.003206, Train acc 0.999100, test acc 0.680200, took 4.18 sec.\n",
      "Epoch 434: T.cost 0.002983, Train acc 0.999120, test acc 0.673300, took 4.31 sec.\n",
      "Epoch 435: T.cost 0.008306, Train acc 0.997640, test acc 0.671500, took 4.24 sec.\n",
      "Epoch 436: T.cost 0.018719, Train acc 0.995380, test acc 0.663400, took 4.29 sec.\n",
      "Epoch 437: T.cost 0.010245, Train acc 0.997000, test acc 0.675000, took 4.27 sec.\n",
      "Epoch 438: T.cost 0.007717, Train acc 0.997760, test acc 0.675400, took 4.32 sec.\n",
      "New LR: 0.000801630713395\n",
      "Epoch 439: T.cost 0.004007, Train acc 0.998800, test acc 0.666800, took 4.33 sec.\n",
      "Epoch 440: T.cost 0.004299, Train acc 0.998720, test acc 0.681500, took 4.44 sec.\n",
      "Epoch 441: T.cost 0.001265, Train acc 0.999540, test acc 0.679000, took 4.26 sec.\n",
      "Epoch 442: T.cost 0.002727, Train acc 0.999060, test acc 0.674600, took 4.37 sec.\n",
      "Epoch 443: T.cost 0.006224, Train acc 0.998240, test acc 0.667200, took 4.33 sec.\n",
      "Epoch 444: T.cost 0.011301, Train acc 0.996840, test acc 0.674300, took 4.21 sec.\n",
      "Epoch 445: T.cost 0.011678, Train acc 0.996860, test acc 0.676000, took 4.21 sec.\n",
      "Epoch 446: T.cost 0.007907, Train acc 0.997420, test acc 0.683600, took 4.3 sec.\n",
      "Epoch 447: T.cost 0.006605, Train acc 0.998340, test acc 0.681000, took 4.33 sec.\n",
      "Epoch 448: T.cost 0.005621, Train acc 0.998300, test acc 0.679200, took 4.29 sec.\n",
      "Epoch 449: T.cost 0.008450, Train acc 0.997240, test acc 0.674100, took 4.45 sec.\n",
      "Epoch 450: T.cost 0.003437, Train acc 0.998900, test acc 0.680800, took 4.37 sec.\n",
      "Epoch 451: T.cost 0.003002, Train acc 0.999200, test acc 0.677800, took 4.33 sec.\n",
      "Epoch 452: T.cost 0.004596, Train acc 0.998480, test acc 0.669600, took 4.29 sec.\n",
      "Epoch 453: T.cost 0.006783, Train acc 0.997680, test acc 0.674700, took 4.39 sec.\n",
      "Epoch 454: T.cost 0.009576, Train acc 0.997340, test acc 0.672100, took 4.41 sec.\n",
      "Epoch 455: T.cost 0.010768, Train acc 0.997060, test acc 0.678900, took 4.37 sec.\n",
      "Epoch 456: T.cost 0.006433, Train acc 0.998120, test acc 0.675400, took 4.31 sec.\n",
      "Epoch 457: T.cost 0.004356, Train acc 0.998780, test acc 0.675400, took 4.42 sec.\n",
      "Epoch 458: T.cost 0.004570, Train acc 0.998600, test acc 0.677700, took 4.39 sec.\n",
      "New LR: 0.000793614418362\n",
      "Epoch 459: T.cost 0.010050, Train acc 0.997240, test acc 0.671600, took 4.39 sec.\n",
      "Epoch 460: T.cost 0.006859, Train acc 0.997940, test acc 0.678600, took 4.37 sec.\n",
      "Epoch 461: T.cost 0.006823, Train acc 0.997980, test acc 0.673800, took 4.26 sec.\n",
      "Epoch 462: T.cost 0.007404, Train acc 0.998000, test acc 0.673300, took 4.35 sec.\n",
      "Epoch 463: T.cost 0.010374, Train acc 0.996800, test acc 0.671500, took 4.3 sec.\n",
      "Epoch 464: T.cost 0.006331, Train acc 0.997920, test acc 0.673700, took 4.31 sec.\n",
      "Epoch 465: T.cost 0.007257, Train acc 0.997860, test acc 0.666300, took 4.35 sec.\n",
      "Epoch 466: T.cost 0.005539, Train acc 0.998460, test acc 0.676000, took 4.34 sec.\n",
      "Epoch 467: T.cost 0.007340, Train acc 0.997880, test acc 0.673100, took 4.33 sec.\n",
      "Epoch 468: T.cost 0.005861, Train acc 0.998200, test acc 0.671500, took 4.3 sec.\n",
      "Epoch 469: T.cost 0.001919, Train acc 0.999440, test acc 0.675800, took 4.3 sec.\n",
      "Epoch 470: T.cost 0.003268, Train acc 0.999120, test acc 0.672800, took 4.42 sec.\n",
      "Epoch 471: T.cost 0.007325, Train acc 0.997840, test acc 0.675100, took 4.47 sec.\n",
      "Epoch 472: T.cost 0.005914, Train acc 0.998340, test acc 0.680500, took 4.33 sec.\n",
      "Epoch 473: T.cost 0.006911, Train acc 0.998040, test acc 0.673300, took 4.37 sec.\n",
      "Epoch 474: T.cost 0.003466, Train acc 0.998780, test acc 0.674500, took 4.33 sec.\n",
      "Epoch 475: T.cost 0.010735, Train acc 0.997280, test acc 0.668100, took 4.3 sec.\n",
      "Epoch 476: T.cost 0.013239, Train acc 0.996580, test acc 0.671400, took 4.29 sec.\n",
      "Epoch 477: T.cost 0.010776, Train acc 0.997100, test acc 0.680400, took 4.26 sec.\n",
      "Epoch 478: T.cost 0.006814, Train acc 0.997920, test acc 0.674900, took 4.26 sec.\n",
      "New LR: 0.000785678280517\n",
      "Epoch 479: T.cost 0.001460, Train acc 0.999500, test acc 0.677400, took 4.36 sec.\n",
      "Epoch 480: T.cost 0.001798, Train acc 0.999540, test acc 0.676600, took 4.32 sec.\n",
      "Epoch 481: T.cost 0.002881, Train acc 0.999200, test acc 0.674500, took 4.34 sec.\n",
      "Epoch 482: T.cost 0.007070, Train acc 0.998100, test acc 0.673600, took 4.48 sec.\n",
      "Epoch 483: T.cost 0.009452, Train acc 0.997400, test acc 0.673200, took 4.25 sec.\n",
      "Epoch 484: T.cost 0.009816, Train acc 0.997640, test acc 0.663900, took 4.37 sec.\n",
      "Epoch 485: T.cost 0.014250, Train acc 0.996440, test acc 0.673000, took 4.36 sec.\n",
      "Epoch 486: T.cost 0.010896, Train acc 0.997100, test acc 0.670500, took 4.35 sec.\n",
      "Epoch 487: T.cost 0.010131, Train acc 0.997300, test acc 0.669400, took 4.27 sec.\n",
      "Epoch 488: T.cost 0.006033, Train acc 0.998320, test acc 0.677700, took 4.28 sec.\n",
      "Epoch 489: T.cost 0.004154, Train acc 0.998740, test acc 0.666900, took 4.49 sec.\n",
      "Epoch 490: T.cost 0.002969, Train acc 0.999100, test acc 0.677500, took 4.28 sec.\n",
      "Epoch 491: T.cost 0.003939, Train acc 0.998520, test acc 0.673600, took 4.32 sec.\n",
      "Epoch 492: T.cost 0.006332, Train acc 0.998460, test acc 0.676300, took 4.28 sec.\n",
      "Epoch 493: T.cost 0.004015, Train acc 0.998640, test acc 0.677200, took 4.32 sec.\n",
      "Epoch 494: T.cost 0.003251, Train acc 0.998900, test acc 0.677400, took 4.27 sec.\n",
      "Epoch 495: T.cost 0.003599, Train acc 0.998860, test acc 0.682400, took 4.24 sec.\n",
      "Epoch 496: T.cost 0.006120, Train acc 0.998180, test acc 0.674400, took 4.27 sec.\n",
      "Epoch 497: T.cost 0.006035, Train acc 0.998320, test acc 0.678900, took 4.27 sec.\n",
      "Epoch 498: T.cost 0.013967, Train acc 0.996140, test acc 0.673600, took 4.25 sec.\n",
      "New LR: 0.000777821493102\n",
      "Epoch 499: T.cost 0.007113, Train acc 0.998160, test acc 0.677600, took 4.27 sec.\n",
      "Epoch 500: T.cost 0.005864, Train acc 0.998200, test acc 0.678500, took 4.26 sec.\n",
      "Epoch 501: T.cost 0.004075, Train acc 0.998880, test acc 0.675800, took 4.25 sec.\n",
      "Epoch 502: T.cost 0.004166, Train acc 0.998620, test acc 0.674000, took 4.27 sec.\n",
      "Epoch 503: T.cost 0.002271, Train acc 0.999100, test acc 0.675300, took 4.25 sec.\n",
      "Epoch 504: T.cost 0.002448, Train acc 0.999260, test acc 0.679100, took 4.26 sec.\n",
      "Epoch 505: T.cost 0.003758, Train acc 0.998940, test acc 0.671500, took 4.29 sec.\n",
      "Epoch 506: T.cost 0.015400, Train acc 0.995800, test acc 0.675100, took 4.23 sec.\n",
      "Epoch 507: T.cost 0.009190, Train acc 0.997380, test acc 0.675100, took 4.23 sec.\n",
      "Epoch 508: T.cost 0.007011, Train acc 0.997920, test acc 0.672600, took 4.22 sec.\n",
      "Epoch 509: T.cost 0.007994, Train acc 0.997760, test acc 0.675700, took 4.22 sec.\n",
      "Epoch 510: T.cost 0.007801, Train acc 0.997780, test acc 0.667700, took 4.24 sec.\n",
      "Epoch 511: T.cost 0.005624, Train acc 0.998600, test acc 0.677800, took 4.21 sec.\n",
      "Epoch 512: T.cost 0.003567, Train acc 0.998980, test acc 0.679100, took 4.25 sec.\n",
      "Epoch 513: T.cost 0.004010, Train acc 0.998960, test acc 0.679100, took 4.24 sec.\n",
      "Epoch 514: T.cost 0.003660, Train acc 0.998900, test acc 0.680700, took 4.21 sec.\n",
      "Epoch 515: T.cost 0.007249, Train acc 0.998060, test acc 0.670800, took 4.28 sec.\n",
      "Epoch 516: T.cost 0.007282, Train acc 0.997900, test acc 0.672800, took 4.24 sec.\n",
      "Epoch 517: T.cost 0.007166, Train acc 0.998060, test acc 0.669100, took 4.24 sec.\n",
      "Epoch 518: T.cost 0.007760, Train acc 0.998000, test acc 0.679300, took 4.23 sec.\n",
      "New LR: 0.000770043306984\n",
      "Epoch 519: T.cost 0.007201, Train acc 0.998140, test acc 0.671500, took 4.22 sec.\n",
      "Epoch 520: T.cost 0.003976, Train acc 0.998700, test acc 0.679200, took 4.33 sec.\n",
      "Epoch 521: T.cost 0.009865, Train acc 0.997640, test acc 0.676100, took 4.16 sec.\n",
      "Epoch 522: T.cost 0.008309, Train acc 0.997880, test acc 0.677400, took 4.17 sec.\n",
      "Epoch 523: T.cost 0.002478, Train acc 0.999220, test acc 0.676700, took 4.46 sec.\n",
      "Epoch 524: T.cost 0.005438, Train acc 0.998420, test acc 0.682600, took 4.4 sec.\n",
      "Epoch 525: T.cost 0.002970, Train acc 0.999220, test acc 0.681300, took 4.21 sec.\n",
      "Epoch 526: T.cost 0.004849, Train acc 0.998740, test acc 0.678600, took 4.3 sec.\n",
      "Epoch 527: T.cost 0.007396, Train acc 0.997880, test acc 0.672700, took 4.24 sec.\n",
      "Epoch 528: T.cost 0.007486, Train acc 0.997980, test acc 0.673200, took 4.24 sec.\n",
      "Epoch 529: T.cost 0.008139, Train acc 0.997500, test acc 0.674400, took 4.24 sec.\n",
      "Epoch 530: T.cost 0.005495, Train acc 0.998560, test acc 0.675400, took 4.23 sec.\n",
      "Epoch 531: T.cost 0.006366, Train acc 0.998120, test acc 0.675400, took 4.26 sec.\n",
      "Epoch 532: T.cost 0.004957, Train acc 0.998480, test acc 0.679500, took 4.32 sec.\n",
      "Epoch 533: T.cost 0.009378, Train acc 0.997580, test acc 0.677700, took 4.22 sec.\n",
      "Epoch 534: T.cost 0.003132, Train acc 0.999040, test acc 0.674400, took 4.26 sec.\n",
      "Epoch 535: T.cost 0.002919, Train acc 0.999020, test acc 0.676700, took 4.26 sec.\n",
      "Epoch 536: T.cost 0.006626, Train acc 0.998300, test acc 0.670900, took 4.27 sec.\n",
      "Epoch 537: T.cost 0.016285, Train acc 0.995680, test acc 0.669300, took 4.27 sec.\n",
      "Epoch 538: T.cost 0.003397, Train acc 0.998940, test acc 0.678600, took 4.33 sec.\n",
      "New LR: 0.000762342857779\n",
      "Epoch 539: T.cost 0.001901, Train acc 0.999440, test acc 0.679500, took 4.28 sec.\n",
      "Epoch 540: T.cost 0.003075, Train acc 0.999080, test acc 0.677700, took 4.26 sec.\n",
      "Epoch 541: T.cost 0.002507, Train acc 0.999340, test acc 0.680100, took 4.24 sec.\n",
      "Epoch 542: T.cost 0.002522, Train acc 0.999300, test acc 0.679600, took 4.32 sec.\n",
      "Epoch 543: T.cost 0.006092, Train acc 0.998180, test acc 0.672600, took 4.39 sec.\n",
      "Epoch 544: T.cost 0.011098, Train acc 0.997060, test acc 0.671200, took 4.24 sec.\n",
      "Epoch 545: T.cost 0.006838, Train acc 0.997920, test acc 0.679100, took 4.34 sec.\n",
      "Epoch 546: T.cost 0.005140, Train acc 0.998520, test acc 0.680100, took 4.45 sec.\n",
      "Epoch 547: T.cost 0.006076, Train acc 0.998160, test acc 0.673000, took 4.25 sec.\n",
      "Epoch 548: T.cost 0.012009, Train acc 0.997360, test acc 0.679200, took 4.26 sec.\n",
      "Epoch 549: T.cost 0.008626, Train acc 0.997680, test acc 0.675400, took 4.25 sec.\n",
      "Epoch 550: T.cost 0.005069, Train acc 0.998400, test acc 0.680300, took 4.23 sec.\n",
      "Epoch 551: T.cost 0.005637, Train acc 0.998400, test acc 0.673500, took 4.24 sec.\n",
      "Epoch 552: T.cost 0.003900, Train acc 0.998960, test acc 0.680000, took 4.22 sec.\n",
      "Epoch 553: T.cost 0.003907, Train acc 0.998800, test acc 0.677900, took 4.21 sec.\n",
      "Epoch 554: T.cost 0.007501, Train acc 0.997760, test acc 0.681600, took 4.22 sec.\n",
      "Epoch 555: T.cost 0.006362, Train acc 0.998340, test acc 0.674900, took 4.22 sec.\n",
      "Epoch 556: T.cost 0.010603, Train acc 0.997340, test acc 0.679700, took 4.25 sec.\n",
      "Epoch 557: T.cost 0.007738, Train acc 0.997920, test acc 0.678600, took 4.24 sec.\n",
      "Epoch 558: T.cost 0.001045, Train acc 0.999640, test acc 0.683200, took 4.25 sec.\n",
      "New LR: 0.00075471945398\n",
      "Epoch 559: T.cost 0.001786, Train acc 0.999480, test acc 0.671200, took 4.23 sec.\n",
      "Epoch 560: T.cost 0.010920, Train acc 0.996780, test acc 0.680600, took 4.34 sec.\n",
      "Epoch 561: T.cost 0.006833, Train acc 0.998260, test acc 0.681300, took 4.42 sec.\n",
      "Epoch 562: T.cost 0.004072, Train acc 0.998800, test acc 0.682500, took 4.22 sec.\n",
      "Epoch 563: T.cost 0.003424, Train acc 0.998980, test acc 0.680800, took 4.37 sec.\n",
      "Epoch 564: T.cost 0.003229, Train acc 0.998920, test acc 0.682900, took 4.5 sec.\n",
      "Epoch 565: T.cost 0.001714, Train acc 0.999440, test acc 0.683600, took 4.34 sec.\n",
      "Epoch 566: T.cost 0.002302, Train acc 0.999480, test acc 0.681900, took 4.28 sec.\n",
      "Epoch 567: T.cost 0.003592, Train acc 0.998900, test acc 0.672400, took 4.41 sec.\n",
      "Epoch 568: T.cost 0.001867, Train acc 0.999380, test acc 0.678000, took 4.27 sec.\n",
      "Epoch 569: T.cost 0.010332, Train acc 0.997220, test acc 0.669100, took 4.31 sec.\n",
      "Epoch 570: T.cost 0.017680, Train acc 0.996040, test acc 0.671500, took 4.29 sec.\n",
      "Epoch 571: T.cost 0.006178, Train acc 0.998220, test acc 0.678600, took 4.28 sec.\n",
      "Epoch 572: T.cost 0.002694, Train acc 0.999300, test acc 0.681500, took 4.26 sec.\n",
      "Epoch 573: T.cost 0.002497, Train acc 0.999260, test acc 0.680400, took 4.28 sec.\n",
      "Epoch 574: T.cost 0.002116, Train acc 0.999480, test acc 0.677100, took 4.32 sec.\n",
      "Epoch 575: T.cost 0.006212, Train acc 0.998260, test acc 0.679200, took 4.31 sec.\n",
      "Epoch 576: T.cost 0.008654, Train acc 0.997860, test acc 0.671900, took 4.39 sec.\n",
      "Epoch 577: T.cost 0.007290, Train acc 0.997880, test acc 0.673500, took 4.33 sec.\n",
      "Epoch 578: T.cost 0.004654, Train acc 0.998560, test acc 0.673300, took 4.3 sec.\n",
      "New LR: 0.000747172231204\n",
      "Epoch 579: T.cost 0.002501, Train acc 0.999240, test acc 0.678300, took 4.27 sec.\n",
      "Epoch 580: T.cost 0.008366, Train acc 0.997700, test acc 0.674900, took 4.28 sec.\n",
      "Epoch 581: T.cost 0.004781, Train acc 0.998720, test acc 0.676900, took 4.29 sec.\n",
      "Epoch 582: T.cost 0.004576, Train acc 0.998820, test acc 0.676800, took 4.29 sec.\n",
      "Epoch 583: T.cost 0.002620, Train acc 0.999260, test acc 0.679500, took 4.28 sec.\n",
      "Epoch 584: T.cost 0.001222, Train acc 0.999640, test acc 0.682400, took 4.28 sec.\n",
      "Epoch 585: T.cost 0.000159, Train acc 0.999980, test acc 0.682900, took 4.26 sec.\n",
      "Epoch 586: T.cost 0.000136, Train acc 0.999960, test acc 0.682500, took 4.23 sec.\n",
      "Epoch 587: T.cost 0.000017, Train acc 1.000000, test acc 0.683000, took 4.3 sec.\n",
      "Epoch 588: T.cost 0.000006, Train acc 1.000000, test acc 0.682900, took 4.25 sec.\n",
      "Epoch 589: T.cost 0.000005, Train acc 1.000000, test acc 0.683200, took 4.27 sec.\n",
      "Epoch 590: T.cost 0.000005, Train acc 1.000000, test acc 0.683100, took 4.36 sec.\n",
      "Epoch 591: T.cost 0.000004, Train acc 1.000000, test acc 0.682700, took 4.27 sec.\n",
      "Epoch 592: T.cost 0.000004, Train acc 1.000000, test acc 0.682400, took 4.29 sec.\n",
      "Epoch 593: T.cost 0.000003, Train acc 1.000000, test acc 0.682300, took 4.33 sec.\n",
      "Epoch 594: T.cost 0.000003, Train acc 1.000000, test acc 0.682700, took 4.35 sec.\n",
      "Epoch 595: T.cost 0.000003, Train acc 1.000000, test acc 0.682900, took 4.3 sec.\n",
      "Epoch 596: T.cost 0.000003, Train acc 1.000000, test acc 0.682600, took 4.29 sec.\n",
      "Epoch 597: T.cost 0.000002, Train acc 1.000000, test acc 0.682700, took 4.34 sec.\n",
      "Epoch 598: T.cost 0.000002, Train acc 1.000000, test acc 0.682800, took 4.35 sec.\n",
      "New LR: 0.000739700497943\n",
      "Epoch 599: T.cost 0.000002, Train acc 1.000000, test acc 0.682800, took 4.34 sec.\n",
      "Epoch 600: T.cost 0.000002, Train acc 1.000000, test acc 0.682700, took 4.34 sec.\n",
      "Epoch 601: T.cost 0.000002, Train acc 1.000000, test acc 0.682800, took 4.4 sec.\n",
      "Epoch 602: T.cost 0.000002, Train acc 1.000000, test acc 0.682800, took 4.36 sec.\n",
      "Epoch 603: T.cost 0.000002, Train acc 1.000000, test acc 0.682800, took 4.41 sec.\n",
      "Epoch 604: T.cost 0.000002, Train acc 1.000000, test acc 0.682900, took 4.35 sec.\n",
      "Epoch 605: T.cost 0.000001, Train acc 1.000000, test acc 0.682900, took 4.43 sec.\n",
      "Epoch 606: T.cost 0.000001, Train acc 1.000000, test acc 0.683200, took 4.35 sec.\n",
      "Epoch 607: T.cost 0.000001, Train acc 1.000000, test acc 0.682800, took 4.5 sec.\n",
      "Epoch 608: T.cost 0.000001, Train acc 1.000000, test acc 0.682700, took 4.36 sec.\n",
      "Epoch 609: T.cost 0.000001, Train acc 1.000000, test acc 0.682700, took 4.35 sec.\n",
      "Epoch 610: T.cost 0.000001, Train acc 1.000000, test acc 0.682700, took 4.29 sec.\n",
      "Epoch 611: T.cost 0.000001, Train acc 1.000000, test acc 0.682400, took 4.4 sec.\n",
      "Epoch 612: T.cost 0.000001, Train acc 1.000000, test acc 0.682300, took 4.42 sec.\n",
      "Epoch 613: T.cost 0.000001, Train acc 1.000000, test acc 0.682400, took 4.31 sec.\n",
      "Epoch 614: T.cost 0.000001, Train acc 1.000000, test acc 0.682600, took 4.32 sec.\n",
      "Epoch 615: T.cost 0.000001, Train acc 1.000000, test acc 0.682900, took 4.27 sec.\n",
      "Epoch 616: T.cost 0.000001, Train acc 1.000000, test acc 0.683400, took 4.31 sec.\n",
      "Epoch 617: T.cost 0.000001, Train acc 1.000000, test acc 0.683700, took 4.4 sec.\n",
      "Epoch 618: T.cost 0.000001, Train acc 1.000000, test acc 0.683700, took 4.44 sec.\n",
      "New LR: 0.000732303505065\n",
      "Epoch 619: T.cost 0.000001, Train acc 1.000000, test acc 0.683900, took 4.43 sec.\n",
      "Epoch 620: T.cost 0.000001, Train acc 1.000000, test acc 0.684000, took 4.41 sec.\n",
      "Epoch 621: T.cost 0.000001, Train acc 1.000000, test acc 0.684000, took 4.56 sec.\n",
      "Epoch 622: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.38 sec.\n",
      "Epoch 623: T.cost 0.000000, Train acc 1.000000, test acc 0.684200, took 4.28 sec.\n",
      "Epoch 624: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.42 sec.\n",
      "Epoch 625: T.cost 0.000000, Train acc 1.000000, test acc 0.684600, took 4.36 sec.\n",
      "Epoch 626: T.cost 0.000000, Train acc 1.000000, test acc 0.684800, took 4.56 sec.\n",
      "Epoch 627: T.cost 0.000000, Train acc 1.000000, test acc 0.684500, took 4.63 sec.\n",
      "Epoch 628: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.6 sec.\n",
      "Epoch 629: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.66 sec.\n",
      "Epoch 630: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.38 sec.\n",
      "Epoch 631: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.45 sec.\n",
      "Epoch 632: T.cost 0.000000, Train acc 1.000000, test acc 0.684200, took 4.27 sec.\n",
      "Epoch 633: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.24 sec.\n",
      "Epoch 634: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.46 sec.\n",
      "Epoch 635: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.53 sec.\n",
      "Epoch 636: T.cost 0.000000, Train acc 1.000000, test acc 0.684500, took 4.38 sec.\n",
      "Epoch 637: T.cost 0.000000, Train acc 1.000000, test acc 0.684700, took 4.4 sec.\n",
      "Epoch 638: T.cost 0.000000, Train acc 1.000000, test acc 0.684900, took 4.29 sec.\n",
      "New LR: 0.000724980445812\n",
      "Epoch 639: T.cost 0.000000, Train acc 1.000000, test acc 0.685000, took 4.26 sec.\n",
      "Epoch 640: T.cost 0.000000, Train acc 1.000000, test acc 0.685200, took 4.27 sec.\n",
      "Epoch 641: T.cost 0.000000, Train acc 1.000000, test acc 0.685300, took 4.48 sec.\n",
      "Epoch 642: T.cost 0.000000, Train acc 1.000000, test acc 0.685600, took 4.34 sec.\n",
      "Epoch 643: T.cost 0.000000, Train acc 1.000000, test acc 0.685700, took 4.53 sec.\n",
      "Epoch 644: T.cost 0.000000, Train acc 1.000000, test acc 0.686000, took 4.48 sec.\n",
      "Epoch 645: T.cost 0.000000, Train acc 1.000000, test acc 0.686000, took 4.4 sec.\n",
      "Epoch 646: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.36 sec.\n",
      "Epoch 647: T.cost 0.000000, Train acc 1.000000, test acc 0.686700, took 4.34 sec.\n",
      "Epoch 648: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.37 sec.\n",
      "Epoch 649: T.cost 0.000000, Train acc 1.000000, test acc 0.686400, took 4.39 sec.\n",
      "Epoch 650: T.cost 0.000000, Train acc 1.000000, test acc 0.686300, took 4.49 sec.\n",
      "Epoch 651: T.cost 0.000000, Train acc 1.000000, test acc 0.686700, took 4.43 sec.\n",
      "Epoch 652: T.cost 0.000000, Train acc 1.000000, test acc 0.686900, took 4.43 sec.\n",
      "Epoch 653: T.cost 0.000000, Train acc 1.000000, test acc 0.686600, took 4.43 sec.\n",
      "Epoch 654: T.cost 0.000000, Train acc 1.000000, test acc 0.686900, took 4.4 sec.\n",
      "Epoch 655: T.cost 0.000000, Train acc 1.000000, test acc 0.687200, took 4.31 sec.\n",
      "Epoch 656: T.cost 0.000000, Train acc 1.000000, test acc 0.687300, took 4.34 sec.\n",
      "Epoch 657: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.33 sec.\n",
      "Epoch 658: T.cost 0.000000, Train acc 1.000000, test acc 0.687600, took 4.33 sec.\n",
      "New LR: 0.000717730628676\n",
      "Epoch 659: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.43 sec.\n",
      "Epoch 660: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.46 sec.\n",
      "Epoch 661: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.5 sec.\n",
      "Epoch 662: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.31 sec.\n",
      "Epoch 663: T.cost 0.000000, Train acc 1.000000, test acc 0.687600, took 4.29 sec.\n",
      "Epoch 664: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.38 sec.\n",
      "Epoch 665: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.38 sec.\n",
      "Epoch 666: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.36 sec.\n",
      "Epoch 667: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.29 sec.\n",
      "Epoch 668: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.33 sec.\n",
      "Epoch 669: T.cost 0.000000, Train acc 1.000000, test acc 0.687600, took 4.33 sec.\n",
      "Epoch 670: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.33 sec.\n",
      "Epoch 671: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.46 sec.\n",
      "Epoch 672: T.cost 0.000000, Train acc 1.000000, test acc 0.687100, took 4.45 sec.\n",
      "Epoch 673: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.36 sec.\n",
      "Epoch 674: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.39 sec.\n",
      "Epoch 675: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.5 sec.\n",
      "Epoch 676: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.44 sec.\n",
      "Epoch 677: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.5 sec.\n",
      "Epoch 678: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.36 sec.\n",
      "New LR: 0.000710553304525\n",
      "Epoch 679: T.cost 0.000000, Train acc 1.000000, test acc 0.688700, took 4.44 sec.\n",
      "Epoch 680: T.cost 0.000000, Train acc 1.000000, test acc 0.689100, took 4.37 sec.\n",
      "Epoch 681: T.cost 0.000000, Train acc 1.000000, test acc 0.689100, took 4.37 sec.\n",
      "Epoch 682: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.37 sec.\n",
      "Epoch 683: T.cost 0.000000, Train acc 1.000000, test acc 0.689300, took 4.35 sec.\n",
      "Epoch 684: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.35 sec.\n",
      "Epoch 685: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.36 sec.\n",
      "Epoch 686: T.cost 0.000000, Train acc 1.000000, test acc 0.689500, took 4.29 sec.\n",
      "Epoch 687: T.cost 0.000000, Train acc 1.000000, test acc 0.689200, took 4.44 sec.\n",
      "Epoch 688: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.38 sec.\n",
      "Epoch 689: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.37 sec.\n",
      "Epoch 690: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.4 sec.\n",
      "Epoch 691: T.cost 0.000000, Train acc 1.000000, test acc 0.689700, took 4.38 sec.\n",
      "Epoch 692: T.cost 0.000000, Train acc 1.000000, test acc 0.689800, took 4.53 sec.\n",
      "Epoch 693: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.56 sec.\n",
      "Epoch 694: T.cost 0.000000, Train acc 1.000000, test acc 0.689800, took 4.45 sec.\n",
      "Epoch 695: T.cost 0.000000, Train acc 1.000000, test acc 0.690100, took 4.49 sec.\n",
      "Epoch 696: T.cost 0.000000, Train acc 1.000000, test acc 0.690800, took 4.62 sec.\n",
      "Epoch 697: T.cost 0.000000, Train acc 1.000000, test acc 0.689800, took 4.55 sec.\n",
      "Epoch 698: T.cost 0.000000, Train acc 1.000000, test acc 0.690300, took 4.63 sec.\n",
      "New LR: 0.000703447781852\n",
      "Epoch 699: T.cost 0.000000, Train acc 1.000000, test acc 0.690000, took 4.65 sec.\n",
      "Epoch 700: T.cost 0.000000, Train acc 1.000000, test acc 0.689800, took 4.63 sec.\n",
      "Epoch 701: T.cost 0.000000, Train acc 1.000000, test acc 0.689900, took 4.62 sec.\n",
      "Epoch 702: T.cost 0.000000, Train acc 1.000000, test acc 0.690000, took 4.52 sec.\n",
      "Epoch 703: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.42 sec.\n",
      "Epoch 704: T.cost 0.000000, Train acc 1.000000, test acc 0.689500, took 4.43 sec.\n",
      "Epoch 705: T.cost 0.000000, Train acc 1.000000, test acc 0.689200, took 4.33 sec.\n",
      "Epoch 706: T.cost 0.000000, Train acc 1.000000, test acc 0.689300, took 4.37 sec.\n",
      "Epoch 707: T.cost 0.000000, Train acc 1.000000, test acc 0.688800, took 4.45 sec.\n",
      "Epoch 708: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.45 sec.\n",
      "Epoch 709: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.41 sec.\n",
      "Epoch 710: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.39 sec.\n",
      "Epoch 711: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.34 sec.\n",
      "Epoch 712: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.37 sec.\n",
      "Epoch 713: T.cost 0.000000, Train acc 1.000000, test acc 0.688200, took 4.43 sec.\n",
      "Epoch 714: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.31 sec.\n",
      "Epoch 715: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.35 sec.\n",
      "Epoch 716: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.45 sec.\n",
      "Epoch 717: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.32 sec.\n",
      "Epoch 718: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.37 sec.\n",
      "New LR: 0.000696413311525\n",
      "Epoch 719: T.cost 0.000000, Train acc 1.000000, test acc 0.688800, took 4.46 sec.\n",
      "Epoch 720: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.4 sec.\n",
      "Epoch 721: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.34 sec.\n",
      "Epoch 722: T.cost 0.000000, Train acc 1.000000, test acc 0.688700, took 4.47 sec.\n",
      "Epoch 723: T.cost 0.000000, Train acc 1.000000, test acc 0.689000, took 4.38 sec.\n",
      "Epoch 724: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.38 sec.\n",
      "Epoch 725: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.46 sec.\n",
      "Epoch 726: T.cost 0.000000, Train acc 1.000000, test acc 0.688800, took 4.37 sec.\n",
      "Epoch 727: T.cost 0.000000, Train acc 1.000000, test acc 0.686800, took 4.35 sec.\n",
      "Epoch 728: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.39 sec.\n",
      "Epoch 729: T.cost 0.000000, Train acc 1.000000, test acc 0.687200, took 4.4 sec.\n",
      "Epoch 730: T.cost 0.000000, Train acc 1.000000, test acc 0.686700, took 4.41 sec.\n",
      "Epoch 731: T.cost 0.067294, Train acc 0.989280, test acc 0.667600, took 4.36 sec.\n",
      "Epoch 732: T.cost 0.022857, Train acc 0.994940, test acc 0.676700, took 4.38 sec.\n",
      "Epoch 733: T.cost 0.004703, Train acc 0.998640, test acc 0.676700, took 4.43 sec.\n",
      "Epoch 734: T.cost 0.002311, Train acc 0.999360, test acc 0.673500, took 4.39 sec.\n",
      "Epoch 735: T.cost 0.003129, Train acc 0.999200, test acc 0.676500, took 4.45 sec.\n",
      "Epoch 736: T.cost 0.002817, Train acc 0.999060, test acc 0.680100, took 4.33 sec.\n",
      "Epoch 737: T.cost 0.001827, Train acc 0.999460, test acc 0.679400, took 4.38 sec.\n",
      "Epoch 738: T.cost 0.002397, Train acc 0.999340, test acc 0.680500, took 4.41 sec.\n",
      "New LR: 0.000689449202036\n",
      "Epoch 739: T.cost 0.001965, Train acc 0.999340, test acc 0.676600, took 4.33 sec.\n",
      "Epoch 740: T.cost 0.003170, Train acc 0.999200, test acc 0.678300, took 4.33 sec.\n",
      "Epoch 741: T.cost 0.003215, Train acc 0.999100, test acc 0.679100, took 4.35 sec.\n",
      "Epoch 742: T.cost 0.007455, Train acc 0.998080, test acc 0.675900, took 4.31 sec.\n",
      "Epoch 743: T.cost 0.007848, Train acc 0.997980, test acc 0.680500, took 4.32 sec.\n",
      "Epoch 744: T.cost 0.003828, Train acc 0.998940, test acc 0.680800, took 4.34 sec.\n",
      "Epoch 745: T.cost 0.003216, Train acc 0.999040, test acc 0.683600, took 4.33 sec.\n",
      "Epoch 746: T.cost 0.003780, Train acc 0.998940, test acc 0.679800, took 4.38 sec.\n",
      "Epoch 747: T.cost 0.006903, Train acc 0.998200, test acc 0.679100, took 4.35 sec.\n",
      "Epoch 748: T.cost 0.007037, Train acc 0.998240, test acc 0.678200, took 4.33 sec.\n",
      "Epoch 749: T.cost 0.003361, Train acc 0.999000, test acc 0.676400, took 4.33 sec.\n",
      "Epoch 750: T.cost 0.004519, Train acc 0.998840, test acc 0.680900, took 4.38 sec.\n",
      "Epoch 751: T.cost 0.008992, Train acc 0.997760, test acc 0.674100, took 4.33 sec.\n",
      "Epoch 752: T.cost 0.008151, Train acc 0.998080, test acc 0.678000, took 4.52 sec.\n",
      "Epoch 753: T.cost 0.004706, Train acc 0.998880, test acc 0.681900, took 4.35 sec.\n",
      "Epoch 754: T.cost 0.001245, Train acc 0.999680, test acc 0.679500, took 4.34 sec.\n",
      "Epoch 755: T.cost 0.001883, Train acc 0.999380, test acc 0.681900, took 4.33 sec.\n",
      "Epoch 756: T.cost 0.007558, Train acc 0.998120, test acc 0.682400, took 4.35 sec.\n",
      "Epoch 757: T.cost 0.006569, Train acc 0.998060, test acc 0.677500, took 4.39 sec.\n",
      "Epoch 758: T.cost 0.003533, Train acc 0.999020, test acc 0.674300, took 4.46 sec.\n",
      "New LR: 0.000682554704254\n",
      "Epoch 759: T.cost 0.004794, Train acc 0.998540, test acc 0.674100, took 4.43 sec.\n",
      "Epoch 760: T.cost 0.001656, Train acc 0.999440, test acc 0.680500, took 4.34 sec.\n",
      "Epoch 761: T.cost 0.002434, Train acc 0.999140, test acc 0.682300, took 4.32 sec.\n",
      "Epoch 762: T.cost 0.005047, Train acc 0.998680, test acc 0.675400, took 4.36 sec.\n",
      "Epoch 763: T.cost 0.006755, Train acc 0.998320, test acc 0.676300, took 4.5 sec.\n",
      "Epoch 764: T.cost 0.018964, Train acc 0.995980, test acc 0.678600, took 4.65 sec.\n",
      "Epoch 765: T.cost 0.006346, Train acc 0.998420, test acc 0.678500, took 4.49 sec.\n",
      "Epoch 766: T.cost 0.003744, Train acc 0.999020, test acc 0.681600, took 4.5 sec.\n",
      "Epoch 767: T.cost 0.004046, Train acc 0.998600, test acc 0.674800, took 4.45 sec.\n",
      "Epoch 768: T.cost 0.002052, Train acc 0.999440, test acc 0.681100, took 4.51 sec.\n",
      "Epoch 769: T.cost 0.001336, Train acc 0.999600, test acc 0.676400, took 4.33 sec.\n",
      "Epoch 770: T.cost 0.001749, Train acc 0.999540, test acc 0.677200, took 4.31 sec.\n",
      "Epoch 771: T.cost 0.001528, Train acc 0.999560, test acc 0.676100, took 4.37 sec.\n",
      "Epoch 772: T.cost 0.003022, Train acc 0.999100, test acc 0.680600, took 4.41 sec.\n",
      "Epoch 773: T.cost 0.009738, Train acc 0.997460, test acc 0.674400, took 4.34 sec.\n",
      "Epoch 774: T.cost 0.013425, Train acc 0.996800, test acc 0.677100, took 4.34 sec.\n",
      "Epoch 775: T.cost 0.003583, Train acc 0.998840, test acc 0.678800, took 4.38 sec.\n",
      "Epoch 776: T.cost 0.002580, Train acc 0.999180, test acc 0.678500, took 4.33 sec.\n",
      "Epoch 777: T.cost 0.002731, Train acc 0.999260, test acc 0.679100, took 4.36 sec.\n",
      "Epoch 778: T.cost 0.000945, Train acc 0.999660, test acc 0.675400, took 4.32 sec.\n",
      "New LR: 0.000675729184295\n",
      "Epoch 779: T.cost 0.001413, Train acc 0.999520, test acc 0.667300, took 4.32 sec.\n",
      "Epoch 780: T.cost 0.003167, Train acc 0.999140, test acc 0.676200, took 4.49 sec.\n",
      "Epoch 781: T.cost 0.006014, Train acc 0.998800, test acc 0.669100, took 4.28 sec.\n",
      "Epoch 782: T.cost 0.011218, Train acc 0.997520, test acc 0.679700, took 4.28 sec.\n",
      "Epoch 783: T.cost 0.003986, Train acc 0.998920, test acc 0.670000, took 4.29 sec.\n",
      "Epoch 784: T.cost 0.005098, Train acc 0.998520, test acc 0.679000, took 4.35 sec.\n",
      "Epoch 785: T.cost 0.006594, Train acc 0.998300, test acc 0.678100, took 4.3 sec.\n",
      "Epoch 786: T.cost 0.003154, Train acc 0.999020, test acc 0.677000, took 4.33 sec.\n",
      "Epoch 787: T.cost 0.004751, Train acc 0.998640, test acc 0.676000, took 4.25 sec.\n",
      "Epoch 788: T.cost 0.000972, Train acc 0.999700, test acc 0.681600, took 4.26 sec.\n",
      "Epoch 789: T.cost 0.000114, Train acc 0.999940, test acc 0.683000, took 4.25 sec.\n",
      "Epoch 790: T.cost 0.000027, Train acc 1.000000, test acc 0.679700, took 4.28 sec.\n",
      "Epoch 791: T.cost 0.000005, Train acc 1.000000, test acc 0.680000, took 4.38 sec.\n",
      "Epoch 792: T.cost 0.000004, Train acc 1.000000, test acc 0.680400, took 4.34 sec.\n",
      "Epoch 793: T.cost 0.000003, Train acc 1.000000, test acc 0.680400, took 4.3 sec.\n",
      "Epoch 794: T.cost 0.000003, Train acc 1.000000, test acc 0.680500, took 4.29 sec.\n",
      "Epoch 795: T.cost 0.000003, Train acc 1.000000, test acc 0.680600, took 4.27 sec.\n",
      "Epoch 796: T.cost 0.000002, Train acc 1.000000, test acc 0.680900, took 4.27 sec.\n",
      "Epoch 797: T.cost 0.000002, Train acc 1.000000, test acc 0.680800, took 4.29 sec.\n",
      "Epoch 798: T.cost 0.000002, Train acc 1.000000, test acc 0.681000, took 4.31 sec.\n",
      "New LR: 0.000668971893028\n",
      "Epoch 799: T.cost 0.000002, Train acc 1.000000, test acc 0.680900, took 4.37 sec.\n",
      "Epoch 800: T.cost 0.000002, Train acc 1.000000, test acc 0.681000, took 4.39 sec.\n",
      "Epoch 801: T.cost 0.000002, Train acc 1.000000, test acc 0.680900, took 4.46 sec.\n",
      "Epoch 802: T.cost 0.000002, Train acc 1.000000, test acc 0.681100, took 4.33 sec.\n",
      "Epoch 803: T.cost 0.000001, Train acc 1.000000, test acc 0.680800, took 4.41 sec.\n",
      "Epoch 804: T.cost 0.000001, Train acc 1.000000, test acc 0.681200, took 4.48 sec.\n",
      "Epoch 805: T.cost 0.000001, Train acc 1.000000, test acc 0.681300, took 4.3 sec.\n",
      "Epoch 806: T.cost 0.000001, Train acc 1.000000, test acc 0.681200, took 4.35 sec.\n",
      "Epoch 807: T.cost 0.000001, Train acc 1.000000, test acc 0.681300, took 4.35 sec.\n",
      "Epoch 808: T.cost 0.000001, Train acc 1.000000, test acc 0.681300, took 4.37 sec.\n",
      "Epoch 809: T.cost 0.000001, Train acc 1.000000, test acc 0.681400, took 4.3 sec.\n",
      "Epoch 810: T.cost 0.000001, Train acc 1.000000, test acc 0.681400, took 4.32 sec.\n",
      "Epoch 811: T.cost 0.000001, Train acc 1.000000, test acc 0.681400, took 4.34 sec.\n",
      "Epoch 812: T.cost 0.000001, Train acc 1.000000, test acc 0.681300, took 4.36 sec.\n",
      "Epoch 813: T.cost 0.000001, Train acc 1.000000, test acc 0.681100, took 4.37 sec.\n",
      "Epoch 814: T.cost 0.000001, Train acc 1.000000, test acc 0.681300, took 4.35 sec.\n",
      "Epoch 815: T.cost 0.000001, Train acc 1.000000, test acc 0.680900, took 4.32 sec.\n",
      "Epoch 816: T.cost 0.000001, Train acc 1.000000, test acc 0.680700, took 4.4 sec.\n",
      "Epoch 817: T.cost 0.000001, Train acc 1.000000, test acc 0.680500, took 4.44 sec.\n",
      "Epoch 818: T.cost 0.000001, Train acc 1.000000, test acc 0.680800, took 4.3 sec.\n",
      "New LR: 0.000662282196572\n",
      "Epoch 819: T.cost 0.000001, Train acc 1.000000, test acc 0.680800, took 4.27 sec.\n",
      "Epoch 820: T.cost 0.000000, Train acc 1.000000, test acc 0.680700, took 4.25 sec.\n",
      "Epoch 821: T.cost 0.000000, Train acc 1.000000, test acc 0.680700, took 4.26 sec.\n",
      "Epoch 822: T.cost 0.000000, Train acc 1.000000, test acc 0.680900, took 4.31 sec.\n",
      "Epoch 823: T.cost 0.000000, Train acc 1.000000, test acc 0.681200, took 4.31 sec.\n",
      "Epoch 824: T.cost 0.000000, Train acc 1.000000, test acc 0.681200, took 4.27 sec.\n",
      "Epoch 825: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.41 sec.\n",
      "Epoch 826: T.cost 0.000000, Train acc 1.000000, test acc 0.681400, took 4.35 sec.\n",
      "Epoch 827: T.cost 0.000000, Train acc 1.000000, test acc 0.681500, took 4.25 sec.\n",
      "Epoch 828: T.cost 0.000000, Train acc 1.000000, test acc 0.681500, took 4.26 sec.\n",
      "Epoch 829: T.cost 0.000000, Train acc 1.000000, test acc 0.681800, took 4.27 sec.\n",
      "Epoch 830: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.26 sec.\n",
      "Epoch 831: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.3 sec.\n",
      "Epoch 832: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.29 sec.\n",
      "Epoch 833: T.cost 0.000000, Train acc 1.000000, test acc 0.682200, took 4.35 sec.\n",
      "Epoch 834: T.cost 0.000000, Train acc 1.000000, test acc 0.682400, took 4.25 sec.\n",
      "Epoch 835: T.cost 0.000000, Train acc 1.000000, test acc 0.682300, took 4.24 sec.\n",
      "Epoch 836: T.cost 0.000000, Train acc 1.000000, test acc 0.682800, took 4.24 sec.\n",
      "Epoch 837: T.cost 0.000000, Train acc 1.000000, test acc 0.682800, took 4.31 sec.\n",
      "Epoch 838: T.cost 0.000000, Train acc 1.000000, test acc 0.682800, took 4.32 sec.\n",
      "New LR: 0.000655659403419\n",
      "Epoch 839: T.cost 0.000000, Train acc 1.000000, test acc 0.683100, took 4.28 sec.\n",
      "Epoch 840: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.29 sec.\n",
      "Epoch 841: T.cost 0.000000, Train acc 1.000000, test acc 0.683600, took 4.22 sec.\n",
      "Epoch 842: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.31 sec.\n",
      "Epoch 843: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.36 sec.\n",
      "Epoch 844: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.39 sec.\n",
      "Epoch 845: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.27 sec.\n",
      "Epoch 846: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.26 sec.\n",
      "Epoch 847: T.cost 0.000000, Train acc 1.000000, test acc 0.684700, took 4.26 sec.\n",
      "Epoch 848: T.cost 0.000000, Train acc 1.000000, test acc 0.684900, took 4.24 sec.\n",
      "Epoch 849: T.cost 0.000000, Train acc 1.000000, test acc 0.685100, took 4.27 sec.\n",
      "Epoch 850: T.cost 0.000000, Train acc 1.000000, test acc 0.685400, took 4.3 sec.\n",
      "Epoch 851: T.cost 0.000000, Train acc 1.000000, test acc 0.685400, took 4.32 sec.\n",
      "Epoch 852: T.cost 0.000000, Train acc 1.000000, test acc 0.685700, took 4.27 sec.\n",
      "Epoch 853: T.cost 0.000000, Train acc 1.000000, test acc 0.686200, took 4.37 sec.\n",
      "Epoch 854: T.cost 0.000000, Train acc 1.000000, test acc 0.686000, took 4.31 sec.\n",
      "Epoch 855: T.cost 0.000000, Train acc 1.000000, test acc 0.686100, took 4.24 sec.\n",
      "Epoch 856: T.cost 0.000000, Train acc 1.000000, test acc 0.685800, took 4.25 sec.\n",
      "Epoch 857: T.cost 0.000000, Train acc 1.000000, test acc 0.686300, took 4.33 sec.\n",
      "Epoch 858: T.cost 0.000000, Train acc 1.000000, test acc 0.686600, took 4.3 sec.\n",
      "New LR: 0.000649102822063\n",
      "Epoch 859: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.27 sec.\n",
      "Epoch 860: T.cost 0.000000, Train acc 1.000000, test acc 0.686900, took 4.35 sec.\n",
      "Epoch 861: T.cost 0.000000, Train acc 1.000000, test acc 0.686900, took 4.3 sec.\n",
      "Epoch 862: T.cost 0.000000, Train acc 1.000000, test acc 0.686900, took 4.42 sec.\n",
      "Epoch 863: T.cost 0.000000, Train acc 1.000000, test acc 0.687100, took 4.29 sec.\n",
      "Epoch 864: T.cost 0.000000, Train acc 1.000000, test acc 0.687100, took 4.25 sec.\n",
      "Epoch 865: T.cost 0.000000, Train acc 1.000000, test acc 0.687000, took 4.26 sec.\n",
      "Epoch 866: T.cost 0.000000, Train acc 1.000000, test acc 0.686900, took 4.36 sec.\n",
      "Epoch 867: T.cost 0.000000, Train acc 1.000000, test acc 0.687000, took 4.25 sec.\n",
      "Epoch 868: T.cost 0.000000, Train acc 1.000000, test acc 0.686800, took 4.26 sec.\n",
      "Epoch 869: T.cost 0.000000, Train acc 1.000000, test acc 0.687200, took 4.33 sec.\n",
      "Epoch 870: T.cost 0.000000, Train acc 1.000000, test acc 0.687300, took 4.29 sec.\n",
      "Epoch 871: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.3 sec.\n",
      "Epoch 872: T.cost 0.000000, Train acc 1.000000, test acc 0.687600, took 4.3 sec.\n",
      "Epoch 873: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.27 sec.\n",
      "Epoch 874: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.25 sec.\n",
      "Epoch 875: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.29 sec.\n",
      "Epoch 876: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.4 sec.\n",
      "Epoch 877: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.24 sec.\n",
      "Epoch 878: T.cost 0.000000, Train acc 1.000000, test acc 0.688200, took 4.25 sec.\n",
      "New LR: 0.000642611818621\n",
      "Epoch 879: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.26 sec.\n",
      "Epoch 880: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.25 sec.\n",
      "Epoch 881: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.17 sec.\n",
      "Epoch 882: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.19 sec.\n",
      "Epoch 883: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.27 sec.\n",
      "Epoch 884: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.32 sec.\n",
      "Epoch 885: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.29 sec.\n",
      "Epoch 886: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.42 sec.\n",
      "Epoch 887: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.51 sec.\n",
      "Epoch 888: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.37 sec.\n",
      "Epoch 889: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.32 sec.\n",
      "Epoch 890: T.cost 0.000000, Train acc 1.000000, test acc 0.688200, took 4.54 sec.\n",
      "Epoch 891: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.42 sec.\n",
      "Epoch 892: T.cost 0.000000, Train acc 1.000000, test acc 0.688800, took 4.32 sec.\n",
      "Epoch 893: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.25 sec.\n",
      "Epoch 894: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.26 sec.\n",
      "Epoch 895: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.33 sec.\n",
      "Epoch 896: T.cost 0.000000, Train acc 1.000000, test acc 0.688100, took 4.31 sec.\n",
      "Epoch 897: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.41 sec.\n",
      "Epoch 898: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.28 sec.\n",
      "New LR: 0.000636185701587\n",
      "Epoch 899: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.39 sec.\n",
      "Epoch 900: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.26 sec.\n",
      "Epoch 901: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.26 sec.\n",
      "Epoch 902: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.25 sec.\n",
      "Epoch 903: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.27 sec.\n",
      "Epoch 904: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.3 sec.\n",
      "Epoch 905: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.3 sec.\n",
      "Epoch 906: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.5 sec.\n",
      "Epoch 907: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.44 sec.\n",
      "Epoch 908: T.cost 0.000000, Train acc 1.000000, test acc 0.688200, took 4.26 sec.\n",
      "Epoch 909: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.24 sec.\n",
      "Epoch 910: T.cost 0.000000, Train acc 1.000000, test acc 0.688900, took 4.25 sec.\n",
      "Epoch 911: T.cost 0.000000, Train acc 1.000000, test acc 0.689100, took 4.25 sec.\n",
      "Epoch 912: T.cost 0.000000, Train acc 1.000000, test acc 0.688900, took 4.28 sec.\n",
      "Epoch 913: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.37 sec.\n",
      "Epoch 914: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.27 sec.\n",
      "Epoch 915: T.cost 0.000000, Train acc 1.000000, test acc 0.689200, took 4.32 sec.\n",
      "Epoch 916: T.cost 0.000000, Train acc 1.000000, test acc 0.689500, took 4.25 sec.\n",
      "Epoch 917: T.cost 0.000000, Train acc 1.000000, test acc 0.688900, took 4.41 sec.\n",
      "Epoch 918: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.34 sec.\n",
      "New LR: 0.00062982383708\n",
      "Epoch 919: T.cost 0.000000, Train acc 1.000000, test acc 0.689700, took 4.42 sec.\n",
      "Epoch 920: T.cost 0.000000, Train acc 1.000000, test acc 0.690100, took 4.39 sec.\n",
      "Epoch 921: T.cost 0.000000, Train acc 1.000000, test acc 0.690200, took 4.31 sec.\n",
      "Epoch 922: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.31 sec.\n",
      "Epoch 923: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.4 sec.\n",
      "Epoch 924: T.cost 0.000000, Train acc 1.000000, test acc 0.689800, took 4.35 sec.\n",
      "Epoch 925: T.cost 0.000000, Train acc 1.000000, test acc 0.688800, took 4.36 sec.\n",
      "Epoch 926: T.cost 0.000000, Train acc 1.000000, test acc 0.689300, took 4.36 sec.\n",
      "Epoch 927: T.cost 0.000000, Train acc 1.000000, test acc 0.690400, took 4.31 sec.\n",
      "Epoch 928: T.cost 0.000000, Train acc 1.000000, test acc 0.690100, took 4.32 sec.\n",
      "Epoch 929: T.cost 0.000000, Train acc 1.000000, test acc 0.689300, took 4.31 sec.\n",
      "Epoch 930: T.cost 0.000000, Train acc 1.000000, test acc 0.689300, took 4.29 sec.\n",
      "Epoch 931: T.cost 0.000000, Train acc 1.000000, test acc 0.688800, took 4.3 sec.\n",
      "Epoch 932: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.41 sec.\n",
      "Epoch 933: T.cost 0.000000, Train acc 1.000000, test acc 0.690500, took 4.29 sec.\n",
      "Epoch 934: T.cost 0.073086, Train acc 0.989060, test acc 0.670600, took 4.28 sec.\n",
      "Epoch 935: T.cost 0.008489, Train acc 0.998060, test acc 0.676600, took 4.31 sec.\n",
      "Epoch 936: T.cost 0.002363, Train acc 0.999280, test acc 0.676100, took 4.29 sec.\n",
      "Epoch 937: T.cost 0.000125, Train acc 0.999980, test acc 0.680300, took 4.38 sec.\n",
      "Epoch 938: T.cost 0.000014, Train acc 1.000000, test acc 0.680600, took 4.3 sec.\n",
      "New LR: 0.000623525591218\n",
      "Epoch 939: T.cost 0.000006, Train acc 1.000000, test acc 0.681000, took 4.28 sec.\n",
      "Epoch 940: T.cost 0.000005, Train acc 1.000000, test acc 0.681300, took 4.35 sec.\n",
      "Epoch 941: T.cost 0.000004, Train acc 1.000000, test acc 0.681700, took 4.36 sec.\n",
      "Epoch 942: T.cost 0.000004, Train acc 1.000000, test acc 0.681900, took 4.34 sec.\n",
      "Epoch 943: T.cost 0.000003, Train acc 1.000000, test acc 0.681800, took 4.36 sec.\n",
      "Epoch 944: T.cost 0.000003, Train acc 1.000000, test acc 0.681700, took 4.27 sec.\n",
      "Epoch 945: T.cost 0.000003, Train acc 1.000000, test acc 0.681600, took 4.27 sec.\n",
      "Epoch 946: T.cost 0.000002, Train acc 1.000000, test acc 0.681700, took 4.25 sec.\n",
      "Epoch 947: T.cost 0.000002, Train acc 1.000000, test acc 0.681800, took 4.43 sec.\n",
      "Epoch 948: T.cost 0.000002, Train acc 1.000000, test acc 0.682000, took 4.34 sec.\n",
      "Epoch 949: T.cost 0.000002, Train acc 1.000000, test acc 0.682000, took 4.16 sec.\n",
      "Epoch 950: T.cost 0.000002, Train acc 1.000000, test acc 0.681800, took 4.21 sec.\n",
      "Epoch 951: T.cost 0.000002, Train acc 1.000000, test acc 0.682000, took 4.25 sec.\n",
      "Epoch 952: T.cost 0.000002, Train acc 1.000000, test acc 0.682000, took 4.25 sec.\n",
      "Epoch 953: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.27 sec.\n",
      "Epoch 954: T.cost 0.000001, Train acc 1.000000, test acc 0.682300, took 4.27 sec.\n",
      "Epoch 955: T.cost 0.000001, Train acc 1.000000, test acc 0.682200, took 4.35 sec.\n",
      "Epoch 956: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.25 sec.\n",
      "Epoch 957: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.25 sec.\n",
      "Epoch 958: T.cost 0.000001, Train acc 1.000000, test acc 0.682000, took 4.27 sec.\n",
      "New LR: 0.000617290330119\n",
      "Epoch 959: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.28 sec.\n",
      "Epoch 960: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.3 sec.\n",
      "Epoch 961: T.cost 0.000001, Train acc 1.000000, test acc 0.682200, took 4.37 sec.\n",
      "Epoch 962: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.48 sec.\n",
      "Epoch 963: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.44 sec.\n",
      "Epoch 964: T.cost 0.000001, Train acc 1.000000, test acc 0.682100, took 4.32 sec.\n",
      "Epoch 965: T.cost 0.000001, Train acc 1.000000, test acc 0.682000, took 4.4 sec.\n",
      "Epoch 966: T.cost 0.000001, Train acc 1.000000, test acc 0.681700, took 4.29 sec.\n",
      "Epoch 967: T.cost 0.000001, Train acc 1.000000, test acc 0.681700, took 4.49 sec.\n",
      "Epoch 968: T.cost 0.000000, Train acc 1.000000, test acc 0.681700, took 4.37 sec.\n",
      "Epoch 969: T.cost 0.000000, Train acc 1.000000, test acc 0.681700, took 4.27 sec.\n",
      "Epoch 970: T.cost 0.000000, Train acc 1.000000, test acc 0.681600, took 4.28 sec.\n",
      "Epoch 971: T.cost 0.000000, Train acc 1.000000, test acc 0.681400, took 4.29 sec.\n",
      "Epoch 972: T.cost 0.000000, Train acc 1.000000, test acc 0.681500, took 4.48 sec.\n",
      "Epoch 973: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.27 sec.\n",
      "Epoch 974: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.29 sec.\n",
      "Epoch 975: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.29 sec.\n",
      "Epoch 976: T.cost 0.000000, Train acc 1.000000, test acc 0.681800, took 4.27 sec.\n",
      "Epoch 977: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.27 sec.\n",
      "Epoch 978: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.27 sec.\n",
      "New LR: 0.000611117419903\n",
      "Epoch 979: T.cost 0.000000, Train acc 1.000000, test acc 0.682300, took 4.3 sec.\n",
      "Epoch 980: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.33 sec.\n",
      "Epoch 981: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.31 sec.\n",
      "Epoch 982: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.44 sec.\n",
      "Epoch 983: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.23 sec.\n",
      "Epoch 984: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.24 sec.\n",
      "Epoch 985: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.25 sec.\n",
      "Epoch 986: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.26 sec.\n",
      "Epoch 987: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.35 sec.\n",
      "Epoch 988: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.32 sec.\n",
      "Epoch 989: T.cost 0.000000, Train acc 1.000000, test acc 0.681600, took 4.28 sec.\n",
      "Epoch 990: T.cost 0.000000, Train acc 1.000000, test acc 0.681400, took 4.32 sec.\n",
      "Epoch 991: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.37 sec.\n",
      "Epoch 992: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.4 sec.\n",
      "Epoch 993: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.24 sec.\n",
      "Epoch 994: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.24 sec.\n",
      "Epoch 995: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.32 sec.\n",
      "Epoch 996: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.24 sec.\n",
      "Epoch 997: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.46 sec.\n",
      "Epoch 998: T.cost 0.000000, Train acc 1.000000, test acc 0.681500, took 4.29 sec.\n",
      "New LR: 0.000605006226688\n",
      "Epoch 999: T.cost 0.000000, Train acc 1.000000, test acc 0.681800, took 4.36 sec.\n",
      "Epoch 1000: T.cost 0.000000, Train acc 1.000000, test acc 0.681600, took 4.28 sec.\n",
      "Epoch 1001: T.cost 0.000000, Train acc 1.000000, test acc 0.681800, took 4.29 sec.\n",
      "Epoch 1002: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.28 sec.\n",
      "Epoch 1003: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.28 sec.\n",
      "Epoch 1004: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.29 sec.\n",
      "Epoch 1005: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.3 sec.\n",
      "Epoch 1006: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.29 sec.\n",
      "Epoch 1007: T.cost 0.000000, Train acc 1.000000, test acc 0.682400, took 4.26 sec.\n",
      "Epoch 1008: T.cost 0.000000, Train acc 1.000000, test acc 0.682600, took 4.34 sec.\n",
      "Epoch 1009: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.25 sec.\n",
      "Epoch 1010: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.27 sec.\n",
      "Epoch 1011: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.27 sec.\n",
      "Epoch 1012: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.27 sec.\n",
      "Epoch 1013: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.34 sec.\n",
      "Epoch 1014: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.25 sec.\n",
      "Epoch 1015: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.26 sec.\n",
      "Epoch 1016: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.24 sec.\n",
      "Epoch 1017: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.19 sec.\n",
      "Epoch 1018: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.27 sec.\n",
      "New LR: 0.000598956174217\n",
      "Epoch 1019: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.47 sec.\n",
      "Epoch 1020: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.3 sec.\n",
      "Epoch 1021: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.29 sec.\n",
      "Epoch 1022: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.29 sec.\n",
      "Epoch 1023: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.28 sec.\n",
      "Epoch 1024: T.cost 0.000000, Train acc 1.000000, test acc 0.684500, took 4.3 sec.\n",
      "Epoch 1025: T.cost 0.000000, Train acc 1.000000, test acc 0.684900, took 4.28 sec.\n",
      "Epoch 1026: T.cost 0.000000, Train acc 1.000000, test acc 0.684900, took 4.35 sec.\n",
      "Epoch 1027: T.cost 0.000000, Train acc 1.000000, test acc 0.685200, took 4.31 sec.\n",
      "Epoch 1028: T.cost 0.000000, Train acc 1.000000, test acc 0.685100, took 4.32 sec.\n",
      "Epoch 1029: T.cost 0.000000, Train acc 1.000000, test acc 0.684700, took 4.29 sec.\n",
      "Epoch 1030: T.cost 0.000000, Train acc 1.000000, test acc 0.684800, took 4.29 sec.\n",
      "Epoch 1031: T.cost 0.000000, Train acc 1.000000, test acc 0.684900, took 4.28 sec.\n",
      "Epoch 1032: T.cost 0.000000, Train acc 1.000000, test acc 0.684900, took 4.29 sec.\n",
      "Epoch 1033: T.cost 0.000000, Train acc 1.000000, test acc 0.685000, took 4.29 sec.\n",
      "Epoch 1034: T.cost 0.000000, Train acc 1.000000, test acc 0.685100, took 4.3 sec.\n",
      "Epoch 1035: T.cost 0.000000, Train acc 1.000000, test acc 0.685200, took 4.29 sec.\n",
      "Epoch 1036: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.26 sec.\n",
      "Epoch 1037: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.38 sec.\n",
      "Epoch 1038: T.cost 0.000000, Train acc 1.000000, test acc 0.685300, took 4.35 sec.\n",
      "New LR: 0.00059296662861\n",
      "Epoch 1039: T.cost 0.000000, Train acc 1.000000, test acc 0.685300, took 4.34 sec.\n",
      "Epoch 1040: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.31 sec.\n",
      "Epoch 1041: T.cost 0.000000, Train acc 1.000000, test acc 0.685800, took 4.28 sec.\n",
      "Epoch 1042: T.cost 0.000000, Train acc 1.000000, test acc 0.685600, took 4.28 sec.\n",
      "Epoch 1043: T.cost 0.000000, Train acc 1.000000, test acc 0.685600, took 4.27 sec.\n",
      "Epoch 1044: T.cost 0.000000, Train acc 1.000000, test acc 0.686000, took 4.24 sec.\n",
      "Epoch 1045: T.cost 0.000000, Train acc 1.000000, test acc 0.686100, took 4.3 sec.\n",
      "Epoch 1046: T.cost 0.000000, Train acc 1.000000, test acc 0.686100, took 4.27 sec.\n",
      "Epoch 1047: T.cost 0.000000, Train acc 1.000000, test acc 0.686400, took 4.27 sec.\n",
      "Epoch 1048: T.cost 0.000000, Train acc 1.000000, test acc 0.686700, took 4.28 sec.\n",
      "Epoch 1049: T.cost 0.000000, Train acc 1.000000, test acc 0.686700, took 4.28 sec.\n",
      "Epoch 1050: T.cost 0.000000, Train acc 1.000000, test acc 0.686600, took 4.27 sec.\n",
      "Epoch 1051: T.cost 0.000000, Train acc 1.000000, test acc 0.686600, took 4.37 sec.\n",
      "Epoch 1052: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.4 sec.\n",
      "Epoch 1053: T.cost 0.000000, Train acc 1.000000, test acc 0.686800, took 4.33 sec.\n",
      "Epoch 1054: T.cost 0.000000, Train acc 1.000000, test acc 0.686700, took 4.27 sec.\n",
      "Epoch 1055: T.cost 0.000000, Train acc 1.000000, test acc 0.687000, took 4.46 sec.\n",
      "Epoch 1056: T.cost 0.000000, Train acc 1.000000, test acc 0.687600, took 4.36 sec.\n",
      "Epoch 1057: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.23 sec.\n",
      "Epoch 1058: T.cost 0.000000, Train acc 1.000000, test acc 0.687800, took 4.31 sec.\n",
      "New LR: 0.000587036955985\n",
      "Epoch 1059: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.3 sec.\n",
      "Epoch 1060: T.cost 0.000000, Train acc 1.000000, test acc 0.687300, took 4.25 sec.\n",
      "Epoch 1061: T.cost 0.000000, Train acc 1.000000, test acc 0.687200, took 4.26 sec.\n",
      "Epoch 1062: T.cost 0.000000, Train acc 1.000000, test acc 0.687100, took 4.3 sec.\n",
      "Epoch 1063: T.cost 0.000000, Train acc 1.000000, test acc 0.687300, took 4.3 sec.\n",
      "Epoch 1064: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.29 sec.\n",
      "Epoch 1065: T.cost 0.000000, Train acc 1.000000, test acc 0.687300, took 4.23 sec.\n",
      "Epoch 1066: T.cost 0.000000, Train acc 1.000000, test acc 0.687300, took 4.24 sec.\n",
      "Epoch 1067: T.cost 0.000000, Train acc 1.000000, test acc 0.687200, took 4.3 sec.\n",
      "Epoch 1068: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.37 sec.\n",
      "Epoch 1069: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.25 sec.\n",
      "Epoch 1070: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.24 sec.\n",
      "Epoch 1071: T.cost 0.000000, Train acc 1.000000, test acc 0.688700, took 4.29 sec.\n",
      "Epoch 1072: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.29 sec.\n",
      "Epoch 1073: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.25 sec.\n",
      "Epoch 1074: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.24 sec.\n",
      "Epoch 1075: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.25 sec.\n",
      "Epoch 1076: T.cost 0.000000, Train acc 1.000000, test acc 0.688700, took 4.23 sec.\n",
      "Epoch 1077: T.cost 0.000000, Train acc 1.000000, test acc 0.689000, took 4.25 sec.\n",
      "Epoch 1078: T.cost 0.000000, Train acc 1.000000, test acc 0.688500, took 4.24 sec.\n",
      "New LR: 0.000581166580087\n",
      "Epoch 1079: T.cost 0.022854, Train acc 0.997580, test acc 0.656300, took 4.24 sec.\n",
      "Epoch 1080: T.cost 0.059673, Train acc 0.990360, test acc 0.677900, took 4.24 sec.\n",
      "Epoch 1081: T.cost 0.007539, Train acc 0.998260, test acc 0.677200, took 4.24 sec.\n",
      "Epoch 1082: T.cost 0.002833, Train acc 0.999340, test acc 0.676700, took 4.27 sec.\n",
      "Epoch 1083: T.cost 0.001575, Train acc 0.999580, test acc 0.675200, took 4.15 sec.\n",
      "Epoch 1084: T.cost 0.001582, Train acc 0.999560, test acc 0.677300, took 4.2 sec.\n",
      "Epoch 1085: T.cost 0.001562, Train acc 0.999560, test acc 0.679300, took 4.28 sec.\n",
      "Epoch 1086: T.cost 0.002084, Train acc 0.999440, test acc 0.677600, took 4.24 sec.\n",
      "Epoch 1087: T.cost 0.002696, Train acc 0.999320, test acc 0.676800, took 4.34 sec.\n",
      "Epoch 1088: T.cost 0.000981, Train acc 0.999660, test acc 0.678900, took 4.28 sec.\n",
      "Epoch 1089: T.cost 0.000263, Train acc 0.999880, test acc 0.677400, took 4.35 sec.\n",
      "Epoch 1090: T.cost 0.000062, Train acc 1.000000, test acc 0.678000, took 4.25 sec.\n",
      "Epoch 1091: T.cost 0.000003, Train acc 1.000000, test acc 0.677800, took 4.28 sec.\n",
      "Epoch 1092: T.cost 0.000002, Train acc 1.000000, test acc 0.678700, took 4.25 sec.\n",
      "Epoch 1093: T.cost 0.000002, Train acc 1.000000, test acc 0.678800, took 4.24 sec.\n",
      "Epoch 1094: T.cost 0.000002, Train acc 1.000000, test acc 0.679100, took 4.28 sec.\n",
      "Epoch 1095: T.cost 0.000001, Train acc 1.000000, test acc 0.679300, took 4.23 sec.\n",
      "Epoch 1096: T.cost 0.000001, Train acc 1.000000, test acc 0.679200, took 4.22 sec.\n",
      "Epoch 1097: T.cost 0.000001, Train acc 1.000000, test acc 0.679300, took 4.26 sec.\n",
      "Epoch 1098: T.cost 0.000001, Train acc 1.000000, test acc 0.679500, took 4.35 sec.\n",
      "New LR: 0.000575354924658\n",
      "Epoch 1099: T.cost 0.000001, Train acc 1.000000, test acc 0.679900, took 4.28 sec.\n",
      "Epoch 1100: T.cost 0.000001, Train acc 1.000000, test acc 0.680000, took 4.44 sec.\n",
      "Epoch 1101: T.cost 0.000001, Train acc 1.000000, test acc 0.680000, took 4.33 sec.\n",
      "Epoch 1102: T.cost 0.000001, Train acc 1.000000, test acc 0.680200, took 4.26 sec.\n",
      "Epoch 1103: T.cost 0.000001, Train acc 1.000000, test acc 0.680500, took 4.25 sec.\n",
      "Epoch 1104: T.cost 0.000001, Train acc 1.000000, test acc 0.680300, took 4.27 sec.\n",
      "Epoch 1105: T.cost 0.000001, Train acc 1.000000, test acc 0.680700, took 4.26 sec.\n",
      "Epoch 1106: T.cost 0.000001, Train acc 1.000000, test acc 0.680500, took 4.27 sec.\n",
      "Epoch 1107: T.cost 0.000001, Train acc 1.000000, test acc 0.680600, took 4.27 sec.\n",
      "Epoch 1108: T.cost 0.000001, Train acc 1.000000, test acc 0.680700, took 4.3 sec.\n",
      "Epoch 1109: T.cost 0.000000, Train acc 1.000000, test acc 0.680500, took 4.27 sec.\n",
      "Epoch 1110: T.cost 0.000000, Train acc 1.000000, test acc 0.680700, took 4.27 sec.\n",
      "Epoch 1111: T.cost 0.000000, Train acc 1.000000, test acc 0.680500, took 4.28 sec.\n",
      "Epoch 1112: T.cost 0.000000, Train acc 1.000000, test acc 0.680800, took 4.29 sec.\n",
      "Epoch 1113: T.cost 0.000000, Train acc 1.000000, test acc 0.681000, took 4.25 sec.\n",
      "Epoch 1114: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.27 sec.\n",
      "Epoch 1115: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.29 sec.\n",
      "Epoch 1116: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.28 sec.\n",
      "Epoch 1117: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.28 sec.\n",
      "Epoch 1118: T.cost 0.000000, Train acc 1.000000, test acc 0.681000, took 4.31 sec.\n",
      "New LR: 0.000569601355819\n",
      "Epoch 1119: T.cost 0.000000, Train acc 1.000000, test acc 0.680900, took 4.28 sec.\n",
      "Epoch 1120: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.31 sec.\n",
      "Epoch 1121: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.27 sec.\n",
      "Epoch 1122: T.cost 0.000000, Train acc 1.000000, test acc 0.681200, took 4.33 sec.\n",
      "Epoch 1123: T.cost 0.000000, Train acc 1.000000, test acc 0.681400, took 4.27 sec.\n",
      "Epoch 1124: T.cost 0.000000, Train acc 1.000000, test acc 0.681400, took 4.27 sec.\n",
      "Epoch 1125: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.25 sec.\n",
      "Epoch 1126: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.25 sec.\n",
      "Epoch 1127: T.cost 0.000000, Train acc 1.000000, test acc 0.681400, took 4.24 sec.\n",
      "Epoch 1128: T.cost 0.000000, Train acc 1.000000, test acc 0.681300, took 4.24 sec.\n",
      "Epoch 1129: T.cost 0.000000, Train acc 1.000000, test acc 0.681400, took 4.27 sec.\n",
      "Epoch 1130: T.cost 0.000000, Train acc 1.000000, test acc 0.681200, took 4.29 sec.\n",
      "Epoch 1131: T.cost 0.000000, Train acc 1.000000, test acc 0.681500, took 4.26 sec.\n",
      "Epoch 1132: T.cost 0.000000, Train acc 1.000000, test acc 0.681600, took 4.28 sec.\n",
      "Epoch 1133: T.cost 0.000000, Train acc 1.000000, test acc 0.681600, took 4.34 sec.\n",
      "Epoch 1134: T.cost 0.000000, Train acc 1.000000, test acc 0.681800, took 4.31 sec.\n",
      "Epoch 1135: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.28 sec.\n",
      "Epoch 1136: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.27 sec.\n",
      "Epoch 1137: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.28 sec.\n",
      "Epoch 1138: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.28 sec.\n",
      "New LR: 0.000563905354938\n",
      "Epoch 1139: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.28 sec.\n",
      "Epoch 1140: T.cost 0.000000, Train acc 1.000000, test acc 0.681900, took 4.29 sec.\n",
      "Epoch 1141: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.31 sec.\n",
      "Epoch 1142: T.cost 0.000000, Train acc 1.000000, test acc 0.682200, took 4.28 sec.\n",
      "Epoch 1143: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.28 sec.\n",
      "Epoch 1144: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.29 sec.\n",
      "Epoch 1145: T.cost 0.000000, Train acc 1.000000, test acc 0.682200, took 4.32 sec.\n",
      "Epoch 1146: T.cost 0.000000, Train acc 1.000000, test acc 0.682600, took 4.5 sec.\n",
      "Epoch 1147: T.cost 0.000000, Train acc 1.000000, test acc 0.682600, took 4.28 sec.\n",
      "Epoch 1148: T.cost 0.000000, Train acc 1.000000, test acc 0.682900, took 4.19 sec.\n",
      "Epoch 1149: T.cost 0.000000, Train acc 1.000000, test acc 0.683000, took 4.2 sec.\n",
      "Epoch 1150: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.27 sec.\n",
      "Epoch 1151: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.29 sec.\n",
      "Epoch 1152: T.cost 0.000000, Train acc 1.000000, test acc 0.683100, took 4.35 sec.\n",
      "Epoch 1153: T.cost 0.000000, Train acc 1.000000, test acc 0.683000, took 4.31 sec.\n",
      "Epoch 1154: T.cost 0.000000, Train acc 1.000000, test acc 0.682800, took 4.29 sec.\n",
      "Epoch 1155: T.cost 0.000000, Train acc 1.000000, test acc 0.682900, took 4.32 sec.\n",
      "Epoch 1156: T.cost 0.000000, Train acc 1.000000, test acc 0.683000, took 4.28 sec.\n",
      "Epoch 1157: T.cost 0.000000, Train acc 1.000000, test acc 0.683100, took 4.41 sec.\n",
      "Epoch 1158: T.cost 0.000000, Train acc 1.000000, test acc 0.683600, took 4.3 sec.\n",
      "New LR: 0.000558266288135\n",
      "Epoch 1159: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.27 sec.\n",
      "Epoch 1160: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.37 sec.\n",
      "Epoch 1161: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.28 sec.\n",
      "Epoch 1162: T.cost 0.000000, Train acc 1.000000, test acc 0.683600, took 4.29 sec.\n",
      "Epoch 1163: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.27 sec.\n",
      "Epoch 1164: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.27 sec.\n",
      "Epoch 1165: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.38 sec.\n",
      "Epoch 1166: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.24 sec.\n",
      "Epoch 1167: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.26 sec.\n",
      "Epoch 1168: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.25 sec.\n",
      "Epoch 1169: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.27 sec.\n",
      "Epoch 1170: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.31 sec.\n",
      "Epoch 1171: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.26 sec.\n",
      "Epoch 1172: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.24 sec.\n",
      "Epoch 1173: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.23 sec.\n",
      "Epoch 1174: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.22 sec.\n",
      "Epoch 1175: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.23 sec.\n",
      "Epoch 1176: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.27 sec.\n",
      "Epoch 1177: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.28 sec.\n",
      "Epoch 1178: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.25 sec.\n",
      "New LR: 0.000552683636779\n",
      "Epoch 1179: T.cost 0.000000, Train acc 1.000000, test acc 0.684200, took 4.28 sec.\n",
      "Epoch 1180: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.33 sec.\n",
      "Epoch 1181: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.28 sec.\n",
      "Epoch 1182: T.cost 0.000000, Train acc 1.000000, test acc 0.684300, took 4.25 sec.\n",
      "Epoch 1183: T.cost 0.000000, Train acc 1.000000, test acc 0.684200, took 4.24 sec.\n",
      "Epoch 1184: T.cost 0.000000, Train acc 1.000000, test acc 0.684200, took 4.29 sec.\n",
      "Epoch 1185: T.cost 0.000000, Train acc 1.000000, test acc 0.684500, took 4.23 sec.\n",
      "Epoch 1186: T.cost 0.000000, Train acc 1.000000, test acc 0.684600, took 4.27 sec.\n",
      "Epoch 1187: T.cost 0.000000, Train acc 1.000000, test acc 0.684700, took 4.46 sec.\n",
      "Epoch 1188: T.cost 0.000000, Train acc 1.000000, test acc 0.684800, took 4.4 sec.\n",
      "Epoch 1189: T.cost 0.000000, Train acc 1.000000, test acc 0.685000, took 4.27 sec.\n",
      "Epoch 1190: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.26 sec.\n",
      "Epoch 1191: T.cost 0.000000, Train acc 1.000000, test acc 0.685700, took 4.27 sec.\n",
      "Epoch 1192: T.cost 0.000000, Train acc 1.000000, test acc 0.686200, took 4.28 sec.\n",
      "Epoch 1193: T.cost 0.000000, Train acc 1.000000, test acc 0.686300, took 4.29 sec.\n",
      "Epoch 1194: T.cost 0.000000, Train acc 1.000000, test acc 0.686600, took 4.28 sec.\n",
      "Epoch 1195: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.31 sec.\n",
      "Epoch 1196: T.cost 0.000000, Train acc 1.000000, test acc 0.686400, took 4.28 sec.\n",
      "Epoch 1197: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.29 sec.\n",
      "Epoch 1198: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.3 sec.\n",
      "New LR: 0.000547156824614\n",
      "Epoch 1199: T.cost 0.000000, Train acc 1.000000, test acc 0.688400, took 4.3 sec.\n",
      "Epoch 1200: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.35 sec.\n",
      "Epoch 1201: T.cost 0.000000, Train acc 1.000000, test acc 0.688000, took 4.34 sec.\n",
      "Epoch 1202: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.27 sec.\n",
      "Epoch 1203: T.cost 0.000000, Train acc 1.000000, test acc 0.687400, took 4.28 sec.\n",
      "Epoch 1204: T.cost 0.000000, Train acc 1.000000, test acc 0.687900, took 4.35 sec.\n",
      "Epoch 1205: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.34 sec.\n",
      "Epoch 1206: T.cost 0.000000, Train acc 1.000000, test acc 0.688300, took 4.28 sec.\n",
      "Epoch 1207: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.26 sec.\n",
      "Epoch 1208: T.cost 0.000000, Train acc 1.000000, test acc 0.688800, took 4.26 sec.\n",
      "Epoch 1209: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.15 sec.\n",
      "Epoch 1210: T.cost 0.000000, Train acc 1.000000, test acc 0.690200, took 4.15 sec.\n",
      "Epoch 1211: T.cost 0.000000, Train acc 1.000000, test acc 0.690400, took 4.24 sec.\n",
      "Epoch 1212: T.cost 0.000000, Train acc 1.000000, test acc 0.690200, took 4.31 sec.\n",
      "Epoch 1213: T.cost 0.000000, Train acc 1.000000, test acc 0.690000, took 4.28 sec.\n",
      "Epoch 1214: T.cost 0.000000, Train acc 1.000000, test acc 0.690400, took 4.29 sec.\n",
      "Epoch 1215: T.cost 0.000000, Train acc 1.000000, test acc 0.690000, took 4.35 sec.\n",
      "Epoch 1216: T.cost 0.000000, Train acc 1.000000, test acc 0.691300, took 4.27 sec.\n",
      "Epoch 1217: T.cost 0.000000, Train acc 1.000000, test acc 0.690700, took 4.24 sec.\n",
      "Epoch 1218: T.cost 0.000000, Train acc 1.000000, test acc 0.690200, took 4.26 sec.\n",
      "New LR: 0.000541685275384\n",
      "Epoch 1219: T.cost 0.000000, Train acc 1.000000, test acc 0.690000, took 4.23 sec.\n",
      "Epoch 1220: T.cost 0.000000, Train acc 1.000000, test acc 0.690700, took 4.23 sec.\n",
      "Epoch 1221: T.cost 0.000000, Train acc 1.000000, test acc 0.690700, took 4.27 sec.\n",
      "Epoch 1222: T.cost 0.000000, Train acc 1.000000, test acc 0.690700, took 4.29 sec.\n",
      "Epoch 1223: T.cost 0.000000, Train acc 1.000000, test acc 0.691100, took 4.22 sec.\n",
      "Epoch 1224: T.cost 0.000000, Train acc 1.000000, test acc 0.691200, took 4.27 sec.\n",
      "Epoch 1225: T.cost 0.000000, Train acc 1.000000, test acc 0.691200, took 4.24 sec.\n",
      "Epoch 1226: T.cost 0.000000, Train acc 1.000000, test acc 0.691400, took 4.24 sec.\n",
      "Epoch 1227: T.cost 0.000000, Train acc 1.000000, test acc 0.692800, took 4.32 sec.\n",
      "Epoch 1228: T.cost 0.000000, Train acc 1.000000, test acc 0.692000, took 4.38 sec.\n",
      "Epoch 1229: T.cost 0.058339, Train acc 0.991800, test acc 0.666800, took 4.23 sec.\n",
      "Epoch 1230: T.cost 0.016303, Train acc 0.996760, test acc 0.675700, took 4.24 sec.\n",
      "Epoch 1231: T.cost 0.003888, Train acc 0.999100, test acc 0.679900, took 4.24 sec.\n",
      "Epoch 1232: T.cost 0.001201, Train acc 0.999620, test acc 0.679900, took 4.24 sec.\n",
      "Epoch 1233: T.cost 0.001408, Train acc 0.999720, test acc 0.676200, took 4.24 sec.\n",
      "Epoch 1234: T.cost 0.002081, Train acc 0.999560, test acc 0.675500, took 4.24 sec.\n",
      "Epoch 1235: T.cost 0.002712, Train acc 0.999420, test acc 0.683700, took 4.22 sec.\n",
      "Epoch 1236: T.cost 0.001854, Train acc 0.999500, test acc 0.678700, took 4.24 sec.\n",
      "Epoch 1237: T.cost 0.001750, Train acc 0.999560, test acc 0.681600, took 4.25 sec.\n",
      "Epoch 1238: T.cost 0.001183, Train acc 0.999580, test acc 0.677400, took 4.24 sec.\n",
      "New LR: 0.000536268412834\n",
      "Epoch 1239: T.cost 0.002799, Train acc 0.999240, test acc 0.678700, took 4.24 sec.\n",
      "Epoch 1240: T.cost 0.003477, Train acc 0.999000, test acc 0.677800, took 4.23 sec.\n",
      "Epoch 1241: T.cost 0.001872, Train acc 0.999360, test acc 0.673300, took 4.22 sec.\n",
      "Epoch 1242: T.cost 0.004448, Train acc 0.999060, test acc 0.678000, took 4.24 sec.\n",
      "Epoch 1243: T.cost 0.002728, Train acc 0.999220, test acc 0.675100, took 4.23 sec.\n",
      "Epoch 1244: T.cost 0.003834, Train acc 0.999020, test acc 0.679100, took 4.24 sec.\n",
      "Epoch 1245: T.cost 0.003216, Train acc 0.999280, test acc 0.672600, took 4.4 sec.\n",
      "Epoch 1246: T.cost 0.002979, Train acc 0.999100, test acc 0.677300, took 4.25 sec.\n",
      "Epoch 1247: T.cost 0.003448, Train acc 0.999260, test acc 0.674800, took 4.22 sec.\n",
      "Epoch 1248: T.cost 0.002074, Train acc 0.999420, test acc 0.679300, took 4.24 sec.\n",
      "Epoch 1249: T.cost 0.001188, Train acc 0.999640, test acc 0.678700, took 4.32 sec.\n",
      "Epoch 1250: T.cost 0.002427, Train acc 0.999380, test acc 0.673000, took 4.26 sec.\n",
      "Epoch 1251: T.cost 0.001436, Train acc 0.999540, test acc 0.680000, took 4.24 sec.\n",
      "Epoch 1252: T.cost 0.008235, Train acc 0.998140, test acc 0.677900, took 4.31 sec.\n",
      "Epoch 1253: T.cost 0.006338, Train acc 0.998580, test acc 0.667700, took 4.23 sec.\n",
      "Epoch 1254: T.cost 0.003667, Train acc 0.999180, test acc 0.679500, took 4.23 sec.\n",
      "Epoch 1255: T.cost 0.001752, Train acc 0.999540, test acc 0.678800, took 4.23 sec.\n",
      "Epoch 1256: T.cost 0.001023, Train acc 0.999640, test acc 0.680600, took 4.23 sec.\n",
      "Epoch 1257: T.cost 0.000937, Train acc 0.999860, test acc 0.679000, took 4.46 sec.\n",
      "Epoch 1258: T.cost 0.001488, Train acc 0.999640, test acc 0.675100, took 4.32 sec.\n",
      "New LR: 0.000530905718333\n",
      "Epoch 1259: T.cost 0.000993, Train acc 0.999660, test acc 0.684900, took 4.28 sec.\n",
      "Epoch 1260: T.cost 0.001823, Train acc 0.999420, test acc 0.680500, took 4.38 sec.\n",
      "Epoch 1261: T.cost 0.007700, Train acc 0.998020, test acc 0.674900, took 4.3 sec.\n",
      "Epoch 1262: T.cost 0.003652, Train acc 0.999180, test acc 0.675500, took 4.49 sec.\n",
      "Epoch 1263: T.cost 0.001505, Train acc 0.999620, test acc 0.678900, took 4.43 sec.\n",
      "Epoch 1264: T.cost 0.000157, Train acc 0.999960, test acc 0.678700, took 4.31 sec.\n",
      "Epoch 1265: T.cost 0.000318, Train acc 0.999880, test acc 0.682700, took 4.24 sec.\n",
      "Epoch 1266: T.cost 0.001955, Train acc 0.999320, test acc 0.677300, took 4.22 sec.\n",
      "Epoch 1267: T.cost 0.002194, Train acc 0.999460, test acc 0.675600, took 4.29 sec.\n",
      "Epoch 1268: T.cost 0.009786, Train acc 0.997760, test acc 0.676200, took 4.32 sec.\n",
      "Epoch 1269: T.cost 0.004916, Train acc 0.998680, test acc 0.678000, took 4.41 sec.\n",
      "Epoch 1270: T.cost 0.004196, Train acc 0.999000, test acc 0.679100, took 4.26 sec.\n",
      "Epoch 1271: T.cost 0.004036, Train acc 0.999220, test acc 0.675100, took 4.31 sec.\n",
      "Epoch 1272: T.cost 0.002038, Train acc 0.999480, test acc 0.675700, took 4.16 sec.\n",
      "Epoch 1273: T.cost 0.000874, Train acc 0.999700, test acc 0.676100, took 4.18 sec.\n",
      "Epoch 1274: T.cost 0.000032, Train acc 1.000000, test acc 0.678200, took 4.3 sec.\n",
      "Epoch 1275: T.cost 0.000004, Train acc 1.000000, test acc 0.678400, took 4.38 sec.\n",
      "Epoch 1276: T.cost 0.000003, Train acc 1.000000, test acc 0.678400, took 4.23 sec.\n",
      "Epoch 1277: T.cost 0.000002, Train acc 1.000000, test acc 0.678400, took 4.24 sec.\n",
      "Epoch 1278: T.cost 0.000002, Train acc 1.000000, test acc 0.678300, took 4.24 sec.\n",
      "New LR: 0.000525596673251\n",
      "Epoch 1279: T.cost 0.000002, Train acc 1.000000, test acc 0.678600, took 4.27 sec.\n",
      "Epoch 1280: T.cost 0.000001, Train acc 1.000000, test acc 0.678900, took 4.25 sec.\n",
      "Epoch 1281: T.cost 0.000001, Train acc 1.000000, test acc 0.679000, took 4.28 sec.\n",
      "Epoch 1282: T.cost 0.000001, Train acc 1.000000, test acc 0.679100, took 4.29 sec.\n",
      "Epoch 1283: T.cost 0.000001, Train acc 1.000000, test acc 0.679200, took 4.41 sec.\n",
      "Epoch 1284: T.cost 0.000001, Train acc 1.000000, test acc 0.679100, took 4.3 sec.\n",
      "Epoch 1285: T.cost 0.000001, Train acc 1.000000, test acc 0.679300, took 4.25 sec.\n",
      "Epoch 1286: T.cost 0.000001, Train acc 1.000000, test acc 0.679400, took 4.24 sec.\n",
      "Epoch 1287: T.cost 0.000001, Train acc 1.000000, test acc 0.679200, took 4.23 sec.\n",
      "Epoch 1288: T.cost 0.000001, Train acc 1.000000, test acc 0.679000, took 4.29 sec.\n",
      "Epoch 1289: T.cost 0.000001, Train acc 1.000000, test acc 0.679200, took 4.29 sec.\n",
      "Epoch 1290: T.cost 0.000001, Train acc 1.000000, test acc 0.679400, took 4.23 sec.\n",
      "Epoch 1291: T.cost 0.000001, Train acc 1.000000, test acc 0.679700, took 4.25 sec.\n",
      "Epoch 1292: T.cost 0.000001, Train acc 1.000000, test acc 0.679900, took 4.22 sec.\n",
      "Epoch 1293: T.cost 0.000000, Train acc 1.000000, test acc 0.679900, took 4.25 sec.\n",
      "Epoch 1294: T.cost 0.000000, Train acc 1.000000, test acc 0.680200, took 4.24 sec.\n",
      "Epoch 1295: T.cost 0.000000, Train acc 1.000000, test acc 0.680000, took 4.3 sec.\n",
      "Epoch 1296: T.cost 0.000000, Train acc 1.000000, test acc 0.680100, took 4.33 sec.\n",
      "Epoch 1297: T.cost 0.000000, Train acc 1.000000, test acc 0.680200, took 4.34 sec.\n",
      "Epoch 1298: T.cost 0.000000, Train acc 1.000000, test acc 0.680300, took 4.26 sec.\n",
      "New LR: 0.000520340701332\n",
      "Epoch 1299: T.cost 0.000000, Train acc 1.000000, test acc 0.680500, took 4.22 sec.\n",
      "Epoch 1300: T.cost 0.000000, Train acc 1.000000, test acc 0.680700, took 4.36 sec.\n",
      "Epoch 1301: T.cost 0.000000, Train acc 1.000000, test acc 0.680600, took 4.41 sec.\n",
      "Epoch 1302: T.cost 0.000000, Train acc 1.000000, test acc 0.680800, took 4.23 sec.\n",
      "Epoch 1303: T.cost 0.000000, Train acc 1.000000, test acc 0.680700, took 4.24 sec.\n",
      "Epoch 1304: T.cost 0.000000, Train acc 1.000000, test acc 0.680900, took 4.24 sec.\n",
      "Epoch 1305: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.22 sec.\n",
      "Epoch 1306: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.25 sec.\n",
      "Epoch 1307: T.cost 0.000000, Train acc 1.000000, test acc 0.681100, took 4.24 sec.\n",
      "Epoch 1308: T.cost 0.000000, Train acc 1.000000, test acc 0.681200, took 4.25 sec.\n",
      "Epoch 1309: T.cost 0.000000, Train acc 1.000000, test acc 0.681200, took 4.24 sec.\n",
      "Epoch 1310: T.cost 0.000000, Train acc 1.000000, test acc 0.681700, took 4.26 sec.\n",
      "Epoch 1311: T.cost 0.000000, Train acc 1.000000, test acc 0.681700, took 4.25 sec.\n",
      "Epoch 1312: T.cost 0.000000, Train acc 1.000000, test acc 0.681700, took 4.27 sec.\n",
      "Epoch 1313: T.cost 0.000000, Train acc 1.000000, test acc 0.681600, took 4.28 sec.\n",
      "Epoch 1314: T.cost 0.000000, Train acc 1.000000, test acc 0.681700, took 4.27 sec.\n",
      "Epoch 1315: T.cost 0.000000, Train acc 1.000000, test acc 0.681700, took 4.29 sec.\n",
      "Epoch 1316: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.33 sec.\n",
      "Epoch 1317: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.29 sec.\n",
      "Epoch 1318: T.cost 0.000000, Train acc 1.000000, test acc 0.682200, took 4.33 sec.\n",
      "New LR: 0.000515137283946\n",
      "Epoch 1319: T.cost 0.000000, Train acc 1.000000, test acc 0.682300, took 4.36 sec.\n",
      "Epoch 1320: T.cost 0.000000, Train acc 1.000000, test acc 0.682400, took 4.3 sec.\n",
      "Epoch 1321: T.cost 0.000000, Train acc 1.000000, test acc 0.682300, took 4.39 sec.\n",
      "Epoch 1322: T.cost 0.000000, Train acc 1.000000, test acc 0.682300, took 4.31 sec.\n",
      "Epoch 1323: T.cost 0.000000, Train acc 1.000000, test acc 0.682300, took 4.28 sec.\n",
      "Epoch 1324: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.29 sec.\n",
      "Epoch 1325: T.cost 0.000000, Train acc 1.000000, test acc 0.682000, took 4.34 sec.\n",
      "Epoch 1326: T.cost 0.000000, Train acc 1.000000, test acc 0.682100, took 4.29 sec.\n",
      "Epoch 1327: T.cost 0.000000, Train acc 1.000000, test acc 0.682200, took 4.28 sec.\n",
      "Epoch 1328: T.cost 0.000000, Train acc 1.000000, test acc 0.682600, took 4.28 sec.\n",
      "Epoch 1329: T.cost 0.000000, Train acc 1.000000, test acc 0.682700, took 4.3 sec.\n",
      "Epoch 1330: T.cost 0.000000, Train acc 1.000000, test acc 0.682900, took 4.28 sec.\n",
      "Epoch 1331: T.cost 0.000000, Train acc 1.000000, test acc 0.682900, took 4.26 sec.\n",
      "Epoch 1332: T.cost 0.000000, Train acc 1.000000, test acc 0.683000, took 4.29 sec.\n",
      "Epoch 1333: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.27 sec.\n",
      "Epoch 1334: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.28 sec.\n",
      "Epoch 1335: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.28 sec.\n",
      "Epoch 1336: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.22 sec.\n",
      "Epoch 1337: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.2 sec.\n",
      "Epoch 1338: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.28 sec.\n",
      "New LR: 0.000509985902463\n",
      "Epoch 1339: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.27 sec.\n",
      "Epoch 1340: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.26 sec.\n",
      "Epoch 1341: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.26 sec.\n",
      "Epoch 1342: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.27 sec.\n",
      "Epoch 1343: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.27 sec.\n",
      "Epoch 1344: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.26 sec.\n",
      "Epoch 1345: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.27 sec.\n",
      "Epoch 1346: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.37 sec.\n",
      "Epoch 1347: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.26 sec.\n",
      "Epoch 1348: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.24 sec.\n",
      "Epoch 1349: T.cost 0.000000, Train acc 1.000000, test acc 0.684400, took 4.24 sec.\n",
      "Epoch 1350: T.cost 0.000000, Train acc 1.000000, test acc 0.684600, took 4.24 sec.\n",
      "Epoch 1351: T.cost 0.000000, Train acc 1.000000, test acc 0.685000, took 4.21 sec.\n",
      "Epoch 1352: T.cost 0.000000, Train acc 1.000000, test acc 0.685100, took 4.24 sec.\n",
      "Epoch 1353: T.cost 0.000000, Train acc 1.000000, test acc 0.685200, took 4.24 sec.\n",
      "Epoch 1354: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.23 sec.\n",
      "Epoch 1355: T.cost 0.000000, Train acc 1.000000, test acc 0.685300, took 4.24 sec.\n",
      "Epoch 1356: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.24 sec.\n",
      "Epoch 1357: T.cost 0.000000, Train acc 1.000000, test acc 0.685300, took 4.23 sec.\n",
      "Epoch 1358: T.cost 0.000000, Train acc 1.000000, test acc 0.685300, took 4.3 sec.\n",
      "New LR: 0.000504886038252\n",
      "Epoch 1359: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.31 sec.\n",
      "Epoch 1360: T.cost 0.000000, Train acc 1.000000, test acc 0.685500, took 4.29 sec.\n",
      "Epoch 1361: T.cost 0.000000, Train acc 1.000000, test acc 0.686000, took 4.31 sec.\n",
      "Epoch 1362: T.cost 0.000000, Train acc 1.000000, test acc 0.686300, took 4.3 sec.\n",
      "Epoch 1363: T.cost 0.000000, Train acc 1.000000, test acc 0.686400, took 4.41 sec.\n",
      "Epoch 1364: T.cost 0.000000, Train acc 1.000000, test acc 0.686700, took 4.28 sec.\n",
      "Epoch 1365: T.cost 0.000000, Train acc 1.000000, test acc 0.686800, took 4.28 sec.\n",
      "Epoch 1366: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.28 sec.\n",
      "Epoch 1367: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.29 sec.\n",
      "Epoch 1368: T.cost 0.000000, Train acc 1.000000, test acc 0.686400, took 4.28 sec.\n",
      "Epoch 1369: T.cost 0.000000, Train acc 1.000000, test acc 0.686800, took 4.29 sec.\n",
      "Epoch 1370: T.cost 0.000000, Train acc 1.000000, test acc 0.686500, took 4.27 sec.\n",
      "Epoch 1371: T.cost 0.000000, Train acc 1.000000, test acc 0.686400, took 4.3 sec.\n",
      "Epoch 1372: T.cost 0.000000, Train acc 1.000000, test acc 0.686800, took 4.27 sec.\n",
      "Epoch 1373: T.cost 0.000000, Train acc 1.000000, test acc 0.687100, took 4.24 sec.\n",
      "Epoch 1374: T.cost 0.000000, Train acc 1.000000, test acc 0.687300, took 4.33 sec.\n",
      "Epoch 1375: T.cost 0.000000, Train acc 1.000000, test acc 0.687500, took 4.34 sec.\n",
      "Epoch 1376: T.cost 0.000000, Train acc 1.000000, test acc 0.687200, took 4.28 sec.\n",
      "Epoch 1377: T.cost 0.000000, Train acc 1.000000, test acc 0.687400, took 4.25 sec.\n",
      "Epoch 1378: T.cost 0.000000, Train acc 1.000000, test acc 0.687700, took 4.25 sec.\n",
      "New LR: 0.000499837172683\n",
      "Epoch 1379: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.31 sec.\n",
      "Epoch 1380: T.cost 0.000000, Train acc 1.000000, test acc 0.688700, took 4.33 sec.\n",
      "Epoch 1381: T.cost 0.000000, Train acc 1.000000, test acc 0.688600, took 4.44 sec.\n",
      "Epoch 1382: T.cost 0.000000, Train acc 1.000000, test acc 0.689200, took 4.24 sec.\n",
      "Epoch 1383: T.cost 0.000000, Train acc 1.000000, test acc 0.689100, took 4.24 sec.\n",
      "Epoch 1384: T.cost 0.000000, Train acc 1.000000, test acc 0.689300, took 4.25 sec.\n",
      "Epoch 1385: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.25 sec.\n",
      "Epoch 1386: T.cost 0.000000, Train acc 1.000000, test acc 0.689100, took 4.3 sec.\n",
      "Epoch 1387: T.cost 0.000000, Train acc 1.000000, test acc 0.689200, took 4.29 sec.\n",
      "Epoch 1388: T.cost 0.000000, Train acc 1.000000, test acc 0.689800, took 4.37 sec.\n",
      "Epoch 1389: T.cost 0.000000, Train acc 1.000000, test acc 0.689400, took 4.24 sec.\n",
      "Epoch 1390: T.cost 0.000000, Train acc 1.000000, test acc 0.689700, took 4.25 sec.\n",
      "Epoch 1391: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.25 sec.\n",
      "Epoch 1392: T.cost 0.000000, Train acc 1.000000, test acc 0.689900, took 4.28 sec.\n",
      "Epoch 1393: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.38 sec.\n",
      "Epoch 1394: T.cost 0.000000, Train acc 1.000000, test acc 0.690100, took 4.26 sec.\n",
      "Epoch 1395: T.cost 0.000000, Train acc 1.000000, test acc 0.690400, took 4.23 sec.\n",
      "Epoch 1396: T.cost 0.000000, Train acc 1.000000, test acc 0.690500, took 4.3 sec.\n",
      "Epoch 1397: T.cost 0.000000, Train acc 1.000000, test acc 0.690600, took 4.24 sec.\n",
      "Epoch 1398: T.cost 0.000000, Train acc 1.000000, test acc 0.690400, took 4.41 sec.\n",
      "New LR: 0.000494838787126\n",
      "Epoch 1399: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.38 sec.\n",
      "Epoch 1400: T.cost 0.000000, Train acc 1.000000, test acc 0.689700, took 4.24 sec.\n",
      "Epoch 1401: T.cost 0.000000, Train acc 1.000000, test acc 0.690100, took 4.17 sec.\n",
      "Epoch 1402: T.cost 0.000000, Train acc 1.000000, test acc 0.690300, took 4.21 sec.\n",
      "Epoch 1403: T.cost 0.000000, Train acc 1.000000, test acc 0.690900, took 4.24 sec.\n",
      "Epoch 1404: T.cost 0.000000, Train acc 1.000000, test acc 0.690300, took 4.25 sec.\n",
      "Epoch 1405: T.cost 0.000000, Train acc 1.000000, test acc 0.690500, took 4.24 sec.\n",
      "Epoch 1406: T.cost 0.000000, Train acc 1.000000, test acc 0.690400, took 4.23 sec.\n",
      "Epoch 1407: T.cost 0.000000, Train acc 1.000000, test acc 0.690500, took 4.23 sec.\n",
      "Epoch 1408: T.cost 0.000000, Train acc 1.000000, test acc 0.690900, took 4.36 sec.\n",
      "Epoch 1409: T.cost 0.000000, Train acc 1.000000, test acc 0.690300, took 4.25 sec.\n",
      "Epoch 1410: T.cost 0.000000, Train acc 1.000000, test acc 0.690600, took 4.23 sec.\n",
      "Epoch 1411: T.cost 0.000000, Train acc 1.000000, test acc 0.690100, took 4.23 sec.\n",
      "Epoch 1412: T.cost 0.000000, Train acc 1.000000, test acc 0.689800, took 4.23 sec.\n",
      "Epoch 1413: T.cost 0.000000, Train acc 1.000000, test acc 0.689600, took 4.38 sec.\n",
      "Epoch 1414: T.cost 0.000000, Train acc 1.000000, test acc 0.690800, took 4.35 sec.\n",
      "Epoch 1415: T.cost 0.065374, Train acc 0.991060, test acc 0.677800, took 4.27 sec.\n",
      "Epoch 1416: T.cost 0.007827, Train acc 0.998320, test acc 0.681100, took 4.27 sec.\n",
      "Epoch 1417: T.cost 0.002483, Train acc 0.999540, test acc 0.684900, took 4.28 sec.\n",
      "Epoch 1418: T.cost 0.000886, Train acc 0.999720, test acc 0.680000, took 4.28 sec.\n",
      "New LR: 0.000489890420577\n",
      "Epoch 1419: T.cost 0.000119, Train acc 0.999960, test acc 0.681600, took 4.28 sec.\n",
      "Epoch 1420: T.cost 0.000033, Train acc 0.999980, test acc 0.681900, took 4.33 sec.\n",
      "Epoch 1421: T.cost 0.000010, Train acc 1.000000, test acc 0.683500, took 4.4 sec.\n",
      "Epoch 1422: T.cost 0.000002, Train acc 1.000000, test acc 0.683300, took 4.28 sec.\n",
      "Epoch 1423: T.cost 0.000001, Train acc 1.000000, test acc 0.683300, took 4.31 sec.\n",
      "Epoch 1424: T.cost 0.000001, Train acc 1.000000, test acc 0.683400, took 4.25 sec.\n",
      "Epoch 1425: T.cost 0.000001, Train acc 1.000000, test acc 0.683200, took 4.24 sec.\n",
      "Epoch 1426: T.cost 0.000001, Train acc 1.000000, test acc 0.683400, took 4.22 sec.\n",
      "Epoch 1427: T.cost 0.000001, Train acc 1.000000, test acc 0.683400, took 4.23 sec.\n",
      "Epoch 1428: T.cost 0.000001, Train acc 1.000000, test acc 0.683100, took 4.24 sec.\n",
      "Epoch 1429: T.cost 0.000001, Train acc 1.000000, test acc 0.683200, took 4.26 sec.\n",
      "Epoch 1430: T.cost 0.000001, Train acc 1.000000, test acc 0.683100, took 4.23 sec.\n",
      "Epoch 1431: T.cost 0.000001, Train acc 1.000000, test acc 0.683100, took 4.24 sec.\n",
      "Epoch 1432: T.cost 0.000001, Train acc 1.000000, test acc 0.683200, took 4.25 sec.\n",
      "Epoch 1433: T.cost 0.000001, Train acc 1.000000, test acc 0.683100, took 4.23 sec.\n",
      "Epoch 1434: T.cost 0.000001, Train acc 1.000000, test acc 0.683100, took 4.24 sec.\n",
      "Epoch 1435: T.cost 0.000001, Train acc 1.000000, test acc 0.683000, took 4.28 sec.\n",
      "Epoch 1436: T.cost 0.000000, Train acc 1.000000, test acc 0.683000, took 4.27 sec.\n",
      "Epoch 1437: T.cost 0.000000, Train acc 1.000000, test acc 0.683100, took 4.29 sec.\n",
      "Epoch 1438: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.29 sec.\n",
      "New LR: 0.000484991496778\n",
      "Epoch 1439: T.cost 0.000000, Train acc 1.000000, test acc 0.683600, took 4.28 sec.\n",
      "Epoch 1440: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.4 sec.\n",
      "Epoch 1441: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.36 sec.\n",
      "Epoch 1442: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.29 sec.\n",
      "Epoch 1443: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.26 sec.\n",
      "Epoch 1444: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.24 sec.\n",
      "Epoch 1445: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.22 sec.\n",
      "Epoch 1446: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.44 sec.\n",
      "Epoch 1447: T.cost 0.000000, Train acc 1.000000, test acc 0.683100, took 4.32 sec.\n",
      "Epoch 1448: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.26 sec.\n",
      "Epoch 1449: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.25 sec.\n",
      "Epoch 1450: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.24 sec.\n",
      "Epoch 1451: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.25 sec.\n",
      "Epoch 1452: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.25 sec.\n",
      "Epoch 1453: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.25 sec.\n",
      "Epoch 1454: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.23 sec.\n",
      "Epoch 1455: T.cost 0.000000, Train acc 1.000000, test acc 0.683600, took 4.23 sec.\n",
      "Epoch 1456: T.cost 0.000000, Train acc 1.000000, test acc 0.683600, took 4.24 sec.\n",
      "Epoch 1457: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.24 sec.\n",
      "Epoch 1458: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.43 sec.\n",
      "New LR: 0.000480141583539\n",
      "Epoch 1459: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.3 sec.\n",
      "Epoch 1460: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.32 sec.\n",
      "Epoch 1461: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.24 sec.\n",
      "Epoch 1462: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.23 sec.\n",
      "Epoch 1463: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.24 sec.\n",
      "Epoch 1464: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.18 sec.\n",
      "Epoch 1465: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.17 sec.\n",
      "Epoch 1466: T.cost 0.000000, Train acc 1.000000, test acc 0.684200, took 4.23 sec.\n",
      "Epoch 1467: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.26 sec.\n",
      "Epoch 1468: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.29 sec.\n",
      "Epoch 1469: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.29 sec.\n",
      "Epoch 1470: T.cost 0.000000, Train acc 1.000000, test acc 0.683900, took 4.3 sec.\n",
      "Epoch 1471: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.26 sec.\n",
      "Epoch 1472: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.25 sec.\n",
      "Epoch 1473: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.26 sec.\n",
      "Epoch 1474: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.24 sec.\n",
      "Epoch 1475: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.24 sec.\n",
      "Epoch 1476: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.24 sec.\n",
      "Epoch 1477: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.34 sec.\n",
      "Epoch 1478: T.cost 0.000000, Train acc 1.000000, test acc 0.683100, took 4.27 sec.\n",
      "New LR: 0.000475340162229\n",
      "Epoch 1479: T.cost 0.000000, Train acc 1.000000, test acc 0.683200, took 4.25 sec.\n",
      "Epoch 1480: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.28 sec.\n",
      "Epoch 1481: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.24 sec.\n",
      "Epoch 1482: T.cost 0.000000, Train acc 1.000000, test acc 0.683300, took 4.26 sec.\n",
      "Epoch 1483: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.25 sec.\n",
      "Epoch 1484: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.24 sec.\n",
      "Epoch 1485: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.29 sec.\n",
      "Epoch 1486: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.25 sec.\n",
      "Epoch 1487: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.38 sec.\n",
      "Epoch 1488: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.33 sec.\n",
      "Epoch 1489: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.3 sec.\n",
      "Epoch 1490: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.23 sec.\n",
      "Epoch 1491: T.cost 0.000000, Train acc 1.000000, test acc 0.683400, took 4.25 sec.\n",
      "Epoch 1492: T.cost 0.000000, Train acc 1.000000, test acc 0.683500, took 4.22 sec.\n",
      "Epoch 1493: T.cost 0.000000, Train acc 1.000000, test acc 0.683800, took 4.29 sec.\n",
      "Epoch 1494: T.cost 0.000000, Train acc 1.000000, test acc 0.683700, took 4.26 sec.\n",
      "Epoch 1495: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.24 sec.\n",
      "Epoch 1496: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.22 sec.\n",
      "Epoch 1497: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.22 sec.\n",
      "Epoch 1498: T.cost 0.000000, Train acc 1.000000, test acc 0.684100, took 4.27 sec.\n",
      "New LR: 0.000470586771844\n",
      "Epoch 1499: T.cost 0.000000, Train acc 1.000000, test acc 0.684000, took 4.27 sec.\n",
      "\n",
      "Total time spent: 6429.5 seconds\n",
      "Traing Acc: 1.0\n",
      "Test Acc: 0.684\n",
      "\n"
     ]
    }
   ],
   "source": [
    "np.set_printoptions(formatter={'float': '{: 0.4f}'.format}, suppress=True)\n",
    "train_accs, test_accs = [], []\n",
    "total_time = 0\n",
    "param_outputs, disc_outputs = [], []\n",
    "disc_dist_t_1 = None\n",
    "quantized_bins = []\n",
    "try:\n",
    "    for n in range(NUM_EPOCHS):\n",
    "        start_time = time.time()\n",
    "        train_cost, train_acc, param_output, disc_output = train_epoch(data['X_train'], data['y_train'])\n",
    "        test_acc, test_transform = eval_epoch(data['X_test'], data['y_test'])\n",
    "        test_accs += [test_acc]\n",
    "        train_accs += [train_acc]\n",
    "\n",
    "        if DISC:\n",
    "            param_outputs = np.append(param_outputs, param_output)\n",
    "            disc_outputs = np.append(disc_outputs, disc_output)\n",
    "\n",
    "        if (n+1) % 20 == 0:\n",
    "            new_lr = sh_lr.get_value() * 0.99\n",
    "            print \"New LR:\", new_lr\n",
    "            sh_lr.set_value(lasagne.utils.floatX(new_lr))\n",
    "        \n",
    "        # Non-uniform Quantization\n",
    "        if DISC:\n",
    "            if n>0 and np.mod(n, 10) == 0:\n",
    "                dist = disc_output.reshape((-1, 6))\n",
    "                q_bins = find_quantization_bins(dist, sharedBins=sharedBins)\n",
    "                quantized_bins.append(q_bins)\n",
    "\n",
    "        time_spent = time.time() - start_time\n",
    "        total_time += time_spent\n",
    "        print \"Epoch {0}: T.cost {1:0.6f}, Train acc {2:0.6f}, test acc {3:0.6f}, took {4:.3} sec.\".format(\n",
    "                n, train_cost, train_acc, test_acc, time_spent)\n",
    "\n",
    "except KeyboardInterrupt:\n",
    "    pass\n",
    "print \"\\nTotal time spent: {0:.5} seconds\\nTraing Acc: {1}\\nTest Acc: {2}\\n\".format(total_time, train_acc, test_acc) \n",
    "\n",
    "if DISC:\n",
    "    story = {'train_accs': train_accs,\n",
    "             'test_accs': test_accs,\n",
    "             'epoch_reached': n, \n",
    "             'total_time': total_time,\n",
    "             'disc_enabled': DISC,\n",
    "             'learning_rate': LEARNING_RATE,\n",
    "             'batch_size': BATCH_SIZE,\n",
    "             'dense_params': param_output,\n",
    "             'disc_params': disc_output,\n",
    "             'quantized_bins': quantized_bins}\n",
    "else:\n",
    "    story = {'train_accs': train_accs,\n",
    "             'test_accs': test_accs,\n",
    "             'epoch_reached': n, \n",
    "             'total_time': total_time,\n",
    "             'disc_enabled': DISC,\n",
    "             'learning_rate': LEARNING_RATE,\n",
    "             'batch_size': BATCH_SIZE,\n",
    "             'disc_params': disc_output}   \n",
    "\n",
    "with open(TEST_NAME + '.model', 'wb') as fp:\n",
    "  pickle.dump(story, fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plot Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdUAAAHDCAYAAAByNoBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xd4VFX+x/H3SShJCBBI6EWKRECaRAUUEAsqrquIWIIs\nZWVRV1YFZdWf7toXLBRxrYuCiqKIoiKoiIIiEMAEkNB7DyS0QBop5/fHkDEhPUxJZj6v55knM/ee\ne+93Bs0n595zzxhrLSIiInLuArxdgIiIiK9QqIqIiLiIQlVERMRFFKoiIiIuolAVERFxEYWqiIiI\niyhURUREXEShKiIi4iIKVRERERdRqIqIiLiI10LVGHO/MWanMSbNGBNjjLmkhPbVjDEvGGN2GWPS\njTE7jDHDPFSuiIhIiap446DGmDuACcBIYCUwGvjeGBNprU0qYrPPgHrAcGA70Aj1tEVEpAIx3phQ\n3xgTA6yw1j545rUB9gJTrLUvFdL+euBjoJW19rhHixURESklj/f0jDFVgSjgx9xl1pHsC4EeRWz2\nZ+A34FFjzD5jzGZjzMvGmCC3FywiIlJK3jj9GwEEAofOWn4IuKCIbVoBvYB0oP+ZfbwJ1AXudk+Z\nIiIiZeOVa6rlEADkAIOstacAjDFjgM+MMX+31macvYExJhy4DtiFI4xFRMT/BAEtgO+ttUfcfTBv\nhGoSkA00OGt5AyChiG0OAvtzA/WMjYABmuIYuHS264CPzq1UERHxEXfhGJvjVh4PVWttpjEmFrga\n+BqcA5WuBqYUsdlSYKAxJsRam3pm2QU4eq/7ithmF8CMGTNo166di6r3rNGjRzNp0iRvl3FOKvt7\nUP3eV9nfQ2WvHyr3e9i4cSODBw+GM5ngbt46/TsRmH4mXHNvqQkBpgMYY8YBja21Q8+0/xh4Ephm\njHkax601LwHvFnbq94x0gHbt2tG1a1c3vQ33ql27dqWtPVdlfw+q3/sq+3uo7PWDb7wHPHQZ0Cuh\naq2dZYyJAJ7Fcdp3DXCdtTbxTJOGQLM87VOMMX2B14BVwBHgU+BfHi1cRESkGF4bqGStfQN4o4h1\nwwtZtgXHdVIREZEKSTMSiYiIuIhCtQKLjo72dgnnrLK/B9XvfZX9PVT2+sE33oOneGWaQk8wxnQF\nYmNjY33hAruIiJRDXFwcUVFRAFHW2jh3H089VRERERdRqIqIiLiIQlVERMRFFKoiIiIuolAVERFx\nkcryLTXiRXv27CEpKcnbZYiIn4uIiKB58+beLqNYClUp1p49e2jXrh2pqaklNxYRcaOQkBA2btxY\noYNVoSrFSkpKIjU1tVJ/24+IVH653zaTlJSkUJXKrzJ/24+IiKdooJKIiIiLKFRFRERcRKEqIiLi\nIgpVERERF1GoioiIuIhCVURExEUUqiIiIi6iUBUREXERhaqIlNuJEycICAggICCAKVOmuO04/fv3\nJyAgQBOQSIWnUBXxst27dzuD6Vwe3mSMqdT797TcPxLK8tizZ4+3y5ZS0DSFIhXAuYaGt0PHE8c3\nxnj9fbpKWd+Lr7xvf6BQFfGyJk2asG7duiLXd+jQAWMMF198MdOmTfNgZSWrXbs2OTk5bj/OnDlz\n3H4MT7PWYoxh+fLlhIaGlti+SZMmHqhKzpVCVcTLqlSpQvv27UtsV6NGjVK1k8qlbdu21KpVy9tl\niIvomqqIiIiLKFRFfMjZo2T37NnDmDFjaNeuHTVr1iQgIIDff//d2f7IkSP873//Izo6mnbt2hEa\nGkpQUBBNmjThxhtv5MMPPyQ7O7vI45U0+vfVV18lICCAwMBAkpOTycrK4rXXXqNbt26EhYVRs2ZN\noqKimDBhApmZmaV+XyXVsGTJEm699VaaNGlCUFAQ5513HnfffTc7d+4s8TM8efIkTz75JBdeeCE1\natSgfv36XHnllcycOROAr776ynm8vJ+lp51dR3Z2Nm+++Sa9e/emfv36BAYGMmbMGGf7Ll26EBAQ\nwIABAwCIj49n5MiRnH/++YSEhBAQEEBycnKB48yePZv+/fs7P8t69erRu3dvJk+eTHp6epH1nf1v\nn5aWxosvvsgll1xCeHi420eMe4tO/4r4kLwDYH766ScGDBjAyZMn863Pq3Xr1iQnJxdYnpCQwPz5\n85k/fz7vvPMOc+fOJSwsrNjjluTYsWNcf/31xMTE5Gu/evVqVq9ezbfffst3331HlSoFfy2VZmBP\n7vrx48fz5JNPYq11rtu3bx/Tpk3jiy++4Mcffyzy1pxt27Zx9dVXs3fvXuf+0tPT+eWXX/j555+Z\nO3cut912W6nfsycYY0hOTqZXr14FPtuz2+Wu+/jjj7n77rs5ffp0vvV5nTp1igEDBrBw4cJ8648e\nPcrSpUv59ddfee211/j222+JjIwstsa9e/cycOBANm/e7NxPRfn8XE2hKuJjrLUkJiZy++23ExgY\nyHPPPccVV1xB9erVWb16NXXr1s3X/oorrqBfv3507tyZ+vXrk5qays6dO5k2bRqLFy9m2bJlDB8+\n/JwHC911113ExsZy33330b9/f+rXr8/27dv5z3/+Q1xcHIsWLWLSpEmMHTu2yPdVklmzZrFs2TJ6\n9OjB/fffT9u2bTl16hSffvopb731FsnJyQwdOrTQgWFpaWlcf/31zkC97bbbGDJkCI0bN2bnzp28\n8cYbfPrpp6Xq7XraqFGjWLduHXfeeSd33XUXTZo04cCBA2RlZRVou2HDBu6++24iIiIYO3Ys3bp1\nA2D58uVUq1bN2W7gwIEsXLgQYwzdu3fngQceIDIyksOHD/Phhx8yc+ZMdu7cyVVXXcW6deuoU6dO\nkfXdddddbN++nXvvvZdbbrmFiIgIdu7cSXh4uOs/DG+z1vrkA+gK2NjYWCvlFxsba/U5epcxxgYE\nBNgrr7yyxLb9+/e3xhhrjLH16tWz27ZtK7Z9SesnT57sPH5cXFyB9cePH3euf/XVV4vc3hhjq1Sp\nYufNm1egzalTp2zLli2tMca2aNGi2Pd10UUXFVtDQECAHTRokM3JySnQbuzYsc52ixcvLrD+6aef\ndq5/7rnnCq1j6NChzvcTEBBg165dW2i7kuS+n4CAALt8+XIbHx9f7GPPnj0F9vHll1/me9+TJk0q\n9phdunRx1h4ZGWmTkpKKbDtjxgznvm+99dZCP89XXnnF2ebee+8tsD7vv31AQID94osvSvHJFK28\nv4tytwO6Wg9kj66pivggYwxPP/00rVu3LrZdSesffPBBWrZsCcCXX355TvUMHz6cG264ocC6GjVq\ncO+99wKOa8D79u0r93HCwsJ45513Cj21+PDDDzufL1myJN86ay3/+9//MMZwwQUX8MQTTxS6/ylT\nphR7Grw8LrvsMjp27Fjs48EHHyx2H1FRUTz00EOlOp4xhkmTJhXbS3zjjTcACA0NLfbzvOSSS7DW\n8uGHH+a7zHD28W699VZuueWWUtVX2en0r3hMaips2uTtKorXti2EhHi7CteIjo4u8zYJCQkkJyfn\nGzTUtGlTdu3axdq1a8+pnkGDBhW5Lioqyvl8x44dNG3atFzH+POf/0yNGjUKXdegQQPnadEdO3bk\nW7dx40YOHDiAMYZBgwYVeb2vVq1a3Hzzzbz//vvlqq8wpbm2WFKb4j7bs4WFhdGvX78i1586dYoV\nK1ZgjOHmm28ucLkgrxEjRrBq1SrS0tJYtmwZ11133TnXV9kpVMVjNm2CPL87K6TYWPCF6WWbNGlS\n7DWuvGbPns3UqVNZunQpKSkphbYxxpCUlHRONbVt27bIdXl/cRfV4znXY+Qe58CBAwWOER8f73we\nVcJ/pBdffLFLQ/X48ePUrFnznPbRqVOnUrUzxnDhhRcWG9IbN24kJycHY4zzemtR8q6Pj48vMlRL\nW58vUKiKx7Rt6witiqyE38mVRmkCNTs7m+joaGbPng0UP8LWWktaWto51RRSzCmAvHMXF3cLz7kc\nI+9xzj7GsWPHnM/r1atX7D5KWl9WthQDsEpS2j+gStP26NGjzuf169cvtm3Dhg0L3a6sx/QlClXx\nmJAQ3+gFVgaBgYEltpk8eTKzZ892ju78xz/+wSWXXEKjRo3yhdPNN9/M3LlzXfLLX9yjNP/e5Wnr\nqtteynLMyk6hKuKnpk6dijGGLl26sGTJkiK/6aa4HoivyNuTSkxMLLZtSesru7yn4g8dOlRs24SE\nhEK382ca/Svih7Kzs9myZQsAAwYMKDJQs7Ky+P333332Rv1cF154ofN5bAnXKH777Td3l+NV7dq1\nc/73sGLFimLbrly50vm8Q4cObq2rslCoivih7Oxs5+ncogYnAcyYMeOcBg5VFu3bt6dRo0aAY7ah\nok51Jycn89VXX3myNI8LDQ2lW7duWGv5+uuv811vPtu7774LQHBwMJdddpmnSqzQFKoifqhatWo0\na9YMay2zZ88mNTW1QJv4+HgeeeQRjDE+fz3VGMOIESOw1rJ582aef/75QtuNGjWK48ePe7g6z7v/\n/vsBx0jse+65p9B//4kTJzpvvRkyZMg5j2D2FbqmKuKnhgwZwgsvvMC2bdvo3r07Y8eOpV27dqSk\npPD999/z2muvERgYSMeOHb06cbynPProo8yYMYOdO3fy1FNPER8fz7Bhw2jYsCG7du3iv//9L4sX\nL6Zbt27O06KuOC2+YcOGUgVS06ZNqV279jkfrzQGDRrEBx98wIIFC5g9ezY9e/bkwQcfpE2bNs5p\nCj/++GMAGjduzAsvvOCRuioDhapIJeCOnuITTzzhnNt3/fr1DB06NN/62rVr88knn/DWW2/5VKgW\n9VmGhITw3Xffcc0117Bv3z4+++wzPvvsM+d6YwwDBw7kjjvuYODAgQAEBQWdcy2lPW06ffp0hgwZ\ncs7HK63Zs2czcOBAfvjhB2JiYli+fHm+9cYYWrVqxfz58/3qlpmS6PSvSAWXe/9oaXtFpW0bFBTE\nokWLeOmll+jSpQshISGEhoZywQUX8NBDD7FmzRrnzfwl7bOkdaWdNehc9uOKz6dNmzbEx8fzf//3\nf7Rr147g4GDCw8Pp1asX06ZN49NPP813jflceo55/11LehQ1kKws/12UtX1oaCjfffcds2bN4s9/\n/jONGjWiWrVqhIeH07NnTyZOnEh8fDxt2rRxyfF8hfHVayXGmK5AbGxsbJFf8yQli4uLIyoqCn2O\nIg5jx45lwoQJ1K5du9hBPOJa5f1dlLsdEGWtjXNbgWeopyoiUkrZ2dnOCTMuvfRSb5cjFZBCVUTk\njJ07dxZ73XH06NHs3r0bgGHDhnmoKqlMNFBJROSMKVOm8M033xAdHc1ll11Gw4YNycjIID4+nvfe\ne4/ly5c7J5q/8847vV2uVEAKVRGRPHbs2FHofaq5g266du3KnDlz/G4AjpSOQlVE5IwHH3yQpk2b\nsmDBAnbs2EFiYiIZGRmEh4dz0UUXcdtttzF48OAiR+OKKFRFRM5o0aIFDz/8MA8//LC3S5FKSn9u\niYiIuIhCVURExEUUqiIiIi6iUBUREXERhaqIiIiLeC1UjTH3G2N2GmPSjDExxphLiml7hTEm56xH\ntjGmvidrFhERKY5XQtUYcwcwAXgKuAhYC3xvjIkoZjMLtAEannk0stYednetIiIipeWtnupo4G1r\n7QfW2k3AvUAq8NcStku01h7Ofbi9ShERkTLweKgaY6oCUcCPucusYwbrhUCP4jYF1hhjDhhjFhhj\nSvfNviIiIh7ijZ5qBBAIHDpr+SEcp3ULcxC4B7gVGADsBRYbY7q4q0gREZGyqhTTFFprtwBb8iyK\nMca0xnEaeah3qhIREcnPG6GaBGQDDc5a3gBIKMN+VgKXl9Ro9OjR1K5dO9+y6OhooqOjy3AoERGp\n6GbOnMnMmTPzLTtx4oRHa/B4qFprM40xscDVwNcAxvEdSlcDU8qwqy44TgsXa9KkSXTt2rU8pYqI\nSCVSWIcpLi6OqKgoj9XgrdO/E4HpZ8J1JY7TuCHAdABjzDigsbV26JnXDwI7gfVAEPA34Eqgr8cr\nFxERKYJXbqmx1s4CHgGeBVYDnYDrrLWJZ5o0BJrl2aQajvtafwcWAx2Bq621iz1Usojb7N69m4CA\ngHN++Jq1a9eW+TP4619LuitPxL289n+itfYNa20La22wtbaHtfa3POuGW2uvyvP6ZWttG2ttDWtt\nPWvt1dbaX7xTuYjrGWPO6VERQvXEiRPOcJsypSxXcopX1s9CxJsqxehfEV/WpEkT1q1bV+T6Dh06\nYIzh4osvZtq0aR6srHzcEWyDBw/m0UcfLbFdnTp1XH5skbJQqIp4WZUqVWjfvn2J7WrUqFGqdr4o\nPDzcb9+7VC7eP2ckIj7BMTGaiH9TqIr4qE8//ZQBAwbQrFkzgoKCCA8Pp0ePHrz44oukpKQUu+36\n9eu57777aN++PTVr1iQoKIimTZvStWtX7rnnHubMmUNOTo6zfVhYGHXr1sUYg7WWhx56qMAgojFj\nxrj7LReqS5cuBAQEMGDAAADi4+MZOXIk559/PiEhIQQEBJCcnAzAV1995az3999/Jzs7mzfffJPe\nvXtTv359AgMDC30fBw8e5J///CedO3emdu3ahISE0Lp1a+6++25iY2OLrS8sLCzf57Ns2TIGDRpE\nixYtCAoK0intSkanf0V8zOHDh7n55ptZsWJFvuubx48fZ+XKlaxYsYI33niDb775ho4dOxbY/t13\n3+W+++4jKysr3/YHDx7k4MGDrFmzhqlTp7J3714aN24MUGCQUEUaMJS3to8//pi7776b06dP51tf\n2DbJycn06tWLmJiYYt/PnDlz+Mtf/kJqamq+drt27WLatGlMnz6dxx57jBdeeKHE+l5++WUef/zx\nfL3+kJCQsr1h8SqFqogPSUlJoXfv3mzZsoVq1aoxfPhwrr76alq2bElaWho//fQTkydPZu/evfTr\n14/Vq1dTr1495/Y7d+7k/vvvJzs7m2bNmjFq1CiioqIIDw8nNTWVLVu2sHjxYr7++ut8x42JieH4\n8eP06NEDYwyPPvoogwcPztcmIqK4b3Z0vw0bNnD33XcTERHB2LFj6datGwDLly+nWrVqBdqPGjWK\ndevWceedd3LXXXfRpEkTDhw4QFZWlrPNkiVLuP3228nJySE4OJjRo0dz/fXXExQUxMqVKxk3bhwH\nDhxg/Pjx1KpVq9jBVgsXLmT9+vW0bduWMWPG0LlzZzIyMli5cqXrPwxxH2utTz6AroCNjY21Un6x\nsbFWn6N3GWNsQECAvfLKK0ts+/e//90aY2yjRo3s5s2bC22zefNmGx4ebgMCAuyoUaPyrZs4caI1\nxtiqVavaPXv2FHmclJQUm5mZmW/Z8ePHnbW++uqrpXhnxVuzZo1zf0OGDLHx8fElPtLT0wvsp0uX\nLtYYY40xNjIy0iYlJRV5zC+//NJ5zICAADtp0qRia4yMjLTGGBscHGxjYmIKrD98+LBt1aqVNcbY\n6tWrF/qZhoWF2YCAAGuMsZdffrlNTU0txafjf8r7uyh3O6Cr9UD26JqqiI9ISkpi2rRpGGOYMGEC\nkZGRhbaLjIzkkUcewVrLhx9+mO9UY0KCY/rt5s2b06xZs0K3B8cpySpVPHei68MPP6Rjx44lPjZv\n3lzkPowxTJo0ifDw8FIdMyoqioceeqjI9T/88ANbt27FGMOYMWOcPd+86tWrx2uvvQZAZmYm77zz\nTqH7stZijOGdd94hODi4VPVJxaTTv+IxqZmpbEra5O0yitU2oi0hVSvnNawFCxaQnp5OYGAg/fv3\nL7Zt7969ATh58iTr16+nQ4cOADRq1AhwzPK0aNEirrzySvcWXUqluUZbUpuwsDD69etX6mMOGjSo\n2PULFy50Pi9uJqcbbriBxo0bc/DgQRYuXMhzzz1XoI0xho4dO+q2IR+gUBWP2ZS0iah3PDexdXnE\njoyla6PK+QUMv/3mmJQsJyeHGjVqlHq7hIQEZ6jedtttPPnkk6SmptK3b1/69u3LjTfeSK9evejY\nsaPXBiA9+OCDTJw4sdzbG2O48MILy1R/p06dil0fHx8PQN26dWnVqlWxbS+99FK+/PJL5zblOZ5U\nDgpV8Zi2EW2JHVn87QXe1jairbdLKLfDhw87n5clPFJTU53PmzRpwpw5cxgyZAiHDx9mwYIFfP/9\n94Cjp3fttdcyYsQIrrnmGtcV7iFlvTWlpPZHjx4FoH79+iXuq2HDhoDjs87MzKRq1arnXJ9UTApV\n8ZiQqiGVthdYGWRnZwNQvXp1YmNjSz0Zw3nnnZfvdd++fdmxYwefffYZ3377LUuWLOHgwYOcOHGC\nWbNmMWvWLG699VZmzpzp0euq5yowMNAt7V3Vey9rfVIxVZ7/I0SkWLkDcDIyMjjvvPPO6f7G4OBg\nhgwZwpAhQwDYvn07c+fO5bXXXmPXrl188cUXjBs3jn/9618uqb0yqlu3LgCHDh0qsW3uALCQkJBC\ne6niOzT6V8RHXHTRRc7nS5cudem+W7duzUMPPcSKFSucYTJr1qx8bSrShA+ekHsd+ujRo+zYsaPY\ntqtWrcIY49xGfJdCVcRH9OvXz9kLevXVV91yjIiICDp16oS1lqSkpHzrgoKCnM8zMjLccvyKJO91\n5ffee6/Idt9++y379+8vsI34JoWqiI9o3Lgxw4cPx1rLt99+yzPPPFNs+wMHDvDBBx/kWzZv3jyO\nHDlS5DaHDx9m7dq1GGNo2bJlvnXVqlVzDrbZvn17Od9F5dG3b18iIyOx1jJp0iRWrVpVoM3hw4d5\n4IEHAKhatSojR470dJniYbqmKuJDJkyYwPLly4mPj+eZZ57hu+++Y/jw4XTs2JHg4GCOHTvGunXr\n+O6771i4cCF9+vRxXjcFmDp1KgMHDuT666+nb9++tG/fnrCwME6cOMHatWuZMmUKR48exRjDfffd\nV+D4l112GfPmzWPmzJn06NGDSy65hOrVqwOO0cOlnXjhbEeOHGH9+vUltqtWrRpt2rQp1zHKY+rU\nqVx55ZWkpaXRp08fRo8eTb9+/ahevTorVqxg/Pjx7N+/H2MMzz77bLETaoiP8MS0Td54oGkKXULT\nFHpf7hR7pZmm0Fprjx07Zvv16+ecai93+7yP3HUDBw7Mt23//v1L3C4wMNA+/vjjhR57yZIltmrV\nqoVuP3r06DK979xpCsvyaNmyZYH95E5TeMstt5R4zLzTFK5du7ZUdc6ZM8eGhoYW+p5zP68nnnii\nyO1zpyks6+fjbyrLNIXqqYpUcLkDgEo7ECgsLIz58+ezaNEiZsyYwa+//kpCQgLp6emEhYXRunVr\nunfvzo033shVV12Vb9upU6cyb948Fi1aRHx8PAkJCSQmJlK1alWaN29Oz549GTlyJFFRhU/i0bNn\nT5YsWcLEiROJiYnh8OHDzm+EKc9AprJuU1T7s79Fp6R9lOW4/fv3Z+vWrUyaNInvvvuOXbt2kZWV\nRaNGjejTpw9///vf6dpVt5L5C2NLeS9bZWOM6QrExsbG6j/ocxAXF0dUVBT6HEXEm8r7uyh3OyDK\nWhvntgLP0EAlERERF1GoioiIuIhCVURExEUUqiIiIi6iUBUREXERhaqIiIiLKFRFRERcRKEqIiLi\nIgpVERERF1GoioiIuIhCVURExEUUqiIiIi6iUBUREXERhaqIiIiLKFRFRERcRF9SLqWyceNGb5cg\nIn6ssvwOUqhKsSIiIggJCWHw4MHeLkVE/FxISAgRERHeLqNYClUpVvPmzdm4cSNJSUneLkVE/FxE\nRATNmzf3dhnFUqhKiZo3b17h/0MWEakINFBJRETERRSqIiIiLqJQFRERcRGFqoiIiIsoVEVERFxE\noSoiIuIiClUREREXUaiKiIi4iEJVRETERRSqIiIiLqJQFRERcRGFqoiIiIsoVEVERFxEoSoiIuIi\nXgtVY8z9xpidxpg0Y0yMMeaSUm53uTEm0xgT5+4aRUREysIroWqMuQOYADwFXASsBb43xhT7le7G\nmNrA+8BCtxcpIiJSRt7qqY4G3rbWfmCt3QTcC6QCfy1hu7eAj4AYN9cnIiJSZh4PVWNMVSAK+DF3\nmbXW4uh99ihmu+FAS+AZd9coIiJSHlW8cMwIIBA4dNbyQ8AFhW1gjGkD/Afoaa3NMca4t0IREZFy\nqPCjf40xAThO+T5lrd2eu9iLJYmIiBTKGz3VJCAbaHDW8gZAQiHtawIXA12MMa+fWRYAGGPMaeBa\na+3iog42evRoateunW9ZdHQ00dHR5ateREQqpJkzZzJz5sx8y06cOOHRGozjcqZnGWNigBXW2gfP\nvDbAHmCKtfbls9oaoN1Zu7gfuBK4FdhlrU0r5BhdgdjY2Fi6du3qhnchIiIVXVxcHFFRUQBR1lq3\n34rpjZ4qwERgujEmFliJYzRwCDAdwBgzDmhsrR16ZhDThrwbG2MOA+nW2o0erVpERKQYXglVa+2s\nM/ekPovjtO8a4DprbeKZJg2BZt6oTUREpLy81VPFWvsG8EYR64aXsO0z6NYaERGpYCr86N9z5Y1r\nxiIi4p98PlSzcrK8XYKIiPgJnw/V09mnvV2CiIj4CZ8PVfVURUTEUxSqIiIiLuLzoZqZrVAVERHP\n8P1QzVKoioiIZ/h8qGYoVEVExEN8PlR1+ldERDxFoSoiIuIiPh+qOv0rIiKe4vOhqp6qiIh4is+H\n6ulMhaqIiHiG74eqTv+KiIiH+HyoZmRlersEERHxEz4fqikZqd4uQURE/ITPh2ry6WRvlyAiIn7C\n50P1pEJVREQ8xA9C9aS3SxARET/h86GakZXh7RJERMRP+H6oZitURUTEM3w+VE8rVEVExEN8PlQz\nstK9XYKIiPgJnw/V0znqqYqIiGf4fKjmvaa65cgWGk1oxNG0o16sSEREfJXPh2pmnp7qJ/GfkHAq\ngeV7l3uxIhER8VU+H6oZOX9cU61RtQYAqZmaulBERFzP50M1b081pGoIACmZKc5l2TnZJKUmebwu\nERHxPX4VqsYYANIy05zLHlnwCPVerseeE3s8XpuIiPgW3w9V+0eo5oZp3qkL52yaA8B5k8+jwxsd\nGPT5IM9pbvFwAAAgAElEQVQWKCIiPsP3QzVPTzX3WurJjD9CNe+p4PWJ65kZP9P52lrLjN9nkJWj\nLzoXqezSMtM0bam4ne+Hqk3HWgv80UNNzvjjm2tSTqcUuh3ADzt+4C9z/sJ7q99zb5Ei4nYh/wnh\nwjcu9HYZ4uN8PlQBTmefxlrLi0tfBOBI2pF864ry8bqPAU3KL+Irth/b7u0SxMf5RaimZaWRmJro\nfL3z+E7nc4stcrv3174PwLrD64rd/5qENZhnDFuPbD3HSkVEpDLzi1Ct/3J9GrzSwPl62d5lvBv3\nLuYZQ47NKXSbT+I/cT7/X9z/nM+Ppx9n/K/jyc7J5vGFjzNvyzx+2f0LACv3ryQ7J7vIOqy1RR4P\nHL3m1lNaM3/rfN6Ne7fU709ERCoGvwjVzJxM5/PbL7wdgBFzRxTZPjsnmy83fVlg+eakzQyZM4TH\nf3ycwXMGM37peG6ceSOBJhCAwXMGU+W5KiScSiiw7anTp7jts9sIfDaQ6Wumk3I6hfG/jmfl/pVY\na2n5akveX/M+O47t4E8f/4kRc0eQmJJYYD/FOZlxkvRK9AUC249ud55iL42DJw8Wew1cRMTb/CJU\n8+rRtEeJbRbuWMin6z8tsLzt622Zu2UukL8nO2XllHztFu1cxM2f3OwcNfxu3LvUHFeTzzd+DsDw\nr4bzyrJXePzHx+k2tRvpWensOr6Lkd+MzLeffcn7nM83JW2i+aTmnEg/ka9Nckayc+BVrfG16Ple\nzxLfX1kknEpw2+QYvaf35q4v7ip1+8YTG3P1B1e7pRYREVfwu1A9lnasxDajvh0FQPPazQEwGNYd\nKvq66pYjW/K9HrNgDF9v/pqfd/3MruO7iu0VA1zw3wsKXb5s7zLn8w/WfsDe5L38duA3jqUdcw6w\nqvtiXWqPr81LS18CIPZgbAnvrmhrE9by9eavSTiVwP9iHae8G01oRMNXGpZq+0OnDnHq9KkCy9cd\nWlfogLDcnvjOYzsLnDZfm7CWF355ocA2K/avyPfHhkhldTLjJCv3r/R2GeJifheqLeu0LLHNtqPb\nALgg/AL+3fvfWCyd3upU6mPknv695sNraPlq4cd7+uennc/3Ju8ttM2SPUucz0OrhQKO+2rrvlSX\n6s9XJysni2zrCKNHFz7qbFuaPxzyijsYR3pWOl3e7sLNn9zMiK9HMPKbkc5Tydk2m+V7l/NO7DvF\n7qfhhIa0ntLaeR/w1iNbGfT5IDq91Yl//vDPAu1ze/KtprTiiulX5FvX98O+PLnoSQDeW/1evm8W\najapWb62n8R/woPfPlim91zRZedkE/laJEt2Lym5MfBqzKuYZ4ybqxJXGvbVMLpN7VZg+dS4qTy+\n8HEvVCSu4FehunjoYoZ2HsrNF9ycb/nIriMLbV83uC5VAqqUev+3tb/tnOo726frP+XU6VM8/P3D\nPPHTEwDc/MkfteedxCKvx3/843/I9Kx01ias5Z3Ydzh1+lSB6705Noeod6IY8fUfvencXmXe076X\nvXcZ93xzT6HHO3X6FJOWTwLgcMphbvrkJsARjLmTacTsi2Fj4kY2JG5wbpd35PXSvUuJPRDL/K3z\ngT+ug59IP8HdX99d5LEBoj+PZsrKKfSZ3oc3V72Z77399au/sjFxY6HbbU7aTINXGnAk9Uih670p\nJTOFrUe38szPz5Sq/dTVUwF49IdHS2gpFUXuH++Z2Zn5lv9t7t8Yv3S8N0oSF/CrUL2ixRUYY+ja\nqGu+5a//6XV6Ne9VoP0r175CgPnjI+rWpOBflXl9fGvpB90AXNPqmhLbdJ/anYkxEws/XhGDfLJz\nsvls/We8/dvbBL8QTJe3u3DPN/fQfWp3Gk1oxKr9q5xtc6du/GjdR85lwVWDAYh6J6rQ/Vtrsdby\nw/YfMM8YRnw9gjELxjjXL961mE/iP2H3id3OZSv2r6D9G+258I0LWbJ7SaGjoC/+38X86eM/sfv4\nbo6nHwccvWj44xfQ2XYc2+F8/vPun/n7/L87X29I3MC0NdN47MfHePu3t53LY/bFsHzvcr7c9CWH\nUw6zcMfCYkdte0NuPXkH2RUnd4KTl5a95LaayuOity/i5aUve7uMCql6YHVA35rla/wmVA89csj5\nvFb1WvnWVQmowvy75rPx/o3MuGUGAK9e/ypNazXloe4PseaeNdinLDEjYpzbRIRE8MuwX5yv373p\n3Xy92twRwbl6n9ebX4b9Qrcm3Uh+LJmcf+fwcI+HnevHXjbW+fzRyx9l86jNgGPqxKLkXvs9W3p2\nOrfPvp17592bb3nuvi6dein/mP8PkjOSC/0fOve66OGUwwXW7Tq+i4BnAwh4NoBrZ1wLUOigrujP\no4usu/f03s5rtoVp8WoL5/OrPrgKoMjeZOsprQss25y0mddXvs6u47sA+Hrz19w7715+3vUzAD3e\n7cFl713mrPvOz++kynNVnMFUEeSGaXGTkxTWHij2ti1PW5Owhn8uLHjqX6B6FUeopmSmkJmdyY87\nfmRq3FQvVyXnqvTnNiu54CrBzueF/fIMrRZK24i2tI1oy12d/hiRWqNaDTo37JyvbYuwFux80DGB\nxMt9X+bSJpfS+7zeAHx717c0Cm3EqgOrGDV/FBnZjtmYfh7m+IWeN5jb12sPwLxB87iu9XW8vMzx\nF/2wLsOIDI/k2tbXsmD7gjK/1xm/zyixzX9X/ZeNSRsL7QHm7cmerc/0PmWupzBnB35Jzr7u3GRi\nEw6cPFBo2xeXvsi0NdPo3CD/v1uf9/sQERLhfL06YXW+9d9t+45+bfqVqS53yT0lGLMvhqNpR6kb\nXJe0zDT+tehfDOsyjOlrpjPu6nEkpiay5ciWfIPl0jLTqFGthrdKl1LK7ammnE5h1PxRzi/3yJWR\nlUGOzSG4ajDWWue3bEnF5jehGlQlyPk8PCQcgPFXj+fvl/y9qE0KlfBwgvP0KMAjlz2Sb/31518P\nQOeGnRnRdQSr9q8qsufQvHZz7FN/BPyOB3YwZsEYWtVpBcBnt31G7fG1y1RfWfy488dCl+f9Fh+A\nIZ2H8MHaDwC8+uUC9ULqOWfGKipQAfaf3A/A2kNrC6wr7vagGz6+Id+/hzfl7XmuSVjD4ZTDpJxO\nYcLyCUxYPgHA+fNs32z5hjs63OGROiujJ396kuevet7bZTh7qjd9chObkjYVWB/0guN3Vq/mvViy\nZwm/DPuFXucVvEwlFYupSKe8XMkY0xWIZSTQmHy/LHNsDvuT99OsdrMit68oth3dRkZWBh3e7OBc\n9tltn/H15q/58PcPXX68sKCwP65njoxj7pa5/PuKf7N873JeXPoiX23+ytm2e9PuxOyLKWpXALx9\n49v0adGnyNuGyuL8uucXeW3VVTbdv4kLIs691nO15ciWc/rMKsofB7kjktOeSMv3h603awEIqRrC\ngTEHqB3kvj9aS3LH7DuYtX5Wqdv3bdWXBX8p+5krV8rMzmT3id2cX/d8r9ZRFnFxcURFRQFEWWvj\n3H08n7+mGmCrF1xmAipFoIIjSC6s/8c3a9inLAPbD8x3re3gwwcL3fa7u75j3X3rGH/1eDo1yH9L\nUIMajmkbm9ZqCsDKESt5steTDO442NnmokYX8e8r/g1Aj2Y9OK/2efn2kTu4a1DHQUy7eRr3RN3D\nvEHzePGaF/km+hvsU5aRUSNpVsvxWd/Q5gYAHrv8sVK997ODYeatM4to6TrL9y13+zFK4+wRoWVV\n0QZeFXZ93ptSM1PpNrWb8zq7N4QHh5ep/Q87fqDLW12YuHwik2Mms/PYzpI3crGnFz9Nm9faOAc4\nSkE+H6p9Euaw7R/u7d14QsPQ/BMwjL1sLNe2vpYj/zxCWFCYc3ne3kCrOq3oUL8Dj/Z8lLX3rs0X\nUv3b9gccA7KO/vMolzS5hOeues556rdOUJ0CNTzZ+0nn86taXsUjlz2Cfcry0YCPGNZlGG/d+BY3\ntLmBf17+T/4U+Sdn2+CqwdinLJc3uxygwOjrs426ZBTfRH9TYPnFjS/GPmWZfN3kfMs/GvBRgbZl\nuRUKHLdPGUy+0cTeVNpRv0WpOa4mE5dPZOa6mazcv5KsnCw2Jm5k/K/jmbdlXpG3Y7lLWb9sIjUz\nlePpx106eKxW9Vpc1PAi5+uEUwn0eb8PbV5rQ/9P+vPvRf8mZl8MK/atIDkj2TnK3V3yjvMoSas6\nrRh39ThqB9Xm4QUPM/r70bSa0oqub3fl0R8e5atNX3nkD5dNRxynqe/8/E63H6uy8vlrqlUyGtC6\nbsERopXNb3/7Ld8v/KjGUXw/+PsC7W6MvJHZG2YDRU900aNpD2fohFYLpU7wHwGaO+Vh7MiCMzPV\nq1GPNfesITE1sVS3A52takBVwHG9+aMBHzF/63xi9sXk+zquudFzuTHyRufrBy59gNUJq/NNL5l7\nLeq9m95j+EXDAVi6ZynrE9fz/FXP02taL25tdyuTr59MnaA6PLX4KeoG12XxrsV8u+3bQmvbN3of\n1824joU7FjK8y3C2Ht1KZHik817lVftXUb9GfdrVa1fm910euT3Vvq36snjX4hJDdvU9q6lfoz4/\n7fyJv8z5C2lZaTy84I/R5QEmIN+1/aoBVenTog8XN76YEV1HOK/ju1LeQLrh4xvo1KATLcNacn7d\n87HWElw1mKV7l7LlyBYahjYkMjwScNx/PX/rfDKyMwitFsoNbW6ga8OujL18bL5b3MpjUMdBxI6M\nxRhDdk42s9bP4vklz/PV5q/4avNXPPfLc8621QOrY7Fc1/o6mtRsQo1qNbgx8kb6tOhzTjXkyp24\nJZd9ypKdk827q98tcF/29gcc/4881vMxMrIyOHX6FNPXTOfn3T/z6opXnbdSta/XnsjwSG6KvIkd\nx3Zwbetr6dm8p8sGOUUEOwb6fb35a06kn/Dq6fOKyuevqV55ZSw//VR8z8gX5F4vWjJ8CSmnU7ju\n/OsKbZeYkkiNajX46PePGPnNSJb9dRk9mv0RWB+s/YChXw51yzWw9Kx01iSsoXvT7s5le07sYfGu\nxQz9cii9mvfil+G/FLMHh98P/U7ntzqz5p41BUZmb07aTNvX2/LZbZ8xsP3AAtt+tv4zbp99u/P1\nS9e8ROeGnbm29bWMnDsy3zcSFWbsZWMZ1HEQ7eu1JzElkQahDcrcKy6NZXuXcfl7lxN/XzwX1r/Q\n+e/7wlUvOCcCubzZ5QzqOIj7Lr7P+Uvz0KlDNJzQkGFdhjH+6vFsO7qN7ce2k5aZhsVy0wU3kZSa\nxE87f+LzjZ/z655fARjaeSiDOw0u1x9LRcnKyaLqc1WZdN0kNiRu4OTpk6xNWMuBkweoWb0m6Vnp\nRIZHEhkeSdzBOHYd38X5dc8ntFoovZr3omVYS37e/TOrDqxiU9ImWoS14Ja2tzC402Da12tf5v8+\nQ/8TynNXPsfoHqMLXZ+RlcHPu38mKyeLncd2kpmTyb7kfaw6sIrElET2n9xPckYyjWs2JuV0Cj2b\n96RucF0Opxymaa2mRIZHcmu7W6lVvRb1atQrsZ5/zP8Hi3Ytct7qlvdM0o87fuSaDx3/FgZDzlNF\n3yZlrSX+cDzfbPmGuIQ4Vu5fyZ4Tewg0gWTbbGpUrUGPZj24tPGl9DqvF5c3u5ya1WuW5aNzeuLH\nJ/jPr/8BHH+odW7QmeSMZDJzMmlaqylZOVmEBYVRL6QeESER1K9Rn2a1mpGRncGXm77kwMkDNAht\nQLcm3TiRfoKwoDC6Ne1GpwadaFyzcblqKomnr6mWKVSNMYHAf6y1FX7altxQ7dUrll9+8f1Q3ZS0\niQGfDmDV31aV6nYKay0x+2Lo3rR7gb9iPT18PzM7kz7v9+HV61/l4sYXn/P+Dp48SKOajYpc//6a\n9wmtFsprK1/jizu+oG5wXcDRS3/4+4dpVLMR7eu1J9AEkp6Vzraj27im1TUs2bOEpxc/na/XGB4c\nzk0X3MRjPR9z9rRc4eddP9Pn/T5sHrWZyPBIXln2CokpibzY90WycrKKDfJ5W+bRo1kP5/sqzsGT\nB3kn9h1eXPoiaVlpjOk+hsd6Pkad4Drn/MdCWmYaIf8J4cNbPmRwp8Elb1CMWetnMSlmEusPr+fk\n6ZNUC6zGhfUupGWdlnRv0p2QqiH0a9Ov2B538AvBvHjNizzQ7YFy1XA6+zSfrf+M1Qmr2X1iN7M3\nzKZxzca0qtOKkxknnaPNDYYb2tzA6ze8znlh5xW5v/u+uY+VB1ZyJPUIr1z7Sr4/AudtmceNM29k\ndPfRPNHrCecdC6VhrSU1M5VqgdX4btt3/C/uf6w9tJY9J/YAULNaTaI7RNO+XntubX+rc1xFaTyz\n+BnnFKt/avMnqgRUoWVYS06ePsnuE7sJrRZKUmoSSalJJGckk5aZxrF0x7Spbeq24dIml3Is/Rg/\n7/qZlMw/vnHKFf+NFKVChyqAMWaltfZSN9XjMrmh2r17LMuX+36oimecOn2KeVvmsSFxA+3rtef3\nQ7/zwe8fsC95Hxc1vIjgqsE8fcXT9G3dF3D01nJ7YGXxj/n/4L+r/suOB3aUar7qc2WtZcLyCfxr\n0b9Iz0qndZ3WPNjNMZ9ynxZ96NigY5n3mZyRTO3xtfnk1k9cdotPZnYmC7Yv4MedP/L99u+pFliN\ndYfWkW2zCTSB3NDmBuqF1GNE1xHO+zvPCzuPusF1qfZcNSZdN4n7L73/nOuw1nIo5VC+sQ7H04/z\n086f2J+8n3G/jiM5I5m/dPoLr//pday1BAbknxDmb1//jd8P/86KESsK7D8jK4NHFz7Ks1c+W2Cy\nmvLIzsnmePpxjqYdZdyv45i2ZhrguPwT1SiK9Kx0Hu7xMDe0uaHYP8ofX/g445eOZ9XfVpXqD2Br\nLVuPbqVqQNV8/x1nZGUQGBBIWmYaJzJOUCeojtvura4Mofo0kAlMA5xfSWKtTXZpZecoN1SjomL5\n7TeFqrhPelY6/1nyH77Z8o1zQonzap9HRnaGc67l29rfxoRrJxQ66jwxJZGR34wk7mAcreu0ZtGu\nRc51Jx8/6fwyBU/YlLSJcb+OY/7W+fnu6b0g/AIahDbAYGhdpzVxCXHUr1Gf0GqhNKjRgFZ1WtGx\nfkfahLehVZ1WJGckczz9OOdNPo/Pb/+cAe0GuK3mzOxMdhzbwewNs/l84+dsSNzgnHQFHD3HsZeN\n5aVlL/Hmn97k3ovLNvFIeew9sZdbPr2F2IOxztOwZ1+uGP7VcLYc2cLSvy51ez1nO519mqTUJB5Z\n8AiLdy3m4CnHHQSNazbm+8Hf06F+h0K3G/P9GL7d9i0b7y98Pu2KqDKEamEn9621NrCQ5V6TG6qd\nOsWydq1CVTzDWsvcLXP5YuMXzlsmYg/GsvbQWo6nH6d29dq0rtuaTg06cfDkQZJSk4r8ur7/6/l/\nvHB1wa+/85TcEbBzNs1hcsxkmtduTkpmCj/t/AlwjKbNHdiWV42qNfKd2pt28zSGdRnmqbI5dfoU\nk2McI8SDqwTz5KInnd+49OUdX3Jz25uL29xlcmwOd31xF3M3zyUlM4WQqiHcd/F9XNz4Yno178Vj\nPz7G7uO7SzWOwN2ycrJ4c9WbPPDdH6fGG9RowKhLR7EveR8D2g2gcc3GdHzTccaiotwHXRoVPlQr\ni9xQbdculg0bFKriXUdSj/B/P/4fiamJJGckF5jN6oleT/D8Vc+zP3k/u0/spmFoQ7eMyHWF5Ixk\nTmacpHHNxs5r7/uS9/Ht1m95/MfHOZKWf57mGbfMyDf1p6elnE7BYok7GEev5r28Mt3f7uO7af9G\n+wJzbV/e7HJ+/euvHq+nKNuObuODtR8wa/0sNh/ZXGQ7hWrRyhWqxphmQO58WT9ba/eXYx/3A48A\nDYG1wD+stYVOOmuMuRx4EWgLhAC7gbettZMLa39mm65A7Pnnx7J1q0JVKpZtR7fxy+5fGNZl2Dnf\nJlLRJGckUyWgCiczTnIo5RAd6nfwufdYHgmnEnhl2SvEH44nIiTC+c1QFTWgrLUs3rUYYwxfbPyC\n09mniQyP5NIml9KzeU9vl1dqng7VMg/vM8bcDLwL/ApYYLIx5m5r7dwy7OMOYAIwElgJjAa+N8ZE\nWmsLm5w1BXgN+P3M857AO8aYU9baYr/WIfPc7qEXcYvz655fqaZ6K4vcgTUhVUNoENrAy9VUHA1D\nG/LKta84X4cFheW7J7uiMcZwZcsrAVx2b64/KM811TjgdmvttjOvzwdmWWtL3R00xsQAK6y1D555\nbYC9wBRrbam+ENIY8zlwylo7tIj1XYHYxo1j2b9fPVUREX9UGeb+DcwNVIAzz0u9H2NMVSAKcF5U\nso5kXwj0KGq7s/Zx0Zm2i0tqq56qiIh4SnlC9bAxZoQxJuDM424gsQzbRwCBwKGzlh/CcX21SMaY\nvcaYdBynjF+31k4r6WAKVRER8ZTyTJlyL/AR8PqZ13GAp4b29QRCge7Ai8aYbdbaT4vbIMt7X/8p\nIiJ+pkyhaowJACKstd2NMaEA1tpTJWx2tiQgGzh7BEMDIKG4Da21u888XW+MaQg8DRQbqqmpo7np\npvyTPkdHRxMdHV2GkkVEpKKbOXMmM2fm/4rIEydOeLSG8gxUWmut7Vxyy2L3UdhApT04Biq9XMp9\n/BsYZq0t9Ga+3IFKAQGxZGdroJKIiD+q8LfUAFuNMefnHaxUDhOB6caYWP64pSYEmA5gjBkHNM4d\n2WuM+TuO0N10ZvsrgIeBIu9TzZWTA9nZEFih5nsSERFfVJ5QrQusMcYsI//cv6We3NNaO8sYEwE8\ni+O07xrgOmtt7oCnhkDeSVIDgHFACyAL2A6Mtda+U5rjnT4NwaX/PmAREZFyKU+ovn/mcU6stW8A\nbxSxbvhZr/8L/Le8x1KoioiIJ5R1oFIg0L4yfJ9qXqdPe7sCERHxB2W6T9Vamw1c6aZa3EahKiIi\nnlCeyR/mG2OeMMY0NsbUyn24vDIXUqiKiIgnlOea6r/P/HwOx4T65szPCju+VqEqIiKeUJY5ey8E\nsNYGACHW2gBrbeCZ11e7q0BXUKiKiIgnlOX074d5ni87a91EF9TiNgpVERHxhLKEqinieWGvK5SM\nDG9XICIi/qAsoWqLeF7Y6wpFPVUREfGEsgxUCjbGdMTRK837HKBCT62gUBUREU8oU6gCX+d5nfe5\neqoiIuL3Sh2q1toWbqzDrRSqIiLiCeWZ/KHSUaiKiIgnKFRFRERcRKEqIiLiIj4fqlWqKFRFRMQz\nFKoiIiIu4vOhWrWqZlQSERHPUKiKiIi4iM+HarVqClUREfEMnw/V6tUhLc3bVYiIiD9QqIqIiLiI\nQlVERMRFfD5Ug4IUqiIi4hk+H6rqqYqIiKcoVEVERFxEoSoiIuIiPh+quqYqIiKe4vOhqp6qiIh4\nikJVRETERRSqIiIiLqJQFRERcRGfD1UNVBIREU/x+VCtXh0yMyE729uViIiIr/OLUAX1VkVExP0U\nqiIiIi7i86EaFOT4qVAVERF38/lQVU9VREQ8RaEqIiLiIgpVERERF/H5UNU1VRER8RSfD1X1VEVE\nxFP8JlTT071bh4iI+D6/CVX1VEVExN0UqiIiIi7i86FqjCbVFxERz/D5UAUIDlaoioiI+ylURURE\nXEShKiIi4iIKVRERERdRqIqIiLiIQlVERMRFFKoiIiIu4jehmprq7SpERMTXeS1UjTH3G2N2GmPS\njDExxphLiml7izFmgTHmsDHmhDFmmTHm2tIeKyREPVUREXE/r4SqMeYOYALwFHARsBb43hgTUcQm\nvYEFQD+gK7AImGuM6Vya4ylURUTEE7zVUx0NvG2t/cBauwm4F0gF/lpYY2vtaGvtK9baWGvtdmvt\nE8BW4M+lOVhIiE7/ioiI+3k8VI0xVYEo4MfcZdZaCywEepRyHwaoCRwtTXtdUxUREU/wRk81AggE\nDp21/BDQsJT7GAvUAGaVprFO/4qIiCdU8XYBZWWMGQT8C7jJWptUmm10+ldERDzBG6GaBGQDDc5a\n3gBIKG5DY8ydwDvAQGvtotIcbPTo0Rw7VpvkZLjpJsey6OhooqOjy1q3iIhUYDNnzmTmzJn5lp04\nccKjNRjH5UzPMsbEACustQ+eeW2APcAUa+3LRWwTDUwF7rDWflOKY3QFYmNjY/n9964MHw6ZmVCl\n0vXNRUSkvOLi4oiKigKIstbGuft43hr9OxH4mzFmiDGmLfAWEAJMBzDGjDPGvJ/b+Mwp3/eBh4FV\nxpgGZx61SnOw4GDHT11XFRERd/JKqFprZwGPAM8Cq4FOwHXW2sQzTRoCzfJs8jccg5teBw7keUwu\nzfFCQhw/dV1VRETcyWsnQ621bwBvFLFu+FmvrzyXY+WGqnqqIiLiTn4x9696qiIi4gl+Eao1azp+\nengQmIiI+Bm/CNX69R0/Dx/2bh0iIuLb/CJUw8MhIAAOnT2Hk4iIiAv5RagGBkJYGBw/7u1KRETE\nl/lFqAKEhsLJk96uQkREfJlfheqpU96uQkREfJnfhGrNmuqpioiIe/lNqKqnKiIi7qZQFRERcRG/\nCVWd/hUREXfzm1BVT1VERNzNr0JVPVUREXEnvwnVmjXVUxUREffym1BVT1VERNzNr0I1JQVycrxd\niYiI+Cq/CdWaNcFafaeqiIi4j9+Eamio46euq4qIiLv4XajquqqIiLiL34RqzZqOnykp3q1DRER8\nl9+Eqk7/ioiIuylURUREXEShKiIi4iJ+E6o1ajh+KlRFRMRd/CZUq1aFatU0UElERNzHb0IV9E01\nIiLiXgpVERERF1GoioiIuIhCVURExEX8KlRr1oTkZG9XISIivsqvQjU8HI4e9XYVIiLiq/wuVJOS\nvF2FiIj4Kr8L1SNHvF2FiIj4Kr8K1YgIhaqIiLiPX4VqeDikpkJamrcrERERX+RXoRoR4fiZmOjd\nOkRExDf5Vah27Oj4GRvr3TpERMQ3+VWoNm7smFT/wAFvVyIiIr7Ir0LVGKhdG06c8HYlIiLii/wq\nVGIMN0EAABaKSURBVMERqsePe7sKERHxRX4XqmFh6qmKiIh7+F2oqqcqIiLu4pehqp6qiIi4g9+F\naliYeqoiIuIefheq6qmKiIi7+GWoqqcqIiLu4HehqtG/IiLiLn4XqrVrOybUP33a25WIiIiv8ctQ\nBfVWRUTE9fwuVENDHT9TU71bh4iI+B6/C9WgIMfP9HTv1iEiIr7Hb0NVX1QuIiKu5rehqp6qiIi4\nmkJVRETERbwWqsaY+40xO40xacaYGGPMJcW0bWiM+cgYs9kYk22MmVje4ypURUTEXbwSqsaYO4AJ\nwFPARcBa4HtjTEQRm1QHDgPPAWvO5djBwY6fClUREXE1b/VURwNvW2s/sNZuAu4FUoG/FtbYWrvb\nWjvaWjsDSD6XA6unKiIi7uLxUDXGVAWigB9zl1lrLbAQ6OHu42v0r4iIuIs3eqoRQCBw6Kzlh4CG\n7j54lSoQGKieqoiIuJ7fjf4FR29VoSoiIq5WxQvHTAKygQZnLW8AJLj6YKNHj6Z27oS/ZwQERJOe\nHu3qQ4mIiBfNnDmTmTNn5lt2wsMTvRvH5UzPMsbEACustQ+eeW2APcAUa+3LJWy7CFhtrR1TQruu\nQGxsbCxdu3bNt65OHcd3qi5fDt27n8s7ERGRiiwuLo6oqCiAKGttnLuP563TvxOBvxljhhhj2gJv\nASHAdABjzDhjzPt5NzDGdDbGdAFCgXpnXrcrz8Fzv6T83XfLXb+IiEgB3jj9i7V21pl7Up/Fcdp3\nDXCdtTbxTJOGQLOzNlsN5HaruwKDgN1Aq/LWUbVqebcUEREpyGsDlay1b1hrW1hrg621Pay1v+VZ\nN9xae9VZ7QOstYFnPcoVqEOHOn6GhJzLOxAREcnPL0f/Tp8OHTrA6dPerkRERHyJX4YqQPXquq1G\nRERcy29DNSgIMjK8XYWIiPgSvw3V6tUVqiIi4loKVRERERfx61DVNVUREXElvw1VXVMVERFX89tQ\n1elfERFxNYWqiIiIi/h1qOqaqoiIuJLfhqquqYqIiKv5bajq9K+IiLiaX4eqTv+KiIgr+XWoqqcq\nIiKu5LehqmuqIiLian4bqrk9VWtLbisiIlIafh2qAAEBMHu2d2sRERHf4LehWqvWH8+ff957dYjI\n/7d351FSVGcfx78PiyAqIoIQVBYNIAaD4JKooKKIintMjHGJRs2JJi4vZjPqmxA9MWIUNCYu0agB\nXKIeNGiMCKK+CAFlE5VFkUUioIzDjsgy9/3jqU4XPT3DwNR0dQ+/zzl9urvqdvVTvdRz761bVSL1\nx06bVPfZJ/t4v/3Si0NEROoPJVVgxYr04hARkfpjp02qnTplu4BXr043FhERqR922qTaqBGsXAk3\n3QTvvadRwCIiUns7bVIFMIPXXvPHr76abiwiIlL6duqkCnDBBX6/bl26cYiISOnb6ZPqFVf4/apV\n6cYhIiKlb6dPqk2awG67QXl52pGIiEip2+mTKkDLljqsRkREak9JFdhrL7VURUSk9pRU8ZaqkqqI\niNSWkipKqiIikgwlVZRURUQkGUqqKKmKiEgylFRRUhURkWQoqZI9pKaiIu1IRESklCmp4kk1BJ1V\nSUREakdJFU+qAK1awRtvpBuLiIiULiVVskm1ogJGj043FhERKV1KqvgZlTI2bEgvDhERKW1KqmRb\nqgBDh8KWLenFIiIipUtJFb9KTdxHH6UTh4iIlDYlVcAMJk3KDlJavTrdeEREpDQpqUa+8Q3Yf39/\nrENrRERkRyipxjRv7vdqqYqIyI5QUo3Zc0/vCv7887QjERGRUqSkGtOoEXzlK/Dxx2lHIiIipUhJ\nNUeHDkqqIiKyY5RUc7RvD4sWpR2FiIiUIiXVHN26weuvQ5cucNhhMH582hGJiEipUFLNcf31fv/h\nhzBtGhx7bLrxiIhI6VBSzbHHHpWntW4Nn3xS+FhERKS0KKnmsXw5zJsHs2f787IyeOaZdGMSEZHi\np6SaR6tWcOCBcNBBMGcOHHAADBwIb74J770Hd97pl4kTqUsbN0IIaUdRO0uXwvr1aUchUjhKqtvQ\ntStceqk/7tMHDjkEfv5zePVVn7ZihR+Cc++98NJLW7921iy44Ybs89dfh5kzCxG1lLovv4QmTeD+\n+9OOpHbatYNTTkk7CvfZZ/DWW2lHIfWdkmoN/OxnW19zFaB/f+jc2S8b16EDXHstnHYarFkDDz8M\n5eVw8cUweLAn3hCgb1/o0SOddZDSkmndPfVUunEkoVhG0Pfp4+f4LhZlZfDCC2lHsX1GjvSzzuny\nmFVLLama2U/MbIGZfWFmk8zsiG2UP97MpprZBjP7wMwuKVSsu+7qA5VmzfJ9q3fe6dPnzatctnlz\n+OEPYe+94YMPfNoTT0CD2Ce9Zg28+CL89a9w1VW+0Zkzx1vBmZP5r1wJDzzwJACDBsFdd8HatfDQ\nQ77ca66p3AU9eLAfBpSkEHwkdD7z5m37MnlPPvlksgEVWFrxf/GF369dW7vllPrnD8mtQ+b/WGhV\nxX/hhXDmmaWxKymzDn/8oz+v7e+yXgshFPwGfBfYAHwfOAh4ECgHWlVRviOwFrgD6Ar8BNgEnFTN\ne/QCwtSpU0PS1q8PoUGDEDzlbP/t5ZernnfMMSE89ljm+RmhrCw773vf8/vOnf3+xRdDuO02fzxm\nTLbckiX+vG/fEJ5/PoSXXsrGvXx5dj02bAihoqL6dR03zpc5alTleZn3q84ZZ5xR5bxNm0JYsaL6\n16etuvjr0rx5/tl26VK75aQVf0ZNfiPbktQ6ZGLZ1m8+aVXF36OHx1NeXth4dkRmHU44wWNevDjl\ngLbD1KlTAxCAXqEA+S2tlupA4MEQwrAQwhzgSmA9cFkV5a8C5ocQfhFCmBtC+DPwbLScgtt1V++e\nGz4cGjaEe+7xWvDbb1cuu88+ladVt49pwoTsPlzwQVMZmQpvpuV4+ulw443++KSTPBbw1590Erz2\nGpx9NgwYAJdc4hdjb93au6DHjoWmTaFjRx+IdfnlcN113r1TXg6vvAL/+ld2BPSIEd7t8/rrfhWf\neO166VIYNsxbruXlMGSI12grKjyGXr28Rd63r0/76CP//K67zrvV163Ltvo3b/ZN38KFvl8RfD/0\n4sXw7rswbpxPmzEDbrsNHnvMu+Lz2bzZY37uOR/0k7FhAyxY4LH86EfeS1BsttVS3bLFdy/MmeO9\nGkuXFi62mirWLsING9KOwDVt6vdlZenGsT0yPW5r1qQbRzFrVOg3NLPGwGHAbZlpIYRgZmOBo6p4\n2TeBsTnTRgND6yTIGmjSBC66yG9xEyd6EjjySN/Q9e7t3btz58JPfwpXX+3JeNUq6NTJN+6XXurJ\noWtXOPhgTwI7IrMRe+WVyvOGDcs+njnTky5kz3O8YIHfZ7p3cj39tN/37Vt5Xrt2+V+zdKknhenT\n/Qa+T2vKlK3L9e/vnxv4RQ02b87OO+ecyp/HP//p+6/jbr3V//DPPAPHHecDyd5/3+d961t+37q1\nV1IyFYVp0+Avf/GKRM+eXu7KK/OvS6Flkuq6dX575BG/itKMGX7Wr2bNvKKzbBlMmuQVkT/8wb/b\noUOzG+w0FUvyyrV2rVeM09asmd8vW+bjMzJmzvTf6fLlsGkTjBkD557rZ3lLW6birqRatYInVaAV\n0BD4NGf6p3jXbj5tqyjf3MyahBC+TDbEHXfUUX4D31CDJ4DTTvOEussuPlJ48mQ44ghPBJ995i23\nBx+EFi28ldaypc+/+Wb4zne8BTtqlCfiu+7yZHPvvdCvn29EhwyBRx+Fm26C3/3O37dJE2/ttW/v\nyWr+fG+ZTZzorb64du1gyZLs8/POyybSzHJatIA2bbyCUBO33155Wm5ChWxCha0TKuSvYOQmVIBf\n/zr7+J138sezfLnfMnr18vuyMt9wjRnjG42ePb2Fa+aVn2nT8i+vLmVGia9aBfvum93XnmtsrKr5\n4x/7/QMPeKJt3Di9+GHrmKdMyX6mO7KcJNfhmmu896Z79+SWWZ2q4m/c2O9vusmT/PTpfijfpEmV\ny954o1eW9967bmOtSmYdMgPo3njDv8sGDXbsO83VsaNv8+oDCwU+EM7MvgJ8AhwVQpgcmz4YODaE\nUKm1amZzgUdCCINj004FXgSa5UuqZnY0MGHEiBF069atDtak7g0cOJChQ/M3xkPwlmCmlbhxoyfK\nXr2yP/J163yAVZcu3vJp3NiTK3irdsMG7xKOq6jw5e67ryfyCRN84NXAgd4q79fPy61f78tu1Mhb\nfv36+YUIMjXutWv9T/jUUwMZMGAoixb5KR9Hj4ajj/bkNn68t7rGjYNbboGXX/YNnZmP1Bw92lul\np57qo2B794YTT/Rybdt61/HIkXDZZd4N2qWL1/p/8xuvaCxa5BuoefPgiiv8s5k82V+/cmX2LFnd\nu3srdvp0n57zLZBihwitW/t30bKlV8jWrvXjpcG7f4cP94rOHnt4tzx4JS0r3fiTkew67L57oQfa\nVB//brt5ZbJ1a2+95htQ1apV2t3Edfs7uvVWr+jUhdmzZ3ORdykeE0KYuK3ytZVGUm2M7z89N4Qw\nKjb9MWDPEMI5eV7zBjA1hHB9bNqlwNAQwl655aP5FwCPJxu9iIiUqAtDCE/U9ZsUvPs3hLDJzKYC\nJwKjAMzMoudV7NHj38CpOdP6R9OrMhq4EFiIjzQWEZGdT1P8CJLRhXizgrdUAczsPOAxfNTvW3jf\nwreBg0IIy83s90C7EMIlUfmOwLvAfcAjeAK+GxgQQsgdwCQiIpKKNAYqEUJ42sxaAbcAbYAZwMkh\nhMwwkrbA/rHyC83sNLxT/1rgP8DlSqgiIlJMUmmpioiI1Ec696+IiEhC6mVS3d7zCheKmf3KzN4y\ns9Vm9qmZPWdmlQ7pNrNbzGyJma03szFm9tWc+U3M7M9mVmZma8zsWTPLc+6mumVmN5hZhZkNyZle\n1PGbWTszGx69/3oze8fMepXCOphZAzO71czmR7HNM7Ob85QrmvjNrI+ZjTKzT6Lfy5l1Ea+Z7WVm\nj5vZKjNbYWYPm1nOQWPJxm9mjcxssJnNNLO1UZm/RYcOFkX821qHPGUfiMpcWyzrUMPfUDcz+4eZ\nrYy+i8lmtl/B4y/EuRALeWM7zytc4NheAi4GugGH4MfZLgR2jZX5ZRTv6UB34HngI2CXWJn7o9cd\nB/QEJgLjC7wuRwDzgenAkFKJH2gBLAAexs/s1QHoB3QqhXUAbgQ+A04B2gPfAlYDVxdr/FGstwBn\nAVuAM3PmJxIv8C9gGnA4cDTwATCiLuMHmuOjSs8FOgNHApOAt3KWkVr8NfkOYuXOwf/Ti4Fri2Ud\navAbOhAoA34PfB3oFP2eWhU6/jrdAKRxi37Q98SeGz6w6Rdpx5Yn1lZABdA7Nm0JMDD2vDnwBXBe\n7PmXwDmxMl2j5RxZoLh3B+YCJwCvsXVSLer4gduBN7ZRpmjXAXgBeChn2rPAsBKJvyLPBrHW8eIV\n1QqgZ6zMycBmoG1dxp+nzOH4hn+/You/unUA9gU+jmJZQCypFtM6VPEbehL4WzWvKVj89ar717Ln\nFX41My34J1PdeYXT1AK/ekI5gJl1wkc+x+NfDUwmG//h+KjteJm5+J+hUOv4Z+CFEMK4+MQSif8M\nYIqZPW3eBT/NzK7IzCyBdZgInGhmnaN4ewDH4L0gpRD/VhKM95vAihDC9Njix+L/r0JfRTXzv86c\nn+swijx+MzNgGHBHCGF2niJFuw5R7KcBH5rZy9H/epKZnZVG/PUqqVL9eYXbFj6cqkU/hLuBN0MI\ns6LJbfEvsLr42wAbow1PVWXqjJmdDxwK/CrP7KKPHzgAv+rRXPwEIvcDfzSzi6P5xb4OtwN/B+aY\n2UZgKnB3CCFzOfNijz9XUvG2xbvF/yuEsAWvsBZsncysCf4dPRFCyJwMsS3FH/8NeIx/qmJ+Ma/D\nPnjv2S/xyuVJwHPASDPrE4utIPGncpyqAH4ii4PxVkZJiHb63w30CyFsSjueHdQA39/1v9Hzd8ys\nO34ikuHphVVj3wUuAM4HZuEVnHvMbEkIoRTir7fMrBHwDF5J+HHK4dSYmR2GH//fM+1YdlCmcfh8\nCCFzVr6Z5ud/vxIYn0Yw9UUZvi+jTc70NsCywoeTn5n9CRgAHB9CiF8Jcxm+D7i6+JcBu5hZ82rK\n1JXDgNbANDPbZGab8J3+10Wtpk8p7vgBlgK53Vuz8UE/UPzfwR3A7SGEZ0II74cQHsdPipLpOSj2\n+HMlFe8yvMXyX2bWEGhJAdYpllD3B/rHWqmZ2Io5/t74/3px7H/dARhiZvNj8RXrOpTh+z239b8u\nSPz1KqlGrafMeYWBrc4rXOdXJ6iJKKGeBfQNIXwcnxdCWIB/efH4m+P9+Zn4p+I/oHiZrviPp7pz\nISdhLD5q+VCgR3SbAowAeoQQ5hd5/AATqHyJwa7AIiiJ76AZXnGMqyD6L5dA/FtJMN5/Ay3MLN7a\nOhFP2JOpQ7GEegBwYghhRU6Roo4f35f6dbL/6R744LE78IE6UMTrEG3336by/7oL0f+aQsaf5Kiy\nYrgB5+FXwYkfUvM50LoIYrsPWAH0wWtImVvTWJlfRPGegSew54EP2frwgvvw0XnH463HCRT4kJpY\nLLmjf4s6fnzQy5d4y+5AvCt1DXB+KawD8Cg+uGIA3po4B98PdFuxxg/shm+oD8UrAP8TPd8/yXjx\n/WlT8MO9jsH3mw+vy/jxXWj/wDfeh7D1/7pxMcRfk+8gT/mtRv+mvQ41+A2djR9KeQX+v74a2Ihf\nYrSg8dfpBiCtG74/YyE+LP/fwOFpxxTFVYG3MnJv388pNwivKa7Hj4H7as78JsC9eLfHGryWvE9K\n6zSOWFIthfjxhDQziu994LI8ZYpyHaKNy5Bo47AOTz6/BRoVa/z4LoJ8v/1HkowXH3U7AliFV14f\nwq+3XGfx4xWb3HmZ58cWQ/w1/Q5yys+nclItyu8gVuZS/LjSdfixpqenEb/O/SsiIpKQerVPVURE\nJE1KqiIiIglRUhUREUmIkqqIiEhClFRFREQSoqQqIiKSECVVERGRhCipioiIJERJVUREJCG69JtI\nkTOzhfgpN7/AT+4dgItDCO8n+B4dgBkhhL2SWqbIzkhJVaT4BeC8EMK7BXgfEakFdf+KlAarNMGs\nwsxuNbNpZjbHzC6IzTvZzKaa2Qwze83MusXm/cDMpkfz3jKz9tlZNsjMppjZB2Z2SgHWS6ReUUtV\npDT83czi3b9HR9O3hBB6mVknYIqZvYl3Ez+OXyVlVpRsnwW+ZmbHAzfjl8T6zMyaRstpA+yJdwEP\nMrOTgXvwyyeKSA3pKjUiRc7MFgBn5nb/mlkF0D6E8J/o+UhgJLASuD6EcEKsbDnQHb8O5foQwqCc\nZXUAZoUQdoueNwfKQgi71NmKidRD6v4VKQ2Vun/zTM+0YqsrX50vY4+3AA13YBkiOzUlVZHS9gMA\nM+sI9Ab+D5gEdDezg6N55wOfhBCWAC8AF5lZ22jerrEu4NxEvCOJWWSnpn2qIsUvUHmf6vXRvIZm\nNg1oBlwTQlgMYGYXAsPNrCGwAvgOQAhhvJn9FhhtZgFvnX479j657ysi20H7VEVKVLRPtUUIYXXa\nsYiIU/evSOlSjVikyKilKiIikhC1VEVERBKipCoiIpIQJVUREZGEKKmKiIgkRElVREQkIUqqIiIi\nCVFSFRERSYiSqoiISEKUVEVERBLy/wLBHVmjlEbYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd9f0468450>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5,5))\n",
    "plt.plot(1-np.array(train_accs), label='Training Error')\n",
    "plt.plot(1-np.array(test_accs), label='Test Error')\n",
    "plt.legend(fontsize=20)\n",
    "plt.xlabel('Epoch', fontsize=8)\n",
    "plt.ylabel('Error', fontsize=8)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'reshape'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-dcfef8167e06>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Histograms\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdense_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparam_outputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mdisc_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdisc_outputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mbin_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m100\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'reshape'"
     ]
    }
   ],
   "source": [
    "# Histograms\n",
    "dense_params = param_outputs.reshape((-1, 6))\n",
    "disc_params = disc_outputs.reshape((-1, 6))\n",
    "\n",
    "bin_count = 100\n",
    "plot_over = True\n",
    "\n",
    "for i in range(0, 6):\n",
    "    dns = dense_params[:, i]\n",
    "    \n",
    "    \n",
    "    dsc = disc_params[:, i]\n",
    "    print len(np.unique(dsc))\n",
    "    #PS: Using normed histograms to plot them over\n",
    "    # Theta x Dense\n",
    "    plt.figure()\n",
    "    n, bins, patches = plt.hist(dns, bins=bin_count, normed=plot_over, histtype='stepfilled')\n",
    "    plt.setp(patches, 'facecolor', 'r', 'alpha', 0.55)\n",
    "    if not plot_over:\n",
    "        plt.xlabel(('Theta({0}) - Discrete Output').format(i+1))\n",
    "        plt.ylabel('Frequency (Consider bin size)')\n",
    "        plt.grid(True)\n",
    "        plt.figure()\n",
    "    \n",
    "    # Theta x Discrete\n",
    "    n, bins, patches = plt.hist(dsc, bins=np.unique(dsc), normed=plot_over, histtype='stepfilled')\n",
    "    plt.setp(patches, 'facecolor', 'g', 'alpha', 0.55)\n",
    "    if not plot_over:\n",
    "        plt.xlabel(('Theta({0}) - Discrete Output').format(i+1))\n",
    "    else:\n",
    "        plt.xlabel(('Theta({0})').format(i+1))\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.grid(True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(7,14))\n",
    "for i in range(3):\n",
    "    plt.subplot(321+i*2)\n",
    "    plt.imshow(data['X_test'][i].reshape(DIM, DIM), cmap='gray', interpolation='none')\n",
    "    if i == 0:\n",
    "        plt.title('Original 60x60', fontsize=20)\n",
    "    plt.axis('off')\n",
    "    plt.subplot(322+i*2)\n",
    "    plt.imshow(test_transform[i].reshape(DIM//3, DIM//3), cmap='gray', interpolation='none')\n",
    "    if i == 0:\n",
    "        plt.title('Transformed 20x20', fontsize=20)\n",
    "    plt.axis('off')\n",
    "plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}