run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import skimage\n",
    "import matplotlib.pyplot as plt\n",
    "import imp\n",
    "import core\n",
    "import resnet\n",
    "import utils\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "wd = 'E:/new_data/kaggle/planet/'\n",
    "train_set = pd.read_csv(wd + 'train_v2.csv')\n",
    "train_set['tags'] = train_set['tags'].apply(lambda x: x.split(' '))\n",
    "test_set = pd.read_csv(wd+'sample_submission_v2.csv')\n",
    "train_tags = ['clear', 'partly_cloudy', 'haze', 'cloudy', 'primary', 'agriculture', 'road', 'water',\n",
    "             'cultivation', 'habitation', 'bare_ground', 'selective_logging', 'artisinal_mine', \n",
    "              'blooming', 'slash_burn', 'conventional_mine', 'blow_down']\n",
    "label_map = {l: i for i, l in enumerate(train_tags)}\n",
    "inv_label_map = {i: l for l, i in label_map.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 载入数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "from time import time\n",
    "file_all = train_set['image_name'].values\n",
    "y_all = utils.get_y(train_set['tags'].values, label_map)\n",
    "x_tr, x_vl, y_tr, y_vl = train_test_split(file_all, y_all, test_size=0.8, random_state=int(time()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn, optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision import transforms\n",
    "import random\n",
    "from PIL import Image\n",
    "from time import time\n",
    "import mydataset\n",
    "imp.reload(utils)\n",
    "imp.reload(core)\n",
    "imp.reload(mydataset)\n",
    "imp.reload(resnet)\n",
    "import gc\n",
    "for i in range(10):\n",
    "    gc.collect()\n",
    "\n",
    "# 超参数\n",
    "batch_size = 256\n",
    "pic_size = (64, 64)\n",
    "learning_rate = 1e-3\n",
    "num_epoches = 1000\n",
    "tolerance = 15\n",
    "lr_tolerance = 15\n",
    "best_model = core.BestModel()\n",
    "transform1 = transforms.Compose([\n",
    "#     transforms.RandomCrop(pic_size),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "])\n",
    "transform2 = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n",
    "])\n",
    "\n",
    "def transform_tr(img, pic_size):\n",
    "    img = img.resize(pic_size)\n",
    "    img = img.transpose(Image.FLIP_LEFT_RIGHT)  if random.randint(0, 1) > .5 else img\n",
    "    img = img.transpose(Image.FLIP_TOP_BOTTOM)  if random.randint(0, 1) > .5 else img\n",
    "    img.rotate(np.random.random() * 45)\n",
    "    img_tensor = transform2(img)\n",
    "    return img_tensor\n",
    "\n",
    "def transform_vl(img, pic_size):\n",
    "    img = img.resize(pic_size)\n",
    "    img_tensor = transform2(img)\n",
    "    return img_tensor\n",
    "\n",
    "probs = [0.5, 0.6, 0.8, 0.8, 0.5, 0.5, 0.6, 0.6, 0.8, 0.8, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Update Model!\n",
      "[1/1000] 92s, Loss: 0.6165, Val-Loss: 0.5070, Best Val-Loss: 0.5070, Val-F2: 0.451\n",
      "Update Model!\n",
      "[2/1000] 93s, Loss: 0.3582, Val-Loss: 0.2051, Best Val-Loss: 0.2051, Val-F2: 0.813\n",
      "Update Model!\n",
      "[3/1000] 95s, Loss: 0.2059, Val-Loss: 0.1651, Best Val-Loss: 0.1651, Val-F2: 0.839\n",
      "Update Model!\n",
      "[4/1000] 94s, Loss: 0.1838, Val-Loss: 0.1566, Best Val-Loss: 0.1566, Val-F2: 0.843\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-148-5c637624685d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[0mval_loader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalid_loader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx_vl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_vl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtransform_vl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpic_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m     \u001b[0mloss_vl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf2_vl\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mval_loader\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m     \u001b[0mbest_model\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss_vl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mavg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf2_vl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mI:\\Workspaces\\python\\kaggle\\Planet\\pytorch\\core.py\u001b[0m in \u001b[0;36mvalidate\u001b[1;34m(self, val_loader)\u001b[0m\n\u001b[0;32m    162\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    163\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 164\u001b[1;33m             \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcriterion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    165\u001b[0m             \u001b[0mloss_vl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    166\u001b[0m             \u001b[0mf2_vl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mJ:\\Anaconda3\\envs\\py36\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 206\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    207\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    208\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mJ:\\Anaconda3\\envs\\py36\\lib\\site-packages\\torch\\nn\\modules\\loss.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, target)\u001b[0m\n\u001b[0;32m     34\u001b[0m         \u001b[0m_assert_no_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[0mbackend_fn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 36\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mbackend_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize_average\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     37\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mJ:\\Anaconda3\\envs\\py36\\lib\\site-packages\\torch\\nn\\_functions\\thnn\\loss.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, target)\u001b[0m\n\u001b[0;32m     22\u001b[0m         \u001b[1;32massert\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnelement\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnelement\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_resize_weight\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 24\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBCELoss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     25\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_unresize_weight\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     26\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mJ:\\Anaconda3\\envs\\py36\\lib\\site-packages\\torch\\nn\\_functions\\thnn\\auto.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, target)\u001b[0m\n\u001b[0;32m     39\u001b[0m         \u001b[0moutput\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnew\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m         getattr(self._backend, update_output.name)(self._backend.library_state, input, target,\n\u001b[1;32m---> 41\u001b[1;33m                                                    output, *self.additional_args)\n\u001b[0m\u001b[0;32m     42\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = core.MyNet(17).cuda()\n",
    "#model = resnet.ResNet(pic_width=pic_size[0], num_classes=17, block=resnet.BasicBlock).cuda()\n",
    "criterion = nn.BCELoss(weight=None).cuda()\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "history = core.History(['train loss', 'valid loss'])\n",
    "\n",
    "estimator = core.Estimator(model, criterion, optimizer)\n",
    "for epoch in range(num_epoches):\n",
    "    time_st = time()\n",
    "    train_loader = utils.weighted_train_loader(x_tr, y_tr, probs, transform_tr, batch_size, pic_size)\n",
    "    loss_tr = estimator.train(train_loader)\n",
    "    \n",
    "    val_loader = utils.valid_loader(x_vl, y_vl, transform_vl, batch_size, pic_size)\n",
    "    loss_vl, f2_vl = estimator.validate(val_loader)\n",
    "    best_model.update(loss_vl.avg, f2_vl, estimator.model)\n",
    "    \n",
    "    if epoch > lr_tolerance and best_model.lrcount > lr_tolerance:\n",
    "        optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * 0.1\n",
    "        print('New Learn Rate: {}!'.format(new_lr))\n",
    "        best_model.lrcount = 0\n",
    "        \n",
    "    if epoch > tolerance and best_model.nobetter > tolerance:\n",
    "        print('Early Stop in Epoch: {}, Best Val-Loss: {:.6f}, Best F2: {:.6f}'.format(\n",
    "            epoch+1, best_model.best_loss, best_model.best_f2.value(bestf2=True)))\n",
    "        break\n",
    "        \n",
    "    print('[{}/{}] {}s, Loss: {:.4f}, Val-Loss: {:.4f}, Best Val-Loss: {:.4f}, Val-F2: {:.3f}'.format(\n",
    "        epoch+1, num_epoches, int(time() - time_st), loss_tr.avg, loss_vl.avg,\n",
    "        best_model.best_loss, f2_vl.value(0.3)))\n",
    "    history.update({'train loss': loss_tr.avg, 'valid loss': loss_vl.avg})\n",
    "    gc.collect()\n",
    "history.plot(['train loss', 'valid loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmMAAAHjCAYAAABvkBg4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl41NXd///nO8mEZIBkwiJbguyyi4qAogJVVNxAxaLF\nXWppXVt7V9ve9md3qbb+tFWpWpf2rlo3XCpqxSKgqGyyyhb2nbAkYUvIcr5/fGYwhiyTzEwmy+tx\nXVwz81nPxCUvzjmf9zHnHCIiIiISHwnxboCIiIhIU6YwJiIiIhJHCmMiIiIicaQwJiIiIhJHCmMi\nIiIicaQwJiIiIhJHCmMiIiIicaQwJiIiIhJHCmMiIiIicZQU7wbURJs2bVyXLl3i3QwRERGRai1c\nuHCPc65tdcc1qDDWpUsXFixYEO9miIiIiFTLzDaFc5yGKUVERETiSGFMREREJI4UxkRERETiqEHN\nGatIUVERW7dupaCgIN5NadBSUlLIzMzE5/PFuykiIiJNSoMPY1u3bqVly5Z06dIFM4t3cxok5xx7\n9+5l69atdO3aNd7NERERaVIa/DBlQUEBrVu3VhCLgJnRunVr9S6KiIjEQYMPY4CCWBToZygiIhIf\njSKMiYiIiDRUCmMRys3N5YknnqjVuRdddBG5ublhH//AAw/w8MMP1+peIiIiUj8pjEWoqjBWXFxc\n5bnTp08nEAjEolkiIiLSQDT4pynL+uU7K/hqe35Ur9m3Yxr/36X9Kt1/3333sW7dOgYNGsTo0aO5\n+OKLuf/++8nIyGDVqlWsWbOGcePGsWXLFgoKCrjrrru49dZbga+Xdzp48CBjxozhrLPOYu7cuXTq\n1Im33nqL1NTUSu+7ePFiJk+ezOHDh+nevTvPPvssGRkZPPbYY0ydOpWkpCT69u3Lyy+/zKxZs7jr\nrrsAb27Y7NmzadmyZVR/TiIiIlI76hmL0IMPPkj37t1ZvHgxDz30EACLFi3i0UcfZc2aNQA8++yz\nLFy4kAULFvDYY4+xd+/e466zdu1abrvtNlasWEEgEOD111+v8r7XX389U6ZMYenSpQwYMIBf/vKX\nx9rz5ZdfsnTpUqZOnQrAww8/zOOPP87ixYuZM2dOlSFPRERE6laj6hmrqgerLg0ZMuQb9boee+wx\npk2bBsCWLVtYu3YtrVu3/sY5Xbt2ZdCgQQCcdtppbNy4sdLr5+XlkZuby4gRIwC44YYbuOqqqwAY\nOHAgEydOZNy4cYwbNw6A4cOH86Mf/YiJEydyxRVXkJmZGbXvKiIiIpFRz1gMNG/e/Nj7jz/+mBkz\nZvDZZ5+xZMkSTjnllArreTVr1uzY+8TExGrnm1Xm3Xff5bbbbmPRokWcfvrpFBcXc9999/HMM89w\n5MgRhg8fzqpVq2p1bREREYm+sMKYmV1oZqvNLNvM7qtg/1gzW2pmi81sgZmdVd25ZtbKzD40s7XB\n14zofKW61bJlSw4cOFDp/ry8PDIyMvD7/axatYrPP/884nump6eTkZHBnDlzAPjHP/7BiBEjKC0t\nZcuWLYwaNYopU6aQl5fHwYMHWbduHQMGDODee+/l9NNPVxgTERGpR6odpjSzROBxYDSwFZhvZm87\n574qc9hHwNvOOWdmA4FXgN7VnHsf8JFz7sFgSLsPuDeaX64utG7dmuHDh9O/f3/GjBnDxRdf/I39\nF154IVOnTqVPnz6cdNJJDBs2LCr3feGFF45N4O/WrRvPPfccJSUlXHvtteTl5eGc48477yQQCHD/\n/fczc+ZMEhIS6NevH2PGjIlKG0RERCRy5pyr+gCzM4AHnHMXBD//FMA59/sqjn/WOdenqnPNbDUw\n0jm3w8w6AB87506qqi2DBw92CxYs+Ma2lStX0qdPnzC+avWcc5SUOhITrElWpI/mz1JERKSpM7OF\nzrnB1R0XzjBlJ2BLmc9bg9vK3/ByM1sFvAvcHMa57ZxzO4LvdwLtwmhLTO05WMhXO/IprTqfioiI\niERN1CbwO+emOed6A+OAX9fwXAdUGIHM7NbgPLQFOTk5UWhp5RITvB9HSWlpTO8jIiIiEhJOGNsG\nZJX5nBncViHn3Gygm5m1qebcXcHhSYKvuyu53lPOucHOucFt27YNo7m1l5jgDU2WqGtMRERE6kg4\nYWw+0NPMuppZMnA18HbZA8yshwUnWZnZqUAzYG81574N3BB8fwPwVqRfJlJJwTBWrDAmIiIidaTa\npymdc8VmdjvwAZCINzl/hZlNDu6fClwJXG9mRcARYEJw6LHCc4OXfhB4xcxuATYB347yd6sx9YyJ\niIhIXQurAr9zbjowvdy2qWXeTwGmhHtucPte4NyaNDbWFMZERESkrqkCfxmJdTRM2aJFCwC2b9/O\n+PHjKzxm5MiRlC/jUdV2ERERaZgUxspIMCPRrM56xjp27Mhrr71WJ/cSERGR+qlRLRTOe/fBzmUR\nXaLr0WISEgySEr0N7QfAmAcrPf6+++4jKyuL2267DYAHHniAFi1aMHnyZMaOHcv+/fspKiriN7/5\nDWPHjv3GuRs3buSSSy5h+fLlHDlyhJtuuoklS5bQu3dvjhw5Um1bX3rpJX73u9/hnOPiiy9mypQp\nlJSUcMstt7BgwQLMjJtvvpkf/vCHPPbYY0ydOpWkpCT69u3Lyy+/XPsfkoiIiERN4wpj0WBQzaIE\n3zBhwgTuvvvuY2HslVde4YMPPiAlJYVp06aRlpbGnj17GDZsGJdddlmllf2ffPJJ/H4/K1euZOnS\npZx66qlV3nf79u3ce++9LFy4kIyMDM4//3zefPNNsrKy2LZtG8uXLwcgNzcXgAcffJANGzbQrFmz\nY9tEREQk/hpXGKuiBytcO3MOUuqgxwktwjr+lFNOYffu3Wzfvp2cnBwyMjLIysqiqKiIn/3sZ8ye\nPZuEhAS2bdvGrl27aN++fYXXmT17NnfeeScAAwcOZODAgVXed/78+YwcOZJQ7bWJEycye/Zs7r//\nftavX88dd9zBxRdfzPnnn3/smhMnTmTcuHGMGzcu3B+HiIiIxJjmjJWTlFDzOWNXXXUVr732Gv/6\n17+YMGECAP/85z/Jyclh4cKFLF68mHbt2lFQUBCLJn9DRkYGS5YsYeTIkUydOpVJkyYB8O6773Lb\nbbexaNEiTj/9dIqLi2PeFhEREamewlg5iQkJNV4OacKECbz88su89tprXHXVVQDk5eVxwgkn4PP5\nmDlzJps2baryGueccw4vvvgiAMuXL2fp0qVVHj9kyBBmzZrFnj17KCkp4aWXXmLEiBHs2bOH0tJS\nrrzySn7zm9+waNEiSktL2bJlC6NGjWLKlCnk5eVx8ODBGn1HERERiY3GNUwZBYnBnjHnXKXzu8rr\n168fBw4coFOnTnTo0AHwhg0vvfRSBgwYwODBg+ndu3eV1/j+97/PTTfdRJ8+fejTpw+nnXZalcd3\n6NCBBx98kFGjRh2bwD927FiWLFnCTTfdRGkwUP7+97+npKSEa6+9lry8PJxz3HnnnQQCgbC+m4iI\niMSWuZrMVo+zwYMHu/I1tlauXEmfPn2ido+cA4XsyDtC3w5pJCU2rY7DaP8sRUREmjIzW+icG1zd\ncU0rbYQhSVX4RUREpA4pjJWjJZFERESkLjWKMBbNodZjSyI1oOHbaGhIw9UiIiKNSYMPYykpKezd\nuzdqYaIpDlM659i7dy8pKSnxboqIiEiT0+CfpszMzGTr1q3k5ORE5XqlpY5deQUU5vhokdLgfzxh\nS0lJITMzM97NEBERaXIafNrw+Xx07do1atcrKXVc8vPp3PGtnvxodK+oXVdERESkIg1+mDLaEhOM\ntBQfuYePxrspIiIi0gQojFUgw+8j93BRvJshIiIiTYDCWAXS/cnsV8+YiIiI1AGFsQpk+H3kHVHP\nmIiIiMSewlgFAqk+9YyJiIhInVAYq0DAn0zuIfWMiYiISOwpjFUgw5/MgcJiikpK490UERERaeQU\nxioQ8PsANG9MREREYk5hrAKhMKbyFiIiIhJrCmMVCPiTAVT4VURERGJOYawCGcGesf3qGRMREZEY\nUxirQIZ6xkRERKSOKIxVIF1zxkRERKSOKIxVoGWzJBITjNwj6hkTERGR2FIYq4CZBavwq2dMRERE\nYkthrBIBv09zxkRERCTmFMYqkeFP1pwxERERiTmFsUoE/BqmFBERkdhTGKtEwJ9MnoYpRUREJMYU\nxiqhCfwiIiJSFxTGKpHRPJkjRSUUFJXEuykiIiLSiCmMVSK0WHjeEfWOiYiISOwojFUikOotibRf\n88ZEREQkhhTGKpGhJZFERESkDiiMVeLr9SnVMyYiIiKxozBWiQx/aJhSPWMiIiISOwpjlQiFMQ1T\nioiISCwpjFUixZdAclKChilFREQkphTGKmFmZPh96hkTERGRmFIYq0IgNVmlLURERCSmwgpjZnah\nma02s2wzu6+C/RPNbKmZLTOzuWZ2cnD7SWa2uMyffDO7O7jvATPbVmbfRdH9apELqGdMREREYiyp\nugPMLBF4HBgNbAXmm9nbzrmvyhy2ARjhnNtvZmOAp4ChzrnVwKAy19kGTCtz3iPOuYej81WiL+D3\nsWHPoXg3Q0RERBqxcHrGhgDZzrn1zrmjwMvA2LIHOOfmOuf2Bz9+DmRWcJ1zgXXOuU2RNLguZfiT\nVdpCREREYiqcMNYJ2FLm89bgtsrcArxXwfargZfKbbsjOLz5rJllVHQxM7vVzBaY2YKcnJwwmhs9\nAX8yeYeLcM7V6X1FRESk6YjqBH4zG4UXxu4ttz0ZuAx4tczmJ4FueMOYO4A/VnRN59xTzrnBzrnB\nbdu2jWZzqxXw+zhaUsrhoyV1el8RERFpOsIJY9uArDKfM4PbvsHMBgLPAGOdc3vL7R4DLHLO7Qpt\ncM7tcs6VOOdKgafxhkPrldD6lHqiUkRERGIlnDA2H+hpZl2DPVxXA2+XPcDMOgNvANc559ZUcI1r\nKDdEaWYdyny8HFhek4bXhfRUVeEXERGR2Kr2aUrnXLGZ3Q58ACQCzzrnVpjZ5OD+qcAvgNbAE2YG\nUOycGwxgZs3xnsT8XrlL/8HMBgEO2FjB/rjLOLZYuMKYiIiIxEa1YQzAOTcdmF5u29Qy7ycBkyo5\n9xBeUCu//boatTQOMpoHe8aOaJhSREREYkMV+KsQSA3NGVPPmIiIiMSGwlgV0kPDlIfUMyYiIiKx\noTBWhWZJifiTE8k9op4xERERiQ2FsWp4VfjVMyYiIiKxoTBWjYDfR57mjImIiEiMKIxVI+D3qWdM\nREREYkZhrBoBf7LqjImIiEjMKIxVI5Dq0wR+ERERiRmFsWpk+JPJPXyU0lIX76aIiIhII6QwVo2A\n30epgwOFxfFuioiIiDRCCmPVCPhDi4VrEr+IiIhEn8JYNbRYuIiIiMSSwlg1Av7Q+pTqGRMREZHo\nUxirxtfDlOoZExERkehTGKtGhuaMiYiISAwpjFUjLSUJgP3qGRMREZEYUBirRlJiAmkpSeSp8KuI\niIjEgMJYGAL+ZE3gFxERkZhQGAtDht+nYUoRERGJCYWxMKT7k8lTz5iIiIjEgMJYGNQzJiIiIrGi\nMBaG0GLhIiIiItGmMBaG9FQf+QXFFJeUxrspIiIi0sgojIUhtD6lyluIiIhItCmMheHYkkgKYyIi\nIhJlCmNhCC0WrnljIiIiEm0KY2HI0GLhIiIiEiMKY2EI9YypvIWIiIhEm8JYGI7NGdMwpYiIiESZ\nwlgYWjZLIsE0TCkiIiLRpzAWhoQE02LhIiIiEhMKY2EK+H0qbSEiIiJRpzAWpkCqT3PGREREJOoU\nxsKU4U9m/yH1jImIiEh0KYyFKd3v03JIIiIiEnUKY2HK0AR+ERERiQGFsTBl+H0cPlpCYXFJvJsi\nIiIijYjCWJjSg4Vf81RrTERERKJIYSxMGVoSSURERGJAYSxMgVQtiSQiIiLRpzAWJi0WLiIiIrGg\nMBamjObBOWNH1DMmIiIi0aMwFqZAqnrGREREJPoUxsLkT04kOTFBtcZEREQkqsIKY2Z2oZmtNrNs\nM7uvgv0TzWypmS0zs7lmdnKZfRuD2xeb2YIy21uZ2Ydmtjb4mhGdrxQbZuZV4VfPmIiIiERRtWHM\nzBKBx4ExQF/gGjPrW+6wDcAI59wA4NfAU+X2j3LODXLODS6z7T7gI+dcT+Cj4Od6LcPvU8+YiIiI\nRFU4PWNDgGzn3Hrn3FHgZWBs2QOcc3Odc/uDHz8HMsO47ljgheD7F4Bx4TU5fgKpyeSqZ0xERESi\nKJww1gnYUubz1uC2ytwCvFfmswNmmNlCM7u1zPZ2zrkdwfc7gXYVXczMbjWzBWa2ICcnJ4zmxk7A\n71MYExERkaiK6gR+MxuFF8buLbP5LOfcILxhztvM7Jzy5znnHF5oO45z7inn3GDn3OC2bdtGs7k1\npsXCRUREJNrCCWPbgKwynzOD277BzAYCzwBjnXN7Q9udc9uCr7uBaXjDngC7zKxD8NwOwO7afIG6\nFPD7yD1ShJcdRURERCIXThibD/Q0s65mlgxcDbxd9gAz6wy8AVznnFtTZntzM2sZeg+cDywP7n4b\nuCH4/gbgrUi+SF0I+JM5WlzKkaKSeDdFREREGomk6g5wzhWb2e3AB0Ai8KxzboWZTQ7unwr8AmgN\nPGFmAMXBJyfbAdOC25KAF51z7wcv/SDwipndAmwCvh3VbxYDoSWRcg8X4U+u9kcnIiIiUq2wEoVz\nbjowvdy2qWXeTwImVXDeeuDk8tuD+/YC59aksfGWcWx9yqN0DKTGuTUiIiLSGKgCfw0E/MH1KfVE\npYiIiESJwlgNBPxan1JERESiS2GsBjKCPWMqbyEiIiLRojBWA+mpXs9Y3hH1jImIiEh0KIzVQIov\nkVRfIvsPqWdMREREokNhrIYygoVfRURERKJBYayG0v3J5GrOmIiIiESJwlgNZfh9eppSREREokZh\nrIYCfp96xkRERCRqFMZqKOBPJlc9YyIiIhIlCmM1FJrA75yLd1NERESkEVAYq6FAajIlpY4DhcXx\nboqIiIg0AgpjNRRaEin3kIYqRUREJHIKYzUUWiw894gm8YuIiEjkFMZqKEOLhYuIiEgUKYzV0LGe\nMZW3EBERkShQGKuhY3PG1DMmIiIiUaAwVkOB1NAwpXrGREREJHIKYzWUlJhAy2ZJ6hkTERGRqFAY\nq4VAcy2JJCIiItGhMFYLgdRkco+oZ0xEREQipzBWCwG/T6UtREREJCoUxmohw5+sYUoRERGJCoWx\nWgj4fZrALyIiIlGhMFYLAX8y+QVFlJS6eDdFREREGjiFsVoIpPpwDvI1iV9EREQipDBWCxnNVfhV\nREREokNhrBZC61PqiUoRERGJlMJYLYSWRMo7op4xERERiYzCWC1khHrGDqlnTERERCKjMFYLAb/X\nM6Yq/CIiIhIphbFaSEvxkWCo8KuIiIhETGGsFhISjPRUn56mFBERkYgpjNVSwJ+sKvwiIiISMYWx\nWtKSSCIiIhINCmO1FEj1kavSFiIiIhIhhbFayvAnq7SFiIiIRExhrJa8OWPqGRMREZHIKIzVUsDv\n49DREo4Wl8a7KSIiItKAKYzVUsaxwq/qHRMREZHaUxirpfTgkkh5eqJSREREIqAwVkuhnrH9CmMi\nIiISAYWxWjq2WLgm8YuIiEgEFMZqKT3V6xnTMKWIiIhEIqwwZmYXmtlqM8s2s/sq2D/RzJaa2TIz\nm2tmJwe3Z5nZTDP7ysxWmNldZc55wMy2mdni4J+Love1Yi+juXrGREREJHJJ1R1gZonA48BoYCsw\n38zeds59VeawDcAI59x+MxsDPAUMBYqBe5xzi8ysJbDQzD4sc+4jzrmHo/mF6krz5ESSEozcI+oZ\nExERkdoLp2dsCJDtnFvvnDsKvAyMLXuAc26uc25/8OPnQGZw+w7n3KLg+wPASqBTtBofT2amwq8i\nIiISsXDCWCdgS5nPW6k6UN0CvFd+o5l1AU4Bviiz+Y7g8OazZpZR0cXM7FYzW2BmC3JycsJobt3R\nYuEiIiISqahO4DezUXhh7N5y21sArwN3O+fyg5ufBLoBg4AdwB8ruqZz7inn3GDn3OC2bdtGs7kR\ny/D7NGdMREREIhJOGNsGZJX5nBnc9g1mNhB4BhjrnNtbZrsPL4j90zn3Rmi7c26Xc67EOVcKPI03\nHNqgeMOU6hkTERGR2gsnjM0HeppZVzNLBq4G3i57gJl1Bt4ArnPOrSmz3YC/ASudc38qd06HMh8v\nB5bX7ivETyBVw5QiIiISmWqfpnTOFZvZ7cAHQCLwrHNuhZlNDu6fCvwCaA084eUvip1zg4HhwHXA\nMjNbHLzkz5xz04E/mNkgwAEbge9F9ZvVgYzmyRqmFBERkYhUG8YAguFperltU8u8nwRMquC8TwCr\n5JrX1ail9VB6qo/C4lIKikpI8SXGuzkiIiLSAKkCfwS0JJKIiIhESmEsAscWCz+keWMiIiJSOwpj\nEUgPhrHcI+oZExERkdpRGItAaJhST1SKiIhIbSmMRSAQ6hlTGBMREZFaUhiLgCbwi4iISKQUxiKQ\n4kskxZegxcJFRESk1hTGIhRI1ZJIIiIiUnsKYxEK+H3sVxgTERGRWlIYi1DA7yNPpS1ERESklhTG\nIpThT1bPmIiIiNSawliEAv5kTeAXERGRWlMYi1DA7yP3cBHOuXg3RURERBoghbEIZfh9FJc6DhYW\nx7spIiIi0gApjEUokKolkURERKT2FMYipCWRREREJBIKYxHKaK4lkURERKT2FMYiFEgN9owdUc+Y\niIiI1JzCWIQC/tCcMfWMiYiISM0pjJV1cDcse61Gp6Snas6YiIiI1J7CWFmLX4TXb4F968M+JTkp\ngRbNkjRnTERERGpFYays/ld6r8tfr9Fp6ak+9YyJiIhIrSiMlRXIgqxhsKxmYSyjuU9zxkRERKRW\nFMbKGzAeclbCrhVhn6LFwkVERKS2FMbK6zsOLLFGE/nTU33kqbSFiIiI1ILCWHkt2kK3Ed68sTAX\n//Z6xjRMKSIiIjWnMFaR/uMhdxNsXRDW4QG/1zNWUhpeeBMREREJURirSJ9LILEZLA9vqDLgT8Y5\nOFCgoUoRERGpGYWxiqSkQ6/zYcU0KC2p9vCM4GLhmsQvIiIiNaUwVpn+4+HgLtg4p9pDA/5QFX7N\nGxMREZGaURirTK8LILllWE9Vfr0+pXrGREREpGYUxirjS4XeF8PKt6G4sMpDA6mhYUr1jImIiEjN\nKIxVZcB4KMiD7I+qPCxDPWMiIiJSSwpjVek2ElJbVftUZVqqDzPNGRMREZGaUxirSqIP+o2DVdOh\n8GDlhyUYnVv5+XzDvjpsnIiIiDQGCmPV6T8eio/A6veqPOy6YScyb8M+lmzJraOGiYiISGOgMFad\nzmdAWqdqhyonnJ5Fy2ZJPD1nfR01TERERBoDhbHqJCRAv8u9SfyHKx+GbJni4ztDO/Pe8p1s2Xe4\nDhsoIiIiDZnCWDgGjIfSIq/MRRVuHN4FA577dGOdNEtEREQaPoWxcHQYBK26V1sAtkN6Kpee3JF/\nzd9M3hGVuRAREZHqKYyFw8zrHdv4CeTvqPLQSWd35dDREl6at7mOGiciIiINmcJYuPqPB5y3eHgV\n+nVMZ3iP1jz36QaOFpeGd23nIm+fiIiINEgKY+Fq2wvaD6j2qUqASWd3Y1d+If9eur366376KPyx\nNxzVpH8REZGmSGGsJvqPh20LYV/V5StG9mpLzxNa8NTs9biqer02fgIzHoCDO73rioiISJMTVhgz\nswvNbLWZZZvZfRXsn2hmS81smZnNNbOTqzvXzFqZ2Ydmtjb4mhGdrxRD/a/0Xpe/XuVhZsZ3z+nG\nqp0H+DR7b8UHHcyB1ydBepb3ecvnUWyoiIiINBTVhjEzSwQeB8YAfYFrzKxvucM2ACOccwOAXwNP\nhXHufcBHzrmewEfBz/VbIMsrArus6jAGMHZQR9q2bMZTFRWBLS2Fabd6dcsm/B+07Q2bv4hBg0VE\nRKS+C6dnbAiQ7Zxb75w7CrwMjC17gHNurnNuf/Dj50BmGOeOBV4Ivn8BGFf7r1GH+l8JOSth14oq\nD2uWlMiNZ3Zh9pocVu3M/+bOTx+Bdf+FMQ9Ch4GQNRS2zvNCmoiIiDQp4YSxTsCWMp+3BrdV5hYg\ntJBjVee2c86F6kTsBNqF0Zb463c5WGK1NccAJg7tTKovkWfmbPh646a58N/fQL8r4LSbvG2dh0FB\nHuxZHaNGi4iISH0V1Qn8ZjYKL4zdW5PznDfLvcKZ7mZ2q5ktMLMFOTk5UWhlhJq3gW4jvXlj1ZSk\nCPiT+fbgTN5avI1d+QVwaA+8dgtkdIFLH/Xql4HXMwawWfPGREREmppwwtg2IKvM58zgtm8ws4HA\nM8BY59zeMM7dZWYdgud2AHZXdHPn3FPOucHOucFt27YNo7l1YMB4yN0EWxdUe+jNZ3WlpNTxwqfr\nYdr34PAeuOp5SEn7+qBW3cDfBrZo3piIiEhTE04Ymw/0NLOuZpYMXA18Y5FGM+sMvAFc55xbE+a5\nbwM3BN/fALxV+69Rx3pfAonNwqo5dmLr5lzQrz2+L/4M2TPgwt9Dh5O/eZCZN1SpMCYiItLkVBvG\nnHPFwO3AB8BK4BXn3Aozm2xmk4OH/QJoDTxhZovNbEFV5wbPeRAYbWZrgfOCnxuGlDTodb5Xjb+0\npNrD7z5pH3e4l9nQbjQMvqXig7KGevXLDlbYQSgiIiKNVFI4BznnpgPTy22bWub9JGBSuOcGt+8F\nzq1JY+uV/uNh5TuwcY43h6wyh/dx0py72JHUju/n38C/Sx1JiXb8cZ2Hea9bvoA+l8aixSIiIlIP\nqQJ/bfW6AJJbwrJXKz+mtBSmTYZDOWwY+Tir9ifwwYpdFR/b4WRv6FOT+EVERJoUhbHa8qVC74vh\nq3eguLDiYz77M6z9AC74HUOHf4sTW/t5ak4lSyQlNYOOp8CWebFtt4iIiNQrCmORGDAeCvO8ifnl\nbf4CZvwS+o6F0yeRmGBMOqsrS7bksmDT/uOPB+g8FHYshqKC2LZbRERE6g2FsUh0GwmprY4vAHt4\nH7x2s7eL6ztqAAAgAElEQVR80mV/PlZPbPxpWWT4fTw1u5KFxrOGQclR2P5lTJstIiIi9YfCWCQS\nfdBvHKx+DwoPetucgze/Dwd3wfjnICX92OGpyYlcO+xEZqzcxfqcg8dfL2uI96pFw0VERJoMhbFI\n9R8PxUe8QAbw2V9gzftwwW+h06nHHX79GV3wJSTwt082HLeP5m2gdQ/NGxMREWlCFMYi1fkMSOvk\nFYDdMh9mPOCVphhya4WHt23ZjMtP6cRrC7ey92AFE/+zgsVfq1lqSURERBoHhbFIJSRA/yu8Sfyv\n3ghpHeGyv3y97mQFJp3dlcLiUv7v883H78waAof3wt7s2LVZRERE6g2FsWjoPx5Ki715Ylc9D6mB\nKg/v2a4l3+p9An//bCMFReUq+IeKv6remIiISJOgMBYNHU6GAVfBJY9Ap9PCOmXS2V3Ze+go074s\nt+Z6656QmqFJ/CIiIk2Ewlg0mMGVz8Cp14V9yhndWtO/UxpPz1lPaWmZ+WEJCd46lZrELyIi0iQo\njMWJmfHds7uxPucQM1eXWxw8awjsWePVKxMREZFGTWEsji4a0IGO6Sn8dXa5JZKyyiwaLiIiIo2a\nwlgc+RIT+O453Zi3YR9PzylTlb/TqZDg0yR+ERGRJiAp3g1o6m44owsLNu3nd9NX0S4thbGDOnmL\nkHc4WfPGREREmgD1jMVZQoLxx6tOZkjXVvz41SXMXbfH25E1FLYvguKj8W2giEhjUVIMRQXxboXI\ncRTG6oEUXyJPXzeYLq2b871/LGT1zgPQeSgUF8COJfFunohI4/Dx7+Bv58W7FSLHURirJ9L9Pp6/\neQipvkRufG4euwKDvB2qNyYiEh07l8GuFVBSFO+WiHyDwlg90imQyvM3DeFAQTE3vLKJ0kAXPVEp\nIhIt+dvBlcKBHfFuicg3KIzVM307pjH12tPI3n2QTwu74zZr0XARkajI3+695m2NbztEylEYq4fO\n6tmGP4wfyPv5J2KHdlO6d0O8myQi0rAVHYEjwULauVvi2xaRchTG6qkrTs2k/7DzAZj+3ptxbo2I\nSANXdmgyT2FM6heFsXrs6ovPpyChOXmrP+Efn22Md3NERBqu/LJhTMOUUr8ojNVjlpBIctdhjEhd\nxy/eXsEHK3bGu0kiIg1TaL5YcguFMal3FMbquYTOw+hUtIkzOyZx50tfsnDT/ng3SUSk4TkQDGOd\nTlUYk3pHYay+yxqK4XhyRAkd0lOY9MJ81uccjHerREQalvztkNwSTujrzRnTU+pSjyiM1XedTgNL\nJG3PIl64eQgJZtzw3DxyDhTGu2UiIg1H/nZI6wDpmXD0IBTkxrtFIscojNV3zVpA+/6w+XNObN2c\nv914OjkHCrnlhfkcKiyOd+tERBqGAzsgraMXxkBDlVKvKIw1BFnDYNtCKCliUFaAx79zKsu35XH7\ni4soLimNd+tEROq//O3QsiOkd/Y+K4xJPaIw1hB0HgpFh7111YBz+7Tj1+P6M3N1Dg/9Z3WcGyci\nUs+VlsCBnV8PU4LCmNQrCmMNQdZQ73XLvGObJg49kQmDs3hmzgZW7siPU8NERBqAg7vBlXjDlM3b\nQmKyCr9KvaIw1hCkZ0JaJmz5/Bub7xvTm/RUHz+ftozSUj0ZJCJSoVBZi5YdISEB0jqpZ0zqFYWx\nhqLzUCi3aHhG82R+dlEfFm3O5ZUF+lueiEiFQtX30zp6r+mZWp9S6hWFsYYia5j3t7tyXetXntqJ\noV1b8fv3VrH3oMpdiIgcJ1R9PxTGAp3VMyb1isJYQ5E1xHstM28MwMz47eX9OXy0mN9NXxWHhomI\n1HMHtkOCD/xtvM/pmV6pi5Ki+LZLJEhhrKFo1x98zWHz58ft6nFCS757djdeX7SVz9btjUPjRETq\nsfzt0LKDN18Mgk9Uuq97zETiTGGsoUhMgszBx03iD7njWz3JzEjl/reWc7RYtcdERI4JVd8PUXkL\nqWcUxhqSzsNg1wooPHDcrtTkRH49tj/Zuw/y9Jz1cWiciEg9Faq+H5Ke5b0qjEk9oTDWkGQNAVcK\nW+dXuHtU7xMY0789j320ls17D9dx40RE6iHnvq6+H5LWyXvN2xyfNomUozDWkGSeDthxk/jL+sWl\nfUlKMH7x9nKcU+0xEWniCvK8FUzKDlMm+73J/OoZk3pCYawhSUmHdv0qnMQf0iE9lR+O7sXHq3N4\nf/nOOmyciEg9VL6sRUh6psKY1BsKYw1N1lDYusBba60SN57ZhT4d0njgnRUcLCyuw8aJiNQzZavv\nl6UwJvWIwlhDkzUUjh7wJvJXIikxgd9d3p/dBwr503/W1GHjRETqmfLV90PSs7wwpukcUg8ojDU0\nnUOLhn9R5WGndM7gO0M68/zcDSzfllcHDRMRqYdCw5QtO3xze3omHD0IBbl13yaRchTGGprAidCi\nfbVhDOAnF/SmVfNkfv7mckq0kLiINEUHtnuT9ZOSv7k9VGtMa1RKPRBWGDOzC81stZllm9l9Fezv\nbWafmVmhmf24zPaTzGxxmT/5ZnZ3cN8DZratzL6Love1GjGzrxcNr0a638fPL+7Dki25vDRPj3CL\nSBOUv/34IUqAgGqNSf1RbRgzs0TgcWAM0Be4xsz6ljtsH3An8HDZjc651c65Qc65QcBpwGFgWplD\nHgntd85Nj+B7NC1ZQ736OGEs5TFuUCfO6NaaKe+vIueAFhIXkSYmf0fFYUyFX6UeCadnbAiQ7Zxb\n75w7CrwMjC17gHNut3NuPlDVqqvnAuucc5tq3VrxZA3zXsMYqjQzfnN5fwqLSvntu1/FuGEiIvXM\ngUp6xvxtILEZ5GmYUuIvnDDWCSj7b+vW4Laauhp4qdy2O8xsqZk9a2YZFZ1kZrea2QIzW5CTk1OL\n2zZCHQZCUmpYQ5UA3du2YPKIbry5eDtzs/fEuHEiIvVEUQEc3nt8WQvwFg1P76SeMakX6mQCv5kl\nA5cBr5bZ/CTQDRgE7AD+WNG5zrmnnHODnXOD27ZtG/O2NgiJPuh0aqWLhlfkB6N6cGJrP//75nIK\niyuvUSYi0mgcCJW16FDx/vRM9YxJvRBOGNsGZJX5nBncVhNjgEXOuV2hDc65Xc65EudcKfA03nCo\nhCtrKOxYCkcPhXV4ii+RX43tz/o9h/jrLC0kLiJNQGXV90PSO6tnTOqFcMLYfKCnmXUN9nBdDbxd\nw/tcQ7khSjMr+1eVy4HlNbxm09Z5GLgS2LYo7FNG9GrLxQM78JeZ2WzcE16IExFpsEI9YxUNU4LX\nM3ZgJxQfrbs2iVSg2jDmnCsGbgc+AFYCrzjnVpjZZDObDGBm7c1sK/Aj4H/NbKuZpQX3NQdGA2+U\nu/QfzGyZmS0FRgE/jNq3agoyT/deN8yu0Wm/uKQvyYkJ/PzNZRSVlMagYSIi9US1PWOZgPt6ySSR\nOAlrzphzbrpzrpdzrrtz7rfBbVOdc1OD73c65zKdc2nOuUDwfX5w3yHnXGvnXF65a17nnBvgnBvo\nnLvMObcj2l+uUfO3gh6jYd5TcGR/2Ke1S0vhpxf15tPsvVzz1Ofszi+IYSNFROIofzskt4CUtIr3\nhwq/aqhS4kwV+Buy8x6AgjyY86canTZx6Ik8ds0prNiez8V//oR5G/bFpHkiInF1YPvxyyCVpVpj\nUk8ojDVk7fvDyVfDF3+t8ZIel53ckTdvG06LZkl85+nPefaTDTgtmCsijUll1fdD0oNVmrQkksSZ\nwlhDN+rn3uvM39X41JPat+St24cz8qQT+NW/v+Lufy3m8NHiKDdQRCROKqu+H+JLheZtVd5C4k5h\nrKELZMHQW2HJS7Cz5g+kpqX4eOq60/ifC07i7SXbufzxuWzQk5Yi0tCVlsDBnVWHMQjWGtMwpcSX\nwlhjcNaPvAmqMx6o1ekJCcZto3rwwk1D2H2ggMv+/AkffrWr+hNFROqrQzlQWlz1nDFQGJN6QWGs\nMfC3grPvgewPa1zqoqxzerXlnTvOokub5nz37wt4+IPVlJRqHpmINEDVlbUISc/ywpjmzEocKYw1\nFkO+B2mZ8OEvoLT29cMyM/y8OvkMJgzO4i8zs7nxuXnsP6SCiCLSwIQdxjKh6FCNSgSJRJvCWGPh\nS4Fv/Ry2fwlfTYvoUim+RKaMH8jvrxjAF+v3ccmfP2HZ1rzqTxQRqS+qq74fcqy8hSbxS/wojDUm\nAyfACf3go19FZXmPa4Z05tXJZ+Cc48qpc/nX/M1RaKSISB3I3w4JSd7TklVR4VepBxTGGpOERBj9\nS9i/ERY+F5VLnpwV4J07zuL0Lhnc+/oyfvrGUgqKSqJybRGRmMkPFnxNqObXnAq/Sj2gMNbY9DgP\nupwNs6ZAQX5ULtm6RTP+fvNQfjCyOy/N28KEv37GjrwjUbm2iEhMVFd9P6R5G0hspmFKiSuFscbG\nDEb/Cg7vhbmPRe2yiQnGTy7szdRrTyN790Eu/fMnfLF+b9SuLyISVdVV3w8xU3kLiTuFscao06nQ\n7wr47HGvAnUUXdi/PW/dPpy0VB8Tn/mC5z7VMkoiUs84V331/bIUxiTOFMYaq3Pvh5Ii+Pj3Ub90\njxNa8uZt3jJKv3znK+55ZYnmkYlI/VGQ55WrCDeMBbK0PqXElcJYY9WqGwy+Gb78B+SsjvrlQ8so\n/fC8Xrzx5TaufHIuW/Ydjvp9RERq7FhZizDmjIE3if/gTigujF2bRKqgMNaYjfgJ+JrDjF/G5PIJ\nCcZd5/XkbzcMZvPew1z2l0/4NHtPTO4lIhK2YwVfO4V3fKi8Reg8kTqmMNaYNW8DZ90Fq9+FzZ/H\n7Dbn9mnHW7cPp02LZlz3ty94evZ6zSMTkfg5FsbC7RlTrTGJL4Wxxm7YD6BFe/jP/TFde61b2xZM\nu204F/Rrz2+nr+TOlxdz+GhxzO4nIlKp2gxTgsKYxI3CWGOX3BxG/RS2zoNV/47prVo0S+KJiafy\nkwtP4t9Lt3PFE3PZtPdQTO8pInKc/G3gbwNJzcI7PjScqTAmcaIw1hQMuhba9PLmjpXEtrfKzPjB\nyB48f9MQduQVcOmfP+Hj1btjek8RkW/I3xH+ECV4a/s2PwHytOSbxIfCWFOQmATnPQB718KXf6+T\nW47o1ZZ3bj+LjoFUbnp+Po/PzNY8MhGpGwe2hz95P0S1xiSOFMaaipMugqxh8PGDcLRuhg47t/bz\nxg/O5NKBHXnog9V89+8L2bxX5S9EJMbyw1wKqSyFMYkjhbGmIrRM0sFdXmX+OuJPTuLRqwfxvxf3\nYc7aHL71x4/52bRlWttSRGKjqMBbDi7cgq8h6VleGFMPvsSBwlhT0nko9L4EPn0UDubU2W3NjEln\nd2POT0bxnaGdeXXBFkY89DG/fGcFOQdUZFFEoij0JGWNw1gmFB2GI/uj3yaRaiiMNTXnPQBFR2D2\nH+r81iekpfCrsf2Z+eORXD6oE3//bBPn/GEmU95fRe7ho3XeHhFphGpa1iIkECpvoWWRpO4pjDU1\nbXrCaTfA/L/Bynfi0oTMDD9Txg/kwx+ew/n92jF11jrOnjKTR2es5UBBUVzaJCKNRE2r74eECr9q\njUqJA4Wxpmj0r6DTqfDqTbD6vbg1o1vbFjx69Sm8f9c5DO/RhkdmrOHsP8xk6qx1KhhbEyVFcGhv\nvFshUj/UtPp+iAq/ShwpjDVFzVrCta9D+/7wyvWwdkZcm3NS+5ZMve403rn9LAZlBXjwvVWc84eP\nef7TDRQWl8S1bQ3C7IfhsVOg8EC8WyISfwd2eGvyNkur2Xn+1pCUomFKiQuFsaYqJR2umwZte8PL\n34F1/413ixiQmc7zNw3h1cln0L1tcx545ytGPfQxL83bzKa9h9iVX0De4SIKikpUs6ysVe9CYR6s\njO0KCyINQv42b/K+Wc3OM1N5C4mbpHg3QOIoNQOufwteuBReugYmvgpdz4l3qzi9SytevnUYc9ft\n5aEPVvPTN5Ydd4wZNEtKIMWXSEpSIqnJiV9/9n29PeD30adDGv07pdGnQxr+5Eb2r/yBnbAr+PNZ\n+i8YdE182yMSbzWtvl+WwpjESSP7zSQ15m/lBbLnL4YXJ3jDlyeeGe9WYWYM79GGM7u35osN+9i2\n/wgFxSUUFJVSUFRCYVEJR4q+/lxQHHwtKqGwqJS9B49SUFTCnoOFvDx/S/Ca0K1Nc/p1TKdfxzT6\nd/JeA/7kOH/bCIR6NHteANkfeuGsZfv4tkkkng7sgC5n1e7c9CxY+5/otkckDApjAs3bwPVve4Hs\nn1fBtW94NcnqATNjWLfWtT7fOcfO/AKWb8tnxfY8lm/LZ8HGfby9ZPuxYzoFUunXMY1+HdPp38l7\nbZfWDKvpMEc8ZM/w1tQ7/9ew9gNY/jqccVu8WyUSH6WlXhiraVmLkPQsrzB2cWH4i4yLRIHCmHha\ntoMb3gkGsvFw3ZuQeVq8WxUxM6NDeiod0lMZ3bfdse37Dh1lxfY8VmzPZ/m2PL7ans9/vtp1bH+b\nFslcfkonfjCyBxnN62nPWWkJrJsJPc+HtidBh0Gw9BWFMWm6DuVAaXHNC76GhMpb5G+DVt2i1y6R\naiiMydfSOgQD2UXwf5d7vWUdB8W7VTHRqnkyZ/dsy9k92x7bdrCwmJU78lmxLY/5G/fzt0828PK8\nLXxvRDduPqtr/Ztvtn0xHNkHPc7zPg+cAB/8FHLWQNte8W2bSDzkb/NeIw1jeVsVxqRO6WlK+ab0\nTl4ga5YOfx8LO4+fPN9YtWiWxOldWnHj8K48PvFU3r/7HIZ2a83D/1nDiIc+5h+fb6KopDTezfxa\n9gzAoPso73P/K8ESYNkrcW2WSNzUtvp+SNkwJlKHFMbkeIHOcMPbkNzcC2S7vor8mgdzYO+6yK9T\nh3q1a8kzNwzmtcln0KW1n/vfXM7oP83inSXbKS2tB6U11n3k9Vw2b+N9btkOuo30hipV+kOaotpW\n3w9RGJM4URiTirXq6vWQJfjg75dBzuqanX/0kFdM9oOfw5NnwcM94PGhsG99bNobQ4O7tOKV753B\nszcOJsWXyB0vfcllj3/C7DU58at3dmQ/bJ3/9RBlyIBvQ+4m2DIvPu0SCde2hbD4xeheM387JCRB\n87bVH1uRpGbQoh3kbo5uu0SqoTAmlWvdHW78N2BeLbI92ZUfW1IMW+bDrIfguYvhwRPhn1fCvKfA\nnwEjf+b9T3LWQ3XW/GgyM77Vux3v3nk2j0w4mdzDRVz/7DwmPvMFS7bk1n2D1s8CV3p8GOtzCSSl\nejXHROqzGb+Et++EoiPRu+aBHdCiPSRE8KtNtcYkDurZjGSpd9r0/PopyxcuhZve9Sa2Ogd71sL6\nj70/G+dAYb53TvuBcMYPvCGzrGGQ7Pe2F+bD50/A2fdAmx7x+T4RSkwwLj8lk4sGdODFLzbzl/9m\nM/bxT7loQHvuOf8kurdtUTcNyZ7hzevrNPib25u1hN4XwYppMGYKJPrqpj0iNVGQB5s+9Z583LoA\nup4dneuGqu9HIj0zOlMzRGpAYUyqd0LvYKX+S+D5S70q/es/hgPB+RmBE6Hf5V746joCmldSF2z4\n3bDgWZg1Ba58uo4aHxvNkhK5aXhXrhqcxdOz1/PMnPV8sGIX3x6cyV3n9qJ9ekrsbu4cZH8E3UZA\nYgX/CQ+c4NUby/4ITrowdu0Qqa11//WCGMDmz6IYxnZAu76RXSM9C9b8x/vvrCHUGpRGQcOUEp72\n/b1AVlIIa973isJe+ijcuRjuXgqXPQb9r6g8iAG0aAtDboVlr8LuVXXX9hhq0SyJH47uxayfjOK6\nYSfy2sKtjHr4Y6bOWhe7Jy9zVnlBuPwQZUj3b3mLHmuoUuqr1e97y7G17Q2b5kbnms55c8ZqO3k/\nJD0Tio/A4X3RaZdIGBTGJHwdToZ7VsP/rIOrnofTbvQm+tfE8Lu8pzRnPRiLFsZNmxbNeOCyfvz3\nnpGc3bMND763ikse+4T5G2PwP/TsGd5rj3Mr3p/o83oqV0+Hgvzo318kEqUl3pJDPc/3li3aOt+b\ncxqpwnwoOlT7shYh6Vnea96WyNskEiaFMamZhMTIJsf6W8Gw73tzmnatiF676omsVn6eun4wT18/\nmIOFxVw19TPufW0p+w8djd5Nsmd4PQqhx/ArMnACFBfAqn9H774i0bB1vlesuNeF0PkMOHoQdi6N\n/Lr5wRpj0ZgzBgpjUqcUxqTunXEbNEuDj38f75bEhnOM7tuOD390Dt8b0Y3XF23lW3/8mFcXbIm8\nFMbRQ96wTvdKesVCMk+HjC5ezTGpvaOHGlx9vHpvzfvek9U9zoUTz/S2bf4s8utGWn0/5FjPmJ6o\nlLoTVhgzswvNbLWZZZvZfRXs721mn5lZoZn9uNy+jWa2zMwWm9mCMttbmdmHZrY2+JoR+deRBiE1\nwwtkK9+BHUvi3Zro+u9v4cnhcPQw/uQkfjqmD/++8yy6t23B/7y2lAl//Zw1uw7U/vobP4WSo5UP\nUYaYeTXHNsyCAztrf7+mzDl45Xp4aiQUFcS7NY3H6ve9EJaS7gWnwInRmTcWafX9EH8rrzyMwpjU\noWrDmJklAo8DY4C+wDVmVv5xlX3AncDDlVxmlHNukHOu7HP49wEfOed6Ah8FP0tTMez7kBKAmY2o\nd2ze0zD7D7B7hfc0Y1Dv9mm88r0zmHLlANbsPsBFj85hyvurOHK0pOb3WPeR94vixOHVHzvw214t\nsjJtkRpY9po3JFyY75VhkMjt3wg5K70hypATz4TNn0e+akR+lMKYWbDWmIYppe6E0zM2BMh2zq13\nzh0FXgbGlj3AObfbOTcfKKrBvccCLwTfvwCMq8G50tClpMOZd8Ca97xK3A3d6vfgvZ94v2Ta9oH5\nT3/jl0tCgjHh9M589KMRjDulE09+vI7Rj8ziv6t21ew+2TOgy3DwhVE6o01P6HiKnqqsjcP74P37\noMMgSEqBtR/Gu0WNw5oPvNeyYazzGXB4j1e3MBL527yniMP5b6M6KvwqdSycMNYJKPtXhK3BbeFy\nwAwzW2hmt5bZ3s45F/yrDDuBdhWdbGa3mtkCM1uQk5NTg9tKvTf0e5DaCmb+Lt4ticy2RfDazV6x\n2/HPwpBJ3vDr1gXHHdq6RTMevupkXr51GCm+RG5+fgGT/7GQHXlhVCHfvxH2Zlde0qIiA77ttaWm\ny1k1df/5XyjIhbGPe0/8ZSuMRcWa96F1T291j5Bj88YiHKo8sANaRjhfLCSQpTAmdaouJvCf5Zwb\nhDfMeZuZnVP+AOfNaq6wj9o595RzbrBzbnDbtrVcb0zqp2YtvVIX2TNg8xfxbk3t7N8EL04Afxv4\nzite2Y6BEyC5pdc7Volh3Voz/c6z+cmFJ/Hxmt2c+8dZPDNnPcVV1SbL/sh7rUkY638lWIIm8tfE\n+lmw+J9ez237/l4Jhr3ZmsgfqcIDsPGT4wsRt+7hrSW5KcJJ/NGovh+SngUHd2muoNSZcMLYNiCr\nzOfM4LawOOe2BV93A9Pwhj0BdplZB4Dg6+5wrymNyJDvev8j/rgB9o4d2Q//vAqKC2Hiq9Ay2Lnb\nrCUMusYr33Gw8t7c5KQEfjCyBx/+cARDu7biN++u5Kbn53OosJKaS9kfQXpn75dXuFq281ZGWPbq\ncXNynHNMX7aDTXsPhX+9xq7oCPz7bsjoCiPu9baFwm+ovpvUzrqZ3sMnvcqFMTPoPCzynrH8HZAW\n4XyxkFB5i/ywf9WJRCScMDYf6GlmXc0sGbgaeDuci5tZczNrGXoPnA8sD+5+G7gh+P4G4K2aNFwa\nieTmcNYPg+tbNqBJ0sWF8K/rYN96uPr/vCWjyjp9kveL58u/V3uprFZ+nr3xdH53+QA+zd7Dd57+\nnL0HC8vd76j3ZGSPc2u+RMvACZC7CbbMO7apsLiEe15Zwg/+uYjRj8zmkQ/XUFBUiwcKGptZf/D+\nmV76/4Mv1dvWuju06q55Y5Fa8743VzRr2PH7Op8JuZshr5bhp7jQm3cWafX9kGO1xjRUKXWj2jDm\nnCsGbgc+AFYCrzjnVpjZZDObDGBm7c1sK/Aj4H/NbKuZpeHNA/vEzJYA84B3nXPvBy/9IDDazNYC\n5wU/S1M0+GZo0c6bOxbpE1V1wTl4+w5vcfRxT3hrdZbX9iRvnc75z4ZVXdzM+M7Qzvz1usGs2nmA\n8VM/Y8u+w18fsHWeVxyzJkOUIb0v9p7ADE7k33/oKNf9bR5vfLmN20f14MJ+7Xn0o7Wc/8hsZq5q\nwh3UO5fD3Mdg0ESvN7Gsnud7/7yPHq7oTKlOaak3eb/H6IrXUz3xDO+1tvXGolXWIkRhTOpYWHPG\nnHPTnXO9nHPdnXO/DW6b6pybGny/0zmX6ZxLc84Fgu/zg09gnhz80y90bvCcvc65c51zPZ1z5znn\ntBBYU+VLhbPvgU2fwIbZ8W5N9Wb+1gs23/pfr3xEZYZ8F/K3ej0CYRrdtx3/nDSUfYeOcsWTc/lq\ne3A5o+wZXqHMioJfdZq1hN4XwYppbNi1nyuenMviLbk8evUgfnzBSTx2zSm8OGkovkTjpufnc+vf\nF7B1fxMLHaUl8M6dXrmV839z/P6eo70VDTZ+Uvdtawy2LfR6rk4aU/H+dgMguUXt640dq74fpTCW\n1gkwhTGpM6rAL/XDqTd4/wOs771ji/4Bsx+CU6+Hs39c9bG9xkBaZpUT+SsyuEsrXp18BkkJxoS/\nfsZn6/Z688WyhkJKWu3aPXACHNnHI1OfJO9IES99dyhjB309pHNmjza8d9c53Hthb+as3cN5f5rF\nEx9nc7Q4Roud1zfzn/ECw4UPekU/yztxOPj8eqqytta8D5ZYxXqqSZA1pPY9Y8eq70dpmDKpmddb\nr1pjUkcUxqR+8KV4vWNbPvcKm9ZH2R/BO3d5SxFd/Kfq524lJsHgG735cDWsodSrXUte//6ZtEtP\n4Z5n/+Ot3df9W7Vu+rT8XuxzLRmX+CnTfnAmp514fOBITkrg+yO7M+OeEYzsdQJ/eH81Yx6dzdzs\nPbmmhKkAACAASURBVLW+b4OQtxU++pX3z3XA+IqP8aV4vZJr/1O//7JQX61536snllrFQiudz4Td\nX3k13moq2sOUoMKvUqcUxqT+OOU675Hy+tg7tnMZvHIDnNAHrnoeEn3hnXfqDZDg83peaqhjIJXX\nJp/B1a28IPfukX41voZzjj99uIYfvvYV85uPZBQLOLF51RP1OwVSmXrdaTx30+kUlzq+88wX3PHS\nl+zKb4SP+TsH797jrVRwSTUBu+for2u9Sfhyt8Cu5dDrgqqPC80b21KLMjf5O8DX3HtAIFpU+FXq\nkMKY1B9JyXDO/3jDRWv/E+/WfC1vG/zz297cq++8UrOhwhYnQL/LYfGLUHiwxrcO+JP5QedN5CUE\nuH1mMX/6cE3Yi40XFP2/9u47PKpqa+Dwb6cXQgKEBBISeu8QugUQBBEEkS4INrBe9eq1X/Wz61UR\nRUUQkN6xACoCIoK0hN4hIEgJJLRUUmd/f+yJhJZMkpnMBNb7PHlmcubMmT2HQ2bNLmvl8NTsrXy2\n4gADoqrQecATqOx02LvYpud3qhvC0qdv4ekutVm666RtudBKm93fm16bTi+bwur5qdXV3LrStVka\n5M6ZvNZ8sVzhLc0Xl6LMG0s6buaLFXalcX5ygzFX+2IorksSjAnX0myI+VBc+Y5r/BFMT4KZA0zC\nynvnQWAR5qS0ftjUNyxKWSKLBY+/VhLQ8Hb6tYzksxUHePm7nQUGRGdSMhj6zQZ+3HaC57vX5YN7\nmuBZtY05t4Voh4+nO093qcOyZ26hVbVyvL1kDz0/X0P04etgvc2Fc/DT81C5KbR5tOD9y1WF4LqS\n4qKw9i+F8jUKzo/n6QvhLYo2byw5zr5DlGB66bPTIe2MfY8rxFVIMCZci7unSbYZtw32LnFuW3Ky\nYO59EL8HBkwx2diLokorUyop+pvCB5hxWyHtDG61u/JhvyY81rEmszb+zWMzNl8zL9jBhBTu/nIt\nO44n8sWQFjzWsRZKKdNr0HiAWbGau/rMRlUr+DNpRCu+HtaS5PRs+o9bx5OztvD7vvjCT/LPSHGN\nQHvZ6+aDttdnV0+3cDW1u5qi4UXo5bwhZaaa663OHbb1WkW2gxNbCp9CJOmE/Sbv5wqy5jqXeWOi\nBEgwJlxP4wHmW/Tv75n8RM6gtcnEfmgl9Bpz7VVgtlDK9I7F7y78EEzsCkBBzc4opXi+ez1e79WA\nZXtOcd/EjSReyLpk97UHT3P3F3+SlpnNrJFtubPJZb0FTQaY+VE7FxThbSi6NazEsn/fwmMda/L7\n3nhGTI6m1TvLeW7eNlbutSEwO7Ac/lcTlr5S6Ne3q8N/wuYp0O4xCGtm+/Nq326S+ZaGFCyu4NDv\nkJNR8HyxXFXbgyUbjl9Z1/WaLBbTM2avtBa5JNeYKEESjAnX4+4Bt75oJv3usanYg31Zcswigi3T\nzRy2FsOKf8xG/UwOq0KmuSB2uRlG8w/+Z9P9Harz2aDmbDl6jgHj1nEy0UysnxdzlPsmbiS0rA/f\nPdaBFpFXWbkWXBvCmsOOoteq9PPy4Pnu9Yj5bxcmDo/itvohLN11kvu/jabl28v499ytrNhziozs\ny3ru9i+F2YNNvrT1X8A+2/Ov2VV2hlkVGxQJHV8q3HMj25l8WJLiwjb7fgbvsheLgRckog2gClen\nMu20CeDs3TMWaO0ZOy89Y8LxbOybF6KENepr8nn9/h7U7wVu7o5/zZxs2Dkf/vgIzhyAJoOgk516\ncLz8oPlQ2DDO9hp6F87DsWhTLuoyvZqGUd7fi1HTNnHPV2vpUj+EKeuOcHPtYMYOaUGgbz6rPRsP\ngKUvQcI+UymgiLw93Lmtfii31Q8lIzuHP2NPs2T7SZbtPsnCzccJ8PGga/1QejSuzC06Gq8FI8xQ\n7+A5MP0e+OExeHQtBFQqchuKZPXH5t936AJTjqswPLxMdv4Dy0zvqT0njF9vLBaz2KHWbbavPvYN\ngtCGhatTmZtjzN5zxnzLmdxy0jMmSoD0jAnX5OYOHV+EhL2m4LYj5WSZZK5jo+C7USbhY/9voc9X\n9v2wbfWg6XXb9K1t+/+1CnTONUsgdagVzOyRbcnIzmHKuiMMbh3JpBGt8g/EABrdA8oNthe9d+xy\n3h7udK4XyscDmhLzalcm39+K7g0rsWJvPPOmf4Waex9HvGrxW6vxpPsEQ79Jpij3wpElOxQdvxdW\nf2IC0qKUlgIzbyzxqLk2xbXFbYGUU2a+WGFEtoOj0TaVEQPsn30/l1KSa0yUGAnGhOtq0AdCGpre\nMVv/MBdGdgZET4TPWsCPT5iUFQNnwKjVJh2Fm53/e5SvYQKATd+aALAgscvNEE+VqGvu0ig8kB+e\nuInxw1ry7t2N8HS3oc0BoaZ3Z8dch0yk9/Jwo1PdEP7XvykxfVP5yvtzjvvV496MF3hgzgGi3l7O\nzEM+Jtv9X6tg7Ri7t+GqLBZT8si7DHR/r+jH+SfFhQxV5mv/UhP01+5auOdVbQdZqXBym2372zv7\nfl6BEdIzJkqEBGPCdbm5QaeXTJLN+febOVxnDxU/gMi6AOvHwZhmsOTfJhfYkHkwchXU72n/ICyv\n1g9DyknYsyj//bSG2N+gxq0FDvGEB/lye8NKZsWkrZoMhPN/Fy3Bpq12LsBz4YO4RbSi2tNLWflq\nb6Y+0JpmEUG8/N0OxiV3MAH3b2/DsU2Oa0euTZPN++327iVz8AotMNx8SZB8Y/nb97OZA3a18lL5\nibTOL7N13lhynCm15F+xcK9jC0n8KkqIBGPCtdXrCVEPmgLNPzwOnzWHj+vBvPth4wQ4tcv2Ya7M\nVFj7OXzaBH55weTcGvYdPLQc6txeMvN/anWBoKoFZ+RP2GeKjNcsxirO/NS7Ezx87TpUeYnt82DB\nQxDZFu6dD94BeLq7cUudiky+vxU9m1Tm/V/28bn/k+iASrDgAZPTzVGSTsDyN6D6rdB0cPGPV7ur\nyYflyDaXZkknTAkvW1dR5lW2svm/aWu+sSRrjjFHzCsNjIDUeMi6DqtPCJciE/iFa1PKlKnp8RGc\n3m9yPP29zqQm2LXQ7OMTZOaZVG1vfio3vbQ3KSPZBG7rxpq8UtVvhVsnQ7WbSv79uLlDq4dg2X9N\nIBl6jRJHufU5i5NSIz/eASYg27XQDBd6eNnv2Ftnmcn5VTvAkDlXTJL3dHdjzKDmlPH24OPVR/Fr\n8ioPHHgM9dNz0He8/dqRy2KBn/5jUlL0HG2foLt2V/jzUzPMWr9X8Y93vcnNul/Y+WK5ItvDgaW2\nLZLIzb7vCLnpLZKOQ4WajnkNIZBgTJQWbm4QUs/8tHrQ/JE+f8QMZRz50+Tv2v+z2dfTzyRardrB\n5NTaMA7Sz5teqVueh8g2zn0vzYeaCgMbJ0CvT6++T+xyk+09KNJx7WgywKweXTsG2j5W+JWFV7Nl\nOvzwhBleHTTLrCK9Cnc3xXt9G+Pv7cFba/6iSsQIum2fZIqhNx1U/HbkyroA3z9qSkB1fct+H6gR\nbcx8vgO/SjB2NfuXmh7goq7WrdoOts00X8AKOkZyHFSsV7TXKcg/ucaOSjAmHEqCMVE6KWWGMspV\ng2bWYafkU2ZJ/JF1Jjj7/T1Am2/nt/7H1L5zBX7lTd6x7XOh6/9dWdw4M830/LV60LHtqNnZBBW/\nvQ1/fgaN+0PLEVC5SdGOt2mKyd9VsxMMmmnK2+RDKcWrd9YnwMeDR5dbWBoUTa0lz6KqtLLPB19K\nPMwabGqddn0T2j9Z/GPmcvc07/PAcklxcbnMNJPsteWIop+Xf+aNrS04GEs64bjhfEn8KkqIzBkT\n14+AULMKsseH8OgaeOEwPL0Dhsx2nUAsV+uHzIqxrbOufOzIWpO13FFDlLncPeGBpean3p2wdQZ8\nfTOM72QCq8KU/ImZZFYq1upiesQKCMRyKaV4uksdXr6zIcPPP0xqtsIy/0HIzizim7I6tRsm3GaG\nggdOgw5PFSkwsFj0lclrc9XqCsknzGuIi/76w9R0LMp8sVwVapoJ+QXNG0tPgswUxw1Tlg0HVPGC\nsdOx5suVEPmQYExcv3yDHDvMVxxhzSE86ur1KmOXg4ePGWZ1NKXMJPu7x8Gze+GOD80H6aJ/wcd1\nTU/XiS35H2PjBFj8DNTpDoNmgKdPoZvx0M01eLJvJ57LeBC3uC1kLHuziG8I01s10Vq26IGfCz2M\naLFoNh05y5uLdtP+/d/o8P5v7D15lYn6uXnKZFXlpfb/bKoUVC3GnEylzDzQglZUJufmGHNAWgsw\ncykDKhU919ipXfDNbTCtD5w7bNemieuLBGNCOEvrh00m+EO/X7r94AoTiNnYu2Q3vuWgzSiTFf/B\n5SbtxLY5ML4jfH2Lycl2+erB9V/BT89B3TthwFSTMLeIBreOpMeAUczMuQ3vDZ+TvKsIQc7GCTCz\nP5SvBg//ZoJeG5gA7BxvLd5Nhw9+456v1jF9/REahZfF3U0xZMIG9p1MvvRJZSubAvCSb+wirc18\nsZqdi78opGp7SPw7/14pR2XfzyuwStFKIp2Ohal9zP9j5Q4rivEFQ1z3JBgTwlka9AG/YBNA5Dp3\nxExaLmp2eHtQCiJaQZ8vTG9Zj49M5YAl/za9ZT88YfKCrR0Lv7xoep76f1usQCzXXU3DqDzgEw7o\nKmTOH0nCSRs/BC058PMLJjCs3Q3u/8XkA8uH1prNf5/j7cW7uemD37jnq7VMW3eEhmFlGT2wKTH/\n7cI3w1sxe2Q7PN0VgyesvzIgq93V5C67cL6I7/g6E7fN9FbVLeIqyrwi25nb/HrHHJV9P6+i5Bo7\n/zdM7W0qaNz3I7R7HHYuMPMXhbgKCcaEcBZPH2hxnxnWyf3m/U9KCycGY3n5BpkevEfWwEO/mVJK\nOxfAN53h11dMQNlvsl1TY3RqXI3UXl9TxpJC7IThHD2Tmv8TMpLNRP0N46Dt42ao1LvMVXfVWrPl\nnwBsJX2/XMvUdUeoX7ksnwy4GIDd3bwKZX1MepTqwf7/BGRDLg/Iat9uPnAPrbTX2y/d9i8F1MUq\nBcVRqTF4BeRfpzLphLkNCCv+611LbjBma7Lp5JMmEMtINnkMK9Yxcxb9guHX/zqk6oUo/SQYE8KZ\noh4wtzGTzG3sCpNoMri289p0NUpBlZbQeyw8uw/u/AQ6vgT3TLS9CHQhNIu6ifh2r9IuZxPzv3yV\ngwnXWExw/ihM6m7m2fUcDd3fvST5p9aac6mZbPzrLO8sMQHY3V+uZcq6w9SrFMDH/ZsS/WoXJo5o\nRd8WFwOwy1UP9mfWw23xuDwgC48yee5kqNLY/7NJK1PGDtnw3dwhonX+PWPJJ8C3fJHmKdosMMIs\nqEk9XfC+aWdh2t1mZffQ+SbnIZhSa51eMml49v3kuLaKUktSWwjhTEERJvXG5qlw87NwaBU0vse1\nUyX4lHV82g0gottTJMet5vEjUxn+VX1efWggDcPypAE5vglmDkJnp5PSbzYHAlpxePMxDp9J4/Dp\nVI6cSeWv06kkpZu6pp7uiptrV+SZrnXo2iC04ILql6lRsQyzHm7L4AnrGTJhPbNGtqVOaIBZ9Xpg\nmUku68hSWq4u+aRZ7HHba/Y7ZtV2JvVK2tmrl1VKOuG4yfu5AiPMbeLR/IPM9CSY3hfOHIR755pA\nMq8Ww00ZtmWvmx5VB3yJEaWXBGNCOFvrh2DfElj6MmQmOy5nUmmjFAEDxpP9RTvev/Ap/ceH8lrf\nVmRmW/Dct4hu+1/nrAriEcsbbJ2WBazNfRrhQb5Uq+DPXc3CqFbBn2oV/GlVrTyBfsX7AMwNyAaN\nX8/g8daArFZXM3R7cjuENSv++17xJuz+weRFa3Zv6fnQ3r/U3Nbpbr9j5uYb+3s91Otx5eNJJxw7\nXwwuzTUW3uLq+2SmwcyBcHIHDJwONTpeuY+7p8krOHsIbJ5iKnEIYSXBmBDOVr0jVKht/kArd5O9\nXhj+FfDoN4HIqb15zWMaT8x05zH3H3necw473OryVeibNAoNp6c14KoW7EdEeT+8PRxQp9CqRsUy\nzB55MSCbO7QtNcH0jhU3GNs2G1Z/DGUqmbQiqz82VSOaDnL9oGz/UgiMhJAG9jtmeEtw9zLzxq4W\njCXH2bxitsjyZuG/muwMmDPU5ES755v8Fy/U7WECzN/fhyYDTVkyIZA5Y0I4n5vbxW/JEW2uzMh/\no6txK+qmZ+iZvYytkWN43nMOOQ3vofFLf/DlqO683acxD91cgy4NQqkVEuDQQOyfJlkDMnc3xcAZ\nsaSHNC1+vrETW0wAVvUmeGYnDJkHfhXgxydgbBRsmQE52fZ5A/aWlW4WMdTpZt8hdk8fCGtx9Xlj\n2RmQmuD4YUrfcuDpf/UVlTnZsOBBs/Dmrs+gcb/8j6UU3P62afefYxzTXlEqSTAmhCtoNtistmrQ\n29ktcU2dXobwKILiN0LHl3DvN9Gxk7ZtUKNiGWaNbIubUkxNqIM+HmPmNhVFSgLMHmqyzg+YYnrB\n6twOD6+EwbNNHcwfHoMvWpneM8ulVQG01sTGp/DTjjjOpxWzekFRHF4NWWlQ145DlLmqtoO4rZB5\n2ara5JPm1tHDlEpZV1Re1jNmscAPj8OeRdDtPbMy2hZVWkLDviY1TO5q0BtZVjqsfBdOH3B2S5xK\ngjEhXIFPIPx7j0m6Kq7k7gn3zoOHVkDHF11mgUNNa0C2zq0FSluI27yk8AfJyYJ5wyHttJlv5B98\n8TGlzLDXqD9g4AzTQ/PdKCxjW3Pwt8mMW7mfh6ZE0+KtZXT5ZBWPzdhMny/+5K/TBaQDsROtNWtj\nT5OyfZFpW3Gy7l9LZHuwZMOxmEu35wYyZR2Y1iJXUMSlPWNam5x222dDp1eg3WOFO95tr5n3tPId\n+7azNNo+G1Z9AJPvMGXMblASjAnhKjy8XCbIcEl+5aFKlLNbcYWaFcvw6sh7OUdZtqyYy4FTyQU/\nKa+lL5uUB3d9fs05Z2dSM/nVEsV7kV/zYeCrHDiTQc0/nua2lX2IPPELXetV5MN7mjB+WEuS0rPp\n++WfxBwuYi+djVIzsnl85maGfLOexO2LiXZvxg+7zpCedY1ankUV0RpQV9apTC6BHGO58iZ+1RqW\nvw4xE6H9v+CW/xT+eOWrQ+uRZuj5Rq5tqrVZYVqhFrh5wLd3Qtx2Z7fKKWQCvxBCFFPN0ECS63Sh\n3f7l3D5+LTNHtqd2qA2Ts7dMh43jod0T0GQAADkWzeEzqWw6fI6YI2eJOXyOQ9aeLi93NxqFtyen\ndQ96uG+k4YGveO3MR5CwBBq+APV7U/ex9oyYHM2Qbzbwcf+m9Gpq/2Dlr9OpjJoWQ2x8Cu+1V4Rv\nPsPUnGZ8PXsrgb6e9GkWxoBWEZemIikq3yAIbQhHLkv+WhLZ93MFVjHzvLIuwLqxZr5X1IPQ9c2i\nf4G65TnYOh2WvQZDF9i3vaXFod8hYQ/0/tLUyJ1yF0zpZZLlXmvl6nVKgjEhhLCDgEY9YP9CGnCI\nwRPcmfVwm0sCMq01SReyOZWczsnEdLKObKTj2qc5XCaK/528i7gv/uRUYjoJKRnkWEyW9iA/T6Kq\nlqN/VARR1crRODwQH8/cBQoN4fb7YNd3Zphn3ggIaUjVji+y8JFujJy+iSdnbeHYuQs8cmsNlJ16\nXVfujedfs7fg7qaY+kAbboqbDMALTz7FLfHuzIk+yqzoo0yxlpYa2CqC3k3Di5dWJLIdbJ1hhnRz\nV5UmnQBPP5N019Fyc40te80Ez00GmTJhxTmnfuXh5udg2X/h4Eqo2ck+bS1NNowzc2Ub3WPmgN7/\nkwnGpvY2AerludquY0qXotIMUVFROiYmpuAdhRCipKWdhQ9rcDbqGbpt64DWmrY1KhCflMGp5HRO\nJaWTnmUBoCLnWeT9Clnag3vdPsAnMJjQsj7WH28iy/vRsmo5agSXwc3Nhg98Sw7sXAir3oczsVCp\nCZk3v8iz2yqzaHscg1tH8lbvhni4F31misWi+fL3WD5etp/6lcry9bCWRJT3gwm3AdoUZrc6n5bJ\nD1tPMCf6KLvjkvD2cKN7o0oMjIqgbY0Ktr2nvHYugPkPmJJcVVqabfNGmCGtf20u8nuy2eE1ZggN\noF5P6D8F3O3Ql5GVDmNbgW8gjPzjxkoafOYgfN4Cbn3BLNDJlXjM9JClnIIhc6FaB+e10Q6UUpu0\n1gXOr5BgTAgh7OWbLmDJIbbPIp6YuZn0rBxCrEFWpbLeJtgq40bHdQ/if3YXWSOW4l2lqf1ePycb\ndswzQdm5w+iw5iwoex/PbQ3h1johfHFvC8p4Fz6ISE7P4tm52/h19yn6NAvjvT4N8D28AjZNNik9\nOr96zblTO48nMif6KN9vPU5yejYR5X0Z0DKCe1pWISzI17YGJMXBJ/VMWoj2T5ptE283OchGLC70\n+ym0xOMwuoFJyDx4Fnh42+/Y2+fBwoegzzizqvpG8dN/IGYyPLMLAkIvfSz5pOkhSzxmzneNjs5o\noV1IMCaEECVt1Ydmmf5zB65dOmfxM6YWab/J0KivY9qRk2VSYPzxIZz/m9NBTXkm4U5OV2zHpPtb\nUTnQxiAIOJiQwsipMRw+k8bbncszyP031OZpZgJ9mUrQYpgp5eWZ/zHTs3L4ZedJ5kQfZd2hM7gp\nuLNJGP/r1yTP0Gs+xjSFkIYweKb5fXRjk/ai73ib30uxHIuB0Eb2T6liscCETqb25ZMxBZ7H68KF\n8/BJA6jfC/p+ffV9UuLNcOXZQzBoBtTqUrJttBNbg7EbqE9UCCEcrHZXQJskoFez6VsTiHV42nGB\nGJh5VS2GwROboOenBFvOMM3zXd469zxvfT6e3SeSbDrMst2n6Dt2NQ1SNxBdYyKD/7wDtepDCKlv\n0nA8s9P0itkQQPh4utOneTizRrblj/90YuQtNVm8/QQPTonmQqYNKzAj25sVlRaL+UmOg4ASmLyf\nq0qUY3LbubmZHr+kY7D+K/sf3xVtmQ5ZqdD2kWvvUyYEhi+G4NowazDs+7nk2ucEEowJIYS9VGpq\nErdeLRv/3xtgyXNmqMuexbTz4+EFUfebeVU9PqKp/1m+zH6N5K+7s2XNtT/cLBbNhCV/sm3GKyxz\nf5LPLe9S/tx2E0Q+tRWGLTS9GkUs0RRZwY8X76jHR/2asvbgGe7/diNpmQVUF6jaDi6chdP7TU42\nS5bjs++XlOo3m5qea0abHrLrmSUHNn5tFmUUVMrKvwIMX2R6JOcMhd0/lkwbnUCCMSGEsBc3N6jV\nFWJXXJolPykO5g4zKRL6TQQ3x5dsuoSHN7R+GM9ntpF061vUdjtO8+WDiBt7BxyNvrifxULq7qVs\n/agn92/syXOe86gQ2QD6f2vm9nR5HcpVs1uz7mlZhdEDmrHxr7OMmBxNakY+Adk/RcPX5kn4WoI9\nY47W9U1TZWDVh85uiWPt+wnO/w1tH7Vtf99ycN/3pk7pvBGwY75Dm+csEowJIYQ91e4K6ecvZozP\nzjCBWEYKDJppPlycxdOXsp3+hdezO5gVNBKvhB0wsQt6en9Y9SFZnzbDf+4AqqVuY2/1+9BPbMJ9\nxI/Q8G7Ty+YAfZqHM3pgM2IOn2XE5I2kXCsgq1DT9DoeWVey2fdLSsW6pqRSzESz0vB6tX6cSRVS\n907bn+MTaFJdRLaFhQ/D1lmOa5+TSDAmhBD2VLMTKDczVKk1LHkWjkXD3eMgtIGzWwdAmTJl6f/k\nB3zWaAHvZw0i7dB6WPkOWxL9edntaWKHbqTRiDGo4Fol0p7ezcL5bHBzNv99nuGTNpKcnnXlTkqZ\noa2/15ds9v2S1PElcPeG5W84uyWOEbcdjqwx1QcKmxrEO8CURKt2M3z/KGya4pg2OokkfRVCCHvy\nLQcRbSB2mRlG2zLNpH1ocJezW3YJD3c33ujXmq9Dgmn18+2UJZXQiJqMG9qiUKst7aVnkzDcleLJ\nWVu4b9JGpjzQmrI+l81Jq9oe9vxoeh2Vu5nkfT0JCIUOT8Hv75o5hpFtnN0i+9owziTqbTGsaM/3\n8ochc8z8sUX/gpxMaP2wfdvoJNIzJoQQ9la7K8Rtg59fgNrdoOPLBT/HCZRSPHJrTUYP60D/zm2Y\nM7KtUwKxXHc0rszYIS3YcSyRYRM3knjhsh6yyHbmds9iCKhU8nPvSkL7J0zKkF9fNT2r14uUeJMD\nr9mQ4g3Ve/qa4f66PUyx9rVj7ddGJ5JgTAgh7K1WV3NbrprJg+XimdW7NazEs7fXtS3fl4N1b1SJ\nr4a2ZPeJRIZN3EBiWp6ArFJj8AqAzOSSTWtRkrz8TUb6YxtNL+D1Imay6clqk086C1t5eJsqCA16\nw6+vwG9vl/rA1bX/QgghRGlUqTF0e8/McfEtgdqJ15muDUIZN7Qle+OSuXfies6nZZoH3Nwv1iu8\nnibvX675UKhY38wdy7rg7NYUX3YGRH9jvqQE17bPMT28TOLkFvfBH/8zyZQtNuSrc1ESjAkhhL0p\nBe0eg/I1nN2SUuu2+qF8fV9L9p9KYciEDZxNtQZkVa1DlddzMObmblJdnD0E71WBL9vDwpGw9nNT\nVLy05SLb9R2kxtuezsJWbu7Q6zO46RlTmmv+AybwK4VsCsaUUt2VUvuUUrFKqRev8ng9pdQ6pVSG\nUuq5PNsjlFIrlVK7lVK7lFJP5XnsDaXUcaXUVutPD/u8JSGEENeDTnVDmHBfFAcTUhgyYT1nUjIu\n5hu7Xocpc9W53aRzaP8vCAyHv1abeWTT+sD/asJHdWF6P9N7tmM+JOx3zZ4hrWH9lxBcF2p2tv/x\nlYIub5gqBru/h5kDTRqZUqbA2pRKKXdgP9AVOAZEA4O11rvz7BMCVAX6AOe01h9Zt1cGKmutNyul\nAoBNQB+t9W6l1BtASu6+tpDalEIIceNZc+A0D06JploFf2Y80JzgP16Fdk/Yb8irtEg9A6d2wMmd\ncGonnNwBCftMNQIAD19TqqpSI6h2i8kPV9gUEvZ2ZB1M7g49R0PUA459rS0z4McnIawZ3DsfKaUT\n7AAAEVhJREFU/Mo79vVsYGttSlv+lVoDsVrrQ9YDzwZ6A/8EY1rreCBeKXVJFjetdRwQZ72frJTa\nA4Tnfa4QQgiRn5tqBzN5RCsemBLN4Imbmfnwh1QM8HZ2s0qefwWo0dH85MrOhNP7TIB2cocJ1vYs\nhs1TYeU7cOvz0HiA84Ky9V+CTxA0GeT412p+r5mjOe9+mNQdhn1nehVLAVuGKcOBo3l+P2bdVihK\nqWpAc2BDns1PKqW2K6UmKaWcmJZaCCGEK2tfK5hv72/NsXMXGDR+HcfOpTm7Sa7Bw8ssGGk2GLq/\na2o5Pn8IBs0yiVK/fxS+aA3bZkNOAfU/7e3cEdi7GFqOAC+/knnNenea2qlJJ2BSNzh9oGRet5hK\nZAK/UqoMsAB4WmudZN38FVADaIbpPfv4Gs8dqZSKUUrFJCQklERzhRBCuKC2NSow5YHWnErK4I5P\nVzN/0zEKmmpjD7Hxybzy3Q6+33KcHEspSKGgFNTrAaP+gIEzTKLV70bBl21g25ySm1sWPQFQJZ+Y\ntdpNcP8SyE43AdmJLSX7+kVgSzB2HIjI83sV6zabKKU8MYHYDK31wtztWutTWuscrbUFmIAZDr2C\n1nq81jpKax1VsWJFW19WCCHEdah19fL89K+bqV+5LM/N28aoaZs4neKYFXQZ2TmMXrafHmPWMDv6\nKE/P2UqPMatZuutkiQSBxaYU1O9pDcqmg4cPfDcSvmgD2+c5NijLSIFNU03licAqjnuda6ncFB5Y\nCp7+8G1P+OuPkm9DIdgSjEUDtZVS1ZVSXsAgwKZMdEopBUwE9mitP7nssbxLYe4GdtrWZCGEEDey\nyAp+zBrZlld61Of3/Ql0G/0Hv+w8adfX2HDoDD3GrGbMigP0aFyJ9S/dxtghzcnKsTBq2ib6fLmW\nNQdKSYoJNzeo3wtGrYYBU8HdExY+BF+2NSsxHRGUbZsFGYnQ9jH7H9tWFWrCg0tNYfLp98Bu102i\nW+BqSgBr2olPAXdgktb6HaXUIwBa63FKqUpADFAWsAApQAOgCbAa2GHdDvCy1vonpdQ0zBClBg4D\no6wT/q9JVlMKIYTIa/+pZP49dys7jyfRt0U4r/dqSKCvZ8FPvIbEtCze/2UPszYeJaK8L2/3acyt\ndS6OymTnWFi45Thjlh/g+PkLtKtRgee61aVl1VI07dliMdn9f38fEvaYtBO3Pm9WX9qjxJTFAl+0\nAu+y8PBvpofOmdLOwswBcHwT9BpjEsWWEFtXU9oUjLkKCcaEEEJcLivHwue/xfLFylhCArz5X7+m\n3FQ7uFDH0FqzeHsc/7doN+fSMnnopuo81aU2fl5XX4WYkZ3DrA1/M3ZlLKdTMulSP4Rnb69L/cpl\n7fGWSobFYnJzrfoAEvZCxXomKGtwd/FKeB1YBjP6Qd9voEl/+7W3ODJTYe59ELscuvwf3PR0ibys\nBGNCCCFuKNuOnuffc7dyMCGV4e2q8uId9fH1Krin59i5NP77/U5W7kugcXgg7/VtTKPwQJteMzUj\nm2/XHubrVQdJzsimV5Mwnulah+rB/sV9OyXHkmOCst8/MGkyKtSGDk9Bk4FmtWZhTbsb4vfAU9uL\n9nxHyc6E7x+BnQug/ZPQ9S2H99pJMCaEEOKGk56Vw4e/7GPSn39RPdifjwc0pUXk1YcQs3MsfLv2\nMB//uh+l4Lnb6zK8fTXc3Qr/AZ2YlsX41QeZtOYwmTkWBkRV4cnOtQkL8i3uWyo5uUHZmtEmZ1lA\nGLR/AloMB+8yth0jfq9Ztdn5VbjlP45tb1FYLPDz82alZ68xJu2GA0kwJoQQ4oa19uBp/jNvO3GJ\nF3i0Y02euq0OXh4Xh952Hk/kpYU72HE8kc71QnirTyPC7RA4JSRn8MXKWGZu+BsUDG1TlUc71ixd\nSWq1hoMrYPVoOLLGJG1tMwpajzKJZ/Oz6Gkzef+ZXeBfuKHiEqM1bJ8DDfs6vOdOgjEhhBA3tOT0\nLN5avJu5MceoX7ksnwxoStUKfoxetp9Jfx6mnJ8X/3dXQ3o0roSy83DVsXNpfLbiAPM3HcNNKW6p\nU5GeTSrTtUEoAT5FX2BQ4o5uhDWfwr4lJl9Zi+HQ7nEIirhy37Sz8EkDaNwPeo8t+ba6IAnGhBBC\nCGD57lO8uHAHiRcyqeDvzcmkdAa3juTF7vUI9HNsYHQoIYU5MUdZvC2O4+cv4OXhRue6IfRqGkbn\neiE2zWlzCfF74c9PYcc883vjAWZeWUi9i/usGW0Klz+6FkIbOqWZrkaCMSGEEMLqbGomr/2wk0MJ\nqbxxV0NaVy/ZItIWi2bL0fMs2naCJTviSEjOwM/Lna4NQunZJIxb6gTj7VEKArPzf8O6L2DTFMi+\nAPV6wk3PmCSrY5qa3F7DFzm7lS5DgjEhhBDCBeVYNBv+OsOibXH8vDOO82lZBPh40L1hJXo1DaN9\nzQp4uJdItcKiSz0NG76GjeMh/bzJVXZ6HwyeDXXvcHbrXIYEY0IIIYSLy8qxsCb2NIu2nWDZrlMk\nZ2RT3t+LOxpV4s4mlYko54eflzt+Xh74eLoVa26b1poLWTmcTc3kXGoWZ9MyOZ+Waf09k7NpmaSk\nZ9OqennubFyZID8bJrdnJJtesnVjwbccPPJn8XKUXWckGBNCCCFKkfSsHFbtT2DRthMs33OK9CzL\nFfv4errj5+WOr1furQe+nm74eXmYbdbHLRrOpWVyLi2Ts6lZnEs19zOyrzwmmHRbQb6eeLq7EZ+c\ngae7omPdEPo0C+e2+iH4eBYwhJqTDToHPErRqtESIMGYEEIIUUqlZmSz9uAZzqVmciErh7TMHC5k\nZpvbrBwuZJptaVlm+8V9zK1SUN7PiyA/T8r7e1HOz8vc+ntdut36e1lfT9zdFFprdp1I4vstx/lx\n2wnikzMI8Page6NK9GkeTtsaFYqUh+1GJcGYEEIIIYosx6JZd/AM3289zi87T5KSkU1oWW/uahpG\n72bhNAwra/eUILZKy8wmLjGdMt4ehJb1cUobbCHBmBBCCCHsIj0rh+V7TvH9lhOs2h9PVo6mVkgZ\n7m4ezl1Nw4go72fX1zqZmM6JxAvEnU8nLvECcYnpxCWmc+K8uZ94IQsww6sdagbTP6oK3RpWKng4\ntYRJMCaEEEIIuzuXmsmSHXH8sPU40YfPARBVtRxta1RAKZPg3qI1GnNfW+9bLNZbrS/Znm3RxCdl\n/BN0nU3NvOI1y/l5UjnQl7AgHyoF+vxz/8iZNOZvOsaxcxcI8PGgV9Mw+resQrOIIKf12uUlwZgQ\nQgghHOro2TR+3HaC77cc50B8CgBuCpRS5haFUqYHy00p3JRCgXWb2cfdTRFcxpvKgT5UDvIlLNCH\nSoHmtnKQL5UDffLt8bJYNOv/OsP8mGP8tDOO9CwLtULK0L9lFe5uEU5IgPOGMSUYE0IIIUSJ0Vo7\nvTcqOT2LJdvjmLfpGJuOnMPdTdGxTkX6R1Whc73QS+qTlgQJxoQQQghxwzqYkML8TcdYuPkYp5Iy\nKO/vRe9mYfRvGUGDsLIl0gYJxoQQQghxw8vOsbA69jTzY46xbPcpMnMsNAwryyt31qd9zWCHvrat\nwZiHQ1shhBBCCOFEHu5udKobQqe6IZxLzeTHbSeYt+kofl6uEwK5TkuEEEIIIRyonL8Xw9tXY3j7\nas5uyiWkgJQQQgghhBNJMCaEEEII4UQSjAkhhBBCOJEEY0IIIYQQTiTBmBBCCCGEE0kwJoQQQgjh\nRBKMCSGEEEI4kQRjQgghhBBOJMGYEEIIIYQTSTAmhBBCCOFEEowJIYQQQjiRBGNCCCGEEE4kwZgQ\nQgghhBNJMCaEEEII4UQSjAkhhBBCOJEEY0IIIYQQTiTBmBBCCCGEE0kwJoQQQgjhREpr7ew22Ewp\nlQAccfDLBAOnHfwapZ2co/zJ+SmYnKP8yfkpmJyj/Mn5KVhJnKOqWuuKBe1UqoKxkqCUitFaRzm7\nHa5MzlH+5PwUTM5R/uT8FEzOUf7k/BTMlc6RDFMKIYQQQjiRBGNCCCGEEE4kwdiVxju7AaWAnKP8\nyfkpmJyj/Mn5KZico/zJ+SmYy5wjmTMmhBBCCOFE0jMmhBBCCOFEEowJIYQQQjiRBGN5KKW6K6X2\nKaVilVIvOrs9rkYpdVgptUMptVUpFePs9rgCpdQkpVS8Umpnnm3llVLLlFIHrLflnNlGZ7rG+XlD\nKXXceh1tVUr1cGYbnU0pFaGUWqmU2q2U2qWUesq6Xa4j8j0/ch0BSikfpdRGpdQ26/n5P+t2uX6s\n8jlHLnMNyZwxK6WUO7Af6AocA6KBwVrr3U5tmAtRSh0GorTWkkjQSil1C5ACTNVaN7Ju+xA4q7V+\n3xrUl9Nav+DMdjrLNc7PG0CK1vojZ7bNVSilKgOVtdablVIBwCagDzACuY7yOz8DkOsIpZQC/LXW\nKUopT2AN8BTQF7l+gHzPUXdc5BqSnrGLWgOxWutDWutMYDbQ28ltEi5Oa/0HcPayzb2BKdb7UzAf\nHDeka5wfkYfWOk5rvdl6PxnYA4Qj1xGQ7/kRgDZSrL96Wn80cv38I59z5DIkGLsoHDia5/djyH/4\ny2lguVJqk1JqpLMb48JCtdZx1vsngVBnNsZFPamU2m4dxrxhh08up5SqBjQHNiDX0RUuOz8g1xFg\nRnaUUluBeGCZ1lqun8tc4xyBi1xDEoyJwrhJa90MuAN43DoEJfKhzTwAl/oG5gK+AmoAzYA44GPn\nNsc1KKXKAAuAp7XWSXkfk+voqudHriMrrXWO9W9zFaC1UqrRZY/f8NfPNc6Ry1xDEoxddByIyPN7\nFes2YaW1Pm69jQe+wwztiiudss5zyZ3vEu/k9rgUrfUp6x9GCzABuY6wzmNZAMzQWi+0bpbryOpq\n50euoytprc8DKzFzoeT6uYq858iVriEJxi6KBmorpaorpbyAQcCPTm6Ty1BK+Vsnz6KU8gduB3bm\n/6wb1o/AcOv94cAPTmyLy8n9gLC6mxv8OrJOLp4I7NFaf5LnIbmOuPb5kevIUEpVVEoFWe/7Yhah\n7UWun39c6xy50jUkqynzsC5r/RRwByZprd9xcpNchlKqBqY3DMADmCnnB5RSs4COQDBwCngd+B6Y\nC0QCR4ABWusbchL7Nc5PR8ywgAYOA6PyzG254SilbgJWAzsAi3Xzy5h5UTf8dZTP+RmMXEcopZpg\nJui7YzpY5mqt31RKVUCuHyDfczQNF7mGJBgTQgghhHAiGaYUQgghhHAiCcaEEEIIIZxIgjEhhBBC\nCCeSYEwIIYQQwokkGBNCCCGEcCIJxoQQogBKqY5KqcXObocQ4vokwZgQQgghhBNJMCaEuG4opYYq\npTYqpbYqpb62FgdOUUqNVkrtUkqtUEpVtO7bTCm13lok+LvcIsFKqVpKqeVKqW1Kqc1KqZrWw5dR\nSs1XSu1VSs2wZoYXQohik2BMCHFdUErVBwYCHawFgXOAewF/IEZr3RBYhakCADAVeEFr3QST3T13\n+wzgC611U6A9poAwQHPgaaABprhwB4e/KSHEDcHD2Q0QQgg7uQ1oCURbO618McWRLcAc6z7TgYVK\nqUAgSGu9yrp9CjDPWn81XGv9HYDWOh3AeryNWutj1t+3AtWANY5/W0KI650EY0KI64UCpmitX7pk\no1L/vWy/otaAy8hzPwf5+ymEsBMZphRCXC9WAP2UUiEASqnySqmqmL9z/az7DAHWaK0TgXNKqZut\n24cBq7TWycAxpVQf6zG8lVJ+JfouhBA3HPlmJ4S4LmitdyulXgV+VUq5AVnA40Aq0Nr6WDxmXhnA\ncGCcNdg6BNxv3T4M+Fop9ab1GP1L8G0IIW5ASuui9tgLIYTrU0qlaK3LOLsdQghxLTJMKYQQQgjh\nRNIzJoQQQgjhRNIzJoQQQgjhRBKMCSGEEEI4kQRjQgghhBBOJMGYEEIIIYQTSTAmhBBCCOFE/w91\n8ndq58IzVgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x135120a1dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "history.plot(['train loss', 'valid loss'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch import nn, optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision import transforms\n",
    "import random\n",
    "import model as m\n",
    "from PIL import Image\n",
    "from time import time\n",
    "import mydataset\n",
    "import gc\n",
    "imp.reload(mydataset)\n",
    "imp.reload(core)\n",
    "imp.reload(utils)\n",
    "imp.reload(m)\n",
    "for i in range(10):\n",
    "    gc.collect()\n",
    "\n",
    "# 超参数\n",
    "batch_size = 256\n",
    "pic_size = (64, 64)\n",
    "learning_rate = 1e-3\n",
    "num_epoches = 1000\n",
    "tolerance = 15\n",
    "lr_tolerance = 7\n",
    "transform1 = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "])\n",
    "transform2 = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n",
    "])\n",
    "\n",
    "def transform_tr(img, pic_size):\n",
    "    img = img.resize(pic_size)\n",
    "    img = img.transpose(Image.FLIP_LEFT_RIGHT)  if random.randint(0, 1) > .5 else img\n",
    "    img = img.transpose(Image.FLIP_TOP_BOTTOM)  if random.randint(0, 1) > .5 else img\n",
    "    img.rotate(np.random.random() * 45)\n",
    "    img_tensor = transform2(img)\n",
    "    return img_tensor\n",
    "\n",
    "def transform_vl(img, pic_size):\n",
    "    img = img.resize(pic_size)\n",
    "    img_tensor = transform2(img)\n",
    "    return img_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 3\n",
       " 0\n",
       " 5\n",
       "[torch.LongTensor of size 3]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.Tensor([3,0, 5.9])\n",
    "a = torch.autograd.Variable(a)\n",
    "a.long()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Update Model!\n",
      "1 [1/1000] 94s, Loss: 0.3031, Val-Loss: 0.2100, Best Val-Loss: 0.2100, Val-F2: 0.860\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "J:\\Anaconda3\\envs\\py36\\lib\\site-packages\\sklearn\\metrics\\classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [2/1000] 93s, Loss: 0.2287, Val-Loss: 0.2197, Best Val-Loss: 0.2100, Val-F2: 0.849\n",
      "Update Model!\n",
      "1 [3/1000] 93s, Loss: 0.2041, Val-Loss: 0.1255, Best Val-Loss: 0.1255, Val-F2: 0.915\n",
      "1 [4/1000] 95s, Loss: 0.1926, Val-Loss: 0.1721, Best Val-Loss: 0.1255, Val-F2: 0.879\n",
      "1 [5/1000] 94s, Loss: 0.1842, Val-Loss: 0.3108, Best Val-Loss: 0.1255, Val-F2: 0.754\n",
      "1 [6/1000] 93s, Loss: 0.1748, Val-Loss: 0.1499, Best Val-Loss: 0.1255, Val-F2: 0.897\n",
      "1 [7/1000] 91s, Loss: 0.1742, Val-Loss: 0.1457, Best Val-Loss: 0.1255, Val-F2: 0.899\n",
      "1 [8/1000] 90s, Loss: 0.1648, Val-Loss: 0.1721, Best Val-Loss: 0.1255, Val-F2: 0.884\n",
      "1 [9/1000] 91s, Loss: 0.1624, Val-Loss: 0.1388, Best Val-Loss: 0.1255, Val-F2: 0.902\n",
      "1 [10/1000] 94s, Loss: 0.1586, Val-Loss: 0.1435, Best Val-Loss: 0.1255, Val-F2: 0.898\n",
      "New Learn Rate: 0.0001!\n",
      "1 [11/1000] 95s, Loss: 0.1570, Val-Loss: 0.1309, Best Val-Loss: 0.1255, Val-F2: 0.918\n",
      "1 [12/1000] 94s, Loss: 0.1441, Val-Loss: 0.1324, Best Val-Loss: 0.1255, Val-F2: 0.913\n",
      "1 [13/1000] 95s, Loss: 0.1378, Val-Loss: 0.1303, Best Val-Loss: 0.1255, Val-F2: 0.913\n",
      "1 [14/1000] 93s, Loss: 0.1374, Val-Loss: 0.1284, Best Val-Loss: 0.1255, Val-F2: 0.914\n",
      "1 [15/1000] 94s, Loss: 0.1368, Val-Loss: 0.1304, Best Val-Loss: 0.1255, Val-F2: 0.912\n",
      "Update Model!\n",
      "1 [16/1000] 94s, Loss: 0.1352, Val-Loss: 0.1167, Best Val-Loss: 0.1167, Val-F2: 0.923\n",
      "1 [17/1000] 94s, Loss: 0.1340, Val-Loss: 0.1274, Best Val-Loss: 0.1167, Val-F2: 0.915\n",
      "1 [18/1000] 94s, Loss: 0.1315, Val-Loss: 0.1303, Best Val-Loss: 0.1167, Val-F2: 0.913\n",
      "1 [19/1000] 94s, Loss: 0.1353, Val-Loss: 0.1298, Best Val-Loss: 0.1167, Val-F2: 0.913\n",
      "1 [20/1000] 94s, Loss: 0.1311, Val-Loss: 0.1352, Best Val-Loss: 0.1167, Val-F2: 0.908\n",
      "1 [21/1000] 94s, Loss: 0.1293, Val-Loss: 0.1267, Best Val-Loss: 0.1167, Val-F2: 0.915\n",
      "1 [22/1000] 94s, Loss: 0.1301, Val-Loss: 0.1262, Best Val-Loss: 0.1167, Val-F2: 0.916\n",
      "1 [23/1000] 94s, Loss: 0.1310, Val-Loss: 0.1383, Best Val-Loss: 0.1167, Val-F2: 0.906\n",
      "New Learn Rate: 1e-05!\n",
      "1 [24/1000] 94s, Loss: 0.1297, Val-Loss: 0.1214, Best Val-Loss: 0.1167, Val-F2: 0.917\n",
      "1 [25/1000] 94s, Loss: 0.1264, Val-Loss: 0.1186, Best Val-Loss: 0.1167, Val-F2: 0.920\n",
      "1 [26/1000] 94s, Loss: 0.1285, Val-Loss: 0.1221, Best Val-Loss: 0.1167, Val-F2: 0.918\n",
      "1 [27/1000] 94s, Loss: 0.1236, Val-Loss: 0.1190, Best Val-Loss: 0.1167, Val-F2: 0.921\n",
      "1 [28/1000] 94s, Loss: 0.1279, Val-Loss: 0.1168, Best Val-Loss: 0.1167, Val-F2: 0.923\n",
      "Update Model!\n",
      "1 [29/1000] 94s, Loss: 0.1289, Val-Loss: 0.1143, Best Val-Loss: 0.1143, Val-F2: 0.923\n",
      "1 [30/1000] 94s, Loss: 0.1297, Val-Loss: 0.1179, Best Val-Loss: 0.1143, Val-F2: 0.920\n",
      "1 [31/1000] 94s, Loss: 0.1268, Val-Loss: 0.1306, Best Val-Loss: 0.1143, Val-F2: 0.912\n",
      "1 [32/1000] 94s, Loss: 0.1285, Val-Loss: 0.1248, Best Val-Loss: 0.1143, Val-F2: 0.916\n",
      "1 [33/1000] 94s, Loss: 0.1265, Val-Loss: 0.1244, Best Val-Loss: 0.1143, Val-F2: 0.917\n",
      "1 [34/1000] 94s, Loss: 0.1254, Val-Loss: 0.1175, Best Val-Loss: 0.1143, Val-F2: 0.922\n",
      "1 [35/1000] 94s, Loss: 0.1272, Val-Loss: 0.1260, Best Val-Loss: 0.1143, Val-F2: 0.916\n",
      "1 [36/1000] 94s, Loss: 0.1257, Val-Loss: 0.1190, Best Val-Loss: 0.1143, Val-F2: 0.921\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "1 [37/1000] 91s, Loss: 0.1284, Val-Loss: 0.1215, Best Val-Loss: 0.1143, Val-F2: 0.919\n",
      "1 [38/1000] 90s, Loss: 0.1277, Val-Loss: 0.1189, Best Val-Loss: 0.1143, Val-F2: 0.920\n",
      "1 [39/1000] 90s, Loss: 0.1261, Val-Loss: 0.1189, Best Val-Loss: 0.1143, Val-F2: 0.921\n",
      "1 [40/1000] 90s, Loss: 0.1283, Val-Loss: 0.1171, Best Val-Loss: 0.1143, Val-F2: 0.922\n",
      "1 [41/1000] 90s, Loss: 0.1254, Val-Loss: 0.1201, Best Val-Loss: 0.1143, Val-F2: 0.920\n",
      "1 [42/1000] 90s, Loss: 0.1301, Val-Loss: 0.1282, Best Val-Loss: 0.1143, Val-F2: 0.913\n",
      "1 [43/1000] 90s, Loss: 0.1270, Val-Loss: 0.1224, Best Val-Loss: 0.1143, Val-F2: 0.918\n",
      "1 [44/1000] 90s, Loss: 0.1254, Val-Loss: 0.1233, Best Val-Loss: 0.1143, Val-F2: 0.918\n",
      "New Learn Rate: 1.0000000000000002e-07!\n",
      "Early Stop in Epoch: 45, Best Val-Loss: 0.114303, Best F2: 0.950785\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'MyNetfirst4' object has no attribute 'MyNetfirst4'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-183-f1bd281ddb5d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     18\u001b[0m     \u001b[1;31m# model = model.MyNet(17).cuda()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m     \u001b[1;31m# model = model.ResNet(num_classes=17, block=resnet.BasicBlock).cuda()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 20\u001b[1;33m     \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mMyNetfirst4\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     21\u001b[0m     \u001b[0mcriterion\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mBCELoss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m     \u001b[1;31m# criterion = nn.CrossEntropyLoss(weight=None).cuda()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mJ:\\Anaconda3\\envs\\py36\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, name)\u001b[0m\n\u001b[0;32m    236\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mmodules\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    237\u001b[0m         raise AttributeError(\"'{}' object has no attribute '{}'\".format(\n\u001b[1;32m--> 238\u001b[1;33m             type(self).__name__, name))\n\u001b[0m\u001b[0;32m    239\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'MyNetfirst4' object has no attribute 'MyNetfirst4'"
     ]
    }
   ],
   "source": [
    "n_splits = 5\n",
    "model_name = 'first4'\n",
    "date = '716'\n",
    "probs1 = [1.42,   5.61,  14.96,  19.24]\n",
    "probs = [0.5, 0.6, 0.8, 0.8, 0.5, 0.5, 0.6, 0.6, 0.8, 0.8, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from itertools import islice\n",
    "file_all = train_set['image_name'].values\n",
    "test_file_all = test_set['image_name'].values\n",
    "y_all = utils.get_y(train_set['tags'].values, label_map)[:, 0:4]\n",
    "pred_tr = np.zeros((file_all.shape[0], 4))\n",
    "pred_ts = np.zeros((test_file_all.shape[0], 4))\n",
    "kf = KFold(n_splits = n_splits)\n",
    "\n",
    "k_now = 0\n",
    "for i_tr, i_vl in islice(kf.split(y_all), 0, None):\n",
    "    # model = m.MyNet(17).cuda()\n",
    "    # model = m.ResNet(num_classes=17, block=resnet.BasicBlock).cuda()\n",
    "    model = m.MyNetfirst4(4).cuda()\n",
    "    criterion = nn.BCELoss(weight=None).cuda()\n",
    "    # criterion = nn.CrossEntropyLoss(weight=None).cuda()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    estimator = core.Estimator(model, criterion, optimizer, 4)\n",
    "    best_model = core.BestModel()\n",
    "    \n",
    "    for epoch in range(num_epoches):\n",
    "        time_st = time()\n",
    "        # 训练\n",
    "        train_loader = utils.weighted_train_loader(file_all[i_tr], y_all[i_tr], probs1, transform_tr, batch_size, pic_size)\n",
    "        loss_tr = estimator.train(train_loader)\n",
    "\n",
    "        # 验证\n",
    "        val_loader = utils.valid_loader(file_all[i_vl], y_all[i_vl], transform_vl, batch_size, pic_size)\n",
    "        loss_vl, f2_vl = estimator.validate(val_loader)\n",
    "        best_model.update(loss_vl.avg, f2_vl, estimator.model)\n",
    "        \n",
    "        # 若验证结果提升缓慢，减小学习率\n",
    "        if epoch > lr_tolerance and best_model.lrcount > lr_tolerance:\n",
    "            optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * 0.1\n",
    "            print('New Learn Rate: {}!'.format(optimizer.param_groups[0]['lr']))\n",
    "            best_model.lrcount = 0\n",
    "        \n",
    "        # 若验证结果不再提升，保存模型、验证结果、预测结果，跳出迭代\n",
    "        if epoch > tolerance and best_model.nobetter > tolerance:\n",
    "            print('Early Stop in Epoch: {}, Best Val-Loss: {:.6f}, Best F2: {:.6f}'.format(\n",
    "                epoch+1, best_model.best_loss, best_model.best_f2.value(bestf2=True)))\n",
    "            pred_tr[i_vl, :] = f2_vl.preds\n",
    "            best_model.save('./model/model{}-date{}-kf{}.pth'.format(model_name, date, k_now+1))\n",
    "                                                               \n",
    "            tst_loader = utils.test_loader(test_file_all, transform_vl, batch_size, pic_size)\n",
    "            estimator.model.load_state_dict(best_model.best_model)                                       \n",
    "            pred_ts_temp = estimator.predict(tst_loader)\n",
    "            pred_ts =  pred_ts_temp / float(n_splits)\n",
    "            break\n",
    "        \n",
    "        # 打印每一次迭代的训练验证成绩\n",
    "        print('{} [{}/{}] {}s, Loss: {:.4f}, Val-Loss: {:.4f}, Best Val-Loss: {:.4f}, Val-F2: {:.3f}'.format(\n",
    "            k_now+1, epoch+1, num_epoches, int(time() - time_st), loss_tr.avg, loss_vl.avg, \n",
    "            best_model.best_loss, f2_vl.value(0.3)))\n",
    "        gc.collect()\n",
    "    k_now += 1\n",
    "\n",
    "# 序列化验证和预测结果，用于stacking\n",
    "np.save('./pred/model{}_date{}_pred_train.npy'.format(model_name, date), pred_tr)\n",
    "np.save('./pred/model{}_date{}_pred_test.npy'.format(model_name, date), pred_ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Update Model!\n",
      "1 [1/1000] 362s, Loss: 0.3675, Val-Loss: 0.1649, Best Val-Loss: 0.1649, Val-F2: 0.83\n",
      "Update Model!\n",
      "1 [2/1000] 184s, Loss: 0.2144, Val-Loss: 0.1512, Best Val-Loss: 0.1512, Val-F2: 0.85\n",
      "Update Model!\n",
      "1 [3/1000] 136s, Loss: 0.2001, Val-Loss: 0.1428, Best Val-Loss: 0.1428, Val-F2: 0.86\n",
      "Update Model!\n",
      "1 [4/1000] 116s, Loss: 0.1891, Val-Loss: 0.1397, Best Val-Loss: 0.1397, Val-F2: 0.86\n",
      "Update Model!\n",
      "1 [5/1000] 110s, Loss: 0.1811, Val-Loss: 0.1271, Best Val-Loss: 0.1271, Val-F2: 0.88\n",
      "1 [6/1000] 103s, Loss: 0.1758, Val-Loss: 0.1353, Best Val-Loss: 0.1271, Val-F2: 0.87\n",
      "Update Model!\n",
      "1 [7/1000] 99s, Loss: 0.1726, Val-Loss: 0.1258, Best Val-Loss: 0.1258, Val-F2: 0.88\n",
      "1 [8/1000] 97s, Loss: 0.1674, Val-Loss: 0.1281, Best Val-Loss: 0.1258, Val-F2: 0.88\n",
      "Update Model!\n",
      "1 [9/1000] 96s, Loss: 0.1666, Val-Loss: 0.1157, Best Val-Loss: 0.1157, Val-F2: 0.89\n",
      "1 [10/1000] 94s, Loss: 0.1620, Val-Loss: 0.1249, Best Val-Loss: 0.1157, Val-F2: 0.88\n",
      "1 [11/1000] 133s, Loss: 0.1592, Val-Loss: 0.1178, Best Val-Loss: 0.1157, Val-F2: 0.89\n",
      "1 [12/1000] 116s, Loss: 0.1554, Val-Loss: 0.1223, Best Val-Loss: 0.1157, Val-F2: 0.88\n",
      "Update Model!\n",
      "1 [13/1000] 108s, Loss: 0.1524, Val-Loss: 0.1106, Best Val-Loss: 0.1106, Val-F2: 0.90\n",
      "Update Model!\n",
      "1 [14/1000] 103s, Loss: 0.1509, Val-Loss: 0.1104, Best Val-Loss: 0.1104, Val-F2: 0.90\n",
      "1 [15/1000] 120s, Loss: 0.1501, Val-Loss: 0.1138, Best Val-Loss: 0.1104, Val-F2: 0.89\n",
      "Update Model!\n",
      "1 [16/1000] 119s, Loss: 0.1466, Val-Loss: 0.1047, Best Val-Loss: 0.1047, Val-F2: 0.90\n",
      "1 [17/1000] 141s, Loss: 0.1471, Val-Loss: 0.1058, Best Val-Loss: 0.1047, Val-F2: 0.90\n",
      "1 [18/1000] 160s, Loss: 0.1457, Val-Loss: 0.1061, Best Val-Loss: 0.1047, Val-F2: 0.90\n",
      "1 [19/1000] 146s, Loss: 0.1439, Val-Loss: 0.1094, Best Val-Loss: 0.1047, Val-F2: 0.90\n",
      "1 [20/1000] 145s, Loss: 0.1424, Val-Loss: 0.1115, Best Val-Loss: 0.1047, Val-F2: 0.90\n",
      "1 [21/1000] 145s, Loss: 0.1417, Val-Loss: 0.1059, Best Val-Loss: 0.1047, Val-F2: 0.90\n",
      "1 [22/1000] 182s, Loss: 0.1409, Val-Loss: 0.1103, Best Val-Loss: 0.1047, Val-F2: 0.89\n",
      "Update Model!\n",
      "1 [23/1000] 188s, Loss: 0.1378, Val-Loss: 0.1046, Best Val-Loss: 0.1046, Val-F2: 0.90\n",
      "Update Model!\n",
      "1 [24/1000] 152s, Loss: 0.1374, Val-Loss: 0.1020, Best Val-Loss: 0.1020, Val-F2: 0.91\n",
      "Update Model!\n",
      "1 [25/1000] 129s, Loss: 0.1375, Val-Loss: 0.1015, Best Val-Loss: 0.1015, Val-F2: 0.91\n",
      "Update Model!\n",
      "1 [26/1000] 118s, Loss: 0.1347, Val-Loss: 0.1008, Best Val-Loss: 0.1008, Val-F2: 0.91\n",
      "Update Model!\n",
      "1 [27/1000] 119s, Loss: 0.1341, Val-Loss: 0.0990, Best Val-Loss: 0.0990, Val-F2: 0.91\n",
      "1 [28/1000] 251s, Loss: 0.1335, Val-Loss: 0.1003, Best Val-Loss: 0.0990, Val-F2: 0.91\n",
      "1 [29/1000] 180s, Loss: 0.1323, Val-Loss: 0.1069, Best Val-Loss: 0.0990, Val-F2: 0.90\n",
      "1 [30/1000] 164s, Loss: 0.1313, Val-Loss: 0.1011, Best Val-Loss: 0.0990, Val-F2: 0.91\n",
      "1 [31/1000] 292s, Loss: 0.1298, Val-Loss: 0.1015, Best Val-Loss: 0.0990, Val-F2: 0.91\n",
      "1 [32/1000] 217s, Loss: 0.1296, Val-Loss: 0.1038, Best Val-Loss: 0.0990, Val-F2: 0.91\n",
      "1 [33/1000] 212s, Loss: 0.1278, Val-Loss: 0.1000, Best Val-Loss: 0.0990, Val-F2: 0.91\n",
      "1 [34/1000] 205s, Loss: 0.1251, Val-Loss: 0.1033, Best Val-Loss: 0.0990, Val-F2: 0.90\n",
      "New Learn Rate: 0.0001!\n",
      "1 [35/1000] 232s, Loss: 0.1244, Val-Loss: 0.1005, Best Val-Loss: 0.0990, Val-F2: 0.91\n",
      "Update Model!\n",
      "1 [36/1000] 199s, Loss: 0.1208, Val-Loss: 0.0960, Best Val-Loss: 0.0960, Val-F2: 0.92\n",
      "Update Model!\n",
      "1 [37/1000] 315s, Loss: 0.1202, Val-Loss: 0.0954, Best Val-Loss: 0.0954, Val-F2: 0.92\n",
      "Update Model!\n",
      "1 [38/1000] 212s, Loss: 0.1175, Val-Loss: 0.0949, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [39/1000] 189s, Loss: 0.1176, Val-Loss: 0.0954, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [40/1000] 204s, Loss: 0.1167, Val-Loss: 0.0956, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [41/1000] 215s, Loss: 0.1176, Val-Loss: 0.0959, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [42/1000] 216s, Loss: 0.1155, Val-Loss: 0.0954, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "Update Model!\n",
      "1 [43/1000] 262s, Loss: 0.1152, Val-Loss: 0.0949, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [44/1000] 296s, Loss: 0.1150, Val-Loss: 0.0965, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [45/1000] 322s, Loss: 0.1150, Val-Loss: 0.0953, Best Val-Loss: 0.0949, Val-F2: 0.91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "J:\\Anaconda3\\envs\\py36\\lib\\site-packages\\sklearn\\metrics\\classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [46/1000] 234s, Loss: 0.1135, Val-Loss: 0.0965, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [47/1000] 290s, Loss: 0.1142, Val-Loss: 0.0968, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [48/1000] 350s, Loss: 0.1127, Val-Loss: 0.0968, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [49/1000] 367s, Loss: 0.1131, Val-Loss: 0.0962, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [50/1000] 348s, Loss: 0.1137, Val-Loss: 0.0955, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "New Learn Rate: 1e-05!\n",
      "1 [51/1000] 313s, Loss: 0.1133, Val-Loss: 0.0949, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [52/1000] 285s, Loss: 0.1142, Val-Loss: 0.0954, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [53/1000] 319s, Loss: 0.1123, Val-Loss: 0.0954, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [54/1000] 235s, Loss: 0.1112, Val-Loss: 0.0954, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [55/1000] 159s, Loss: 0.1118, Val-Loss: 0.0954, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "1 [56/1000] 149s, Loss: 0.1118, Val-Loss: 0.0956, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [57/1000] 153s, Loss: 0.1125, Val-Loss: 0.0954, Best Val-Loss: 0.0949, Val-F2: 0.92\n",
      "1 [58/1000] 148s, Loss: 0.1126, Val-Loss: 0.0960, Best Val-Loss: 0.0949, Val-F2: 0.91\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "Early Stop in Epoch: 59, Best Val-Loss: 0.094861, Best F2: 0.918830\n",
      "Update Model!\n",
      "2 [1/1000] 381s, Loss: 0.3769, Val-Loss: 0.1749, Best Val-Loss: 0.1749, Val-F2: 0.82\n",
      "Update Model!\n",
      "2 [2/1000] 203s, Loss: 0.2169, Val-Loss: 0.1609, Best Val-Loss: 0.1609, Val-F2: 0.84\n",
      "Update Model!\n",
      "2 [3/1000] 192s, Loss: 0.1980, Val-Loss: 0.1490, Best Val-Loss: 0.1490, Val-F2: 0.85\n",
      "2 [4/1000] 194s, Loss: 0.1896, Val-Loss: 0.1498, Best Val-Loss: 0.1490, Val-F2: 0.85\n",
      "Update Model!\n",
      "2 [5/1000] 184s, Loss: 0.1824, Val-Loss: 0.1317, Best Val-Loss: 0.1317, Val-F2: 0.88\n",
      "Update Model!\n",
      "2 [6/1000] 153s, Loss: 0.1779, Val-Loss: 0.1314, Best Val-Loss: 0.1314, Val-F2: 0.87\n",
      "2 [7/1000] 128s, Loss: 0.1726, Val-Loss: 0.1353, Best Val-Loss: 0.1314, Val-F2: 0.86\n",
      "Update Model!\n",
      "2 [8/1000] 138s, Loss: 0.1699, Val-Loss: 0.1199, Best Val-Loss: 0.1199, Val-F2: 0.88\n",
      "Update Model!\n",
      "2 [9/1000] 152s, Loss: 0.1664, Val-Loss: 0.1188, Best Val-Loss: 0.1188, Val-F2: 0.89\n",
      "2 [10/1000] 148s, Loss: 0.1627, Val-Loss: 0.1203, Best Val-Loss: 0.1188, Val-F2: 0.88\n",
      "Update Model!\n",
      "2 [11/1000] 207s, Loss: 0.1604, Val-Loss: 0.1168, Best Val-Loss: 0.1168, Val-F2: 0.89\n",
      "Update Model!\n",
      "2 [12/1000] 186s, Loss: 0.1596, Val-Loss: 0.1149, Best Val-Loss: 0.1149, Val-F2: 0.90\n",
      "Update Model!\n",
      "2 [13/1000] 183s, Loss: 0.1542, Val-Loss: 0.1133, Best Val-Loss: 0.1133, Val-F2: 0.89\n",
      "Update Model!\n",
      "2 [14/1000] 186s, Loss: 0.1537, Val-Loss: 0.1083, Best Val-Loss: 0.1083, Val-F2: 0.90\n",
      "2 [15/1000] 207s, Loss: 0.1525, Val-Loss: 0.1147, Best Val-Loss: 0.1083, Val-F2: 0.89\n",
      "2 [16/1000] 209s, Loss: 0.1496, Val-Loss: 0.1088, Best Val-Loss: 0.1083, Val-F2: 0.90\n",
      "Update Model!\n",
      "2 [17/1000] 209s, Loss: 0.1482, Val-Loss: 0.1074, Best Val-Loss: 0.1074, Val-F2: 0.90\n",
      "2 [18/1000] 388s, Loss: 0.1469, Val-Loss: 0.1117, Best Val-Loss: 0.1074, Val-F2: 0.90\n",
      "Update Model!\n",
      "2 [19/1000] 379s, Loss: 0.1437, Val-Loss: 0.1072, Best Val-Loss: 0.1072, Val-F2: 0.90\n",
      "Update Model!\n",
      "2 [20/1000] 438s, Loss: 0.1431, Val-Loss: 0.1044, Best Val-Loss: 0.1044, Val-F2: 0.90\n",
      "2 [21/1000] 500s, Loss: 0.1416, Val-Loss: 0.1064, Best Val-Loss: 0.1044, Val-F2: 0.90\n",
      "Update Model!\n",
      "2 [22/1000] 383s, Loss: 0.1389, Val-Loss: 0.1017, Best Val-Loss: 0.1017, Val-F2: 0.91\n",
      "2 [23/1000] 387s, Loss: 0.1383, Val-Loss: 0.1066, Best Val-Loss: 0.1017, Val-F2: 0.90\n",
      "2 [24/1000] 203s, Loss: 0.1391, Val-Loss: 0.1072, Best Val-Loss: 0.1017, Val-F2: 0.90\n",
      "2 [25/1000] 147s, Loss: 0.1368, Val-Loss: 0.1054, Best Val-Loss: 0.1017, Val-F2: 0.90\n",
      "Update Model!\n",
      "2 [26/1000] 123s, Loss: 0.1361, Val-Loss: 0.0991, Best Val-Loss: 0.0991, Val-F2: 0.91\n",
      "2 [27/1000] 112s, Loss: 0.1344, Val-Loss: 0.1057, Best Val-Loss: 0.0991, Val-F2: 0.90\n",
      "2 [28/1000] 107s, Loss: 0.1329, Val-Loss: 0.1020, Best Val-Loss: 0.0991, Val-F2: 0.91\n",
      "2 [29/1000] 104s, Loss: 0.1310, Val-Loss: 0.1013, Best Val-Loss: 0.0991, Val-F2: 0.91\n",
      "2 [30/1000] 103s, Loss: 0.1320, Val-Loss: 0.1045, Best Val-Loss: 0.0991, Val-F2: 0.90\n",
      "2 [31/1000] 108s, Loss: 0.1303, Val-Loss: 0.1035, Best Val-Loss: 0.0991, Val-F2: 0.91\n",
      "2 [32/1000] 104s, Loss: 0.1297, Val-Loss: 0.1017, Best Val-Loss: 0.0991, Val-F2: 0.91\n",
      "2 [33/1000] 103s, Loss: 0.1288, Val-Loss: 0.1022, Best Val-Loss: 0.0991, Val-F2: 0.91\n",
      "New Learn Rate: 0.0001!\n",
      "2 [34/1000] 113s, Loss: 0.1281, Val-Loss: 0.1046, Best Val-Loss: 0.0991, Val-F2: 0.90\n",
      "Update Model!\n",
      "2 [35/1000] 164s, Loss: 0.1231, Val-Loss: 0.0969, Best Val-Loss: 0.0969, Val-F2: 0.91\n",
      "Update Model!\n",
      "2 [36/1000] 161s, Loss: 0.1202, Val-Loss: 0.0963, Best Val-Loss: 0.0963, Val-F2: 0.91\n",
      "Update Model!\n",
      "2 [37/1000] 166s, Loss: 0.1189, Val-Loss: 0.0959, Best Val-Loss: 0.0959, Val-F2: 0.91\n",
      "Update Model!\n",
      "2 [38/1000] 156s, Loss: 0.1189, Val-Loss: 0.0955, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "2 [39/1000] 165s, Loss: 0.1187, Val-Loss: 0.0963, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "2 [40/1000] 197s, Loss: 0.1190, Val-Loss: 0.0961, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "2 [41/1000] 188s, Loss: 0.1177, Val-Loss: 0.0963, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "2 [42/1000] 1672s, Loss: 0.1170, Val-Loss: 0.0966, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "2 [43/1000] 291s, Loss: 0.1175, Val-Loss: 0.0960, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "2 [44/1000] 190s, Loss: 0.1177, Val-Loss: 0.0955, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "2 [45/1000] 223s, Loss: 0.1162, Val-Loss: 0.0960, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "New Learn Rate: 1e-05!\n",
      "2 [46/1000] 533s, Loss: 0.1171, Val-Loss: 0.0962, Best Val-Loss: 0.0955, Val-F2: 0.91\n",
      "Update Model!\n",
      "2 [47/1000] 389s, Loss: 0.1155, Val-Loss: 0.0953, Best Val-Loss: 0.0953, Val-F2: 0.92\n",
      "2 [48/1000] 218s, Loss: 0.1150, Val-Loss: 0.0954, Best Val-Loss: 0.0953, Val-F2: 0.92\n",
      "2 [49/1000] 165s, Loss: 0.1152, Val-Loss: 0.0960, Best Val-Loss: 0.0953, Val-F2: 0.91\n",
      "2 [50/1000] 273s, Loss: 0.1150, Val-Loss: 0.0963, Best Val-Loss: 0.0953, Val-F2: 0.91\n",
      "2 [51/1000] 569s, Loss: 0.1148, Val-Loss: 0.0958, Best Val-Loss: 0.0953, Val-F2: 0.91\n",
      "2 [52/1000] 440s, Loss: 0.1147, Val-Loss: 0.0955, Best Val-Loss: 0.0953, Val-F2: 0.91\n",
      "2 [53/1000] 432s, Loss: 0.1164, Val-Loss: 0.0960, Best Val-Loss: 0.0953, Val-F2: 0.91\n",
      "2 [54/1000] 399s, Loss: 0.1136, Val-Loss: 0.0962, Best Val-Loss: 0.0953, Val-F2: 0.91\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "2 [55/1000] 522s, Loss: 0.1138, Val-Loss: 0.0957, Best Val-Loss: 0.0953, Val-F2: 0.92\n",
      "2 [56/1000] 363s, Loss: 0.1146, Val-Loss: 0.0957, Best Val-Loss: 0.0953, Val-F2: 0.91\n"
     ]
    }
   ],
   "source": [
    "n_splits = 5\n",
    "model_name = 'resnet1'\n",
    "date = '715'\n",
    "probs = [0.5, 0.6, 0.8, 0.8, 0.5, 0.5, 0.6, 0.6, 0.8, 0.8, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from itertools import islice\n",
    "file_all = train_set['image_name'].values\n",
    "test_file_all = test_set['image_name'].values\n",
    "y_all = utils.get_y(train_set['tags'].values, label_map)\n",
    "pred_tr = np.zeros((file_all.shape[0], 17))\n",
    "pred_ts = np.zeros((test_file_all.shape[0], 17))\n",
    "kf = KFold(n_splits = n_splits)\n",
    "\n",
    "k_now = 0\n",
    "for i_tr, i_vl in islice(kf.split(y_all), 0, None):\n",
    "    # model = core.MyNet(17).cuda()\n",
    "    model = resnet.ResNet(pic_width=pic_size[0], num_classes=17, block=resnet.BasicBlock).cuda()\n",
    "    criterion = nn.BCELoss(weight=None).cuda()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    estimator = core.Estimator(model, criterion, optimizer)\n",
    "    best_model = core.BestModel()\n",
    "    \n",
    "    for epoch in range(num_epoches):\n",
    "        time_st = time()\n",
    "        # 训练\n",
    "        train_loader = utils.weighted_train_loader(file_all[i_tr], y_all[i_tr], probs, transform_tr, batch_size, pic_size)\n",
    "        loss_tr = estimator.train(train_loader)\n",
    "\n",
    "        # 验证\n",
    "        val_loader = utils.valid_loader(file_all[i_vl], y_all[i_vl], transform_vl, batch_size, pic_size)\n",
    "        loss_vl, f2_vl = estimator.validate(val_loader)\n",
    "        best_model.update(loss_vl.avg, f2_vl, estimator.model)\n",
    "        \n",
    "        # 若验证结果提升缓慢，减小学习率\n",
    "        if epoch > lr_tolerance and best_model.lrcount > lr_tolerance:\n",
    "            optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * 0.1\n",
    "            print('New Learn Rate: {}!'.format(optimizer.param_groups[0]['lr']))\n",
    "            best_model.lrcount = 0\n",
    "        \n",
    "        # 若验证结果不再提升，保存模型、验证结果、预测结果，跳出迭代\n",
    "        if epoch > tolerance and best_model.nobetter > tolerance:\n",
    "            print('Early Stop in Epoch: {}, Best Val-Loss: {:.6f}, Best F2: {:.6f}'.format(\n",
    "                epoch+1, best_model.best_loss, best_model.best_f2.value(bestf2=True)))\n",
    "            pred_tr[i_vl, :] = f2_vl.preds\n",
    "            best_model.save('./model/model{}-date{}-kf{}.pth'.format(model_name, date, k_now+1))\n",
    "                                                               \n",
    "            tst_loader = utils.test_loader(test_file_all, transform_vl, batch_size, pic_size)\n",
    "            estimator.model.load_state_dict(best_model.best_model)                                       \n",
    "            pred_ts_temp = estimator.predict(tst_loader)\n",
    "            pred_ts =  pred_ts_temp / float(n_splits)\n",
    "            break\n",
    "        \n",
    "        # 打印每一次迭代的训练验证成绩\n",
    "        print('{} [{}/{}] {}s, Loss: {:.4f}, Val-Loss: {:.4f}, Best Val-Loss: {:.4f}, Val-F2: {:.3f}'.format(\n",
    "            k_now+1, epoch+1, num_epoches, int(time() - time_st), loss_tr.avg, loss_vl.avg, \n",
    "            best_model.best_loss, f2_vl.value(0.3)))\n",
    "        gc.collect()\n",
    "    k_now += 1\n",
    "\n",
    "# 序列化验证和预测结果，用于stacking\n",
    "np.save('./pred/model{}_date{}_pred_train.npy'.format(model_name, date), pred_tr)\n",
    "np.save('./pred/model{}_date{}_pred_test.npy'.format(model_name, date), pred_ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imp.reload(core)\n",
    "imp.reload(utils)\n",
    "imp.reload(mydataset)\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from itertools import islice\n",
    "file_all = train_set['image_name'].values\n",
    "test_file_all = test_set['image_name'].values\n",
    "y_all = utils.get_y(train_set['tags'].values, label_map)\n",
    "kf = KFold(n_splits = 5)\n",
    "\n",
    "pred_tr = np.zeros((file_all.shape[0], 17))\n",
    "pred_ts = np.zeros((test_file_all.shape[0], 17))\n",
    "model = core.MyNet(17).cuda()\n",
    "criterion = nn.BCELoss(weight=None).cuda()\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "estimator = core.Estimator(model, criterion, optimizer)\n",
    "k_now = 1\n",
    "for i_tr, i_vl in islice(kf.split(y_all), 0, None):\n",
    "    best_model = core.BestModel()\n",
    "    estimator.model.load_state_dict(torch.load('modelweightedNN1-date710-kf{}.pth'.format(k_now)))\n",
    "    val_loader = utils.valid_loader(file_all[i_vl], y_all[i_vl], transform_vl, batch_size, pic_size)\n",
    "    loss_vl, f2_vl = estimator.validate(val_loader)\n",
    "    pred_tr[i_vl] = f2_vl.preds\n",
    "    \n",
    "    tst_loader = utils.test_loader(test_file_all, transform_vl, batch_size, pic_size)\n",
    "    pred_ts += estimator.predict(tst_loader) / 5.0\n",
    "    k_now += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.912293546792\n"
     ]
    }
   ],
   "source": [
    "th1 = utils.f2_opti_score(y_all, pred_tr, thresholds = np.arange(0, 1, 0.01), num_classes=17)\n",
    "th2 = utils.f2_opti_score(y_all, pred_tr, thresholds = np.arange(1, 0, -0.01), num_classes=17)\n",
    "th = (th1 + th2) / 2.0\n",
    "print(utils.f2_score(y_all, pred_tr, th))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imp.reload(utils)\n",
    "submit_df = utils.to_submit(pred_ts, th, test_set, inv_label_map)\n",
    "submit_df.to_csv('./submit/model{}_date{}_no{}.csv'.format(model_name, date, 1), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('./pred/model{}_date{}_pred_train.npy'.format(model_name, date), pred_tr)\n",
    "np.save('./pred/model{}_date{}_pred_test.npy'.format(model_name, date), pred_ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.921965630042\n"
     ]
    }
   ],
   "source": [
    "y_all = utils.get_y(train_set['tags'].values, label_map)\n",
    "wnnts = np.load('./pred/modelweightedNN1_date710_pred_test.npy')\n",
    "wnntr = np.load('./pred/modelweightedNN1_date710_pred_train.npy')\n",
    "th1 = utils.f2_opti_score(y_all, wnntr, thresholds = np.arange(0, 1, 0.01), num_classes=17)\n",
    "th2 = utils.f2_opti_score(y_all, wnntr, thresholds = np.arange(1, 0, -0.01), num_classes=17)\n",
    "th = (th1 + th2) / 2.0\n",
    "print(utils.f2_score(y_all, wnntr, th))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "imp.reload(utils)\n",
    "sample_weight_factors = np.zeros(wnntr.shape[0]) + 0.1\n",
    "wnntr_01 = utils.to_01(wnntr, th)\n",
    "sample_weight_factors[(wnntr_01 != y_all).sum(axis=1) > 0] = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import nn, optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision import transforms\n",
    "import random\n",
    "import model as m\n",
    "from PIL import Image\n",
    "from time import time\n",
    "import mydataset\n",
    "import gc\n",
    "imp.reload(mydataset)\n",
    "imp.reload(core)\n",
    "imp.reload(utils)\n",
    "imp.reload(m)\n",
    "for i in range(10):\n",
    "    gc.collect()\n",
    "\n",
    "# 超参数\n",
    "batch_size = 256\n",
    "pic_size = (64, 64)\n",
    "learning_rate = 1e-3\n",
    "num_epoches = 1000\n",
    "tolerance = 15\n",
    "lr_tolerance = 7\n",
    "transform1 = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "])\n",
    "transform2 = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])\n",
    "])\n",
    "\n",
    "def transform_tr(img, pic_size):\n",
    "    img = img.resize(pic_size)\n",
    "    img = img.transpose(Image.FLIP_LEFT_RIGHT)  if random.randint(0, 1) > .5 else img\n",
    "    img = img.transpose(Image.FLIP_TOP_BOTTOM)  if random.randint(0, 1) > .5 else img\n",
    "    img.rotate(np.random.random() * 45)\n",
    "    img_tensor = transform2(img)\n",
    "    return img_tensor\n",
    "\n",
    "def transform_vl(img, pic_size):\n",
    "    img = img.resize(pic_size)\n",
    "    img_tensor = transform2(img)\n",
    "    return img_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Update Model!\n",
      "1 [1/1000] 93s, Loss: 0.3804, Val-Loss: 0.1811, Best Val-Loss: 0.1811, Val-F2: 0.806\n",
      "Update Model!\n",
      "1 [2/1000] 95s, Loss: 0.2187, Val-Loss: 0.1419, Best Val-Loss: 0.1419, Val-F2: 0.863\n",
      "1 [3/1000] 94s, Loss: 0.2085, Val-Loss: 0.1445, Best Val-Loss: 0.1419, Val-F2: 0.856\n",
      "Update Model!\n",
      "1 [4/1000] 95s, Loss: 0.2004, Val-Loss: 0.1298, Best Val-Loss: 0.1298, Val-F2: 0.878\n",
      "1 [5/1000] 93s, Loss: 0.1936, Val-Loss: 0.1337, Best Val-Loss: 0.1298, Val-F2: 0.868\n",
      "Update Model!\n",
      "1 [6/1000] 93s, Loss: 0.1889, Val-Loss: 0.1293, Best Val-Loss: 0.1293, Val-F2: 0.884\n",
      "Update Model!\n",
      "1 [7/1000] 94s, Loss: 0.1856, Val-Loss: 0.1270, Best Val-Loss: 0.1270, Val-F2: 0.881\n",
      "Update Model!\n",
      "1 [8/1000] 96s, Loss: 0.1831, Val-Loss: 0.1178, Best Val-Loss: 0.1178, Val-F2: 0.894\n",
      "1 [9/1000] 99s, Loss: 0.1798, Val-Loss: 0.1186, Best Val-Loss: 0.1178, Val-F2: 0.891\n",
      "Update Model!\n",
      "1 [10/1000] 95s, Loss: 0.1762, Val-Loss: 0.1082, Best Val-Loss: 0.1082, Val-F2: 0.901\n",
      "1 [11/1000] 95s, Loss: 0.1755, Val-Loss: 0.1097, Best Val-Loss: 0.1082, Val-F2: 0.900\n",
      "1 [12/1000] 94s, Loss: 0.1717, Val-Loss: 0.1111, Best Val-Loss: 0.1082, Val-F2: 0.901\n",
      "1 [13/1000] 94s, Loss: 0.1705, Val-Loss: 0.1125, Best Val-Loss: 0.1082, Val-F2: 0.898\n",
      "Update Model!\n",
      "1 [14/1000] 93s, Loss: 0.1690, Val-Loss: 0.1064, Best Val-Loss: 0.1064, Val-F2: 0.901\n",
      "1 [15/1000] 95s, Loss: 0.1660, Val-Loss: 0.1069, Best Val-Loss: 0.1064, Val-F2: 0.901\n",
      "Update Model!\n",
      "1 [16/1000] 93s, Loss: 0.1650, Val-Loss: 0.1040, Best Val-Loss: 0.1040, Val-F2: 0.904\n",
      "1 [17/1000] 95s, Loss: 0.1651, Val-Loss: 0.1067, Best Val-Loss: 0.1040, Val-F2: 0.902\n",
      "Update Model!\n",
      "1 [18/1000] 93s, Loss: 0.1612, Val-Loss: 0.1033, Best Val-Loss: 0.1033, Val-F2: 0.906\n",
      "1 [19/1000] 95s, Loss: 0.1613, Val-Loss: 0.1039, Best Val-Loss: 0.1033, Val-F2: 0.900\n",
      "Update Model!\n",
      "1 [20/1000] 95s, Loss: 0.1591, Val-Loss: 0.1018, Best Val-Loss: 0.1018, Val-F2: 0.907\n",
      "Update Model!\n",
      "1 [21/1000] 94s, Loss: 0.1587, Val-Loss: 0.1000, Best Val-Loss: 0.1000, Val-F2: 0.907\n",
      "1 [22/1000] 92s, Loss: 0.1584, Val-Loss: 0.1087, Best Val-Loss: 0.1000, Val-F2: 0.897\n",
      "1 [23/1000] 95s, Loss: 0.1560, Val-Loss: 0.1113, Best Val-Loss: 0.1000, Val-F2: 0.889\n",
      "Update Model!\n",
      "1 [24/1000] 92s, Loss: 0.1554, Val-Loss: 0.0989, Best Val-Loss: 0.0989, Val-F2: 0.908\n",
      "1 [25/1000] 93s, Loss: 0.1529, Val-Loss: 0.1008, Best Val-Loss: 0.0989, Val-F2: 0.906\n",
      "Update Model!\n",
      "1 [26/1000] 93s, Loss: 0.1540, Val-Loss: 0.0975, Best Val-Loss: 0.0975, Val-F2: 0.910\n",
      "1 [27/1000] 94s, Loss: 0.1529, Val-Loss: 0.1031, Best Val-Loss: 0.0975, Val-F2: 0.903\n",
      "1 [28/1000] 92s, Loss: 0.1512, Val-Loss: 0.1001, Best Val-Loss: 0.0975, Val-F2: 0.909\n",
      "1 [29/1000] 93s, Loss: 0.1494, Val-Loss: 0.0986, Best Val-Loss: 0.0975, Val-F2: 0.909\n",
      "1 [30/1000] 92s, Loss: 0.1487, Val-Loss: 0.1025, Best Val-Loss: 0.0975, Val-F2: 0.906\n",
      "1 [31/1000] 93s, Loss: 0.1471, Val-Loss: 0.0984, Best Val-Loss: 0.0975, Val-F2: 0.906\n",
      "Update Model!\n",
      "1 [32/1000] 92s, Loss: 0.1473, Val-Loss: 0.0961, Best Val-Loss: 0.0961, Val-F2: 0.911\n",
      "Update Model!\n",
      "1 [33/1000] 92s, Loss: 0.1451, Val-Loss: 0.0950, Best Val-Loss: 0.0950, Val-F2: 0.913\n",
      "1 [34/1000] 92s, Loss: 0.1457, Val-Loss: 0.1019, Best Val-Loss: 0.0950, Val-F2: 0.905\n",
      "1 [35/1000] 94s, Loss: 0.1450, Val-Loss: 0.0998, Best Val-Loss: 0.0950, Val-F2: 0.908\n",
      "1 [36/1000] 93s, Loss: 0.1449, Val-Loss: 0.1000, Best Val-Loss: 0.0950, Val-F2: 0.907\n",
      "1 [37/1000] 94s, Loss: 0.1426, Val-Loss: 0.1000, Best Val-Loss: 0.0950, Val-F2: 0.907\n",
      "1 [38/1000] 94s, Loss: 0.1422, Val-Loss: 0.0988, Best Val-Loss: 0.0950, Val-F2: 0.907\n",
      "1 [39/1000] 93s, Loss: 0.1418, Val-Loss: 0.0978, Best Val-Loss: 0.0950, Val-F2: 0.912\n",
      "1 [40/1000] 91s, Loss: 0.1404, Val-Loss: 0.0972, Best Val-Loss: 0.0950, Val-F2: 0.911\n",
      "Update Model!\n",
      "1 [41/1000] 91s, Loss: 0.1404, Val-Loss: 0.0950, Best Val-Loss: 0.0950, Val-F2: 0.913\n",
      "1 [42/1000] 97s, Loss: 0.1389, Val-Loss: 0.0985, Best Val-Loss: 0.0950, Val-F2: 0.908\n",
      "1 [43/1000] 93s, Loss: 0.1390, Val-Loss: 0.0969, Best Val-Loss: 0.0950, Val-F2: 0.913\n",
      "1 [44/1000] 94s, Loss: 0.1395, Val-Loss: 0.0960, Best Val-Loss: 0.0950, Val-F2: 0.912\n",
      "1 [45/1000] 96s, Loss: 0.1363, Val-Loss: 0.0987, Best Val-Loss: 0.0950, Val-F2: 0.908\n",
      "1 [46/1000] 93s, Loss: 0.1352, Val-Loss: 0.0995, Best Val-Loss: 0.0950, Val-F2: 0.909\n",
      "1 [47/1000] 92s, Loss: 0.1360, Val-Loss: 0.0975, Best Val-Loss: 0.0950, Val-F2: 0.912\n",
      "1 [48/1000] 92s, Loss: 0.1348, Val-Loss: 0.0961, Best Val-Loss: 0.0950, Val-F2: 0.911\n",
      "New Learn Rate: 0.0001!\n",
      "1 [49/1000] 92s, Loss: 0.1345, Val-Loss: 0.0980, Best Val-Loss: 0.0950, Val-F2: 0.910\n",
      "Update Model!\n",
      "1 [50/1000] 92s, Loss: 0.1297, Val-Loss: 0.0937, Best Val-Loss: 0.0937, Val-F2: 0.915\n",
      "Update Model!\n",
      "1 [51/1000] 92s, Loss: 0.1267, Val-Loss: 0.0932, Best Val-Loss: 0.0932, Val-F2: 0.916\n",
      "1 [52/1000] 93s, Loss: 0.1283, Val-Loss: 0.0935, Best Val-Loss: 0.0932, Val-F2: 0.915\n",
      "1 [53/1000] 93s, Loss: 0.1257, Val-Loss: 0.0937, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [54/1000] 92s, Loss: 0.1245, Val-Loss: 0.0939, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [55/1000] 92s, Loss: 0.1254, Val-Loss: 0.0940, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [56/1000] 92s, Loss: 0.1245, Val-Loss: 0.0941, Best Val-Loss: 0.0932, Val-F2: 0.915\n",
      "1 [57/1000] 94s, Loss: 0.1253, Val-Loss: 0.0948, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [58/1000] 96s, Loss: 0.1228, Val-Loss: 0.0944, Best Val-Loss: 0.0932, Val-F2: 0.915\n",
      "New Learn Rate: 1e-05!\n",
      "1 [59/1000] 94s, Loss: 0.1231, Val-Loss: 0.0943, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [60/1000] 96s, Loss: 0.1231, Val-Loss: 0.0943, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [61/1000] 95s, Loss: 0.1225, Val-Loss: 0.0943, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [62/1000] 96s, Loss: 0.1237, Val-Loss: 0.0940, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [63/1000] 95s, Loss: 0.1230, Val-Loss: 0.0943, Best Val-Loss: 0.0932, Val-F2: 0.915\n",
      "1 [64/1000] 93s, Loss: 0.1219, Val-Loss: 0.0943, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [65/1000] 94s, Loss: 0.1212, Val-Loss: 0.0943, Best Val-Loss: 0.0932, Val-F2: 0.914\n",
      "1 [66/1000] 97s, Loss: 0.1215, Val-Loss: 0.0940, Best Val-Loss: 0.0932, Val-F2: 0.915\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "Early Stop in Epoch: 67, Best Val-Loss: 0.093215, Best F2: 0.917407\n",
      "Update Model!\n",
      "2 [1/1000] 92s, Loss: 0.3766, Val-Loss: 0.1996, Best Val-Loss: 0.1996, Val-F2: 0.805\n",
      "Update Model!\n",
      "2 [2/1000] 93s, Loss: 0.2236, Val-Loss: 0.1492, Best Val-Loss: 0.1492, Val-F2: 0.853\n",
      "Update Model!\n",
      "2 [3/1000] 92s, Loss: 0.2115, Val-Loss: 0.1465, Best Val-Loss: 0.1465, Val-F2: 0.851\n",
      "Update Model!\n",
      "2 [4/1000] 92s, Loss: 0.2029, Val-Loss: 0.1329, Best Val-Loss: 0.1329, Val-F2: 0.877\n",
      "2 [5/1000] 92s, Loss: 0.1967, Val-Loss: 0.1333, Best Val-Loss: 0.1329, Val-F2: 0.877\n",
      "Update Model!\n",
      "2 [6/1000] 92s, Loss: 0.1899, Val-Loss: 0.1279, Best Val-Loss: 0.1279, Val-F2: 0.882\n",
      "Update Model!\n",
      "2 [7/1000] 92s, Loss: 0.1857, Val-Loss: 0.1241, Best Val-Loss: 0.1241, Val-F2: 0.884\n",
      "Update Model!\n",
      "2 [8/1000] 92s, Loss: 0.1829, Val-Loss: 0.1192, Best Val-Loss: 0.1192, Val-F2: 0.892\n",
      "Update Model!\n",
      "2 [9/1000] 92s, Loss: 0.1791, Val-Loss: 0.1162, Best Val-Loss: 0.1162, Val-F2: 0.890\n",
      "2 [10/1000] 92s, Loss: 0.1769, Val-Loss: 0.1169, Best Val-Loss: 0.1162, Val-F2: 0.888\n",
      "2 [11/1000] 92s, Loss: 0.1753, Val-Loss: 0.1176, Best Val-Loss: 0.1162, Val-F2: 0.885\n",
      "2 [12/1000] 92s, Loss: 0.1727, Val-Loss: 0.1195, Best Val-Loss: 0.1162, Val-F2: 0.882\n",
      "Update Model!\n",
      "2 [13/1000] 92s, Loss: 0.1702, Val-Loss: 0.1066, Best Val-Loss: 0.1066, Val-F2: 0.903\n",
      "2 [14/1000] 92s, Loss: 0.1692, Val-Loss: 0.1090, Best Val-Loss: 0.1066, Val-F2: 0.895\n",
      "2 [15/1000] 92s, Loss: 0.1662, Val-Loss: 0.1099, Best Val-Loss: 0.1066, Val-F2: 0.899\n",
      "2 [16/1000] 92s, Loss: 0.1673, Val-Loss: 0.1073, Best Val-Loss: 0.1066, Val-F2: 0.901\n",
      "Update Model!\n",
      "2 [17/1000] 92s, Loss: 0.1649, Val-Loss: 0.1050, Best Val-Loss: 0.1050, Val-F2: 0.902\n",
      "Update Model!\n",
      "2 [18/1000] 92s, Loss: 0.1617, Val-Loss: 0.1045, Best Val-Loss: 0.1045, Val-F2: 0.904\n",
      "2 [19/1000] 92s, Loss: 0.1612, Val-Loss: 0.1055, Best Val-Loss: 0.1045, Val-F2: 0.907\n",
      "2 [20/1000] 92s, Loss: 0.1594, Val-Loss: 0.1050, Best Val-Loss: 0.1045, Val-F2: 0.906\n",
      "Update Model!\n",
      "2 [21/1000] 92s, Loss: 0.1591, Val-Loss: 0.1011, Best Val-Loss: 0.1011, Val-F2: 0.909\n",
      "Update Model!\n",
      "2 [22/1000] 92s, Loss: 0.1584, Val-Loss: 0.1000, Best Val-Loss: 0.1000, Val-F2: 0.909\n",
      "2 [23/1000] 91s, Loss: 0.1555, Val-Loss: 0.1063, Best Val-Loss: 0.1000, Val-F2: 0.899\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 [24/1000] 91s, Loss: 0.1567, Val-Loss: 0.1056, Best Val-Loss: 0.1000, Val-F2: 0.901\n",
      "2 [25/1000] 91s, Loss: 0.1539, Val-Loss: 0.1010, Best Val-Loss: 0.1000, Val-F2: 0.910\n",
      "2 [26/1000] 91s, Loss: 0.1541, Val-Loss: 0.1045, Best Val-Loss: 0.1000, Val-F2: 0.899\n",
      "Update Model!\n",
      "2 [27/1000] 91s, Loss: 0.1541, Val-Loss: 0.0991, Best Val-Loss: 0.0991, Val-F2: 0.909\n",
      "2 [28/1000] 91s, Loss: 0.1518, Val-Loss: 0.1023, Best Val-Loss: 0.0991, Val-F2: 0.907\n",
      "Update Model!\n",
      "2 [29/1000] 91s, Loss: 0.1511, Val-Loss: 0.0990, Best Val-Loss: 0.0990, Val-F2: 0.912\n",
      "Update Model!\n",
      "2 [30/1000] 91s, Loss: 0.1497, Val-Loss: 0.0982, Best Val-Loss: 0.0982, Val-F2: 0.909\n",
      "2 [31/1000] 91s, Loss: 0.1492, Val-Loss: 0.0993, Best Val-Loss: 0.0982, Val-F2: 0.911\n",
      "Update Model!\n",
      "2 [32/1000] 91s, Loss: 0.1496, Val-Loss: 0.0980, Best Val-Loss: 0.0980, Val-F2: 0.909\n",
      "2 [33/1000] 91s, Loss: 0.1463, Val-Loss: 0.1013, Best Val-Loss: 0.0980, Val-F2: 0.906\n",
      "2 [34/1000] 94s, Loss: 0.1457, Val-Loss: 0.0985, Best Val-Loss: 0.0980, Val-F2: 0.911\n",
      "2 [35/1000] 95s, Loss: 0.1450, Val-Loss: 0.0997, Best Val-Loss: 0.0980, Val-F2: 0.909\n",
      "Update Model!\n",
      "2 [36/1000] 97s, Loss: 0.1446, Val-Loss: 0.0978, Best Val-Loss: 0.0978, Val-F2: 0.911\n",
      "2 [37/1000] 96s, Loss: 0.1431, Val-Loss: 0.0981, Best Val-Loss: 0.0978, Val-F2: 0.910\n",
      "2 [38/1000] 96s, Loss: 0.1432, Val-Loss: 0.1016, Best Val-Loss: 0.0978, Val-F2: 0.905\n",
      "Update Model!\n",
      "2 [39/1000] 95s, Loss: 0.1419, Val-Loss: 0.0961, Best Val-Loss: 0.0961, Val-F2: 0.914\n",
      "2 [40/1000] 96s, Loss: 0.1415, Val-Loss: 0.0993, Best Val-Loss: 0.0961, Val-F2: 0.907\n",
      "2 [41/1000] 95s, Loss: 0.1410, Val-Loss: 0.0989, Best Val-Loss: 0.0961, Val-F2: 0.912\n",
      "2 [42/1000] 97s, Loss: 0.1409, Val-Loss: 0.1004, Best Val-Loss: 0.0961, Val-F2: 0.909\n",
      "2 [43/1000] 95s, Loss: 0.1382, Val-Loss: 0.0993, Best Val-Loss: 0.0961, Val-F2: 0.908\n",
      "2 [44/1000] 97s, Loss: 0.1373, Val-Loss: 0.0984, Best Val-Loss: 0.0961, Val-F2: 0.910\n",
      "2 [45/1000] 97s, Loss: 0.1375, Val-Loss: 0.0983, Best Val-Loss: 0.0961, Val-F2: 0.910\n",
      "2 [46/1000] 98s, Loss: 0.1369, Val-Loss: 0.1010, Best Val-Loss: 0.0961, Val-F2: 0.904\n",
      "New Learn Rate: 0.0001!\n",
      "2 [47/1000] 98s, Loss: 0.1365, Val-Loss: 0.0985, Best Val-Loss: 0.0961, Val-F2: 0.910\n",
      "Update Model!\n",
      "2 [48/1000] 98s, Loss: 0.1312, Val-Loss: 0.0941, Best Val-Loss: 0.0941, Val-F2: 0.914\n",
      "2 [49/1000] 98s, Loss: 0.1296, Val-Loss: 0.0942, Best Val-Loss: 0.0941, Val-F2: 0.913\n",
      "2 [50/1000] 98s, Loss: 0.1270, Val-Loss: 0.0942, Best Val-Loss: 0.0941, Val-F2: 0.914\n",
      "2 [51/1000] 96s, Loss: 0.1271, Val-Loss: 0.0943, Best Val-Loss: 0.0941, Val-F2: 0.913\n",
      "2 [52/1000] 95s, Loss: 0.1262, Val-Loss: 0.0943, Best Val-Loss: 0.0941, Val-F2: 0.914\n",
      "2 [53/1000] 94s, Loss: 0.1255, Val-Loss: 0.0944, Best Val-Loss: 0.0941, Val-F2: 0.914\n",
      "2 [54/1000] 95s, Loss: 0.1255, Val-Loss: 0.0946, Best Val-Loss: 0.0941, Val-F2: 0.913\n",
      "2 [55/1000] 95s, Loss: 0.1249, Val-Loss: 0.0945, Best Val-Loss: 0.0941, Val-F2: 0.913\n",
      "New Learn Rate: 1e-05!\n",
      "2 [56/1000] 95s, Loss: 0.1251, Val-Loss: 0.0942, Best Val-Loss: 0.0941, Val-F2: 0.915\n",
      "2 [57/1000] 95s, Loss: 0.1243, Val-Loss: 0.0943, Best Val-Loss: 0.0941, Val-F2: 0.914\n",
      "2 [58/1000] 95s, Loss: 0.1238, Val-Loss: 0.0944, Best Val-Loss: 0.0941, Val-F2: 0.914\n",
      "Update Model!\n",
      "2 [59/1000] 93s, Loss: 0.1241, Val-Loss: 0.0940, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [60/1000] 93s, Loss: 0.1249, Val-Loss: 0.0942, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [61/1000] 94s, Loss: 0.1246, Val-Loss: 0.0943, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [62/1000] 95s, Loss: 0.1240, Val-Loss: 0.0941, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [63/1000] 96s, Loss: 0.1252, Val-Loss: 0.0942, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [64/1000] 93s, Loss: 0.1242, Val-Loss: 0.0942, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [65/1000] 93s, Loss: 0.1232, Val-Loss: 0.0946, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [66/1000] 93s, Loss: 0.1238, Val-Loss: 0.0948, Best Val-Loss: 0.0940, Val-F2: 0.913\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "2 [67/1000] 92s, Loss: 0.1239, Val-Loss: 0.0946, Best Val-Loss: 0.0940, Val-F2: 0.913\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "J:\\Anaconda3\\envs\\py36\\lib\\site-packages\\sklearn\\metrics\\classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 [68/1000] 92s, Loss: 0.1242, Val-Loss: 0.0944, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [69/1000] 92s, Loss: 0.1237, Val-Loss: 0.0942, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [70/1000] 94s, Loss: 0.1242, Val-Loss: 0.0943, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [71/1000] 96s, Loss: 0.1234, Val-Loss: 0.0945, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [72/1000] 93s, Loss: 0.1235, Val-Loss: 0.0945, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [73/1000] 92s, Loss: 0.1236, Val-Loss: 0.0944, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "2 [74/1000] 92s, Loss: 0.1238, Val-Loss: 0.0946, Best Val-Loss: 0.0940, Val-F2: 0.914\n",
      "New Learn Rate: 1.0000000000000002e-07!\n",
      "Early Stop in Epoch: 75, Best Val-Loss: 0.094038, Best F2: 0.915030\n",
      "Update Model!\n",
      "3 [1/1000] 91s, Loss: 0.3836, Val-Loss: 0.1700, Best Val-Loss: 0.1700, Val-F2: 0.818\n",
      "Update Model!\n",
      "3 [2/1000] 90s, Loss: 0.2292, Val-Loss: 0.1511, Best Val-Loss: 0.1511, Val-F2: 0.849\n",
      "Update Model!\n",
      "3 [3/1000] 90s, Loss: 0.2152, Val-Loss: 0.1428, Best Val-Loss: 0.1428, Val-F2: 0.855\n",
      "Update Model!\n",
      "3 [4/1000] 90s, Loss: 0.2045, Val-Loss: 0.1353, Best Val-Loss: 0.1353, Val-F2: 0.870\n",
      "Update Model!\n",
      "3 [5/1000] 92s, Loss: 0.1989, Val-Loss: 0.1316, Best Val-Loss: 0.1316, Val-F2: 0.874\n",
      "3 [6/1000] 96s, Loss: 0.1929, Val-Loss: 0.1321, Best Val-Loss: 0.1316, Val-F2: 0.866\n",
      "3 [7/1000] 96s, Loss: 0.1877, Val-Loss: 0.1327, Best Val-Loss: 0.1316, Val-F2: 0.863\n",
      "Update Model!\n",
      "3 [8/1000] 95s, Loss: 0.1853, Val-Loss: 0.1230, Best Val-Loss: 0.1230, Val-F2: 0.883\n",
      "Update Model!\n",
      "3 [9/1000] 97s, Loss: 0.1825, Val-Loss: 0.1148, Best Val-Loss: 0.1148, Val-F2: 0.896\n",
      "3 [10/1000] 96s, Loss: 0.1796, Val-Loss: 0.1190, Best Val-Loss: 0.1148, Val-F2: 0.891\n",
      "3 [11/1000] 96s, Loss: 0.1761, Val-Loss: 0.1172, Best Val-Loss: 0.1148, Val-F2: 0.893\n",
      "Update Model!\n",
      "3 [12/1000] 97s, Loss: 0.1739, Val-Loss: 0.1107, Best Val-Loss: 0.1107, Val-F2: 0.898\n",
      "Update Model!\n",
      "3 [13/1000] 95s, Loss: 0.1727, Val-Loss: 0.1091, Best Val-Loss: 0.1091, Val-F2: 0.904\n",
      "Update Model!\n",
      "3 [14/1000] 97s, Loss: 0.1704, Val-Loss: 0.1079, Best Val-Loss: 0.1079, Val-F2: 0.900\n",
      "Update Model!\n",
      "3 [15/1000] 96s, Loss: 0.1677, Val-Loss: 0.1073, Best Val-Loss: 0.1073, Val-F2: 0.902\n",
      "Update Model!\n",
      "3 [16/1000] 97s, Loss: 0.1676, Val-Loss: 0.1065, Best Val-Loss: 0.1065, Val-F2: 0.904\n",
      "3 [17/1000] 98s, Loss: 0.1648, Val-Loss: 0.1101, Best Val-Loss: 0.1065, Val-F2: 0.900\n",
      "Update Model!\n",
      "3 [18/1000] 96s, Loss: 0.1630, Val-Loss: 0.1041, Best Val-Loss: 0.1041, Val-F2: 0.906\n",
      "3 [19/1000] 96s, Loss: 0.1641, Val-Loss: 0.1050, Best Val-Loss: 0.1041, Val-F2: 0.905\n",
      "Update Model!\n",
      "3 [20/1000] 98s, Loss: 0.1608, Val-Loss: 0.1024, Best Val-Loss: 0.1024, Val-F2: 0.906\n",
      "3 [21/1000] 96s, Loss: 0.1598, Val-Loss: 0.1024, Best Val-Loss: 0.1024, Val-F2: 0.905\n",
      "3 [22/1000] 97s, Loss: 0.1599, Val-Loss: 0.1032, Best Val-Loss: 0.1024, Val-F2: 0.906\n",
      "3 [23/1000] 95s, Loss: 0.1579, Val-Loss: 0.1079, Best Val-Loss: 0.1024, Val-F2: 0.903\n",
      "3 [24/1000] 95s, Loss: 0.1573, Val-Loss: 0.1031, Best Val-Loss: 0.1024, Val-F2: 0.904\n",
      "3 [25/1000] 96s, Loss: 0.1542, Val-Loss: 0.1024, Best Val-Loss: 0.1024, Val-F2: 0.905\n",
      "Update Model!\n",
      "3 [26/1000] 95s, Loss: 0.1552, Val-Loss: 0.1010, Best Val-Loss: 0.1010, Val-F2: 0.906\n",
      "Update Model!\n",
      "3 [27/1000] 96s, Loss: 0.1542, Val-Loss: 0.0990, Best Val-Loss: 0.0990, Val-F2: 0.911\n",
      "Update Model!\n",
      "3 [28/1000] 96s, Loss: 0.1524, Val-Loss: 0.0980, Best Val-Loss: 0.0980, Val-F2: 0.913\n",
      "3 [29/1000] 97s, Loss: 0.1517, Val-Loss: 0.1062, Best Val-Loss: 0.0980, Val-F2: 0.902\n",
      "3 [30/1000] 96s, Loss: 0.1503, Val-Loss: 0.1012, Best Val-Loss: 0.0980, Val-F2: 0.902\n",
      "3 [31/1000] 97s, Loss: 0.1497, Val-Loss: 0.0983, Best Val-Loss: 0.0980, Val-F2: 0.910\n",
      "3 [32/1000] 97s, Loss: 0.1489, Val-Loss: 0.1013, Best Val-Loss: 0.0980, Val-F2: 0.906\n",
      "3 [33/1000] 97s, Loss: 0.1482, Val-Loss: 0.1044, Best Val-Loss: 0.0980, Val-F2: 0.903\n",
      "3 [34/1000] 95s, Loss: 0.1479, Val-Loss: 0.1002, Best Val-Loss: 0.0980, Val-F2: 0.907\n",
      "Update Model!\n",
      "3 [35/1000] 96s, Loss: 0.1460, Val-Loss: 0.0974, Best Val-Loss: 0.0974, Val-F2: 0.912\n",
      "3 [36/1000] 95s, Loss: 0.1466, Val-Loss: 0.1002, Best Val-Loss: 0.0974, Val-F2: 0.905\n",
      "3 [37/1000] 95s, Loss: 0.1455, Val-Loss: 0.0991, Best Val-Loss: 0.0974, Val-F2: 0.909\n",
      "3 [38/1000] 95s, Loss: 0.1448, Val-Loss: 0.1006, Best Val-Loss: 0.0974, Val-F2: 0.904\n",
      "3 [39/1000] 95s, Loss: 0.1432, Val-Loss: 0.0975, Best Val-Loss: 0.0974, Val-F2: 0.910\n",
      "3 [40/1000] 99s, Loss: 0.1424, Val-Loss: 0.0994, Best Val-Loss: 0.0974, Val-F2: 0.910\n",
      "3 [41/1000] 98s, Loss: 0.1442, Val-Loss: 0.1038, Best Val-Loss: 0.0974, Val-F2: 0.903\n",
      "3 [42/1000] 95s, Loss: 0.1409, Val-Loss: 0.0978, Best Val-Loss: 0.0974, Val-F2: 0.911\n",
      "New Learn Rate: 0.0001!\n",
      "3 [43/1000] 95s, Loss: 0.1405, Val-Loss: 0.0981, Best Val-Loss: 0.0974, Val-F2: 0.911\n",
      "Update Model!\n",
      "3 [44/1000] 95s, Loss: 0.1351, Val-Loss: 0.0945, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [45/1000] 95s, Loss: 0.1342, Val-Loss: 0.0948, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "3 [46/1000] 95s, Loss: 0.1317, Val-Loss: 0.0946, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [47/1000] 95s, Loss: 0.1321, Val-Loss: 0.0947, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [48/1000] 96s, Loss: 0.1315, Val-Loss: 0.0951, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "3 [49/1000] 96s, Loss: 0.1315, Val-Loss: 0.0950, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "Update Model!\n",
      "3 [50/1000] 98s, Loss: 0.1312, Val-Loss: 0.0945, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [51/1000] 95s, Loss: 0.1315, Val-Loss: 0.0949, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "3 [52/1000] 95s, Loss: 0.1303, Val-Loss: 0.0951, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "3 [53/1000] 95s, Loss: 0.1294, Val-Loss: 0.0949, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "3 [54/1000] 95s, Loss: 0.1290, Val-Loss: 0.0948, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [55/1000] 95s, Loss: 0.1291, Val-Loss: 0.0953, Best Val-Loss: 0.0945, Val-F2: 0.912\n",
      "3 [56/1000] 94s, Loss: 0.1294, Val-Loss: 0.0948, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [57/1000] 93s, Loss: 0.1284, Val-Loss: 0.0952, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "New Learn Rate: 1e-05!\n",
      "3 [58/1000] 94s, Loss: 0.1282, Val-Loss: 0.0950, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [59/1000] 94s, Loss: 0.1285, Val-Loss: 0.0947, Best Val-Loss: 0.0945, Val-F2: 0.915\n",
      "3 [60/1000] 94s, Loss: 0.1277, Val-Loss: 0.0950, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [61/1000] 96s, Loss: 0.1274, Val-Loss: 0.0953, Best Val-Loss: 0.0945, Val-F2: 0.913\n",
      "3 [62/1000] 93s, Loss: 0.1273, Val-Loss: 0.0947, Best Val-Loss: 0.0945, Val-F2: 0.915\n",
      "3 [63/1000] 93s, Loss: 0.1277, Val-Loss: 0.0950, Best Val-Loss: 0.0945, Val-F2: 0.915\n",
      "3 [64/1000] 93s, Loss: 0.1280, Val-Loss: 0.0951, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "3 [65/1000] 93s, Loss: 0.1274, Val-Loss: 0.0949, Best Val-Loss: 0.0945, Val-F2: 0.914\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "Early Stop in Epoch: 66, Best Val-Loss: 0.094503, Best F2: 0.915086\n",
      "Update Model!\n",
      "4 [1/1000] 94s, Loss: 0.3748, Val-Loss: 0.1718, Best Val-Loss: 0.1718, Val-F2: 0.816\n",
      "Update Model!\n",
      "4 [2/1000] 94s, Loss: 0.2188, Val-Loss: 0.1437, Best Val-Loss: 0.1437, Val-F2: 0.858\n",
      "Update Model!\n",
      "4 [3/1000] 94s, Loss: 0.2070, Val-Loss: 0.1373, Best Val-Loss: 0.1373, Val-F2: 0.862\n",
      "Update Model!\n",
      "4 [4/1000] 94s, Loss: 0.1987, Val-Loss: 0.1280, Best Val-Loss: 0.1280, Val-F2: 0.882\n",
      "Update Model!\n",
      "4 [5/1000] 94s, Loss: 0.1943, Val-Loss: 0.1249, Best Val-Loss: 0.1249, Val-F2: 0.882\n",
      "4 [6/1000] 94s, Loss: 0.1885, Val-Loss: 0.1315, Best Val-Loss: 0.1249, Val-F2: 0.867\n",
      "Update Model!\n",
      "4 [7/1000] 95s, Loss: 0.1847, Val-Loss: 0.1209, Best Val-Loss: 0.1209, Val-F2: 0.883\n",
      "Update Model!\n",
      "4 [8/1000] 94s, Loss: 0.1826, Val-Loss: 0.1185, Best Val-Loss: 0.1185, Val-F2: 0.891\n",
      "Update Model!\n",
      "4 [9/1000] 94s, Loss: 0.1790, Val-Loss: 0.1113, Best Val-Loss: 0.1113, Val-F2: 0.899\n",
      "4 [10/1000] 94s, Loss: 0.1761, Val-Loss: 0.1372, Best Val-Loss: 0.1113, Val-F2: 0.860\n",
      "Update Model!\n",
      "4 [11/1000] 95s, Loss: 0.1739, Val-Loss: 0.1108, Best Val-Loss: 0.1108, Val-F2: 0.898\n",
      "Update Model!\n",
      "4 [12/1000] 94s, Loss: 0.1714, Val-Loss: 0.1073, Best Val-Loss: 0.1073, Val-F2: 0.901\n",
      "4 [13/1000] 94s, Loss: 0.1683, Val-Loss: 0.1103, Best Val-Loss: 0.1073, Val-F2: 0.892\n",
      "Update Model!\n",
      "4 [14/1000] 94s, Loss: 0.1678, Val-Loss: 0.1048, Best Val-Loss: 0.1048, Val-F2: 0.908\n",
      "Update Model!\n",
      "4 [15/1000] 95s, Loss: 0.1677, Val-Loss: 0.1041, Best Val-Loss: 0.1041, Val-F2: 0.904\n",
      "Update Model!\n",
      "4 [16/1000] 94s, Loss: 0.1662, Val-Loss: 0.1031, Best Val-Loss: 0.1031, Val-F2: 0.905\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 [17/1000] 94s, Loss: 0.1628, Val-Loss: 0.1049, Best Val-Loss: 0.1031, Val-F2: 0.905\n",
      "4 [18/1000] 94s, Loss: 0.1624, Val-Loss: 0.1140, Best Val-Loss: 0.1031, Val-F2: 0.888\n",
      "4 [19/1000] 96s, Loss: 0.1614, Val-Loss: 0.1044, Best Val-Loss: 0.1031, Val-F2: 0.903\n",
      "4 [20/1000] 95s, Loss: 0.1588, Val-Loss: 0.1033, Best Val-Loss: 0.1031, Val-F2: 0.902\n",
      "4 [21/1000] 95s, Loss: 0.1587, Val-Loss: 0.1038, Best Val-Loss: 0.1031, Val-F2: 0.900\n",
      "Update Model!\n",
      "4 [22/1000] 95s, Loss: 0.1563, Val-Loss: 0.1010, Best Val-Loss: 0.1010, Val-F2: 0.905\n",
      "Update Model!\n",
      "4 [23/1000] 95s, Loss: 0.1554, Val-Loss: 0.0978, Best Val-Loss: 0.0978, Val-F2: 0.909\n",
      "4 [24/1000] 94s, Loss: 0.1526, Val-Loss: 0.1009, Best Val-Loss: 0.0978, Val-F2: 0.904\n",
      "4 [25/1000] 95s, Loss: 0.1540, Val-Loss: 0.0984, Best Val-Loss: 0.0978, Val-F2: 0.910\n",
      "Update Model!\n",
      "4 [26/1000] 95s, Loss: 0.1523, Val-Loss: 0.0970, Best Val-Loss: 0.0970, Val-F2: 0.910\n",
      "4 [27/1000] 94s, Loss: 0.1522, Val-Loss: 0.0975, Best Val-Loss: 0.0970, Val-F2: 0.912\n",
      "4 [28/1000] 94s, Loss: 0.1520, Val-Loss: 0.0996, Best Val-Loss: 0.0970, Val-F2: 0.906\n",
      "4 [29/1000] 95s, Loss: 0.1493, Val-Loss: 0.0971, Best Val-Loss: 0.0970, Val-F2: 0.910\n",
      "Update Model!\n",
      "4 [30/1000] 95s, Loss: 0.1492, Val-Loss: 0.0962, Best Val-Loss: 0.0962, Val-F2: 0.911\n",
      "4 [31/1000] 95s, Loss: 0.1481, Val-Loss: 0.0964, Best Val-Loss: 0.0962, Val-F2: 0.909\n",
      "4 [32/1000] 94s, Loss: 0.1456, Val-Loss: 0.0965, Best Val-Loss: 0.0962, Val-F2: 0.909\n",
      "4 [33/1000] 94s, Loss: 0.1461, Val-Loss: 0.0969, Best Val-Loss: 0.0962, Val-F2: 0.912\n",
      "4 [34/1000] 94s, Loss: 0.1442, Val-Loss: 0.0982, Best Val-Loss: 0.0962, Val-F2: 0.908\n",
      "4 [35/1000] 95s, Loss: 0.1450, Val-Loss: 0.0981, Best Val-Loss: 0.0962, Val-F2: 0.906\n",
      "Update Model!\n",
      "4 [36/1000] 95s, Loss: 0.1443, Val-Loss: 0.0954, Best Val-Loss: 0.0954, Val-F2: 0.910\n",
      "4 [37/1000] 94s, Loss: 0.1437, Val-Loss: 0.0978, Best Val-Loss: 0.0954, Val-F2: 0.908\n",
      "4 [38/1000] 94s, Loss: 0.1420, Val-Loss: 0.0987, Best Val-Loss: 0.0954, Val-F2: 0.908\n",
      "Update Model!\n",
      "4 [39/1000] 95s, Loss: 0.1406, Val-Loss: 0.0948, Best Val-Loss: 0.0948, Val-F2: 0.911\n",
      "4 [40/1000] 94s, Loss: 0.1405, Val-Loss: 0.0957, Best Val-Loss: 0.0948, Val-F2: 0.911\n",
      "4 [41/1000] 95s, Loss: 0.1402, Val-Loss: 0.0966, Best Val-Loss: 0.0948, Val-F2: 0.909\n",
      "4 [42/1000] 96s, Loss: 0.1396, Val-Loss: 0.0963, Best Val-Loss: 0.0948, Val-F2: 0.909\n",
      "4 [43/1000] 101s, Loss: 0.1385, Val-Loss: 0.0978, Best Val-Loss: 0.0948, Val-F2: 0.909\n",
      "4 [44/1000] 100s, Loss: 0.1367, Val-Loss: 0.0971, Best Val-Loss: 0.0948, Val-F2: 0.907\n",
      "4 [45/1000] 98s, Loss: 0.1376, Val-Loss: 0.0970, Best Val-Loss: 0.0948, Val-F2: 0.908\n",
      "4 [46/1000] 95s, Loss: 0.1356, Val-Loss: 0.0972, Best Val-Loss: 0.0948, Val-F2: 0.909\n",
      "New Learn Rate: 0.0001!\n",
      "4 [47/1000] 95s, Loss: 0.1371, Val-Loss: 0.0958, Best Val-Loss: 0.0948, Val-F2: 0.912\n",
      "Update Model!\n",
      "4 [48/1000] 98s, Loss: 0.1300, Val-Loss: 0.0927, Best Val-Loss: 0.0927, Val-F2: 0.914\n",
      "4 [49/1000] 97s, Loss: 0.1280, Val-Loss: 0.0929, Best Val-Loss: 0.0927, Val-F2: 0.914\n",
      "4 [50/1000] 99s, Loss: 0.1271, Val-Loss: 0.0930, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [51/1000] 97s, Loss: 0.1271, Val-Loss: 0.0929, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "Update Model!\n",
      "4 [52/1000] 96s, Loss: 0.1270, Val-Loss: 0.0927, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [53/1000] 94s, Loss: 0.1262, Val-Loss: 0.0928, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "Update Model!\n",
      "4 [54/1000] 96s, Loss: 0.1247, Val-Loss: 0.0927, Best Val-Loss: 0.0927, Val-F2: 0.914\n",
      "4 [55/1000] 93s, Loss: 0.1260, Val-Loss: 0.0934, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [56/1000] 93s, Loss: 0.1257, Val-Loss: 0.0933, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [57/1000] 96s, Loss: 0.1249, Val-Loss: 0.0930, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [58/1000] 98s, Loss: 0.1237, Val-Loss: 0.0934, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [59/1000] 97s, Loss: 0.1244, Val-Loss: 0.0934, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [60/1000] 95s, Loss: 0.1232, Val-Loss: 0.0933, Best Val-Loss: 0.0927, Val-F2: 0.914\n",
      "4 [61/1000] 96s, Loss: 0.1229, Val-Loss: 0.0936, Best Val-Loss: 0.0927, Val-F2: 0.912\n",
      "New Learn Rate: 1e-05!\n",
      "4 [62/1000] 94s, Loss: 0.1223, Val-Loss: 0.0936, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [63/1000] 94s, Loss: 0.1212, Val-Loss: 0.0935, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [64/1000] 93s, Loss: 0.1220, Val-Loss: 0.0933, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [65/1000] 93s, Loss: 0.1230, Val-Loss: 0.0934, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [66/1000] 93s, Loss: 0.1231, Val-Loss: 0.0935, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [67/1000] 93s, Loss: 0.1219, Val-Loss: 0.0933, Best Val-Loss: 0.0927, Val-F2: 0.914\n",
      "4 [68/1000] 93s, Loss: 0.1226, Val-Loss: 0.0935, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "4 [69/1000] 93s, Loss: 0.1212, Val-Loss: 0.0935, Best Val-Loss: 0.0927, Val-F2: 0.913\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "Early Stop in Epoch: 70, Best Val-Loss: 0.092702, Best F2: 0.916406\n",
      "Update Model!\n",
      "5 [1/1000] 92s, Loss: 0.3761, Val-Loss: 0.1658, Best Val-Loss: 0.1658, Val-F2: 0.836\n",
      "5 [2/1000] 94s, Loss: 0.2219, Val-Loss: 0.2055, Best Val-Loss: 0.1658, Val-F2: 0.805\n",
      "Update Model!\n",
      "5 [3/1000] 93s, Loss: 0.2089, Val-Loss: 0.1422, Best Val-Loss: 0.1422, Val-F2: 0.852\n",
      "Update Model!\n",
      "5 [4/1000] 93s, Loss: 0.2016, Val-Loss: 0.1291, Best Val-Loss: 0.1291, Val-F2: 0.879\n",
      "5 [5/1000] 95s, Loss: 0.1937, Val-Loss: 0.1315, Best Val-Loss: 0.1291, Val-F2: 0.867\n",
      "Update Model!\n",
      "5 [6/1000] 95s, Loss: 0.1898, Val-Loss: 0.1240, Best Val-Loss: 0.1240, Val-F2: 0.887\n",
      "5 [7/1000] 94s, Loss: 0.1843, Val-Loss: 0.1250, Best Val-Loss: 0.1240, Val-F2: 0.880\n",
      "Update Model!\n",
      "5 [8/1000] 93s, Loss: 0.1823, Val-Loss: 0.1227, Best Val-Loss: 0.1227, Val-F2: 0.884\n",
      "Update Model!\n",
      "5 [9/1000] 94s, Loss: 0.1796, Val-Loss: 0.1143, Best Val-Loss: 0.1143, Val-F2: 0.893\n",
      "Update Model!\n",
      "5 [10/1000] 94s, Loss: 0.1780, Val-Loss: 0.1104, Best Val-Loss: 0.1104, Val-F2: 0.899\n",
      "Update Model!\n",
      "5 [11/1000] 93s, Loss: 0.1729, Val-Loss: 0.1078, Best Val-Loss: 0.1078, Val-F2: 0.904\n",
      "5 [12/1000] 93s, Loss: 0.1716, Val-Loss: 0.1089, Best Val-Loss: 0.1078, Val-F2: 0.901\n",
      "5 [13/1000] 93s, Loss: 0.1698, Val-Loss: 0.1090, Best Val-Loss: 0.1078, Val-F2: 0.899\n",
      "Update Model!\n",
      "5 [14/1000] 93s, Loss: 0.1690, Val-Loss: 0.1054, Best Val-Loss: 0.1054, Val-F2: 0.906\n",
      "5 [15/1000] 94s, Loss: 0.1661, Val-Loss: 0.1057, Best Val-Loss: 0.1054, Val-F2: 0.904\n",
      "5 [16/1000] 94s, Loss: 0.1652, Val-Loss: 0.1059, Best Val-Loss: 0.1054, Val-F2: 0.905\n",
      "Update Model!\n",
      "5 [17/1000] 94s, Loss: 0.1646, Val-Loss: 0.1051, Best Val-Loss: 0.1051, Val-F2: 0.903\n",
      "Update Model!\n",
      "5 [18/1000] 94s, Loss: 0.1627, Val-Loss: 0.1016, Best Val-Loss: 0.1016, Val-F2: 0.908\n",
      "5 [19/1000] 94s, Loss: 0.1617, Val-Loss: 0.1022, Best Val-Loss: 0.1016, Val-F2: 0.908\n",
      "Update Model!\n",
      "5 [20/1000] 93s, Loss: 0.1587, Val-Loss: 0.1003, Best Val-Loss: 0.1003, Val-F2: 0.906\n",
      "5 [21/1000] 93s, Loss: 0.1588, Val-Loss: 0.1017, Best Val-Loss: 0.1003, Val-F2: 0.904\n",
      "5 [22/1000] 93s, Loss: 0.1567, Val-Loss: 0.1004, Best Val-Loss: 0.1003, Val-F2: 0.908\n",
      "5 [23/1000] 93s, Loss: 0.1565, Val-Loss: 0.1036, Best Val-Loss: 0.1003, Val-F2: 0.904\n",
      "Update Model!\n",
      "5 [24/1000] 94s, Loss: 0.1555, Val-Loss: 0.0978, Best Val-Loss: 0.0978, Val-F2: 0.910\n",
      "5 [25/1000] 94s, Loss: 0.1544, Val-Loss: 0.1006, Best Val-Loss: 0.0978, Val-F2: 0.903\n",
      "5 [26/1000] 94s, Loss: 0.1538, Val-Loss: 0.0991, Best Val-Loss: 0.0978, Val-F2: 0.908\n",
      "5 [27/1000] 94s, Loss: 0.1523, Val-Loss: 0.1010, Best Val-Loss: 0.0978, Val-F2: 0.906\n",
      "Update Model!\n",
      "5 [28/1000] 94s, Loss: 0.1518, Val-Loss: 0.0973, Best Val-Loss: 0.0973, Val-F2: 0.907\n",
      "Update Model!\n",
      "5 [29/1000] 94s, Loss: 0.1520, Val-Loss: 0.0963, Best Val-Loss: 0.0963, Val-F2: 0.910\n",
      "5 [30/1000] 94s, Loss: 0.1494, Val-Loss: 0.0988, Best Val-Loss: 0.0963, Val-F2: 0.909\n",
      "5 [31/1000] 94s, Loss: 0.1491, Val-Loss: 0.0974, Best Val-Loss: 0.0963, Val-F2: 0.909\n",
      "5 [32/1000] 94s, Loss: 0.1473, Val-Loss: 0.0985, Best Val-Loss: 0.0963, Val-F2: 0.909\n",
      "Update Model!\n",
      "5 [33/1000] 94s, Loss: 0.1481, Val-Loss: 0.0950, Best Val-Loss: 0.0950, Val-F2: 0.911\n",
      "5 [34/1000] 94s, Loss: 0.1452, Val-Loss: 0.0971, Best Val-Loss: 0.0950, Val-F2: 0.909\n",
      "5 [35/1000] 94s, Loss: 0.1446, Val-Loss: 0.1003, Best Val-Loss: 0.0950, Val-F2: 0.904\n",
      "Update Model!\n",
      "5 [36/1000] 94s, Loss: 0.1448, Val-Loss: 0.0943, Best Val-Loss: 0.0943, Val-F2: 0.913\n",
      "5 [37/1000] 94s, Loss: 0.1438, Val-Loss: 0.0953, Best Val-Loss: 0.0943, Val-F2: 0.910\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 [38/1000] 94s, Loss: 0.1429, Val-Loss: 0.0972, Best Val-Loss: 0.0943, Val-F2: 0.909\n",
      "5 [39/1000] 94s, Loss: 0.1428, Val-Loss: 0.0975, Best Val-Loss: 0.0943, Val-F2: 0.908\n",
      "5 [40/1000] 94s, Loss: 0.1417, Val-Loss: 0.0958, Best Val-Loss: 0.0943, Val-F2: 0.909\n",
      "5 [41/1000] 94s, Loss: 0.1407, Val-Loss: 0.0945, Best Val-Loss: 0.0943, Val-F2: 0.913\n",
      "5 [42/1000] 94s, Loss: 0.1407, Val-Loss: 0.0983, Best Val-Loss: 0.0943, Val-F2: 0.909\n",
      "5 [43/1000] 94s, Loss: 0.1397, Val-Loss: 0.0948, Best Val-Loss: 0.0943, Val-F2: 0.914\n",
      "New Learn Rate: 0.0001!\n",
      "5 [44/1000] 94s, Loss: 0.1389, Val-Loss: 0.1003, Best Val-Loss: 0.0943, Val-F2: 0.905\n",
      "Update Model!\n",
      "5 [45/1000] 94s, Loss: 0.1335, Val-Loss: 0.0925, Best Val-Loss: 0.0925, Val-F2: 0.915\n",
      "5 [46/1000] 94s, Loss: 0.1321, Val-Loss: 0.0926, Best Val-Loss: 0.0925, Val-F2: 0.913\n",
      "Update Model!\n",
      "5 [47/1000] 94s, Loss: 0.1305, Val-Loss: 0.0918, Best Val-Loss: 0.0918, Val-F2: 0.916\n",
      "5 [48/1000] 94s, Loss: 0.1306, Val-Loss: 0.0922, Best Val-Loss: 0.0918, Val-F2: 0.916\n",
      "5 [49/1000] 94s, Loss: 0.1294, Val-Loss: 0.0925, Best Val-Loss: 0.0918, Val-F2: 0.914\n",
      "5 [50/1000] 94s, Loss: 0.1302, Val-Loss: 0.0919, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [51/1000] 94s, Loss: 0.1299, Val-Loss: 0.0922, Best Val-Loss: 0.0918, Val-F2: 0.916\n",
      "5 [52/1000] 94s, Loss: 0.1283, Val-Loss: 0.0924, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [53/1000] 94s, Loss: 0.1281, Val-Loss: 0.0928, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [54/1000] 94s, Loss: 0.1270, Val-Loss: 0.0923, Best Val-Loss: 0.0918, Val-F2: 0.914\n",
      "New Learn Rate: 1e-05!\n",
      "5 [55/1000] 94s, Loss: 0.1273, Val-Loss: 0.0925, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [56/1000] 94s, Loss: 0.1272, Val-Loss: 0.0926, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [57/1000] 94s, Loss: 0.1266, Val-Loss: 0.0926, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [58/1000] 94s, Loss: 0.1274, Val-Loss: 0.0923, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [59/1000] 94s, Loss: 0.1261, Val-Loss: 0.0924, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [60/1000] 94s, Loss: 0.1264, Val-Loss: 0.0923, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [61/1000] 94s, Loss: 0.1269, Val-Loss: 0.0924, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "5 [62/1000] 94s, Loss: 0.1264, Val-Loss: 0.0923, Best Val-Loss: 0.0918, Val-F2: 0.915\n",
      "New Learn Rate: 1.0000000000000002e-06!\n",
      "Early Stop in Epoch: 63, Best Val-Loss: 0.091807, Best F2: 0.917102\n"
     ]
    }
   ],
   "source": [
    "n_splits = 5\n",
    "model_name = 'nn_boost'\n",
    "date = '716'\n",
    "probs = [0.5, 0.6, 0.8, 0.8, 0.5, 0.5, 0.6, 0.6, 0.8, 0.8, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from itertools import islice\n",
    "file_all = train_set['image_name'].values\n",
    "test_file_all = test_set['image_name'].values\n",
    "y_all = utils.get_y(train_set['tags'].values, label_map)\n",
    "pred_tr = np.zeros((file_all.shape[0], 17))\n",
    "pred_ts = np.zeros((test_file_all.shape[0], 17))\n",
    "kf = KFold(n_splits = n_splits)\n",
    "\n",
    "k_now = 0\n",
    "for i_tr, i_vl in islice(kf.split(y_all), 0, None):\n",
    "    model = m.MyNet(17).cuda()\n",
    "    # model = m.ResNet(pic_width=pic_size[0], num_classes=17, block=resnet.BasicBlock).cuda()\n",
    "    criterion = nn.BCELoss(weight=None).cuda()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    estimator = core.Estimator(model, criterion, optimizer,num_classes=17)\n",
    "    best_model = core.BestModel()\n",
    "    \n",
    "    for epoch in range(num_epoches):\n",
    "        time_st = time()\n",
    "        # 训练\n",
    "        train_loader = utils.weighted_train_loader(\n",
    "            file_all[i_tr], y_all[i_tr], probs, transform_tr, batch_size, pic_size, factor=sample_weight_factors[i_tr])\n",
    "        loss_tr = estimator.train(train_loader)\n",
    "\n",
    "        # 验证\n",
    "        val_loader = utils.valid_loader(file_all[i_vl], y_all[i_vl], transform_vl, batch_size, pic_size)\n",
    "        loss_vl, f2_vl = estimator.validate(val_loader)\n",
    "        best_model.update(loss_vl.avg, f2_vl, estimator.model)\n",
    "        \n",
    "        # 若验证结果提升缓慢，减小学习率\n",
    "        if epoch > lr_tolerance and best_model.lrcount > lr_tolerance:\n",
    "            optimizer.param_groups[0]['lr'] = optimizer.param_groups[0]['lr'] * 0.1\n",
    "            print('New Learn Rate: {}!'.format(optimizer.param_groups[0]['lr']))\n",
    "            best_model.lrcount = 0\n",
    "        \n",
    "        # 若验证结果不再提升，保存模型、验证结果、预测结果，跳出迭代\n",
    "        if epoch > tolerance and best_model.nobetter > tolerance:\n",
    "            print('Early Stop in Epoch: {}, Best Val-Loss: {:.6f}, Best F2: {:.6f}'.format(\n",
    "                epoch+1, best_model.best_loss, best_model.best_f2.value(bestf2=True)))\n",
    "            pred_tr[i_vl, :] = f2_vl.preds\n",
    "            best_model.save('./model/model{}-date{}-kf{}.pth'.format(model_name, date, k_now+1))\n",
    "                                                               \n",
    "            tst_loader = utils.test_loader(test_file_all, transform_vl, batch_size, pic_size)\n",
    "            estimator.model.load_state_dict(best_model.best_model)                                       \n",
    "            pred_ts_temp = estimator.predict(tst_loader)\n",
    "            pred_ts =  pred_ts_temp / float(n_splits)\n",
    "            break\n",
    "        \n",
    "        # 打印每一次迭代的训练验证成绩\n",
    "        print('{} [{}/{}] {}s, Loss: {:.4f}, Val-Loss: {:.4f}, Best Val-Loss: {:.4f}, Val-F2: {:.3f}'.format(\n",
    "            k_now+1, epoch+1, num_epoches, int(time() - time_st), loss_tr.avg, loss_vl.avg, \n",
    "            best_model.best_loss, f2_vl.value(0.3)))\n",
    "        gc.collect()\n",
    "    k_now += 1\n",
    "\n",
    "# 序列化验证和预测结果，用于stacking\n",
    "np.save('./pred/model{}_date{}_pred_train.npy'.format(model_name, date), pred_tr)\n",
    "np.save('./pred/model{}_date{}_pred_test.npy'.format(model_name, date), pred_ts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1. ,  1. ,  0.1, ...,  1. ,  1. ,  0.1])"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_weight_factors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "torch.load('tensors.pt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}