shopee-pytorch-xlmroberta-doubles-relativebinclf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "textile-superintendent",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:17.504036Z",
     "iopub.status.busy": "2021-04-10T13:07:17.502534Z",
     "iopub.status.idle": "2021-04-10T13:07:20.015695Z",
     "shell.execute_reply": "2021-04-10T13:07:20.015063Z"
    },
    "papermill": {
     "duration": 2.536659,
     "end_time": "2021-04-10T13:07:20.015868",
     "exception": false,
     "start_time": "2021-04-10T13:07:17.479209",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/shopee-generate-data-for-triplet-loss/train_triplets_imgs.csv\r\n",
      "sample_submission.csv  test.csv  test_images  train.csv  train_images\r\n",
      "../input/shopee-generate-data-for-triplet-loss/train_triplets_titles.csv\r\n",
      "../input/shopee-pytorch-siamese-triplet-loss-xlmrobe-cd1846/xlmroberta_256_fold0.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../input/shopee-generate-data-for-triplet-loss/train_triplets_imgs.csv\n",
    "!ls ../input/shopee-product-matching/\n",
    "!ls ../input/shopee-generate-data-for-triplet-loss/train_triplets_titles.csv\n",
    "!ls ../input/shopee-pytorch-siamese-triplet-loss-xlmrobe-cd1846/xlmroberta_256_fold0.pth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "major-federal",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:20.077726Z",
     "iopub.status.busy": "2021-04-10T13:07:20.077117Z",
     "iopub.status.idle": "2021-04-10T13:07:28.709948Z",
     "shell.execute_reply": "2021-04-10T13:07:28.708887Z"
    },
    "papermill": {
     "duration": 8.661927,
     "end_time": "2021-04-10T13:07:28.710085",
     "exception": false,
     "start_time": "2021-04-10T13:07:20.048158",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import sys\n",
    "# sys.path.append ('/kaggle/input/pytorch-images-seresnet')\n",
    "import os\n",
    "import gc\n",
    "import time\n",
    "import math\n",
    "import random\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from   torch.nn import init\n",
    "from   torch.nn import CrossEntropyLoss, MSELoss\n",
    "from   torch.nn.modules.loss import _WeightedLoss\n",
    "import torch.nn.functional as F\n",
    "from   torch.nn import Parameter\n",
    "from   torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from   transformers import AdamW, get_cosine_schedule_with_warmup\n",
    "from   torch.cuda.amp import autocast, GradScaler\n",
    "\n",
    "from   transformers import BertForSequenceClassification, BertConfig, AutoTokenizer, AutoModelForSequenceClassification\n",
    "from   transformers import RobertaTokenizer, RobertaForSequenceClassification, XLMRobertaModel\n",
    "\n",
    "from   sklearn.model_selection import StratifiedKFold, GroupKFold\n",
    "# import timm\n",
    "\n",
    "import albumentations as A\n",
    "from   albumentations import *\n",
    "from   albumentations.pytorch import ToTensorV2\n",
    "from   albumentations.core.transforms_interface import DualTransform\n",
    "from   albumentations.augmentations import functional as AF\n",
    "import cv2\n",
    "\n",
    "from   tqdm import tqdm\n",
    "from   pprint import pprint\n",
    "from   functools import partial\n",
    "import matplotlib.pyplot as plt\n",
    "# from GPUtil import showUtilization as gpu_usage\n",
    "# from   numba import cuda\n",
    "import warnings\n",
    "warnings.filterwarnings (\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "increasing-booking",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:29.320616Z",
     "iopub.status.busy": "2021-04-10T13:07:29.319709Z",
     "iopub.status.idle": "2021-04-10T13:07:29.323287Z",
     "shell.execute_reply": "2021-04-10T13:07:29.322454Z"
    },
    "papermill": {
     "duration": 0.592806,
     "end_time": "2021-04-10T13:07:29.323445",
     "exception": false,
     "start_time": "2021-04-10T13:07:28.730639",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CFG:\n",
    "    device       = torch.device ('cuda' if torch.cuda.is_available () else 'cpu')\n",
    "    num_workers  = 8\n",
    "    model_name   = 'xlm-roberta-large'\n",
    "    bert_model_name = '/kaggle/input/xlm-roberta-large'\n",
    "    size         = 128\n",
    "    isTrain      = True\n",
    "    isFreeze     = True\n",
    "    lr           = 5e-5\n",
    "    epochs       = 1\n",
    "    warmup_steps = 0                     # if float: these many epochs are with frozen model at the beginning, if int = actual steps\n",
    "    eval_steps   = 0.5                   # if float: these many epochs are with frozen model at the beginning, if int = actual steps \n",
    "    lr_num_cycles= 0.5\n",
    "    epochsNx     = 1\n",
    "    weight_decay = 1e-6\n",
    "    max_grad_norm= 1000.0\n",
    "    seed         = 42\n",
    "    n_fold       = 10\n",
    "    train_fold   = [0]                      # [0, 1, 2, 3, 4]\n",
    "    print_every  = 100\n",
    "    adam_epsilon = 1e-8\n",
    "    train_batch_size = 64\n",
    "    eval_batch_size  = 64\n",
    "    target_size      = 1\n",
    "    model_infer_path_prefix = \".\"\n",
    "    model_train_path_prefix = \".\"\n",
    "    text_triplets_csv= \"../input/shopee-generate-data-for-triplet-loss/train_triplets_titles.csv\"\n",
    "    train_path       = '../input/shopee-product-matching/train_images'\n",
    "    # train_csv        = '../input/vinbigdata-chest-xray-abnormalities-detection/train.csv'\n",
    "    # test_path        = '../input/vinbigdata-chest-xray-resized-png-1024x1024/test'\n",
    "    output_dir       = './results'        # output directory        \n",
    "    max_steps        = 0\n",
    "    MODEL            = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "innocent-comparative",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:29.369537Z",
     "iopub.status.busy": "2021-04-10T13:07:29.368746Z",
     "iopub.status.idle": "2021-04-10T13:07:29.371591Z",
     "shell.execute_reply": "2021-04-10T13:07:29.371144Z"
    },
    "papermill": {
     "duration": 0.028692,
     "end_time": "2021-04-10T13:07:29.371707",
     "exception": false,
     "start_time": "2021-04-10T13:07:29.343015",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def init_logger (log_file=CFG.output_dir+'train.log'):\n",
    "    \n",
    "    from logging import getLogger, INFO, FileHandler,  Formatter,  StreamHandler\n",
    "    logger = getLogger (__name__)\n",
    "    logger.setLevel (INFO)\n",
    "    handler1 = StreamHandler ()\n",
    "    handler1.setFormatter (Formatter (\"%(message)s\"))\n",
    "    handler2 = FileHandler (filename=log_file)\n",
    "    handler2.setFormatter (Formatter (\"%(message)s\"))\n",
    "    logger.addHandler (handler1)\n",
    "    logger.addHandler (handler2)\n",
    "    return logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "funny-kitty",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:29.415447Z",
     "iopub.status.busy": "2021-04-10T13:07:29.414818Z",
     "iopub.status.idle": "2021-04-10T13:07:29.417232Z",
     "shell.execute_reply": "2021-04-10T13:07:29.417754Z"
    },
    "papermill": {
     "duration": 0.026671,
     "end_time": "2021-04-10T13:07:29.417874",
     "exception": false,
     "start_time": "2021-04-10T13:07:29.391203",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_everything (seed):\n",
    "    \n",
    "    random.seed (seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed (seed)\n",
    "    torch.manual_seed (seed)\n",
    "    torch.cuda.manual_seed (seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tutorial-collective",
   "metadata": {
    "papermill": {
     "duration": 0.019635,
     "end_time": "2021-04-10T13:07:29.457058",
     "exception": false,
     "start_time": "2021-04-10T13:07:29.437423",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "applied-spelling",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:29.503071Z",
     "iopub.status.busy": "2021-04-10T13:07:29.502581Z",
     "iopub.status.idle": "2021-04-10T13:07:29.865685Z",
     "shell.execute_reply": "2021-04-10T13:07:29.866148Z"
    },
    "papermill": {
     "duration": 0.391435,
     "end_time": "2021-04-10T13:07:29.868004",
     "exception": false,
     "start_time": "2021-04-10T13:07:29.476569",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anchor</th>\n",
       "      <th>positive</th>\n",
       "      <th>negative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30825</th>\n",
       "      <td>TOTEBAG KPOP PAKAI RESLETING / TOTE BAG KPOP /...</td>\n",
       "      <td>RESLETING TAS TOTE TOTEBAG KPOP / TOTE BAG KPO...</td>\n",
       "      <td>[BPOM] Cosrx Clear Fit Master Patch isi 18pcs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30826</th>\n",
       "      <td>Stroller Koper Cabin Size Kereta Dorong Bayi B...</td>\n",
       "      <td>Dorongan kereta bayi stroller BabyDoes Pronto ...</td>\n",
       "      <td>Masker Gelatin Organic 10 Gram by Poupeepou</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30827</th>\n",
       "      <td>Dakron isi Boneka 100grm</td>\n",
       "      <td>Dakron Super 100 gram/isi boneka</td>\n",
       "      <td>Lakban Besar OPP Isolasi Selotip TEBAL EKA TAP...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30828</th>\n",
       "      <td>Cussons Baby Hair and Body Wash 100ml + 100ml</td>\n",
       "      <td>Cussons Baby Hair &amp; Body Wash Mild Gentle 200ml</td>\n",
       "      <td>Stelan baju tidur 3in1~ XL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30829</th>\n",
       "      <td>Cokelat SNICKERS 35 Gram</td>\n",
       "      <td>Coklat SNICKERS 35gr import Inggris PROMO!!</td>\n",
       "      <td>Sendok Makan Bayi dan Anak Motif Tangan Mickey...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  anchor  \\\n",
       "30825  TOTEBAG KPOP PAKAI RESLETING / TOTE BAG KPOP /...   \n",
       "30826  Stroller Koper Cabin Size Kereta Dorong Bayi B...   \n",
       "30827                           Dakron isi Boneka 100grm   \n",
       "30828      Cussons Baby Hair and Body Wash 100ml + 100ml   \n",
       "30829                           Cokelat SNICKERS 35 Gram   \n",
       "\n",
       "                                                positive  \\\n",
       "30825  RESLETING TAS TOTE TOTEBAG KPOP / TOTE BAG KPO...   \n",
       "30826  Dorongan kereta bayi stroller BabyDoes Pronto ...   \n",
       "30827                   Dakron Super 100 gram/isi boneka   \n",
       "30828    Cussons Baby Hair & Body Wash Mild Gentle 200ml   \n",
       "30829        Coklat SNICKERS 35gr import Inggris PROMO!!   \n",
       "\n",
       "                                                negative  \n",
       "30825      [BPOM] Cosrx Clear Fit Master Patch isi 18pcs  \n",
       "30826        Masker Gelatin Organic 10 Gram by Poupeepou  \n",
       "30827  Lakban Besar OPP Isolasi Selotip TEBAL EKA TAP...  \n",
       "30828                         Stelan baju tidur 3in1~ XL  \n",
       "30829  Sendok Makan Bayi dan Anak Motif Tangan Mickey...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see image triplets\n",
    "df = pd.read_csv (CFG.text_triplets_csv)\n",
    "TRAIN_DF = df.iloc[:(df.shape[0]*9//10)]\n",
    "TEST_DF  = df.iloc[(df.shape[0]*9//10):]\n",
    "del df\n",
    "gc.collect ()\n",
    "TEST_DF.head ()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "promising-salad",
   "metadata": {
    "papermill": {
     "duration": 0.020007,
     "end_time": "2021-04-10T13:07:29.908667",
     "exception": false,
     "start_time": "2021-04-10T13:07:29.888660",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Xlm-Roberta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "significant-pottery",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:29.952910Z",
     "iopub.status.busy": "2021-04-10T13:07:29.952324Z",
     "iopub.status.idle": "2021-04-10T13:07:31.641207Z",
     "shell.execute_reply": "2021-04-10T13:07:31.640061Z"
    },
    "papermill": {
     "duration": 1.712582,
     "end_time": "2021-04-10T13:07:31.641353",
     "exception": false,
     "start_time": "2021-04-10T13:07:29.928771",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "max_len         = CFG.size\n",
    "tokenizer       = AutoTokenizer.from_pretrained (CFG.bert_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "tender-mercy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:31.688785Z",
     "iopub.status.busy": "2021-04-10T13:07:31.687938Z",
     "iopub.status.idle": "2021-04-10T13:07:31.690692Z",
     "shell.execute_reply": "2021-04-10T13:07:31.690233Z"
    },
    "papermill": {
     "duration": 0.028519,
     "end_time": "2021-04-10T13:07:31.690797",
     "exception": false,
     "start_time": "2021-04-10T13:07:31.662278",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def encode (premise, hypothesis):\n",
    "    \n",
    "    encoded_dict = tokenizer (\n",
    "        premise,                   # 1st of the Sentence pair to encode.\n",
    "        hypothesis,                # 2nd of the Sentence pair to encode.\n",
    "        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "        truncation=True,           # just max_len will not automatically truncate\n",
    "        max_length = max_len,      # Pad & truncate all sentences.\n",
    "        padding='max_length',\n",
    "        return_attention_mask = True,   # Construct attn. masks.\n",
    "        return_tensors = 'pt',     # Return pytorch tensors.\n",
    "    ) \n",
    "    # print ('encoded_dict =', encoded_dict)\n",
    "    # 1-D tensors are expected for a sample. Hence squeeze these 2-D tensors e.g [1,256] shaped tensors to 1-D [256] shape \n",
    "    for k in encoded_dict:\n",
    "        encoded_dict[k] = torch.squeeze (encoded_dict[k])\n",
    "    return encoded_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "capable-toronto",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:31.738822Z",
     "iopub.status.busy": "2021-04-10T13:07:31.737181Z",
     "iopub.status.idle": "2021-04-10T13:07:31.739585Z",
     "shell.execute_reply": "2021-04-10T13:07:31.740005Z"
    },
    "papermill": {
     "duration": 0.029003,
     "end_time": "2021-04-10T13:07:31.740132",
     "exception": false,
     "start_time": "2021-04-10T13:07:31.711129",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class DoubleTextDataset (Dataset):\n",
    "    \n",
    "    def __init__(self, df=TRAIN_DF):\n",
    "        self.df = df   # pd.read_csv (img_triplets_csv).reset_index (drop=True)\n",
    "        return\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        \n",
    "        triplet  = self.df.iloc[index]\n",
    "        anchor   = triplet['anchor']\n",
    "        positive = triplet['positive']\n",
    "        negative = triplet['negative']\n",
    "        \n",
    "        positive = encode (anchor, positive)\n",
    "        negative = encode (anchor, negative)\n",
    "        return (positive, negative)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "sudden-demographic",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:31.788653Z",
     "iopub.status.busy": "2021-04-10T13:07:31.787877Z",
     "iopub.status.idle": "2021-04-10T13:07:31.990884Z",
     "shell.execute_reply": "2021-04-10T13:07:31.991278Z"
    },
    "papermill": {
     "duration": 0.230808,
     "end_time": "2021-04-10T13:07:31.991446",
     "exception": false,
     "start_time": "2021-04-10T13:07:31.760638",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+ve =  {'input_ids': tensor([     0,  65693,   3065,   6901,  75385,  90349, 190878,    102,    482,\n",
      "             6, 194817,  85207,      2,      2,    378, 118978,    147,     45,\n",
      "         39400,    268,  65693,   3065,   6901,  75385,    248,    636,   3334,\n",
      "           482,      6, 194817,  85207,      2,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0])}\n",
      "-ve =  {'input_ids': tensor([     0,  65693,   3065,   6901,  75385,  90349, 190878,    102,    482,\n",
      "             6, 194817,  85207,      2,      2,  34994,  34324,  17715, 139087,\n",
      "            62,  74368,  48153, 173103,      6, 135775, 186104,   9780,      2,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0])}\n",
      "+ve =  {'input_ids': tensor([     0,   1735,  16840, 166564,  15491,  96957,    441,  30535,   7237,\n",
      "            20,    262,   1596,  44808,     66,   1136,     73,    619,  65967,\n",
      "            20,   4267,    461,   9091, 180203,  51578,  16035,      2,      2,\n",
      "          6206,     73,  44808,     66,   1735,  16840,   3036,  73720,   6206,\n",
      "        204228,   1236, 166564,   1781,   1328,   1197,   7237,      2,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0])}\n",
      "-ve =  {'input_ids': tensor([     0,   1735,  16840, 166564,  15491,  96957,    441,  30535,   7237,\n",
      "            20,    262,   1596,  44808,     66,   1136,     73,    619,  65967,\n",
      "            20,   4267,    461,   9091, 180203,  51578,  16035,      2,      2,\n",
      "           382,  36719,      7,   5452,   2278,    382,  23195,  77670,   1019,\n",
      "          1819,  74322,      2,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1,      1,      1,      1,      1,      1,      1,      1,\n",
      "             1,      1]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
      "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
      "        0, 0, 0, 0, 0, 0, 0, 0])}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def plot_txt (dataset_show):\n",
    "        \n",
    "    for i in range (2):\n",
    "            idx = np.random.randint (0, len (dataset_show))\n",
    "            dict1, dict2 = dataset_show[idx] \n",
    "            print ('+ve = ', dict1)\n",
    "            print ('-ve = ', dict2)\n",
    "    return \n",
    "\n",
    "TR_DATASET = DoubleTextDataset ()\n",
    "plot_txt (TR_DATASET)\n",
    "del TR_DATASET\n",
    "gc.collect ()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "several-optimum",
   "metadata": {
    "papermill": {
     "duration": 0.024027,
     "end_time": "2021-04-10T13:07:32.037050",
     "exception": false,
     "start_time": "2021-04-10T13:07:32.013023",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "> # Loss Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "mysterious-tiffany",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:32.083906Z",
     "iopub.status.busy": "2021-04-10T13:07:32.083348Z",
     "iopub.status.idle": "2021-04-10T13:07:32.086784Z",
     "shell.execute_reply": "2021-04-10T13:07:32.087278Z"
    },
    "papermill": {
     "duration": 0.0288,
     "end_time": "2021-04-10T13:07:32.087439",
     "exception": false,
     "start_time": "2021-04-10T13:07:32.058639",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_criterion ():\n",
    "    \n",
    "    criterion = nn.BCEWithLogitsLoss ()\n",
    "    return criterion"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "compliant-footwear",
   "metadata": {
    "papermill": {
     "duration": 0.021099,
     "end_time": "2021-04-10T13:07:32.129856",
     "exception": false,
     "start_time": "2021-04-10T13:07:32.108757",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "norwegian-colleague",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:32.184679Z",
     "iopub.status.busy": "2021-04-10T13:07:32.184020Z",
     "iopub.status.idle": "2021-04-10T13:07:32.188020Z",
     "shell.execute_reply": "2021-04-10T13:07:32.187582Z"
    },
    "papermill": {
     "duration": 0.036614,
     "end_time": "2021-04-10T13:07:32.188129",
     "exception": false,
     "start_time": "2021-04-10T13:07:32.151515",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MyGAPModelForSeqClf (nn.Module):\n",
    "    \n",
    "    def __init__(self, bert_model_name=CFG.bert_model_name, outputCount=3, #CFG.target_size, \n",
    "                 drop_prob=0.2, nonlin=nn.SiLU ()):\n",
    "        \n",
    "        super (MyGAPModelForSeqClf, self).__init__()\n",
    "        self.model       = AutoModelForSequenceClassification.from_pretrained (bert_model_name).base_model  # adding .base_model if using pretrained XLMRobertaForSequenceClassification\n",
    "        self.drop_prob   = drop_prob\n",
    "        self.nonlin      = nonlin\n",
    "        self.outputCount = outputCount\n",
    "        hidden_size      = self.model.config.hidden_size\n",
    "        self.dense       = nn.Linear (hidden_size, hidden_size)\n",
    "        self.batchnorm   = nn.BatchNorm1d (hidden_size)\n",
    "        self.outDense    = nn.Linear (hidden_size, outputCount)\n",
    "        self.dropout     = nn.Dropout (drop_prob)\n",
    "        # self.outActivtn  = nn.LogSoftmax (dim=1)\n",
    "        # self.NLLLoss     = nn.NLLLoss ()\n",
    "        return\n",
    "    \n",
    "    def freeze (self):\n",
    "        \n",
    "        for param in self.model.base_model.parameters ():\n",
    "            param.requires_grad = False\n",
    "        return\n",
    "    \n",
    "    def unfreeze (self):\n",
    "        \n",
    "        for param in self.model.base_model.parameters ():\n",
    "            param.requires_grad = True\n",
    "        return\n",
    "    \n",
    "    def forward (self, input_ids, attention_mask, token_type_ids=None, labels=None, **kwargs):\n",
    "        \n",
    "        last_hidden_states = None\n",
    "        \n",
    "        # The base bert model do not take labels as input\n",
    "        if token_type_ids is None:\n",
    "            moutput = self.model (input_ids=input_ids, attention_mask=attention_mask)\n",
    "            last_hidden_states = moutput[0]\n",
    "        else:\n",
    "            moutput = self.model (input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "            last_hidden_states = moutput[0]\n",
    "        #print('last_hidden_states.size=', last_hidden_states.size())\n",
    "        \n",
    "        # GAP: last_hidden_states shape = batch_size * max_seq_len * emb_dim(1024?)\n",
    "        # output shape = batch_size * emb_dim(1024?)  i.e avg across the sequence\n",
    "        last_hidden_states = torch.mean (last_hidden_states, 1)             #;print('GAP last_hidden_states.size=', last_hidden_states.size())\n",
    "        # fcnn\n",
    "        X = self.dropout (self.nonlin (self.batchnorm (self.dense (last_hidden_states))))        #;print('X.size=', X.size())\n",
    "        out_logits = self.outDense (X)                #;print('out_logits.size=', out_logits.size())\n",
    "        \n",
    "        \"\"\"if labels is None:\n",
    "            \n",
    "            # return a named tuple\n",
    "            Logits = namedtuple ('Logits',['logits'])\n",
    "            out_logits = Logits (out_logits)\n",
    "            return out_logits\n",
    "        log_ps = self.outActivtn (out_logits)         #;print('log_ps.size=', log_ps.size())\n",
    "        batchLoss = self.NLLLoss (log_ps, labels)\n",
    "        \n",
    "        # return a named tuple\n",
    "        Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])\n",
    "        loss_logits = Loss_Logits (batchLoss, out_logits)\n",
    "        return loss_logits \"\"\"\n",
    "        return out_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "structural-moisture",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:07:32.236364Z",
     "iopub.status.busy": "2021-04-10T13:07:32.235803Z",
     "iopub.status.idle": "2021-04-10T13:08:31.808746Z",
     "shell.execute_reply": "2021-04-10T13:08:31.807687Z"
    },
    "papermill": {
     "duration": 59.598796,
     "end_time": "2021-04-10T13:08:31.808896",
     "exception": false,
     "start_time": "2021-04-10T13:07:32.210100",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at /kaggle/input/xlm-roberta-large were not used when initializing XLMRobertaForSequenceClassification: ['pooler.dense.weight', 'pooler.dense.bias']\n",
      "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at /kaggle/input/xlm-roberta-large and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = MyGAPModelForSeqClf ()\n",
    "\n",
    "# load the pretrained model which was trained this code only (by commenting out these 4 line)\n",
    "try:\n",
    "    model.load_state_dict (torch.load (\"../input/robertagapxnlimnlirishi/roberta-gap-xnli-mnli-rishi/my_model.bin\"))\n",
    "except:\n",
    "    model.load_state_dict (torch.load (\"../input/robertagapxnlimnlirishi/roberta-gap-xnli-mnli-rishi/my_model.bin\", map_location='cpu'))\n",
    "model.outDense = nn.Linear (model.model.config.hidden_size, CFG.target_size)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings (\"ignore\")\n",
    "model.to (CFG.device)\n",
    "CFG.MODEL = model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alternative-viking",
   "metadata": {
    "papermill": {
     "duration": 0.021765,
     "end_time": "2021-04-10T13:08:31.853102",
     "exception": false,
     "start_time": "2021-04-10T13:08:31.831337",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Trainer Helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "static-ballet",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:08:31.903475Z",
     "iopub.status.busy": "2021-04-10T13:08:31.901926Z",
     "iopub.status.idle": "2021-04-10T13:08:31.904541Z",
     "shell.execute_reply": "2021-04-10T13:08:31.904983Z"
    },
    "papermill": {
     "duration": 0.030045,
     "end_time": "2021-04-10T13:08:31.905111",
     "exception": false,
     "start_time": "2021-04-10T13:08:31.875066",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def format_time (elapsed):\n",
    "    '''\n",
    "    Takes a time in seconds and returns a string hh:mm:ss\n",
    "    '''\n",
    "    \n",
    "    # Round to the nearest second.\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    \n",
    "    # Format as hh:mm:ss\n",
    "    return str (datetime.timedelta (seconds=elapsed_rounded))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "unlimited-mailman",
   "metadata": {
    "papermill": {
     "duration": 0.022061,
     "end_time": "2021-04-10T13:08:31.949236",
     "exception": false,
     "start_time": "2021-04-10T13:08:31.927175",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "played-affairs",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:08:32.024056Z",
     "iopub.status.busy": "2021-04-10T13:08:31.996892Z",
     "iopub.status.idle": "2021-04-10T13:08:32.047948Z",
     "shell.execute_reply": "2021-04-10T13:08:32.048326Z"
    },
    "papermill": {
     "duration": 0.077263,
     "end_time": "2021-04-10T13:08:32.048466",
     "exception": false,
     "start_time": "2021-04-10T13:08:31.971203",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MyTrainer:\n",
    "    \n",
    "    def __init__(self, fold, model, train_dataset, eval_dataset, criterion, \n",
    "                 checkpoint_path=None, isResume=False):\n",
    "        \n",
    "        self.fold             = fold\n",
    "        self.start_epoch      = 0\n",
    "        self.model            = model\n",
    "        # load checkpoint\n",
    "        if checkpoint_path is not None:\n",
    "            if isResume:\n",
    "                self.start_epoch = self.load_checkpoint (checkpoint_path, isResume=True) + 1\n",
    "            else:\n",
    "                self.load_checkpoint (checkpoint_path, isResume=False)\n",
    "        self.model            = self.model.to (CFG.device)\n",
    "        if CFG.isFreeze:\n",
    "            self.model.freeze ()\n",
    "        else:\n",
    "            self.model.unfreeze ()\n",
    "        CFG.MODEL             = self.model\n",
    "        self.train_dataset    = train_dataset\n",
    "        self.eval_dataset     = eval_dataset\n",
    "        self.criterion        = criterion\n",
    "        self.isTrained        = False\n",
    "        self.device           = CFG.device\n",
    "        self.optimizer        = AdamW (self.model.parameters (), lr=CFG.lr, eps=CFG.adam_epsilon, weight_decay=CFG.weight_decay)\n",
    "        self.epochs           = CFG.epochs\n",
    "        self.set_dataLoaders ()\n",
    "        self.training_stats   = []\n",
    "        self.modelFile        = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}.pth\"\n",
    "        if eval_dataset is not None:\n",
    "            self.minLossModelFile = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_min_val_loss.pth\"\n",
    "            self.maxAccModelFile  = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_max_val_acc.pth\"\n",
    "        else:\n",
    "            self.minLossModelFile = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_min_tr_loss.pth\"\n",
    "            self.maxAccModelFile  = f\"{CFG.model_train_path_prefix}/{CFG.model_name}_{CFG.size}_fold{self.fold}_max_tr_acc.pth\"\n",
    "        \n",
    "        self.min_val_loss         = 9999\n",
    "        self.min_train_loss       = 9999\n",
    "        self.max_val_acc          = -1    \n",
    "        return\n",
    "    \n",
    "    def set_dataLoaders (self):\n",
    "        # Create the DataLoaders for our training and validation sets.\n",
    "        \n",
    "        if isinstance (self.train_dataset, torch.utils.data.IterableDataset):\n",
    "            train_sampler = None\n",
    "        else:\n",
    "            train_sampler = RandomSampler (self.train_dataset)           # Better use RandomSampler\n",
    "        train_dataloader  = DataLoader (\n",
    "                    self.train_dataset,                                  # The training samples.\n",
    "                    sampler     = train_sampler,                           \n",
    "                    batch_size  = CFG.train_batch_size,\n",
    "                    num_workers = CFG.num_workers,\n",
    "                    pin_memory  = True\n",
    "        )\n",
    "        # train_dataloader  = DataLoader (self.train_dataset, batch_size=CFG.train_batch_size) # TODO: comment this\n",
    "        validation_dataloader = None\n",
    "        if self.eval_dataset:\n",
    "            validation_dataloader = DataLoader (\n",
    "                        self.eval_dataset, \n",
    "                        sampler     = SequentialSampler (self.eval_dataset),\n",
    "                        batch_size  = CFG.eval_batch_size,\n",
    "                        num_workers = CFG.num_workers,\n",
    "                        pin_memory  = False\n",
    "            )\n",
    "            # validation_dataloader  = DataLoader (self.eval_dataset, batch_size=CFG.eval_batch_size) # TODO: comment this\n",
    "        \n",
    "        if type (CFG.warmup_steps) is float:\n",
    "            CFG.warmup_steps = int (CFG.warmup_steps * len (train_dataloader))\n",
    "        # Total number of training steps is [number of batches] x [number of epochs]\n",
    "        num_training_steps = len (train_dataloader) * self.epochs        \n",
    "        lr_scheduler = get_cosine_schedule_with_warmup (self.optimizer, num_cycles=CFG.lr_num_cycles,\n",
    "                        num_warmup_steps=CFG.warmup_steps, num_training_steps=num_training_steps)\n",
    "        \n",
    "        if type (CFG.eval_steps) is float:\n",
    "            CFG.eval_steps = int (CFG.eval_steps * len (train_dataloader))\n",
    "        self.train_dataloader, self.validation_dataloader, self.lr_scheduler, self.num_training_steps=train_dataloader, validation_dataloader, lr_scheduler, num_training_steps\n",
    "        return\n",
    "            \n",
    "    def test_iterate_dataloader (self):\n",
    "        \n",
    "        for step, batch in enumerate (self.train_dataloader):\n",
    "            print (step)\n",
    "            print (batch)\n",
    "            break\n",
    "        return\n",
    "    \n",
    "    def save_checkpoint (self, epoch, path):\n",
    "        \n",
    "        checkpoint = {\n",
    "            'epoch'               : epoch,\n",
    "            'model_state_dict'    : self.model.state_dict (),\n",
    "            'optimizer_state_dict': self.optimizer.state_dict (),\n",
    "            'lr_sched_state_dict' : self.lr_scheduler.state_dict (),\n",
    "            'training_stats'      : self.training_stats,\n",
    "            'max_val_acc'         : self.max_val_acc,\n",
    "            'min_train_loss'      : self.min_train_loss,\n",
    "            'min_val_loss'        : self.min_val_loss,\n",
    "        }\n",
    "        torch.save (checkpoint, path)\n",
    "        gc.collect (); torch.cuda.empty_cache ()\n",
    "        print (\"saved checkpoint\", path)\n",
    "        return\n",
    "    \n",
    "    def load_checkpoint (self, path, isResume=False):\n",
    "        \n",
    "        epoch      = 0\n",
    "        checkpoint = torch.load (path, map_location=torch.device ('cpu'))\n",
    "        self.model.load_state_dict (checkpoint['model_state_dict'])\n",
    "        if isResume:\n",
    "            \n",
    "            self.optimizer.load_state_dict (checkpoint['optimizer_state_dict'])\n",
    "            self.lr_scheduler.load_state_dict (checkpoint['lr_sched_state_dict'])\n",
    "            epoch = checkpoint['epoch']\n",
    "            self.training_stats  = checkpoint['training_stats']\n",
    "            self.min_val_loss    = checkpoint['min_val_loss']\n",
    "            self.min_train_loss  = checkpoint['min_train_loss']\n",
    "            self.max_val_acc     = checkpoint['max_val_acc']\n",
    "            print (\"Loaded model, optimizer, and lr_scheduler from -\", path)\n",
    "        else:\n",
    "            print (\"Loaded model from -\", path)\n",
    "            \n",
    "        self.model.train ()\n",
    "        return epoch\n",
    "    \n",
    "    def train (self):\n",
    "        \n",
    "        seed_everything (seed=CFG.seed)\n",
    "        step             = 0\n",
    "        total_t0         = time.time ()\n",
    "        scaler           = GradScaler()\n",
    "        for epoch_i in range (self.start_epoch, self.epochs):\n",
    "            \n",
    "            avg_epoch_train_loss   = 0\n",
    "            total_epoch_train_loss = 0\n",
    "            print('======== Epoch {:} / {:} ========'.format (epoch_i + 1, self.epochs))\n",
    "            t0 = time.time ()\n",
    "            self.model.train ()\n",
    "            for stp, batch in tqdm (enumerate (self.train_dataloader), total=len(self.train_dataloader)):\n",
    "                \n",
    "                # Print Stats\n",
    "                # if step % CFG.print_every == 0:\n",
    "                #     elapsed = format_time (time.time() - t0)\n",
    "                #     print ('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format (step, len (self.train_dataloader), elapsed))                \n",
    "                if ((CFG.max_steps > 0 and CFG.max_steps < step) or \n",
    "                    (CFG.eval_steps>0 and stp==CFG.eval_steps)): # or step==0):   # TODO: rm this comment\n",
    "                    \n",
    "                    self.save_checkpoint (epoch_i, self.modelFile)\n",
    "                    training_time = format_time (time.time () - t0)            \n",
    "                    if self.validation_dataloader:\n",
    "                        \n",
    "                        avg_val_loss, avg_val_accuracy, validation_time = self.evaluate (epoch_i, avg_epoch_train_loss, training_time)\n",
    "                        # save this model if the eval loss decreases from the minimum so far\n",
    "                        checkpoint_epoch = epoch_i\n",
    "                        if stp==CFG.eval_steps:\n",
    "                            # don't count this epoch in the checkpoint since this epoch \n",
    "                            # has not completed. Hence, checkpoint at prev epoch\n",
    "                            checkpoint_epoch = epoch_i-1\n",
    "                        if avg_val_loss < self.min_val_loss:                             \n",
    "                            self.min_val_loss = avg_val_loss\n",
    "                            self.save_checkpoint (checkpoint_epoch, self.minLossModelFile)\n",
    "                        if avg_val_accuracy > self.max_val_acc:\n",
    "                            self.max_val_acc = avg_val_accuracy\n",
    "                            self.save_checkpoint (checkpoint_epoch, self.maxAccModelFile)\n",
    "                    if CFG.max_steps > 0 and CFG.max_steps < step:\n",
    "                        \n",
    "                        print (\"\")\n",
    "                        print (\"Training complete!\")\n",
    "                        print (\"Total training took {:} (h:mm:ss)\".format (format_time (time.time ()-total_t0)))\n",
    "                        self.isTrained = True\n",
    "                        self.model.cpu ()\n",
    "                        self.model.eval ()\n",
    "                        self.save_checkpoint (epoch_i, self.modelFile)\n",
    "                        try:\n",
    "                            torch.cuda.empty_cache ()\n",
    "                            self.plot_train_stats (self.training_stats)\n",
    "                        except:\n",
    "                            pass\n",
    "                        return pd.DataFrame (self.training_stats)\n",
    "                \n",
    "                ########################################################\n",
    "                # Train\n",
    "                ########################################################\n",
    "                # self.model.zero_grad ()\n",
    "                for i in [0,1]:\n",
    "                    for k in batch[i]:\n",
    "                        batch[i][k] = batch[i][k].to (self.device)\n",
    "                \n",
    "                pos_txt = batch[0]\n",
    "                neg_txt = batch[1]\n",
    "                with autocast():\n",
    "                    \n",
    "                    # print (\"pos_txt['input_ids'].size() =\", pos_txt['input_ids'].size ())\n",
    "                    bs     = pos_txt['input_ids'].size (0)\n",
    "                    labels = torch.ones ((bs, 1)).to (CFG.device)\n",
    "                    logits = self.model (**pos_txt)            \n",
    "                    loss   = self.criterion (logits, labels)\n",
    "                    scaler.scale (loss).backward ()\n",
    "                    # torch.nn.utils.clip_grad_norm_ (self.model.parameters (), CFG.max_grad_norm)\n",
    "                    scaler.step (self.optimizer)\n",
    "                    scaler.update ()\n",
    "                    \n",
    "                    labels = torch.zeros ((bs, 1)).to (CFG.device)\n",
    "                    logits = self.model (**neg_txt)                    \n",
    "                    loss   = self.criterion (logits, labels)\n",
    "                    scaler.scale (loss).backward ()\n",
    "                    # torch.nn.utils.clip_grad_norm_ (self.model.parameters (), CFG.max_grad_norm)\n",
    "                    scaler.step (self.optimizer)\n",
    "                    scaler.update ()\n",
    "                    \n",
    "                    self.optimizer.zero_grad ()\n",
    "                    self.lr_scheduler.step ()\n",
    "                    \n",
    "                total_epoch_train_loss += loss.cpu ().item ()\n",
    "                avg_epoch_train_loss    = total_epoch_train_loss / (stp+1)\n",
    "                step += 1\n",
    "            # all steps of an epoch end\n",
    "            \n",
    "            # Measure how long this epoch took.\n",
    "            training_time = format_time (time.time () - t0)            \n",
    "            print (\"  Average training loss: {0:.4f}\".format (avg_epoch_train_loss))\n",
    "            print (\"  Training epcoh took: {:}\".format (training_time))            \n",
    "            if self.validation_dataloader:    \n",
    "                \n",
    "                avg_val_loss, avg_val_accuracy, validation_time = self.evaluate (epoch_i, avg_epoch_train_loss, training_time)\n",
    "                # save this epoch's model if the eval loss decreases from the minimum so far\n",
    "                if avg_val_loss < self.min_val_loss:                    \n",
    "                    self.min_val_loss = avg_val_loss\n",
    "                    self.save_checkpoint (epoch_i, self.minLossModelFile)\n",
    "                if avg_val_accuracy > self.max_val_acc:\n",
    "                    self.max_val_acc = avg_val_accuracy\n",
    "                    self.save_checkpoint (epoch_i, self.maxAccModelFile)\n",
    "            else:                \n",
    "                training_stats.append ({\n",
    "                    'epoch'         : epoch_i + 1,\n",
    "                    'training_loss' : avg_epoch_train_loss,\n",
    "                    'training_time' : training_time,\n",
    "                })\n",
    "                if avg_train_loss < self.min_train_loss:                     \n",
    "                    self.min_train_loss = avg_train_loss\n",
    "                    self.save_checkpoint (epoch_i, self.minLossModelFile)\n",
    "            self.save_checkpoint (epoch_i, self.modelFile)\n",
    "            # 1 epoch end\n",
    "        # all epochs end\n",
    "        \n",
    "        # just get the best class thresholds at the end\n",
    "        if self.validation_dataloader:\n",
    "            print ('At training end, threshold Adjustment (last row of the train summary DF)')\n",
    "            print (self.evaluate (epoch_i, avg_epoch_train_loss, training_time, isThreshAdjust=True))\n",
    "            print ('<: avg_val_loss, avg_val_accuracy, validation_time')\n",
    "        \n",
    "        print (\"***** Training complete! *****\")\n",
    "        print (\"Total training took {:} (h:mm:ss)\".format (format_time (time.time ()-total_t0)))\n",
    "        self.isTrained = True\n",
    "        self.model.cpu ()\n",
    "        self.model.eval ()\n",
    "        try:\n",
    "            torch.cuda.empty_cache ()\n",
    "            self.plot_train_stats (self.training_stats)\n",
    "        except:\n",
    "            pass\n",
    "        return pd.DataFrame (self.training_stats)\n",
    "    \n",
    "    def evaluate (self, epoch_i, avg_train_loss=999, training_time=999, isThreshAdjust=False):\n",
    "        \n",
    "        t0           = time.time ()\n",
    "        all_labels   = []\n",
    "        all_pred_prs = []\n",
    "        # Put the model in evaluation mode--the dropout layers behave differently\n",
    "        # during evaluation.\n",
    "        self.model.eval ()\n",
    "        \n",
    "        # Tracking variables\n",
    "        total_eval_accuracy  = 0\n",
    "        total_eval_loss      = 0\n",
    "        nb_eval_steps        = 0\n",
    "        correct_pred_count   = 0\n",
    "        total_pred_count     = 0\n",
    "        # Evaluate data for one epoch\n",
    "        for batch in self.validation_dataloader:\n",
    "            with torch.no_grad ():\n",
    "                \n",
    "                for i in [0,1]:\n",
    "                    for k in batch[i]:\n",
    "                        batch[i][k] = batch[i][k].to (self.device)\n",
    "                \n",
    "                pos_txt = batch[0]\n",
    "                neg_txt = batch[1]\n",
    "                # print (\"pos_txt['input_ids'].size() =\", pos_txt['input_ids'].size ())\n",
    "                bs     = pos_txt['input_ids'].size (0)\n",
    "                labels = torch.ones ((bs, 1)).to (CFG.device)\n",
    "                logits1= self.model (**pos_txt)            \n",
    "                loss1  = self.criterion (logits1, labels)\n",
    "                \n",
    "                labels = torch.zeros ((bs, 1)).to (CFG.device)\n",
    "                logits0= self.model (**neg_txt)\n",
    "                loss0  = self.criterion (logits0, labels)\n",
    "                \n",
    "            correct_pred_count += np.sum ((torch.sigmoid (logits1).detach ().cpu ().numpy () >= 0.5) + 0.0) + np.sum ((torch.sigmoid (logits0).detach ().cpu ().numpy () < 0.5) + 0.0)\n",
    "            total_pred_count   += logits0.shape[0]\n",
    "            total_eval_loss    += (loss1.item () + loss0.item ()) / 2.0\n",
    "        \n",
    "        avg_val_loss     = total_eval_loss / len (self.validation_dataloader)\n",
    "        avg_val_accuracy = correct_pred_count / total_pred_count\n",
    "        print (\"Val Loss: {0:.4f}\".format (avg_val_loss))\n",
    "        print (\"Val Accuracy: {0:.4f}\".format (avg_val_accuracy))\n",
    "        validation_time = format_time (time.time () - t0)\n",
    "        self.training_stats.append ({\n",
    "                'epoch'         : epoch_i + 1,\n",
    "                'training_loss' : avg_train_loss,\n",
    "                'eval_loss'     : avg_val_loss,\n",
    "                'eval_accuracy' : avg_val_accuracy,\n",
    "                'training_time' : training_time,\n",
    "                'eval_time'     : validation_time                   \n",
    "        })\n",
    "        self.model.train ()\n",
    "        print (\"Validation took {:} (h:mm:ss)\".format (format_time (time.time () - t0)))\n",
    "        return avg_val_loss, avg_val_accuracy, validation_time\n",
    "        \n",
    "        \n",
    "    def plot_train_stats (self, training_stats):\n",
    "        \"\"\"\n",
    "        Draw Classification Report curve\n",
    "        \"\"\"\n",
    "        \n",
    "        accuracies = eval_losses = tr_losses = epochs = -1\n",
    "        epochs = len (training_stats)\n",
    "        if 'eval_accuracy' in training_stats[0]:\n",
    "            accuracies = [e['eval_accuracy'] for e in training_stats]\n",
    "            sns.lineplot (x=np.arange(1, epochs + 1), y=accuracies, label='val_accuracy')\n",
    "        if 'eval_loss' in training_stats[0]:\n",
    "            eval_losses= [e['eval_loss'] for e in training_stats]\n",
    "        if 'training_loss'  in training_stats[0]:\n",
    "            tr_losses  = [e['training_loss'] for e in training_stats]\n",
    "            sns.lineplot (x=np.arange(1, epochs + 1), y=tr_losses,  label='tr_losses')\n",
    "            \n",
    "        plt.show ()\n",
    "        print ('accuracies :', accuracies)        \n",
    "        print ('eval_losses:', eval_losses)\n",
    "        print ('tr_losses  :', tr_losses)\n",
    "        return\n",
    "    \n",
    "    def get_trained_model (self):\n",
    "        \n",
    "        if self.isTrained:\n",
    "            return self.model.eval ()\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "pending-canal",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:08:32.097326Z",
     "iopub.status.busy": "2021-04-10T13:08:32.096824Z",
     "iopub.status.idle": "2021-04-10T13:08:32.100232Z",
     "shell.execute_reply": "2021-04-10T13:08:32.100656Z"
    },
    "papermill": {
     "duration": 0.029719,
     "end_time": "2021-04-10T13:08:32.100781",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.071062",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def free_gpu_cache ():\n",
    "    \n",
    "    # print(\"Initial GPU Usage\")\n",
    "    # gpu_usage()                             \n",
    "\n",
    "    torch.cuda.empty_cache()\n",
    "\n",
    "    # cuda.select_device(0)\n",
    "    # cuda.close()\n",
    "    # cuda.select_device(0)\n",
    "\n",
    "    # print(\"GPU Usage after emptying the cache\")\n",
    "    # gpu_usage()\n",
    "    return\n",
    "\n",
    "# free_gpu_cache()           "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "given-driver",
   "metadata": {
    "papermill": {
     "duration": 0.021982,
     "end_time": "2021-04-10T13:08:32.144877",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.122895",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bigger-brick",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:08:32.196803Z",
     "iopub.status.busy": "2021-04-10T13:08:32.196120Z",
     "iopub.status.idle": "2021-04-10T13:08:32.198928Z",
     "shell.execute_reply": "2021-04-10T13:08:32.198499Z"
    },
    "papermill": {
     "duration": 0.031567,
     "end_time": "2021-04-10T13:08:32.199026",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.167459",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_fold_loop (checkpoint_path=None):\n",
    "\n",
    "    print (f\"========== training ==========\")\n",
    "    fold = 0 # fold is not used\n",
    "    criterion      = get_criterion ()\n",
    "    model          = CFG.MODEL\n",
    "    if model is None and checkpoint_path is None:\n",
    "        print (\"CFG.MODEL is None\")\n",
    "        model      = getModel (fold, isTrain=True)\n",
    "        model      = model.float()\n",
    "    elif model is not None and checkpoint_path is not None:\n",
    "        pass\n",
    "        # checkpoint_path = None\n",
    "    elif model is None and checkpoint_path is not None:\n",
    "        print (\"CFG.MODEL is None\")\n",
    "        model      = getModel (fold, isTrain=False)\n",
    "        model      = model.float()\n",
    "        \n",
    "    train_dataset  = DoubleTextDataset ()\n",
    "    valid_dataset  = DoubleTextDataset (TEST_DF)\n",
    "    trainer        = MyTrainer (\n",
    "        fold            = fold,\n",
    "        model           = model,\n",
    "        train_dataset   = train_dataset,\n",
    "        eval_dataset    = valid_dataset,\n",
    "        criterion       = criterion,\n",
    "        checkpoint_path = checkpoint_path\n",
    "    )\n",
    "    metrics = trainer.train ()\n",
    "    return metrics\n",
    "    \n",
    "    # To plot lr uncomment this\n",
    "    # lrs = []\n",
    "    # for i in range (CFG.epochs*len (trainer.train_dataloader)):\n",
    "    #     trainer.lr_scheduler.step ()\n",
    "    #     lrs.append (trainer.optimizer.param_groups[0][\"lr\"])\n",
    "    # print (lrs)\n",
    "    # plt.plot (lrs)\n",
    "    # plt.show ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "portuguese-figure",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:08:32.249091Z",
     "iopub.status.busy": "2021-04-10T13:08:32.248525Z",
     "iopub.status.idle": "2021-04-10T13:08:32.251419Z",
     "shell.execute_reply": "2021-04-10T13:08:32.250896Z"
    },
    "papermill": {
     "duration": 0.029986,
     "end_time": "2021-04-10T13:08:32.251521",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.221535",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_main (checkpoint_path=None):\n",
    "    \n",
    "    # print (f\"========== train_main() ==========\")\n",
    "    if CFG.isTrain:        \n",
    "        valid_scores_df = pd.DataFrame ()\n",
    "        for fold in range (CFG.n_fold):\n",
    "            if fold in CFG.train_fold:\n",
    "                \n",
    "                valid_scores_fold_df = train_fold_loop (checkpoint_path)\n",
    "                # valid_scores_fold = np.array (valid_scores_fold).reshape ((1, -1))\n",
    "                valid_scores_df = valid_scores_df.append (valid_scores_fold_df)\n",
    "                \n",
    "        print (f\"========== CV ==========\")\n",
    "        # print (valid_scores_df)\n",
    "        # valid_scores = np.vstack (valid_scores)\n",
    "        # valid_scores = np.mean (valid_scores, axis=0)\n",
    "        valid_scores = valid_scores_df.iloc[-1, :]  #.mean ()\n",
    "        print (\"CV Scores :-\");  print (valid_scores)\n",
    "    return valid_scores_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "intended-opinion",
   "metadata": {
    "papermill": {
     "duration": 0.022612,
     "end_time": "2021-04-10T13:08:32.296494",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.273882",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Single config training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sharp-glory",
   "metadata": {
    "papermill": {
     "duration": 0.022705,
     "end_time": "2021-04-10T13:08:32.341955",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.319250",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "!mkdir -p /kaggle/working/Output/\n",
    "!touch /kaggle/working/Output/train.log\n",
    "gc.collect ()\n",
    "model_names = timm.list_models (pretrained=True)\n",
    "model_names = timm.list_models ('*resnet*', pretrained=True)\n",
    "pprint (model_names)\n",
    "LOGGER = init_logger ()\n",
    "seed_everything (seed=CFG.seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "failing-trust",
   "metadata": {
    "papermill": {
     "duration": 0.023148,
     "end_time": "2021-04-10T13:08:32.387650",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.364502",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "valid_scores_df = train_main ()\n",
    "\n",
    "valid_scores_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "injured-mainstream",
   "metadata": {
    "papermill": {
     "duration": 0.023122,
     "end_time": "2021-04-10T13:08:32.433851",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.410729",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# To train, uncomment these"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "harmful-product",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:08:32.486190Z",
     "iopub.status.busy": "2021-04-10T13:08:32.485323Z",
     "iopub.status.idle": "2021-04-10T13:08:33.234277Z",
     "shell.execute_reply": "2021-04-10T13:08:33.233837Z"
    },
    "papermill": {
     "duration": 0.777405,
     "end_time": "2021-04-10T13:08:33.234444",
     "exception": false,
     "start_time": "2021-04-10T13:08:32.457039",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../input/shopee-pytorch-siamese-triplet-loss-xlmrobe-cd1846/xlmroberta_256_fold0.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../input/shopee-pytorch-siamese-triplet-loss-xlmrobe-cd1846/xlmroberta_256_fold0.pth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "touched-letter",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:08:33.542944Z",
     "iopub.status.busy": "2021-04-10T13:08:33.542051Z",
     "iopub.status.idle": "2021-04-10T13:27:33.931967Z",
     "shell.execute_reply": "2021-04-10T13:27:33.932416Z"
    },
    "papermill": {
     "duration": 1140.674632,
     "end_time": "2021-04-10T13:27:33.932595",
     "exception": false,
     "start_time": "2021-04-10T13:08:33.257963",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Training, freeze=True *****\n",
      "========== training ==========\n",
      "======== Epoch 1 / 1 ========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 241/482 [07:02<07:01,  1.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved checkpoint ./xlm-roberta-large_128_fold0.pth\n",
      "Val Loss: 0.7172\n",
      "Val Accuracy: 0.9101\n",
      "Validation took 0:01:23 (h:mm:ss)\n",
      "saved checkpoint ./xlm-roberta-large_128_fold0_min_val_loss.pth\n",
      "saved checkpoint ./xlm-roberta-large_128_fold0_max_val_acc.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 482/482 [16:02<00:00,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Average training loss: 0.6425\n",
      "  Training epcoh took: 0:16:03\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val Loss: 0.7373\n",
      "Val Accuracy: 0.7031\n",
      "Validation took 0:01:24 (h:mm:ss)\n",
      "saved checkpoint ./xlm-roberta-large_128_fold0.pth\n",
      "At training end, threshold Adjustment (last row of the train summary DF)\n",
      "Val Loss: 0.7373\n",
      "Val Accuracy: 0.7031\n",
      "Validation took 0:01:24 (h:mm:ss)\n",
      "(0.7372929117193928, 0.703065693430657, '0:01:24')\n",
      "<: avg_val_loss, avg_val_accuracy, validation_time\n",
      "***** Training complete! *****\n",
      "Total training took 0:18:58 (h:mm:ss)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAAna0lEQVR4nO3deXxU9b3/8ddnlhB2QghBAQUVZBWQEIK4U5WqLVoLCbigLNGqtJda+6OtVxG5atVbtdaFRQQUCGhrr1qttYKlCwECsiMKuBC0EMImsmT7/v7IQIeYkAlMcpLJ+/l45JGZ8z1n5p3D4Z0z50zOmHMOERGJXT6vA4iISPVS0YuIxDgVvYhIjFPRi4jEOBW9iEiMC3gdoKxWrVq5Dh06eB1DRKROWbFixS7nXFJ5Y7Wu6Dt06EBOTo7XMURE6hQz+7yiMR26ERGJcSp6EZEYp6IXEYlxte4YvYjULoWFheTm5nL48GGvowgQHx9Pu3btCAaDES+joheRE8rNzaVp06Z06NABM/M6Tr3mnCM/P5/c3Fw6duwY8XI6dCMiJ3T48GESExNV8rWAmZGYmFjlV1cqehGplEq+9jiZf4uYKfriEscjb28kd89Br6OIiNQqMVP0X+w+yLxlX5A+JZsv8lX2IiJHxUzRd2zVmLlj0/imoIj0qUv4dNc3XkcSEY80adLE6wi1SswUPUCPts2ZOyaNI0UlpE9ZwuadB7yOJCL1WFFRkdcRgBh8e2W305uRlZnGiGlLyZi6hDlj0ji3TVOvY4nEhAffXM+GL/dH9TG7nd6MB77X/YTzTJgwgfbt23PXXXcBMHHiRAKBAIsWLWLPnj0UFhYyefJkhgwZUunzHThwgCFDhpS73OzZs3niiScwM8477zxefvllduzYwR133MHWrVsBeP755zn99NO59tprWbduHQBPPPEEBw4cYOLEiVx66aX07t2bf/zjHwwfPpzOnTszefJkCgoKSExMZM6cOSQnJ3PgwAHGjRtHTk4OZsYDDzzAvn37WLNmDU899RQA06ZNY8OGDTz55JMnu3qBCPfozWywmW0ys81mNqGc8TPN7H0zW2NmH5hZu7CxkWb2Sehr5CmljVDn5KZkZabhM2P4tOyob5giUrPS09NZsGDBsfsLFixg5MiRvP7666xcuZJFixZxzz33EMlnYMfHx5e73Pr165k8eTILFy5k9erVPP300wD8+Mc/5pJLLmH16tWsXLmS7t1P/EsJoKCggJycHO655x4uvPBCsrOz+fDDD8nIyOCxxx4D4KGHHqJ58+asXbuWNWvWcPnllzNs2DDefPNNCgsLAXjppZcYNWrUyayy41S6R29mfuBZ4AogF1huZm845zaEzfYEMNs5N8vMLgceAW42s5bAA0AK4IAVoWX3nHLySpzTugkLbh/AiGnZDJ+WzSuj+9OzXfPqflqRmFbZnnd16dOnDzt37uTLL78kLy+PhIQE2rRpw/jx41m8eDE+n4/t27ezY8cO2rRpc8LHcs7xy1/+8lvLLVy4kKFDh9KqVSsAWrZsCcDChQuZPXs2AH6/n+bNm7Nnz4krLD09/djt3Nxc0tPT+eqrrygoKDj2h05//etfycrKOjZfQkICAJdffjlvvfUWXbt2pbCwkJ49e1ZxbX1bJHv0qcBm59xW51wBkAWUfX3UDVgYur0obPwq4D3n3O5Qub8HDD7l1BHq0Kox828fQJMGAUZMz+bDL6r994uIVJOhQ4fy2muvMX/+fNLT05kzZw55eXmsWLGCVatWkZycHNEfEp3scuECgQAlJSXH7pddvnHjxsdujxs3jrvvvpu1a9cyZcqUSp9rzJgxzJw5k5deeonbbrutSrkqEknRtwW2hd3PDU0Ltxr4Qej29UBTM0uMcFnMLNPMcswsJy8vL9LsEWnfshEL7hhAQqM4bn5xGTmf7Y7q44tIzUhPTycrK4vXXnuNoUOHsm/fPlq3bk0wGGTRokV8/nmFl2M/TkXLXX755bz66qvk5+cDsHt3aVcMGjSI559/HoDi4mL27dtHcnIyO3fuJD8/nyNHjvDWW2+d8Pnati2tvVmzZh2bfsUVV/Dss88eu3/0VUL//v3Ztm0bc+fOZfjw4ZGunhOK1rtufgZcYmYfApcA24HiSBd2zk11zqU451KSksr9gJRT0rZFQxbcPoDWTRtwy4xlZG/Nj/pziEj16t69O19//TVt27bltNNO48YbbyQnJ4eePXsye/ZsunTpEtHjVLRc9+7d+dWvfsUll1xCr169+OlPfwrA008/zaJFi+jZsyd9+/Zlw4YNBINB7r//flJTU7niiitO+NwTJ05k6NCh9O3b99hhIYD77ruPPXv20KNHD3r16sWiRYuOjQ0bNoyBAwceO5xzqqyykxdmNgCY6Jy7KnT/FwDOuUcqmL8J8JFzrp2ZDQcudc7dHhqbAnzgnJtX0fOlpKS46vqEqZ37DzNi+lJy9xzkxZH9GHhOq8oXEqnnNm7cSNeuXb2OUa9ce+21jB8/nkGDBpU7Xt6/iZmtcM6llDd/JHv0y4FOZtbRzOKADOCNMk/QysyOPtYvgBmh2+8CV5pZgpklAFeGpnmidbN4sjLT6JDYmFEzl/PBpp1eRRER+Za9e/fSuXNnGjZsWGHJn4xK33XjnCsys7spLWg/MMM5t97MJgE5zrk3gEuBR8zMAYuBu0LL7jazhyj9ZQEwyTnn6UHyVk0aMHdsGjdNX0rm7BU8f9P5DOqa7GUkEakGa9eu5eabbz5uWoMGDVi6dKlHiSrXokULPv7446g/bqWHbmpadR66Cbf3YAG3zFjGxq/288zw8xnc48RvyRKpr3TopvapjkM3MalFozheGdOfnm2bc9fclby15kuvI4mIVIt6W/QAzeKDzB7dn/PPaMGP533IHz/c7nUkEZGoq9dFD9CkQYBZo1Lp3zGR8QtW8WrOtsoXEhGpQ+p90QM0igsw49Z+XHhOK+59bQ1zl37hdSQRkahR0Yc0jPMz7ZYULjs3iV++vpbZSz7zOpKIUPqWw+eee65Ky+h69MdT0YeJD/p54ea+fKdrMvf/33qm/32r15FE6r2Kir62XOu9Loi569GfqgYBP8/deD4/yfqQyX/aSFGJ445LzvY6lkjt8M4E+Pfa6D5mm57w3UcrHJ4wYQJbtmyhd+/eBINB4uPjSUhI4KOPPqr0PefOOX7+85/zzjvvYGbcd999x64kmZ6ezv79+ykqKuL555/nggsuYPTo0ceuDz9q1CjGjx/Pli1buOuuu8jLy6NRo0ZMmzaNLl268Oqrr/Lggw8eu6Ll4sWLo7teokhFX464gI9nhvdh/ILVPPrORxQWlTBuUCevY4nUS48++ijr1q1j1apVfPDBB1xzzTWsW7fu2OV+T+QPf/gDq1atYvXq1ezatYt+/fpx8cUXM3fuXK666ip+9atfUVxczMGDB1m1ahXbt28/9mEie/fuBSAzM5MXXniBTp06sXTpUu68804WLlzIpEmTePfdd2nbtu2xeWsrFX0FAn4fT6X3Jugz/ve9jyksLmH8FZ0xM6+jiXjnBHveNSU1NTWikgeOfcqT3+8nOTmZSy65hOXLl9OvXz9GjRpFYWEh1113Hb179+ass85i69atjBs3jmuuuYYrr7ySAwcO8K9//YuhQ4cee8wjR44AMHDgQG699VaGDRvGD37wg4oi1Ao6Rn8Cfp/x+NBepKe057cLN/PrP2+K6BNsRKT6hF/r/WRdfPHFLF68mLZt23Lrrbcye/ZsEhISWL16NZdeeikvvPACY8aMoaSkhBYtWrBq1apjXxs3bgTghRdeYPLkyWzbto2+ffseu7xxbaSir4TfZzzyg57c2P8MXvjbFib/aaPKXqQGNW3alK+//vqklr3ooouYP38+xcXF5OXlsXjxYlJTU/n8889JTk5m7NixjBkzhpUrV7Jr1y5KSkq44YYbmDx5MitXrqRZs2Z07NiRV199FSg95r969WoAtmzZQv/+/Zk0aRJJSUls21Z7/wZHh24i4PMZk6/rQdDv48V/fEphcQkTv9cdn0+HcUSqW2JiIgMHDqRHjx40bNiQ5OTIL0J4/fXXs2TJEnr16oWZ8dhjj9GmTRtmzZrF448/TjAYpEmTJsyePZvt27dz2223HfvkqEceKb0S+5w5c/jRj37E5MmTKSwsJCMjg169enHvvffyySef4Jxj0KBB9OrVq1p+/miotxc1OxnOOR5+eyPT/v4pw1PP4H+u66Gyl5ini5rVPlW9qJn26KvAzPjl1V0J+n0898EWCotL+PUN5+FX2YtILaairyIz496rziUu4OOpv35CUXEJTwztRcCv0x0iNSk/P7/cD+d4//33SUxM9CBR7aWiPwlmxn99pzNBv4/H391EUYnjyfTeBFX2EqOcc7XurcWJiYmsWrXK6xg17mQOt6voT8Fdl51D0G88/PZHFBaX8Mzw84kLqOwltsTHx5Ofn09iYmKtK/v6xjlHfn4+8fHxVVpORX+KMi8+m6Dfx4NvbuBHr6zguZvOp0HA73Uskahp164dubm55OXleR1FKP3F265duyoto6KPgtsGdiTg9/Hff1xH5uwVTLm5L/FBlb3EhmAwGPFfokrtpOMMUXJz2pn8+oaeLP4kj9GzlnOooNjrSCIigIo+qtL7ncETP+zFki353PrSMr45osuoioj3VPRRdkPfdjyZ3pucz/cwcsYyvj5c6HUkEannVPTVYEjvtjwzvA+rtu3l5heXse+Qyl5EvKOiryZX9zyN5248n/Vf7uOm6UvZe7DA60giUk+p6KvRld3bMPXmFDbt+Jrh05aSf+CI15FEpB5S0Vezy7q0ZvotKWzNO8Dwadnkfa2yF5GapaKvARd3TuKlW/uxbfchMqYuYcf+w15HEpF6JKKiN7PBZrbJzDab2YRyxs8ws0Vm9qGZrTGzq0PTO5jZITNbFfp6Ido/QF1xwTmtmDUqlX/vO0z6lCV8ufeQ15FEpJ6otOjNzA88C3wX6AYMN7NuZWa7D1jgnOsDZADPhY1tcc71Dn3dEaXcdVJqx5bMHp1K/oEC0qcuYdvug15HEpF6IJI9+lRgs3Nuq3OuAMgChpSZxwHNQrebA19GL2Js6XtmS14e0599BwvJmJrNF/kqexGpXpEUfVsg/MMQc0PTwk0EbjKzXOBtYFzYWMfQIZ2/mdlF5T2BmWWaWY6Z5dSHCyf1bt+CuWPT+KagiGFTlrA174DXkUQkhkXrZOxwYKZzrh1wNfCymfmAr4AzQod0fgrMNbNmZRd2zk11zqU451KSkpKiFKl269G2OfPGplFYXEL61Gw27zy5Dz8WEalMJEW/HWgfdr9daFq40cACAOfcEiAeaOWcO+Kcyw9NXwFsATqfauhY0fW0ZmRlpuEcpE/J5qN/7/c6kojEoEiKfjnQycw6mlkcpSdb3ygzzxfAIAAz60pp0eeZWVLoZC5mdhbQCdgarfCxoFNyU+bfnkbAbwyfms36L/d5HUlEYkylRe+cKwLuBt4FNlL67pr1ZjbJzL4fmu0eYKyZrQbmAbe60s+7uhhYY2argNeAO5xzu6vh56jTzk5qwvzMATQM+hkxbSlrcvd6HUlEYoidzOcPVqeUlBSXk5PjdQxPbNt9kOHTstl3sJBZo1M5/4wEryOJSB1hZiuccynljekvY2uR9i0bMf/2AbRsEsctLy5j+Wd68SMip05FX8u0bdGQ+ZkDaN20ASNnLGPJlnyvI4lIHaeir4XaNI8n6/Y02rZoyG0zl/H3T2L/bwtEpPqo6Gup1k3jycpMo0NiY0bPymHRpp1eRxKROkpFX4slNmnAvLFpdGrdhNtnr+C9DTu8jiQidZCKvpZLaBzH3DFpdD2tKT96ZQXvrP3K60giUseo6OuA5o2CvDymP+e1a87d8z7kzdW6ZpyIRE5FX0c0iw8ye3R/+p6RwE+yPuT1D3O9jiQidYSKvg5p0iDAzFH96N8xkZ8uWM2CnG2VLyQi9Z6Kvo5pFBdgxq39uPCcVvz8tTXMWfq515FEpJZT0ddBDeP8TLslhcu7tOZXr69j5j8/9TqSiNRiKvo6Kj7o54Wb+nJlt2QmvrmB6X/XRUFFpHwq+josLuDj2RvP55qepzH5Txt57oPNXkcSkVoo4HUAOTVBv4+nM3oT8BuP/XkThUWOn3ynk9exRKQWUdHHgIDfx2+G9cbvM57868cUlZTw0ys6Y2ZeRxORWkBFHyP8PuOJH/Yizu/jmYWbKSguYcLgLip7EVHRxxKfz3j4+p4E/MaUv22loKiE+6/tprIXqedU9DHG5zMeGtKDoN/HS//8jMLiEiZ9vwc+n8pepL5S0ccgM+P+a7sR5/cxZfFWioodD1/fU2UvUk+p6GOUmTHhu10I+n38btFmCosdj/3wPPwqe5F6R0Ufw8yMn111LkG/79i7cf53aC8Cfv35hEh9oqKvB37ynU4E/Mbj726iqNjxVEZvgip7kXpDRV9P3HXZOcT5ffzP2xspLC7hdyPOJy6gshepD/Q/vR4Ze/FZTPxeN/6yYQd3vLKCw4XFXkcSkRqgoq9nbh3YkcnX9WDhRzsZOztHZS9SD6jo66Gb0s7ksRvO4x+bdzFq5nIOFhR5HUlEqpGKvp4a1q89vxnWi+yt+dw6YzkHjqjsRWJVREVvZoPNbJOZbTazCeWMn2Fmi8zsQzNbY2ZXh439IrTcJjO7Kprh5dRc36cdT2X0YcUXe7jlxaXsP1zodSQRqQaVFr2Z+YFnge8C3YDhZtatzGz3AQucc32ADOC50LLdQve7A4OB50KPJ7XE93udzu+G92FN7j5ufnEZ+w6q7EViTSR79KnAZufcVudcAZAFDCkzjwOahW43B74M3R4CZDnnjjjnPgU2hx5PapHv9jyN52/qy4Yv93Hji9ns+abA60giEkWRFH1bYFvY/dzQtHATgZvMLBd4GxhXhWUxs0wzyzGznLy8vAijSzRd0S2ZqTen8PGOAwyflk3+gSNeRxKRKInWydjhwEznXDvgauBlM4v4sZ1zU51zKc65lKSkpChFkqq6rEtrXhyZwqe7viFjajY7vz7sdSQRiYJIyng70D7sfrvQtHCjgQUAzrklQDzQKsJlpRa5qFMSL93Wj9w9h8iYks2/96nsReq6SIp+OdDJzDqaWRylJ1ffKDPPF8AgADPrSmnR54XmyzCzBmbWEegELItWeKkeF5zdilmjUtmx/zDpU5ewfe8hryOJyCmotOidc0XA3cC7wEZK312z3swmmdn3Q7PdA4w1s9XAPOBWV2o9pXv6G4A/A3c55/SnmHVAaseWvDymP7sPFJA+ZQnbdh/0OpKInCRzznmd4TgpKSkuJyfH6xgSsiZ3LzdNX0qTBgHmZaZxZmJjryOJSDnMbIVzLqW8Mf1lrJzQee1aMHdsGocKixk2ZQlb8g54HUlEqkhFL5Xq0bY58zLTKCp2ZEzN5pMdX3sdSUSqQEUvEenSphlZmWkAZEzNZuNX+z1OJCKRUtFLxDolN2V+ZhpBv4/h07JZt32f15FEJAIqeqmSs5KaMP/2NBrHBRgxLZvV2/Z6HUlEKqGilyo7M7ExWZlpNG8U5KbpS1nx+R6vI4nICajo5aS0b9mI+ZkDSGwSxy0vLmXZp7u9jiQiFVDRy0k7vUVD5t8+gOTm8YycsYx/bdnldSQRKYeKXk5JcrN45mcOoH3Lhtz20nIWf6yrj4rUNip6OWVJTRswb2waHVs1ZszsHBZ9tNPrSCISRkUvUZHYpLTsOyc3IfPlHP6y/t9eRxKREBW9RE1C4zjmjEmj2+nNuXPOSt5e+5XXkUQEFb1EWfOGQV4enUqv9i0YN+9D/m+VPn5AxGsqeom6ZvFBZo1Kpe+ZCYyfv4rfr8j1OpJIvaail2rRpEGAmbf1Y8DZifzstdXMX/6F15FE6i0VvVSbRnEBXhzZj4s6JfH/fr+WV7I/9zqSSL2kopdqFR/0M/Xmvgzq0pr7/riOl/75qdeRROodFb1Uu/ign+dv6stV3ZN58M0NTF28xetIIvWKil5qRFzAx+9GnM81PU/j4bc/4tlFm72OJFJvBLwOIPVH0O/j6YzeBPzG4+9uoqCohP/6TifMzOtoIjFNRS81KuD38ZthvUtL//1PKCop4WdXnquyF6lGKnqpcX6f8dgN5xH0G88u2kJhseMX3+2ishepJip68YTPZ/zPdT0J+n1MXbyVgqISHvheN5W9SDVQ0YtnfD7jwe93J+DzMeOfn1JUUsKk7/fA51PZi0STil48ZWb897VdCQaMKX/bSmGR4+Ef9MSvsheJGhW9eM7MmDC4C3F+H88s3ExhcQmPD+2lsheJEhW91Apmxj1XnkvQ7+M3731MYYnjyWG9CPj1px4ipyqiojezwcDTgB+Y7px7tMz4k8BlobuNgNbOuRahsWJgbWjsC+fc96OQW2LUjwd1IuA3HvvzJoqKS/jt8D4EVfYip6TSojczP/AscAWQCyw3szeccxuOzuOcGx82/zigT9hDHHLO9Y5aYol5d156DnF+H5P/tJE756zkdyP60CDg9zqWSJ0Vya5SKrDZObfVOVcAZAFDTjD/cGBeNMJJ/TXmorOYNKQ7723YwR0vr+BwYbHXkUTqrEiKvi2wLex+bmjat5jZmUBHYGHY5HgzyzGzbDO7roLlMkPz5OTl5UWWXGLeLQM68PD1PVm0KY+xs3M4VKCyFzkZ0T74mQG85pwL/x95pnMuBRgBPGVmZ5ddyDk31TmX4pxLSUpKinIkqctG9D+Dx354Hv/YvItRM5dzsKDI60gidU4kRb8daB92v11oWnkyKHPYxjm3PfR9K/ABxx+/F6nUsJT2/GZYL5Z+ms/IGcs4cERlL1IVkRT9cqCTmXU0szhKy/yNsjOZWRcgAVgSNi3BzBqEbrcCBgIbyi4rUpnr+7Tj6Yw+rPxiLze/uJT9hwu9jiRSZ1Ra9M65IuBu4F1gI7DAObfezCaZWfhbJTOALOecC5vWFcgxs9XAIuDR8HfriFTF93qdzrMj+rA2dx83TV/KvoMqe5FI2PG97L2UlBSXk5PjdQypxf66YQd3zlnJOa2b8MqY/rRsHOd1JBHPmdmK0PnQb9Ffokid851uyUy9pS+b8w4wYlo2uw4c8TqSSK2mopc66dJzWzNjZD8+y/+GjKnZ7Nx/2OtIIrWWil7qrAs7tWLmbal8ufcQGVOz+fc+lb1IeVT0UqelnZXI7FGp7Pz6CMOmLCF3z0GvI4nUOip6qfNSOrRk9uhU9hwsIH1KNtt2q+xFwqnoJSacf0YCc8b058CRIoZNWcJnu77xOpJIraGil5hxXrsWzB3bn8OFxQybsoTNOw94HUmkVlDRS0zpfnpzsjIHUOIcGVOz+XjH115HEvGcil5izrltmpKVmYbPIGNqNhu+3O91JBFPqeglJp3Tuinzbx9Ag4CPEdOzWbd9n9eRRDyjopeY1bFVY+ZnDqBxXIAR07JZtW2v15FEPKGil5h2RmIj5t+eRvNGQW6avpScz3Z7HUmkxqnoJea1S2jEgtsHkNS0AbfMWMbSrfleRxKpUSp6qRdOa96QrMw0Tmsez8iXlvHPzbu8jiRSY1T0Um8kN4snK3MAZ7RsxKiZy/nbx/p8YqkfVPRSryQ1bcC8sWmcldSEsbNyeH/jDq8jiVQ7Fb3UO4lNGjBvbH/ObdOUO15Zwbvr/+11JJFqpaKXeqlFozheGdOf7qc35645K/nTmq+8jiRSbVT0Um81bxjk5dGp9G7fgnHzVvLHD7d7HUmkWqjopV5rGh9k1qhU+nVoyfgFq3htRa7XkUSiTkUv9V7jBgFm3pbKwLNbce9rq5m37AuvI4lElYpeBGgY52f6yBQu7pTEL/6wlpeXfOZ1JJGoUdGLhMQH/Uy9pS/f6dqa//6/9bz4j0+9jiQSFSp6kTANAn6eu7Evg7u34aG3NjDlb1u8jiRyylT0ImXEBXw8M6IP1553Go+88xHPvP+J15FETknA6wAitVHQ7+Op9N4E/T7+972PKSwuYfwVnTEzr6OJVJmKXqQCAb+PJ4b2IuAzfrtwM4Uljp9fda7KXuqciIrezAYDTwN+YLpz7tEy408Cl4XuNgJaO+dahMZGAveFxiY752ZFIbdIjfD7jF/fcB4Bv4/nP9jC71fk4vep6KV6dD+9GdNH9ov641Za9GbmB54FrgBygeVm9oZzbsPReZxz48PmHwf0Cd1uCTwApAAOWBFadk9UfwqRauTzGQ9f34Ozkxrrw8alWp3RslG1PG4ke/SpwGbn3FYAM8sChgAbKph/OKXlDnAV8J5zbndo2feAwcC8UwktUtPMjDEXneV1DJGTEsm7btoC28Lu54amfYuZnQl0BBZWZVkzyzSzHDPLycvTNcJFRKIp2m+vzABec84VV2Uh59xU51yKcy4lKSkpypFEROq3SIp+O9A+7H670LTyZHD8YZmqLCsiItUgkqJfDnQys45mFkdpmb9RdiYz6wIkAEvCJr8LXGlmCWaWAFwZmiYiIjWk0pOxzrkiM7ub0oL2AzOcc+vNbBKQ45w7WvoZQJZzzoUtu9vMHqL0lwXApKMnZkVEpGZYWC/XCikpKS4nJ8frGCIidYqZrXDOpZQ3pmvdiIjEOBW9iEiMU9GLiMQ4Fb2ISIxT0YuIxDgVvYhIjFPRi4jEOBW9iEiMU9GLiMQ4Fb2ISIxT0YuIxDgVvYhIjFPRi4jEOBW9iEiMU9GLiMQ4Fb2ISIxT0YuIxDgVvYhIjFPRi4jEOBW9iEiMU9GLiMQ4Fb2ISIxT0YuIxDgVvYhIjFPRi4jEOBW9iEiMU9GLiMS4iIrezAab2SYz22xmEyqYZ5iZbTCz9WY2N2x6sZmtCn29Ea3gIiISmUBlM5iZH3gWuALIBZab2RvOuQ1h83QCfgEMdM7tMbPWYQ9xyDnXO7qxRUQkUpHs0acCm51zW51zBUAWMKTMPGOBZ51zewCcczujG1NERE5WJEXfFtgWdj83NC1cZ6Czmf3TzLLNbHDYWLyZ5YSmX1feE5hZZmienLy8vKrkFxGRSlR66KYKj9MJuBRoByw2s57Oub3Amc657WZ2FrDQzNY657aEL+ycmwpMBUhJSXFRyiQiIkS2R78daB92v11oWrhc4A3nXKFz7lPgY0qLH+fc9tD3rcAHQJ9TzCwiIlUQSdEvBzqZWUcziwMygLLvnvkjpXvzmFkrSg/lbDWzBDNrEDZ9ILABERGpMZUeunHOFZnZ3cC7gB+Y4Zxbb2aTgBzn3BuhsSvNbANQDNzrnMs3swuAKWZWQukvlUfD360j4rniQig8VPpVdOg/t4/dP3z8mCvxOrHEsiatoccNUX9Yc652HRJPSUlxOTk5XscQrzgHxQWhcj0MhQe/XbbfGjt4/P1j84WPlS3z0Jgr9vonFvmPtikw9v2TWtTMVjjnUsobi9bJWIllzkHRkXJKs7yyrWjvuIKx8OlHx052rznQEILxoe9hX4F4aJJcwdjR2ycaCz1GsCH4/NFdtyLhrHq2LxV9XVVSUlqSkZZtxHvHFRQxJ/PKz8qUZvzxBdowocxYo/8U6reWOzrWqPx5A/FgFu21LBITVPTRVFJSwXHeSPaAqzhWdPjkMprv26V59HZcI2jcqvyx8sq2vD3n8LFAA5WvSC0Q+0VfXBSlso3guG9xwcll9AWOP4QQvrca3wwCyeWPlT28UNGec/iYP6jyFalnYqfov8mHmVd/+7hvSeHJPZ4/ruLSPHrIocJjuSc4vFDemD8Y3XUhIhImdoo+EAetOkV2LLfCsbDC1kk3EYkRsVP0DZpC+itepxARqXX0wSMiIjFORS8iEuNU9CIiMU5FLyIS41T0IiIxTkUvIhLjVPQiIjFORS8iEuNq3fXozSwP+PwUHqIVsCtKcaJJuapGuapGuaomFnOd6ZxLKm+g1hX9qTKznIouvu8l5aoa5aoa5aqa+pZLh25ERGKcil5EJMbFYtFP9TpABZSrapSrapSraupVrpg7Ri8iIseLxT16EREJo6IXEYlxdabozWyGme00s3UVjJuZ/dbMNpvZGjM7P2xspJl9EvoaWcO5bgzlWWtm/zKzXmFjn4WmrzKznBrOdamZ7Qs99yozuz9sbLCZbQqtywk1nOvesEzrzKzYzFqGxqpzfbU3s0VmtsHM1pvZT8qZp0a3sQgzebV9RZKtxrexCHPV+DZmZvFmtszMVodyPVjOPA3MbH5onSw1sw5hY78ITd9kZldVOYBzrk58ARcD5wPrKhi/GngHMCANWBqa3hLYGvqeELqdUIO5Ljj6fMB3j+YK3f8MaOXR+roUeKuc6X5gC3AWEAesBrrVVK4y834PWFhD6+s04PzQ7abAx2V/7prexiLM5NX2FUm2Gt/GIsnlxTYW2maahG4HgaVAWpl57gReCN3OAOaHbncLraMGQMfQuvNX5fnrzB69c24xsPsEswwBZrtS2UALMzsNuAp4zzm32zm3B3gPGFxTuZxz/wo9L0A20C5az30quU4gFdjsnNvqnCsAsihdt17kGg7Mi9Zzn4hz7ivn3MrQ7a+BjUDbMrPV6DYWSSYPt69I1ldFqm0bO4lcNbKNhbaZA6G7wdBX2XfCDAFmhW6/BgwyMwtNz3LOHXHOfQpspnQdRqzOFH0E2gLbwu7nhqZVNN0LoyndIzzKAX8xsxVmlulBngGhl5LvmFn30LRasb7MrBGlZfn7sMk1sr5CL5n7ULrXFc6zbewEmcJ5sn1Vks2zbayydVbT25iZ+c1sFbCT0h2DCrcv51wRsA9IJArrK3Y+HLyWM7PLKP2PeGHY5Audc9vNrDXwnpl9FNrjrQkrKb02xgEzuxr4I9Cphp47Et8D/umcC9/7r/b1ZWZNKP2P/1/Ouf3RfOyTFUkmr7avSrJ5to1F+O9Yo9uYc64Y6G1mLYDXzayHc67cc1XRFkt79NuB9mH324WmVTS9xpjZecB0YIhzLv/odOfc9tD3ncDrVPHl2Klwzu0/+lLSOfc2EDSzVtSC9RWSQZmX1NW9vswsSGk5zHHO/aGcWWp8G4sgk2fbV2XZvNrGIllnITW+jYUeey+wiG8f3ju2XswsADQH8onG+or2SYfq/AI6UPHJxWs4/kTZstD0lsCnlJ4kSwjdblmDuc6g9JjaBWWmNwaaht3+FzC4BnO14T9/MJcKfBFadwFKTyZ25D8nyrrXVK7QeHNKj+M3rqn1FfrZZwNPnWCeGt3GIszkyfYVYbYa38YiyeXFNgYkAS1CtxsCfweuLTPPXRx/MnZB6HZ3jj8Zu5UqnoytM4duzGwepWfxW5lZLvAApSc0cM69ALxN6bsiNgMHgdtCY7vN7CFgeeihJrnjX6pVd677KT3O9lzpeRWKXOnV6ZIpffkGpRv+XOfcn2sw1w+BH5lZEXAIyHClW1WRmd0NvEvpuyNmOOfW12AugOuBvzjnvglbtFrXFzAQuBlYGzqOCvBLSovUq20skkyebF8RZvNiG4skF9T8NnYaMMvM/JQeSVngnHvLzCYBOc65N4AXgZfNbDOlv4QyQpnXm9kCYANQBNzlSg8DRUyXQBARiXGxdIxeRETKoaIXEYlxKnoRkRinohcRiXEqehGRGKeiFxGJcSp6EZEY9/8B+VQ+O+BvHZQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracies : [0.9100729927007299, 0.703065693430657, 0.703065693430657]\n",
      "eval_losses: [0.7172352180436805, 0.7372929117193928, 0.7372929117193928]\n",
      "tr_losses  : [0.6347438450670836, 0.6425418000498253, 0.6425418000498253]\n",
      "========== CV ==========\n",
      "CV Scores :-\n",
      "epoch                   1\n",
      "training_loss    0.642542\n",
      "eval_loss        0.737293\n",
      "eval_accuracy    0.703066\n",
      "training_time     0:16:03\n",
      "eval_time         0:01:24\n",
      "Name: 2, dtype: object\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>training_loss</th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_accuracy</th>\n",
       "      <th>training_time</th>\n",
       "      <th>eval_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.634744</td>\n",
       "      <td>0.717235</td>\n",
       "      <td>0.910073</td>\n",
       "      <td>0:07:11</td>\n",
       "      <td>0:01:23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.642542</td>\n",
       "      <td>0.737293</td>\n",
       "      <td>0.703066</td>\n",
       "      <td>0:16:03</td>\n",
       "      <td>0:01:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.642542</td>\n",
       "      <td>0.737293</td>\n",
       "      <td>0.703066</td>\n",
       "      <td>0:16:03</td>\n",
       "      <td>0:01:24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  training_loss  eval_loss  eval_accuracy training_time eval_time\n",
       "0      1       0.634744   0.717235       0.910073       0:07:11   0:01:23\n",
       "1      1       0.642542   0.737293       0.703066       0:16:03   0:01:24\n",
       "2      1       0.642542   0.737293       0.703066       0:16:03   0:01:24"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect (); torch.cuda.empty_cache ()\n",
    "# CFG.warmup_steps = 0.5\n",
    "# CFG.eval_steps   = 0\n",
    "# CFG.num_workers  = 8\n",
    "# CFG.train_batch_size = 32\n",
    "# CFG.eval_batch_size  = 32\n",
    "# CFG.freeze = True\n",
    "# CFG.epochs = 1\n",
    "# CFG.lr     = 5e-5\n",
    "print (f\"***** Training, freeze={CFG.isFreeze} *****\")\n",
    "valid_scores_df = train_main () \n",
    "valid_scores_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "metropolitan-nigeria",
   "metadata": {
    "papermill": {
     "duration": 0.149543,
     "end_time": "2021-04-10T13:27:34.238301",
     "exception": false,
     "start_time": "2021-04-10T13:27:34.088758",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "gc.collect (); torch.cuda.empty_cache ()\n",
    "CFG.warmup_steps = 0.5\n",
    "CFG.eval_steps   = 0.5\n",
    "CFG.num_workers = 8\n",
    "CFG.train_batch_size = 1\n",
    "CFG.eval_batch_size  = 1\n",
    "CFG.isFreeze = False\n",
    "CFG.epochs = 2\n",
    "print (f\"***** Training, freeze={CFG.isFreeze} *****\")\n",
    "valid_scores_df = train_main (\"../input/shopee-pytorch-siamese-triplet-loss-xlmrobe-cd1846/xlmroberta_256_fold0.pth\")\n",
    "valid_scores_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dependent-grain",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-10T13:27:34.540203Z",
     "iopub.status.busy": "2021-04-10T13:27:34.539609Z",
     "iopub.status.idle": "2021-04-10T13:27:34.544076Z",
     "shell.execute_reply": "2021-04-10T13:27:34.543360Z"
    },
    "papermill": {
     "duration": 0.157294,
     "end_time": "2021-04-10T13:27:34.544193",
     "exception": false,
     "start_time": "2021-04-10T13:27:34.386899",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done !\n"
     ]
    }
   ],
   "source": [
    "print ('Done !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "practical-singapore",
   "metadata": {
    "papermill": {
     "duration": 0.15473,
     "end_time": "2021-04-10T13:27:34.851413",
     "exception": false,
     "start_time": "2021-04-10T13:27:34.696683",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1224.96961,
   "end_time": "2021-04-10T13:27:37.415565",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-04-10T13:07:12.445955",
   "version": "2.3.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}