notebooks/Sequence Regression Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adapted from [this example script](https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/examples/run_classifier.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Local network environment settings\n",
    "import os\n",
    "os.environ[\"http_proxy\"] = \"127.0.0.1:11233\"\n",
    "os.environ[\"https_proxy\"] = \"127.0.0.1:11233\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logger settings and Constants: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "                    datefmt = '%m/%d/%Y %H:%M:%S',\n",
    "                    level = logging.INFO)\n",
    "logger = logging.getLogger(\"regressor\")\n",
    "\n",
    "FP16 = False\n",
    "BATCH_SIZE = 32\n",
    "SEED = 42\n",
    "WARMUP_PROPORTION = 0.1\n",
    "PYTORCH_PRETRAINED_BERT_CACHE = \"/mnt/Intel/bert_tmp\"\n",
    "LOSS_SCALE = 0. # Loss scaling to improve fp16 numeric stability. Only used when fp16 set to True.\n",
    "MAX_SEQ_LENGTH = 100\n",
    "\n",
    "DATA_PATH = \"douban_ratings.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from pytorch_pretrained_bert.modeling import PreTrainedBertModel, BertModel\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "from pytorch_pretrained_bert.optimization import BertAdam, warmup_linear, SCHEDULES\n",
    "from fastprogress import master_bar, progress_bar\n",
    "from sklearn.model_selection import StratifiedShuffleSplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 12:44:10 - INFO - regressor -   device: cuda n_gpu: 1, 16-bits training: False\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "logger.info(\"device: {} n_gpu: {}, 16-bits training: {}\".format(\n",
    "    device, n_gpu, FP16))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Set random seeds:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "if n_gpu > 0:\n",
    "    torch.cuda.manual_seed_all(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Definitions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regression model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertForSequenceRegression(PreTrainedBertModel):\n",
    "    def __init__(self, config):\n",
    "        super(BertForSequenceRegression, self).__init__(config)\n",
    "        self.bert = BertModel(config)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "        self.regressor = nn.Linear(config.hidden_size, 1)\n",
    "        self.apply(self.init_bert_weights)\n",
    "        self.loss_fct = torch.nn.MSELoss()\n",
    "\n",
    "    def forward(self, input_ids, token_type_ids=None, attention_mask=None, targets=None):\n",
    "        _, pooled_output = self.bert(input_ids, token_type_ids, attention_mask, output_all_encoded_layers=False)\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        outputs = self.regressor(pooled_output).clamp(-1, 1)\n",
    "        if targets is not None:\n",
    "            loss = self.loss_fct(outputs.view(-1), targets.view(-1))\n",
    "            return loss\n",
    "        else:\n",
    "            return outputs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Classes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InputExample(object):\n",
    "    \"\"\"A single training/test example for simple sequence classification.\"\"\"\n",
    "\n",
    "    def __init__(self, guid, text, target=None):\n",
    "        self.guid = guid\n",
    "        self.text = text\n",
    "        self.target = target\n",
    "\n",
    "\n",
    "class InputFeatures(object):\n",
    "    \"\"\"A single set of features of data.\"\"\"\n",
    "\n",
    "    def __init__(self, input_ids, input_mask, segment_ids, target):\n",
    "        self.input_ids = input_ids\n",
    "        self.input_mask = input_mask\n",
    "        self.segment_ids = segment_ids\n",
    "        self.target = target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Processing Class and Function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DoubanRatingProcessor:\n",
    "    \"\"\"Processor for the Douban movie ratings data set.\"\"\"\n",
    "    def __init__(self, sample_ratio: float = 0.05):\n",
    "        df_ratings = self.filter_entries(pd.read_csv(DATA_PATH)).sample(frac=sample_ratio)\n",
    "        df_ratings[\"rating\"] = ((df_ratings[\"rating\"] - 3) / 2).astype(\"float32\")\n",
    "        assert df_ratings.rating.max() <= 1\n",
    "        assert df_ratings.rating.min() >= -1\n",
    "        assert df_ratings.isnull().sum().sum() == 0\n",
    "        texts = df_ratings[\"comment\"].values\n",
    "        # Split the dataset\n",
    "        sss = StratifiedShuffleSplit(n_splits=1, test_size=0.4, random_state=888)\n",
    "        train_idx, test_idx = next(sss.split(df_ratings, df_ratings.rating))\n",
    "        texts_train, texts_test = texts[train_idx], texts[test_idx]\n",
    "        y_train = df_ratings.iloc[train_idx][[\"rating\"]].copy().values\n",
    "        y_test = df_ratings.iloc[test_idx][[\"rating\"]].copy().values\n",
    "        sss = StratifiedShuffleSplit(n_splits=1, test_size=0.5, random_state=888)\n",
    "        val_idx, test_idx = next(sss.split(y_test, y_test))\n",
    "        texts_valid, texts_test = texts_test[val_idx], texts_test[test_idx]\n",
    "        y_valid, y_test = y_test[val_idx], y_test[test_idx]\n",
    "        self.x_train, self.x_valid, self.x_test = (\n",
    "            texts_train, texts_valid, texts_test)\n",
    "        self.y_train, self.y_valid, self.y_test = (\n",
    "            y_train, y_valid, y_test)\n",
    "\n",
    "    @classmethod\n",
    "    def filter_entries(cls, df_ratings, min_len=3, max_len=1000):\n",
    "        lengths = df_ratings.comment.str.len()\n",
    "        flags = (lengths >= min_len) & (lengths <= max_len)\n",
    "        assert flags.isnull().sum() == 0\n",
    "        return df_ratings.loc[flags].copy()\n",
    "        \n",
    "    def get_train_examples(self):\n",
    "        return self._create_examples(self.x_train, self.y_train)\n",
    "\n",
    "    def get_dev_examples(self):\n",
    "        return self._create_examples(self.x_valid, self.y_valid)\n",
    "\n",
    "    def get_test_examples(self):\n",
    "        return self._create_examples(self.x_test, self.y_test)\n",
    "    \n",
    "    def _create_examples(self, x, y):\n",
    "        \"\"\"Creates examples for the training and dev sets.\"\"\"\n",
    "        examples = []\n",
    "        for (i, (texts, target)) in enumerate(zip(x, y)):\n",
    "            examples.append(\n",
    "                InputExample(guid=i, text=texts, target=target))\n",
    "        return examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_examples_to_features(examples, max_seq_length, tokenizer):\n",
    "    \"\"\"Loads a data file into a list of `InputBatch`s.\"\"\"\n",
    "    \n",
    "    features = []\n",
    "    for (ex_index, example) in enumerate(examples):\n",
    "        tokens = tokenizer.tokenize(example.text)\n",
    "        \n",
    "        if len(tokens) > max_seq_length - 2:\n",
    "            tokens = tokens[:(max_seq_length - 2)]\n",
    "\n",
    "        # The convention in BERT is:\n",
    "        # (a) For sequence pairs:\n",
    "        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]\n",
    "        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1\n",
    "        # (b) For single sequences:\n",
    "        #  tokens:   [CLS] the dog is hairy . [SEP]\n",
    "        #  type_ids: 0   0   0   0  0     0 0\n",
    "        #\n",
    "        # Where \"type_ids\" are used to indicate whether this is the first\n",
    "        # sequence or the second sequence. The embedding vectors for `type=0` and\n",
    "        # `type=1` were learned during pre-training and are added to the wordpiece\n",
    "        # embedding vector (and position vector). This is not *strictly* necessary\n",
    "        # since the [SEP] token unambigiously separates the sequences, but it makes\n",
    "        # it easier for the model to learn the concept of sequences.\n",
    "        #\n",
    "        # For classification tasks, the first vector (corresponding to [CLS]) is\n",
    "        # used as as the \"sentence vector\". Note that this only makes sense because\n",
    "        # the entire model is fine-tuned.\n",
    "        tokens = [\"[CLS]\"] + tokens + [\"[SEP]\"]\n",
    "        segment_ids = [0] * len(tokens)\n",
    "\n",
    "        input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "        # The mask has 1 for real tokens and 0 for padding tokens. Only real\n",
    "        # tokens are attended to.\n",
    "        input_mask = [1] * len(input_ids)\n",
    "\n",
    "        # Zero-pad up to the sequence length.\n",
    "        padding = [0] * (max_seq_length - len(input_ids))\n",
    "        input_ids += padding\n",
    "        input_mask += padding\n",
    "        segment_ids += padding\n",
    "\n",
    "        assert len(input_ids) == max_seq_length\n",
    "        assert len(input_mask) == max_seq_length\n",
    "        assert len(segment_ids) == max_seq_length\n",
    "\n",
    "        if ex_index < 5:\n",
    "            logger.info(\"*** Example ***\")\n",
    "            logger.info(\"guid: %s\" % (example.guid))\n",
    "            logger.info(\"tokens: %s\" % \" \".join(\n",
    "                    [str(x) for x in tokens]))\n",
    "            logger.info(\"input_ids: %s\" % \" \".join([str(x) for x in input_ids]))\n",
    "            logger.info(\"input_mask: %s\" % \" \".join([str(x) for x in input_mask]))\n",
    "            logger.info(\n",
    "                    \"segment_ids: %s\" % \" \".join([str(x) for x in segment_ids]))\n",
    "            logger.info(\"target: %s\" % (example.target))\n",
    "\n",
    "        features.append(\n",
    "                InputFeatures(input_ids=input_ids,\n",
    "                              input_mask=input_mask,\n",
    "                              segment_ids=segment_ids,\n",
    "                              target=example.target))\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FreezableBertAdam(BertAdam):\n",
    "    def get_lr(self):\n",
    "        lr = []\n",
    "        for group in self.param_groups:\n",
    "            for p in group['params']:\n",
    "                state = self.state[p]\n",
    "                if len(state) == 0:\n",
    "                    continue\n",
    "                if group['t_total'] != -1:\n",
    "                    schedule_fct = SCHEDULES[group['schedule']]\n",
    "                    lr_scheduled = group['lr'] * schedule_fct(state['step']/group['t_total'], group['warmup'])\n",
    "                else:\n",
    "                    lr_scheduled = group['lr']\n",
    "                lr.append(lr_scheduled)\n",
    "        return lr    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Utility functions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def children(m):\n",
    "    return m if isinstance(m, (list, tuple)) else list(m.children())\n",
    "\n",
    "\n",
    "def set_trainable_attr(m, b):\n",
    "    m.trainable = b\n",
    "    for p in m.parameters():\n",
    "        p.requires_grad = b\n",
    "\n",
    "\n",
    "def apply_leaf(m, f):\n",
    "    c = children(m)\n",
    "    if isinstance(m, nn.Module):\n",
    "        f(m)\n",
    "    if len(c) > 0:\n",
    "        for l in c:\n",
    "            apply_leaf(l, f)\n",
    "\n",
    "\n",
    "def set_trainable(l, b):\n",
    "    apply_leaf(l, lambda m: set_trainable_attr(m, b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_model_parameters(model):\n",
    "    logger.info(\n",
    "        \"# of paramters: {:,d}\".format(\n",
    "            sum(p.numel() for p in model.parameters())))\n",
    "    logger.info(\n",
    "        \"# of trainable paramters: {:,d}\".format(\n",
    "            sum(p.numel() for p in model.parameters() if p.requires_grad)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 12:44:12 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt from cache at /mnt/Intel/bert_tmp/8a0c070123c1f794c42a29c6904beb7c1b8715741e235bee04aca2c7636fc83f.9b42061518a39ca00b8b52059fd2bede8daa613f8a8671500e518a8c29de8c00\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\n",
    "    \"bert-base-chinese\", do_lower_case=True, \n",
    "    cache_dir=PYTORCH_PRETRAINED_BERT_CACHE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_examples = DoubanRatingProcessor().get_train_examples()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 12:44:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 12:44:16 - INFO - regressor -   guid: 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   tokens: [CLS] 影 片 只 体 现 了 他 们 共 同 跨 越 赛 场 上 的 障 碍 ， 生 活 中 的 镜 头 几 乎 全 给 了 pierre 。 去 戏 剧 化 和 小 细 节 跟 体 育 类 型 有 点 不 兼 容 。 两 小 时 的 片 长 剧 情 和 节 奏 安 排 得 也 不 妥 帖 。 [SEP]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_ids: 101 2512 4275 1372 860 4385 749 800 812 1066 1398 6659 6632 6612 1767 677 4638 7397 4809 8024 4495 3833 704 4638 7262 1928 1126 725 1059 5314 749 8744 511 1343 2767 1196 1265 1469 2207 5301 5688 6656 860 5509 5102 1798 3300 4157 679 1076 2159 511 697 2207 3198 4638 4275 7270 1196 2658 1469 5688 1941 2128 2961 2533 738 679 1980 2365 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   target: [-0.5]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 12:44:16 - INFO - regressor -   guid: 1\n",
      "02/09/2019 12:44:16 - INFO - regressor -   tokens: [CLS] 乔 恩 的 演 技 没 的 说 哦 [SEP]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_ids: 101 730 2617 4638 4028 2825 3766 4638 6432 1521 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   target: [0.5]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 12:44:16 - INFO - regressor -   guid: 2\n",
      "02/09/2019 12:44:16 - INFO - regressor -   tokens: [CLS] 片 里 有 两 个 英 雄 救 美 的 情 节 ， 第 一 个 让 电 影 步 入 俗 套 ， 第 二 个 让 电 影 起 死 回 生 . . . . 不 过 不 难 看 ， 最 喜 欢 三 个 男 主 唯 一 同 框 的 那 场 戏 ， 特 别 是 罗 素 · 克 劳 , 简 直 飚 了 ！ 斯 特 朗 先 生 苏 出 新 高 度 ！ ！ 说 真 的 ， 躺 在 砧 板 上 胖 头 鱼 [SEP]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_ids: 101 4275 7027 3300 697 702 5739 7413 3131 5401 4638 2658 5688 8024 5018 671 702 6375 4510 2512 3635 1057 921 1947 8024 5018 753 702 6375 4510 2512 6629 3647 1726 4495 119 119 119 119 679 6814 679 7410 4692 8024 3297 1599 3614 676 702 4511 712 1546 671 1398 3427 4638 6929 1767 2767 8024 4294 1166 3221 5384 5162 185 1046 1227 117 5042 4684 7605 749 8013 3172 4294 3306 1044 4495 5722 1139 3173 7770 2428 8013 8013 6432 4696 4638 8024 6720 1762 4784 3352 677 5523 1928 7824 102\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      "02/09/2019 12:44:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   target: [0.5]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 12:44:16 - INFO - regressor -   guid: 3\n",
      "02/09/2019 12:44:16 - INFO - regressor -   tokens: [CLS] 讲 的 有 点 快 。 。 。 而 且 深 度 不 太 够 。 讲 的 慢 些 细 些 会 更 好 吧 。 [SEP]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_ids: 101 6382 4638 3300 4157 2571 511 511 511 5445 684 3918 2428 679 1922 1916 511 6382 4638 2714 763 5301 763 833 3291 1962 1416 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   target: [0.]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 12:44:16 - INFO - regressor -   guid: 4\n",
      "02/09/2019 12:44:16 - INFO - regressor -   tokens: [CLS] 政 治 倾 向 太 明 显 ， 这 种 非 黑 即 白 的 设 定 一 旦 不 能 接 受 ， 整 个 观 影 过 程 感 觉 在 听 某 些 传 销 的 讲 课 。 [SEP]\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_ids: 101 3124 3780 967 1403 1922 3209 3227 8024 6821 4905 7478 7946 1315 4635 4638 6392 2137 671 3190 679 5543 2970 1358 8024 3146 702 6225 2512 6814 4923 2697 6230 1762 1420 3378 763 837 7218 4638 6382 6440 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 12:44:16 - INFO - regressor -   target: [0.5]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features = convert_examples_to_features(\n",
    "    train_examples, MAX_SEQ_LENGTH, tokenizer)\n",
    "del train_examples\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 12:44:31 - INFO - pytorch_pretrained_bert.modeling -   loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese.tar.gz from cache at /mnt/Intel/bert_tmp/42d4a64dda3243ffeca7ec268d5544122e67d9d06b971608796b483925716512.02ac7d664cff08d793eb00d6aac1d04368a1322435e5fe0a27c70b0b3a85327f\n",
      "02/09/2019 12:44:31 - INFO - pytorch_pretrained_bert.modeling -   extracting archive file /mnt/Intel/bert_tmp/42d4a64dda3243ffeca7ec268d5544122e67d9d06b971608796b483925716512.02ac7d664cff08d793eb00d6aac1d04368a1322435e5fe0a27c70b0b3a85327f to temp dir /tmp/tmpw3rlljf2\n",
      "02/09/2019 12:44:33 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "02/09/2019 12:44:35 - INFO - pytorch_pretrained_bert.modeling -   Weights of BertForSequenceRegression not initialized from pretrained model: ['regressor.weight', 'regressor.bias']\n",
      "02/09/2019 12:44:35 - INFO - pytorch_pretrained_bert.modeling -   Weights from pretrained model not used in BertForSequenceRegression: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceRegression(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(21128, 768)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): BertLayerNorm()\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): BertLayerNorm()\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): BertLayerNorm()\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (regressor): Linear(in_features=768, out_features=1, bias=True)\n",
       "  (loss_fct): MSELoss()\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Prepare model\n",
    "model = BertForSequenceRegression.from_pretrained(\n",
    "    \"bert-base-chinese\",\n",
    "    cache_dir=PYTORCH_PRETRAINED_BERT_CACHE)\n",
    "if FP16:\n",
    "    model.half()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare optimizer\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_optimizer(num_train_optimization_steps: int, learning_rate: float):\n",
    "    grouped_parameters = [\n",
    "       x for x in optimizer_grouped_parameters if any([p.requires_grad for p in x[\"params\"]])\n",
    "    ]\n",
    "    for group in grouped_parameters:\n",
    "        group['lr'] = learning_rate\n",
    "    if FP16:\n",
    "        try:\n",
    "            from apex.optimizers import FP16_Optimizer\n",
    "            from apex.optimizers import FusedAdam\n",
    "        except ImportError:\n",
    "            raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex \"\n",
    "                              \"to use distributed and fp16 training.\")\n",
    "\n",
    "        optimizer = FusedAdam(grouped_parameters,\n",
    "                              lr=learning_rate, bias_correction=False,\n",
    "                              max_grad_norm=1.0)\n",
    "        if args.loss_scale == 0:\n",
    "            optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)\n",
    "        else:\n",
    "            optimizer = FP16_Optimizer(optimizer, static_loss_scale=LOSS_SCALE)\n",
    "\n",
    "    else:\n",
    "        optimizer = FreezableBertAdam(grouped_parameters,\n",
    "                             lr=learning_rate, warmup=WARMUP_PROPORTION,\n",
    "                             t_total=num_train_optimization_steps)\n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model: nn.Module, num_epochs: int, learning_rate: float):\n",
    "    num_train_optimization_steps = len(train_dataloader) * num_epochs \n",
    "    optimizer = get_optimizer(num_train_optimization_steps, learning_rate)\n",
    "    assert all([x[\"lr\"] == learning_rate for x in optimizer.param_groups])\n",
    "    global_step = 0\n",
    "    nb_tr_steps = 0\n",
    "    tr_loss = 0\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(train_features))\n",
    "    logger.info(\"  Batch size = %d\", BATCH_SIZE)\n",
    "    logger.info(\"  Num steps = %d\", num_train_optimization_steps)    \n",
    "    model.train()\n",
    "    mb = master_bar(range(num_epochs))\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0    \n",
    "    for _ in mb:\n",
    "        for step, batch in enumerate(progress_bar(train_dataloader, parent=mb)):\n",
    "            batch = tuple(t.to(device) for t in batch)\n",
    "            input_ids, input_mask, segment_ids, target = batch\n",
    "            loss = model(input_ids, segment_ids, input_mask, target)\n",
    "            if n_gpu > 1:\n",
    "                loss = loss.mean() # mean() to average on multi-gpu.\n",
    "\n",
    "            if FP16:\n",
    "                optimizer.backward(loss)\n",
    "            else:\n",
    "                loss.backward()\n",
    "\n",
    "            if tr_loss == 0:\n",
    "                tr_loss = loss.item()\n",
    "            else:\n",
    "                tr_loss = tr_loss * 0.9 + loss.item() * 0.1\n",
    "            nb_tr_examples += input_ids.size(0)\n",
    "            nb_tr_steps += 1\n",
    "            if FP16:\n",
    "                # modify learning rate with special warm up BERT uses\n",
    "                # if args.fp16 is False, BertAdam is used that handles this automatically\n",
    "                lr_this_step = (\n",
    "                     LR * warmup_linear(global_step/num_train_optimization_steps, WARMUP_PROPORTION))\n",
    "                for param_group in optimizer.param_groups:\n",
    "                    param_group['lr'] = lr_this_step\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            global_step += 1\n",
    "            mb.child.comment = f'loss: {tr_loss:.4f} lr: {optimizer.get_lr()[0]:.2E}'\n",
    "    logger.info(\"  train loss = %.4f\", tr_loss) \n",
    "    return tr_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)\n",
    "all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)\n",
    "all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)\n",
    "all_targets = torch.tensor([f.target for f in train_features], dtype=torch.float)\n",
    "train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_targets)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 12:44:37 - INFO - regressor -   # of paramters: 102,268,417\n",
      "02/09/2019 12:44:37 - INFO - regressor -   # of trainable paramters: 591,361\n",
      "02/09/2019 12:44:37 - INFO - regressor -   ***** Running training *****\n",
      "02/09/2019 12:44:37 - INFO - regressor -     Num examples = 86871\n",
      "02/09/2019 12:44:37 - INFO - regressor -     Batch size = 32\n",
      "02/09/2019 12:44:37 - INFO - regressor -     Num steps = 5430\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 16:16 <p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 13:00:54 - INFO - regressor -     train loss = 0.1961\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.1960972243634961"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train only the \"pooler\" and the final linear layer\n",
    "set_trainable(model, True)\n",
    "set_trainable(model.bert.embeddings, False)\n",
    "set_trainable(model.bert.encoder, False)\n",
    "count_model_parameters(model)\n",
    "train(model, num_epochs = 2, learning_rate = 5e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self\n",
    "output_model_file = \"./regressor_stage1.pth\"\n",
    "# torch.save(model_to_save.state_dict(), output_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(output_model_file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "446"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 13:28:16 - INFO - regressor -   # of paramters: 102,268,417\n",
      "02/09/2019 13:28:16 - INFO - regressor -   # of trainable paramters: 14,767,105\n",
      "02/09/2019 13:28:16 - INFO - regressor -   ***** Running training *****\n",
      "02/09/2019 13:28:16 - INFO - regressor -     Num examples = 86871\n",
      "02/09/2019 13:28:16 - INFO - regressor -     Batch size = 32\n",
      "02/09/2019 13:28:16 - INFO - regressor -     Num steps = 5430\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 22:09 <p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 13:50:25 - INFO - regressor -     train loss = 0.2017\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.20167179324832113"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the last two layer, too\n",
    "set_trainable(model.bert.encoder.layer[11], True)\n",
    "set_trainable(model.bert.encoder.layer[10], True)\n",
    "count_model_parameters(model)\n",
    "train(model, num_epochs = 2, learning_rate = 5e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self\n",
    "output_model_file = \"./regressor_stage2.pth\"\n",
    "# torch.save(model_to_save.state_dict(), output_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 13:55:17 - INFO - regressor -   # of paramters: 102,268,417\n",
      "02/09/2019 13:55:17 - INFO - regressor -   # of trainable paramters: 102,268,417\n",
      "02/09/2019 13:55:17 - INFO - regressor -   ***** Running training *****\n",
      "02/09/2019 13:55:17 - INFO - regressor -     Num examples = 86871\n",
      "02/09/2019 13:55:17 - INFO - regressor -     Batch size = 32\n",
      "02/09/2019 13:55:17 - INFO - regressor -     Num steps = 2715\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Total time: 27:06 <p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 14:22:24 - INFO - regressor -     train loss = 0.1713\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.17134070469037616"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train all layers\n",
    "set_trainable(model, True)\n",
    "count_model_parameters(model)\n",
    "train(model, num_epochs = 1, learning_rate = 1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self\n",
    "output_model_file = \"./regressor_stage3.pth\"\n",
    "# torch.save(model_to_save.state_dict(), output_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "# model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self\n",
    "# output_model_file = \"./pytorch_model.bin\"\n",
    "# torch.save(model_to_save.state_dict(), output_model_file)\n",
    "\n",
    "# Load a trained model that you have fine-tuned\n",
    "# model_state_dict = torch.load(output_model_file)\n",
    "# model = BertForSequenceClassification.from_pretrained(args.bert_model, state_dict=model_state_dict, num_labels=num_labels)\n",
    "# model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_features\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evauluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 13:51:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 13:51:16 - INFO - regressor -   guid: 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   tokens: [CLS] 舍 不 得 辜 负 这 段 相 遇 ， 但 继 续 下 去 是 一 个 太 大 的 决 定 。 我 努 力 过 了 ， 不 喜 欢 你 ， 真 希 望 能 结 束 于 谈 天 说 地 。 怪 这 个 城 市 太 快 吧 ， 在 世 界 的 其 他 地 方 我 们 还 生 活 在 昨 天 。 [SEP]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_ids: 101 5650 679 2533 6790 6566 6821 3667 4685 6878 8024 852 5326 5330 678 1343 3221 671 702 1922 1920 4638 1104 2137 511 2769 1222 1213 6814 749 8024 679 1599 3614 872 8024 4696 2361 3307 5543 5310 3338 754 6448 1921 6432 1765 511 2597 6821 702 1814 2356 1922 2571 1416 8024 1762 686 4518 4638 1071 800 1765 3175 2769 812 6820 4495 3833 1762 3219 1921 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   target: [0.]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 13:51:16 - INFO - regressor -   guid: 1\n",
      "02/09/2019 13:51:16 - INFO - regressor -   tokens: [CLS] 那 一 段 模 仿 《 性 爱 宝 典 》 ！ ！ ！ ！ ！ ！ [SEP]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_ids: 101 6929 671 3667 3563 820 517 2595 4263 2140 1073 518 8013 8013 8013 8013 8013 8013 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   target: [0.]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 13:51:16 - INFO - regressor -   guid: 2\n",
      "02/09/2019 13:51:16 - INFO - regressor -   tokens: [CLS] 我 应 该 有 看 过 这 片 ， 不 过 不 知 是 不 是 真 的 与 我 脑 中 的 那 片 是 同 一 部 片 而 已 ， 只 记 得 女 主 在 海 滩 过 裸 着 上 身 躺 在 躺 椅 上 ， 不 过 为 什 么 要 安 排 这 一 幕 ， 没 有 裸 也 没 影 响 片 啊 ， 裸 来 干 吗 ？ [SEP]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_ids: 101 2769 2418 6421 3300 4692 6814 6821 4275 8024 679 6814 679 4761 3221 679 3221 4696 4638 680 2769 5554 704 4638 6929 4275 3221 1398 671 6956 4275 5445 2347 8024 1372 6381 2533 1957 712 1762 3862 4013 6814 6180 4708 677 6716 6720 1762 6720 3488 677 8024 679 6814 711 784 720 6206 2128 2961 6821 671 2391 8024 3766 3300 6180 738 3766 2512 1510 4275 1557 8024 6180 3341 2397 1408 8043 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   target: [0.]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 13:51:16 - INFO - regressor -   guid: 3\n",
      "02/09/2019 13:51:16 - INFO - regressor -   tokens: [CLS] 挺 有 想 法 的 ， 主 角 的 口 炮 挺 有 深 度 的 。 缺 的 是 人 物 造 型 不 是 太 惊 悚 ， 有 几 个 坑 没 填 平 。 [SEP]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_ids: 101 2923 3300 2682 3791 4638 8024 712 6235 4638 1366 4152 2923 3300 3918 2428 4638 511 5375 4638 3221 782 4289 6863 1798 679 3221 1922 2661 2639 8024 3300 1126 702 1778 3766 1856 2398 511 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   target: [0.]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   *** Example ***\n",
      "02/09/2019 13:51:16 - INFO - regressor -   guid: 4\n",
      "02/09/2019 13:51:16 - INFO - regressor -   tokens: [CLS] 虽 然 背 景 人 物 阴 暗 扭 曲 ， 但 我 感 受 到 的 是 他 们 对 自 由 对 梦 想 无 限 的 追 求 ， 无 所 羁 绊 ， 真 爱 无 欲 无 求 [SEP]\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_ids: 101 6006 4197 5520 3250 782 4289 7346 3266 2814 3289 8024 852 2769 2697 1358 1168 4638 3221 800 812 2190 5632 4507 2190 3457 2682 3187 7361 4638 6841 3724 8024 3187 2792 5396 5304 8024 4696 4263 3187 3617 3187 3724 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "02/09/2019 13:51:16 - INFO - regressor -   target: [1.]\n"
     ]
    }
   ],
   "source": [
    "eval_examples = DoubanRatingProcessor().get_dev_examples()\n",
    "eval_features = convert_examples_to_features(\n",
    "    eval_examples, MAX_SEQ_LENGTH, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "02/09/2019 14:25:07 - INFO - regressor -   ***** Running evaluation *****\n",
      "02/09/2019 14:25:07 - INFO - regressor -     Num examples = 28957\n",
      "02/09/2019 14:25:07 - INFO - regressor -     Batch size = 160\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='181' class='' max='181', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      100.00% [181/181 02:35<00:00 0.1671]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.16707682765979134"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logger.info(\"***** Running evaluation *****\")\n",
    "logger.info(\"  Num examples = %d\", len(eval_examples))\n",
    "logger.info(\"  Batch size = %d\", BATCH_SIZE * 5)\n",
    "all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)\n",
    "all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)\n",
    "all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)\n",
    "all_targets = torch.tensor([f.target for f in eval_features], dtype=torch.float)\n",
    "eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_targets)\n",
    "# Run prediction for full data\n",
    "eval_sampler = SequentialSampler(eval_data)\n",
    "eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=BATCH_SIZE * 5)\n",
    "\n",
    "model.eval()\n",
    "eval_loss, eval_accuracy = 0, 0\n",
    "nb_eval_steps, nb_eval_examples = 0, 0\n",
    "\n",
    "mb = progress_bar(eval_dataloader)\n",
    "for input_ids, input_mask, segment_ids, targets in mb:\n",
    "    input_ids = input_ids.to(device)\n",
    "    input_mask = input_mask.to(device)\n",
    "    segment_ids = segment_ids.to(device)\n",
    "    targets = targets.to(device)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        tmp_eval_loss = model(input_ids, segment_ids, input_mask, targets)\n",
    "        # outputs = model(input_ids, segment_ids, input_mask)\n",
    "\n",
    "    # outputs = outputs.detach().cpu().numpy()\n",
    "    # targets = targets.to('cpu').numpy()\n",
    "    # tmp_eval_accuracy = accuracy(logits, label_ids)\n",
    "\n",
    "    eval_loss += tmp_eval_loss.mean().item()\n",
    "    # eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "    nb_eval_examples += input_ids.size(0)\n",
    "    nb_eval_steps += 1\n",
    "    mb.comment = f'{eval_loss / nb_eval_steps:.4f}'\n",
    "\n",
    "eval_loss / nb_eval_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}