bert_finetune_stsb_sentence_transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bert_finetune_stsb_sentence_transformers.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "pSRBtRSeD9aM",
        "colab_type": "code",
        "outputId": "191abe50-5f57-4f6a-b94c-4ca672daac35",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 609
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ec/e7/0a1babead1b79afabb654fbec0a052e0d833ba4205a6dfd98b1aeda9c82e/transformers-2.2.0-py3-none-any.whl (360kB)\n",
            "\r\u001b[K     |█                               | 10kB 33.8MB/s eta 0:00:01\r\u001b[K     |█▉                              | 20kB 6.1MB/s eta 0:00:01\r\u001b[K     |██▊                             | 30kB 8.6MB/s eta 0:00:01\r\u001b[K     |███▋                            | 40kB 5.7MB/s eta 0:00:01\r\u001b[K     |████▌                           | 51kB 7.0MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 61kB 8.2MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 71kB 9.4MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 81kB 10.5MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 92kB 11.7MB/s eta 0:00:01\r\u001b[K     |█████████                       | 102kB 9.2MB/s eta 0:00:01\r\u001b[K     |██████████                      | 112kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████                     | 122kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 133kB 9.2MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 143kB 9.2MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 153kB 9.2MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 163kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 174kB 9.2MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 184kB 9.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 194kB 9.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 204kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 215kB 9.2MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 225kB 9.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 235kB 9.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 245kB 9.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 256kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 266kB 9.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 276kB 9.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 286kB 9.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 296kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 307kB 9.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 317kB 9.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 327kB 9.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 337kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 348kB 9.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 358kB 9.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 368kB 9.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Collecting regex\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e3/8e/cbf2295643d7265e7883326fb4654e643bfc93b3a8a8274d8010a39d8804/regex-2019.11.1-cp36-cp36m-manylinux1_x86_64.whl (643kB)\n",
            "\u001b[K     |████████████████████████████████| 645kB 67.9MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/3d/efb655a670b98f62ec32d66954e1109f403db4d937c50d779a75b9763a29/sentencepiece-0.1.83-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 56.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.10.18)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1f/8e/ed5364a06a9ba720fddd9820155cc57300d28f5f43a6fd7b7e817177e642/sacremoses-0.0.35.tar.gz (859kB)\n",
            "\u001b[K     |████████████████████████████████| 860kB 74.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.9.11)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: botocore<1.14.0,>=1.13.18 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.13.18)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.2.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.0)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.18->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<2.8.1,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.18->boto3->transformers) (2.6.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.35-cp36-none-any.whl size=883999 sha256=9375db6b0c7b19355873311a1f57f5cd1a6d16dde7d820ad9925cc6f9e6a9db2\n",
            "  Stored in directory: /root/.cache/pip/wheels/63/2a/db/63e2909042c634ef551d0d9ac825b2b0b32dede4a6d87ddc94\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: regex, sentencepiece, sacremoses, transformers\n",
            "Successfully installed regex-2019.11.1 sacremoses-0.0.35 sentencepiece-0.1.83 transformers-2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "70BCXzd-D8cn",
        "colab_type": "code",
        "outputId": "7c0c8856-6e09-442a-da5f-98f74b8e0adc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 612
        }
      },
      "source": [
        "!pip install -U sentence-transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentence-transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/85/17/9edba42c29fda04f2eb8597bb4de380f0f43d65e317969070c04510d93eb/sentence-transformers-0.2.3.tar.gz (45kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 4.1MB/s \n",
            "\u001b[?25hCollecting pytorch-transformers==1.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/89/ad0d6bb932d0a51793eaabcf1617a36ff530dc9ab9e38f765a35dc293306/pytorch_transformers-1.1.0-py3-none-any.whl (158kB)\n",
            "\u001b[K     |████████████████████████████████| 163kB 21.1MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (4.28.1)\n",
            "Requirement already satisfied, skipping upgrade: torch>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.3.1)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.17.4)\n",
            "Requirement already satisfied, skipping upgrade: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (0.21.3)\n",
            "Requirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.3.2)\n",
            "Requirement already satisfied, skipping upgrade: nltk in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (3.2.5)\n",
            "Requirement already satisfied, skipping upgrade: sentencepiece in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers==1.1.0->sentence-transformers) (0.1.83)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers==1.1.0->sentence-transformers) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers==1.1.0->sentence-transformers) (1.10.18)\n",
            "Requirement already satisfied, skipping upgrade: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers==1.1.0->sentence-transformers) (2019.11.1)\n",
            "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sentence-transformers) (0.14.0)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from nltk->sentence-transformers) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers==1.1.0->sentence-transformers) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers==1.1.0->sentence-transformers) (2.8)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers==1.1.0->sentence-transformers) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers==1.1.0->sentence-transformers) (2019.9.11)\n",
            "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers==1.1.0->sentence-transformers) (0.9.4)\n",
            "Requirement already satisfied, skipping upgrade: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers==1.1.0->sentence-transformers) (0.2.1)\n",
            "Requirement already satisfied, skipping upgrade: botocore<1.14.0,>=1.13.18 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers==1.1.0->sentence-transformers) (1.13.18)\n",
            "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.18->boto3->pytorch-transformers==1.1.0->sentence-transformers) (0.15.2)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil<2.8.1,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.18->boto3->pytorch-transformers==1.1.0->sentence-transformers) (2.6.1)\n",
            "Building wheels for collected packages: sentence-transformers\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-0.2.3-cp36-none-any.whl size=57413 sha256=17cb18ca192fafc7dd6dcff567850a7a9f7598747e683a27cc7711d9e0d95e99\n",
            "  Stored in directory: /root/.cache/pip/wheels/20/89/88/95d8a3d4034bb77f52a6f66efdbfc623240221183dc0001c31\n",
            "Successfully built sentence-transformers\n",
            "Installing collected packages: pytorch-transformers, sentence-transformers\n",
            "Successfully installed pytorch-transformers-1.1.0 sentence-transformers-0.2.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofenonJJDvyA",
        "colab_type": "code",
        "outputId": "e0dd0310-f0c7-45a2-984c-75a64cf8c8c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 443
        }
      },
      "source": [
        "from torch.utils.data import DataLoader\n",
        "import math\n",
        "from sentence_transformers import SentenceTransformer,  SentencesDataset, LoggingHandler, losses, models\n",
        "from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator\n",
        "from sentence_transformers.readers import STSDataReader\n",
        "import logging\n",
        "from datetime import datetime\n",
        "\n",
        "\n",
        "#### Just some code to print debug information to stdout\n",
        "logging.basicConfig(format='%(asctime)s - %(message)s',\n",
        "                    datefmt='%Y-%m-%d %H:%M:%S',\n",
        "                    level=logging.INFO,\n",
        "                    handlers=[LoggingHandler()])\n",
        "#### /print debug information to stdout\n",
        "\n",
        "# Read the dataset\n",
        "train_batch_size = 16\n",
        "num_epochs = 4\n",
        "model_save_path = 'output/training_stsbenchmark_bert-'+datetime.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
        "sts_reader = STSDataReader('stsbenchmark', normalize_scores=True)\n",
        "\n",
        "# Use BERT for mapping tokens to embeddings\n",
        "word_embedding_model = models.BERT('bert-base-uncased')\n",
        "\n",
        "# Apply mean pooling to get one fixed sized sentence vector\n",
        "pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),\n",
        "                               pooling_mode_mean_tokens=True,\n",
        "                               pooling_mode_cls_token=False,\n",
        "                               pooling_mode_max_tokens=False)\n",
        "\n",
        "model = SentenceTransformer(modules=[word_embedding_model, pooling_model])\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 08:19:18 - loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at /root/.cache/torch/pytorch_transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.bf3b9ea126d8c0001ee8a1e8b92229871d06d36d8808208cc2449280da87785c\n",
            "2019-11-28 08:19:18 - Model config {\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"finetuning_task\": null,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"num_labels\": 2,\n",
            "  \"output_attentions\": false,\n",
            "  \"output_hidden_states\": false,\n",
            "  \"torchscript\": false,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "2019-11-28 08:19:19 - loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at /root/.cache/torch/pytorch_transformers/aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n",
            "2019-11-28 08:19:21 - loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.cache/torch/pytorch_transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
            "2019-11-28 08:19:21 - Use pytorch device: cpu\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WolSRSxoFIRP",
        "colab_type": "code",
        "outputId": "f651a5e3-584d-4bd2-91e1-54244daa93f6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 152
        }
      },
      "source": [
        "import urllib.request\n",
        "import zipfile\n",
        "import os\n",
        "#folder_path = os.path.dirname(os.path.realpath(__file__))\n",
        "folder_path = \"./\"\n",
        "print('Beginning download of datasets')\n",
        "\n",
        "datasets = ['stsbenchmark.zip']\n",
        "server = \"https://public.ukp.informatik.tu-darmstadt.de/reimers/sentence-transformers/datasets/\"\n",
        "\n",
        "for dataset in datasets:\n",
        "    print(\"Download\", dataset)\n",
        "    url = server+dataset\n",
        "    dataset_path = os.path.join(folder_path, dataset)\n",
        "    urllib.request.urlretrieve(url, dataset_path)\n",
        "\n",
        "    print(\"Extract\", dataset)\n",
        "    with zipfile.ZipFile(dataset_path, \"r\") as zip_ref:\n",
        "        zip_ref.extractall(folder_path)\n",
        "    os.remove(dataset_path)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Beginning download of datasets\n",
            "Download AllNLI.zip\n",
            "Extract AllNLI.zip\n",
            "Download stsbenchmark.zip\n",
            "Extract stsbenchmark.zip\n",
            "Download wikipedia-sections-triplets.zip\n",
            "Extract wikipedia-sections-triplets.zip\n",
            "All datasets downloaded and extracted\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VMlLtd7LEM3t",
        "colab_type": "code",
        "outputId": "4d613a8c-e3a2-408d-c774-5615f1c386ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "\n",
        "# Convert the dataset to a DataLoader ready for training\n",
        "logging.info(\"Read STSbenchmark train dataset\")\n",
        "train_data = SentencesDataset(sts_reader.get_examples('sts-train.csv'), model)\n",
        "train_dataloader = DataLoader(train_data, shuffle=True, batch_size=train_batch_size)\n",
        "train_loss = losses.CosineSimilarityLoss(model=model)\n",
        "\n",
        "\n",
        "logging.info(\"Read STSbenchmark dev dataset\")\n",
        "dev_data = SentencesDataset(examples=sts_reader.get_examples('sts-dev.csv'), model=model)\n",
        "dev_dataloader = DataLoader(dev_data, shuffle=False, batch_size=train_batch_size)\n",
        "evaluator = EmbeddingSimilarityEvaluator(dev_dataloader)\n",
        "\n",
        "\n",
        "# Configure the training. We skip evaluation in this example\n",
        "warmup_steps = math.ceil(len(train_data)*num_epochs/train_batch_size*0.1) #10% of train data for warm-up\n",
        "logging.info(\"Warmup-steps: {}\".format(warmup_steps))\n",
        "\n",
        "\n",
        "# Train the model\n",
        "model.fit(train_objectives=[(train_dataloader, train_loss)],\n",
        "          evaluator=evaluator,\n",
        "          epochs=num_epochs,\n",
        "          evaluation_steps=1000,\n",
        "          warmup_steps=warmup_steps,\n",
        "          output_path=model_save_path)\n",
        "\n",
        "\n",
        "##############################################################################\n",
        "#\n",
        "# Load the stored model and evaluate its performance on STS benchmark dataset\n",
        "#\n",
        "##############################################################################\n",
        "\n",
        "model = SentenceTransformer(model_save_path)\n",
        "test_data = SentencesDataset(examples=sts_reader.get_examples(\"sts-test.csv\"), model=model)\n",
        "test_dataloader = DataLoader(test_data, shuffle=False, batch_size=train_batch_size)\n",
        "evaluator = EmbeddingSimilarityEvaluator(test_dataloader)\n",
        "model.evaluate(evaluator)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Convert dataset:   2%|▏         | 98/5749 [00:00<00:06, 844.45it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 08:19:27 - Read STSbenchmark train dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Convert dataset: 100%|██████████| 5749/5749 [00:02<00:00, 2636.34it/s]\n",
            "Convert dataset:  26%|██▌       | 391/1500 [00:00<00:00, 3899.20it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 08:19:29 - Num sentences: 5749\n",
            "2019-11-28 08:19:29 - Sentences 0 longer than max_seqence_length: 0\n",
            "2019-11-28 08:19:29 - Sentences 1 longer than max_seqence_length: 0\n",
            "2019-11-28 08:19:29 - Read STSbenchmark dev dataset\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Convert dataset: 100%|██████████| 1500/1500 [00:00<00:00, 2396.09it/s]\n",
            "Epoch:   0%|          | 0/4 [00:00<?, ?it/s]\n",
            "Iteration:   0%|          | 0/360 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 08:19:30 - Num sentences: 1500\n",
            "2019-11-28 08:19:30 - Sentences 0 longer than max_seqence_length: 0\n",
            "2019-11-28 08:19:30 - Sentences 1 longer than max_seqence_length: 0\n",
            "2019-11-28 08:19:30 - Warmup-steps: 144\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration:   0%|          | 1/360 [00:08<52:08,  8.71s/it]\u001b[A\n",
            "Iteration:   1%|          | 2/360 [00:19<56:29,  9.47s/it]\u001b[A\n",
            "Iteration:   1%|          | 3/360 [00:29<56:41,  9.53s/it]\u001b[A\n",
            "Iteration:   1%|          | 4/360 [00:38<54:42,  9.22s/it]\u001b[A\n",
            "Iteration:   1%|▏         | 5/360 [00:45<51:01,  8.62s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 6/360 [00:55<54:08,  9.18s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 7/360 [01:02<49:28,  8.41s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 8/360 [01:12<52:17,  8.91s/it]\u001b[A\n",
            "Iteration:   2%|▎         | 9/360 [01:18<47:40,  8.15s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 10/360 [01:27<48:30,  8.31s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 11/360 [01:35<48:02,  8.26s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 12/360 [01:45<51:22,  8.86s/it]\u001b[A\n",
            "Iteration:   4%|▎         | 13/360 [01:55<52:33,  9.09s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 14/360 [02:03<50:42,  8.79s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 15/360 [02:11<48:45,  8.48s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 16/360 [02:20<48:56,  8.54s/it]\u001b[A\n",
            "Iteration:   5%|▍         | 17/360 [02:29<50:46,  8.88s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 18/360 [02:37<47:44,  8.38s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 19/360 [02:46<48:55,  8.61s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 20/360 [02:55<49:58,  8.82s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 21/360 [03:03<48:22,  8.56s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 22/360 [03:09<43:21,  7.70s/it]\u001b[A\n",
            "Iteration:   6%|▋         | 23/360 [03:15<41:09,  7.33s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 24/360 [03:25<44:43,  7.99s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 25/360 [03:34<46:33,  8.34s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 26/360 [03:43<47:39,  8.56s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 27/360 [03:51<47:23,  8.54s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 28/360 [04:00<48:00,  8.68s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 29/360 [04:10<48:44,  8.83s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 30/360 [04:18<48:27,  8.81s/it]\u001b[A\n",
            "Iteration:   9%|▊         | 31/360 [04:25<44:48,  8.17s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 32/360 [04:33<44:22,  8.12s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 33/360 [04:42<46:08,  8.47s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 34/360 [04:50<44:03,  8.11s/it]\u001b[A\n",
            "Iteration:  10%|▉         | 35/360 [04:57<43:17,  7.99s/it]\u001b[A\n",
            "Iteration:  10%|█         | 36/360 [05:07<46:07,  8.54s/it]\u001b[A\n",
            "Iteration:  10%|█         | 37/360 [05:16<46:45,  8.68s/it]\u001b[A\n",
            "Iteration:  11%|█         | 38/360 [05:25<46:18,  8.63s/it]\u001b[A\n",
            "Iteration:  11%|█         | 39/360 [05:34<46:41,  8.73s/it]\u001b[A\n",
            "Iteration:  11%|█         | 40/360 [05:42<46:14,  8.67s/it]\u001b[A\n",
            "Iteration:  11%|█▏        | 41/360 [05:51<46:38,  8.77s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 42/360 [05:57<41:35,  7.85s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 43/360 [06:07<44:40,  8.45s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 44/360 [06:14<43:12,  8.21s/it]\u001b[A\n",
            "Iteration:  12%|█▎        | 45/360 [06:26<48:04,  9.16s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 46/360 [06:35<49:01,  9.37s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 47/360 [06:44<47:13,  9.05s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 48/360 [06:54<48:10,  9.26s/it]\u001b[A\n",
            "Iteration:  14%|█▎        | 49/360 [07:02<46:08,  8.90s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 50/360 [07:10<45:35,  8.82s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 51/360 [07:18<43:51,  8.52s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 52/360 [07:28<46:30,  9.06s/it]\u001b[A\n",
            "Iteration:  15%|█▍        | 53/360 [07:37<45:46,  8.95s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 54/360 [07:49<50:34,  9.92s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 55/360 [07:57<46:33,  9.16s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 56/360 [08:06<46:37,  9.20s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 57/360 [08:14<45:28,  9.00s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 58/360 [08:23<44:40,  8.88s/it]\u001b[A\n",
            "Iteration:  16%|█▋        | 59/360 [08:32<44:37,  8.89s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 60/360 [08:38<40:04,  8.01s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 61/360 [08:48<43:03,  8.64s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 62/360 [08:56<41:53,  8.43s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 63/360 [09:05<42:00,  8.49s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 64/360 [09:12<39:30,  8.01s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 65/360 [09:19<38:07,  7.75s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 66/360 [09:28<40:44,  8.31s/it]\u001b[A\n",
            "Iteration:  19%|█▊        | 67/360 [09:38<42:52,  8.78s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 68/360 [09:45<40:06,  8.24s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 69/360 [09:52<38:22,  7.91s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 70/360 [10:00<38:05,  7.88s/it]\u001b[A\n",
            "Iteration:  20%|█▉        | 71/360 [10:06<34:32,  7.17s/it]\u001b[A\n",
            "Iteration:  20%|██        | 72/360 [10:13<34:45,  7.24s/it]\u001b[A\n",
            "Iteration:  20%|██        | 73/360 [10:22<37:42,  7.88s/it]\u001b[A\n",
            "Iteration:  21%|██        | 74/360 [10:30<37:14,  7.81s/it]\u001b[A\n",
            "Iteration:  21%|██        | 75/360 [10:39<38:30,  8.11s/it]\u001b[A\n",
            "Iteration:  21%|██        | 76/360 [10:48<39:34,  8.36s/it]\u001b[A\n",
            "Iteration:  21%|██▏       | 77/360 [10:57<40:02,  8.49s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 78/360 [11:05<39:45,  8.46s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 79/360 [11:12<36:56,  7.89s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 80/360 [11:20<37:24,  8.02s/it]\u001b[A\n",
            "Iteration:  22%|██▎       | 81/360 [11:27<35:32,  7.64s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 82/360 [11:36<38:14,  8.25s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 83/360 [11:45<38:22,  8.31s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 84/360 [11:54<39:08,  8.51s/it]\u001b[A\n",
            "Iteration:  24%|██▎       | 85/360 [12:00<35:28,  7.74s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 86/360 [12:08<35:45,  7.83s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 87/360 [12:16<36:18,  7.98s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 88/360 [12:23<34:32,  7.62s/it]\u001b[A\n",
            "Iteration:  25%|██▍       | 89/360 [12:31<34:45,  7.69s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 90/360 [12:39<35:00,  7.78s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 91/360 [12:47<36:03,  8.04s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 92/360 [12:55<35:12,  7.88s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 93/360 [13:05<37:41,  8.47s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 94/360 [13:12<36:06,  8.14s/it]\u001b[A\n",
            "Iteration:  26%|██▋       | 95/360 [13:20<35:29,  8.03s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 96/360 [13:29<36:26,  8.28s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 97/360 [13:37<36:28,  8.32s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 98/360 [13:45<35:36,  8.16s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 99/360 [13:53<35:51,  8.25s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 100/360 [14:01<35:04,  8.10s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 101/360 [14:11<37:05,  8.59s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 102/360 [14:20<38:07,  8.87s/it]\u001b[A\n",
            "Iteration:  29%|██▊       | 103/360 [14:29<38:10,  8.91s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 104/360 [14:38<38:07,  8.93s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 105/360 [14:46<35:43,  8.41s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 106/360 [14:54<35:24,  8.37s/it]\u001b[A\n",
            "Iteration:  30%|██▉       | 107/360 [15:01<34:11,  8.11s/it]\u001b[A\n",
            "Iteration:  30%|███       | 108/360 [15:10<34:18,  8.17s/it]\u001b[A\n",
            "Iteration:  30%|███       | 109/360 [15:17<33:29,  8.01s/it]\u001b[A\n",
            "Iteration:  31%|███       | 110/360 [15:23<31:01,  7.44s/it]\u001b[A\n",
            "Iteration:  31%|███       | 111/360 [15:31<31:16,  7.53s/it]\u001b[A\n",
            "Iteration:  31%|███       | 112/360 [15:39<31:39,  7.66s/it]\u001b[A\n",
            "Iteration:  31%|███▏      | 113/360 [15:48<33:02,  8.03s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 114/360 [15:56<33:14,  8.11s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 115/360 [16:05<33:45,  8.27s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 116/360 [16:13<33:43,  8.29s/it]\u001b[A\n",
            "Iteration:  32%|███▎      | 117/360 [16:23<34:48,  8.60s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 118/360 [16:31<34:15,  8.49s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 119/360 [16:40<34:41,  8.64s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 120/360 [16:48<34:37,  8.66s/it]\u001b[A\n",
            "Iteration:  34%|███▎      | 121/360 [16:59<36:38,  9.20s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 122/360 [17:06<34:27,  8.69s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 123/360 [17:14<33:28,  8.48s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 124/360 [17:23<33:29,  8.51s/it]\u001b[A\n",
            "Iteration:  35%|███▍      | 125/360 [17:33<34:56,  8.92s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 126/360 [17:39<31:36,  8.11s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 127/360 [17:45<28:39,  7.38s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 128/360 [17:51<27:39,  7.15s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 129/360 [18:00<28:50,  7.49s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 130/360 [18:08<29:23,  7.67s/it]\u001b[A\n",
            "Iteration:  36%|███▋      | 131/360 [18:16<30:28,  7.99s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 132/360 [18:24<29:17,  7.71s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 133/360 [18:32<30:01,  7.94s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 134/360 [18:42<32:35,  8.65s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 135/360 [18:51<32:02,  8.55s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 136/360 [18:59<31:53,  8.54s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 137/360 [19:08<32:14,  8.67s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 138/360 [19:16<31:28,  8.51s/it]\u001b[A\n",
            "Iteration:  39%|███▊      | 139/360 [19:24<30:52,  8.38s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 140/360 [19:31<28:33,  7.79s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 141/360 [19:40<30:05,  8.24s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 142/360 [19:50<31:58,  8.80s/it]\u001b[A\n",
            "Iteration:  40%|███▉      | 143/360 [20:00<32:28,  8.98s/it]\u001b[A\n",
            "Iteration:  40%|████      | 144/360 [20:08<31:17,  8.69s/it]\u001b[A\n",
            "Iteration:  40%|████      | 145/360 [20:17<31:24,  8.76s/it]\u001b[A\n",
            "Iteration:  41%|████      | 146/360 [20:25<31:13,  8.76s/it]\u001b[A\n",
            "Iteration:  41%|████      | 147/360 [20:33<30:26,  8.58s/it]\u001b[A\n",
            "Iteration:  41%|████      | 148/360 [20:41<29:42,  8.41s/it]\u001b[A\n",
            "Iteration:  41%|████▏     | 149/360 [20:49<28:29,  8.10s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 150/360 [20:59<30:58,  8.85s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 151/360 [21:07<29:37,  8.50s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 152/360 [21:14<27:39,  7.98s/it]\u001b[A\n",
            "Iteration:  42%|████▎     | 153/360 [21:23<28:54,  8.38s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 154/360 [21:31<28:25,  8.28s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 155/360 [21:40<28:19,  8.29s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 156/360 [21:45<25:43,  7.57s/it]\u001b[A\n",
            "Iteration:  44%|████▎     | 157/360 [21:53<25:39,  7.58s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 158/360 [22:00<24:54,  7.40s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 159/360 [22:07<24:39,  7.36s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 160/360 [22:16<25:48,  7.74s/it]\u001b[A\n",
            "Iteration:  45%|████▍     | 161/360 [22:24<25:37,  7.73s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 162/360 [22:30<24:32,  7.44s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 163/360 [22:39<26:01,  7.93s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 164/360 [22:46<24:30,  7.50s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 165/360 [22:52<22:59,  7.07s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 166/360 [23:01<24:47,  7.67s/it]\u001b[A\n",
            "Iteration:  46%|████▋     | 167/360 [23:07<22:40,  7.05s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 168/360 [23:15<23:22,  7.30s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 169/360 [23:23<24:19,  7.64s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 170/360 [23:32<25:24,  8.02s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 171/360 [23:42<27:20,  8.68s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 172/360 [23:51<27:19,  8.72s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 173/360 [23:59<26:42,  8.57s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 174/360 [24:06<25:22,  8.18s/it]\u001b[A\n",
            "Iteration:  49%|████▊     | 175/360 [24:16<26:02,  8.45s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 176/360 [24:26<27:19,  8.91s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 177/360 [24:35<27:18,  8.95s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 178/360 [24:43<26:42,  8.80s/it]\u001b[A\n",
            "Iteration:  50%|████▉     | 179/360 [24:53<27:47,  9.21s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 180/360 [25:01<26:01,  8.68s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 181/360 [25:10<26:55,  9.03s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 182/360 [25:19<26:03,  8.78s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 183/360 [25:26<24:48,  8.41s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 184/360 [25:33<23:33,  8.03s/it]\u001b[A\n",
            "Iteration:  51%|█████▏    | 185/360 [25:42<23:48,  8.16s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 186/360 [25:51<24:24,  8.42s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 187/360 [25:59<24:06,  8.36s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 188/360 [26:07<23:43,  8.28s/it]\u001b[A\n",
            "Iteration:  52%|█████▎    | 189/360 [26:16<23:51,  8.37s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 190/360 [26:23<22:55,  8.09s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 191/360 [26:30<21:54,  7.78s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 192/360 [26:39<22:44,  8.12s/it]\u001b[A\n",
            "Iteration:  54%|█████▎    | 193/360 [26:47<22:29,  8.08s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 194/360 [26:56<22:52,  8.27s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 195/360 [27:04<23:01,  8.37s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 196/360 [27:13<22:57,  8.40s/it]\u001b[A\n",
            "Iteration:  55%|█████▍    | 197/360 [27:23<24:19,  8.95s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 198/360 [27:32<24:21,  9.02s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 199/360 [27:41<23:31,  8.77s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 200/360 [27:49<23:11,  8.70s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 201/360 [27:55<20:48,  7.85s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 202/360 [28:04<21:16,  8.08s/it]\u001b[A\n",
            "Iteration:  56%|█████▋    | 203/360 [28:12<21:38,  8.27s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 204/360 [28:20<20:43,  7.97s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 205/360 [28:29<21:29,  8.32s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 206/360 [28:37<21:21,  8.32s/it]\u001b[A\n",
            "Iteration:  57%|█████▊    | 207/360 [28:46<21:39,  8.49s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 208/360 [28:54<21:20,  8.42s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 209/360 [29:03<21:12,  8.43s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 210/360 [29:11<21:16,  8.51s/it]\u001b[A\n",
            "Iteration:  59%|█████▊    | 211/360 [29:20<20:58,  8.45s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 212/360 [29:29<21:44,  8.81s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 213/360 [29:38<21:46,  8.89s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 214/360 [29:46<21:00,  8.63s/it]\u001b[A\n",
            "Iteration:  60%|█████▉    | 215/360 [29:53<19:36,  8.12s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 216/360 [30:01<18:51,  7.86s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 217/360 [30:10<20:10,  8.46s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 218/360 [30:22<22:07,  9.35s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 219/360 [30:29<20:10,  8.59s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 220/360 [30:37<20:03,  8.60s/it]\u001b[A\n",
            "Iteration:  61%|██████▏   | 221/360 [30:47<20:35,  8.89s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 222/360 [30:56<20:30,  8.91s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 223/360 [31:05<20:38,  9.04s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 224/360 [31:13<19:55,  8.79s/it]\u001b[A\n",
            "Iteration:  62%|██████▎   | 225/360 [31:22<19:39,  8.74s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 226/360 [31:31<19:30,  8.74s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 227/360 [31:39<19:20,  8.72s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 228/360 [31:46<17:50,  8.11s/it]\u001b[A\n",
            "Iteration:  64%|██████▎   | 229/360 [31:54<17:20,  7.94s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 230/360 [32:01<17:01,  7.86s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 231/360 [32:10<17:09,  7.98s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 232/360 [32:18<17:07,  8.03s/it]\u001b[A\n",
            "Iteration:  65%|██████▍   | 233/360 [32:25<16:37,  7.85s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 234/360 [32:33<16:23,  7.81s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 235/360 [32:41<16:22,  7.86s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 236/360 [32:49<16:28,  7.97s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 237/360 [32:57<16:37,  8.11s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 238/360 [33:06<16:48,  8.27s/it]\u001b[A\n",
            "Iteration:  66%|██████▋   | 239/360 [33:15<17:13,  8.54s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 240/360 [33:22<16:06,  8.06s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 241/360 [33:30<15:38,  7.89s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 242/360 [33:38<15:47,  8.03s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 243/360 [33:45<15:06,  7.75s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 244/360 [33:53<14:55,  7.72s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 245/360 [34:01<15:07,  7.90s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 246/360 [34:10<15:26,  8.13s/it]\u001b[A\n",
            "Iteration:  69%|██████▊   | 247/360 [34:19<15:40,  8.33s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 248/360 [34:28<16:11,  8.67s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 249/360 [34:36<15:42,  8.49s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 250/360 [34:46<16:08,  8.80s/it]\u001b[A\n",
            "Iteration:  70%|██████▉   | 251/360 [34:56<17:04,  9.40s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 252/360 [35:06<16:55,  9.40s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 253/360 [35:13<15:36,  8.75s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 254/360 [35:20<14:34,  8.25s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 255/360 [35:28<14:24,  8.23s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 256/360 [35:37<14:20,  8.27s/it]\u001b[A\n",
            "Iteration:  71%|███████▏  | 257/360 [35:43<13:04,  7.61s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 258/360 [35:50<12:48,  7.54s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 259/360 [35:59<13:28,  8.01s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 260/360 [36:07<13:01,  7.81s/it]\u001b[A\n",
            "Iteration:  72%|███████▎  | 261/360 [36:14<12:40,  7.68s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 262/360 [36:24<13:33,  8.31s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 263/360 [36:31<12:55,  8.00s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 264/360 [36:41<13:35,  8.49s/it]\u001b[A\n",
            "Iteration:  74%|███████▎  | 265/360 [36:49<13:29,  8.52s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 266/360 [36:57<13:13,  8.44s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 267/360 [37:07<13:34,  8.76s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 268/360 [37:15<12:55,  8.43s/it]\u001b[A\n",
            "Iteration:  75%|███████▍  | 269/360 [37:25<13:35,  8.96s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 270/360 [37:34<13:37,  9.08s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 271/360 [37:39<11:44,  7.91s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 272/360 [37:49<12:09,  8.29s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 273/360 [37:57<12:14,  8.44s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 274/360 [38:06<12:20,  8.61s/it]\u001b[A\n",
            "Iteration:  76%|███████▋  | 275/360 [38:14<11:46,  8.31s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 276/360 [38:19<10:20,  7.38s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 277/360 [38:27<10:18,  7.45s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 278/360 [38:34<10:05,  7.38s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 279/360 [38:41<09:55,  7.35s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 280/360 [38:49<09:57,  7.47s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 281/360 [38:58<10:23,  7.89s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 282/360 [39:06<10:08,  7.80s/it]\u001b[A\n",
            "Iteration:  79%|███████▊  | 283/360 [39:16<10:50,  8.45s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 284/360 [39:24<10:52,  8.58s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 285/360 [39:30<09:47,  7.83s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 286/360 [39:38<09:38,  7.82s/it]\u001b[A\n",
            "Iteration:  80%|███████▉  | 287/360 [39:47<09:48,  8.06s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 288/360 [39:56<09:54,  8.26s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 289/360 [40:04<09:44,  8.24s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 290/360 [40:13<09:48,  8.40s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 291/360 [40:20<09:18,  8.09s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 292/360 [40:28<09:14,  8.16s/it]\u001b[A\n",
            "Iteration:  81%|████████▏ | 293/360 [40:36<09:04,  8.12s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 294/360 [40:46<09:23,  8.53s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 295/360 [40:54<09:02,  8.35s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 296/360 [41:02<08:46,  8.23s/it]\u001b[A\n",
            "Iteration:  82%|████████▎ | 297/360 [41:09<08:15,  7.87s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 298/360 [41:18<08:37,  8.34s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 299/360 [41:26<08:26,  8.30s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 300/360 [41:34<07:59,  7.99s/it]\u001b[A\n",
            "Iteration:  84%|████████▎ | 301/360 [41:40<07:31,  7.65s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 302/360 [41:49<07:39,  7.93s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 303/360 [41:56<07:10,  7.55s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 304/360 [42:02<06:39,  7.14s/it]\u001b[A\n",
            "Iteration:  85%|████████▍ | 305/360 [42:11<06:59,  7.63s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 306/360 [42:19<07:01,  7.80s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 307/360 [42:26<06:40,  7.56s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 308/360 [42:32<06:16,  7.23s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 309/360 [42:41<06:26,  7.57s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 310/360 [42:50<06:44,  8.09s/it]\u001b[A\n",
            "Iteration:  86%|████████▋ | 311/360 [42:57<06:21,  7.79s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 312/360 [43:04<05:59,  7.50s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 313/360 [43:13<06:09,  7.85s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 314/360 [43:21<06:15,  8.15s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 315/360 [43:28<05:48,  7.75s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 316/360 [43:37<05:53,  8.04s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 317/360 [43:45<05:49,  8.13s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 318/360 [43:55<05:55,  8.47s/it]\u001b[A\n",
            "Iteration:  89%|████████▊ | 319/360 [44:01<05:18,  7.76s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 320/360 [44:09<05:16,  7.91s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 321/360 [44:18<05:25,  8.34s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 322/360 [44:26<05:15,  8.30s/it]\u001b[A\n",
            "Iteration:  90%|████████▉ | 323/360 [44:34<04:54,  7.96s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 324/360 [44:41<04:44,  7.91s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 325/360 [44:52<05:00,  8.57s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 326/360 [44:59<04:42,  8.32s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 327/360 [45:11<05:09,  9.38s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 328/360 [45:19<04:44,  8.89s/it]\u001b[A\n",
            "Iteration:  91%|█████████▏| 329/360 [45:27<04:26,  8.59s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 330/360 [45:35<04:11,  8.38s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 331/360 [45:43<03:58,  8.24s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 332/360 [45:50<03:44,  8.00s/it]\u001b[A\n",
            "Iteration:  92%|█████████▎| 333/360 [46:01<03:56,  8.77s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 334/360 [46:08<03:35,  8.27s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 335/360 [46:15<03:22,  8.10s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 336/360 [46:24<03:19,  8.33s/it]\u001b[A\n",
            "Iteration:  94%|█████████▎| 337/360 [46:38<03:47,  9.89s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 338/360 [46:45<03:16,  8.94s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 339/360 [46:53<03:05,  8.82s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 340/360 [47:01<02:48,  8.44s/it]\u001b[A\n",
            "Iteration:  95%|█████████▍| 341/360 [47:10<02:48,  8.86s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 342/360 [47:19<02:36,  8.69s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 343/360 [47:27<02:23,  8.43s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 344/360 [47:35<02:12,  8.30s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 345/360 [47:44<02:07,  8.51s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 346/360 [47:49<01:46,  7.60s/it]\u001b[A\n",
            "Iteration:  96%|█████████▋| 347/360 [47:55<01:30,  6.95s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 348/360 [48:03<01:30,  7.55s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 349/360 [48:12<01:25,  7.75s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 350/360 [48:20<01:18,  7.81s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 351/360 [48:28<01:10,  7.86s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 352/360 [48:35<01:02,  7.76s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 353/360 [48:43<00:54,  7.72s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 354/360 [48:51<00:48,  8.01s/it]\u001b[A\n",
            "Iteration:  99%|█████████▊| 355/360 [48:59<00:38,  7.75s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 356/360 [49:07<00:32,  8.07s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 357/360 [49:16<00:24,  8.22s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 358/360 [49:26<00:17,  8.83s/it]\u001b[A\n",
            "Iteration: 100%|█████████▉| 359/360 [49:36<00:09,  9.07s/it]\u001b[A\n",
            "Iteration: 100%|██████████| 360/360 [49:40<00:00,  7.58s/it]\u001b[A\n",
            "Epoch:   0%|          | 0/4 [49:40<?, ?it/s]\n",
            "Convert Evaluating:   0%|          | 0/94 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 09:09:11 - Evaluation the model on  dataset after epoch 0:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Convert Evaluating:   1%|          | 1/94 [00:00<01:16,  1.22it/s]\u001b[A\n",
            "Convert Evaluating:   2%|▏         | 2/94 [00:01<01:13,  1.25it/s]\u001b[A\n",
            "Convert Evaluating:   3%|▎         | 3/94 [00:02<01:16,  1.20it/s]\u001b[A\n",
            "Convert Evaluating:   4%|▍         | 4/94 [00:03<01:14,  1.21it/s]\u001b[A\n",
            "Convert Evaluating:   5%|▌         | 5/94 [00:04<01:15,  1.17it/s]\u001b[A\n",
            "Convert Evaluating:   6%|▋         | 6/94 [00:05<01:21,  1.08it/s]\u001b[A\n",
            "Convert Evaluating:   7%|▋         | 7/94 [00:06<01:18,  1.10it/s]\u001b[A\n",
            "Convert Evaluating:   9%|▊         | 8/94 [00:07<01:16,  1.13it/s]\u001b[A\n",
            "Convert Evaluating:  10%|▉         | 9/94 [00:07<01:14,  1.14it/s]\u001b[A\n",
            "Convert Evaluating:  11%|█         | 10/94 [00:08<01:09,  1.21it/s]\u001b[A\n",
            "Convert Evaluating:  12%|█▏        | 11/94 [00:09<01:07,  1.22it/s]\u001b[A\n",
            "Convert Evaluating:  13%|█▎        | 12/94 [00:10<01:11,  1.15it/s]\u001b[A\n",
            "Convert Evaluating:  14%|█▍        | 13/94 [00:11<01:09,  1.17it/s]\u001b[A\n",
            "Convert Evaluating:  15%|█▍        | 14/94 [00:12<01:10,  1.13it/s]\u001b[A\n",
            "Convert Evaluating:  16%|█▌        | 15/94 [00:13<01:15,  1.05it/s]\u001b[A\n",
            "Convert Evaluating:  17%|█▋        | 16/94 [00:14<01:18,  1.00s/it]\u001b[A\n",
            "Convert Evaluating:  18%|█▊        | 17/94 [00:15<01:21,  1.06s/it]\u001b[A\n",
            "Convert Evaluating:  19%|█▉        | 18/94 [00:16<01:20,  1.06s/it]\u001b[A\n",
            "Convert Evaluating:  20%|██        | 19/94 [00:17<01:21,  1.09s/it]\u001b[A\n",
            "Convert Evaluating:  21%|██▏       | 20/94 [00:18<01:20,  1.08s/it]\u001b[A\n",
            "Convert Evaluating:  22%|██▏       | 21/94 [00:20<01:24,  1.16s/it]\u001b[A\n",
            "Convert Evaluating:  23%|██▎       | 22/94 [00:21<01:19,  1.11s/it]\u001b[A\n",
            "Convert Evaluating:  24%|██▍       | 23/94 [00:22<01:18,  1.11s/it]\u001b[A\n",
            "Convert Evaluating:  26%|██▌       | 24/94 [00:23<01:20,  1.16s/it]\u001b[A\n",
            "Convert Evaluating:  27%|██▋       | 25/94 [00:24<01:15,  1.09s/it]\u001b[A\n",
            "Convert Evaluating:  28%|██▊       | 26/94 [00:25<01:18,  1.15s/it]\u001b[A\n",
            "Convert Evaluating:  29%|██▊       | 27/94 [00:26<01:13,  1.09s/it]\u001b[A\n",
            "Convert Evaluating:  30%|██▉       | 28/94 [00:27<01:15,  1.14s/it]\u001b[A\n",
            "Convert Evaluating:  31%|███       | 29/94 [00:29<01:20,  1.23s/it]\u001b[A\n",
            "Convert Evaluating:  32%|███▏      | 30/94 [00:30<01:15,  1.18s/it]\u001b[A\n",
            "Convert Evaluating:  33%|███▎      | 31/94 [00:31<01:15,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  34%|███▍      | 32/94 [00:32<01:13,  1.18s/it]\u001b[A\n",
            "Convert Evaluating:  35%|███▌      | 33/94 [00:34<01:13,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  36%|███▌      | 34/94 [00:35<01:13,  1.23s/it]\u001b[A\n",
            "Convert Evaluating:  37%|███▋      | 35/94 [00:36<01:10,  1.19s/it]\u001b[A\n",
            "Convert Evaluating:  38%|███▊      | 36/94 [00:37<01:09,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  39%|███▉      | 37/94 [00:38<01:08,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  40%|████      | 38/94 [00:40<01:07,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  41%|████▏     | 39/94 [00:41<01:05,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  43%|████▎     | 40/94 [00:43<01:21,  1.51s/it]\u001b[A\n",
            "Convert Evaluating:  44%|████▎     | 41/94 [00:45<01:30,  1.70s/it]\u001b[A\n",
            "Convert Evaluating:  45%|████▍     | 42/94 [00:47<01:29,  1.72s/it]\u001b[A\n",
            "Convert Evaluating:  46%|████▌     | 43/94 [00:49<01:30,  1.78s/it]\u001b[A\n",
            "Convert Evaluating:  47%|████▋     | 44/94 [00:51<01:28,  1.76s/it]\u001b[A\n",
            "Convert Evaluating:  48%|████▊     | 45/94 [00:52<01:26,  1.78s/it]\u001b[A\n",
            "Convert Evaluating:  49%|████▉     | 46/94 [00:54<01:22,  1.71s/it]\u001b[A\n",
            "Convert Evaluating:  50%|█████     | 47/94 [00:56<01:21,  1.73s/it]\u001b[A\n",
            "Convert Evaluating:  51%|█████     | 48/94 [00:59<01:35,  2.07s/it]\u001b[A\n",
            "Convert Evaluating:  52%|█████▏    | 49/94 [01:01<01:30,  2.02s/it]\u001b[A\n",
            "Convert Evaluating:  53%|█████▎    | 50/94 [01:02<01:24,  1.92s/it]\u001b[A\n",
            "Convert Evaluating:  54%|█████▍    | 51/94 [01:04<01:18,  1.82s/it]\u001b[A\n",
            "Convert Evaluating:  55%|█████▌    | 52/94 [01:05<01:14,  1.77s/it]\u001b[A\n",
            "Convert Evaluating:  56%|█████▋    | 53/94 [01:07<01:11,  1.74s/it]\u001b[A\n",
            "Convert Evaluating:  57%|█████▋    | 54/94 [01:09<01:08,  1.71s/it]\u001b[A\n",
            "Convert Evaluating:  59%|█████▊    | 55/94 [01:11<01:10,  1.80s/it]\u001b[A\n",
            "Convert Evaluating:  60%|█████▉    | 56/94 [01:13<01:08,  1.81s/it]\u001b[A\n",
            "Convert Evaluating:  61%|██████    | 57/94 [01:14<01:05,  1.76s/it]\u001b[A\n",
            "Convert Evaluating:  62%|██████▏   | 58/94 [01:16<01:05,  1.83s/it]\u001b[A\n",
            "Convert Evaluating:  63%|██████▎   | 59/94 [01:18<01:02,  1.80s/it]\u001b[A\n",
            "Convert Evaluating:  64%|██████▍   | 60/94 [01:20<01:00,  1.77s/it]\u001b[A\n",
            "Convert Evaluating:  65%|██████▍   | 61/94 [01:21<00:57,  1.75s/it]\u001b[A\n",
            "Convert Evaluating:  66%|██████▌   | 62/94 [01:23<00:54,  1.71s/it]\u001b[A\n",
            "Convert Evaluating:  67%|██████▋   | 63/94 [01:25<00:54,  1.77s/it]\u001b[A\n",
            "Convert Evaluating:  68%|██████▊   | 64/94 [01:27<00:56,  1.87s/it]\u001b[A\n",
            "Convert Evaluating:  69%|██████▉   | 65/94 [01:29<00:57,  1.98s/it]\u001b[A\n",
            "Convert Evaluating:  70%|███████   | 66/94 [01:31<00:55,  1.98s/it]\u001b[A\n",
            "Convert Evaluating:  71%|███████▏  | 67/94 [01:33<00:54,  2.02s/it]\u001b[A\n",
            "Convert Evaluating:  72%|███████▏  | 68/94 [01:36<00:54,  2.09s/it]\u001b[A\n",
            "Convert Evaluating:  73%|███████▎  | 69/94 [01:38<00:52,  2.12s/it]\u001b[A\n",
            "Convert Evaluating:  74%|███████▍  | 70/94 [01:40<00:53,  2.23s/it]\u001b[A\n",
            "Convert Evaluating:  76%|███████▌  | 71/94 [01:42<00:49,  2.16s/it]\u001b[A\n",
            "Convert Evaluating:  77%|███████▋  | 72/94 [01:45<00:48,  2.20s/it]\u001b[A\n",
            "Convert Evaluating:  78%|███████▊  | 73/94 [01:47<00:48,  2.31s/it]\u001b[A\n",
            "Convert Evaluating:  79%|███████▊  | 74/94 [01:49<00:44,  2.22s/it]\u001b[A\n",
            "Convert Evaluating:  80%|███████▉  | 75/94 [01:51<00:41,  2.18s/it]\u001b[A\n",
            "Convert Evaluating:  81%|████████  | 76/94 [01:53<00:38,  2.15s/it]\u001b[A\n",
            "Convert Evaluating:  82%|████████▏ | 77/94 [01:56<00:37,  2.19s/it]\u001b[A\n",
            "Convert Evaluating:  83%|████████▎ | 78/94 [01:58<00:35,  2.20s/it]\u001b[A\n",
            "Convert Evaluating:  84%|████████▍ | 79/94 [01:59<00:30,  2.00s/it]\u001b[A\n",
            "Convert Evaluating:  85%|████████▌ | 80/94 [02:00<00:23,  1.71s/it]\u001b[A\n",
            "Convert Evaluating:  86%|████████▌ | 81/94 [02:01<00:19,  1.50s/it]\u001b[A\n",
            "Convert Evaluating:  87%|████████▋ | 82/94 [02:02<00:15,  1.32s/it]\u001b[A\n",
            "Convert Evaluating:  88%|████████▊ | 83/94 [02:03<00:13,  1.18s/it]\u001b[A\n",
            "Convert Evaluating:  89%|████████▉ | 84/94 [02:04<00:11,  1.14s/it]\u001b[A\n",
            "Convert Evaluating:  90%|█████████ | 85/94 [02:05<00:09,  1.10s/it]\u001b[A\n",
            "Convert Evaluating:  91%|█████████▏| 86/94 [02:06<00:08,  1.12s/it]\u001b[A\n",
            "Convert Evaluating:  93%|█████████▎| 87/94 [02:07<00:07,  1.10s/it]\u001b[A\n",
            "Convert Evaluating:  94%|█████████▎| 88/94 [02:08<00:06,  1.10s/it]\u001b[A\n",
            "Convert Evaluating:  95%|█████████▍| 89/94 [02:09<00:05,  1.05s/it]\u001b[A\n",
            "Convert Evaluating:  96%|█████████▌| 90/94 [02:10<00:04,  1.00s/it]\u001b[A\n",
            "Convert Evaluating:  97%|█████████▋| 91/94 [02:11<00:02,  1.03it/s]\u001b[A\n",
            "Convert Evaluating:  98%|█████████▊| 92/94 [02:12<00:02,  1.05s/it]\u001b[A\n",
            "Convert Evaluating:  99%|█████████▉| 93/94 [02:14<00:01,  1.05s/it]\u001b[A\n",
            "Convert Evaluating: 100%|██████████| 94/94 [02:14<00:00,  1.08it/s]\u001b[A\n",
            "Epoch:   0%|          | 0/4 [51:55<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 09:11:25 - Cosine-Similarity :\tPearson: 0.8651\tSpearman: 0.8670\n",
            "2019-11-28 09:11:25 - Manhattan-Distance:\tPearson: 0.8333\tSpearman: 0.8393\n",
            "2019-11-28 09:11:25 - Euclidean-Distance:\tPearson: 0.8341\tSpearman: 0.8399\n",
            "2019-11-28 09:11:25 - Dot-Product-Similarity:\tPearson: 0.7937\tSpearman: 0.7984\n",
            "2019-11-28 09:11:25 - Save model to output/training_stsbenchmark_bert-2019-11-28_08-19-18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch:  25%|██▌       | 1/4 [51:56<2:35:48, 3116.10s/it]\n",
            "\n",
            "Epoch:  25%|██▌       | 1/4 [51:56<2:35:48, 3116.10s/it]\n",
            "Iteration:   0%|          | 0/360 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 09:11:26 - Restart data_iterator\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration:   0%|          | 1/360 [00:06<37:49,  6.32s/it]\u001b[A\n",
            "Iteration:   1%|          | 2/360 [00:16<44:19,  7.43s/it]\u001b[A\n",
            "Iteration:   1%|          | 3/360 [00:24<45:55,  7.72s/it]\u001b[A\n",
            "Iteration:   1%|          | 4/360 [00:32<45:42,  7.70s/it]\u001b[A\n",
            "Iteration:   1%|▏         | 5/360 [00:40<47:03,  7.95s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 6/360 [00:49<48:10,  8.17s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 7/360 [00:57<48:17,  8.21s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 8/360 [01:06<48:30,  8.27s/it]\u001b[A\n",
            "Iteration:   2%|▎         | 9/360 [01:14<47:50,  8.18s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 10/360 [01:21<45:43,  7.84s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 11/360 [01:30<47:35,  8.18s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 12/360 [01:37<45:40,  7.88s/it]\u001b[A\n",
            "Iteration:   4%|▎         | 13/360 [01:44<43:13,  7.48s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 14/360 [01:51<42:16,  7.33s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 15/360 [01:58<41:57,  7.30s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 16/360 [02:05<41:07,  7.17s/it]\u001b[A\n",
            "Iteration:   5%|▍         | 17/360 [02:13<43:16,  7.57s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 18/360 [02:18<38:55,  6.83s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 19/360 [02:26<40:12,  7.08s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 20/360 [02:34<41:09,  7.26s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 21/360 [02:42<42:13,  7.47s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 22/360 [02:51<45:44,  8.12s/it]\u001b[A\n",
            "Iteration:   6%|▋         | 23/360 [02:59<44:45,  7.97s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 24/360 [03:06<43:36,  7.79s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 25/360 [03:14<43:18,  7.76s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 26/360 [03:21<42:32,  7.64s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 27/360 [03:30<44:32,  8.03s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 28/360 [03:39<45:45,  8.27s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 29/360 [03:46<44:02,  7.98s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 30/360 [03:56<45:58,  8.36s/it]\u001b[A\n",
            "Iteration:   9%|▊         | 31/360 [04:04<45:39,  8.33s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 32/360 [04:10<42:25,  7.76s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 33/360 [04:18<42:36,  7.82s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 34/360 [04:28<45:09,  8.31s/it]\u001b[A\n",
            "Iteration:  10%|▉         | 35/360 [04:35<42:48,  7.90s/it]\u001b[A\n",
            "Iteration:  10%|█         | 36/360 [04:42<41:37,  7.71s/it]\u001b[A\n",
            "Iteration:  10%|█         | 37/360 [04:51<43:15,  8.03s/it]\u001b[A\n",
            "Iteration:  11%|█         | 38/360 [04:57<40:07,  7.48s/it]\u001b[A\n",
            "Iteration:  11%|█         | 39/360 [05:04<39:48,  7.44s/it]\u001b[A\n",
            "Iteration:  11%|█         | 40/360 [05:13<41:26,  7.77s/it]\u001b[A\n",
            "Iteration:  11%|█▏        | 41/360 [05:21<42:39,  8.02s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 42/360 [05:30<43:22,  8.18s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 43/360 [05:37<42:14,  8.00s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 44/360 [05:44<40:16,  7.65s/it]\u001b[A\n",
            "Iteration:  12%|█▎        | 45/360 [05:53<42:08,  8.03s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 46/360 [06:01<42:04,  8.04s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 47/360 [06:10<42:54,  8.23s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 48/360 [06:15<38:26,  7.39s/it]\u001b[A\n",
            "Iteration:  14%|█▎        | 49/360 [06:23<38:11,  7.37s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 50/360 [06:30<38:42,  7.49s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 51/360 [06:38<39:16,  7.63s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 52/360 [06:52<47:48,  9.31s/it]\u001b[A\n",
            "Iteration:  15%|█▍        | 53/360 [07:00<46:50,  9.15s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 54/360 [07:09<45:11,  8.86s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 55/360 [07:18<46:07,  9.07s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 56/360 [07:26<43:36,  8.61s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 57/360 [07:34<43:41,  8.65s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 58/360 [07:42<42:30,  8.45s/it]\u001b[A\n",
            "Iteration:  16%|█▋        | 59/360 [07:52<43:45,  8.72s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 60/360 [07:58<40:05,  8.02s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 61/360 [08:07<40:51,  8.20s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 62/360 [08:12<36:40,  7.38s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 63/360 [08:19<36:22,  7.35s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 64/360 [08:29<38:43,  7.85s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 65/360 [08:36<38:15,  7.78s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 66/360 [08:47<42:24,  8.65s/it]\u001b[A\n",
            "Iteration:  19%|█▊        | 67/360 [08:57<44:13,  9.06s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 68/360 [09:06<43:54,  9.02s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 69/360 [09:13<41:05,  8.47s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 70/360 [09:21<40:00,  8.28s/it]\u001b[A\n",
            "Iteration:  20%|█▉        | 71/360 [09:29<39:08,  8.13s/it]\u001b[A\n",
            "Iteration:  20%|██        | 72/360 [09:36<38:10,  7.95s/it]\u001b[A\n",
            "Iteration:  20%|██        | 73/360 [09:45<39:01,  8.16s/it]\u001b[A\n",
            "Iteration:  21%|██        | 74/360 [09:53<39:38,  8.32s/it]\u001b[A\n",
            "Iteration:  21%|██        | 75/360 [10:00<36:19,  7.65s/it]\u001b[A\n",
            "Iteration:  21%|██        | 76/360 [10:09<38:58,  8.24s/it]\u001b[A\n",
            "Iteration:  21%|██▏       | 77/360 [10:17<38:56,  8.26s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 78/360 [10:26<38:45,  8.25s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 79/360 [10:31<35:08,  7.50s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 80/360 [10:40<36:40,  7.86s/it]\u001b[A\n",
            "Iteration:  22%|██▎       | 81/360 [10:50<39:03,  8.40s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 82/360 [10:58<38:14,  8.25s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 83/360 [11:06<38:10,  8.27s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 84/360 [11:14<38:22,  8.34s/it]\u001b[A\n",
            "Iteration:  24%|██▎       | 85/360 [11:23<38:10,  8.33s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 86/360 [11:30<36:24,  7.97s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 87/360 [11:38<36:43,  8.07s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 88/360 [11:46<36:36,  8.07s/it]\u001b[A\n",
            "Iteration:  25%|██▍       | 89/360 [11:58<41:05,  9.10s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 90/360 [12:05<37:48,  8.40s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 91/360 [12:12<36:33,  8.15s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 92/360 [12:20<35:26,  7.93s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 93/360 [12:25<32:15,  7.25s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 94/360 [12:33<32:56,  7.43s/it]\u001b[A\n",
            "Iteration:  26%|██▋       | 95/360 [12:41<34:06,  7.72s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 96/360 [12:49<33:06,  7.52s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 97/360 [12:57<34:15,  7.82s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 98/360 [13:03<31:41,  7.26s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 99/360 [13:12<34:27,  7.92s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 100/360 [13:21<35:33,  8.20s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 101/360 [13:29<35:14,  8.16s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 102/360 [13:35<32:07,  7.47s/it]\u001b[A\n",
            "Iteration:  29%|██▊       | 103/360 [13:43<32:40,  7.63s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 104/360 [13:53<35:09,  8.24s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 105/360 [14:00<34:07,  8.03s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 106/360 [14:09<34:43,  8.20s/it]\u001b[A\n",
            "Iteration:  30%|██▉       | 107/360 [14:17<34:39,  8.22s/it]\u001b[A\n",
            "Iteration:  30%|███       | 108/360 [14:27<36:03,  8.59s/it]\u001b[A\n",
            "Iteration:  30%|███       | 109/360 [14:34<34:16,  8.19s/it]\u001b[A\n",
            "Iteration:  31%|███       | 110/360 [14:42<34:24,  8.26s/it]\u001b[A\n",
            "Iteration:  31%|███       | 111/360 [14:51<34:08,  8.23s/it]\u001b[A\n",
            "Iteration:  31%|███       | 112/360 [15:02<37:53,  9.17s/it]\u001b[A\n",
            "Iteration:  31%|███▏      | 113/360 [15:11<37:49,  9.19s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 114/360 [15:20<37:00,  9.03s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 115/360 [15:28<35:15,  8.63s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 116/360 [15:37<36:21,  8.94s/it]\u001b[A\n",
            "Iteration:  32%|███▎      | 117/360 [15:44<33:58,  8.39s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 118/360 [15:52<33:35,  8.33s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 119/360 [16:00<32:41,  8.14s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 120/360 [16:10<35:00,  8.75s/it]\u001b[A\n",
            "Iteration:  34%|███▎      | 121/360 [16:20<35:48,  8.99s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 122/360 [16:28<34:01,  8.58s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 123/360 [16:37<34:40,  8.78s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 124/360 [16:43<31:29,  8.00s/it]\u001b[A\n",
            "Iteration:  35%|███▍      | 125/360 [16:52<32:05,  8.19s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 126/360 [17:01<33:46,  8.66s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 127/360 [17:13<36:52,  9.49s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 128/360 [17:21<35:31,  9.19s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 129/360 [17:29<34:06,  8.86s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 130/360 [17:38<33:10,  8.65s/it]\u001b[A\n",
            "Iteration:  36%|███▋      | 131/360 [17:48<35:15,  9.24s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 132/360 [17:58<35:51,  9.44s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 133/360 [18:08<36:25,  9.63s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 134/360 [18:18<36:09,  9.60s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 135/360 [18:26<34:08,  9.10s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 136/360 [18:34<32:56,  8.82s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 137/360 [18:42<31:48,  8.56s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 138/360 [18:53<34:28,  9.32s/it]\u001b[A\n",
            "Iteration:  39%|███▊      | 139/360 [19:02<34:12,  9.29s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 140/360 [19:12<35:06,  9.58s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 141/360 [19:22<34:42,  9.51s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 142/360 [19:31<34:34,  9.51s/it]\u001b[A\n",
            "Iteration:  40%|███▉      | 143/360 [19:39<32:17,  8.93s/it]\u001b[A\n",
            "Iteration:  40%|████      | 144/360 [19:48<32:22,  8.99s/it]\u001b[A\n",
            "Iteration:  40%|████      | 145/360 [19:58<33:26,  9.33s/it]\u001b[A\n",
            "Iteration:  41%|████      | 146/360 [20:08<34:05,  9.56s/it]\u001b[A\n",
            "Iteration:  41%|████      | 147/360 [20:17<32:49,  9.25s/it]\u001b[A\n",
            "Iteration:  41%|████      | 148/360 [20:27<33:33,  9.50s/it]\u001b[A\n",
            "Iteration:  41%|████▏     | 149/360 [20:35<32:22,  9.20s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 150/360 [20:44<32:17,  9.23s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 151/360 [20:54<32:25,  9.31s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 152/360 [21:03<32:13,  9.29s/it]\u001b[A\n",
            "Iteration:  42%|████▎     | 153/360 [21:12<31:50,  9.23s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 154/360 [21:21<31:09,  9.08s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 155/360 [21:30<31:21,  9.18s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 156/360 [21:40<31:31,  9.27s/it]\u001b[A\n",
            "Iteration:  44%|████▎     | 157/360 [21:49<30:47,  9.10s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 158/360 [21:58<30:44,  9.13s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 159/360 [22:10<33:16,  9.93s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 160/360 [22:20<33:49, 10.15s/it]\u001b[A\n",
            "Iteration:  45%|████▍     | 161/360 [22:26<29:39,  8.94s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 162/360 [22:34<28:04,  8.51s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 163/360 [22:43<28:08,  8.57s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 164/360 [22:52<28:50,  8.83s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 165/360 [23:02<29:43,  9.14s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 166/360 [23:13<31:18,  9.68s/it]\u001b[A\n",
            "Iteration:  46%|████▋     | 167/360 [23:22<30:21,  9.44s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 168/360 [23:30<28:41,  8.97s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 169/360 [23:37<27:25,  8.62s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 170/360 [23:47<28:20,  8.95s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 171/360 [23:57<28:49,  9.15s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 172/360 [24:04<27:20,  8.73s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 173/360 [24:15<28:45,  9.23s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 174/360 [24:24<28:36,  9.23s/it]\u001b[A\n",
            "Iteration:  49%|████▊     | 175/360 [24:33<28:04,  9.10s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 176/360 [24:42<27:42,  9.03s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 177/360 [24:53<29:35,  9.70s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 178/360 [25:02<29:11,  9.62s/it]\u001b[A\n",
            "Iteration:  50%|████▉     | 179/360 [25:11<28:22,  9.40s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 180/360 [25:20<27:37,  9.21s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 181/360 [25:30<28:03,  9.40s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 182/360 [25:40<28:20,  9.55s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 183/360 [25:49<27:26,  9.30s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 184/360 [25:59<28:28,  9.71s/it]\u001b[A\n",
            "Iteration:  51%|█████▏    | 185/360 [26:09<28:45,  9.86s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 186/360 [26:18<27:39,  9.54s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 187/360 [26:28<27:51,  9.66s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 188/360 [26:35<25:36,  8.93s/it]\u001b[A\n",
            "Iteration:  52%|█████▎    | 189/360 [26:44<24:57,  8.76s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 190/360 [26:54<26:21,  9.30s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 191/360 [27:02<24:43,  8.78s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 192/360 [27:13<26:39,  9.52s/it]\u001b[A\n",
            "Iteration:  54%|█████▎    | 193/360 [27:22<26:04,  9.37s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 194/360 [27:32<26:19,  9.51s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 195/360 [27:41<25:37,  9.32s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 196/360 [27:49<24:28,  8.96s/it]\u001b[A\n",
            "Iteration:  55%|█████▍    | 197/360 [28:00<25:47,  9.50s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 198/360 [28:09<25:46,  9.55s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 199/360 [28:18<25:12,  9.39s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 200/360 [28:25<22:28,  8.43s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 201/360 [28:33<22:31,  8.50s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 202/360 [28:43<23:20,  8.86s/it]\u001b[A\n",
            "Iteration:  56%|█████▋    | 203/360 [28:51<22:10,  8.48s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 204/360 [28:59<21:36,  8.31s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 205/360 [29:07<21:48,  8.44s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 206/360 [29:16<21:34,  8.41s/it]\u001b[A\n",
            "Iteration:  57%|█████▊    | 207/360 [29:24<21:38,  8.48s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 208/360 [29:32<20:49,  8.22s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 209/360 [29:41<21:37,  8.59s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 210/360 [29:50<21:47,  8.72s/it]\u001b[A\n",
            "Iteration:  59%|█████▊    | 211/360 [29:59<21:45,  8.76s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 212/360 [30:09<22:09,  8.98s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 213/360 [30:18<22:32,  9.20s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 214/360 [30:25<20:09,  8.29s/it]\u001b[A\n",
            "Iteration:  60%|█████▉    | 215/360 [30:33<19:53,  8.23s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 216/360 [30:43<21:08,  8.81s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 217/360 [30:53<21:54,  9.19s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 218/360 [31:00<20:06,  8.50s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 219/360 [31:09<20:21,  8.66s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 220/360 [31:19<20:57,  8.98s/it]\u001b[A\n",
            "Iteration:  61%|██████▏   | 221/360 [31:29<21:45,  9.39s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 222/360 [31:38<21:05,  9.17s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 223/360 [31:47<20:51,  9.14s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 224/360 [31:56<21:04,  9.30s/it]\u001b[A\n",
            "Iteration:  62%|██████▎   | 225/360 [32:05<20:12,  8.98s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 226/360 [32:16<21:36,  9.68s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 227/360 [32:25<21:00,  9.48s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 228/360 [32:34<20:47,  9.45s/it]\u001b[A\n",
            "Iteration:  64%|██████▎   | 229/360 [32:44<20:44,  9.50s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 230/360 [32:55<21:26,  9.90s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 231/360 [33:04<21:02,  9.78s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 232/360 [33:13<20:04,  9.41s/it]\u001b[A\n",
            "Iteration:  65%|██████▍   | 233/360 [33:22<19:52,  9.39s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 234/360 [33:31<19:20,  9.21s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 235/360 [33:39<18:31,  8.89s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 236/360 [33:49<18:55,  9.16s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 237/360 [33:57<18:17,  8.93s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 238/360 [34:07<18:23,  9.05s/it]\u001b[A\n",
            "Iteration:  66%|██████▋   | 239/360 [34:16<18:15,  9.05s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 240/360 [34:26<18:47,  9.40s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 241/360 [34:37<19:38,  9.90s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 242/360 [34:47<19:28,  9.91s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 243/360 [34:54<18:00,  9.23s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 244/360 [35:03<17:18,  8.95s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 245/360 [35:09<15:39,  8.17s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 246/360 [35:18<15:46,  8.30s/it]\u001b[A\n",
            "Iteration:  69%|██████▊   | 247/360 [35:26<15:44,  8.36s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 248/360 [35:35<15:57,  8.55s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 249/360 [35:46<16:58,  9.18s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 250/360 [35:53<15:40,  8.55s/it]\u001b[A\n",
            "Iteration:  70%|██████▉   | 251/360 [36:00<14:53,  8.20s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 252/360 [36:10<15:28,  8.60s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 253/360 [36:21<16:41,  9.36s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 254/360 [36:30<16:35,  9.39s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 255/360 [36:38<15:25,  8.81s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 256/360 [36:44<14:04,  8.12s/it]\u001b[A\n",
            "Iteration:  71%|███████▏  | 257/360 [36:56<15:29,  9.02s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 258/360 [37:02<14:06,  8.30s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 259/360 [37:08<12:55,  7.68s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 260/360 [37:17<13:05,  7.86s/it]\u001b[A\n",
            "Iteration:  72%|███████▎  | 261/360 [37:27<14:03,  8.52s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 262/360 [37:38<15:31,  9.50s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 263/360 [37:49<15:43,  9.73s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 264/360 [37:59<15:48,  9.88s/it]\u001b[A\n",
            "Iteration:  74%|███████▎  | 265/360 [38:06<14:03,  8.87s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 266/360 [38:13<13:17,  8.49s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 267/360 [38:20<12:12,  7.88s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 268/360 [38:29<12:47,  8.34s/it]\u001b[A\n",
            "Iteration:  75%|███████▍  | 269/360 [38:38<12:53,  8.50s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 270/360 [38:49<13:55,  9.28s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 271/360 [38:58<13:48,  9.31s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 272/360 [39:06<12:43,  8.68s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 273/360 [39:14<12:28,  8.61s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 274/360 [39:24<12:47,  8.92s/it]\u001b[A\n",
            "Iteration:  76%|███████▋  | 275/360 [39:32<12:27,  8.79s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 276/360 [39:41<12:09,  8.68s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 277/360 [39:51<12:44,  9.21s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 278/360 [40:02<13:24,  9.82s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 279/360 [40:10<12:31,  9.28s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 280/360 [40:21<13:01,  9.77s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 281/360 [40:31<13:02,  9.91s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 282/360 [40:43<13:31, 10.41s/it]\u001b[A\n",
            "Iteration:  79%|███████▊  | 283/360 [40:54<13:38, 10.62s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 284/360 [41:07<14:14, 11.25s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 285/360 [41:16<13:18, 10.65s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 286/360 [41:27<13:03, 10.59s/it]\u001b[A\n",
            "Iteration:  80%|███████▉  | 287/360 [41:36<12:24, 10.20s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 288/360 [41:46<12:10, 10.14s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 289/360 [41:52<10:36,  8.97s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 290/360 [42:03<11:09,  9.56s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 291/360 [42:11<10:21,  9.01s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 292/360 [42:20<10:20,  9.12s/it]\u001b[A\n",
            "Iteration:  81%|████████▏ | 293/360 [42:29<10:15,  9.19s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 294/360 [42:39<10:10,  9.25s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 295/360 [42:49<10:21,  9.56s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 296/360 [43:00<10:29,  9.84s/it]\u001b[A\n",
            "Iteration:  82%|████████▎ | 297/360 [43:09<10:10,  9.69s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 298/360 [43:22<10:55, 10.58s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 299/360 [43:30<10:11, 10.02s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 300/360 [43:38<09:18,  9.31s/it]\u001b[A\n",
            "Iteration:  84%|████████▎ | 301/360 [43:47<08:57,  9.11s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 302/360 [43:56<08:54,  9.22s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 303/360 [44:05<08:34,  9.02s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 304/360 [44:14<08:28,  9.08s/it]\u001b[A\n",
            "Iteration:  85%|████████▍ | 305/360 [44:23<08:17,  9.05s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 306/360 [44:33<08:21,  9.29s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 307/360 [44:42<08:17,  9.38s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 308/360 [44:52<08:10,  9.44s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 309/360 [45:02<08:16,  9.74s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 310/360 [45:11<07:53,  9.47s/it]\u001b[A\n",
            "Iteration:  86%|████████▋ | 311/360 [45:20<07:27,  9.14s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 312/360 [45:30<07:38,  9.56s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 313/360 [45:38<07:12,  9.21s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 314/360 [45:48<07:05,  9.26s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 315/360 [45:56<06:48,  9.07s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 316/360 [46:07<07:04,  9.65s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 317/360 [46:16<06:34,  9.18s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 318/360 [46:27<06:53,  9.84s/it]\u001b[A\n",
            "Iteration:  89%|████████▊ | 319/360 [46:38<06:55, 10.13s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 320/360 [46:44<06:02,  9.07s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 321/360 [46:53<05:54,  9.08s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 322/360 [47:03<05:52,  9.27s/it]\u001b[A\n",
            "Iteration:  90%|████████▉ | 323/360 [47:15<06:11, 10.04s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 324/360 [47:26<06:15, 10.42s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 325/360 [47:33<05:21,  9.19s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 326/360 [47:42<05:10,  9.15s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 327/360 [47:52<05:14,  9.54s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 328/360 [47:59<04:43,  8.85s/it]\u001b[A\n",
            "Iteration:  91%|█████████▏| 329/360 [48:08<04:36,  8.91s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 330/360 [48:18<04:37,  9.25s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 331/360 [48:28<04:27,  9.21s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 332/360 [48:38<04:30,  9.65s/it]\u001b[A\n",
            "Iteration:  92%|█████████▎| 333/360 [48:49<04:25,  9.84s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 334/360 [48:55<03:47,  8.75s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 335/360 [49:06<03:55,  9.42s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 336/360 [49:15<03:45,  9.41s/it]\u001b[A\n",
            "Iteration:  94%|█████████▎| 337/360 [49:25<03:38,  9.49s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 338/360 [49:35<03:32,  9.67s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 339/360 [49:44<03:22,  9.64s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 340/360 [49:53<03:05,  9.29s/it]\u001b[A\n",
            "Iteration:  95%|█████████▍| 341/360 [50:01<02:47,  8.82s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 342/360 [50:08<02:31,  8.44s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 343/360 [50:19<02:34,  9.07s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 344/360 [50:26<02:16,  8.53s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 345/360 [50:35<02:10,  8.73s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 346/360 [50:45<02:07,  9.12s/it]\u001b[A\n",
            "Iteration:  96%|█████████▋| 347/360 [50:56<02:04,  9.59s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 348/360 [51:04<01:49,  9.09s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 349/360 [51:13<01:41,  9.22s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 350/360 [51:22<01:31,  9.19s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 351/360 [51:31<01:21,  9.06s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 352/360 [51:41<01:13,  9.22s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 353/360 [51:49<01:01,  8.86s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 354/360 [51:59<00:56,  9.34s/it]\u001b[A\n",
            "Iteration:  99%|█████████▊| 355/360 [52:07<00:43,  8.77s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 356/360 [52:15<00:34,  8.71s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 357/360 [52:25<00:26,  8.95s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 358/360 [52:33<00:17,  8.87s/it]\u001b[A\n",
            "Iteration: 100%|█████████▉| 359/360 [52:41<00:08,  8.50s/it]\u001b[A\n",
            "Iteration: 100%|██████████| 360/360 [52:45<00:00,  7.15s/it]\u001b[A\n",
            "Epoch:  25%|██▌       | 1/4 [1:44:41<2:35:48, 3116.10s/it]\n",
            "Convert Evaluating:   0%|          | 0/94 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 10:04:12 - Evaluation the model on  dataset after epoch 1:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Convert Evaluating:   1%|          | 1/94 [00:00<01:27,  1.06it/s]\u001b[A\n",
            "Convert Evaluating:   2%|▏         | 2/94 [00:01<01:24,  1.09it/s]\u001b[A\n",
            "Convert Evaluating:   3%|▎         | 3/94 [00:02<01:28,  1.03it/s]\u001b[A\n",
            "Convert Evaluating:   4%|▍         | 4/94 [00:03<01:27,  1.03it/s]\u001b[A\n",
            "Convert Evaluating:   5%|▌         | 5/94 [00:04<01:29,  1.00s/it]\u001b[A\n",
            "Convert Evaluating:   6%|▋         | 6/94 [00:06<01:35,  1.09s/it]\u001b[A\n",
            "Convert Evaluating:   7%|▋         | 7/94 [00:07<01:34,  1.08s/it]\u001b[A\n",
            "Convert Evaluating:   9%|▊         | 8/94 [00:08<01:31,  1.06s/it]\u001b[A\n",
            "Convert Evaluating:  10%|▉         | 9/94 [00:09<01:29,  1.05s/it]\u001b[A\n",
            "Convert Evaluating:  11%|█         | 10/94 [00:10<01:22,  1.02it/s]\u001b[A\n",
            "Convert Evaluating:  12%|█▏        | 11/94 [00:11<01:21,  1.02it/s]\u001b[A\n",
            "Convert Evaluating:  13%|█▎        | 12/94 [00:12<01:25,  1.04s/it]\u001b[A\n",
            "Convert Evaluating:  14%|█▍        | 13/94 [00:13<01:23,  1.02s/it]\u001b[A\n",
            "Convert Evaluating:  15%|█▍        | 14/94 [00:14<01:22,  1.04s/it]\u001b[A\n",
            "Convert Evaluating:  16%|█▌        | 15/94 [00:15<01:28,  1.12s/it]\u001b[A\n",
            "Convert Evaluating:  17%|█▋        | 16/94 [00:17<01:32,  1.18s/it]\u001b[A\n",
            "Convert Evaluating:  18%|█▊        | 17/94 [00:18<01:35,  1.24s/it]\u001b[A\n",
            "Convert Evaluating:  19%|█▉        | 18/94 [00:19<01:33,  1.23s/it]\u001b[A\n",
            "Convert Evaluating:  20%|██        | 19/94 [00:20<01:34,  1.26s/it]\u001b[A\n",
            "Convert Evaluating:  21%|██▏       | 20/94 [00:22<01:33,  1.26s/it]\u001b[A\n",
            "Convert Evaluating:  22%|██▏       | 21/94 [00:23<01:36,  1.33s/it]\u001b[A\n",
            "Convert Evaluating:  23%|██▎       | 22/94 [00:24<01:32,  1.28s/it]\u001b[A\n",
            "Convert Evaluating:  24%|██▍       | 23/94 [00:26<01:31,  1.29s/it]\u001b[A\n",
            "Convert Evaluating:  26%|██▌       | 24/94 [00:27<01:35,  1.37s/it]\u001b[A\n",
            "Convert Evaluating:  27%|██▋       | 25/94 [00:28<01:31,  1.32s/it]\u001b[A\n",
            "Convert Evaluating:  28%|██▊       | 26/94 [00:30<01:34,  1.39s/it]\u001b[A\n",
            "Convert Evaluating:  29%|██▊       | 27/94 [00:31<01:27,  1.31s/it]\u001b[A\n",
            "Convert Evaluating:  30%|██▉       | 28/94 [00:33<01:29,  1.35s/it]\u001b[A\n",
            "Convert Evaluating:  31%|███       | 29/94 [00:34<01:33,  1.43s/it]\u001b[A\n",
            "Convert Evaluating:  32%|███▏      | 30/94 [00:35<01:28,  1.38s/it]\u001b[A\n",
            "Convert Evaluating:  33%|███▎      | 31/94 [00:37<01:29,  1.42s/it]\u001b[A\n",
            "Convert Evaluating:  34%|███▍      | 32/94 [00:38<01:26,  1.39s/it]\u001b[A\n",
            "Convert Evaluating:  35%|███▌      | 33/94 [00:40<01:26,  1.41s/it]\u001b[A\n",
            "Convert Evaluating:  36%|███▌      | 34/94 [00:41<01:26,  1.44s/it]\u001b[A\n",
            "Convert Evaluating:  37%|███▋      | 35/94 [00:42<01:22,  1.39s/it]\u001b[A\n",
            "Convert Evaluating:  38%|███▊      | 36/94 [00:44<01:21,  1.41s/it]\u001b[A\n",
            "Convert Evaluating:  39%|███▉      | 37/94 [00:45<01:19,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  40%|████      | 38/94 [00:47<01:18,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  41%|████▏     | 39/94 [00:48<01:17,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  43%|████▎     | 40/94 [00:51<01:35,  1.76s/it]\u001b[A\n",
            "Convert Evaluating:  44%|████▎     | 41/94 [00:53<01:44,  1.97s/it]\u001b[A\n",
            "Convert Evaluating:  45%|████▍     | 42/94 [00:55<01:43,  1.98s/it]\u001b[A\n",
            "Convert Evaluating:  46%|████▌     | 43/94 [00:58<01:46,  2.08s/it]\u001b[A\n",
            "Convert Evaluating:  47%|████▋     | 44/94 [01:00<01:44,  2.08s/it]\u001b[A\n",
            "Convert Evaluating:  48%|████▊     | 45/94 [01:02<01:43,  2.11s/it]\u001b[A\n",
            "Convert Evaluating:  49%|████▉     | 46/94 [01:04<01:38,  2.05s/it]\u001b[A\n",
            "Convert Evaluating:  50%|█████     | 47/94 [01:06<01:38,  2.09s/it]\u001b[A\n",
            "Convert Evaluating:  51%|█████     | 48/94 [01:09<01:53,  2.46s/it]\u001b[A\n",
            "Convert Evaluating:  52%|█████▏    | 49/94 [01:12<01:49,  2.43s/it]\u001b[A\n",
            "Convert Evaluating:  53%|█████▎    | 50/94 [01:14<01:41,  2.31s/it]\u001b[A\n",
            "Convert Evaluating:  54%|█████▍    | 51/94 [01:16<01:34,  2.20s/it]\u001b[A\n",
            "Convert Evaluating:  55%|█████▌    | 52/94 [01:17<01:29,  2.13s/it]\u001b[A\n",
            "Convert Evaluating:  56%|█████▋    | 53/94 [01:20<01:26,  2.11s/it]\u001b[A\n",
            "Convert Evaluating:  57%|█████▋    | 54/94 [01:21<01:22,  2.05s/it]\u001b[A\n",
            "Convert Evaluating:  59%|█████▊    | 55/94 [01:24<01:24,  2.17s/it]\u001b[A\n",
            "Convert Evaluating:  60%|█████▉    | 56/94 [01:26<01:22,  2.18s/it]\u001b[A\n",
            "Convert Evaluating:  61%|██████    | 57/94 [01:28<01:19,  2.15s/it]\u001b[A\n",
            "Convert Evaluating:  62%|██████▏   | 58/94 [01:30<01:18,  2.19s/it]\u001b[A\n",
            "Convert Evaluating:  63%|██████▎   | 59/94 [01:33<01:15,  2.16s/it]\u001b[A\n",
            "Convert Evaluating:  64%|██████▍   | 60/94 [01:35<01:11,  2.11s/it]\u001b[A\n",
            "Convert Evaluating:  65%|██████▍   | 61/94 [01:37<01:09,  2.10s/it]\u001b[A\n",
            "Convert Evaluating:  66%|██████▌   | 62/94 [01:39<01:05,  2.06s/it]\u001b[A\n",
            "Convert Evaluating:  67%|██████▋   | 63/94 [01:41<01:05,  2.12s/it]\u001b[A\n",
            "Convert Evaluating:  68%|██████▊   | 64/94 [01:43<01:07,  2.25s/it]\u001b[A\n",
            "Convert Evaluating:  69%|██████▉   | 65/94 [01:46<01:08,  2.35s/it]\u001b[A\n",
            "Convert Evaluating:  70%|███████   | 66/94 [01:48<01:06,  2.36s/it]\u001b[A\n",
            "Convert Evaluating:  71%|███████▏  | 67/94 [01:51<01:04,  2.39s/it]\u001b[A\n",
            "Convert Evaluating:  72%|███████▏  | 68/94 [01:53<01:04,  2.46s/it]\u001b[A\n",
            "Convert Evaluating:  73%|███████▎  | 69/94 [01:56<01:02,  2.50s/it]\u001b[A\n",
            "Convert Evaluating:  74%|███████▍  | 70/94 [01:59<01:03,  2.65s/it]\u001b[A\n",
            "Convert Evaluating:  76%|███████▌  | 71/94 [02:01<00:58,  2.56s/it]\u001b[A\n",
            "Convert Evaluating:  77%|███████▋  | 72/94 [02:04<00:57,  2.59s/it]\u001b[A\n",
            "Convert Evaluating:  78%|███████▊  | 73/94 [02:07<00:57,  2.72s/it]\u001b[A\n",
            "Convert Evaluating:  79%|███████▊  | 74/94 [02:10<00:52,  2.63s/it]\u001b[A\n",
            "Convert Evaluating:  80%|███████▉  | 75/94 [02:12<00:49,  2.60s/it]\u001b[A\n",
            "Convert Evaluating:  81%|████████  | 76/94 [02:15<00:46,  2.57s/it]\u001b[A\n",
            "Convert Evaluating:  82%|████████▏ | 77/94 [02:17<00:44,  2.63s/it]\u001b[A\n",
            "Convert Evaluating:  83%|████████▎ | 78/94 [02:20<00:42,  2.63s/it]\u001b[A\n",
            "Convert Evaluating:  84%|████████▍ | 79/94 [02:22<00:35,  2.36s/it]\u001b[A\n",
            "Convert Evaluating:  85%|████████▌ | 80/94 [02:23<00:28,  2.00s/it]\u001b[A\n",
            "Convert Evaluating:  86%|████████▌ | 81/94 [02:24<00:22,  1.76s/it]\u001b[A\n",
            "Convert Evaluating:  87%|████████▋ | 82/94 [02:25<00:18,  1.55s/it]\u001b[A\n",
            "Convert Evaluating:  88%|████████▊ | 83/94 [02:26<00:15,  1.39s/it]\u001b[A\n",
            "Convert Evaluating:  89%|████████▉ | 84/94 [02:27<00:13,  1.35s/it]\u001b[A\n",
            "Convert Evaluating:  90%|█████████ | 85/94 [02:28<00:11,  1.28s/it]\u001b[A\n",
            "Convert Evaluating:  91%|█████████▏| 86/94 [02:30<00:10,  1.31s/it]\u001b[A\n",
            "Convert Evaluating:  93%|█████████▎| 87/94 [02:31<00:09,  1.30s/it]\u001b[A\n",
            "Convert Evaluating:  94%|█████████▎| 88/94 [02:32<00:07,  1.29s/it]\u001b[A\n",
            "Convert Evaluating:  95%|█████████▍| 89/94 [02:34<00:06,  1.25s/it]\u001b[A\n",
            "Convert Evaluating:  96%|█████████▌| 90/94 [02:35<00:04,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  97%|█████████▋| 91/94 [02:36<00:03,  1.17s/it]\u001b[A\n",
            "Convert Evaluating:  98%|█████████▊| 92/94 [02:37<00:02,  1.27s/it]\u001b[A\n",
            "Convert Evaluating:  99%|█████████▉| 93/94 [02:39<00:01,  1.28s/it]\u001b[A\n",
            "Convert Evaluating: 100%|██████████| 94/94 [02:39<00:00,  1.13s/it]\u001b[A\n",
            "Epoch:  25%|██▌       | 1/4 [1:47:21<2:35:48, 3116.10s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 10:06:52 - Cosine-Similarity :\tPearson: 0.8702\tSpearman: 0.8705\n",
            "2019-11-28 10:06:52 - Manhattan-Distance:\tPearson: 0.8398\tSpearman: 0.8431\n",
            "2019-11-28 10:06:52 - Euclidean-Distance:\tPearson: 0.8402\tSpearman: 0.8432\n",
            "2019-11-28 10:06:52 - Dot-Product-Similarity:\tPearson: 0.8089\tSpearman: 0.8139\n",
            "2019-11-28 10:06:52 - Save model to output/training_stsbenchmark_bert-2019-11-28_08-19-18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch:  50%|█████     | 2/4 [1:47:22<1:45:58, 3179.25s/it]\n",
            "\n",
            "Epoch:  50%|█████     | 2/4 [1:47:22<1:45:58, 3179.25s/it]\n",
            "Iteration:   0%|          | 0/360 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 10:06:53 - Restart data_iterator\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration:   0%|          | 1/360 [00:10<1:04:57, 10.86s/it]\u001b[A\n",
            "Iteration:   1%|          | 2/360 [00:19<1:00:59, 10.22s/it]\u001b[A\n",
            "Iteration:   1%|          | 3/360 [00:28<57:36,  9.68s/it]  \u001b[A\n",
            "Iteration:   1%|          | 4/360 [00:37<57:17,  9.66s/it]\u001b[A\n",
            "Iteration:   1%|▏         | 5/360 [00:45<54:19,  9.18s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 6/360 [00:55<54:48,  9.29s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 7/360 [01:04<54:02,  9.18s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 8/360 [01:12<52:26,  8.94s/it]\u001b[A\n",
            "Iteration:   2%|▎         | 9/360 [01:22<53:47,  9.20s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 10/360 [01:31<52:57,  9.08s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 11/360 [01:38<49:30,  8.51s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 12/360 [01:48<51:32,  8.89s/it]\u001b[A\n",
            "Iteration:   4%|▎         | 13/360 [01:55<48:38,  8.41s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 14/360 [02:06<53:32,  9.28s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 15/360 [02:15<52:47,  9.18s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 16/360 [02:22<49:24,  8.62s/it]\u001b[A\n",
            "Iteration:   5%|▍         | 17/360 [02:29<46:00,  8.05s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 18/360 [02:40<51:14,  8.99s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 19/360 [02:49<51:12,  9.01s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 20/360 [02:59<52:34,  9.28s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 21/360 [03:10<55:37,  9.85s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 22/360 [03:22<57:59, 10.30s/it]\u001b[A\n",
            "Iteration:   6%|▋         | 23/360 [03:33<58:43, 10.46s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 24/360 [03:39<52:04,  9.30s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 25/360 [03:47<48:30,  8.69s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 26/360 [03:55<48:37,  8.74s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 27/360 [04:03<45:50,  8.26s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 28/360 [04:11<45:29,  8.22s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 29/360 [04:21<48:24,  8.77s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 30/360 [04:30<48:52,  8.89s/it]\u001b[A\n",
            "Iteration:   9%|▊         | 31/360 [04:39<49:13,  8.98s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 32/360 [04:48<48:18,  8.84s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 33/360 [04:58<50:40,  9.30s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 34/360 [05:06<48:57,  9.01s/it]\u001b[A\n",
            "Iteration:  10%|▉         | 35/360 [05:16<50:23,  9.30s/it]\u001b[A\n",
            "Iteration:  10%|█         | 36/360 [05:24<48:07,  8.91s/it]\u001b[A\n",
            "Iteration:  10%|█         | 37/360 [05:34<49:10,  9.14s/it]\u001b[A\n",
            "Iteration:  11%|█         | 38/360 [05:44<50:07,  9.34s/it]\u001b[A\n",
            "Iteration:  11%|█         | 39/360 [05:52<47:29,  8.88s/it]\u001b[A\n",
            "Iteration:  11%|█         | 40/360 [05:59<45:34,  8.55s/it]\u001b[A\n",
            "Iteration:  11%|█▏        | 41/360 [06:09<47:56,  9.02s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 42/360 [06:18<47:25,  8.95s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 43/360 [06:31<52:53, 10.01s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 44/360 [06:39<50:22,  9.56s/it]\u001b[A\n",
            "Iteration:  12%|█▎        | 45/360 [06:49<50:05,  9.54s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 46/360 [06:57<47:37,  9.10s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 47/360 [07:08<51:04,  9.79s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 48/360 [07:21<54:58, 10.57s/it]\u001b[A\n",
            "Iteration:  14%|█▎        | 49/360 [07:31<54:23, 10.49s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 50/360 [07:38<49:37,  9.60s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 51/360 [07:46<45:57,  8.92s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 52/360 [07:55<46:37,  9.08s/it]\u001b[A\n",
            "Iteration:  15%|█▍        | 53/360 [08:04<45:43,  8.94s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 54/360 [08:14<47:01,  9.22s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 55/360 [08:24<49:07,  9.66s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 56/360 [08:34<48:25,  9.56s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 57/360 [08:42<46:16,  9.16s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 58/360 [08:51<45:59,  9.14s/it]\u001b[A\n",
            "Iteration:  16%|█▋        | 59/360 [08:59<44:38,  8.90s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 60/360 [09:12<49:29,  9.90s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 61/360 [09:21<48:03,  9.64s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 62/360 [09:30<47:12,  9.51s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 63/360 [09:39<47:03,  9.51s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 64/360 [09:48<45:49,  9.29s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 65/360 [09:59<48:01,  9.77s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 66/360 [10:07<45:50,  9.35s/it]\u001b[A\n",
            "Iteration:  19%|█▊        | 67/360 [10:16<43:55,  9.00s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 68/360 [10:25<44:50,  9.21s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 69/360 [10:33<42:56,  8.85s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 70/360 [10:43<43:24,  8.98s/it]\u001b[A\n",
            "Iteration:  20%|█▉        | 71/360 [10:53<44:40,  9.27s/it]\u001b[A\n",
            "Iteration:  20%|██        | 72/360 [11:02<45:20,  9.45s/it]\u001b[A\n",
            "Iteration:  20%|██        | 73/360 [11:12<45:20,  9.48s/it]\u001b[A\n",
            "Iteration:  21%|██        | 74/360 [11:24<49:24, 10.37s/it]\u001b[A\n",
            "Iteration:  21%|██        | 75/360 [11:34<48:45, 10.27s/it]\u001b[A\n",
            "Iteration:  21%|██        | 76/360 [11:44<47:38, 10.06s/it]\u001b[A\n",
            "Iteration:  21%|██▏       | 77/360 [11:54<46:42,  9.90s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 78/360 [12:03<46:09,  9.82s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 79/360 [12:13<45:43,  9.76s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 80/360 [12:22<45:01,  9.65s/it]\u001b[A\n",
            "Iteration:  22%|██▎       | 81/360 [12:30<42:39,  9.17s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 82/360 [12:39<42:05,  9.09s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 83/360 [12:48<41:22,  8.96s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 84/360 [12:54<37:49,  8.22s/it]\u001b[A\n",
            "Iteration:  24%|██▎       | 85/360 [13:04<39:32,  8.63s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 86/360 [13:14<41:50,  9.16s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 87/360 [13:24<42:59,  9.45s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 88/360 [13:33<41:07,  9.07s/it]\u001b[A\n",
            "Iteration:  25%|██▍       | 89/360 [13:41<40:12,  8.90s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 90/360 [13:50<39:33,  8.79s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 91/360 [13:57<38:06,  8.50s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 92/360 [14:08<41:20,  9.26s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 93/360 [14:18<41:45,  9.38s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 94/360 [14:25<38:52,  8.77s/it]\u001b[A\n",
            "Iteration:  26%|██▋       | 95/360 [14:35<40:23,  9.15s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 96/360 [14:47<42:51,  9.74s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 97/360 [14:56<42:33,  9.71s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 98/360 [15:07<43:10,  9.89s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 99/360 [15:14<39:34,  9.10s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 100/360 [15:24<40:32,  9.36s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 101/360 [15:35<43:08,  9.99s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 102/360 [15:43<39:37,  9.21s/it]\u001b[A\n",
            "Iteration:  29%|██▊       | 103/360 [15:52<39:19,  9.18s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 104/360 [16:00<38:02,  8.92s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 105/360 [16:09<38:22,  9.03s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 106/360 [16:20<40:46,  9.63s/it]\u001b[A\n",
            "Iteration:  30%|██▉       | 107/360 [16:30<41:07,  9.75s/it]\u001b[A\n",
            "Iteration:  30%|███       | 108/360 [16:39<39:11,  9.33s/it]\u001b[A\n",
            "Iteration:  30%|███       | 109/360 [16:49<39:43,  9.49s/it]\u001b[A\n",
            "Iteration:  31%|███       | 110/360 [16:56<36:51,  8.85s/it]\u001b[A\n",
            "Iteration:  31%|███       | 111/360 [17:06<37:59,  9.16s/it]\u001b[A\n",
            "Iteration:  31%|███       | 112/360 [17:15<37:14,  9.01s/it]\u001b[A\n",
            "Iteration:  31%|███▏      | 113/360 [17:23<37:00,  8.99s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 114/360 [17:33<37:17,  9.10s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 115/360 [17:42<37:13,  9.12s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 116/360 [17:51<36:30,  8.98s/it]\u001b[A\n",
            "Iteration:  32%|███▎      | 117/360 [18:01<38:32,  9.52s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 118/360 [18:08<35:06,  8.70s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 119/360 [18:17<34:47,  8.66s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 120/360 [18:27<36:49,  9.21s/it]\u001b[A\n",
            "Iteration:  34%|███▎      | 121/360 [18:36<36:23,  9.14s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 122/360 [18:47<37:56,  9.56s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 123/360 [18:55<35:38,  9.02s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 124/360 [19:04<36:24,  9.26s/it]\u001b[A\n",
            "Iteration:  35%|███▍      | 125/360 [19:13<35:51,  9.16s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 126/360 [19:22<35:24,  9.08s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 127/360 [19:30<34:06,  8.78s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 128/360 [19:37<31:57,  8.27s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 129/360 [19:45<31:35,  8.21s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 130/360 [19:55<33:04,  8.63s/it]\u001b[A\n",
            "Iteration:  36%|███▋      | 131/360 [20:05<34:20,  9.00s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 132/360 [20:14<34:17,  9.02s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 133/360 [20:23<34:20,  9.08s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 134/360 [20:31<32:22,  8.60s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 135/360 [20:38<30:31,  8.14s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 136/360 [20:48<32:38,  8.74s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 137/360 [20:54<30:08,  8.11s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 138/360 [21:03<30:37,  8.28s/it]\u001b[A\n",
            "Iteration:  39%|███▊      | 139/360 [21:12<31:27,  8.54s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 140/360 [21:21<31:13,  8.52s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 141/360 [21:31<32:58,  9.03s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 142/360 [21:40<32:44,  9.01s/it]\u001b[A\n",
            "Iteration:  40%|███▉      | 143/360 [21:49<32:55,  9.10s/it]\u001b[A\n",
            "Iteration:  40%|████      | 144/360 [22:01<35:25,  9.84s/it]\u001b[A\n",
            "Iteration:  40%|████      | 145/360 [22:11<35:09,  9.81s/it]\u001b[A\n",
            "Iteration:  41%|████      | 146/360 [22:19<33:39,  9.44s/it]\u001b[A\n",
            "Iteration:  41%|████      | 147/360 [22:29<34:07,  9.61s/it]\u001b[A\n",
            "Iteration:  41%|████      | 148/360 [22:37<31:44,  8.98s/it]\u001b[A\n",
            "Iteration:  41%|████▏     | 149/360 [22:44<29:44,  8.46s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 150/360 [22:50<27:07,  7.75s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 151/360 [22:58<26:59,  7.75s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 152/360 [23:08<29:02,  8.38s/it]\u001b[A\n",
            "Iteration:  42%|████▎     | 153/360 [23:17<29:36,  8.58s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 154/360 [23:26<30:16,  8.82s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 155/360 [23:36<31:24,  9.19s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 156/360 [23:45<30:30,  8.97s/it]\u001b[A\n",
            "Iteration:  44%|████▎     | 157/360 [23:51<28:01,  8.28s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 158/360 [23:59<27:08,  8.06s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 159/360 [24:07<27:34,  8.23s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 160/360 [24:16<27:39,  8.30s/it]\u001b[A\n",
            "Iteration:  45%|████▍     | 161/360 [24:28<31:06,  9.38s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 162/360 [24:38<32:09,  9.75s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 163/360 [24:46<30:15,  9.22s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 164/360 [24:53<27:31,  8.43s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 165/360 [25:02<27:49,  8.56s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 166/360 [25:13<29:50,  9.23s/it]\u001b[A\n",
            "Iteration:  46%|████▋     | 167/360 [25:23<30:47,  9.57s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 168/360 [25:32<30:35,  9.56s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 169/360 [25:42<29:58,  9.42s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 170/360 [25:51<29:48,  9.41s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 171/360 [26:03<31:55, 10.14s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 172/360 [26:12<30:37,  9.77s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 173/360 [26:21<30:03,  9.64s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 174/360 [26:31<30:11,  9.74s/it]\u001b[A\n",
            "Iteration:  49%|████▊     | 175/360 [26:40<29:36,  9.60s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 176/360 [26:51<30:15,  9.87s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 177/360 [26:59<28:53,  9.47s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 178/360 [27:09<28:44,  9.47s/it]\u001b[A\n",
            "Iteration:  50%|████▉     | 179/360 [27:17<27:20,  9.06s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 180/360 [27:26<26:52,  8.96s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 181/360 [27:36<27:49,  9.33s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 182/360 [27:47<29:19,  9.88s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 183/360 [27:58<30:14, 10.25s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 184/360 [28:09<30:55, 10.54s/it]\u001b[A\n",
            "Iteration:  51%|█████▏    | 185/360 [28:20<30:32, 10.47s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 186/360 [28:30<30:21, 10.47s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 187/360 [28:41<30:09, 10.46s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 188/360 [28:48<27:36,  9.63s/it]\u001b[A\n",
            "Iteration:  52%|█████▎    | 189/360 [28:58<27:24,  9.62s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 190/360 [29:05<25:23,  8.96s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 191/360 [29:15<26:07,  9.27s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 192/360 [29:24<25:37,  9.15s/it]\u001b[A\n",
            "Iteration:  54%|█████▎    | 193/360 [29:33<25:32,  9.18s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 194/360 [29:44<26:13,  9.48s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 195/360 [29:52<25:35,  9.30s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 196/360 [30:01<24:35,  8.99s/it]\u001b[A\n",
            "Iteration:  55%|█████▍    | 197/360 [30:11<25:50,  9.51s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 198/360 [30:21<25:52,  9.58s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 199/360 [30:31<26:02,  9.70s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 200/360 [30:41<25:38,  9.61s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 201/360 [30:50<25:25,  9.59s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 202/360 [31:00<25:33,  9.71s/it]\u001b[A\n",
            "Iteration:  56%|█████▋    | 203/360 [31:14<28:27, 10.88s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 204/360 [31:24<27:45, 10.68s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 205/360 [31:35<28:01, 10.85s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 206/360 [31:42<24:35,  9.58s/it]\u001b[A\n",
            "Iteration:  57%|█████▊    | 207/360 [31:51<24:01,  9.42s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 208/360 [32:02<24:56,  9.85s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 209/360 [32:11<24:10,  9.61s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 210/360 [32:20<24:07,  9.65s/it]\u001b[A\n",
            "Iteration:  59%|█████▊    | 211/360 [32:30<23:35,  9.50s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 212/360 [32:39<23:28,  9.51s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 213/360 [32:48<23:03,  9.41s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 214/360 [32:58<22:51,  9.39s/it]\u001b[A\n",
            "Iteration:  60%|█████▉    | 215/360 [33:07<22:26,  9.29s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 216/360 [33:15<21:38,  9.01s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 217/360 [33:26<22:39,  9.51s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 218/360 [33:36<23:05,  9.75s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 219/360 [33:46<22:50,  9.72s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 220/360 [33:53<21:08,  9.06s/it]\u001b[A\n",
            "Iteration:  61%|██████▏   | 221/360 [34:03<21:12,  9.16s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 222/360 [34:19<25:49, 11.23s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 223/360 [34:28<24:05, 10.55s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 224/360 [34:37<22:56, 10.12s/it]\u001b[A\n",
            "Iteration:  62%|██████▎   | 225/360 [34:45<21:26,  9.53s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 226/360 [34:55<21:23,  9.58s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 227/360 [35:04<21:17,  9.60s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 228/360 [35:14<20:55,  9.51s/it]\u001b[A\n",
            "Iteration:  64%|██████▎   | 229/360 [35:24<21:11,  9.70s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 230/360 [35:33<20:46,  9.59s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 231/360 [35:42<20:18,  9.44s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 232/360 [35:51<20:00,  9.38s/it]\u001b[A\n",
            "Iteration:  65%|██████▍   | 233/360 [36:00<19:33,  9.24s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 234/360 [36:09<19:16,  9.18s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 235/360 [36:18<19:03,  9.15s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 236/360 [36:27<18:27,  8.93s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 237/360 [36:36<18:13,  8.89s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 238/360 [36:44<17:32,  8.63s/it]\u001b[A\n",
            "Iteration:  66%|██████▋   | 239/360 [36:52<17:23,  8.62s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 240/360 [37:01<17:13,  8.61s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 241/360 [37:12<18:46,  9.46s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 242/360 [37:20<17:26,  8.87s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 243/360 [37:29<17:42,  9.08s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 244/360 [37:38<17:22,  8.99s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 245/360 [37:48<17:52,  9.32s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 246/360 [37:58<18:07,  9.54s/it]\u001b[A\n",
            "Iteration:  69%|██████▊   | 247/360 [38:06<17:09,  9.11s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 248/360 [38:15<16:44,  8.97s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 249/360 [38:25<17:00,  9.19s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 250/360 [38:33<16:16,  8.88s/it]\u001b[A\n",
            "Iteration:  70%|██████▉   | 251/360 [38:42<16:23,  9.02s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 252/360 [38:52<16:39,  9.26s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 253/360 [39:01<16:32,  9.28s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 254/360 [39:12<16:56,  9.59s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 255/360 [39:19<15:34,  8.90s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 256/360 [39:29<15:47,  9.11s/it]\u001b[A\n",
            "Iteration:  71%|███████▏  | 257/360 [39:37<15:28,  9.02s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 258/360 [39:46<15:01,  8.83s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 259/360 [39:55<14:49,  8.81s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 260/360 [40:04<15:03,  9.04s/it]\u001b[A\n",
            "Iteration:  72%|███████▎  | 261/360 [40:12<14:26,  8.75s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 262/360 [40:23<15:12,  9.31s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 263/360 [40:33<15:21,  9.50s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 264/360 [40:40<14:17,  8.94s/it]\u001b[A\n",
            "Iteration:  74%|███████▎  | 265/360 [40:50<14:25,  9.11s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 266/360 [40:59<14:25,  9.20s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 267/360 [41:09<14:16,  9.20s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 268/360 [41:15<12:49,  8.36s/it]\u001b[A\n",
            "Iteration:  75%|███████▍  | 269/360 [41:22<12:06,  7.98s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 270/360 [41:29<11:24,  7.61s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 271/360 [41:39<12:25,  8.37s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 272/360 [41:48<12:47,  8.72s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 273/360 [41:59<13:27,  9.28s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 274/360 [42:05<12:01,  8.38s/it]\u001b[A\n",
            "Iteration:  76%|███████▋  | 275/360 [42:15<12:13,  8.62s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 276/360 [42:25<12:46,  9.13s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 277/360 [42:31<11:21,  8.21s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 278/360 [42:39<11:17,  8.26s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 279/360 [42:48<11:29,  8.52s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 280/360 [42:59<12:13,  9.17s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 281/360 [43:10<12:51,  9.77s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 282/360 [43:20<12:30,  9.62s/it]\u001b[A\n",
            "Iteration:  79%|███████▊  | 283/360 [43:30<12:36,  9.82s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 284/360 [43:38<11:57,  9.44s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 285/360 [43:47<11:24,  9.13s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 286/360 [43:56<11:20,  9.20s/it]\u001b[A\n",
            "Iteration:  80%|███████▉  | 287/360 [44:04<10:53,  8.95s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 288/360 [44:15<11:21,  9.47s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 289/360 [44:24<11:06,  9.39s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 290/360 [44:34<10:57,  9.39s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 291/360 [44:42<10:30,  9.13s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 292/360 [44:53<10:56,  9.65s/it]\u001b[A\n",
            "Iteration:  81%|████████▏ | 293/360 [45:02<10:30,  9.41s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 294/360 [45:12<10:29,  9.54s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 295/360 [45:22<10:35,  9.77s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 296/360 [45:32<10:27,  9.80s/it]\u001b[A\n",
            "Iteration:  82%|████████▎ | 297/360 [45:44<11:07, 10.60s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 298/360 [45:53<10:22, 10.04s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 299/360 [46:03<10:02,  9.87s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 300/360 [46:13<09:55,  9.92s/it]\u001b[A\n",
            "Iteration:  84%|████████▎ | 301/360 [46:21<09:13,  9.37s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 302/360 [46:32<09:38,  9.97s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 303/360 [46:40<08:56,  9.42s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 304/360 [46:50<08:52,  9.51s/it]\u001b[A\n",
            "Iteration:  85%|████████▍ | 305/360 [47:00<08:50,  9.65s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 306/360 [47:09<08:35,  9.55s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 307/360 [47:19<08:28,  9.60s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 308/360 [47:29<08:20,  9.63s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 309/360 [47:35<07:16,  8.57s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 310/360 [47:43<07:07,  8.55s/it]\u001b[A\n",
            "Iteration:  86%|████████▋ | 311/360 [47:53<07:11,  8.81s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 312/360 [48:00<06:44,  8.43s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 313/360 [48:07<06:12,  7.93s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 314/360 [48:13<05:38,  7.35s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 315/360 [48:21<05:37,  7.51s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 316/360 [48:33<06:25,  8.76s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 317/360 [48:42<06:21,  8.87s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 318/360 [48:49<05:49,  8.32s/it]\u001b[A\n",
            "Iteration:  89%|████████▊ | 319/360 [48:58<05:55,  8.67s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 320/360 [49:09<06:11,  9.28s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 321/360 [49:18<05:57,  9.18s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 322/360 [49:24<05:15,  8.30s/it]\u001b[A\n",
            "Iteration:  90%|████████▉ | 323/360 [49:34<05:23,  8.73s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 324/360 [49:44<05:29,  9.14s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 325/360 [49:53<05:15,  9.02s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 326/360 [50:02<05:05,  9.00s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 327/360 [50:12<05:07,  9.32s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 328/360 [50:23<05:18,  9.95s/it]\u001b[A\n",
            "Iteration:  91%|█████████▏| 329/360 [50:31<04:45,  9.20s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 330/360 [50:40<04:38,  9.29s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 331/360 [50:50<04:37,  9.57s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 332/360 [50:59<04:19,  9.27s/it]\u001b[A\n",
            "Iteration:  92%|█████████▎| 333/360 [51:09<04:16,  9.50s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 334/360 [51:18<04:06,  9.47s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 335/360 [51:27<03:49,  9.20s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 336/360 [51:36<03:41,  9.24s/it]\u001b[A\n",
            "Iteration:  94%|█████████▎| 337/360 [51:43<03:11,  8.35s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 338/360 [51:52<03:09,  8.63s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 339/360 [52:01<03:06,  8.87s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 340/360 [52:09<02:52,  8.61s/it]\u001b[A\n",
            "Iteration:  95%|█████████▍| 341/360 [52:18<02:44,  8.64s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 342/360 [52:28<02:41,  8.95s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 343/360 [52:36<02:28,  8.74s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 344/360 [52:46<02:26,  9.15s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 345/360 [52:53<02:05,  8.40s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 346/360 [53:03<02:03,  8.83s/it]\u001b[A\n",
            "Iteration:  96%|█████████▋| 347/360 [53:16<02:11, 10.15s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 348/360 [53:25<01:59,  9.97s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 349/360 [53:32<01:39,  9.02s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 350/360 [53:43<01:37,  9.71s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 351/360 [53:51<01:21,  9.09s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 352/360 [54:01<01:14,  9.36s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 353/360 [54:11<01:05,  9.40s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 354/360 [54:21<00:57,  9.62s/it]\u001b[A\n",
            "Iteration:  99%|█████████▊| 355/360 [54:30<00:47,  9.48s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 356/360 [54:40<00:38,  9.73s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 357/360 [54:49<00:28,  9.39s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 358/360 [55:00<00:19,  9.87s/it]\u001b[A\n",
            "Iteration: 100%|█████████▉| 359/360 [55:10<00:09,  9.90s/it]\u001b[A\n",
            "Iteration: 100%|██████████| 360/360 [55:14<00:00,  8.29s/it]\u001b[A\n",
            "Epoch:  50%|█████     | 2/4 [2:42:37<1:45:58, 3179.25s/it]\n",
            "Convert Evaluating:   0%|          | 0/94 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:02:08 - Evaluation the model on  dataset after epoch 2:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Convert Evaluating:   1%|          | 1/94 [00:00<01:28,  1.06it/s]\u001b[A\n",
            "Convert Evaluating:   2%|▏         | 2/94 [00:01<01:25,  1.08it/s]\u001b[A\n",
            "Convert Evaluating:   3%|▎         | 3/94 [00:02<01:29,  1.02it/s]\u001b[A\n",
            "Convert Evaluating:   4%|▍         | 4/94 [00:03<01:27,  1.02it/s]\u001b[A\n",
            "Convert Evaluating:   5%|▌         | 5/94 [00:04<01:29,  1.01s/it]\u001b[A\n",
            "Convert Evaluating:   6%|▋         | 6/94 [00:06<01:35,  1.09s/it]\u001b[A\n",
            "Convert Evaluating:   7%|▋         | 7/94 [00:07<01:33,  1.08s/it]\u001b[A\n",
            "Convert Evaluating:   9%|▊         | 8/94 [00:08<01:31,  1.06s/it]\u001b[A\n",
            "Convert Evaluating:  10%|▉         | 9/94 [00:09<01:29,  1.05s/it]\u001b[A\n",
            "Convert Evaluating:  11%|█         | 10/94 [00:10<01:22,  1.02it/s]\u001b[A\n",
            "Convert Evaluating:  12%|█▏        | 11/94 [00:11<01:21,  1.02it/s]\u001b[A\n",
            "Convert Evaluating:  13%|█▎        | 12/94 [00:12<01:25,  1.05s/it]\u001b[A\n",
            "Convert Evaluating:  14%|█▍        | 13/94 [00:13<01:23,  1.04s/it]\u001b[A\n",
            "Convert Evaluating:  15%|█▍        | 14/94 [00:14<01:23,  1.04s/it]\u001b[A\n",
            "Convert Evaluating:  16%|█▌        | 15/94 [00:15<01:28,  1.12s/it]\u001b[A\n",
            "Convert Evaluating:  17%|█▋        | 16/94 [00:17<01:32,  1.19s/it]\u001b[A\n",
            "Convert Evaluating:  18%|█▊        | 17/94 [00:18<01:35,  1.24s/it]\u001b[A\n",
            "Convert Evaluating:  19%|█▉        | 18/94 [00:19<01:33,  1.23s/it]\u001b[A\n",
            "Convert Evaluating:  20%|██        | 19/94 [00:20<01:34,  1.26s/it]\u001b[A\n",
            "Convert Evaluating:  21%|██▏       | 20/94 [00:22<01:33,  1.26s/it]\u001b[A\n",
            "Convert Evaluating:  22%|██▏       | 21/94 [00:23<01:36,  1.33s/it]\u001b[A\n",
            "Convert Evaluating:  23%|██▎       | 22/94 [00:24<01:32,  1.28s/it]\u001b[A\n",
            "Convert Evaluating:  24%|██▍       | 23/94 [00:26<01:30,  1.27s/it]\u001b[A\n",
            "Convert Evaluating:  26%|██▌       | 24/94 [00:27<01:33,  1.34s/it]\u001b[A\n",
            "Convert Evaluating:  27%|██▋       | 25/94 [00:28<01:28,  1.29s/it]\u001b[A\n",
            "Convert Evaluating:  28%|██▊       | 26/94 [00:30<01:32,  1.35s/it]\u001b[A\n",
            "Convert Evaluating:  29%|██▊       | 27/94 [00:31<01:25,  1.28s/it]\u001b[A\n",
            "Convert Evaluating:  30%|██▉       | 28/94 [00:32<01:27,  1.33s/it]\u001b[A\n",
            "Convert Evaluating:  31%|███       | 29/94 [00:34<01:31,  1.42s/it]\u001b[A\n",
            "Convert Evaluating:  32%|███▏      | 30/94 [00:35<01:27,  1.36s/it]\u001b[A\n",
            "Convert Evaluating:  33%|███▎      | 31/94 [00:37<01:28,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  34%|███▍      | 32/94 [00:38<01:26,  1.39s/it]\u001b[A\n",
            "Convert Evaluating:  35%|███▌      | 33/94 [00:40<01:26,  1.41s/it]\u001b[A\n",
            "Convert Evaluating:  36%|███▌      | 34/94 [00:41<01:26,  1.44s/it]\u001b[A\n",
            "Convert Evaluating:  37%|███▋      | 35/94 [00:42<01:22,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  38%|███▊      | 36/94 [00:44<01:22,  1.42s/it]\u001b[A\n",
            "Convert Evaluating:  39%|███▉      | 37/94 [00:45<01:19,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  40%|████      | 38/94 [00:47<01:18,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  41%|████▏     | 39/94 [00:48<01:16,  1.40s/it]\u001b[A\n",
            "Convert Evaluating:  43%|████▎     | 40/94 [00:51<01:35,  1.76s/it]\u001b[A\n",
            "Convert Evaluating:  44%|████▎     | 41/94 [00:53<01:44,  1.98s/it]\u001b[A\n",
            "Convert Evaluating:  45%|████▍     | 42/94 [00:55<01:43,  1.98s/it]\u001b[A\n",
            "Convert Evaluating:  46%|████▌     | 43/94 [00:57<01:45,  2.07s/it]\u001b[A\n",
            "Convert Evaluating:  47%|████▋     | 44/94 [00:59<01:43,  2.07s/it]\u001b[A\n",
            "Convert Evaluating:  48%|████▊     | 45/94 [01:02<01:43,  2.11s/it]\u001b[A\n",
            "Convert Evaluating:  49%|████▉     | 46/94 [01:03<01:38,  2.05s/it]\u001b[A\n",
            "Convert Evaluating:  50%|█████     | 47/94 [01:06<01:40,  2.13s/it]\u001b[A\n",
            "Convert Evaluating:  51%|█████     | 48/94 [01:09<01:54,  2.48s/it]\u001b[A\n",
            "Convert Evaluating:  52%|█████▏    | 49/94 [01:11<01:49,  2.43s/it]\u001b[A\n",
            "Convert Evaluating:  53%|█████▎    | 50/94 [01:13<01:41,  2.31s/it]\u001b[A\n",
            "Convert Evaluating:  54%|█████▍    | 51/94 [01:15<01:34,  2.20s/it]\u001b[A\n",
            "Convert Evaluating:  55%|█████▌    | 52/94 [01:17<01:29,  2.14s/it]\u001b[A\n",
            "Convert Evaluating:  56%|█████▋    | 53/94 [01:19<01:26,  2.12s/it]\u001b[A\n",
            "Convert Evaluating:  57%|█████▋    | 54/94 [01:21<01:22,  2.06s/it]\u001b[A\n",
            "Convert Evaluating:  59%|█████▊    | 55/94 [01:24<01:24,  2.18s/it]\u001b[A\n",
            "Convert Evaluating:  60%|█████▉    | 56/94 [01:26<01:22,  2.18s/it]\u001b[A\n",
            "Convert Evaluating:  61%|██████    | 57/94 [01:28<01:18,  2.12s/it]\u001b[A\n",
            "Convert Evaluating:  62%|██████▏   | 58/94 [01:30<01:17,  2.16s/it]\u001b[A\n",
            "Convert Evaluating:  63%|██████▎   | 59/94 [01:32<01:14,  2.13s/it]\u001b[A\n",
            "Convert Evaluating:  64%|██████▍   | 60/94 [01:34<01:10,  2.09s/it]\u001b[A\n",
            "Convert Evaluating:  65%|██████▍   | 61/94 [01:36<01:08,  2.07s/it]\u001b[A\n",
            "Convert Evaluating:  66%|██████▌   | 62/94 [01:38<01:05,  2.06s/it]\u001b[A\n",
            "Convert Evaluating:  67%|██████▋   | 63/94 [01:41<01:06,  2.14s/it]\u001b[A\n",
            "Convert Evaluating:  68%|██████▊   | 64/94 [01:43<01:07,  2.25s/it]\u001b[A\n",
            "Convert Evaluating:  69%|██████▉   | 65/94 [01:46<01:07,  2.34s/it]\u001b[A\n",
            "Convert Evaluating:  70%|███████   | 66/94 [01:48<01:05,  2.35s/it]\u001b[A\n",
            "Convert Evaluating:  71%|███████▏  | 67/94 [01:51<01:04,  2.38s/it]\u001b[A\n",
            "Convert Evaluating:  72%|███████▏  | 68/94 [01:53<01:03,  2.45s/it]\u001b[A\n",
            "Convert Evaluating:  73%|███████▎  | 69/94 [01:56<01:02,  2.48s/it]\u001b[A\n",
            "Convert Evaluating:  74%|███████▍  | 70/94 [01:59<01:02,  2.62s/it]\u001b[A\n",
            "Convert Evaluating:  76%|███████▌  | 71/94 [02:01<00:58,  2.53s/it]\u001b[A\n",
            "Convert Evaluating:  77%|███████▋  | 72/94 [02:04<00:56,  2.58s/it]\u001b[A\n",
            "Convert Evaluating:  78%|███████▊  | 73/94 [02:07<00:56,  2.70s/it]\u001b[A\n",
            "Convert Evaluating:  79%|███████▊  | 74/94 [02:09<00:52,  2.60s/it]\u001b[A\n",
            "Convert Evaluating:  80%|███████▉  | 75/94 [02:12<00:48,  2.57s/it]\u001b[A\n",
            "Convert Evaluating:  81%|████████  | 76/94 [02:14<00:45,  2.54s/it]\u001b[A\n",
            "Convert Evaluating:  82%|████████▏ | 77/94 [02:17<00:43,  2.59s/it]\u001b[A\n",
            "Convert Evaluating:  83%|████████▎ | 78/94 [02:19<00:41,  2.60s/it]\u001b[A\n",
            "Convert Evaluating:  84%|████████▍ | 79/94 [02:21<00:34,  2.33s/it]\u001b[A\n",
            "Convert Evaluating:  85%|████████▌ | 80/94 [02:22<00:27,  1.99s/it]\u001b[A\n",
            "Convert Evaluating:  86%|████████▌ | 81/94 [02:23<00:22,  1.75s/it]\u001b[A\n",
            "Convert Evaluating:  87%|████████▋ | 82/94 [02:25<00:18,  1.54s/it]\u001b[A\n",
            "Convert Evaluating:  88%|████████▊ | 83/94 [02:26<00:15,  1.38s/it]\u001b[A\n",
            "Convert Evaluating:  89%|████████▉ | 84/94 [02:27<00:13,  1.33s/it]\u001b[A\n",
            "Convert Evaluating:  90%|█████████ | 85/94 [02:28<00:11,  1.28s/it]\u001b[A\n",
            "Convert Evaluating:  91%|█████████▏| 86/94 [02:29<00:10,  1.30s/it]\u001b[A\n",
            "Convert Evaluating:  93%|█████████▎| 87/94 [02:31<00:09,  1.29s/it]\u001b[A\n",
            "Convert Evaluating:  94%|█████████▎| 88/94 [02:32<00:07,  1.29s/it]\u001b[A\n",
            "Convert Evaluating:  95%|█████████▍| 89/94 [02:33<00:06,  1.25s/it]\u001b[A\n",
            "Convert Evaluating:  96%|█████████▌| 90/94 [02:34<00:04,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  97%|█████████▋| 91/94 [02:35<00:03,  1.17s/it]\u001b[A\n",
            "Convert Evaluating:  98%|█████████▊| 92/94 [02:37<00:02,  1.27s/it]\u001b[A\n",
            "Convert Evaluating:  99%|█████████▉| 93/94 [02:38<00:01,  1.30s/it]\u001b[A\n",
            "Convert Evaluating: 100%|██████████| 94/94 [02:39<00:00,  1.14s/it]\u001b[A\n",
            "Epoch:  50%|█████     | 2/4 [2:45:16<1:45:58, 3179.25s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:04:47 - Cosine-Similarity :\tPearson: 0.8727\tSpearman: 0.8732\n",
            "2019-11-28 11:04:47 - Manhattan-Distance:\tPearson: 0.8432\tSpearman: 0.8467\n",
            "2019-11-28 11:04:47 - Euclidean-Distance:\tPearson: 0.8436\tSpearman: 0.8471\n",
            "2019-11-28 11:04:47 - Dot-Product-Similarity:\tPearson: 0.8052\tSpearman: 0.8100\n",
            "2019-11-28 11:04:47 - Save model to output/training_stsbenchmark_bert-2019-11-28_08-19-18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch:  75%|███████▌  | 3/4 [2:45:18<54:28, 3268.07s/it]  \n",
            "Iteration:   0%|          | 0/360 [00:00<?, ?it/s]\u001b[A\n",
            "\n",
            "Epoch:  75%|███████▌  | 3/4 [2:45:18<54:28, 3268.07s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:04:48 - Restart data_iterator\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Iteration:   0%|          | 1/360 [00:11<1:06:27, 11.11s/it]\u001b[A\n",
            "Iteration:   1%|          | 2/360 [00:21<1:04:36, 10.83s/it]\u001b[A\n",
            "Iteration:   1%|          | 3/360 [00:30<1:02:18, 10.47s/it]\u001b[A\n",
            "Iteration:   1%|          | 4/360 [00:38<56:14,  9.48s/it]  \u001b[A\n",
            "Iteration:   1%|▏         | 5/360 [00:46<54:41,  9.24s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 6/360 [00:57<57:08,  9.69s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 7/360 [01:09<1:00:44, 10.32s/it]\u001b[A\n",
            "Iteration:   2%|▏         | 8/360 [01:19<1:00:19, 10.28s/it]\u001b[A\n",
            "Iteration:   2%|▎         | 9/360 [01:29<59:34, 10.18s/it]  \u001b[A\n",
            "Iteration:   3%|▎         | 10/360 [01:38<57:08,  9.79s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 11/360 [01:47<55:27,  9.54s/it]\u001b[A\n",
            "Iteration:   3%|▎         | 12/360 [01:54<52:04,  8.98s/it]\u001b[A\n",
            "Iteration:   4%|▎         | 13/360 [02:03<51:39,  8.93s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 14/360 [02:14<54:21,  9.43s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 15/360 [02:25<56:31,  9.83s/it]\u001b[A\n",
            "Iteration:   4%|▍         | 16/360 [02:32<51:20,  8.96s/it]\u001b[A\n",
            "Iteration:   5%|▍         | 17/360 [02:40<50:35,  8.85s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 18/360 [02:50<52:25,  9.20s/it]\u001b[A\n",
            "Iteration:   5%|▌         | 19/360 [02:59<52:07,  9.17s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 20/360 [03:07<49:22,  8.71s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 21/360 [03:17<50:49,  9.00s/it]\u001b[A\n",
            "Iteration:   6%|▌         | 22/360 [03:28<54:17,  9.64s/it]\u001b[A\n",
            "Iteration:   6%|▋         | 23/360 [03:37<54:09,  9.64s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 24/360 [03:46<52:27,  9.37s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 25/360 [03:55<52:11,  9.35s/it]\u001b[A\n",
            "Iteration:   7%|▋         | 26/360 [04:02<47:19,  8.50s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 27/360 [04:10<46:40,  8.41s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 28/360 [04:21<50:36,  9.15s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 29/360 [04:32<53:20,  9.67s/it]\u001b[A\n",
            "Iteration:   8%|▊         | 30/360 [04:43<54:55,  9.99s/it]\u001b[A\n",
            "Iteration:   9%|▊         | 31/360 [04:54<56:27, 10.30s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 32/360 [05:02<53:29,  9.78s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 33/360 [05:10<49:17,  9.04s/it]\u001b[A\n",
            "Iteration:   9%|▉         | 34/360 [05:18<48:33,  8.94s/it]\u001b[A\n",
            "Iteration:  10%|▉         | 35/360 [05:27<48:08,  8.89s/it]\u001b[A\n",
            "Iteration:  10%|█         | 36/360 [05:35<46:50,  8.67s/it]\u001b[A\n",
            "Iteration:  10%|█         | 37/360 [05:48<52:45,  9.80s/it]\u001b[A\n",
            "Iteration:  11%|█         | 38/360 [05:57<51:48,  9.65s/it]\u001b[A\n",
            "Iteration:  11%|█         | 39/360 [06:06<50:47,  9.49s/it]\u001b[A\n",
            "Iteration:  11%|█         | 40/360 [06:15<50:05,  9.39s/it]\u001b[A\n",
            "Iteration:  11%|█▏        | 41/360 [06:23<47:02,  8.85s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 42/360 [06:31<45:18,  8.55s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 43/360 [06:40<46:04,  8.72s/it]\u001b[A\n",
            "Iteration:  12%|█▏        | 44/360 [06:49<46:16,  8.79s/it]\u001b[A\n",
            "Iteration:  12%|█▎        | 45/360 [06:58<47:37,  9.07s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 46/360 [07:07<45:59,  8.79s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 47/360 [07:13<42:25,  8.13s/it]\u001b[A\n",
            "Iteration:  13%|█▎        | 48/360 [07:23<45:46,  8.80s/it]\u001b[A\n",
            "Iteration:  14%|█▎        | 49/360 [07:31<44:12,  8.53s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 50/360 [07:42<46:44,  9.05s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 51/360 [07:51<47:07,  9.15s/it]\u001b[A\n",
            "Iteration:  14%|█▍        | 52/360 [08:02<49:22,  9.62s/it]\u001b[A\n",
            "Iteration:  15%|█▍        | 53/360 [08:11<49:14,  9.62s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 54/360 [08:22<50:45,  9.95s/it]\u001b[A\n",
            "Iteration:  15%|█▌        | 55/360 [08:31<48:26,  9.53s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 56/360 [08:41<49:20,  9.74s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 57/360 [08:52<51:59, 10.30s/it]\u001b[A\n",
            "Iteration:  16%|█▌        | 58/360 [09:02<50:01,  9.94s/it]\u001b[A\n",
            "Iteration:  16%|█▋        | 59/360 [09:12<50:27, 10.06s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 60/360 [09:21<49:18,  9.86s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 61/360 [09:29<46:31,  9.34s/it]\u001b[A\n",
            "Iteration:  17%|█▋        | 62/360 [09:39<46:30,  9.36s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 63/360 [09:47<44:44,  9.04s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 64/360 [09:56<44:21,  8.99s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 65/360 [10:04<42:40,  8.68s/it]\u001b[A\n",
            "Iteration:  18%|█▊        | 66/360 [10:15<46:00,  9.39s/it]\u001b[A\n",
            "Iteration:  19%|█▊        | 67/360 [10:24<44:37,  9.14s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 68/360 [10:30<40:24,  8.30s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 69/360 [10:38<40:30,  8.35s/it]\u001b[A\n",
            "Iteration:  19%|█▉        | 70/360 [10:49<43:26,  8.99s/it]\u001b[A\n",
            "Iteration:  20%|█▉        | 71/360 [10:57<42:02,  8.73s/it]\u001b[A\n",
            "Iteration:  20%|██        | 72/360 [11:06<42:09,  8.78s/it]\u001b[A\n",
            "Iteration:  20%|██        | 73/360 [11:13<39:32,  8.27s/it]\u001b[A\n",
            "Iteration:  21%|██        | 74/360 [11:20<37:30,  7.87s/it]\u001b[A\n",
            "Iteration:  21%|██        | 75/360 [11:31<41:28,  8.73s/it]\u001b[A\n",
            "Iteration:  21%|██        | 76/360 [11:36<37:04,  7.83s/it]\u001b[A\n",
            "Iteration:  21%|██▏       | 77/360 [11:46<40:02,  8.49s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 78/360 [11:56<40:54,  8.71s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 79/360 [12:04<39:56,  8.53s/it]\u001b[A\n",
            "Iteration:  22%|██▏       | 80/360 [12:11<38:20,  8.22s/it]\u001b[A\n",
            "Iteration:  22%|██▎       | 81/360 [12:22<41:17,  8.88s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 82/360 [12:30<41:04,  8.86s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 83/360 [12:37<38:20,  8.30s/it]\u001b[A\n",
            "Iteration:  23%|██▎       | 84/360 [12:45<36:40,  7.97s/it]\u001b[A\n",
            "Iteration:  24%|██▎       | 85/360 [12:52<35:06,  7.66s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 86/360 [13:00<36:35,  8.01s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 87/360 [13:08<36:03,  7.92s/it]\u001b[A\n",
            "Iteration:  24%|██▍       | 88/360 [13:16<35:51,  7.91s/it]\u001b[A\n",
            "Iteration:  25%|██▍       | 89/360 [13:25<36:51,  8.16s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 90/360 [13:33<37:09,  8.26s/it]\u001b[A\n",
            "Iteration:  25%|██▌       | 91/360 [13:42<37:18,  8.32s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 92/360 [13:49<36:20,  8.14s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 93/360 [13:59<37:30,  8.43s/it]\u001b[A\n",
            "Iteration:  26%|██▌       | 94/360 [14:05<35:04,  7.91s/it]\u001b[A\n",
            "Iteration:  26%|██▋       | 95/360 [14:13<34:51,  7.89s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 96/360 [14:22<35:28,  8.06s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 97/360 [14:30<35:40,  8.14s/it]\u001b[A\n",
            "Iteration:  27%|██▋       | 98/360 [14:40<38:26,  8.80s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 99/360 [14:49<38:12,  8.78s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 100/360 [14:58<37:59,  8.77s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 101/360 [15:07<38:04,  8.82s/it]\u001b[A\n",
            "Iteration:  28%|██▊       | 102/360 [15:16<38:02,  8.85s/it]\u001b[A\n",
            "Iteration:  29%|██▊       | 103/360 [15:24<37:27,  8.74s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 104/360 [15:34<38:50,  9.10s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 105/360 [15:43<38:21,  9.03s/it]\u001b[A\n",
            "Iteration:  29%|██▉       | 106/360 [15:52<38:21,  9.06s/it]\u001b[A\n",
            "Iteration:  30%|██▉       | 107/360 [15:59<35:50,  8.50s/it]\u001b[A\n",
            "Iteration:  30%|███       | 108/360 [16:08<36:19,  8.65s/it]\u001b[A\n",
            "Iteration:  30%|███       | 109/360 [16:18<37:24,  8.94s/it]\u001b[A\n",
            "Iteration:  31%|███       | 110/360 [16:28<38:39,  9.28s/it]\u001b[A\n",
            "Iteration:  31%|███       | 111/360 [16:35<36:29,  8.79s/it]\u001b[A\n",
            "Iteration:  31%|███       | 112/360 [16:44<35:50,  8.67s/it]\u001b[A\n",
            "Iteration:  31%|███▏      | 113/360 [16:51<33:57,  8.25s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 114/360 [17:00<34:42,  8.47s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 115/360 [17:10<35:59,  8.82s/it]\u001b[A\n",
            "Iteration:  32%|███▏      | 116/360 [17:18<34:46,  8.55s/it]\u001b[A\n",
            "Iteration:  32%|███▎      | 117/360 [17:25<33:33,  8.29s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 118/360 [17:33<32:31,  8.06s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 119/360 [17:42<33:17,  8.29s/it]\u001b[A\n",
            "Iteration:  33%|███▎      | 120/360 [17:51<34:38,  8.66s/it]\u001b[A\n",
            "Iteration:  34%|███▎      | 121/360 [18:01<35:54,  9.01s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 122/360 [18:09<34:22,  8.66s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 123/360 [18:18<34:09,  8.65s/it]\u001b[A\n",
            "Iteration:  34%|███▍      | 124/360 [18:26<33:41,  8.56s/it]\u001b[A\n",
            "Iteration:  35%|███▍      | 125/360 [18:35<34:42,  8.86s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 126/360 [18:42<32:21,  8.30s/it]\u001b[A\n",
            "Iteration:  35%|███▌      | 127/360 [18:51<32:39,  8.41s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 128/360 [19:00<32:42,  8.46s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 129/360 [19:08<32:29,  8.44s/it]\u001b[A\n",
            "Iteration:  36%|███▌      | 130/360 [19:16<32:05,  8.37s/it]\u001b[A\n",
            "Iteration:  36%|███▋      | 131/360 [19:26<33:16,  8.72s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 132/360 [19:34<33:04,  8.70s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 133/360 [19:43<33:15,  8.79s/it]\u001b[A\n",
            "Iteration:  37%|███▋      | 134/360 [19:54<35:09,  9.33s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 135/360 [20:05<36:45,  9.80s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 136/360 [20:15<36:51,  9.87s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 137/360 [20:25<36:48,  9.90s/it]\u001b[A\n",
            "Iteration:  38%|███▊      | 138/360 [20:34<35:56,  9.72s/it]\u001b[A\n",
            "Iteration:  39%|███▊      | 139/360 [20:43<34:25,  9.35s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 140/360 [20:50<32:28,  8.85s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 141/360 [20:58<31:15,  8.56s/it]\u001b[A\n",
            "Iteration:  39%|███▉      | 142/360 [21:07<31:05,  8.56s/it]\u001b[A\n",
            "Iteration:  40%|███▉      | 143/360 [21:15<30:18,  8.38s/it]\u001b[A\n",
            "Iteration:  40%|████      | 144/360 [21:22<29:10,  8.11s/it]\u001b[A\n",
            "Iteration:  40%|████      | 145/360 [21:30<28:37,  7.99s/it]\u001b[A\n",
            "Iteration:  41%|████      | 146/360 [21:40<30:25,  8.53s/it]\u001b[A\n",
            "Iteration:  41%|████      | 147/360 [21:48<29:52,  8.42s/it]\u001b[A\n",
            "Iteration:  41%|████      | 148/360 [21:57<30:31,  8.64s/it]\u001b[A\n",
            "Iteration:  41%|████▏     | 149/360 [22:08<32:38,  9.28s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 150/360 [22:17<32:41,  9.34s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 151/360 [22:26<31:39,  9.09s/it]\u001b[A\n",
            "Iteration:  42%|████▏     | 152/360 [22:35<31:49,  9.18s/it]\u001b[A\n",
            "Iteration:  42%|████▎     | 153/360 [22:43<29:50,  8.65s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 154/360 [22:51<29:44,  8.66s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 155/360 [22:59<28:52,  8.45s/it]\u001b[A\n",
            "Iteration:  43%|████▎     | 156/360 [23:08<29:20,  8.63s/it]\u001b[A\n",
            "Iteration:  44%|████▎     | 157/360 [23:16<28:35,  8.45s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 158/360 [23:24<27:42,  8.23s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 159/360 [23:33<28:14,  8.43s/it]\u001b[A\n",
            "Iteration:  44%|████▍     | 160/360 [23:41<27:29,  8.25s/it]\u001b[A\n",
            "Iteration:  45%|████▍     | 161/360 [23:49<27:39,  8.34s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 162/360 [23:58<28:13,  8.55s/it]\u001b[A\n",
            "Iteration:  45%|████▌     | 163/360 [24:06<26:48,  8.16s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 164/360 [24:13<25:41,  7.87s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 165/360 [24:21<25:41,  7.90s/it]\u001b[A\n",
            "Iteration:  46%|████▌     | 166/360 [24:29<25:36,  7.92s/it]\u001b[A\n",
            "Iteration:  46%|████▋     | 167/360 [24:39<27:21,  8.51s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 168/360 [24:47<26:33,  8.30s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 169/360 [24:55<26:12,  8.23s/it]\u001b[A\n",
            "Iteration:  47%|████▋     | 170/360 [25:02<25:18,  7.99s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 171/360 [25:10<25:12,  8.00s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 172/360 [25:19<25:34,  8.16s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 173/360 [25:26<24:59,  8.02s/it]\u001b[A\n",
            "Iteration:  48%|████▊     | 174/360 [25:36<26:53,  8.68s/it]\u001b[A\n",
            "Iteration:  49%|████▊     | 175/360 [25:49<30:19,  9.83s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 176/360 [25:56<27:42,  9.04s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 177/360 [26:04<26:38,  8.74s/it]\u001b[A\n",
            "Iteration:  49%|████▉     | 178/360 [26:12<25:22,  8.37s/it]\u001b[A\n",
            "Iteration:  50%|████▉     | 179/360 [26:21<25:47,  8.55s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 180/360 [26:29<25:06,  8.37s/it]\u001b[A\n",
            "Iteration:  50%|█████     | 181/360 [26:38<25:43,  8.62s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 182/360 [26:46<24:50,  8.37s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 183/360 [26:52<22:47,  7.73s/it]\u001b[A\n",
            "Iteration:  51%|█████     | 184/360 [27:00<22:47,  7.77s/it]\u001b[A\n",
            "Iteration:  51%|█████▏    | 185/360 [27:07<22:08,  7.59s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 186/360 [27:16<22:57,  7.92s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 187/360 [27:24<23:01,  7.98s/it]\u001b[A\n",
            "Iteration:  52%|█████▏    | 188/360 [27:34<24:48,  8.66s/it]\u001b[A\n",
            "Iteration:  52%|█████▎    | 189/360 [27:42<23:46,  8.34s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 190/360 [27:47<21:22,  7.55s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 191/360 [27:53<19:39,  6.98s/it]\u001b[A\n",
            "Iteration:  53%|█████▎    | 192/360 [28:01<20:13,  7.22s/it]\u001b[A\n",
            "Iteration:  54%|█████▎    | 193/360 [28:07<19:31,  7.02s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 194/360 [28:15<20:00,  7.23s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 195/360 [28:23<20:38,  7.51s/it]\u001b[A\n",
            "Iteration:  54%|█████▍    | 196/360 [28:32<21:39,  7.92s/it]\u001b[A\n",
            "Iteration:  55%|█████▍    | 197/360 [28:41<22:24,  8.25s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 198/360 [28:49<21:47,  8.07s/it]\u001b[A\n",
            "Iteration:  55%|█████▌    | 199/360 [28:59<23:02,  8.59s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 200/360 [29:07<22:39,  8.50s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 201/360 [29:14<21:30,  8.12s/it]\u001b[A\n",
            "Iteration:  56%|█████▌    | 202/360 [29:23<22:20,  8.48s/it]\u001b[A\n",
            "Iteration:  56%|█████▋    | 203/360 [29:31<21:28,  8.21s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 204/360 [29:40<22:11,  8.53s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 205/360 [29:49<22:27,  8.70s/it]\u001b[A\n",
            "Iteration:  57%|█████▋    | 206/360 [29:58<21:59,  8.57s/it]\u001b[A\n",
            "Iteration:  57%|█████▊    | 207/360 [30:06<21:35,  8.47s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 208/360 [30:13<20:33,  8.12s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 209/360 [30:22<21:20,  8.48s/it]\u001b[A\n",
            "Iteration:  58%|█████▊    | 210/360 [30:32<22:15,  8.90s/it]\u001b[A\n",
            "Iteration:  59%|█████▊    | 211/360 [30:40<21:06,  8.50s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 212/360 [30:48<20:27,  8.30s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 213/360 [30:55<19:26,  7.94s/it]\u001b[A\n",
            "Iteration:  59%|█████▉    | 214/360 [31:03<19:37,  8.06s/it]\u001b[A\n",
            "Iteration:  60%|█████▉    | 215/360 [31:13<20:52,  8.64s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 216/360 [31:21<20:22,  8.49s/it]\u001b[A\n",
            "Iteration:  60%|██████    | 217/360 [31:29<19:53,  8.35s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 218/360 [31:38<20:22,  8.61s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 219/360 [31:46<19:35,  8.34s/it]\u001b[A\n",
            "Iteration:  61%|██████    | 220/360 [31:55<19:35,  8.39s/it]\u001b[A\n",
            "Iteration:  61%|██████▏   | 221/360 [32:03<19:33,  8.44s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 222/360 [32:11<19:08,  8.33s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 223/360 [32:19<18:39,  8.17s/it]\u001b[A\n",
            "Iteration:  62%|██████▏   | 224/360 [32:26<17:54,  7.90s/it]\u001b[A\n",
            "Iteration:  62%|██████▎   | 225/360 [32:33<16:34,  7.37s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 226/360 [32:41<16:59,  7.61s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 227/360 [32:49<17:20,  7.83s/it]\u001b[A\n",
            "Iteration:  63%|██████▎   | 228/360 [32:56<16:37,  7.56s/it]\u001b[A\n",
            "Iteration:  64%|██████▎   | 229/360 [33:06<18:22,  8.42s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 230/360 [33:15<18:34,  8.57s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 231/360 [33:24<18:12,  8.47s/it]\u001b[A\n",
            "Iteration:  64%|██████▍   | 232/360 [33:30<17:04,  8.01s/it]\u001b[A\n",
            "Iteration:  65%|██████▍   | 233/360 [33:40<17:36,  8.32s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 234/360 [33:47<17:03,  8.12s/it]\u001b[A\n",
            "Iteration:  65%|██████▌   | 235/360 [33:56<17:06,  8.21s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 236/360 [34:04<16:59,  8.22s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 237/360 [34:12<16:53,  8.24s/it]\u001b[A\n",
            "Iteration:  66%|██████▌   | 238/360 [34:21<17:07,  8.42s/it]\u001b[A\n",
            "Iteration:  66%|██████▋   | 239/360 [34:32<18:42,  9.28s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 240/360 [34:38<16:21,  8.18s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 241/360 [34:48<17:37,  8.89s/it]\u001b[A\n",
            "Iteration:  67%|██████▋   | 242/360 [34:55<16:23,  8.34s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 243/360 [35:04<16:16,  8.35s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 244/360 [35:11<15:39,  8.10s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 245/360 [35:20<15:59,  8.34s/it]\u001b[A\n",
            "Iteration:  68%|██████▊   | 246/360 [35:27<14:43,  7.75s/it]\u001b[A\n",
            "Iteration:  69%|██████▊   | 247/360 [35:35<15:00,  7.97s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 248/360 [35:42<14:29,  7.76s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 249/360 [35:53<16:06,  8.71s/it]\u001b[A\n",
            "Iteration:  69%|██████▉   | 250/360 [36:01<15:27,  8.44s/it]\u001b[A\n",
            "Iteration:  70%|██████▉   | 251/360 [36:09<15:03,  8.29s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 252/360 [36:18<15:09,  8.42s/it]\u001b[A\n",
            "Iteration:  70%|███████   | 253/360 [36:27<15:15,  8.56s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 254/360 [36:34<14:20,  8.12s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 255/360 [36:42<14:09,  8.09s/it]\u001b[A\n",
            "Iteration:  71%|███████   | 256/360 [36:49<13:40,  7.89s/it]\u001b[A\n",
            "Iteration:  71%|███████▏  | 257/360 [36:58<13:54,  8.10s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 258/360 [37:06<13:43,  8.07s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 259/360 [37:11<12:22,  7.35s/it]\u001b[A\n",
            "Iteration:  72%|███████▏  | 260/360 [37:19<12:19,  7.40s/it]\u001b[A\n",
            "Iteration:  72%|███████▎  | 261/360 [37:27<12:45,  7.73s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 262/360 [37:34<11:54,  7.29s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 263/360 [37:42<12:10,  7.53s/it]\u001b[A\n",
            "Iteration:  73%|███████▎  | 264/360 [37:51<13:01,  8.14s/it]\u001b[A\n",
            "Iteration:  74%|███████▎  | 265/360 [38:00<13:13,  8.35s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 266/360 [38:09<13:10,  8.41s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 267/360 [38:17<13:04,  8.44s/it]\u001b[A\n",
            "Iteration:  74%|███████▍  | 268/360 [38:27<13:28,  8.78s/it]\u001b[A\n",
            "Iteration:  75%|███████▍  | 269/360 [38:33<11:54,  7.85s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 270/360 [38:40<11:47,  7.86s/it]\u001b[A\n",
            "Iteration:  75%|███████▌  | 271/360 [38:47<11:12,  7.56s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 272/360 [38:55<11:07,  7.58s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 273/360 [39:03<11:10,  7.71s/it]\u001b[A\n",
            "Iteration:  76%|███████▌  | 274/360 [39:11<11:22,  7.94s/it]\u001b[A\n",
            "Iteration:  76%|███████▋  | 275/360 [39:19<11:11,  7.90s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 276/360 [39:28<11:30,  8.22s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 277/360 [39:35<10:53,  7.88s/it]\u001b[A\n",
            "Iteration:  77%|███████▋  | 278/360 [39:43<10:48,  7.91s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 279/360 [39:54<11:56,  8.84s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 280/360 [40:01<11:05,  8.31s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 281/360 [40:11<11:39,  8.85s/it]\u001b[A\n",
            "Iteration:  78%|███████▊  | 282/360 [40:19<11:07,  8.55s/it]\u001b[A\n",
            "Iteration:  79%|███████▊  | 283/360 [40:27<10:48,  8.43s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 284/360 [40:35<10:13,  8.07s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 285/360 [40:43<10:16,  8.22s/it]\u001b[A\n",
            "Iteration:  79%|███████▉  | 286/360 [40:53<10:37,  8.62s/it]\u001b[A\n",
            "Iteration:  80%|███████▉  | 287/360 [41:01<10:10,  8.36s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 288/360 [41:08<09:37,  8.03s/it]\u001b[A\n",
            "Iteration:  80%|████████  | 289/360 [41:17<09:47,  8.27s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 290/360 [41:24<09:24,  8.06s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 291/360 [41:33<09:39,  8.40s/it]\u001b[A\n",
            "Iteration:  81%|████████  | 292/360 [41:42<09:41,  8.54s/it]\u001b[A\n",
            "Iteration:  81%|████████▏ | 293/360 [41:52<09:46,  8.75s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 294/360 [41:59<09:17,  8.45s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 295/360 [42:08<09:22,  8.66s/it]\u001b[A\n",
            "Iteration:  82%|████████▏ | 296/360 [42:16<09:03,  8.48s/it]\u001b[A\n",
            "Iteration:  82%|████████▎ | 297/360 [42:23<08:09,  7.78s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 298/360 [42:31<08:13,  7.97s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 299/360 [42:40<08:29,  8.35s/it]\u001b[A\n",
            "Iteration:  83%|████████▎ | 300/360 [42:49<08:24,  8.41s/it]\u001b[A\n",
            "Iteration:  84%|████████▎ | 301/360 [42:57<08:15,  8.40s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 302/360 [43:05<08:04,  8.35s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 303/360 [43:13<07:40,  8.09s/it]\u001b[A\n",
            "Iteration:  84%|████████▍ | 304/360 [43:22<07:44,  8.29s/it]\u001b[A\n",
            "Iteration:  85%|████████▍ | 305/360 [43:31<07:57,  8.69s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 306/360 [43:38<07:14,  8.05s/it]\u001b[A\n",
            "Iteration:  85%|████████▌ | 307/360 [43:43<06:26,  7.30s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 308/360 [43:54<07:08,  8.24s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 309/360 [44:04<07:29,  8.82s/it]\u001b[A\n",
            "Iteration:  86%|████████▌ | 310/360 [44:11<06:57,  8.35s/it]\u001b[A\n",
            "Iteration:  86%|████████▋ | 311/360 [44:20<06:55,  8.47s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 312/360 [44:27<06:22,  7.98s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 313/360 [44:35<06:18,  8.06s/it]\u001b[A\n",
            "Iteration:  87%|████████▋ | 314/360 [44:41<05:35,  7.30s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 315/360 [44:50<05:55,  7.89s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 316/360 [44:58<05:52,  8.02s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 317/360 [45:07<05:54,  8.25s/it]\u001b[A\n",
            "Iteration:  88%|████████▊ | 318/360 [45:14<05:31,  7.89s/it]\u001b[A\n",
            "Iteration:  89%|████████▊ | 319/360 [45:23<05:35,  8.19s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 320/360 [45:32<05:37,  8.44s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 321/360 [45:41<05:36,  8.64s/it]\u001b[A\n",
            "Iteration:  89%|████████▉ | 322/360 [45:48<05:12,  8.23s/it]\u001b[A\n",
            "Iteration:  90%|████████▉ | 323/360 [45:57<05:05,  8.25s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 324/360 [46:06<05:05,  8.48s/it]\u001b[A\n",
            "Iteration:  90%|█████████ | 325/360 [46:20<06:01, 10.33s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 326/360 [46:29<05:31,  9.76s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 327/360 [46:37<05:11,  9.43s/it]\u001b[A\n",
            "Iteration:  91%|█████████ | 328/360 [46:46<04:58,  9.34s/it]\u001b[A\n",
            "Iteration:  91%|█████████▏| 329/360 [46:55<04:42,  9.13s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 330/360 [47:06<04:46,  9.54s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 331/360 [47:14<04:26,  9.18s/it]\u001b[A\n",
            "Iteration:  92%|█████████▏| 332/360 [47:22<04:08,  8.86s/it]\u001b[A\n",
            "Iteration:  92%|█████████▎| 333/360 [47:34<04:25,  9.84s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 334/360 [47:43<04:06,  9.49s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 335/360 [47:51<03:49,  9.19s/it]\u001b[A\n",
            "Iteration:  93%|█████████▎| 336/360 [47:59<03:30,  8.78s/it]\u001b[A\n",
            "Iteration:  94%|█████████▎| 337/360 [48:07<03:13,  8.40s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 338/360 [48:14<02:54,  7.92s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 339/360 [48:23<02:53,  8.28s/it]\u001b[A\n",
            "Iteration:  94%|█████████▍| 340/360 [48:30<02:40,  8.00s/it]\u001b[A\n",
            "Iteration:  95%|█████████▍| 341/360 [48:38<02:31,  7.98s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 342/360 [48:47<02:27,  8.18s/it]\u001b[A\n",
            "Iteration:  95%|█████████▌| 343/360 [48:55<02:22,  8.36s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 344/360 [49:03<02:11,  8.24s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 345/360 [49:10<01:56,  7.78s/it]\u001b[A\n",
            "Iteration:  96%|█████████▌| 346/360 [49:18<01:50,  7.91s/it]\u001b[A\n",
            "Iteration:  96%|█████████▋| 347/360 [49:24<01:33,  7.23s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 348/360 [49:30<01:24,  7.01s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 349/360 [49:40<01:24,  7.68s/it]\u001b[A\n",
            "Iteration:  97%|█████████▋| 350/360 [49:49<01:21,  8.17s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 351/360 [49:57<01:12,  8.05s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 352/360 [50:03<01:00,  7.61s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 353/360 [50:12<00:55,  7.99s/it]\u001b[A\n",
            "Iteration:  98%|█████████▊| 354/360 [50:20<00:47,  7.95s/it]\u001b[A\n",
            "Iteration:  99%|█████████▊| 355/360 [50:31<00:44,  8.83s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 356/360 [50:38<00:33,  8.30s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 357/360 [50:47<00:25,  8.47s/it]\u001b[A\n",
            "Iteration:  99%|█████████▉| 358/360 [50:56<00:17,  8.68s/it]\u001b[A\n",
            "Iteration: 100%|█████████▉| 359/360 [51:03<00:08,  8.10s/it]\u001b[A\n",
            "Iteration: 100%|██████████| 360/360 [51:07<00:00,  7.00s/it]\u001b[A\n",
            "Epoch:  75%|███████▌  | 3/4 [3:36:25<54:28, 3268.07s/it]\n",
            "Convert Evaluating:   0%|          | 0/94 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:55:56 - Evaluation the model on  dataset after epoch 3:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Convert Evaluating:   1%|          | 1/94 [00:00<01:18,  1.19it/s]\u001b[A\n",
            "Convert Evaluating:   2%|▏         | 2/94 [00:01<01:16,  1.20it/s]\u001b[A\n",
            "Convert Evaluating:   3%|▎         | 3/94 [00:02<01:19,  1.14it/s]\u001b[A\n",
            "Convert Evaluating:   4%|▍         | 4/94 [00:03<01:18,  1.15it/s]\u001b[A\n",
            "Convert Evaluating:   5%|▌         | 5/94 [00:04<01:21,  1.10it/s]\u001b[A\n",
            "Convert Evaluating:   6%|▋         | 6/94 [00:05<01:25,  1.03it/s]\u001b[A\n",
            "Convert Evaluating:   7%|▋         | 7/94 [00:06<01:23,  1.04it/s]\u001b[A\n",
            "Convert Evaluating:   9%|▊         | 8/94 [00:07<01:21,  1.06it/s]\u001b[A\n",
            "Convert Evaluating:  10%|▉         | 9/94 [00:08<01:18,  1.08it/s]\u001b[A\n",
            "Convert Evaluating:  11%|█         | 10/94 [00:09<01:12,  1.16it/s]\u001b[A\n",
            "Convert Evaluating:  12%|█▏        | 11/94 [00:09<01:12,  1.15it/s]\u001b[A\n",
            "Convert Evaluating:  13%|█▎        | 12/94 [00:11<01:16,  1.07it/s]\u001b[A\n",
            "Convert Evaluating:  14%|█▍        | 13/94 [00:11<01:14,  1.09it/s]\u001b[A\n",
            "Convert Evaluating:  15%|█▍        | 14/94 [00:12<01:13,  1.08it/s]\u001b[A\n",
            "Convert Evaluating:  16%|█▌        | 15/94 [00:14<01:19,  1.01s/it]\u001b[A\n",
            "Convert Evaluating:  17%|█▋        | 16/94 [00:15<01:22,  1.06s/it]\u001b[A\n",
            "Convert Evaluating:  18%|█▊        | 17/94 [00:16<01:24,  1.09s/it]\u001b[A\n",
            "Convert Evaluating:  19%|█▉        | 18/94 [00:17<01:22,  1.08s/it]\u001b[A\n",
            "Convert Evaluating:  20%|██        | 19/94 [00:18<01:23,  1.11s/it]\u001b[A\n",
            "Convert Evaluating:  21%|██▏       | 20/94 [00:19<01:23,  1.12s/it]\u001b[A\n",
            "Convert Evaluating:  22%|██▏       | 21/94 [00:21<01:26,  1.18s/it]\u001b[A\n",
            "Convert Evaluating:  23%|██▎       | 22/94 [00:22<01:21,  1.14s/it]\u001b[A\n",
            "Convert Evaluating:  24%|██▍       | 23/94 [00:23<01:21,  1.14s/it]\u001b[A\n",
            "Convert Evaluating:  26%|██▌       | 24/94 [00:24<01:23,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  27%|██▋       | 25/94 [00:25<01:19,  1.15s/it]\u001b[A\n",
            "Convert Evaluating:  28%|██▊       | 26/94 [00:27<01:22,  1.21s/it]\u001b[A\n",
            "Convert Evaluating:  29%|██▊       | 27/94 [00:27<01:16,  1.14s/it]\u001b[A\n",
            "Convert Evaluating:  30%|██▉       | 28/94 [00:29<01:18,  1.18s/it]\u001b[A\n",
            "Convert Evaluating:  31%|███       | 29/94 [00:30<01:22,  1.27s/it]\u001b[A\n",
            "Convert Evaluating:  32%|███▏      | 30/94 [00:31<01:18,  1.23s/it]\u001b[A\n",
            "Convert Evaluating:  33%|███▎      | 31/94 [00:33<01:18,  1.25s/it]\u001b[A\n",
            "Convert Evaluating:  34%|███▍      | 32/94 [00:34<01:17,  1.24s/it]\u001b[A\n",
            "Convert Evaluating:  35%|███▌      | 33/94 [00:35<01:16,  1.25s/it]\u001b[A\n",
            "Convert Evaluating:  36%|███▌      | 34/94 [00:37<01:16,  1.27s/it]\u001b[A\n",
            "Convert Evaluating:  37%|███▋      | 35/94 [00:38<01:13,  1.24s/it]\u001b[A\n",
            "Convert Evaluating:  38%|███▊      | 36/94 [00:39<01:12,  1.25s/it]\u001b[A\n",
            "Convert Evaluating:  39%|███▉      | 37/94 [00:40<01:10,  1.24s/it]\u001b[A\n",
            "Convert Evaluating:  40%|████      | 38/94 [00:41<01:09,  1.24s/it]\u001b[A\n",
            "Convert Evaluating:  41%|████▏     | 39/94 [00:43<01:08,  1.24s/it]\u001b[A\n",
            "Convert Evaluating:  43%|████▎     | 40/94 [00:45<01:26,  1.59s/it]\u001b[A\n",
            "Convert Evaluating:  44%|████▎     | 41/94 [00:47<01:34,  1.78s/it]\u001b[A\n",
            "Convert Evaluating:  45%|████▍     | 42/94 [00:49<01:32,  1.79s/it]\u001b[A\n",
            "Convert Evaluating:  46%|████▌     | 43/94 [00:51<01:34,  1.86s/it]\u001b[A\n",
            "Convert Evaluating:  47%|████▋     | 44/94 [00:53<01:32,  1.85s/it]\u001b[A\n",
            "Convert Evaluating:  48%|████▊     | 45/94 [00:55<01:31,  1.87s/it]\u001b[A\n",
            "Convert Evaluating:  49%|████▉     | 46/94 [00:57<01:27,  1.82s/it]\u001b[A\n",
            "Convert Evaluating:  50%|█████     | 47/94 [00:58<01:26,  1.85s/it]\u001b[A\n",
            "Convert Evaluating:  51%|█████     | 48/94 [01:01<01:40,  2.19s/it]\u001b[A\n",
            "Convert Evaluating:  52%|█████▏    | 49/94 [01:04<01:36,  2.15s/it]\u001b[A\n",
            "Convert Evaluating:  53%|█████▎    | 50/94 [01:05<01:29,  2.04s/it]\u001b[A\n",
            "Convert Evaluating:  54%|█████▍    | 51/94 [01:07<01:23,  1.94s/it]\u001b[A\n",
            "Convert Evaluating:  55%|█████▌    | 52/94 [01:09<01:18,  1.88s/it]\u001b[A\n",
            "Convert Evaluating:  56%|█████▋    | 53/94 [01:11<01:16,  1.87s/it]\u001b[A\n",
            "Convert Evaluating:  57%|█████▋    | 54/94 [01:12<01:12,  1.82s/it]\u001b[A\n",
            "Convert Evaluating:  59%|█████▊    | 55/94 [01:15<01:15,  1.94s/it]\u001b[A\n",
            "Convert Evaluating:  60%|█████▉    | 56/94 [01:16<01:13,  1.95s/it]\u001b[A\n",
            "Convert Evaluating:  61%|██████    | 57/94 [01:18<01:10,  1.90s/it]\u001b[A\n",
            "Convert Evaluating:  62%|██████▏   | 58/94 [01:20<01:10,  1.94s/it]\u001b[A\n",
            "Convert Evaluating:  63%|██████▎   | 59/94 [01:22<01:06,  1.91s/it]\u001b[A\n",
            "Convert Evaluating:  64%|██████▍   | 60/94 [01:24<01:04,  1.88s/it]\u001b[A\n",
            "Convert Evaluating:  65%|██████▍   | 61/94 [01:26<01:01,  1.86s/it]\u001b[A\n",
            "Convert Evaluating:  66%|██████▌   | 62/94 [01:28<00:58,  1.83s/it]\u001b[A\n",
            "Convert Evaluating:  67%|██████▋   | 63/94 [01:30<00:58,  1.88s/it]\u001b[A\n",
            "Convert Evaluating:  68%|██████▊   | 64/94 [01:32<00:59,  1.99s/it]\u001b[A\n",
            "Convert Evaluating:  69%|██████▉   | 65/94 [01:34<01:00,  2.07s/it]\u001b[A\n",
            "Convert Evaluating:  70%|███████   | 66/94 [01:36<00:57,  2.06s/it]\u001b[A\n",
            "Convert Evaluating:  71%|███████▏  | 67/94 [01:38<00:56,  2.09s/it]\u001b[A\n",
            "Convert Evaluating:  72%|███████▏  | 68/94 [01:41<00:55,  2.15s/it]\u001b[A\n",
            "Convert Evaluating:  73%|███████▎  | 69/94 [01:43<00:54,  2.18s/it]\u001b[A\n",
            "Convert Evaluating:  74%|███████▍  | 70/94 [01:45<00:55,  2.31s/it]\u001b[A\n",
            "Convert Evaluating:  76%|███████▌  | 71/94 [01:48<00:51,  2.25s/it]\u001b[A\n",
            "Convert Evaluating:  77%|███████▋  | 72/94 [01:50<00:50,  2.31s/it]\u001b[A\n",
            "Convert Evaluating:  78%|███████▊  | 73/94 [01:53<00:51,  2.43s/it]\u001b[A\n",
            "Convert Evaluating:  79%|███████▊  | 74/94 [01:55<00:46,  2.33s/it]\u001b[A\n",
            "Convert Evaluating:  80%|███████▉  | 75/94 [01:57<00:43,  2.30s/it]\u001b[A\n",
            "Convert Evaluating:  81%|████████  | 76/94 [01:59<00:40,  2.27s/it]\u001b[A\n",
            "Convert Evaluating:  82%|████████▏ | 77/94 [02:02<00:39,  2.33s/it]\u001b[A\n",
            "Convert Evaluating:  83%|████████▎ | 78/94 [02:04<00:37,  2.33s/it]\u001b[A\n",
            "Convert Evaluating:  84%|████████▍ | 79/94 [02:06<00:31,  2.09s/it]\u001b[A\n",
            "Convert Evaluating:  85%|████████▌ | 80/94 [02:07<00:24,  1.77s/it]\u001b[A\n",
            "Convert Evaluating:  86%|████████▌ | 81/94 [02:08<00:20,  1.56s/it]\u001b[A\n",
            "Convert Evaluating:  87%|████████▋ | 82/94 [02:09<00:16,  1.37s/it]\u001b[A\n",
            "Convert Evaluating:  88%|████████▊ | 83/94 [02:09<00:13,  1.22s/it]\u001b[A\n",
            "Convert Evaluating:  89%|████████▉ | 84/94 [02:11<00:12,  1.20s/it]\u001b[A\n",
            "Convert Evaluating:  90%|█████████ | 85/94 [02:12<00:10,  1.14s/it]\u001b[A\n",
            "Convert Evaluating:  91%|█████████▏| 86/94 [02:13<00:09,  1.17s/it]\u001b[A\n",
            "Convert Evaluating:  93%|█████████▎| 87/94 [02:14<00:08,  1.17s/it]\u001b[A\n",
            "Convert Evaluating:  94%|█████████▎| 88/94 [02:15<00:06,  1.15s/it]\u001b[A\n",
            "Convert Evaluating:  95%|█████████▍| 89/94 [02:16<00:05,  1.10s/it]\u001b[A\n",
            "Convert Evaluating:  96%|█████████▌| 90/94 [02:17<00:04,  1.07s/it]\u001b[A\n",
            "Convert Evaluating:  97%|█████████▋| 91/94 [02:18<00:03,  1.04s/it]\u001b[A\n",
            "Convert Evaluating:  98%|█████████▊| 92/94 [02:19<00:02,  1.14s/it]\u001b[A\n",
            "Convert Evaluating:  99%|█████████▉| 93/94 [02:21<00:01,  1.14s/it]\u001b[A\n",
            "Convert Evaluating: 100%|██████████| 94/94 [02:21<00:00,  1.00s/it]\u001b[A\n",
            "Epoch:  75%|███████▌  | 3/4 [3:38:47<54:28, 3268.07s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:58:18 - Cosine-Similarity :\tPearson: 0.8730\tSpearman: 0.8735\n",
            "2019-11-28 11:58:18 - Manhattan-Distance:\tPearson: 0.8450\tSpearman: 0.8482\n",
            "2019-11-28 11:58:18 - Euclidean-Distance:\tPearson: 0.8454\tSpearman: 0.8485\n",
            "2019-11-28 11:58:18 - Dot-Product-Similarity:\tPearson: 0.8080\tSpearman: 0.8119\n",
            "2019-11-28 11:58:18 - Save model to output/training_stsbenchmark_bert-2019-11-28_08-19-18\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rEpoch: 100%|██████████| 4/4 [3:38:48<00:00, 3250.84s/it]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:58:19 - Load pretrained SentenceTransformer: output/training_stsbenchmark_bert-2019-11-28_08-19-18\n",
            "2019-11-28 11:58:19 - Load SentenceTransformer from folder: output/training_stsbenchmark_bert-2019-11-28_08-19-18\n",
            "2019-11-28 11:58:19 - loading configuration file output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/config.json\n",
            "2019-11-28 11:58:19 - Model config {\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"finetuning_task\": null,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"num_labels\": 2,\n",
            "  \"output_attentions\": false,\n",
            "  \"output_hidden_states\": false,\n",
            "  \"torchscript\": false,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "2019-11-28 11:58:19 - loading weights file output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/pytorch_model.bin\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Convert dataset:  28%|██▊       | 392/1379 [00:00<00:00, 3914.37it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:58:20 - Model name 'output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc). Assuming 'output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT' is a path or url to a directory containing tokenizer files.\n",
            "2019-11-28 11:58:20 - loading file output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/vocab.txt\n",
            "2019-11-28 11:58:20 - loading file output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/added_tokens.json\n",
            "2019-11-28 11:58:20 - loading file output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/special_tokens_map.json\n",
            "2019-11-28 11:58:21 - Use pytorch device: cpu\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Convert dataset: 100%|██████████| 1379/1379 [00:00<00:00, 2910.46it/s]\n",
            "Convert Evaluating:   0%|          | 0/87 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 11:58:21 - Num sentences: 1379\n",
            "2019-11-28 11:58:21 - Sentences 0 longer than max_seqence_length: 0\n",
            "2019-11-28 11:58:21 - Sentences 1 longer than max_seqence_length: 0\n",
            "2019-11-28 11:58:21 - Evaluation the model on  dataset:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Convert Evaluating: 100%|██████████| 87/87 [01:58<00:00,  1.13it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2019-11-28 12:00:19 - Cosine-Similarity :\tPearson: 0.8507\tSpearman: 0.8497\n",
            "2019-11-28 12:00:19 - Manhattan-Distance:\tPearson: 0.8381\tSpearman: 0.8384\n",
            "2019-11-28 12:00:19 - Euclidean-Distance:\tPearson: 0.8390\tSpearman: 0.8392\n",
            "2019-11-28 12:00:19 - Dot-Product-Similarity:\tPearson: 0.7777\tSpearman: 0.7690\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8497360050809678"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z7-dl3jkOTM5",
        "colab_type": "code",
        "outputId": "535e0ff1-b3d0-4903-8bac-60542b36a6c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 254
        }
      },
      "source": [
        "!zip -r output.zip output"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  adding: output/ (stored 0%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/ (stored 0%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/config.json (stored 0%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/1_Pooling/ (stored 0%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/1_Pooling/config.json (deflated 47%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/similarity_evaluation_results.csv (deflated 50%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/modules.json (deflated 51%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/ (stored 0%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/config.json (deflated 51%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/vocab.txt (deflated 53%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/pytorch_model.bin (deflated 7%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/special_tokens_map.json (deflated 40%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/added_tokens.json (stored 0%)\n",
            "  adding: output/training_stsbenchmark_bert-2019-11-28_08-19-18/0_BERT/sentence_bert_config.json (deflated 4%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JSrHHk4Y9Ufa",
        "colab_type": "code",
        "outputId": "c5baec2a-d7be-47b9-a4a5-0908ac91706d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i4Knride_M6Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "root_dir = \"/content/gdrive/My Drive/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tp6P-5tP91Aq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp output.zip '/content/gdrive/My Drive/bert_finetuned_stsb.zip'"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}