mask_transformers_en.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mask_transformers_en.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOVTVky2upmuov1A/ZyfNc3",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8d0f9b71a82b4b199991336db49b62c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8c58787f29834df68097a86b8bda2fc4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_030d242c47e446dba3152e5e0bc4e408",
              "IPY_MODEL_04af08e8f81946598f6165e3fa5b42dc"
            ]
          }
        },
        "8c58787f29834df68097a86b8bda2fc4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "030d242c47e446dba3152e5e0bc4e408": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0c49346deaed45fbbb43d11ae8b98dc1",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f4e14f2b32bb411dbbb01757ad77eeb4"
          }
        },
        "04af08e8f81946598f6165e3fa5b42dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_78934383bb3949e8be47df2561362a66",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 232k/232k [00:00&lt;00:00, 4.37MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_390367d8aad442e9ba2f87b7ca68065b"
          }
        },
        "0c49346deaed45fbbb43d11ae8b98dc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f4e14f2b32bb411dbbb01757ad77eeb4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "78934383bb3949e8be47df2561362a66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "390367d8aad442e9ba2f87b7ca68065b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mashyko/NLP_BERT_Transformers/blob/master/mask_transformers_en.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqtlhBawKB0s",
        "colab_type": "code",
        "outputId": "1b44fead-b86a-4553-8441-321e649ca921",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 666
        }
      },
      "source": [
        "! pip install transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ee/fc/bd726a15ab2c66dc09306689d04da07a3770dad724f0883f0a4bfb745087/transformers-2.4.1-py3-none-any.whl (475kB)\n",
            "\u001b[K     |████████████████████████████████| 481kB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 14.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.11.10)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 19.9MB/s \n",
            "\u001b[?25hCollecting tokenizers==0.0.11\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/5e/36/7af38d572c935f8e0462ec7b4f7a46d73a2b3b1a938f50a5e8132d5b2dc5/tokenizers-0.0.11-cp36-cp36m-manylinux1_x86_64.whl (3.1MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1MB 23.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.5)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.2)\n",
            "Requirement already satisfied: botocore<1.15.0,>=1.14.10 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.14.10)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.10->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.10->boto3->transformers) (2.6.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=8b3cff0b128f49fdd3cb3012f224af2cda1a692202ef7bf8dc38e8a7ded40087\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, sentencepiece, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.38 sentencepiece-0.1.85 tokenizers-0.0.11 transformers-2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "33SrRqW2JuOy",
        "colab_type": "code",
        "outputId": "5fd44443-f893-4ac8-9937-08ee586d46e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 234,
          "referenced_widgets": [
            "8d0f9b71a82b4b199991336db49b62c7",
            "8c58787f29834df68097a86b8bda2fc4",
            "030d242c47e446dba3152e5e0bc4e408",
            "04af08e8f81946598f6165e3fa5b42dc",
            "0c49346deaed45fbbb43d11ae8b98dc1",
            "f4e14f2b32bb411dbbb01757ad77eeb4",
            "78934383bb3949e8be47df2561362a66",
            "390367d8aad442e9ba2f87b7ca68065b"
          ]
        }
      },
      "source": [
        "import torch\n",
        "from transformers import BertTokenizer, BertModel, BertForMaskedLM\n",
        "\n",
        "# OPTIONAL: if you want to have more information on what's happening under the hood, activate the logger as follows\n",
        "import logging\n",
        "logging.basicConfig(level=logging.INFO)\n",
        "\n",
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Tokenize input\n",
        "text = \"[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]\"\n",
        "tokenized_text = tokenizer.tokenize(text)\n",
        "\n",
        "# Mask a token that we will try to predict back with `BertForMaskedLM`\n",
        "masked_index = 8\n",
        "tokenized_text[masked_index] = '[MASK]'\n",
        "assert tokenized_text == ['[CLS]', 'who', 'was', 'jim', 'henson', '?', '[SEP]', 'jim', '[MASK]', 'was', 'a', 'puppet', '##eer', '[SEP]']\n",
        "\n",
        "# Convert token to vocabulary indices\n",
        "indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "# Define sentence A and B indices associated to 1st and 2nd sentences (see paper)\n",
        "segments_ids = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]\n",
        "\n",
        "# Convert inputs to PyTorch tensors\n",
        "tokens_tensor = torch.tensor([indexed_tokens])\n",
        "segments_tensors = torch.tensor([segments_ids])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140003108850768 acquired on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmp7tschp0c\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8d0f9b71a82b4b199991336db49b62c7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt in cache at /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
            "INFO:filelock:Lock 140003108850768 released on /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084.lock\n",
            "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.cache/torch/transformers/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3WtRCBD3KVuN",
        "colab_type": "code",
        "outputId": "f6868375-8b21-4b9f-ee92-db163ffac218",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 904
        }
      },
      "source": [
        "# Load pre-trained model (weights)\n",
        "model = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Set the model in evaluation mode to deactivate the DropOut modules\n",
        "# This is IMPORTANT to have reproducible results during evaluation!\n",
        "model.eval()\n",
        "\n",
        "# If you have a GPU, put everything on cuda\n",
        "tokens_tensor = tokens_tensor.to('cuda')\n",
        "segments_tensors = segments_tensors.to('cuda')\n",
        "model.to('cuda')\n",
        "\n",
        "# Predict hidden states features for each layer\n",
        "with torch.no_grad():\n",
        "    # See the models docstrings for the detail of the inputs\n",
        "    outputs = model(tokens_tensor, token_type_ids=segments_tensors)\n",
        "    # Transformers models always output tuples.\n",
        "    # See the models docstrings for the detail of all the outputs\n",
        "    # In our case, the first element is the hidden state of the last layer of the Bert model\n",
        "    encoded_layers = outputs[0]\n",
        "# We have encoded our input sequence in a FloatTensor of shape (batch size, sequence length, model hidden dimension)\n",
        "assert tuple(encoded_layers.shape) == (1, len(indexed_tokens), model.config.hidden_size)\n",
        "print ('encoded layers =', encoded_layers.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.configuration_utils:loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at /root/.cache/torch/transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.8f56353af4a709bf5ff0fbc915d8f5b42bfff892cbb6ac98c3c45f481a03c685\n",
            "INFO:transformers.configuration_utils:Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"do_sample\": false,\n",
            "  \"eos_token_ids\": 0,\n",
            "  \"finetuning_task\": null,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"is_decoder\": false,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"length_penalty\": 1.0,\n",
            "  \"max_length\": 20,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_beams\": 1,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"num_labels\": 2,\n",
            "  \"num_return_sequences\": 1,\n",
            "  \"output_attentions\": false,\n",
            "  \"output_hidden_states\": false,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"pruned_heads\": {},\n",
            "  \"repetition_penalty\": 1.0,\n",
            "  \"temperature\": 1.0,\n",
            "  \"top_k\": 50,\n",
            "  \"top_p\": 1.0,\n",
            "  \"torchscript\": false,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_bfloat16\": false,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "INFO:transformers.modeling_utils:loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at /root/.cache/torch/transformers/aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "encoded layers = torch.Size([1, 14, 768])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pkHvBJuZK6ak",
        "colab_type": "code",
        "outputId": "9aa4cbab-d671-4a0a-b6b7-dd9aa72491c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 938
        }
      },
      "source": [
        "# Load pre-trained model (weights)\n",
        "model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
        "model.eval()\n",
        "\n",
        "# If you have a GPU, put everything on cuda\n",
        "tokens_tensor = tokens_tensor.to('cuda')\n",
        "segments_tensors = segments_tensors.to('cuda')\n",
        "model.to('cuda')\n",
        "\n",
        "# Predict all tokens\n",
        "with torch.no_grad():\n",
        "    outputs = model(tokens_tensor, token_type_ids=segments_tensors)\n",
        "    predictions = outputs[0]\n",
        "\n",
        "# confirm we were able to predict 'henson'\n",
        "predicted_index = torch.argmax(predictions[0, masked_index]).item()\n",
        "predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0]\n",
        "assert predicted_token == 'henson'\n",
        "\n",
        "#_, predicted_indexes = torch.topk(predictions[0, masked_index], k=5)\n",
        "#predicted_tokens = tokenizer.convert_ids_to_tokens(predicted_indexes.tolist())\n",
        "print('Predicted Token =' ,  predicted_token)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.configuration_utils:loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-config.json from cache at /root/.cache/torch/transformers/4dad0251492946e18ac39290fcfe91b89d370fee250efe9521476438fe8ca185.8f56353af4a709bf5ff0fbc915d8f5b42bfff892cbb6ac98c3c45f481a03c685\n",
            "INFO:transformers.configuration_utils:Model config BertConfig {\n",
            "  \"architectures\": [\n",
            "    \"BertForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"do_sample\": false,\n",
            "  \"eos_token_ids\": 0,\n",
            "  \"finetuning_task\": null,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"is_decoder\": false,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"length_penalty\": 1.0,\n",
            "  \"max_length\": 20,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_beams\": 1,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"num_labels\": 2,\n",
            "  \"num_return_sequences\": 1,\n",
            "  \"output_attentions\": false,\n",
            "  \"output_hidden_states\": false,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"pruned_heads\": {},\n",
            "  \"repetition_penalty\": 1.0,\n",
            "  \"temperature\": 1.0,\n",
            "  \"top_k\": 50,\n",
            "  \"top_p\": 1.0,\n",
            "  \"torchscript\": false,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_bfloat16\": false,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "INFO:transformers.modeling_utils:loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-pytorch_model.bin from cache at /root/.cache/torch/transformers/aa1ef1aede4482d0dbcd4d52baad8ae300e60902e88fcb0bebdec09afd232066.36ca03ab34a1a5d5fa7bc3d03d55c4fa650fed07220e2eeebc06ce58d0e9a157\n",
            "INFO:transformers.modeling_utils:Weights of BertForMaskedLM not initialized from pretrained model: ['cls.predictions.decoder.bias']\n",
            "INFO:transformers.modeling_utils:Weights from pretrained model not used in BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Predicted Token = henson\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}