masked_transformers_example.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "file_extension": ".py",
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    },
    "mimetype": "text/x-python",
    "name": "python",
    "npconvert_exporter": "python",
    "pygments_lexer": "ipython3",
    "version": 3,
    "colab": {
      "name": "masked_transformers_example.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mashyko/NLP_BERT_Transformers/blob/master/masked_transformers_example.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c98qQ9NbdSRG",
        "colab_type": "code",
        "outputId": "89e8cfb0-3f96-48f9-8d46-ab306335cd41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "! pip install mecab-python3"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: mecab-python3 in /usr/local/lib/python3.6/dist-packages (0.996.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRpn1B4rdq89",
        "colab_type": "code",
        "outputId": "fcc33600-c419-4488-8f53-f5051672e90b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        }
      },
      "source": [
        "! pip install transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (2.4.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.38)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Requirement already satisfied: tokenizers==0.0.11 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.11)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.11.10)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.5)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.85)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.3.2)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: botocore<1.15.0,>=1.14.10 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.14.10)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.10->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.10->boto3->transformers) (2.6.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WvdVmibYdPpb",
        "colab_type": "code",
        "outputId": "ea0058ac-19cb-43ed-9d44-89d9f631e118",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 63
        }
      },
      "source": [
        "import torch\n",
        "from transformers.tokenization_bert_japanese import BertJapaneseTokenizer\n",
        "from transformers.modeling_bert import BertForMaskedLM"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iacLiohBdPpi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = BertJapaneseTokenizer.from_pretrained('bert-base-japanese-whole-word-masking')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "o_FUuWcRdPpn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = BertForMaskedLM.from_pretrained('bert-base-japanese-whole-word-masking')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M-m-zvw9iUiG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_ids = tokenizer.encode(f'''\n",
        "    宇宙塵は星から生まれ、宇宙に飛んでいきます。これら小さな塵は最終的にはくっつきあって新たな{tokenizer.mask_token}となったり、惑星や月、隕石になったりします。\n",
        "''', return_tensors='pt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QCAnhTO_dPpu",
        "colab_type": "code",
        "outputId": "c26a2509-1a66-4263-d15f-0bcbef40b353",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "print(input_ids)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[    2,  1669, 30810,     9,  1414,    40,  1115,     6,  1669,     7,\n",
            "         15782,    12,  4481,  2610,     8,   875,  3060, 19688,     9,  1140,\n",
            "            81,     7,     9,   609, 28468,  7093,   102,    16,  1485,    18,\n",
            "             4,    13,    58,   790,     6,  4184,    49,    37,     6, 16293,\n",
            "             7,    58,   790,    15,  2610,     8,     3]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HBAjXx7xdPpy",
        "colab_type": "code",
        "outputId": "fc42097c-2b03-48d0-8005-174e03dbcb42",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "print(tokenizer.convert_ids_to_tokens(input_ids[0].tolist()))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[CLS]', '宇宙', '##塵', 'は', '星', 'から', '生まれ', '、', '宇宙', 'に', '飛ん', 'で', 'いき', 'ます', '。', 'これら', '小さな', '塵', 'は', '最終', '的', 'に', 'は', 'く', '##っ', '##つき', 'あっ', 'て', '新た', 'な', '[MASK]', 'と', 'なっ', 'たり', '、', '惑星', 'や', '月', '、', '隕石', 'に', 'なっ', 'たり', 'し', 'ます', '。', '[SEP]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wb7X8CEpdPp2",
        "colab_type": "code",
        "outputId": "62771380-1216-4277-a7dc-9a98ad46c16e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "masked_index = torch.where(input_ids == tokenizer.mask_token_id)[1].tolist()[0]\n",
        "print(masked_index)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZfHrXHEJdPp5",
        "colab_type": "code",
        "outputId": "7e7bf849-25d2-4438-e1b0-7be3d5df0004",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "result = model(input_ids)\n",
        "pred_ids = result[0][:, masked_index].topk(5).indices.tolist()[0]\n",
        "for pred_id in pred_ids:\n",
        "    output_ids = input_ids.tolist()[0]\n",
        "    output_ids[masked_index] = pred_id\n",
        "    print(tokenizer.decode(output_ids))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[CLS] 宇宙塵 は 星 から 生まれ 、 宇宙 に 飛ん で いき ます 。 これら 小さな 塵 は 最終 的 に は くっつき あっ て 新た な 天体 と なっ たり 、 惑星 や 月 、 隕石 に なっ たり し ます 。 [SEP]\n",
            "[CLS] 宇宙塵 は 星 から 生まれ 、 宇宙 に 飛ん で いき ます 。 これら 小さな 塵 は 最終 的 に は くっつき あっ て 新た な 星 と なっ たり 、 惑星 や 月 、 隕石 に なっ たり し ます 。 [SEP]\n",
            "[CLS] 宇宙塵 は 星 から 生まれ 、 宇宙 に 飛ん で いき ます 。 これら 小さな 塵 は 最終 的 に は くっつき あっ て 新た な 塵 と なっ たり 、 惑星 や 月 、 隕石 に なっ たり し ます 。 [SEP]\n",
            "[CLS] 宇宙塵 は 星 から 生まれ 、 宇宙 に 飛ん で いき ます 。 これら 小さな 塵 は 最終 的 に は くっつき あっ て 新た な 物質 と なっ たり 、 惑星 や 月 、 隕石 に なっ たり し ます 。 [SEP]\n",
            "[CLS] 宇宙塵 は 星 から 生まれ 、 宇宙 に 飛ん で いき ます 。 これら 小さな 塵 は 最終 的 に は くっつき あっ て 新た な 惑星 と なっ たり 、 惑星 や 月 、 隕石 に なっ たり し ます 。 [SEP]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-0rb9fLAdPp8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wi_aIZ8OijTu",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    }
  ]
}