macula_gpt_insights.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ryderwishart/biblical-machine-learning/blob/main/macula_gpt_insights.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iAtVZHrlco6O"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install openai transformers torch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RSKKnJImR9HH",
        "outputId": "daa5d8a3-8cf0-4dd9-b6cc-c89af8f53ca5"
      },
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: openai in /usr/local/lib/python3.9/dist-packages (0.27.2)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.27.2)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (1.13.1+cu116)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from openai) (3.8.4)\n",
            "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.9/dist-packages (from openai) (2.27.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from openai) (4.65.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.10.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch) (4.5.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (2.0.12)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (1.8.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (22.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (6.0.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (1.3.3)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (4.0.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 81,
      "metadata": {
        "id": "0HMGZN75cr2M"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline\n",
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jf9ufdZ9eZ-k"
      },
      "source": [
        "## Build Corpus"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "if 'macula-greek.tsv' not in [path for path in os.listdir()]:\n",
        "    !wget -q 'https://raw.githubusercontent.com/Clear-Bible/macula-greek/main/Nestle1904/TSV/macula-greek.tsv'\n",
        "if 'marble-domain-label-mapping.json' not in [path for path in os.listdir()]:\n",
        "    !wget -q 'https://raw.githubusercontent.com/Clear-Bible/macula-greek/main/sources/MARBLE/SDBG/marble-domain-label-mapping.json'"
      ],
      "metadata": {
        "id": "zGHbSUYD_K8a"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "CtUz3kg5cuFe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e58bf307-b21a-4764-96a0-8cb974ef6eb6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['.config',\n",
              " 'marble-domain-label-mapping.json',\n",
              " 'macula-greek.tsv',\n",
              " 'sample_data']"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "os.listdir()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import Macula Greek data\n",
        "mg = pd.read_csv('macula-greek.tsv', index_col='xml:id', sep='\\t', header=0, converters={'*': str}).fillna('missing')\n",
        "# mg['domain'] = mg['domain'].astype(str).fillna('missing')\n",
        "\n",
        "# Extract book, chapter, and verse into separate columns\n",
        "mg[['book', 'chapter', 'verse']] = mg['ref'].str.extract(r'(\\d?[A-Z]+)\\s(\\d+):(\\d+)')\n",
        "\n",
        "# Add columns for book + chapter, and book + chapter + verse for easier grouping\n",
        "mg['book_chapter'] = mg['book'] + ' ' + mg['chapter'].astype(str)\n",
        "mg['book_chapter_verse'] = mg['book_chapter'] + ':' + mg['verse'].astype(str)\n",
        "\n",
        "# Display the updated data frame\n",
        "mg.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 418
        },
        "id": "zfcFUhrl87Xy",
        "outputId": "2809a124-32a0-4903-e3ea-1cbf48e6cef9"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                    ref     role class    type               gloss      text  \\\n",
              "xml:id                                                                         \n",
              "n40001001001  MAT 1:1!1  missing  noun  common          [The] book    Βίβλος   \n",
              "n40001001002  MAT 1:1!2  missing  noun  common  of [the] genealogy  γενέσεως   \n",
              "n40001001003  MAT 1:1!3  missing  noun  proper            of Jesus     Ἰησοῦ   \n",
              "n40001001004  MAT 1:1!4  missing  noun  proper              Christ   Χριστοῦ   \n",
              "n40001001005  MAT 1:1!5  missing  noun  common                 son      υἱοῦ   \n",
              "\n",
              "             after    lemma normalized  strong  ...         domain  \\\n",
              "xml:id                                          ...                  \n",
              "n40001001001         βίβλος     Βίβλος     976  ...         033005   \n",
              "n40001001002        γένεσις   γενέσεως    1078  ...  010002 033003   \n",
              "n40001001003         Ἰησοῦς      Ἰησοῦ    2424  ...         093001   \n",
              "n40001001004        Χριστός    Χριστοῦ    5547  ...         093001   \n",
              "n40001001005           υἱός       υἱοῦ    5207  ...         010002   \n",
              "\n",
              "                       ln    frame  subjref referent book chapter verse  \\\n",
              "xml:id                                                                    \n",
              "n40001001001        33.38  missing  missing  missing  MAT       1     1   \n",
              "n40001001002  10.24 33.19  missing  missing  missing  MAT       1     1   \n",
              "n40001001003      93.169a  missing  missing  missing  MAT       1     1   \n",
              "n40001001004       93.387  missing  missing  missing  MAT       1     1   \n",
              "n40001001005        10.30  missing  missing  missing  MAT       1     1   \n",
              "\n",
              "             book_chapter book_chapter_verse  \n",
              "xml:id                                        \n",
              "n40001001001        MAT 1            MAT 1:1  \n",
              "n40001001002        MAT 1            MAT 1:1  \n",
              "n40001001003        MAT 1            MAT 1:1  \n",
              "n40001001004        MAT 1            MAT 1:1  \n",
              "n40001001005        MAT 1            MAT 1:1  \n",
              "\n",
              "[5 rows x 29 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b9939a09-2ad0-49b1-96c8-7d9d585579c4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ref</th>\n",
              "      <th>role</th>\n",
              "      <th>class</th>\n",
              "      <th>type</th>\n",
              "      <th>gloss</th>\n",
              "      <th>text</th>\n",
              "      <th>after</th>\n",
              "      <th>lemma</th>\n",
              "      <th>normalized</th>\n",
              "      <th>strong</th>\n",
              "      <th>...</th>\n",
              "      <th>domain</th>\n",
              "      <th>ln</th>\n",
              "      <th>frame</th>\n",
              "      <th>subjref</th>\n",
              "      <th>referent</th>\n",
              "      <th>book</th>\n",
              "      <th>chapter</th>\n",
              "      <th>verse</th>\n",
              "      <th>book_chapter</th>\n",
              "      <th>book_chapter_verse</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>xml:id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>n40001001001</th>\n",
              "      <td>MAT 1:1!1</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>common</td>\n",
              "      <td>[The] book</td>\n",
              "      <td>Βίβλος</td>\n",
              "      <td></td>\n",
              "      <td>βίβλος</td>\n",
              "      <td>Βίβλος</td>\n",
              "      <td>976</td>\n",
              "      <td>...</td>\n",
              "      <td>033005</td>\n",
              "      <td>33.38</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001002</th>\n",
              "      <td>MAT 1:1!2</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>common</td>\n",
              "      <td>of [the] genealogy</td>\n",
              "      <td>γενέσεως</td>\n",
              "      <td></td>\n",
              "      <td>γένεσις</td>\n",
              "      <td>γενέσεως</td>\n",
              "      <td>1078</td>\n",
              "      <td>...</td>\n",
              "      <td>010002 033003</td>\n",
              "      <td>10.24 33.19</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001003</th>\n",
              "      <td>MAT 1:1!3</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>proper</td>\n",
              "      <td>of Jesus</td>\n",
              "      <td>Ἰησοῦ</td>\n",
              "      <td></td>\n",
              "      <td>Ἰησοῦς</td>\n",
              "      <td>Ἰησοῦ</td>\n",
              "      <td>2424</td>\n",
              "      <td>...</td>\n",
              "      <td>093001</td>\n",
              "      <td>93.169a</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001004</th>\n",
              "      <td>MAT 1:1!4</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>proper</td>\n",
              "      <td>Christ</td>\n",
              "      <td>Χριστοῦ</td>\n",
              "      <td></td>\n",
              "      <td>Χριστός</td>\n",
              "      <td>Χριστοῦ</td>\n",
              "      <td>5547</td>\n",
              "      <td>...</td>\n",
              "      <td>093001</td>\n",
              "      <td>93.387</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001005</th>\n",
              "      <td>MAT 1:1!5</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>common</td>\n",
              "      <td>son</td>\n",
              "      <td>υἱοῦ</td>\n",
              "      <td></td>\n",
              "      <td>υἱός</td>\n",
              "      <td>υἱοῦ</td>\n",
              "      <td>5207</td>\n",
              "      <td>...</td>\n",
              "      <td>010002</td>\n",
              "      <td>10.30</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 29 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b9939a09-2ad0-49b1-96c8-7d9d585579c4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b9939a09-2ad0-49b1-96c8-7d9d585579c4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b9939a09-2ad0-49b1-96c8-7d9d585579c4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mg.dtypes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "05c2xyhUKad-",
        "outputId": "a7d2b5b0-d9ab-46b5-c79e-9e86b6b4a496"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ref                   object\n",
              "role                  object\n",
              "class                 object\n",
              "type                  object\n",
              "gloss                 object\n",
              "text                  object\n",
              "after                 object\n",
              "lemma                 object\n",
              "normalized            object\n",
              "strong                 int64\n",
              "morph                 object\n",
              "person                object\n",
              "number                object\n",
              "gender                object\n",
              "case                  object\n",
              "tense                 object\n",
              "voice                 object\n",
              "mood                  object\n",
              "degree                object\n",
              "domain                object\n",
              "ln                    object\n",
              "frame                 object\n",
              "subjref               object\n",
              "referent              object\n",
              "book                  object\n",
              "chapter               object\n",
              "verse                 object\n",
              "book_chapter          object\n",
              "book_chapter_verse    object\n",
              "dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Import domain-label mapping\n",
        "import json\n",
        "\n",
        "# Open the JSON file\n",
        "with open('marble-domain-label-mapping.json', 'r') as f:\n",
        "\n",
        "    # Load the contents of the file as a dictionary\n",
        "    domain_labels = json.load(f)\n",
        "\n",
        "domain_labels['missing'] = 'no domain'\n",
        "domain_labels['nan'] = 'no domain'\n",
        "\n",
        "# Display the resulting dictionary\n",
        "count = 0\n",
        "for d, l in domain_labels.items():\n",
        "    print(d, l)\n",
        "    if count > 5:\n",
        "        break\n",
        "    count += 1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Aqu1upkw_4UA",
        "outputId": "e728c5cd-1043-4225-c31c-bf93142c0746"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "001 Geographical Objects and Features\n",
            "001001 Universe, Creation\n",
            "001002 Regions Above the Earth\n",
            "001003 Regions Below the Surface of the Earth\n",
            "001004 Heavenly Bodies\n",
            "001005 Atmospheric Objects\n",
            "001006 The Earth's Surface\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def get_domain_label(domain_string_number):\n",
        "    labels = [domain_labels[label] for label in domain_string_number.split(' ')]\n",
        "    return labels\n",
        "\n",
        "mg['domain_label'] = mg['domain'].apply(get_domain_label)\n",
        "\n",
        "mg.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 522
        },
        "id": "-RF9YDEq_ThY",
        "outputId": "3f219cbe-e936-4eee-e3dc-5643de7f162b"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                    ref     role class    type               gloss      text  \\\n",
              "xml:id                                                                         \n",
              "n40001001001  MAT 1:1!1  missing  noun  common          [The] book    Βίβλος   \n",
              "n40001001002  MAT 1:1!2  missing  noun  common  of [the] genealogy  γενέσεως   \n",
              "n40001001003  MAT 1:1!3  missing  noun  proper            of Jesus     Ἰησοῦ   \n",
              "n40001001004  MAT 1:1!4  missing  noun  proper              Christ   Χριστοῦ   \n",
              "n40001001005  MAT 1:1!5  missing  noun  common                 son      υἱοῦ   \n",
              "\n",
              "             after    lemma normalized  strong  ...           ln    frame  \\\n",
              "xml:id                                          ...                         \n",
              "n40001001001         βίβλος     Βίβλος     976  ...        33.38  missing   \n",
              "n40001001002        γένεσις   γενέσεως    1078  ...  10.24 33.19  missing   \n",
              "n40001001003         Ἰησοῦς      Ἰησοῦ    2424  ...      93.169a  missing   \n",
              "n40001001004        Χριστός    Χριστοῦ    5547  ...       93.387  missing   \n",
              "n40001001005           υἱός       υἱοῦ    5207  ...        10.30  missing   \n",
              "\n",
              "              subjref referent book chapter verse book_chapter  \\\n",
              "xml:id                                                           \n",
              "n40001001001  missing  missing  MAT       1     1        MAT 1   \n",
              "n40001001002  missing  missing  MAT       1     1        MAT 1   \n",
              "n40001001003  missing  missing  MAT       1     1        MAT 1   \n",
              "n40001001004  missing  missing  MAT       1     1        MAT 1   \n",
              "n40001001005  missing  missing  MAT       1     1        MAT 1   \n",
              "\n",
              "             book_chapter_verse  \\\n",
              "xml:id                            \n",
              "n40001001001            MAT 1:1   \n",
              "n40001001002            MAT 1:1   \n",
              "n40001001003            MAT 1:1   \n",
              "n40001001004            MAT 1:1   \n",
              "n40001001005            MAT 1:1   \n",
              "\n",
              "                                                   domain_label  \n",
              "xml:id                                                           \n",
              "n40001001001                                 [Written Language]  \n",
              "n40001001002  [Kinship Relations Involving Successive Genera...  \n",
              "n40001001003                                          [Persons]  \n",
              "n40001001004                                          [Persons]  \n",
              "n40001001005  [Kinship Relations Involving Successive Genera...  \n",
              "\n",
              "[5 rows x 30 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-df0c4553-ce8b-4e77-a610-02bde31d7bf0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ref</th>\n",
              "      <th>role</th>\n",
              "      <th>class</th>\n",
              "      <th>type</th>\n",
              "      <th>gloss</th>\n",
              "      <th>text</th>\n",
              "      <th>after</th>\n",
              "      <th>lemma</th>\n",
              "      <th>normalized</th>\n",
              "      <th>strong</th>\n",
              "      <th>...</th>\n",
              "      <th>ln</th>\n",
              "      <th>frame</th>\n",
              "      <th>subjref</th>\n",
              "      <th>referent</th>\n",
              "      <th>book</th>\n",
              "      <th>chapter</th>\n",
              "      <th>verse</th>\n",
              "      <th>book_chapter</th>\n",
              "      <th>book_chapter_verse</th>\n",
              "      <th>domain_label</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>xml:id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>n40001001001</th>\n",
              "      <td>MAT 1:1!1</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>common</td>\n",
              "      <td>[The] book</td>\n",
              "      <td>Βίβλος</td>\n",
              "      <td></td>\n",
              "      <td>βίβλος</td>\n",
              "      <td>Βίβλος</td>\n",
              "      <td>976</td>\n",
              "      <td>...</td>\n",
              "      <td>33.38</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "      <td>[Written Language]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001002</th>\n",
              "      <td>MAT 1:1!2</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>common</td>\n",
              "      <td>of [the] genealogy</td>\n",
              "      <td>γενέσεως</td>\n",
              "      <td></td>\n",
              "      <td>γένεσις</td>\n",
              "      <td>γενέσεως</td>\n",
              "      <td>1078</td>\n",
              "      <td>...</td>\n",
              "      <td>10.24 33.19</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "      <td>[Kinship Relations Involving Successive Genera...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001003</th>\n",
              "      <td>MAT 1:1!3</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>proper</td>\n",
              "      <td>of Jesus</td>\n",
              "      <td>Ἰησοῦ</td>\n",
              "      <td></td>\n",
              "      <td>Ἰησοῦς</td>\n",
              "      <td>Ἰησοῦ</td>\n",
              "      <td>2424</td>\n",
              "      <td>...</td>\n",
              "      <td>93.169a</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "      <td>[Persons]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001004</th>\n",
              "      <td>MAT 1:1!4</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>proper</td>\n",
              "      <td>Christ</td>\n",
              "      <td>Χριστοῦ</td>\n",
              "      <td></td>\n",
              "      <td>Χριστός</td>\n",
              "      <td>Χριστοῦ</td>\n",
              "      <td>5547</td>\n",
              "      <td>...</td>\n",
              "      <td>93.387</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "      <td>[Persons]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>n40001001005</th>\n",
              "      <td>MAT 1:1!5</td>\n",
              "      <td>missing</td>\n",
              "      <td>noun</td>\n",
              "      <td>common</td>\n",
              "      <td>son</td>\n",
              "      <td>υἱοῦ</td>\n",
              "      <td></td>\n",
              "      <td>υἱός</td>\n",
              "      <td>υἱοῦ</td>\n",
              "      <td>5207</td>\n",
              "      <td>...</td>\n",
              "      <td>10.30</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>missing</td>\n",
              "      <td>MAT</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>MAT 1</td>\n",
              "      <td>MAT 1:1</td>\n",
              "      <td>[Kinship Relations Involving Successive Genera...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 30 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-df0c4553-ce8b-4e77-a610-02bde31d7bf0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-df0c4553-ce8b-4e77-a610-02bde31d7bf0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-df0c4553-ce8b-4e77-a610-02bde31d7bf0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's learn some things about some of the words. First, we need to connect to the OpenAI ChapGPT API using a secret API key. If you don't have one, you will need to sign up for one. You could also just query ChatGPT for free."
      ],
      "metadata": {
        "id": "rG9wANH7RLzS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import getpass\n",
        "secret_key = getpass.getpass('Enter OpenAI secret key: ')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z7VbeVUSAWNL",
        "outputId": "32b069ad-e58a-4d76-f341-99246bf304db"
      },
      "execution_count": 32,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter OpenAI secret key: ··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "ICVRB5VldBf9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "os.environ['OPENAI_API_KEY'] = secret_key"
      ],
      "metadata": {
        "id": "bxWrfpwUCzg1"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mg.loc['n40001001002'].to_dict()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MbfnG2AMgejw",
        "outputId": "f1838d00-9c04-438c-b1a6-0e6e34b7aa49"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'ref': 'MAT 1:1!2',\n",
              " 'role': 'missing',\n",
              " 'class': 'noun',\n",
              " 'type': 'common',\n",
              " 'gloss': 'of [the] genealogy',\n",
              " 'text': 'γενέσεως',\n",
              " 'after': ' ',\n",
              " 'lemma': 'γένεσις',\n",
              " 'normalized': 'γενέσεως',\n",
              " 'strong': 1078,\n",
              " 'morph': 'N-GSF',\n",
              " 'person': 'missing',\n",
              " 'number': 'singular',\n",
              " 'gender': 'feminine',\n",
              " 'case': 'genitive',\n",
              " 'tense': 'missing',\n",
              " 'voice': 'missing',\n",
              " 'mood': 'missing',\n",
              " 'degree': 'missing',\n",
              " 'domain': '010002 033003',\n",
              " 'ln': '10.24 33.19',\n",
              " 'frame': 'missing',\n",
              " 'subjref': 'missing',\n",
              " 'referent': 'missing',\n",
              " 'book': 'MAT',\n",
              " 'chapter': '1',\n",
              " 'verse': '1',\n",
              " 'book_chapter': 'MAT 1',\n",
              " 'book_chapter_verse': 'MAT 1:1',\n",
              " 'domain_label': ['Kinship Relations Involving Successive Generations',\n",
              "  'Discourse Types']}"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Ozzt73PTh-aS"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mg['domain'].head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HKys-eZej5Hk",
        "outputId": "dcf76710-c199-4d15-d4cd-81302dbc3597"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "xml:id\n",
              "n40001001001           033005\n",
              "n40001001002    010002 033003\n",
              "n40001001003           093001\n",
              "n40001001004           093001\n",
              "n40001001005           010002\n",
              "Name: domain, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import openai\n",
        "\n",
        "selected_fields = ['domain_label', 'gloss', 'class', 'type']\n",
        "def expand(word):\n",
        "    prompt = mg.loc[word]['lemma'] + \"'s \" \n",
        "    components = [f\"{key} is {value}\" for key, value in mg.loc[word].to_dict().items() if key in selected_fields]\n",
        "    \n",
        "    lemma_coreferences = []\n",
        "    # for component in components:\n",
        "        # print(component)\n",
        "        # if component == 'nan':\n",
        "        #     print('true', type(component),component)\n",
        "        # else:\n",
        "        #     print('false', type(component),component)\n",
        "    prompt += \", \".join(components)\n",
        "    print('Prompt: ', prompt)\n",
        "\n",
        "    return prompt, lemma_coreferences\n",
        "\n",
        "\n",
        "\n",
        "def generate_prompt(expanded_word_prompt):\n",
        "    return '''Explain what this word means based on the supplied values:\n",
        "\n",
        "Prompt: \"γένεσις, a common noun, '[the] genealogy', ['Kinship Relations Involving Successive Generations', 'Discourse Types']\"\n",
        "Explanation: The lemma γένεσις is a common noun that could be glossed 'genealogy', or 'the genealogy' in this context. Semantically, this word has to do with kinship relations across successive generations, and it refers to a particular kind of discourse or genre (i.e., a genealogical record).\n",
        "Word: \"Ἰησοῦς, a proper noun, 'Jesus', ['Persons']\"\n",
        "Explanation: The lemma Ἰησοῦς is a proper noun that could be glossed 'Jesus', the son of Mary and adopted son of Joseph. This word's semantics indicates a person is in view.\n",
        "Word: {0}\n",
        "Explanation: '''.format(expanded_word_prompt)"
      ],
      "metadata": {
        "id": "9neBjGzXD1Ty"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "expanded_word = expand('n40001001001')\n",
        "input_prompt = expanded_word[0]\n",
        "\n",
        "completion = openai.Completion.create(\n",
        "  model=\"text-davinci-003\",\n",
        "#   model=\"babbage\",\n",
        "  prompt=generate_prompt(input_prompt),\n",
        "  temperature=0.6,\n",
        "  max_tokens=50,\n",
        "  top_p=1,\n",
        "  frequency_penalty=0.0,\n",
        "  presence_penalty=0.6,\n",
        ")\n",
        "\n",
        "text_before_final_period = lambda text: text[:text.rfind('.')+1]\n",
        "print(completion.choices[0].text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EtJtIcv7aeii",
        "outputId": "14a93e09-e131-40da-88bb-e87e56a7fdcf"
      },
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Prompt:  βίβλος's class is noun, type is common, gloss is [The] book, domain_label is ['Written Language']\n",
            "\n",
            "The lemma βίβλος is a common noun that could be glossed 'book', or 'the book' in this context. Semantically, this word refers to a written language object and could refer to a physical book\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text_before_final_period = lambda text: text[:text.rfind('.')+1] # Simple function to ensure trailing content is removed"
      ],
      "metadata": {
        "id": "ySDwX65SReST"
      },
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the tokenizer and model\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"facebook/nllb-200-distilled-600M\")\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(\"facebook/nllb-200-distilled-600M\")"
      ],
      "metadata": {
        "id": "cYbjeox8R3qr"
      },
      "execution_count": 103,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the source language code and target language codes\n",
        "src_lang = \"eng_Latn\"\n",
        "tgt_langs = [\"deu_Latn\", \"spa_Latn\", \"fra_Latn\"] # german, spanish, french\n",
        "\n",
        "# Define the translation prompt\n",
        "translation_prompt = text_before_final_period(completion.choices[0].text)\n",
        "\n",
        "# Encode all Greek/Hebrew tokens as 1_1, 2_2, 3_3, etc.\n",
        "special_id2token = dict()\n",
        "\n",
        "def is_special_token(token):\n",
        "    '''\n",
        "    Any token that contains a Greek or Hebrew unicode character is a special token.\n",
        "    '''\n",
        "    greek_unicode_chars = [chr(i) for i in range(0x0370, 0x03FF)]\n",
        "    hebrew_unicode_chars = [chr(i) for i in range(0x0590, 0x05FF)]\n",
        "    return any([char in token for char in greek_unicode_chars + hebrew_unicode_chars])\n",
        "\n",
        "def encode_non_english_tokens(original_prompt):\n",
        "    prompt = original_prompt\n",
        "    count = 0\n",
        "    for token in original_prompt.split():\n",
        "        if is_special_token(token):\n",
        "            prompt = prompt.replace(token, f\"{count}_{count}\")\n",
        "            special_id2token[f\"{count}_{count}\"] = token\n",
        "            count += 1\n",
        "    return prompt\n",
        "\n",
        "encoded_prompt = encode_non_english_tokens(translation_prompt)\n",
        "encoded_prompt"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "vXqer7uJTY4n",
        "outputId": "dc5fb905-7621-465c-9dcc-05c8de3e4f7d"
      },
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"\\nThe lemma 0_0 is a common noun that could be glossed 'book', or 'the book' in this context.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Instantiate each target language translator\n",
        "deu_translator = pipeline('translation', model=model, tokenizer=tokenizer, src_lang=\"eng_Latn\", tgt_lang='deu_Latn')\n",
        "spa_translator = pipeline('translation', model=model, tokenizer=tokenizer, src_lang=\"eng_Latn\", tgt_lang='spa_Latn')\n",
        "fra_translator = pipeline('translation', model=model, tokenizer=tokenizer, src_lang=\"eng_Latn\", tgt_lang='fra_Latn')"
      ],
      "metadata": {
        "id": "VYyCFDA9auh_"
      },
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# Decode the special tokens\n",
        "def decode_special_tokens(result):\n",
        "    decoded_result = result[0]['translation_text']\n",
        "    for k, v in special_id2token.items():\n",
        "        decoded_result = decoded_result.replace(k, v)\n",
        "    return decoded_result"
      ],
      "metadata": {
        "id": "VW0p_xLyUSv6"
      },
      "execution_count": 106,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Pass prompt through the pipelines\n",
        "deu_result = deu_translator(encoded_prompt)\n",
        "deu_text = deu_result[0]['translation_text']\n",
        "spa_result = spa_translator(encoded_prompt)\n",
        "spa_text = spa_result[0]['translation_text']\n",
        "fra_result = fra_translator(encoded_prompt)\n",
        "fra_text = fra_result[0]['translation_text']\n",
        "\n",
        "print('German: ', decode_special_tokens(deu_result))\n",
        "print('Spanish: ', decode_special_tokens(spa_result))\n",
        "print('French: ', decode_special_tokens(fra_result))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sh42x-Oub-Nv",
        "outputId": "0adcda1d-5735-4d69-93ba-4a9d9f222d4a"
      },
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "German:  Das Lemma βίβλος ist ein allgemeines Nomen, das in diesem Zusammenhang \"Buch\" oder \"das Buch\" genannt werden könnte.\n",
            "Spanish:  El lema βίβλος es un sustantivo común que podría ser glosado 'libro', o 'el libro' en este contexto.\n",
            "French:  Le lemma βίβλος est un nom commun qui pourrait être glossé \"livre\", ou \"le livre\" dans ce contexte.\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}