seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32aca16a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "08e03c22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "caf26f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unicodeToAscii(s):\n",
    "    return ''.join(\n",
    "        c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn'\n",
    "    )\n",
    "\n",
    "# Lowercase, trim, and remove non-letter characters\n",
    "\n",
    "\n",
    "def normalizeString(s):\n",
    "    s = unicodeToAscii(s.lower().strip())\n",
    "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
    "    s = re.sub(r\"[^a-zA-Zа-яА-Я.!?]+\", r\" \", s)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "487d9d8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def readLangs(lang1, lang2, reverse=False):\n",
    "    print(\"Reading lines...\")\n",
    "    # Read the file and split into lines\n",
    "    lines = open('data/%s-%s.txt' % (lang1, lang2), encoding='utf-8').\\\n",
    "        read().strip().split('\\n')\n",
    "    # Split every line into pairs and normalize\n",
    "    pairs = [[normalizeString(s) for s in l.split('\\t')] for l in lines]\n",
    "    #print (pairs[:10])\n",
    "    # Reverse pairs, make Lang instances\n",
    "\n",
    "    return pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ee8901b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LENGTH_WORDS = 10\n",
    "\n",
    "\n",
    "def filterPair(p):\n",
    "    #print(p[0] + \" \" + p[1])\n",
    "    return len(p[0].split(' ')) < MAX_LENGTH_WORDS and \\\n",
    "        len(p[1].split(' ')) < MAX_LENGTH_WORDS \n",
    "\n",
    "\n",
    "def filterPairs(pairs):\n",
    "    return [pair for pair in pairs if filterPair(pair)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "51614b00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading lines...\n",
      "Read 330194 sentence pairs\n",
      "Trimmed to 277796 sentence pairs\n",
      "Counting words...\n",
      "making validation set\n",
      "train data\n",
      "input data: \n",
      "почему вы жили в австралии в прошлом году ?\n",
      "я опоздал на реис .\n",
      "я в этом никого не виню .\n",
      "даваи сделаем последнее усилие .\n",
      "я слышал как вы плакали .\n",
      "мы не должны терять надежду .\n",
      "как вы переносите этот холод ?\n",
      "нам не везло .\n",
      "я не смог бы так жить .\n",
      "ты сеичас куда ?\n",
      "target data\n",
      "why did you live in australia last year ?\n",
      "i missed my flight .\n",
      "i m not blaming anyone for it .\n",
      "let s make one last effort .\n",
      "i heard you crying .\n",
      "we shouldn t lose hope .\n",
      "how do you stand this cold ?\n",
      "we weren t lucky .\n",
      "i couldn t live like that .\n",
      "where are you going now ?\n",
      "validation data\n",
      "input data: \n",
      "почему вы жили в австралии в прошлом году ?\n",
      "я опоздал на реис .\n",
      "я в этом никого не виню .\n",
      "даваи сделаем последнее усилие .\n",
      "я слышал как вы плакали .\n",
      "мы не должны терять надежду .\n",
      "как вы переносите этот холод ?\n",
      "нам не везло .\n",
      "я не смог бы так жить .\n",
      "ты сеичас куда ?\n",
      "target data\n",
      "why did you live in australia last year ?\n",
      "i missed my flight .\n",
      "i m not blaming anyone for it .\n",
      "let s make one last effort .\n",
      "i heard you crying .\n",
      "we shouldn t lose hope .\n",
      "how do you stand this cold ?\n",
      "we weren t lucky .\n",
      "i couldn t live like that .\n",
      "where are you going now ?\n",
      "validation set len = 250017\n",
      "train set len      = 250017\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "def prepareData(lang1, lang2, reverse=False):\n",
    "    pairs = readLangs(lang1, lang2, reverse)\n",
    "    print(\"Read %s sentence pairs\" % len(pairs))\n",
    "    pairs = filterPairs(pairs)\n",
    "    print(\"Trimmed to %s sentence pairs\" % len(pairs))\n",
    "    print(\"Counting words...\")\n",
    "    return pairs\n",
    "\n",
    "def prepairTrainSeq():\n",
    "    pairs = prepareData('eng', 'rus', True)\n",
    "    target_strings = [pair[0] for pair in pairs]\n",
    "    input_strings = [pair[1] for pair in pairs]\n",
    "    return target_strings, input_strings\n",
    "\n",
    "target_strings, input_strings = prepairTrainSeq()\n",
    "\n",
    "#i need that for tokenizer\n",
    "ru_text = \"\"\n",
    "en_text = \"\"\n",
    "for p in input_strings:\n",
    "    ru_text += p + '\\n'\n",
    "for p in target_strings:\n",
    "    en_text += p + '\\n'\n",
    "\n",
    "# calculate 10% of the total number of elements in pairs\n",
    "# randomly select n elements from pairs and move them to val set\\n\",\n",
    "print(\"making validation set\")\n",
    "combined_list = list(zip(target_strings, input_strings))\n",
    "random.shuffle(combined_list)\n",
    "target_strings, input_strings = zip(*combined_list)\n",
    "\n",
    "n = int(len(input_strings) * 0.1)\n",
    "\n",
    "train_target_strings = target_strings[n:]\n",
    "train_input_strings = input_strings[n:]\n",
    "\n",
    "val_target_strings = target_strings[n:]\n",
    "val_input_strings = input_strings[n:]\n",
    "\n",
    "print(\"train data\")\n",
    "print(\"input data: \")\n",
    "for s in train_input_strings[:10]:\n",
    "    print(s)\n",
    "\n",
    "print(\"target data\")\n",
    "for s in train_target_strings[:10]:\n",
    "    print(s)\n",
    "    \n",
    "print(\"validation data\")\n",
    "print(\"input data: \")\n",
    "for s in val_input_strings[:10]:\n",
    "    print(s)\n",
    "\n",
    "print(\"target data\")\n",
    "for s in val_target_strings[:10]:\n",
    "    print(s)\n",
    "print(\"validation set len = \" + str(len(val_input_strings)))\n",
    "print(\"train set len      = \" + str(len(train_input_strings)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "714fb625",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de95884b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install tokenizers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5fc3a4e",
   "metadata": {},
   "source": [
    "here I am building the tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bf6c0308",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tokenizers import BertWordPieceTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e0a26c5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1017537c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "марш !\n",
      "иди .\n",
      "идите .\n",
      "здравствуите .\n",
      "привет !\n",
      "хаи .\n",
      "здрасте .\n",
      "здорово !\n",
      "приветик !\n",
      "беги !\n",
      "бегите !\n",
      "бе\n"
     ]
    }
   ],
   "source": [
    "print(ru_text[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8a829c10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "go .\n",
      "go .\n",
      "go .\n",
      "hi .\n",
      "hi .\n",
      "hi .\n",
      "hi .\n",
      "hi .\n",
      "hi .\n",
      "run !\n",
      "run !\n",
      "run .\n",
      "run .\n",
      "who ?\n",
      "wow !\n",
      "wow !\n",
      "wow !\n",
      "wow !\n",
      "w\n"
     ]
    }
   ],
   "source": [
    "print(en_text[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "86df349e",
   "metadata": {},
   "outputs": [],
   "source": [
    "SOS_token = 0\n",
    "EOS_token = 1\n",
    "\n",
    "def create_tokenizer_from_text(text):\n",
    "    # Create new tokenizer based on input text\n",
    "    tokenizer = BertWordPieceTokenizer(\n",
    "        clean_text=True,\n",
    "        handle_chinese_chars=False,\n",
    "        strip_accents=False,\n",
    "        lowercase=False\n",
    "        )\n",
    "    tokenizer.add_tokens([\"[SOS]\"])#Вот прям 100% тут можно как-то отстрелить себе ногу\n",
    "    tokenizer.add_tokens([\"[EOS]\"])\n",
    "    tokenizer.train_from_iterator([text.split()], vocab_size=5000, special_tokens=[\"[PAD]\", \"[UNK]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"])\n",
    "    # Return tokenizer object\n",
    "    return tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "61232777",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ru_tokenizer = create_tokenizer_from_text(ru_text)\n",
    "en_tokenizer = create_tokenizer_from_text(en_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "80febccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "ru_tokenizer.save(\"ru_tokenizer.json\")\n",
    "en_tokenizer.save(\"en_tokenizer.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c9b2757f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'мальчик': 1900,\n",
       " 'прежнему': 2990,\n",
       " '##ресси': 4114,\n",
       " 'наеди': 4866,\n",
       " 'шахматы': 3502,\n",
       " 'па': 452,\n",
       " 'объяснить': 2132,\n",
       " '##лагода': 1593,\n",
       " 'кататься': 3019,\n",
       " 'дерево': 2123,\n",
       " 'должна': 1517,\n",
       " '##тым': 2479,\n",
       " '##ких': 1228,\n",
       " '##ении': 1016,\n",
       " 'мест': 4483,\n",
       " 'купил': 821,\n",
       " '##рован': 4791,\n",
       " '##фор': 2004,\n",
       " 'твою': 2120,\n",
       " '##рая': 2966,\n",
       " '##ское': 3164,\n",
       " 'докто': 4133,\n",
       " '##циеи': 4992,\n",
       " 'работали': 4826,\n",
       " 'попал': 4126,\n",
       " 'терпеть': 2567,\n",
       " 'спросил': 3051,\n",
       " 'увер': 654,\n",
       " 'принад': 2425,\n",
       " '##лю': 277,\n",
       " '##рели': 2857,\n",
       " 'сплю': 3857,\n",
       " 'сохра': 3569,\n",
       " 'искус': 3931,\n",
       " 'повы': 3874,\n",
       " 'осу': 4804,\n",
       " 'слово': 2056,\n",
       " '##рабо': 1488,\n",
       " 'смогла': 4890,\n",
       " 'друзеи': 1687,\n",
       " 'сэ': 4973,\n",
       " 'равно': 2129,\n",
       " 'баскет': 4371,\n",
       " '##ально': 1290,\n",
       " 'дать': 2115,\n",
       " 'вряд': 3902,\n",
       " 'ненадолго': 4112,\n",
       " 'нас': 323,\n",
       " 'украли': 2988,\n",
       " '##ялись': 2463,\n",
       " 'забыть': 3487,\n",
       " '##лении': 4196,\n",
       " 'ждет': 4330,\n",
       " '##гия': 3241,\n",
       " '##чу': 265,\n",
       " 'япо': 1153,\n",
       " 'проснулся': 4262,\n",
       " 'две': 1325,\n",
       " 'утро': 2384,\n",
       " 'сказали': 1102,\n",
       " 'заму': 1405,\n",
       " 'кана': 1837,\n",
       " 'потеряли': 3892,\n",
       " 'карту': 4351,\n",
       " '##ать': 210,\n",
       " 'повто': 1970,\n",
       " 'попу': 2839,\n",
       " 'следует': 1151,\n",
       " '##мом': 3180,\n",
       " 'конф': 3328,\n",
       " 'канадка': 3787,\n",
       " '##жа': 413,\n",
       " '##оте': 2342,\n",
       " 'уч': 1895,\n",
       " 'отказы': 4614,\n",
       " 'другому': 4789,\n",
       " 'бле': 2819,\n",
       " 'ноги': 2646,\n",
       " 'вку': 1530,\n",
       " 'из': 263,\n",
       " 'какая': 944,\n",
       " 'интере': 725,\n",
       " 'плохое': 3996,\n",
       " 'красное': 4573,\n",
       " 'свет': 1399,\n",
       " 'план': 1413,\n",
       " 'п': 47,\n",
       " 'нужна': 867,\n",
       " 'перестань': 1750,\n",
       " 'соблю': 4687,\n",
       " 'ответ': 700,\n",
       " 'пи': 371,\n",
       " 'еще': 284,\n",
       " 'выгля': 762,\n",
       " 'делается': 3445,\n",
       " 'случаи': 2219,\n",
       " 'опу': 4508,\n",
       " 'съез': 2172,\n",
       " 'выбрать': 3696,\n",
       " 'вечеринка': 4902,\n",
       " 'соба': 642,\n",
       " 'возьму': 2937,\n",
       " 'ценю': 4590,\n",
       " '##разум': 4291,\n",
       " 'оставить': 2282,\n",
       " '##ент': 4198,\n",
       " 'приятно': 2726,\n",
       " '##дения': 3611,\n",
       " 'сюрпри': 4539,\n",
       " '##жешь': 1491,\n",
       " 'уг': 2462,\n",
       " '##тели': 2253,\n",
       " 'людеи': 1330,\n",
       " 'возду': 2970,\n",
       " 'большои': 1755,\n",
       " 'мал': 1298,\n",
       " 'учите': 1853,\n",
       " 'шесть': 1923,\n",
       " '##вление': 3420,\n",
       " 'убили': 4894,\n",
       " 'неи': 1164,\n",
       " 'побе': 1099,\n",
       " 'говорила': 3056,\n",
       " 'нетер': 3566,\n",
       " 'самая': 2984,\n",
       " '##дится': 3107,\n",
       " 'сделаешь': 3360,\n",
       " 'поидем': 1506,\n",
       " 'посуду': 3174,\n",
       " '##ется': 356,\n",
       " '##ными': 1357,\n",
       " '##жда': 1578,\n",
       " 'пото': 3037,\n",
       " 'просты': 4427,\n",
       " 'тысячи': 3474,\n",
       " '##ни': 146,\n",
       " '##за': 422,\n",
       " 'свои': 429,\n",
       " 'справи': 3519,\n",
       " 'зап': 4606,\n",
       " '##еру': 2742,\n",
       " 'всегда': 543,\n",
       " '##жал': 280,\n",
       " 'фотоаппа': 3150,\n",
       " 'проис': 1063,\n",
       " 'яблоко': 2737,\n",
       " 'стране': 4348,\n",
       " 'живем': 3768,\n",
       " 'играете': 4489,\n",
       " '##ным': 574,\n",
       " 'которое': 4834,\n",
       " 'остано': 901,\n",
       " 'ябло': 1286,\n",
       " 'внутри': 3817,\n",
       " 'твое': 1230,\n",
       " '##шается': 4724,\n",
       " 'играет': 2491,\n",
       " '##шие': 1451,\n",
       " 'интересныи': 4888,\n",
       " 'стреля': 4006,\n",
       " '##бра': 877,\n",
       " 'закрыть': 3733,\n",
       " 'пробо': 2954,\n",
       " '##том': 689,\n",
       " 'грузовик': 4367,\n",
       " 'пожар': 3558,\n",
       " 'жарко': 2362,\n",
       " 'ус': 681,\n",
       " 'роди': 1644,\n",
       " 'поо': 2037,\n",
       " '##тныи': 4333,\n",
       " '##фе': 727,\n",
       " 'посмотрите': 2396,\n",
       " 'напиши': 4638,\n",
       " '##ление': 1341,\n",
       " '##рк': 2117,\n",
       " '##щен': 2710,\n",
       " 'мужчина': 2577,\n",
       " 'команда': 3030,\n",
       " '##p': 97,\n",
       " 'следующии': 2087,\n",
       " 'открыл': 3130,\n",
       " 'девушка': 1855,\n",
       " 'опазды': 2124,\n",
       " 'шля': 1608,\n",
       " 'прис': 2345,\n",
       " 'побы': 2768,\n",
       " 'лесу': 4525,\n",
       " 'рев': 4776,\n",
       " '##альше': 1821,\n",
       " '##кры': 494,\n",
       " 'прости': 1503,\n",
       " 'дра': 2640,\n",
       " 'думаете': 1022,\n",
       " 'пришел': 1069,\n",
       " 'посмотреть': 1596,\n",
       " 'знала': 1972,\n",
       " 'выпить': 2885,\n",
       " 'ище': 3100,\n",
       " 'приходите': 3388,\n",
       " 'фо': 664,\n",
       " 'гольф': 3311,\n",
       " 'воити': 3700,\n",
       " '##прав': 1201,\n",
       " 'силь': 1366,\n",
       " '##ств': 753,\n",
       " 'сына': 3285,\n",
       " '##жила': 4433,\n",
       " 'ответить': 2094,\n",
       " 'защи': 3018,\n",
       " 'сообщ': 2031,\n",
       " '##стато': 955,\n",
       " 'внима': 1318,\n",
       " 'уверены': 2418,\n",
       " 'выиг': 1863,\n",
       " '##шке': 2893,\n",
       " 'бывает': 2045,\n",
       " 'оп': 871,\n",
       " 'ши': 3085,\n",
       " 'поезд': 1103,\n",
       " 'выход': 1293,\n",
       " '##ением': 1535,\n",
       " 'н': 45,\n",
       " 'настоя': 1756,\n",
       " 'вели': 1998,\n",
       " 'плакать': 3618,\n",
       " 'фи': 931,\n",
       " 'добро': 2622,\n",
       " 'цели': 4154,\n",
       " 'любит': 1388,\n",
       " 'ответа': 2881,\n",
       " 'разру': 3980,\n",
       " '##жида': 4986,\n",
       " 'покажите': 2846,\n",
       " 'интересу': 2826,\n",
       " 'особ': 2593,\n",
       " 'полно': 1552,\n",
       " 'имел': 3203,\n",
       " 'они': 262,\n",
       " 'обожаю': 3226,\n",
       " 'ошибка': 2591,\n",
       " '##дале': 3259,\n",
       " 'поддер': 3549,\n",
       " 'многи': 3495,\n",
       " 'лед': 4942,\n",
       " 'таи': 2154,\n",
       " 'срав': 4509,\n",
       " '##вая': 842,\n",
       " 'вашего': 2240,\n",
       " 'книг': 2137,\n",
       " 'бутер': 4045,\n",
       " 'вам': 219,\n",
       " 'нашем': 4153,\n",
       " '##ять': 554,\n",
       " 'поб': 2520,\n",
       " 'полностью': 2677,\n",
       " 'завтрак': 2426,\n",
       " 'такую': 3699,\n",
       " 'убе': 1207,\n",
       " 'самолет': 2042,\n",
       " 'умеет': 2447,\n",
       " 'сказ': 327,\n",
       " 'мар': 3779,\n",
       " 'свободен': 3812,\n",
       " 'нравлюсь': 4081,\n",
       " 'другу': 1889,\n",
       " 'ожидал': 2408,\n",
       " '##мал': 400,\n",
       " 'чеи': 3659,\n",
       " 'никого': 1177,\n",
       " 'име': 728,\n",
       " 'сможем': 4146,\n",
       " 'дого': 4515,\n",
       " 'пить': 1590,\n",
       " 'галстук': 2862,\n",
       " '##шая': 754,\n",
       " '[MASK]': 4,\n",
       " '##граем': 3643,\n",
       " 'ре': 367,\n",
       " '##ваюсь': 2286,\n",
       " 'написа': 3736,\n",
       " 'нашел': 1255,\n",
       " 'клу': 3718,\n",
       " '##шеи': 1602,\n",
       " '##лая': 2177,\n",
       " 'задачу': 4401,\n",
       " 'собак': 3340,\n",
       " 'почи': 1202,\n",
       " '##валась': 4512,\n",
       " 'ста': 420,\n",
       " 'рассказы': 1909,\n",
       " 'похожи': 3646,\n",
       " 'заметил': 2691,\n",
       " 'попробовал': 4951,\n",
       " 'ос': 1865,\n",
       " 'рассказать': 2114,\n",
       " 'лишь': 1620,\n",
       " 'рядом': 2244,\n",
       " 'дверь': 779,\n",
       " '##вели': 2505,\n",
       " '##енно': 584,\n",
       " 'защитить': 3609,\n",
       " 'следую': 886,\n",
       " 'сказать': 613,\n",
       " 'поиду': 1367,\n",
       " '##хая': 3048,\n",
       " 'ка': 169,\n",
       " '##альчи': 1155,\n",
       " '##цеи': 1525,\n",
       " 'гряз': 2519,\n",
       " 'чашку': 3402,\n",
       " 'сели': 4879,\n",
       " 'руч': 1919,\n",
       " 'выи': 1001,\n",
       " 'крас': 2405,\n",
       " 'потер': 4928,\n",
       " 'ру': 559,\n",
       " '##влека': 2537,\n",
       " 'дал': 1658,\n",
       " '##рпри': 4510,\n",
       " 'видно': 4245,\n",
       " '##доб': 3143,\n",
       " 'стои': 601,\n",
       " 'услыш': 1652,\n",
       " 'посмотрел': 3365,\n",
       " '##сли': 515,\n",
       " '##пер': 1352,\n",
       " 'поехать': 971,\n",
       " '##гать': 1785,\n",
       " '##сныи': 1335,\n",
       " 'ходить': 1777,\n",
       " 'жале': 2991,\n",
       " '##сто': 203,\n",
       " '##шил': 2365,\n",
       " 'посове': 2527,\n",
       " 'запер': 2723,\n",
       " 'упал': 3824,\n",
       " 'вме': 851,\n",
       " 'сожалею': 3870,\n",
       " 'надеяться': 4106,\n",
       " 'следующеи': 2215,\n",
       " '##стю': 4276,\n",
       " '##редо': 4480,\n",
       " '##таться': 1928,\n",
       " '##пли': 4421,\n",
       " 'другои': 2103,\n",
       " '##куда': 965,\n",
       " 'теле': 608,\n",
       " 'пытаюсь': 2883,\n",
       " '##ка': 157,\n",
       " 'грузови': 3491,\n",
       " '##тер': 781,\n",
       " 'выну': 4380,\n",
       " 'нем': 590,\n",
       " '##ком': 330,\n",
       " 'проголо': 3071,\n",
       " 'по': 122,\n",
       " 'пои': 522,\n",
       " '##вание': 2238,\n",
       " 'присоедини': 3973,\n",
       " '##дание': 1680,\n",
       " 'пробовал': 4505,\n",
       " 'узна': 1453,\n",
       " 'кот': 2107,\n",
       " 'выити': 2797,\n",
       " '##шены': 4815,\n",
       " '##лож': 1085,\n",
       " 'ехать': 1587,\n",
       " '##зво': 1988,\n",
       " 'пожа': 2793,\n",
       " 'яица': 2849,\n",
       " 'поехали': 3728,\n",
       " 'поку': 1289,\n",
       " 'полиция': 2168,\n",
       " '##пно': 3008,\n",
       " 'хватает': 2243,\n",
       " 'важно': 1312,\n",
       " 'выз': 1768,\n",
       " '##кти': 2765,\n",
       " 'вещ': 4723,\n",
       " '##дна': 4195,\n",
       " 'никак': 4582,\n",
       " '##вут': 1234,\n",
       " 'оружи': 3297,\n",
       " 'уд': 2472,\n",
       " 'тра': 1710,\n",
       " 'будем': 883,\n",
       " 'остановить': 2206,\n",
       " '##кала': 4859,\n",
       " 'были': 495,\n",
       " '##дцать': 868,\n",
       " 'дово': 891,\n",
       " 'скажи': 929,\n",
       " 'темноты': 4909,\n",
       " '##чера': 514,\n",
       " 'бостона': 1749,\n",
       " '##реть': 729,\n",
       " '##сил': 856,\n",
       " 'паль': 1606,\n",
       " 'пот': 1944,\n",
       " 'получить': 3843,\n",
       " 'поможете': 4279,\n",
       " 'воскрес': 2022,\n",
       " '##тое': 4379,\n",
       " 'вчераш': 4571,\n",
       " 'празд': 4965,\n",
       " '##ньше': 684,\n",
       " 'осторож': 1633,\n",
       " 'заво': 4870,\n",
       " '##льная': 3510,\n",
       " '##альная': 3255,\n",
       " 'водитель': 4201,\n",
       " 'глубоко': 4324,\n",
       " '##лую': 4104,\n",
       " 'поверить': 1882,\n",
       " 'еды': 2389,\n",
       " 'со': 189,\n",
       " '##стныи': 2796,\n",
       " 'своим': 1566,\n",
       " 'сдела': 558,\n",
       " '##вь': 570,\n",
       " 'ок': 757,\n",
       " 'сфотографи': 4370,\n",
       " '##деи': 1060,\n",
       " 'ем': 2151,\n",
       " 'ис': 656,\n",
       " 'повер': 2668,\n",
       " '##гаи': 3547,\n",
       " 'уго': 1683,\n",
       " 'готовы': 1808,\n",
       " '##где': 3791,\n",
       " 'тор': 3055,\n",
       " 'шум': 2512,\n",
       " 'ел': 2310,\n",
       " 'привлека': 3754,\n",
       " 'три': 487,\n",
       " 'ь': 60,\n",
       " '##кономи': 4474,\n",
       " '##дин': 526,\n",
       " '##стно': 1236,\n",
       " '##нию': 2200,\n",
       " '##уи': 304,\n",
       " 'расхо': 4937,\n",
       " 'расс': 489,\n",
       " 'вернусь': 2000,\n",
       " 'ног': 3798,\n",
       " '##пить': 4202,\n",
       " 'обеспоко': 4887,\n",
       " 'остане': 3573,\n",
       " '##с': 74,\n",
       " 'дол': 274,\n",
       " 'потеря': 954,\n",
       " 'спросить': 2649,\n",
       " 'попо': 4709,\n",
       " '##з': 69,\n",
       " '##шка': 769,\n",
       " 'позволить': 2738,\n",
       " 'правда': 793,\n",
       " '##шили': 3444,\n",
       " 'решать': 4148,\n",
       " '##ву': 347,\n",
       " '##стве': 2136,\n",
       " 'списке': 3536,\n",
       " 'позд': 4068,\n",
       " 'нос': 4390,\n",
       " 'напра': 4453,\n",
       " '##дали': 1242,\n",
       " '##хом': 3401,\n",
       " 'девять': 3332,\n",
       " 'америка': 4158,\n",
       " 'англии': 1179,\n",
       " 'множество': 3999,\n",
       " 'постоя': 2596,\n",
       " '##льз': 768,\n",
       " 'ходите': 4305,\n",
       " '##дись': 3879,\n",
       " 'одино': 1805,\n",
       " 'зря': 3083,\n",
       " 'наве': 3776,\n",
       " '##гнул': 4810,\n",
       " 'вернемся': 3766,\n",
       " 'ресторане': 4918,\n",
       " 'поведение': 4633,\n",
       " 'октябре': 3786,\n",
       " 'поможет': 2800,\n",
       " 'вече': 674,\n",
       " 'замок': 4715,\n",
       " 'прибы': 2748,\n",
       " '##чего': 409,\n",
       " 'французского': 3146,\n",
       " '##спе': 3216,\n",
       " '##m': 106,\n",
       " 'шпи': 3693,\n",
       " 'единствен': 1847,\n",
       " 'недавно': 3932,\n",
       " 'боится': 4410,\n",
       " 'знает': 1044,\n",
       " 'новыи': 1480,\n",
       " '##ство': 589,\n",
       " 'бума': 1748,\n",
       " 'возьмите': 3206,\n",
       " 'пирог': 3654,\n",
       " 'женщин': 4636,\n",
       " 'чрезвы': 4802,\n",
       " 'насчет': 2586,\n",
       " '##нул': 1056,\n",
       " 'ошибок': 3222,\n",
       " 'куплю': 2787,\n",
       " 'закрыты': 4399,\n",
       " 'участи': 4799,\n",
       " 'больнице': 4825,\n",
       " 'работы': 1678,\n",
       " 'пытался': 1918,\n",
       " '##по': 234,\n",
       " 'дождь': 953,\n",
       " 'меся': 1349,\n",
       " 'печенье': 4365,\n",
       " 'забыл': 1126,\n",
       " 'нако': 1991,\n",
       " 'ко': 173,\n",
       " 'пасса': 4392,\n",
       " '##ж': 78,\n",
       " 'сеи': 442,\n",
       " 'комнаты': 3077,\n",
       " 'чу': 568,\n",
       " 'переста': 1362,\n",
       " '##рту': 4547,\n",
       " '##ну': 220,\n",
       " '##цени': 4097,\n",
       " 'спро': 1081,\n",
       " 'вмеши': 4354,\n",
       " 'друз': 784,\n",
       " '##шила': 4309,\n",
       " 'верно': 3619,\n",
       " 'стакан': 3200,\n",
       " 'предложение': 1719,\n",
       " '##рож': 1106,\n",
       " 'отсут': 4103,\n",
       " 'когда': 352,\n",
       " '##няя': 4520,\n",
       " 'опыт': 3819,\n",
       " 'проигры': 4749,\n",
       " 'ящи': 1845,\n",
       " '##лось': 455,\n",
       " 'красныи': 3283,\n",
       " 'такои': 839,\n",
       " '##сси': 2343,\n",
       " 'рассчиты': 2576,\n",
       " '##р': 80,\n",
       " 'чита': 999,\n",
       " 'муз': 1133,\n",
       " 'реб': 921,\n",
       " 'наза': 1224,\n",
       " 'други': 2074,\n",
       " '##стья': 3125,\n",
       " '##сс': 3538,\n",
       " 'занятия': 3955,\n",
       " 'музыка': 4116,\n",
       " 'дни': 4270,\n",
       " '##менить': 4578,\n",
       " 'порядке': 1671,\n",
       " 'бросил': 3367,\n",
       " 'u': 27,\n",
       " 'стал': 1833,\n",
       " 'открыта': 3890,\n",
       " '##б': 88,\n",
       " 'зая': 4932,\n",
       " 'написать': 3473,\n",
       " 'солдаты': 4913,\n",
       " '##вер': 276,\n",
       " 'япон': 2517,\n",
       " 'теря': 3287,\n",
       " 'мальчи': 1158,\n",
       " 'ин': 548,\n",
       " 'мнои': 614,\n",
       " '##казы': 1929,\n",
       " 'коробке': 4002,\n",
       " 'оправ': 4147,\n",
       " 'повезло': 2934,\n",
       " '##яза': 1275,\n",
       " 'видишь': 3126,\n",
       " '##сь': 158,\n",
       " 'терпение': 4596,\n",
       " '##лов': 4714,\n",
       " 'взя': 1513,\n",
       " '##вало': 1471,\n",
       " 'иг': 972,\n",
       " 'самом': 2205,\n",
       " 'уходить': 2442,\n",
       " 'сомневаюсь': 3656,\n",
       " 'осталось': 1809,\n",
       " 'преду': 1958,\n",
       " 'шка': 3633,\n",
       " 'переоде': 4192,\n",
       " 'ро': 418,\n",
       " 'конце': 4519,\n",
       " '##меня': 3940,\n",
       " 'скоро': 903,\n",
       " '##лся': 379,\n",
       " 'разговари': 1170,\n",
       " 'дня': 1573,\n",
       " 'един': 1662,\n",
       " 'надеюсь': 596,\n",
       " 'англи': 1146,\n",
       " 'воскресеньям': 3770,\n",
       " 'тысяч': 4362,\n",
       " 'уроки': 2373,\n",
       " 'девушку': 4530,\n",
       " 'предпочита': 2551,\n",
       " 'последнее': 2981,\n",
       " 'суббо': 4760,\n",
       " 'ваша': 947,\n",
       " 'оптими': 3649,\n",
       " '##ним': 1338,\n",
       " 'сих': 1936,\n",
       " 'поправи': 3661,\n",
       " 'худ': 3862,\n",
       " 'пят': 3157,\n",
       " 'изви': 1295,\n",
       " 'напишите': 4320,\n",
       " '##жды': 1827,\n",
       " 'работают': 4248,\n",
       " 'случится': 1443,\n",
       " '##льси': 2597,\n",
       " '##читал': 3113,\n",
       " 'согласился': 3785,\n",
       " 'доба': 2745,\n",
       " 'доро': 796,\n",
       " '##ству': 1744,\n",
       " 'тво': 297,\n",
       " 'решили': 3131,\n",
       " 'сумасшед': 3191,\n",
       " '##сть': 216,\n",
       " '##фо': 688,\n",
       " '##тишь': 3543,\n",
       " '##нок': 2434,\n",
       " 'стороны': 3767,\n",
       " 'посмотри': 1444,\n",
       " 'огонь': 3869,\n",
       " '##ковь': 3509,\n",
       " '##я': 75,\n",
       " '##пере': 2867,\n",
       " 'скорее': 3154,\n",
       " 'ль': 3394,\n",
       " 'телефо': 993,\n",
       " '##кал': 1615,\n",
       " 'бабушка': 4415,\n",
       " 'оскорб': 4699,\n",
       " 'почто': 4987,\n",
       " 'спокои': 2481,\n",
       " 'подня': 2330,\n",
       " '##жаю': 2323,\n",
       " 'сожале': 3137,\n",
       " 'рождество': 2440,\n",
       " 'сю': 4300,\n",
       " 'прочел': 2746,\n",
       " 'встретимся': 3528,\n",
       " 'расскажу': 3451,\n",
       " '##ехали': 2086,\n",
       " '##рить': 690,\n",
       " 'говорите': 1254,\n",
       " 'будут': 1753,\n",
       " 'окна': 2092,\n",
       " 'удивительно': 4895,\n",
       " '##биться': 3948,\n",
       " '##ться': 243,\n",
       " 'принесите': 3490,\n",
       " 'работал': 2843,\n",
       " 'избе': 3482,\n",
       " 'загля': 4933,\n",
       " '##вил': 1047,\n",
       " 'ума': 2504,\n",
       " '##v': 115,\n",
       " '##вьте': 1708,\n",
       " '##дет': 707,\n",
       " 'вернуться': 1829,\n",
       " '##бно': 1969,\n",
       " 'зна': 206,\n",
       " 'лекар': 2057,\n",
       " '##рин': 1468,\n",
       " '##тка': 2068,\n",
       " 'вести': 2584,\n",
       " 'зонтик': 2690,\n",
       " '##еди': 2262,\n",
       " 'лег': 1169,\n",
       " 'счастлива': 2971,\n",
       " 'телевизору': 3920,\n",
       " 'торопи': 3979,\n",
       " '##l': 104,\n",
       " 'своими': 2754,\n",
       " 'пиццу': 3415,\n",
       " 'любои': 3571,\n",
       " 'чист': 4373,\n",
       " '##рых': 4434,\n",
       " '##тив': 1092,\n",
       " '##ницу': 1498,\n",
       " 'телефону': 3764,\n",
       " '##рию': 4869,\n",
       " 'сделать': 339,\n",
       " 'тебя': 231,\n",
       " 'нельз': 1141,\n",
       " 'обратно': 2148,\n",
       " 'поможешь': 3981,\n",
       " 'велико': 3653,\n",
       " 'ради': 1868,\n",
       " '##чо': 4016,\n",
       " '##w': 117,\n",
       " '##лено': 3129,\n",
       " 'полт': 3726,\n",
       " 'единственныи': 3065,\n",
       " 'хотите': 527,\n",
       " 'праз': 3960,\n",
       " '##рал': 952,\n",
       " 'опла': 4123,\n",
       " 'комп': 1462,\n",
       " 'печ': 2611,\n",
       " 'поцеловать': 4645,\n",
       " 'ним': 1203,\n",
       " 'фут': 1790,\n",
       " 'дума': 282,\n",
       " 'чего': 564,\n",
       " 'верну': 696,\n",
       " 'пра': 827,\n",
       " 'выглядишь': 1775,\n",
       " 'автомоби': 1916,\n",
       " 'двига': 3738,\n",
       " 'родился': 2763,\n",
       " '##вым': 1279,\n",
       " 'зде': 299,\n",
       " 'чест': 4177,\n",
       " 'г': 36,\n",
       " 'просто': 437,\n",
       " 'наде': 461,\n",
       " 'обеща': 1285,\n",
       " 'супер': 3361,\n",
       " 'навести': 3442,\n",
       " 'познакомились': 4409,\n",
       " 'приходит': 4558,\n",
       " 'предлож': 1351,\n",
       " '##o': 103,\n",
       " '##стая': 2360,\n",
       " '##споко': 3856,\n",
       " 'входи': 4762,\n",
       " 'взял': 2273,\n",
       " 'тот': 1138,\n",
       " '##тами': 3072,\n",
       " 'большое': 2564,\n",
       " 'ответи': 1292,\n",
       " 'бассе': 4211,\n",
       " 'состои': 4136,\n",
       " 'понедельника': 2936,\n",
       " 'другом': 2104,\n",
       " 'столом': 4482,\n",
       " 'мужчи': 2043,\n",
       " 'улицу': 2818,\n",
       " 'опре': 3597,\n",
       " 'позвони': 2267,\n",
       " 'постра': 3105,\n",
       " 'двоюрод': 3975,\n",
       " 'звонок': 4078,\n",
       " 'кухне': 3270,\n",
       " 'смерть': 4691,\n",
       " 'проти': 3795,\n",
       " '##рко': 1441,\n",
       " 'ап': 4541,\n",
       " '##ые': 4813,\n",
       " 'такие': 1496,\n",
       " '##кла': 1702,\n",
       " 'послу': 1880,\n",
       " '##мать': 1996,\n",
       " 'ря': 2116,\n",
       " 'наити': 1156,\n",
       " '##ваитесь': 2743,\n",
       " 'расска': 998,\n",
       " 'нормально': 3212,\n",
       " '##лке': 4926,\n",
       " 'ручку': 3309,\n",
       " '##вое': 1383,\n",
       " '##ши': 213,\n",
       " '##ром': 478,\n",
       " 'хожу': 2305,\n",
       " 'поедем': 3243,\n",
       " 'сидеть': 3802,\n",
       " 'автомобиль': 2482,\n",
       " '##выи': 678,\n",
       " 'даваи': 628,\n",
       " 'босто': 421,\n",
       " '##вец': 4616,\n",
       " 'необходимо': 2235,\n",
       " 'будет': 446,\n",
       " 'дали': 3561,\n",
       " 'тринадцать': 4107,\n",
       " '##ы': 85,\n",
       " 'дерева': 3731,\n",
       " '##вы': 305,\n",
       " 'неспра': 4450,\n",
       " 'школе': 1630,\n",
       " 'история': 2250,\n",
       " '##поз': 996,\n",
       " 'защити': 2976,\n",
       " 'огром': 3413,\n",
       " 'странныи': 3553,\n",
       " 'жела': 2184,\n",
       " 'результа': 2387,\n",
       " 'вашем': 3956,\n",
       " 'вместе': 1019,\n",
       " 'чемодан': 3972,\n",
       " 'чисто': 4395,\n",
       " 'получится': 3112,\n",
       " 'спорить': 4634,\n",
       " 'пользоваться': 2927,\n",
       " 'обна': 3160,\n",
       " 'открыт': 4785,\n",
       " 'напуга': 3096,\n",
       " '##сается': 4285,\n",
       " 'будешь': 1032,\n",
       " 'сидел': 4070,\n",
       " 'починить': 2030,\n",
       " '##сти': 293,\n",
       " 'тенни': 1825,\n",
       " '##нечно': 2312,\n",
       " 'горо': 976,\n",
       " '##ваются': 3961,\n",
       " 'т': 50,\n",
       " '##нулись': 4021,\n",
       " 'привык': 2350,\n",
       " 'рубашку': 3173,\n",
       " 'стыдно': 4595,\n",
       " 'виду': 2119,\n",
       " 'прошлои': 1556,\n",
       " 'пиц': 2561,\n",
       " 'жил': 2714,\n",
       " 'попробуи': 3115,\n",
       " 'ветер': 3668,\n",
       " '##даите': 3106,\n",
       " '##чка': 1534,\n",
       " '##нев': 3250,\n",
       " 'ужина': 2833,\n",
       " 'узнал': 2459,\n",
       " 'устав': 2632,\n",
       " 'бывал': 4064,\n",
       " 'серьезно': 2308,\n",
       " '##чала': 1926,\n",
       " 'вроде': 2630,\n",
       " '##щую': 4840,\n",
       " 'один': 624,\n",
       " 'буд': 3912,\n",
       " '##жат': 4990,\n",
       " 'адрес': 2526,\n",
       " 'вопросов': 1886,\n",
       " '##зи': 600,\n",
       " '##щ': 73,\n",
       " 'предпо': 1520,\n",
       " 'горя': 2699,\n",
       " 'выбро': 3826,\n",
       " 'открои': 4218,\n",
       " 'вол': 1616,\n",
       " 'саду': 3001,\n",
       " '##тит': 1406,\n",
       " 'счастливым': 3003,\n",
       " 'ху': 1454,\n",
       " 'компьюте': 3291,\n",
       " 'особенно': 3926,\n",
       " 'познакоми': 1968,\n",
       " 'делаю': 1554,\n",
       " '##тик': 2313,\n",
       " 'спал': 2382,\n",
       " 'слома': 3282,\n",
       " '##ятно': 1124,\n",
       " 'сове': 4385,\n",
       " 'англииском': 4838,\n",
       " 'где': 369,\n",
       " 'уп': 2176,\n",
       " 'часть': 2858,\n",
       " 'нам': 292,\n",
       " 'голодныи': 4216,\n",
       " '##лад': 1484,\n",
       " '##лоси': 1427,\n",
       " '##мо': 177,\n",
       " 'кому': 1339,\n",
       " '##надца': 4000,\n",
       " '##бли': 1067,\n",
       " '##вень': 4511,\n",
       " '##лер': 4096,\n",
       " 'спасти': 3432,\n",
       " 'началь': 2436,\n",
       " 'плать': 1589,\n",
       " 'b': 9,\n",
       " 'стран': 1043,\n",
       " 'выглядит': 1922,\n",
       " '##сибо': 776,\n",
       " '##сьте': 4768,\n",
       " 'следовало': 2260,\n",
       " 'жениться': 3739,\n",
       " 'приз': 2514,\n",
       " 'настоящии': 4185,\n",
       " '##t': 114,\n",
       " '##то': 120,\n",
       " 'деиствительно': 670,\n",
       " 'такая': 1225,\n",
       " '##нет': 1544,\n",
       " 'пиро': 2455,\n",
       " '##рала': 2541,\n",
       " '##ссе': 3476,\n",
       " 'смешно': 3763,\n",
       " 'заи': 1410,\n",
       " 'статью': 4955,\n",
       " '##нил': 1014,\n",
       " '##вои': 1073,\n",
       " 'поспать': 2945,\n",
       " '##вно': 705,\n",
       " '##яс': 809,\n",
       " '##ступ': 1742,\n",
       " 'пожени': 4600,\n",
       " 'сом': 1980,\n",
       " 'думают': 3983,\n",
       " 'намерен': 4914,\n",
       " 'идет': 1344,\n",
       " '##сности': 1617,\n",
       " 'закан': 3793,\n",
       " '##нимался': 4086,\n",
       " '##жите': 587,\n",
       " 'глу': 1049,\n",
       " 'настро': 2711,\n",
       " 'осторожно': 3093,\n",
       " 'голод': 1629,\n",
       " 'ло': 741,\n",
       " 'вода': 2228,\n",
       " '?': 7,\n",
       " 'друзья': 1042,\n",
       " 'переби': 4759,\n",
       " '##пустить': 3205,\n",
       " 'пос': 315,\n",
       " 'даите': 1161,\n",
       " '##рел': 1209,\n",
       " '##ходят': 4031,\n",
       " 'спрят': 3403,\n",
       " '##вались': 2624,\n",
       " '##вает': 812,\n",
       " '##ля': 230,\n",
       " '##але': 1064,\n",
       " 'получил': 1787,\n",
       " '##вку': 3036,\n",
       " '##зом': 3886,\n",
       " 'стороне': 4640,\n",
       " 'засме': 4839,\n",
       " 'будьте': 1810,\n",
       " 'остановились': 4949,\n",
       " 'значение': 4308,\n",
       " 'смы': 2327,\n",
       " '##кет': 4197,\n",
       " 'если': 566,\n",
       " '##пая': 4435,\n",
       " 'поставьте': 3677,\n",
       " 'удиви': 3079,\n",
       " 'ошибки': 2428,\n",
       " '##чно': 390,\n",
       " 'боле': 1178,\n",
       " '##соб': 1518,\n",
       " 'говори': 322,\n",
       " 'собирался': 3231,\n",
       " '##би': 261,\n",
       " 'фотографию': 3228,\n",
       " '##ются': 1000,\n",
       " '##ружи': 1470,\n",
       " 'отли': 1422,\n",
       " 'столько': 2047,\n",
       " '##вается': 1284,\n",
       " '##жим': 3829,\n",
       " 'чувства': 4111,\n",
       " 'листья': 4938,\n",
       " '##мои': 604,\n",
       " 'постараитесь': 3454,\n",
       " '##нт': 3352,\n",
       " 'би': 904,\n",
       " 'ладно': 2349,\n",
       " 'поцело': 2617,\n",
       " 'неве': 2070,\n",
       " 'выучи': 2663,\n",
       " '##си': 233,\n",
       " 'каж': 507,\n",
       " 'последнии': 2156,\n",
       " 'слышал': 937,\n",
       " 'друзьями': 2407,\n",
       " 'выро': 4233,\n",
       " 'зем': 1800,\n",
       " 'опера': 4124,\n",
       " 'живет': 1586,\n",
       " 'лоша': 2324,\n",
       " 'рабо': 301,\n",
       " '##шек': 2336,\n",
       " 'начали': 3076,\n",
       " 'медленно': 3518,\n",
       " 'заходи': 4817,\n",
       " '##довал': 4821,\n",
       " '##рался': 1955,\n",
       " 'год': 2415,\n",
       " 'неболь': 2861,\n",
       " '##скую': 3068,\n",
       " 'бли': 1487,\n",
       " ...}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ru_tokenizer.get_vocab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6bcd29af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5002"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_tokenizer.get_vocab_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "231c9fa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5002"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ru_tokenizer.get_vocab_size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "13e28ce5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['я', 'опоздал', 'на', 'ре', '##и', '##с', '.']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ru_tokenizer.encode(train_input_strings[1]).tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d17952d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i', 'missed', 'my', 'flight', '.']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_tokenizer.encode(train_target_strings[1]).tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "980db267",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderRNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        super(EncoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        output = embedded\n",
    "        output, hidden = self.gru(output, hidden)\n",
    "        return output, hidden\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3dff139c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderRNN(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size):\n",
    "        super(DecoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "        self.embedding = nn.Embedding(output_size, hidden_size)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        output = self.embedding(input).view(1, 1, -1)\n",
    "        output = F.relu(output)\n",
    "        output, hidden = self.gru(output, hidden)\n",
    "        output = self.softmax(self.out(output[0]))\n",
    "        return output, hidden\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c0a148c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LENGTH = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "19e0e52c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttnDecoderRNN(nn.Module):\n",
    "    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):\n",
    "        super(AttnDecoderRNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.output_size = output_size\n",
    "        self.dropout_p = dropout_p\n",
    "        self.max_length = max_length\n",
    "\n",
    "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
    "        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n",
    "        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
    "        self.dropout = nn.Dropout(self.dropout_p)\n",
    "        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n",
    "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
    "\n",
    "    def forward(self, input, hidden, encoder_outputs):\n",
    "        embedded = self.embedding(input).view(1, 1, -1)\n",
    "        embedded = self.dropout(embedded)\n",
    "\n",
    "        attn_weights = F.softmax(\n",
    "            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)\n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
    "                                 encoder_outputs.unsqueeze(0))\n",
    "\n",
    "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
    "        output = self.attn_combine(output).unsqueeze(0)\n",
    "\n",
    "        output = F.relu(output)\n",
    "        output, hidden = self.gru(output, hidden)\n",
    "\n",
    "        output = F.log_softmax(self.out(output[0]), dim=1)\n",
    "        return output, hidden, attn_weights\n",
    "\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7b01f7bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tensorFromSentence(tokenizer, sentence):\n",
    "    indexes = tokenizer.encode(sentence).ids\n",
    "    indexes.append(EOS_token)\n",
    "    indexes = torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)\n",
    "    return indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "edabeef2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#teacher_forcing_ratio = 0.5\n",
    "\n",
    "\n",
    "def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):\n",
    "    encoder_hidden = encoder.initHidden()\n",
    "\n",
    "    encoder_optimizer.zero_grad()\n",
    "    decoder_optimizer.zero_grad()\n",
    "\n",
    "    input_length = input_tensor.size(0)\n",
    "    target_length = target_tensor.size(0)\n",
    "\n",
    "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "    loss = 0\n",
    "    if(input_length >= MAX_LENGTH ):\n",
    "        input_length = (MAX_LENGTH - 1) #better to lose some tokens than fail\n",
    "    for ei in range(input_length):\n",
    "        encoder_output, encoder_hidden = encoder(\n",
    "            input_tensor[ei], encoder_hidden)\n",
    "        encoder_outputs[ei] = encoder_output[0, 0]\n",
    "\n",
    "    decoder_input = torch.tensor([[SOS_token]], device=device)\n",
    "\n",
    "    decoder_hidden = encoder_hidden\n",
    "\n",
    "    #use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
    "    use_teacher_forcing = False\n",
    "    if use_teacher_forcing:\n",
    "        # Teacher forcing: Feed the target as the next input\n",
    "        for di in range(target_length):\n",
    "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
    "                decoder_input, decoder_hidden, encoder_outputs)\n",
    "            loss += criterion(decoder_output, target_tensor[di])\n",
    "            decoder_input = target_tensor[di]  # Teacher forcing\n",
    "\n",
    "    else:\n",
    "        # Without teacher forcing: use its own predictions as the next input\n",
    "        for di in range(target_length):\n",
    "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
    "                decoder_input, decoder_hidden, encoder_outputs)\n",
    "            topv, topi = decoder_output.topk(1)\n",
    "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
    "\n",
    "            loss += criterion(decoder_output, target_tensor[di])\n",
    "            if decoder_input.item() == EOS_token:\n",
    "                break\n",
    "\n",
    "    loss.backward()\n",
    "\n",
    "    encoder_optimizer.step()\n",
    "    decoder_optimizer.step()\n",
    "\n",
    "    return loss.item() / target_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "af860237",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import math\n",
    "\n",
    "\n",
    "def asMinutes(s):\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)\n",
    "\n",
    "\n",
    "def timeSince(since, percent):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    es = s / (percent)\n",
    "    rs = es - s\n",
    "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b5af9ab",
   "metadata": {},
   "source": [
    "#converting strings to tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "aa2c8984",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input_tensors = []\n",
    "train_target_tensors = []\n",
    "for i in train_input_strings:\n",
    "    train_input_tensors.append(tensorFromSentence(ru_tokenizer, i))\n",
    "for i in train_target_strings:\n",
    "    train_target_tensors.append(tensorFromSentence(en_tokenizer, i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "42dddf68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input: мы оба победили.\n",
      "target: we both won.\n"
     ]
    }
   ],
   "source": [
    "#checking if everything is fine\n",
    "n = 256\n",
    "print(\"input: \" + ru_tokenizer.decode(train_input_tensors[n].squeeze(-1).tolist()))\n",
    "print(\"target: \" + en_tokenizer.decode(train_target_tensors[n].squeeze(-1).tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2512f4d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "id": "752c5c5d",
   "metadata": {},
   "source": [
    "# Create data loaders for training sets\n",
    "batch_size = 200\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True,num_workers=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7c47ff10",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "plot_losses = []\n",
    "def trainIters(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01, save_every=1000, batch_size = 200):\n",
    "    start = time.time()\n",
    "    print_loss_total = 0  # Reset every print_every\n",
    "    plot_loss_total = 0  # Reset every plot_every\n",
    "\n",
    "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate, weight_decay = 0.0001 )\n",
    "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate, weight_decay = 0.0001 )\n",
    "    \n",
    "    criterion = nn.NLLLoss()    \n",
    "    for iter in range(1, n_iters + 1):\n",
    "        print(f'{iter} of {n_iters}', end='\\r')\n",
    "        loss  = 0\n",
    "        for i in range(batch_size):\n",
    "            n = random.randint(0, lens(train_input_tensors)-1)\n",
    "            input_tensor = train_input_tensors[n]\n",
    "            target_tensor = train_target_tensors[n]\n",
    "            #print(\"debug\")\n",
    "            #print(\"input: \" + ru_tokenizer.decode(input_tensor.squeeze(-1).tolist()))\n",
    "            #print(\"target: \" + en_tokenizer.decode(target_tensor.squeeze(-1).tolist()))\n",
    "            loss += train(input_tensor, target_tensor, encoder,\n",
    "                     decoder, encoder_optimizer, decoder_optimizer, criterion)\n",
    "        print_loss_total += (loss/batch_size)\n",
    "        plot_loss_total += (loss/batch_size)\n",
    "\n",
    "        if iter % print_every == 0:\n",
    "            print_loss_avg = print_loss_total / print_every\n",
    "            print_loss_total = 0\n",
    "            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),\n",
    "                                         iter, iter / n_iters * 100, print_loss_avg))\n",
    "        if iter % save_every == 0:\n",
    "            model_file_name = f'weights/encoder/model_weights_{iter}.pth'\n",
    "            print(f'Saving model as {model_file_name}')\n",
    "            torch.save(encoder, model_file_name)\n",
    "            model_file_name = f'weights/decoder/model_weights_{iter}.pth'\n",
    "            print(f'Saving model as {model_file_name}')\n",
    "            torch.save(decoder, model_file_name)\n",
    "\n",
    "        if iter % plot_every == 0:\n",
    "            plot_loss_avg = plot_loss_total / plot_every\n",
    "            plot_losses.append(plot_loss_avg)\n",
    "            plot_loss_total = 0\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6154f458",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "mkdir: cannot create directory ‘weights’: File exists\n"
     ]
    }
   ],
   "source": [
    "%mkdir weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "00ba9c65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "mkdir: cannot create directory ‘weights/encoder’: File exists\n"
     ]
    }
   ],
   "source": [
    "%mkdir weights/encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2588db7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "mkdir: cannot create directory ‘weights/decoder’: File exists\n"
     ]
    }
   ],
   "source": [
    "%mkdir weights/decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "f6f63eb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4m 20s (- 429m 31s) (1000 1%) 4.1306\n",
      "8m 47s (- 430m 36s) (2000 2%) 3.4799\n",
      "13m 17s (- 429m 33s) (3000 3%) 3.1480\n",
      "17m 48s (- 427m 24s) (4000 4%) 2.9355\n",
      "22m 17s (- 423m 29s) (5000 5%) 2.7768\n",
      "26m 45s (- 419m 17s) (6000 6%) 2.6343\n",
      "31m 16s (- 415m 26s) (7000 7%) 2.5531\n",
      "35m 45s (- 411m 11s) (8000 8%) 2.4717\n",
      "40m 18s (- 407m 38s) (9000 9%) 2.4076\n",
      "45m 10s (- 406m 35s) (10000 10%) 2.3495\n",
      "Saving model as weights/encoder/model_weights_10000.pth\n",
      "Saving model as weights/decoder/model_weights_10000.pth\n",
      "50m 0s (- 404m 36s) (11000 11%) 2.3122\n",
      "54m 37s (- 400m 33s) (12000 12%) 2.2756\n",
      "59m 15s (- 396m 34s) (13000 13%) 2.2157\n",
      "63m 54s (- 392m 36s) (14000 14%) 2.1883\n",
      "68m 33s (- 388m 27s) (15000 15%) 2.1626\n",
      "73m 12s (- 384m 20s) (16000 16%) 2.1488\n",
      "77m 42s (- 379m 24s) (17000 17%) 2.0958\n",
      "82m 39s (- 376m 34s) (18000 18%) 2.0695\n",
      "87m 40s (- 373m 44s) (19000 19%) 2.0495\n",
      "92m 40s (- 370m 42s) (20000 20%) 2.0189\n",
      "Saving model as weights/encoder/model_weights_20000.pth\n",
      "Saving model as weights/decoder/model_weights_20000.pth\n",
      "97m 40s (- 367m 25s) (21000 21%) 2.0079\n",
      "102m 41s (- 364m 5s) (22000 22%) 2.0220\n",
      "107m 40s (- 360m 29s) (23000 23%) 1.9654\n",
      "112m 40s (- 356m 48s) (24000 24%) 1.9593\n",
      "117m 41s (- 353m 3s) (25000 25%) 1.9643\n",
      "122m 41s (- 349m 11s) (26000 26%) 1.9358\n",
      "127m 42s (- 345m 17s) (27000 27%) 1.9104\n",
      "132m 43s (- 341m 17s) (28000 28%) 1.8823\n",
      "137m 44s (- 337m 14s) (29000 28%) 1.8669\n",
      "142m 46s (- 333m 7s) (30000 30%) 1.8740\n",
      "Saving model as weights/encoder/model_weights_30000.pth\n",
      "Saving model as weights/decoder/model_weights_30000.pth\n",
      "147m 47s (- 328m 56s) (31000 31%) 1.8476\n",
      "152m 48s (- 324m 43s) (32000 32%) 1.8249\n",
      "157m 49s (- 320m 26s) (33000 33%) 1.8000\n",
      "162m 51s (- 316m 8s) (34000 34%) 1.8173\n",
      "167m 52s (- 311m 46s) (35000 35%) 1.8042\n",
      "172m 55s (- 307m 25s) (36000 36%) 1.7676\n",
      "177m 58s (- 303m 2s) (37000 37%) 1.7576\n",
      "183m 1s (- 298m 36s) (38000 38%) 1.7554\n",
      "188m 3s (- 294m 8s) (39000 39%) 1.7403\n",
      "193m 5s (- 289m 38s) (40000 40%) 1.7062\n",
      "Saving model as weights/encoder/model_weights_40000.pth\n",
      "Saving model as weights/decoder/model_weights_40000.pth\n",
      "198m 8s (- 285m 7s) (41000 41%) 1.7223\n",
      "203m 11s (- 280m 35s) (42000 42%) 1.6978\n",
      "208m 13s (- 276m 1s) (43000 43%) 1.6808\n",
      "213m 17s (- 271m 27s) (44000 44%) 1.6647\n",
      "218m 20s (- 266m 51s) (45000 45%) 1.6646\n",
      "223m 22s (- 262m 13s) (46000 46%) 1.6418\n",
      "228m 26s (- 257m 36s) (47000 47%) 1.6340\n",
      "233m 30s (- 252m 57s) (48000 48%) 1.6388\n",
      "238m 33s (- 248m 17s) (49000 49%) 1.6381\n",
      "243m 35s (- 243m 35s) (50000 50%) 1.5834\n",
      "Saving model as weights/encoder/model_weights_50000.pth\n",
      "Saving model as weights/decoder/model_weights_50000.pth\n",
      "248m 39s (- 238m 54s) (51000 51%) 1.5937\n",
      "253m 43s (- 234m 12s) (52000 52%) 1.5573\n",
      "258m 47s (- 229m 29s) (53000 53%) 1.5826\n",
      "263m 51s (- 224m 46s) (54000 54%) 1.5593\n",
      "268m 55s (- 220m 1s) (55000 55%) 1.5396\n",
      "273m 58s (- 215m 15s) (56000 56%) 1.5374\n",
      "279m 1s (- 210m 29s) (57000 56%) 1.5372\n",
      "284m 5s (- 205m 43s) (58000 57%) 1.5179\n",
      "289m 10s (- 200m 57s) (59000 59%) 1.5134\n",
      "294m 14s (- 196m 9s) (60000 60%) 1.5012\n",
      "Saving model as weights/encoder/model_weights_60000.pth\n",
      "Saving model as weights/decoder/model_weights_60000.pth\n",
      "299m 12s (- 191m 17s) (61000 61%) 1.4971\n",
      "304m 9s (- 186m 25s) (62000 62%) 1.4690\n",
      "309m 17s (- 181m 38s) (63000 63%) 1.4643\n",
      "314m 24s (- 176m 51s) (64000 64%) 1.4752\n",
      "319m 31s (- 172m 3s) (65000 65%) 1.4320\n",
      "324m 38s (- 167m 14s) (66000 66%) 1.4223\n",
      "329m 46s (- 162m 25s) (67000 67%) 1.4256\n",
      "334m 54s (- 157m 36s) (68000 68%) 1.4084\n",
      "340m 2s (- 152m 46s) (69000 69%) 1.4257\n",
      "345m 9s (- 147m 55s) (70000 70%) 1.4085\n",
      "Saving model as weights/encoder/model_weights_70000.pth\n",
      "Saving model as weights/decoder/model_weights_70000.pth\n",
      "350m 17s (- 143m 4s) (71000 71%) 1.3939\n",
      "355m 25s (- 138m 13s) (72000 72%) 1.3797\n",
      "360m 32s (- 133m 21s) (73000 73%) 1.3817\n",
      "365m 40s (- 128m 28s) (74000 74%) 1.3757\n",
      "370m 48s (- 123m 36s) (75000 75%) 1.3674\n",
      "375m 54s (- 118m 42s) (76000 76%) 1.3464\n",
      "381m 2s (- 113m 48s) (77000 77%) 1.3481\n",
      "386m 10s (- 108m 55s) (78000 78%) 1.3612\n",
      "391m 18s (- 104m 1s) (79000 79%) 1.3371\n",
      "396m 24s (- 99m 6s) (80000 80%) 1.3280\n",
      "Saving model as weights/encoder/model_weights_80000.pth\n",
      "Saving model as weights/decoder/model_weights_80000.pth\n",
      "401m 33s (- 94m 11s) (81000 81%) 1.3264\n",
      "406m 41s (- 89m 16s) (82000 82%) 1.3101\n",
      "411m 49s (- 84m 20s) (83000 83%) 1.2945\n",
      "416m 57s (- 79m 25s) (84000 84%) 1.2975\n",
      "422m 5s (- 74m 29s) (85000 85%) 1.3113\n",
      "427m 15s (- 69m 33s) (86000 86%) 1.2553\n",
      "432m 25s (- 64m 36s) (87000 87%) 1.2945\n",
      "437m 33s (- 59m 39s) (88000 88%) 1.2920\n",
      "442m 40s (- 54m 42s) (89000 89%) 1.2799\n",
      "447m 48s (- 49m 45s) (90000 90%) 1.2593\n",
      "Saving model as weights/encoder/model_weights_90000.pth\n",
      "Saving model as weights/decoder/model_weights_90000.pth\n",
      "452m 58s (- 44m 47s) (91000 91%) 1.2551\n",
      "458m 7s (- 39m 50s) (92000 92%) 1.2310\n",
      "463m 16s (- 34m 52s) (93000 93%) 1.2007\n",
      "468m 24s (- 29m 53s) (94000 94%) 1.2408\n",
      "473m 31s (- 24m 55s) (95000 95%) 1.2066\n",
      "478m 40s (- 19m 56s) (96000 96%) 1.2315\n",
      "483m 50s (- 14m 57s) (97000 97%) 1.2029\n",
      "488m 58s (- 9m 58s) (98000 98%) 1.2043\n",
      "494m 6s (- 4m 59s) (99000 99%) 1.1948\n",
      "499m 14s (- 0m 0s) (100000 100%) 1.1615\n",
      "Saving model as weights/encoder/model_weights_100000.pth\n",
      "Saving model as weights/decoder/model_weights_100000.pth\n"
     ]
    }
   ],
   "source": [
    "hidden_size = 800\n",
    "encoder1 = EncoderRNN(ru_tokenizer.get_vocab_size(), hidden_size).to(device)\n",
    "attn_decoder1 = AttnDecoderRNN(hidden_size, en_tokenizer.get_vocab_size(), dropout_p=0.5).to(device)\n",
    "\n",
    "trainIters(encoder1, attn_decoder1, 100000, print_every=1000, save_every = 10000, batch_size = 10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4ab9f2c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUPUlEQVR4nO3dd3xT5eIG8CfpSHe66KKDQqFltOxR9iggIoKDq4gCinj1whXc4p63/MRxVRTcXAeiqIAiQ1ZBpIwWCm2RQpmldLCadKZtcn5/tD3NaZK2adOk4/l+PvmYc857Tt4chTy+5x0yQRAEEBEREdmI3NYVICIioo6NYYSIiIhsimGEiIiIbIphhIiIiGyKYYSIiIhsimGEiIiIbIphhIiIiGyKYYSIiIhsyt7WFWgMnU6Hy5cvw93dHTKZzNbVISIiokYQBAGFhYUICgqCXG66/aNNhJHLly8jJCTE1tUgIiKiJsjKykJwcLDJ420ijLi7uwOo+jIeHh42rg0RERE1hlqtRkhIiPg7bkqbCCM1j2Y8PDwYRoiIiNqYhrpYsAMrERER2RTDCBEREdkUwwgRERHZFMMIERER2RTDCBEREdkUwwgRERHZFMMIERER2RTDCBEREdkUwwgRERHZFMMIERER2RTDCBEREdkUwwgRERHZVJtYKK+lfLHvHC5eK8bsYWHo4V//ioJERETUMjp0y8im45fxv8QLOH+12NZVISIi6rA6dBhxdrADAJRWaG1cEyIioo6rQ4cRp+owoqnQ2bgmREREHVezwsiyZcsgk8mwZMkSk2VWr14NmUwmeTk5OTXnYy2GLSNERES21+QOrIcPH8Ynn3yCmJiYBst6eHggIyND3JbJZE39WItSOFRlsTKGESIiIptpUstIUVERZs+ejc8++wxeXl4NlpfJZAgICBBf/v7+TflYi2PLCBERke01KYwsXLgQU6dORVxcXKPKFxUVISwsDCEhIZg+fTrS09PrLa/RaKBWqyWvllDTZ6SMfUaIiIhsxuwwsnbtWhw5cgTx8fGNKh8ZGYkvv/wSGzduxLfffgudTofhw4fj0qVLJs+Jj4+HUqkUXyEhIeZWs1GcxTDClhEiIiJbMSuMZGVlYfHixfjuu+8a3Qk1NjYWc+bMQb9+/TBmzBj88ssv6NSpEz755BOT5yxduhQqlUp8ZWVlmVPNRnNinxEiIiKbM6sDa3JyMvLz8zFgwABxn1arxd69e7FixQpoNBrY2dnVew0HBwf0798fmZmZJssoFAooFApzqtYkTuwzQkREZHNmhZEJEyYgNTVVsu/+++9HVFQUnnnmmQaDCFAVXlJTU3HzzTebV9MW4MTHNERERDZnVhhxd3dHnz59JPtcXV3h4+Mj7p8zZw46d+4s9il57bXXMGzYMERERKCgoADLly/HhQsX8OCDD1roKzRd7WgadmAlIiKyFYsvlHfx4kXI5bVdUW7cuIEFCxYgNzcXXl5eGDhwIPbv349evXpZ+qPNxpYRIiIi25MJgiDYuhINUavVUCqVUKlU8PDwsNh1d53MwwOrkxATrMSvi0Za7LpERETU+N/vjr02jT1bRoiIiGytY4cRR46mISIisrWOHUbsOQMrERGRrXXoMOJc3TJSVs6WESIiIlvp0GFEnIG1kmGEiIjIVjp0GKmZZ6RCK6BSy0c1REREttChw0jNPCMAUFbJMEJERGQLHTqMKOxrv34p+40QERHZRIcOIzKZjCv3EhER2ViHDiNAbb8RhhEiIiLb6PBhpHZ9GvYZISIisoUOH0ZqV+5lywgREZEtdPgwomAYISIisqkOH0ac2YGViIjIpjp8GHFiB1YiIiKb6vBhhKNpiIiIbKvDhxGn6sXyOOkZERGRbTCM2Fe3jHA6eCIiIpvo8GHE2bHqFrBlhIiIyDY6fBipbRlhGCEiIrKFDh9GnKv7jJSxZYSIiMgmOnwY4XTwREREtsUwUh1GSji0l4iIyCY6fBgJ8HACAFy4VmzjmhAREXVMHT6MxAQrAQB/56ihYSdWIiIiq+vwYSTYyxmO9nJUaAXkqzW2rg4REVGH0+HDiEwmg5N91W0o17ITKxERkbV1+DACAIrqTqwajqghIiKyOoYRAI52bBkhIiKyFYYRAAqHqtug4fBeIiIiq2MYAaConhKeLSNERETWxzACwNG+pmWEYYSIiMjaGEYAKDiahoiIyGYYRlAbRjjpGRERkfUxjEAvjPAxDRERkdUxjKC2zwgf0xAREVkfwwhqR9OUlvMxDRERkbUxjACQy2QAgPgtJyEIgo1rQ0RE1LE0K4wsW7YMMpkMS5YsqbfcunXrEBUVBScnJ0RHR2Pz5s3N+ViLy7xSJL5Xl1XasCZEREQdT5PDyOHDh/HJJ58gJiam3nL79+/HrFmzMH/+fBw9ehQzZszAjBkzkJaW1tSPtrjsG6Xi++vF5TasCRERUcfTpDBSVFSE2bNn47PPPoOXl1e9Zd9//33cdNNNeOqpp9CzZ0+8/vrrGDBgAFasWNGkCreEUd19xfcMI0RERNbVpDCycOFCTJ06FXFxcQ2WTUxMNCg3efJkJCYmmjxHo9FArVZLXi3ppVt6ie8ZRoiIiKzL7DCydu1aHDlyBPHx8Y0qn5ubC39/f8k+f39/5ObmmjwnPj4eSqVSfIWEhJhbTbN4uTpibGQnAMANhhEiIiKrMiuMZGVlYfHixfjuu+/g5OTUUnXC0qVLoVKpxFdWVlaLfVYNbxdHAMD1EoYRIiIia7I3p3BycjLy8/MxYMAAcZ9Wq8XevXuxYsUKaDQa2NnZSc4JCAhAXl6eZF9eXh4CAgJMfo5CoYBCoTCnas3m7Mi5RoiIiGzBrJaRCRMmIDU1FSkpKeJr0KBBmD17NlJSUgyCCADExsZi586dkn3bt29HbGxs82puYU4OVXXXVHIWViIiImsyq2XE3d0dffr0kexzdXWFj4+PuH/OnDno3Lmz2Kdk8eLFGDNmDN555x1MnToVa9euRVJSEj799FMLfQXLqFmfpqyCLSNERETWZPEZWC9evIicnBxxe/jw4VizZg0+/fRT9O3bFz/99BM2bNhgEGpsjS0jREREtmFWy4gxCQkJ9W4DwMyZMzFz5szmflSLElfurWTLCBERkTVxbZpqYhipYMsIERGRNTGMVKt9TMOWESIiImtiGKmmcKjpwMqWESIiImtiGKnmZM+WESIiIltgGKlW0zLC0TRERETWxTBSraZlhPOMEBERWRfDSDW2jBAREdkGw0g1BVtGiIiIbIJhpFrN0F4ulEdERGRdDCPV3BRVk9EWl2shCIKNa0NERNRxMIxUc3OqCiNancB+I0RERFbEMFLNpfoxDQAUllXasCZEREQdC8NINblcJj6qKdIwjBAREVkLw4gesd8IwwgREZHVMIzocVVUParhYxoiIiLrYRjR4+bkAIAtI0RERNbEMKLHrbplhH1GiIiIrIdhRA87sBIREVkfw4geV4YRIiIiq2MY0ePO0TRERERWxzCip2YWVo6mISIish6GET2ubBkhIiKyOoYRPTWPadYlX8LVIo2Na0NERNQxMIzoqWkZAYBNxy7bsCZEREQdB8OIHnVphfjey9XRhjUhIiLqOBhG9EQHK21dBSIiog6HYUTPwDBvONpV3RJNpc7GtSEiIuoYGEbqmNDTDwDDCBERkbUwjNShsK9uGanQ2rgmREREHQPDSB0K+6rF8tgyQkREZB0MI3U42rPPCBERkTUxjNQhPqap5GMaIiIia2AYqUPhUNNnhC0jRERE1sAwUgf7jBAREVkXw0gdfExDRERkXQwjddSEkc2pOTauCRERUcfAMFKHVqj6Z1mFDlnXS2xbGSIiog7ArDCycuVKxMTEwMPDAx4eHoiNjcWWLVtMll+9ejVkMpnk5eTk1OxKtyQXRzvxfX5hmQ1rQkRE1DGYFUaCg4OxbNkyJCcnIykpCePHj8f06dORnp5u8hwPDw/k5OSIrwsXLjS70i3ptv6dxfel5ezESkRE1NLszSk8bdo0yfabb76JlStX4sCBA+jdu7fRc2QyGQICAppeQytzcrBDvxBPpGQVYPX+8xjZ3dfWVSIiImrXmtxnRKvVYu3atSguLkZsbKzJckVFRQgLC0NISEiDrSitRU0n1h1/5yFfzUc1RERELcmslhEASE1NRWxsLMrKyuDm5ob169ejV69eRstGRkbiyy+/RExMDFQqFd5++20MHz4c6enpCA4ONvkZGo0GGo1G3Far1eZWs1kqtLWPZwo1lfCz6qcTERF1LGa3jERGRiIlJQUHDx7EI488grlz5+LEiRNGy8bGxmLOnDno168fxowZg19++QWdOnXCJ598Uu9nxMfHQ6lUiq+QkBBzq9ks+hOelWg43wgREVFLMjuMODo6IiIiAgMHDkR8fDz69u2L999/v1HnOjg4oH///sjMzKy33NKlS6FSqcRXVlaWudVsFknLSFmFVT+biIioo2n2PCM6nU7ySKU+Wq0WqampCAwMrLecQqEQhw/XvKypvFL6mIaIiIhajll9RpYuXYopU6YgNDQUhYWFWLNmDRISErBt2zYAwJw5c9C5c2fEx8cDAF577TUMGzYMERERKCgowPLly3HhwgU8+OCDlv8mFiQJI2UMI0RERC3JrDCSn5+POXPmICcnB0qlEjExMdi2bRsmTpwIALh48SLk8trGlhs3bmDBggXIzc2Fl5cXBg4ciP3795vs8NpaBHk647KqahQNH9MQERG1LJkgCIKtK9EQtVoNpVIJlUpllUc2F64VY8zyBADA4xN74NEJ3Vv8M4mIiNqbxv5+c20aI8J8XPHQ6K4AgCL2GSEiImpRDCMmuCmqnmDxMQ0REVHLYhgxwd2pKoyo2YGViIioRTGMmODu5AAAKGIYISIialEMIybwMQ0REZF1MIyY4OFUE0bYMkJERNSSGEZMEB/TcDQNERFRi2IYMcGdLSNERERWwTBiQk0YKdJU4vzVYhvXhoiIqP1iGDHBw9lBfD/27QTbVYSIiKidYxgxwcGOt4aIiMga+IvbSKXlWltXgYiIqF1iGGmka8UaW1eBiIioXWIYaaTrxeW2rgIREVG7xDDSSNcYRoiIiFoEw0g9/jm6q/g+I7fQhjUhIiJqvxhG6vHslCj8a2w3AMD+M9dsXBsiIqL2iWGkHjKZDBN6+gEAzl4psnFtiIiI2ieGkQZ09nQBAOSoylCp1dm4NkRERO0Pw0gD/NwVcLCTQasTkKsus3V1iIiI2h2GkQbI5TIEKp0BAJcLGEaIiIgsjWGkEfzcFQCAK4Wc+IyIiMjSGEYawdetKoxcLWIYISIisjSGkUboVN0yciyrwLYVISIiaocYRhrByaHqNv1yNBsFJZyJlYiIyJIYRhqhu5+7+P7SjVIb1oSIiKj9YRhphFv7BYnv1aUVNqwJERFR+8Mw0ghODnYYFOYFANiQkg1NpdbGNSIiImo/GEYaSensAAD4MekSlm/NsHFtiIiI2g+GkUbyqA4jAPDdwYs2rAkREVH7wjDSSEq9MOLiaGfDmhAREbUvDCONpB9GnBlGiIiILIZhpJE6ezmL750dGEaIiIgshWGkkbr6uorv+ZiGiIjIchhGGinMpzaM8DENERGR5TCMNFIndwWCqx/VKOwZRoiIiCyFYcQMz9wUBQCc9IyIiMiCGEbMoLCvul2aSp2Na0JERNR+mBVGVq5ciZiYGHh4eMDDwwOxsbHYsmVLveesW7cOUVFRcHJyQnR0NDZv3tysCtuSU/UoGk0FwwgREZGlmBVGgoODsWzZMiQnJyMpKQnjx4/H9OnTkZ6ebrT8/v37MWvWLMyfPx9Hjx7FjBkzMGPGDKSlpVmk8tZW0zJyIkeNM1eKbFwbIiKi9kEmCILQnAt4e3tj+fLlmD9/vsGxu+66C8XFxdi0aZO4b9iwYejXrx9WrVrV6M9Qq9VQKpVQqVTw8PBoTnWbJSWrADM++gsAEBXgjt8fHQWtToCjPZ92ERER1dXY3+8m/4pqtVqsXbsWxcXFiI2NNVomMTERcXFxkn2TJ09GYmJiUz/Wphztam/XydxC3LlqP8a9ncAOrURERM1gb+4JqampiI2NRVlZGdzc3LB+/Xr06tXLaNnc3Fz4+/tL9vn7+yM3N7fez9BoNNBoNOK2Wq02t5otQiaTbh+9WAAAyMwvQu8gpfUrRERE1A6Y3TISGRmJlJQUHDx4EI888gjmzp2LEydOWLRS8fHxUCqV4iskJMSi12+qqAB33D3YsC7Ne9BFRETUsZkdRhwdHREREYGBAwciPj4effv2xfvvv2+0bEBAAPLy8iT78vLyEBAQUO9nLF26FCqVSnxlZWWZW80WIZPJsOyOGIP9xZpKG9SGiIiofWh2z0udTid5pKIvNjYWO3fulOzbvn27yT4mNRQKhTh8uObVmhWXM4wQERE1lVlhZOnSpdi7dy/Onz+P1NRULF26FAkJCZg9ezYAYM6cOVi6dKlYfvHixdi6dSveeecdnDx5Eq+88gqSkpKwaNEiy34LK/t49gDJdpGGHViJiIiayqwOrPn5+ZgzZw5ycnKgVCoRExODbdu2YeLEiQCAixcvQi6vzTfDhw/HmjVr8MILL+C5555D9+7dsWHDBvTp08ey38LKxkf5SbaLytgyQkRE1FTNnmfEGlrLPCP6ujz7u/j++Zt7YsHorjasDRERUevT4vOMdHSjuvuK74vYgZWIiKjJGEaa6Mt5gzFrSCgAIL/QeAdeIiIiahjDSBM52MkR17Oq78iOv/PQBp52ERERtUoMI80wONwbAHClUANNJVfyJSIiagqGkWZwdawdjBQbvxPZBaU2rA0REVHbxDDSDHZyGezlVQvW3CipwNeJ521bISIiojaIYaSZKnW1fUUUdrydRERE5uKvpwUVlFbYugpERERtDsOIBX2deAFXOMyXiIjILAwjFrbrZF7DhYiIiEjEMGJhJeVcNI+IiMgcDCMWlqfmYxoiIiJzMIxYWJ66zNZVICIialMYRprprTti4Opoh7sGhQAADp69hkotZ2MlIiJqLIaRZvrH4BCkvjIZN/UJAABcVpXhzc1/27hWREREbQfDiAXI5TJ4uTqK21/9dd52lSEiImpjGEYsxMvFQbKt03EVXyIiosZgGLEQTxdHyfaF6yU2qgkREVHbwjBiIR5O9pLttGyVjWpCRETUtjCMWIhMJpNsf3Pggo1qQkRE1LYwjFjQjsdH47mbowAAh85dh6aSs7ESERE1hGHEgiL83LFgVFc42lfd1rHLE3CtiDOyEhER1YdhxMJkMhkCPJwAADmqMnyw87SNa0RERNS6MYy0gJowAgDqskob1oSIiKj1YxhpAf7K2jBiL5fVU5KIiIgYRlqAv7tCfG9vV3uLNZVaCAInQyMiItLHMNICXBW1c4442FW1jGQXlGLQGzvwxLpjtqoWERFRq2TfcBEyl5ODnfj+68QL+Dqxds6RX45kIyO3EI9O6I7JvQNsUT0iIqJWhS0jLaCLj0u9x9Mvq/HPb5KtVBsiIqLWjWGkBUzuHWAwPTwREREZxzDSAuRyGR6d0N3W1SAiImoTGEZayM3RgbauAhERUZvAMNJCgjydMSc2TNy2l8vgrpA+uuEwXyIiIo6maVFP3xQFTxdH9A/1RA9/d8z69AAKNbUzspaUayXDgImIiDoi/hK2IDeFPR6f2EPcdq/TqbWwrJJhhIiIOjw+prEitzrBo6xCa6OaEBERtR4MI1bk7uQg2dZU6mxUEyIiotaDYcSK6s49wpYRIiIiM8NIfHw8Bg8eDHd3d/j5+WHGjBnIyMio95zVq1dDJpNJXk5OTvWe017V7TNy3xcHbVQTIiKi1sOsMLJnzx4sXLgQBw4cwPbt21FRUYFJkyahuLi43vM8PDyQk5Mjvi5cuFBv+faq7mMadVklnvjxGOZ+eQgJGfk2qhUREZFtmTWUY+vWrZLt1atXw8/PD8nJyRg9erTJ82QyGQICuChc3ZYRAPj5yCUAwJ5TV3B+2VRrV4mIiMjmmtVnRKVSAQC8vb3rLVdUVISwsDCEhIRg+vTpSE9Pr7e8RqOBWq2WvNqDui0jRERE1IwwotPpsGTJEowYMQJ9+vQxWS4yMhJffvklNm7ciG+//RY6nQ7Dhw/HpUuXTJ4THx8PpVIpvkJCQppazVYlUNlwX5kKrQ6XC0qtUBsiIqLWQSY0cU7yRx55BFu2bMG+ffsQHBzc6PMqKirQs2dPzJo1C6+//rrRMhqNBhqNRtxWq9UICQmBSqWCh4dHU6rbKmh1AtYevojn16cZPX5+2VTc/9Uh7M64gp8fGY6BYV5WriEREZHlqNVqKJXKBn+/m9QysmjRImzatAm7d+82K4gAgIODA/r374/MzEyTZRQKBTw8PCSv9sBOLsPsoWEmjyedv47dGVcAAN8e6JidfImIqOMxK4wIgoBFixZh/fr12LVrF8LDw83+QK1Wi9TUVAQGclXbuu5clSi+l8lsWBEiIiIrMms0zcKFC7FmzRps3LgR7u7uyM3NBQAolUo4OzsDAObMmYPOnTsjPj4eAPDaa69h2LBhiIiIQEFBAZYvX44LFy7gwQcftPBXaV/kTCNERNRBmBVGVq5cCQAYO3asZP9XX32FefPmAQAuXrwIuby2weXGjRtYsGABcnNz4eXlhYEDB2L//v3o1atX82rehtnLZajUVXXVCVQ6IUdVZlCmaT15iIiI2p4md2C1psZ2gGkrDpy9hufWp+KN6X0Q280HMa/8gUJNpaRMD383/PHYGBvVkIiIqPlatAMrNc+wrj7Y9cRYDI/whUwmw75nxxuUOZVXhMz8IhvUjoiIyLoYRloBpbMDNv17pMH+/+44hT/Sc21QIyIiIuthGGklegcZNl9tOp6Dh75JxpVCjZEziIiI2geGkVZCVs/omcFv7sDpvELkq8vw/o7TyFcbdnglIiJqq8waTUO2M/G9veL7XRn52LhwhA1rQ0REZDlsGWlFxkf5AQBeuqX+Yc/Hsgrw4c7T1qgSERFRi2MYaUVW3NMfPz8Si3nDuzRY9p3tp6DTtfpR2URERA1iGGlFXBztMTDMG3K5DD383Rosn8O+I0RE1A4wjLRSX8wdjDsHBuPWvkEmy6ReUuFakQb/238eG45mW7F2RERElsMOrK1UiLcL3p7ZF+/vMN035OFvkyXbPm6OGNW9U0tXjYiIyKLYMtLKKRwa/68o/bK6BWtCRETUMhhGWrlZQ0IR7uvaqLLLtpzE/jNXW7hGRERElsUw0sopnR2w+8mx+G2R4XTxxtzz2cEWrhEREZFlMYy0Ef4eCvH9C1N72rAmRERElsUw0kb4uNWGkRn9O0Nhb/pfXVmF1hpVIiIisgiZIAitfuYstVoNpVIJlUoFDw/DBeU6iqMXb6CkXIsREb6o0Orww+EsjOnRCUkXruOxH45Jyt7evzMeGBmOPp2V4r5KrQ5XijQIVDpbu+pERNQBNfb3m0N725D+oV7iewc7Oe4dFgYA0BqZifWXo9n45Wg2zi+biiJNJRIy8vFN4gUcPHcdPzw0DEO7+lit3kRERPVhGGkHwnxc6j3+zM/H8fvxHHH76wMXGEaIiKjVYJ+RdkAmk2FqTKDRY5VanSSIAKi3vwkREZG18Vepnfjg7v449PwEg/1jlicY7JNBhu0n8lCkqbRCzYiIiOrHMNJO2Mll8HN3wsrZAyT7swtKDcr+fOQSFnydhKd/OmZw7NzVYlwt0rRYPYmIiOpiGGlnpkQHol+IZ6PKbk7NlWwfuXgDE9/dgztX7kelVtcCtSMiIjLEMNIOfffg0EaXvVFcjvu/OoQNR7Px/cGLqNQJOH+tBO/tOAV1WUUL1pKIiKgKw0g75Kpo/CCpj3ZnYnfGFSz5IQU3Ssr19p9BzCt/cAI1IiJqcQwj7ZyDnQzfzh+K9FcnGz3++b5z4vsdf+cbHH/qp+MtVjciIiKAYaTd69NZiZHdfeGqsG/Smja/HbvcArUiIiKqxTDSTs0aEgoAeGJipLhPJpPZqjpEREQmMYy0U/+5rQ+SXojDyO6+4r7iJs4rcsuHfyIzv8hSVSMiIpJgGGmnZDIZfPVW+gWaHkbSstV47IcUo8euFmnw0e5M5KvLmnRtIiIihpEOJMS7dg2b5XfGiO/93BXo0sD6NqnZKvR99Q/csXI/Nh2/LI6yWbTmCJZvy8A/v00WywqCgEe/P4plW05a+BsQEVF7xDDSgdw1OASPTuiOdQ/HYuagEHG/r5sCOx4fgx7+bvWeryqtQPKFG1i05ije+SMDAHDg7HUAwNGLBWK51GwVfj12Gav2nLH8lyAionaHYaQDcbCT4/GJPTC4i7fBMXs7ObYsHo3426Mbda3P/jxn8lhZRe3srTqdYH5FiYioQ2EYIZGdXAYXR7tGl4+N32l0vyDUBpByTitPREQNYBjpwPp09gAA3D6gs7jP2aE2jEzpE1Dv+TkqaafV1X9VtZbot4VoKhlGiIiofo2fN5zane/mD8ORrBsYFVE7/NfNqfY/CT93hbHTTHrltxPoH+oleTSjqdQCcKj3vD/Sc7H/zDU8P7UnHOyYj4mIOhr+zd+BKV0cMC7SD/Z6AaB/iJf4Xi43nCTttem9MTDMy2B/jekf/SVpDSlvoGVEEAQ89E0yVu8/j0/Y4ZWIqENiGCEJZ0c7vD2zL26JCcR9w8IMjt83LAxfzB1U7zVWJtSGippg8vvxHAx+cwcOnr0mHlu15wxe+TVd3H77j1PIUZU29ysQEVEbwzBCBu4cGIwV9wxA105uWHFPf8nkaTKZDJ4ujvWef+j8dfG9urQC3xy4gIVrjuBKoQZ3fXoA14o0OH+1GMu2nMT/Ei9Izk3PVovvky/cwMR39+DP01cs9M2IiKg1MiuMxMfHY/DgwXB3d4efnx9mzJiBjIyMBs9bt24doqKi4OTkhOjoaGzevLnJFSbruiUmCKN7+Brsr1n7piG3fbwfL25Ik+z713dHkF+oMVq+ZvkcQRBwx8r9OJ1fhPu+OGRepYmIqE0xK4zs2bMHCxcuxIEDB7B9+3ZUVFRg0qRJKC4uNnnO/v37MWvWLMyfPx9Hjx7FjBkzMGPGDKSlpZk8h1oXuZEF9v5zWx+ENTBrqykHz13HG7+fMHps/v+S0OXZ3/HiRuP/fWTkFuL4pYImfS4REbVOMkF/UggzXblyBX5+ftizZw9Gjx5ttMxdd92F4uJibNq0Sdw3bNgw9OvXD6tWrWrU56jVaiiVSqhUKnh4eDS1utRET/90DD8mXQIAnF82Vdw/fcU+HLukskodzi+bCkEQEL60qlXt6IsT4eHsADsjnWyJiKh1aOzvd7P6jKhUVT9E3t6GM3rWSExMRFxcnGTf5MmTkZiYaPIcjUYDtVoteZHthPm4Gt3/6vQ+8HJxwH9ui4aHU8uPEtcfpbPzZD76vfZHVSvKhjTJRGs6nYAKMydbe/W3dEx6bw9Kypu2mCARETVdk8OITqfDkiVLMGLECPTp08dkudzcXPj7+0v2+fv7Izc31+Q58fHxUCqV4iskJMRkWWp580eGY9aQUHx1/2DJ/n4hnjjy4kTcMzQUmxePavJjGwCI9Hev9/jrm07gil4/kzd+P4HCsqrg8M2BCzh7tfZR4azPDmDs8gRxMb/G+Oqv8ziVV4RNx3LMrDkRETVXk8PIwoULkZaWhrVr11qyPgCApUuXQqVSia+srCyLfwY1npODHeJvj8a4SD+DY7Lq/iTBXi6Y0a+zwfHGurVfUL3Hv9h3Dnd/ekDcLiipkBwv1lTiWFYBPtx5GgfPXUd2QSmSzt8wux4CuJYOEZG1NaltfdGiRdi0aRP27t2L4ODgessGBAQgLy9Psi8vLw8BAaanGlcoFFAozJv9k2zv/hFdkJatwt7TV1Chrf1R93Z1hLq0Ag+O6oqCknKsPWwYLoO9nBu8fnaB6TlIki/cwKu/STvFqssqTJSW0n/EYyfnaHciImszK4wIgoB///vfWL9+PRISEhAeHt7gObGxsdi5cyeWLFki7tu+fTtiY2PNriy1bp4ujvhi3mAUayrR++Vt4v6xPTrhP7dHw8nBDoIgQFOpw/qj2QbnNoW7kz0KyyoNgggA3CgphyAIYuuNKfqrDNuzQywRkdWZ9b+BCxcuxLfffos1a9bA3d0dubm5yM3NRWlp7f+xzpkzB0uXLhW3Fy9ejK1bt+Kdd97ByZMn8corryApKQmLFi2y3LegVsVVYY93ZvYVt2+ODoRT9QJ8MpkMr8/og+dv7okHR4ZDLgO+fmAIvFzqX7/GlM6epltUnl+fhgVfJxnsrzuATL/TagO5RXKNa0XG50ohIiLzmNUysnLlSgDA2LFjJfu/+uorzJs3DwBw8eJFyPWauocPH441a9bghRdewHPPPYfu3btjw4YN9XZ6pbbvjoHBiOvpj8wrhRgQKl3Lxk1hjwWjuwIAnpwcCScHO2RdL2nS5/h7OOFkbqHJ4zv+zkdpuRbOjlVhKE9dhts++gv/GByCJXE9AAAl5bUdXSu1jesz8tz6NHx/6CJW3z8YY430pSEiosYz+zFNQxISEgz2zZw5EzNnzjTno6gdULo4YGCY6WHfAMQWE896Wkbievpjx995Ro+5NWJI8am8QvQN8USxphJzvzyEy6oy/HfHaUzrG4RundwkYaS8kUOCvz90EQDw3vZTDCNERM3E3nrUKrg7OeCOAcEYH+WHCD83yTFXhZ3Rc2YPDYWLg/Fj+jan5eDezw9i6H92SlpRJryzB2UVWsljGk2FfiuJDofOXa93iHCljqNviIiai2GEWo13/tEXX84bjEClk2S/i6Nh4Mh8cwrevC26UTOwfrLnLPZlXkWRxnBCs4nv7RHnKwFqW0YEQcCqPWfwj08S8cIG00sXaBlGiIiareWnzSQyU93w0dXXzaCMvV1Vjj6R07zZebOul+L8tdoJ0wpKKhD37h5EdHLD1vSqifl+Sr6Et/U65OpjywgRUfOxZYRanYfHdAMAjI/yw7GXJsHHzfSw39iuPgCArr6u+O7BoU36vAvXajvPpl9WIzO/SAwiDWHLCBFR87FlhFqd/qFe2P/seHRyV8DBTo5bYoJw6Nx1hHi74MDZa3ikOqwAwMLxEQjydMaU6AB0cmvaRHlf7Dsnvt9z6orJcqrSCng42UvmLanUGXZ4La/UYXdGPkZ374QiTSXu++Ig7h4cgnkjGp6Xh4ioI2IYoVYpSG/+EEd7OZbdEQMAWDguQlLOw8kBc4d3Ebc7ezrXO1NrU3i7OuJUXiEmvbcXM/oF4b939xePGcki+Cn5Ep5bn4owHxex1eWV304wjBARmcDHNNSuPDU50mDf0zdFYubA+pctqI+mQovP9p4FAGxIuSw5ZqxlZPuJqkc8+o9/aqRfVlk8LBERtXUMI9SuzOjfGXueGituDwrzwr/GRuDZKVFNvmZxuVbSUfXtbRnie/0+I4IgYO+pK/BwNj5nStb1Ekz9YB9GLNvV5LoQEbVHDCPU7oT5uGLdw7EYF9kJy6tHwfi4KbD/2fGScg+OrP+xiaNd7R8P/bV0VuzOFN/rh5R9mVcx58tD2Fin9aTG4fPXxfcV1UOI65vDpMaVQg2uF5c3WI6IqK1iGKF2aXAXb3x1/xCE+7qK+4LqrGPz9E1RGBpufIbYbp1csX/peKPH9KlKK8SZif/KvFZv2cd/PCa+v1ZUjvVHLyHqxa3YmJJt8hxVSQUGv7kDA17fzpE7RNRuMYxQhzS9XxAc7eUYVj00uK4KrQDfRozOEQTgkW+PIDO/CJfN6AtyMleNx36oCieL16aYLNf/9T/E96rSikZfn4ioLWEYoQ6lZnXgsZGdAEBcQA8AXpveW3wfHawEAKy4pz8asjU9F3Hv7sGvx4w/njFm3leHGyyj1QnQbwy5UcJHNUTUPnFoL3Uo2x4bjbRsFcb2qFrczllvbZvh3Xyw4/HRWHsoCw+PrZrL5JaYIIzq3gl9X/3D6PUsSRAEPLHuGMordXh7Zl+Ulkv7k9woLgc6tXg1iIisjmGEOhQ/dyeMj6pd+0ZhX9s46OHkAD8PJ7xwSy/JOUpnB/z4z1j845PEZn++r5sjrhYZb+E4nV+EX45U9R/ZdDwHnev0cblRUoGv/jqHtGw13rytj7jiMRFRW8fHNNShaYXa5yCmhuQCwJBwbwTVWcCvKUb3MN20Mem9vZLtuvOR3Cgpx6u/ncDPRy5hxa5MEBG1Fwwj1KFVVNZOWqbfSmKMnV3DKwQbm3RN/7r9Q72MBpKPExoOF5du1IaTQ3rDhPV9sPM0Rr+1GzmqqrKn8wqRqypD1vUSVGqNTBdLRNQKMIxQh1au9wOtv+aMMfby2j8u78zsi8m9/Q3K3DssTLI9vV8QNHqBZ/aQUHxwdz+D897ammGwr64DZ2qHDteEjbre3X4KF6+X4KPdmTh7pQgT39uLYfE7Meqt3ZKhxQDEIcmmNHSciMhSGEaoQ4sJ9mx0WTt5bVi5Y2AwVtwzAG/M6IMnJ/WAg50M79/dD26K2m5YX80bjPfvlo7Gkctl8HRxxJAuxuc3qY9+a0jW9VLsOJGHVXvO4OFvkvFHei6Gx+8Uj18p1GD8O3sk5+uP9nlpYxqGL9uFa0Uao5/1xb5zGBa/E+euFptdTyIic7EDK3Vow7r64PM5g9DNz63BsvZyacuJg51cbAlZMLorFPbSDqWO1Y9nogLccTK3UHJMU1k7UmZUd1/8efqqyc/t4e+GU3lFBvsf/DpJfL81PVdyLC1bbfRaf6TnYlLvAHydeAEA8Pm+c3jmJsOp8l/fdAIA8Opv6Vh9/xCTdSMisgS2jFCHF9fLXzJTqynPVK9vMyc2zOCYfhB5/uaeuH1AZ8RWT6j2yX0DMSDUE6vuHSiW0X908838oehaz+dHBXhItj+bM6jBuppajO+hb5KRX1gmbp+9UoSCknKMWb4bb1QHEH2caI2IrIEtI0SNNC7SD0kvxMHH1bHecgtGd5Vsh/m44pd/jZDsK6+UdiaV12l1sZfLxHVvXPUe/cT19EdcTz/JcXOlXCwQ329Lz4NWdxwXrpXg833nDIY1l1UYdnrV6QSD+hIRNQdbRojM4OumaLCja2No6oQRO71rLonrjpV6rSjODnZYdns0Ivzc8PK0XpDJZGhOFQ7XGYmz4+888f3k9/bi3T9qO9PWXcjvWFYBYl79A1/uOwcAyMwvwk/Jl9jZlYiahWGEyAaW3xkDO7kMz9/cE4C0ZWRJXA8Ee9VOeObkIMfdQ0Kx4/ExCPF2AVC1dk5T/X48x+SxjLxCfKA3h0lpuRal5VrszshHWYUW//ruCIo0lXit+pHOTf/diyfXHcOGlGzkqEqxas8ZFGkqxfMPnr2GRWuOIF9dZvBZREQ1+JiGyAaGR/gi/dXJ4iyqdTvH6k9T72zhmVYvqxofDIo1lbj5gz9x7mox5o8MN+iLUvOoaMff+eLCfw52cswfGQ4AuOvTAwCA9MtqKJ0d8MjYbpjcO8ASX4OI2hG2jBDZiP507nX7YOgv4Kf/viH/e8D0yJc9T41tfOWqFWoqxeG9X1Q/mqmh1euz8vfl2tE7Jy6rcbXOkOFzV4uRklWAxWuPml0HImr/GEaIWoGITtKhxU56o3PqtpoAwMvTesHRXo7AOlPUj6lnuvlO7grJtrKe6e8b4/n1qeL7s3rzkfx85BJi43fiwjXDOUr0J44jIqrBvxmIWoHnp/bEbf074/sFwwAATo61fzSN9Q65f0Q40l+djP6hnuK+526uGnr8n9uijX5G3cc9dcOJudYezjJ5rEIrGD1epKmsWn3YTOqyCklfFCJqXxhGiFoBb1dHvHdXP8R2q5qbxNGu4T+aDnZylFfWRpUFo6qGFN8zNBQ9Az0MytcdBaQ/W2xLqDt8uca/vzfvUU1ZhRYxr/yBPi9vg66Jw5mJqHVjGCFqhRo7fNjU2jqfzRmIObFhmBodaPJc/TDiaka/lMaq28ekxr5Mw9lmEzLykZlfiGtFGoPAkac3EqesUlv3VADA9eJyvL/jtMnJ3oiodWMYIWrDyk38OAd7ueC16X3EocDGKF0cEObjgs6ezujdWVnv52z698h6Z4k1V4be9PhJ569j3leHEffuXgx6cwfe+P1vAECFVoevE89Lpra/VlSOwrLaWWFLyiuxaM0RDHh9O97bcQpzvjhosToSkfUwjBC1csFepgNFZQPzjcwdHgYXRzv8Y1CwwTEZgG1LRmPXk2MkI3vqenZKFPp0VsKrgZln9S2dIl3vZmIv6QrHk/+7FwDwc/Il3LkqUdwvCMCXf1W1qOw7fRUvbUzHwjVHxOOj3tqN6Ff+QNb1EgDAZ3vPYZPevClnrnBhP6K2iPOMELVSXz8wBMcvFSCup5/JMi9P641/fJKIf0+IMHo8UOmMoy9NNNoHRUDt8GI7vadCp9+cAkGoXeivhpdL48PIzdGB0FTq8O72UwCAf47uiu0n8iRlPtx5Gu9UH6/rapFG8nimrjWHLuKZm6KQ24zJ1HQ6AUezCtAz0B0ujvyrkMiW+CeQqJUa3aMTRtczVBcAooOVOP7KJDjU0+FVfxG/0T06Ye+pKwCA+4d3Effb6Q25NXWtOwcGS6aOr09nT2e46PVD8XQxHEZsKogAVdPMq8tML9JXX+fbLak5iOvlX+89AYAfkrKw9JdUjIjwQXRnT3TxccHdQ0LrPYeIWgbDCFEb19CPrr5V9w7A8UsqdPFxRYDeHCURfm4NBo3Jvf2xZsFQ/JR0Cb8czTY4/o9BwSiv1KFviCfkcpmktUFhb2fW4n53f3oAU/qYnqk1R1UKnU7A94cuGhx75LsjeOamKDwytpvBsTNXilCpFRAZ4I7/7T8PAPgr8xr+yrxW9bl1wkhKVgHCvF3MekRFROZjGCHqQFwc7TGsq4/B/n+Pj0CRpgJTo4NMniuTyTC8my/cFQ5Iv6xGRl6h5Hj/UC/M0vsxd1XUtowo7OVIeXkS+ry8rdF13ZKWa/LYtwcu1juB2v9tPYnzV4uxaHwEPt17FoO6eKF/iBcmvLMHAHD8lUnQGVncT1OpFVuS9p+5ins+O4gwHxfseWqcQVlBEPDhrkx08XXFrX1N3zciahjDCBHBVWGPN2YYnyytruhgJbY9Nhr/238eKxPOYFxUJxSWVeKOAdJOsgq9PieO9vJGzWsS5uOCC9dKGlWP1dUtG6b8kJSFjceyUVahwzcHLkiO/ZJ8STKdfY2Ckgr4e1SFkQ3VrT8XrpUgI7cQFVod+uiNOjpy8YbYJ4ZhhKh5GEaIqEnmDu+CuXr9TurSb7mo6QwrlwH6GWBEhI/4iOTMf26GDMBlVSlG/t9ui9SxrML4xGuv/HbC6P6qMFL1+EpdWjvja83on9RXJsHdqar/S55aY3gBImoShhEiahF2ekN0akbz/LpoJG75cB8AILqzEt89OAznrxZD4SCHXfUaPJ5mjNqxtBsltVPVF2oMO9CuOXgRG1IuQ1OhRWSAu7i/UquDvZG+O1tSc1CpEzBNr+WkQquDDDBanqijMvtPw969ezFt2jQEBQVBJpNhw4YN9ZZPSEiATCYzeOXmmn4eTERtn4Ney0jND6/+Yw776rDSxdcVgUpncX9LzAbbWAUltQGkqMxwLZz4LSfxd44aZ68WS/q0vL7pBCq00laYzPwiPPLdEfz7+6NQVV+3UqvDpPf2IuL5LUjJKmiZL0HUBpkdRoqLi9G3b1989NFHZp2XkZGBnJwc8eXnZ3ruBCJq+8I71T9j6+Au3kb3609r72AnnRa/Z6BHsxf4q0+BXsuIqUc8xvwv8QJ+Sr4EoGpRv+0n8hD37h7xeM0w5eyCUpyrXuF4xkd/WaLKBo5fKsCrv6VDVWp6aDRRa2P2Y5opU6ZgypQpZn+Qn58fPD09zT6PiNqmzp7OWPvQMIM5RrYuGYWtabl4aHTXBq/h7GCHCm1tC8XaBcPg4WyP3Rn5WPpLKl69tTce/vZIPVcw9Mq0Xvg44QzyC2v7fLg42qGkXIuC0gp8e+AC1hy8aDBaqCHZN0rx/o7TeG+H4fwpxeVV30FTZ/HAYk0lXC28YOGtK6pCTlFZJZbP7GvRaxO1FKs9tOzXrx8CAwMxceJE/PVX/f9HoNFooFarJS8ianuGdfVBVIB0BeGoAA8sievRqFlPnR3t8PCYqvlC5o8Mh9LFATKZDOOj/HHwuTjc1CcQs4aEGJz39E2R+Gd12BmjN3Hcv8Z2w7wR4XBzkn52TSvNsi0n8cKGNJzIMf/vnBW7M40GEQA4fP4GrheX43e9qesBtOjCfkcu3mixaxNZWot3YA0MDMSqVaswaNAgaDQafP755xg7diwOHjyIAQMGGD0nPj4er776aktXjYhaOScHOzw5qQem9AlA7yAPo2Wen9oLSmdHlFVoxeG+C0Z1hU4QMCLCF0PCvRH14lYAtbPR6s8O+/HsATiTX4Q91TPTtoQXN6Thiz/P4nydYcuXbpSgh39VR9gKrQ7/3XEKo7t3wlC9uWBe+TUdf56+go2LRjZqeHSN0nLjiygStUYt3jISGRmJf/7znxg4cCCGDx+OL7/8EsOHD8d7771n8pylS5dCpVKJr6ysrJauJhG1QtGdlbC3k6NviKfJ0SduCns8OyUKQ8Jr+6A42MmhsLfD6B6d4ORgh5kDg+GmsMfM6gUDozt7imVvjg40Ol29pdUNIgBw6UZty8j/9p/HR7vP4K5PD0jKrN5/HmeuFGNrPZPAGVNcTxj5cOdp/HiYf69S62GTob1DhgzBvn37TB5XKBRQKFqukxoRtW4/PxKL7w9lGaz+W5+amWUD9aa5r/HWnTF447Y+YsvIE5N64O8cNe4Y0BlA1aKBxgzv5oOnb4rCnC8OQm1kdE1zZV0vwaxPD6CkQotwH8PVmfXX5xGMzBhbn5LySmxNy4GqtAJ3Da6dGTczv0hcF+jOgcGQy2WmLkFkNTYJIykpKQgMDLTFRxNRGzAwzBsDw4yPtjHF29URyS/EwdnI0GCZTCZZMNDXTYENC0eI2zWrF+tzdrDDN/OHwk4uw/FXJqPLs79Ljic8ORZj304wq451ffbnOfH92StF4vtKrQ6n8oqgN7AIZRX1P3YRBAGLvj8qbldoBbFz77CuPgjzcTW4jrqswqbzuhDVMDuMFBUVITMzU9w+d+4cUlJS4O3tjdDQUCxduhTZ2dn4+uuvAQD//e9/ER4ejt69e6OsrAyff/45du3ahT/++MNy34KICICPW9NaVG/tG4TdJ/Ph66bAyO6+6F/9WMhOr9Wgq68rzlYPywWq5kexpEK9lpdZnx3A4fPSDqhXCjVYsvYorhWXo1eQB56aFCk+uhIEAQfPXTfoIFsjT61BJ3cFXBztJfOhXC0qZxihVsHsMJKUlIRx42oXjXr88ccBAHPnzsXq1auRk5ODixdrV9IsLy/HE088gezsbLi4uCAmJgY7duyQXIOIyJacHOyw8t6B9Zb55sGh+Dn5Enb+nScuCDhrSKi4cvCtfYPw67HLRs+d0S8IG1Jqj90xIBg/H7lk8rPqBhEA2JByGRevV/U7+fP0VQR7ueB4VgH8PBSIDPDAo3qtInU99E0SisoqsWHhCEnH1mtFGkT4uUEQBFwtKhfncCmv1CElqwD9QjzFqfyJWpJMMPdBpA2o1WoolUqoVCp4eBjvUU9EZG06nYC8wjIEKp2RekmFaSsM+8I9FtcDj06IQPjSzQCqFhDMeGMKBEEQ91nLczdHIdzXDQu+TgIAxPX0h7erA7p2csOyLSfx9sy+GNzFC4vWHEVqtgpzYsPw2vQ+FvnsywWlmPXZAdwzJBT/rB6uTe1fY3+/GXmJiJpILpeJU9n7Kw0fEfUL8cT8UeGQyWQIr36sMy6yavZpmUyGrx8YYr3KArhcUIZcdZm4vePvPPyYdAnLtpwEADy57hj++U0yUrNVAICvEy8YvY6+bw5cwPD4ncjML6q33Ie7MnHhWgniqz+rrmtFGmxMyYamkkOSOyKGESIiC/Bzd8IHs/qL2yvu6Y8NC0eIc4P88shwLBoXgccn9RDLjO7RCY/F9TC4VktZvf88XtyQVm+Zk7nSmWdHv7UbXZ79HXeu3I/LBaVQlVTg3s8P4sekqqHBL25Iw2VVGZ5fn1rvdbW6+qfXf/DrJCxem4L3d5xuxDeh9oar9hIRWcitfYMQ7uOKvaevYFKvAMkxL1dHPDk50uCcB0Z2wd85amxNt+zioXIZoLPAQ/iafipJF25g+LJd6NbJFWeuFGNf5lXMHBgslrteXG7qEgCk86yUVWiRflmNfiGeYifhoxcLAADfH7qIp29q/JBuah/YMkJEZEHRwUosHBfR6I6f7k4OWDQ+wuTxXxeNMHmsPpZe86bGmSu1I4oGvbFDfH+tuBwrE87g8Pnr4r6S8krodAIycgtx6Fzt/kVrjuCOlfvxyd4zAIB/63W+LdbwMU1HxDBCRGRjfTor8fFs48tjhHhJJ0P755iGFxgEAKWzdFZZRxMz2DbHNb3WkOvF5fi/rScxc1UiCkrKMX3FPvR6aRve23EKf2VelZy34+98AMCqhDMoq9DiN71RSOXaxq+WTO0HwwgRUStwc7TxiSA9nB3w4MhwTI0JROabU/CskUcYE6L8DPa9f3c/yXa/UE9LVLNRPvvzLI5dquoE++GuTKOTygGAuqwSG1OyG3VNrU5A8oXrknlSqP1gGCEiamVm9AsS39vJZXjhll746J4BsLeTQ6Y3Leu84V2Q8tJEfDFvsME1BoZ540O9DrV9gpSS4906NW/Stvfu6mvyWPpl6arHz9XTufWZnw2PlVVoodUJWLHrNH5OvoQrhRp0e24z7liZiI93nzEon6sqw8j/24UVuww7v+aqyvDW1pPIum64NlBj3Cgux/7Mq9BZogMOmcQOrERErYSniwMKSiowf2RX3BwdaHJG2X3PjMPVonL0C/EU9znaycVHHJ/eVzWBm5tT7V/xkQFukms8OqE7Fq9NaVI9R0T4YFT3TiaPp1UPDW6qT/acxXs7Tonb4/Vaft7bcQpxvfzg5GCHbp2qvtPKhExculGKt/84hUXju0uu9fC3yUjJKsDHCWfw6ITueHyieaOXZnz8Fy5cK8EHs/rj1r5VIfF0XiEOnLuOe4aESmbppaZjywgRUSuR8ORYbFk8CtHBSkzqHYCBYV5GywV7uUiCCAB8PX8I/D0UWHXvQEzqXTWSx9WxNoz4uikwf2S4uO3n7iT5ka8RqHTCTw/H1lvPL+cNho+rI9z1OsnG3x4tvr9aVNWXpLOnc73XMUU/iADArpP5ku2pH+zDhHf2YO+pKwCAynpaLVKyCsT3H+w8bXKW3BplFVpJK8qF6lFA2/RGO018by9e3JCGdUlc+dhSGEaIiFoJTxdH9Axs2izTw7r64OBzcbipT+2QYhe9RQN93BRYOK521I6niwMen9gDSmcHPDmpB2YNCQEA/Oe2aAzq4o1N/x6J2/t3xpoHh8LLxQHP3BSFCD83jI/yg8LermoiN71HPUPDvXFTb+lw5hdv6dWk79JYc748hKfWHYODXufcTcfrDxsrdp1God5qyHXd+/lBjHprNz7anSlZHNHN0fBBQtIFw2n768ouKMVT647hZK66wbIdGcMIEVE7JQkjro7iBGxAVUtJn85KpLw0EYvGd8ebM6KRuHQ8xlW3lvTprMS7d/XD8AhfHHlxIh4Z2w1/LBmNL+YOEq/Rxac2jDjayyXhBADCfV3x59PSdcj0J4azhHXJl1BQUjuqZ9Gao+jx/BakZatQaaSz66m8IkS/8ofRR0kZuYViwFi+LUNy7IekLGgqtdBfQaUxT2gWrTmCdcmXcMfH+xv7lSxm3+mreGPTCZRXtv5OvwwjRETtlH6LgY+bIxzt5fhwVn+8M7OvuCheTYdY/ant69Ivo9+B1levT4ujvRwPj5auOdPJXYEQ79qhyUPCvTE1OhCeLtJhx/qCvWrrMLm3v+TY9wuGGT0nu6BUsl2u1eGWD/dh1Fu7TX7OLR/uk4SYM1eKMPm/e02WB4Cfk7NRpKldXVkuk6aRfaev4uvE85J9NaGnuHqBQkEQkJlfZDQo6SsoKTf4Xua694uD+HzfOYM6tUYMI0RE7VSwlzNu6h2AuwaFwKX6McO0vkG4Q2/m1OZwVdS2vCjs7KB0cYC/R21A8awz14lWJ8BOLsORFyZi40Ljk7npB5yYYE/xvdLZAYO6GO9DY2yVYwDIUZVJtl0dpUOMd2fU9kXZeLThIcZXizS4UqgRtwtKKnD8UgEqtDoIgoB7vziIlzam4/ilApRX6vDChlRUaKX9WX4+ko24d/fg6Z+O1/tZ/V7bjhHLdjU4s21jNLRuUGvAMEJE1E7JZDKsum8g/u/OmBa5vv78ITUzztYECEc7OeR1nmPUtAbI5TL0rdMBFwAWjusGd70RQANCa8OHt6ujpKXnvmFhZtfXX+kk2d5+Ig+/H8/BrpN5WG9ivhNnve/47vZTGP/OHnF7a3oubl3xF7o/v0XSwbWgpALbT+Th2wMXDa73/s6qzrm/1BN+9BcLPHOl+UGiLUwkx6G9RETUJM5GwshzN/dEV19XxPWqfcRiL5ehUidgYJi30euMjeyEpyZHIirAA3O+PCjuHxDmKb6v6ffw4z9j8euxbDw7JQrfHGh4VeFP7xsIrU6As6Md3toq7QeyOTUXm1PrXxMorpe/ZIZYUx7+9oj4XlOpQ2mF8WntK7X1z1eSekmFb/W+l5O98QnjzFG3daY1YssIERE1ibPeY4+a+TbCfV2x9OaeGNylNnhse2w0npociScmGZ/jI9LfHb2DlLCTyyQ/nAp7O3Fa+2nVc3wMCffGGzOiG732TrivK6ZEB2JspF+j1guaWecRlq+bY6M+R9+Cr5Nww8TjFf1hyMbC1HPrU/GD3pBh/VaSzPxCHL9UIG6nZasw+b292H4iD6pS0yOEKtiBlYiI2isnh8b9hHTr5IaF4yIMAsTL03phUJgX/qU35Lhux87V9w/GU5MjGzVZ2d2DQ7DriTGSjq/Bemv7PH1T1arJ4yJNT9g2WW948oez+kMG40NmujYwg+2bm/822Lf6r3OSPicvbkjDI98mi9uq0gqk1hnlU1ZRdT8EQUDcu3tx64q/kKcug1Yn4OFvk5GRV4gFXyeh76t/ICFDOh9Lja3puXhq3THJSKDWhmGEiIiapGYG1Ka6f0Q4fnpkuGRRv7qzzvYP9TK5CvLiCdLZVh3s5OjayQ399fqa6LfeDO/mi0PPT8AXcwcbdGat0cXXFY+M7YZR3X0xsZc//jHYeGffAaFeGNLF+GMnU1757YTBvi1ptY+JXtyQZnBcXVYBQRAkHVlX7z+P0W/txqUb0tE2L26sPb9uqFuXfAnnrzVtSnxrYJ8RIiJqkphgT7x1R4xkOG5zvXRLL1wt0mDBqIZXJ148oTum9Q1C3LtVnUpr+mnMGhyKDUezEdfT3+AcP/eqTqyuCntxuO3yO2PwVPXoFs/qCd5qRAV4ICZYieOXpC0WJeWVWPvQMMhkQO+Xt6Gk3Hgfkcb4OCETf6TnSWaLrfGv76r6onTxqW3hWZlguD4PAGgqagPI2sOGs8MW6w1Lbm0YRoiIqMn+MTjEotcL8XbB+n8ZH/Zbl1wuQ4SfG3zdFLhapBGnt1e6OGDrktH1nqvfd2NS74DaMOJsOAeKh5PhvhvFFeJooeYEEQAGHWuNaUyrRplep9kXjLSyfPbnWfzfHTEmV1G2JYYRIiJq07YtGYW/cwoxIsKn0efoT16mdHbA7ifHQgbA3s7wcZDMSLeRbn7G+4z4eyjw7JQoPPbDsUbXpan6BitRrhXwd07VVPNlFfV3VN2Ychk+rgq8NE06TX/N/C+2xD4jRETUpvm4KTCyu69kdtiG1J0iPdzXFV18jQeMF2/pBV83BV6e1gu/LhqBubFheGJipEG5cZGdcPC5ONzW3zKTyhkT4l37SCz+9hj0DHAXt8u1Ojy57himfbjP5Plf/nXOYN8tH+5D31f/wKFz1y1bWTMwjBARUYfzaHXn149nD2iwbA9/dxx+fgLuHxGOmGBPvDq9D7xca4f8fvfgUMzoF4T37uon7nv+5p5NqteCUeFwsDMeqv5zWzRmDQkVtz2c7SXT7QPAT8mXDEbkNORKYRlUpRWSGXWtjWGEiIg6nMUTuuPQ8xNwc3Rgo8rX1+oyIsIX/727PzxdagPKgtFdcfj5ONwSI73+nqfGmnycNKZHJ8waEoq9T4+TLHJY467BIXDXGx7t4ewAfw8ng3KG9ZN+3qxPD+CVX9ORdb0EWl3tSJ2a9YpsgWGEiIg6HDu5TBxZ01I6uSuw4p4B+PS+gejs6YwfHhqGMB9XfHSP8daY/z0wBF07uSFQ6YzAOlPXT+rlDzu5TNKnxc3Rvt5FBwGgf6gn7hvWRbIv8ew1rN5/Hit2ZeJ6cTl0QlW/GG8X8yd4sxR2YCUiImpBk3oHYJLeZGqeTfjRr1nLR7/FRC6X1RtGwnxc8MNDsbh0w/hInB+SslChq+o74+3iaLTzrrWwZYSIiKiVeevOvgj3dcWLt/TCC1N74oER4QCqZoiNCVZi3vAuAIDBXbwxpIs3ZvQLMrhGD393ONpXTQT3yrReiNLr7FrjlyNVC/b5utnuEQ3AlhEiIqJWZ2CYF3Y/OdZgv5ODHX5dNFLcdrCT48eHYwEAqdkqnLlSLB7Tn9l23ohwzIntgrWHsxAV6I7bP94vuW6gZ8s+smoIW0aIiIjagc/nDsaHs/qL2yXl0hlX5XIZ7hkaigGhXnjpll5w0+sMG+IlHZVjbQwjREREVvbDQ8MwJNwbC8d1g7ODHR6La3ghwIaE+7piWt8gccTNAL01eup6YGQ4Ul+ZhKcmRyK8ej0eW5IJrXkZv2pqtRpKpRIqlQoeHh62rg4REZHFVGh1cLBg59HrxeX4/fhl3DYgWNL6YQuN/f1mnxEiIiIbsmQQAQBvV0fcF9vFotdsaXxMQ0RERDbFMEJEREQ2xTBCRERENsUwQkRERDbFMEJEREQ2xTBCRERENmV2GNm7dy+mTZuGoKAgyGQybNiwocFzEhISMGDAACgUCkRERGD16tVNqCoRERG1R2aHkeLiYvTt2xcfffRRo8qfO3cOU6dOxbhx45CSkoIlS5bgwQcfxLZt28yuLBEREbU/Zk96NmXKFEyZMqXR5VetWoXw8HC88847AICePXti3759eO+99zB58mRzP56IiIjamRbvM5KYmIi4uDjJvsmTJyMxMdHkORqNBmq1WvIiIiKi9qnFw0hubi78/f0l+/z9/aFWq1FaWmr0nPj4eCiVSvEVEhLS0tUkIiIiG2mVo2mWLl0KlUolvrKysmxdJSIiImohLb5QXkBAAPLy8iT78vLy4OHhAWdnZ6PnKBQKKBSKlq4aERERtQItHkZiY2OxefNmyb7t27cjNja20dcQBAEA2HeEiIioDan53a75HTfF7DBSVFSEzMxMcfvcuXNISUmBt7c3QkNDsXTpUmRnZ+Prr78GADz88MNYsWIFnn76aTzwwAPYtWsXfvzxR/z++++N/szCwkIAYN8RIiKiNqiwsBBKpdLkcZnQUFypIyEhAePGjTPYP3fuXKxevRrz5s3D+fPnkZCQIDnnsccew4kTJxAcHIwXX3wR8+bNa/Rn6nQ6XL58Ge7u7pDJZOZUt15qtRohISHIysqCh4eHxa5LhnivrYP32Tp4n62D99l6WupeC4KAwsJCBAUFQS433U3V7DDSnqjVaiiVSqhUKv6H3sJ4r62D99k6eJ+tg/fZemx9r1vlaBoiIiLqOBhGiIiIyKY6dBhRKBR4+eWXOYzYCnivrYP32Tp4n62D99l6bH2vO3SfESIiIrK9Dt0yQkRERLbHMEJEREQ2xTBCRERENsUwQkRERDbVocPIRx99hC5dusDJyQlDhw7FoUOHbF2lNiM+Ph6DBw+Gu7s7/Pz8MGPGDGRkZEjKlJWVYeHChfDx8YGbmxvuuOMOg0UTL168iKlTp8LFxQV+fn546qmnUFlZac2v0qYsW7YMMpkMS5YsEffxPltOdnY27r33Xvj4+MDZ2RnR0dFISkoSjwuCgJdeegmBgYFwdnZGXFwcTp8+LbnG9evXMXv2bHh4eMDT0xPz589HUVGRtb9Kq6XVavHiiy8iPDwczs7O6NatG15//XXJ2iW8z02zd+9eTJs2DUFBQZDJZNiwYYPkuKXu6/HjxzFq1Cg4OTkhJCQEb731VvMrL3RQa9euFRwdHYUvv/xSSE9PFxYsWCB4enoKeXl5tq5amzB58mThq6++EtLS0oSUlBTh5ptvFkJDQ4WioiKxzMMPPyyEhIQIO3fuFJKSkoRhw4YJw4cPF49XVlYKffr0EeLi4oSjR48KmzdvFnx9fYWlS5fa4iu1eocOHRK6dOkixMTECIsXLxb38z5bxvXr14WwsDBh3rx5wsGDB4WzZ88K27ZtEzIzM8Uyy5YtE5RKpbBhwwbh2LFjwq233iqEh4cLpaWlYpmbbrpJ6Nu3r3DgwAHhzz//FCIiIoRZs2bZ4iu1Sm+++abg4+MjbNq0STh37pywbt06wc3NTXj//ffFMrzPTbN582bh+eefF3755RcBgLB+/XrJcUvcV5VKJfj7+wuzZ88W0tLShO+//15wdnYWPvnkk2bVvcOGkSFDhggLFy4Ut7VarRAUFCTEx8fbsFZtV35+vgBA2LNnjyAIglBQUCA4ODgI69atE8v8/fffAgAhMTFREISqPzhyuVzIzc0Vy6xcuVLw8PAQNBqNdb9AK1dYWCh0795d2L59uzBmzBgxjPA+W84zzzwjjBw50uRxnU4nBAQECMuXLxf3FRQUCAqFQvj+++8FQRCEEydOCACEw4cPi2W2bNkiyGQyITs7u+Uq34ZMnTpVeOCBByT7br/9dmH27NmCIPA+W0rdMGKp+/rxxx8LXl5ekr87nnnmGSEyMrJZ9e2Qj2nKy8uRnJyMuLg4cZ9cLkdcXBwSExNtWLO2S6VSAQC8vb0BAMnJyaioqJDc46ioKISGhor3ODExEdHR0fD39xfLTJ48GWq1Gunp6Vasfeu3cOFCTJ06VXI/Ad5nS/r1118xaNAgzJw5E35+fujfvz8+++wz8fi5c+eQm5sruddKpRJDhw6V3GtPT08MGjRILBMXFwe5XI6DBw9a78u0YsOHD8fOnTtx6tQpAMCxY8ewb98+TJkyBQDvc0ux1H1NTEzE6NGj4ejoKJaZPHkyMjIycOPGjSbXz77JZ7ZhV69ehVarlfzlDAD+/v44efKkjWrVdul0OixZsgQjRoxAnz59AAC5ublwdHSEp6enpKy/vz9yc3PFMsb+HdQcoypr167FkSNHcPjwYYNjvM+Wc/bsWaxcuRKPP/44nnvuORw+fBiPPvooHB0dMXfuXPFeGbuX+vfaz89Pctze3h7e3t6819WeffZZqNVqREVFwc7ODlqtFm+++SZmz54NALzPLcRS9zU3Nxfh4eEG16g55uXl1aT6dcgwQpa1cOFCpKWlYd++fbauSruTlZWFxYsXY/v27XBycrJ1ddo1nU6HQYMG4T//+Q8AoH///khLS8OqVaswd+5cG9eu/fjxxx/x3XffYc2aNejduzdSUlKwZMkSBAUF8T53YB3yMY2vry/s7OwMRhzk5eUhICDARrVqmxYtWoRNmzZh9+7dCA4OFvcHBASgvLwcBQUFkvL69zggIMDov4OaY1T1GCY/Px8DBgyAvb097O3tsWfPHnzwwQewt7eHv78/77OFBAYGolevXpJ9PXv2xMWLFwHU3qv6/t4ICAhAfn6+5HhlZSWuX7/Oe13tqaeewrPPPou7774b0dHRuO+++/DYY48hPj4eAO9zS7HUfW2pv086ZBhxdHTEwIEDsXPnTnGfTqfDzp07ERsba8OatR2CIGDRokVYv349du3aZdBsN3DgQDg4OEjucUZGBi5evCje49jYWKSmpkr+49++fTs8PDwMfhQ6qgkTJiA1NRUpKSnia9CgQZg9e7b4nvfZMkaMGGEwPP3UqVMICwsDAISHhyMgIEByr9VqNQ4ePCi51wUFBUhOThbL7Nq1CzqdDkOHDrXCt2j9SkpKIJdLf3rs7Oyg0+kA8D63FEvd19jYWOzduxcVFRVime3btyMyMrLJj2gAdOyhvQqFQli9erVw4sQJ4aGHHhI8PT0lIw7ItEceeURQKpVCQkKCkJOTI75KSkrEMg8//LAQGhoq7Nq1S0hKShJiY2OF2NhY8XjNkNNJkyYJKSkpwtatW4VOnTpxyGkD9EfTCALvs6UcOnRIsLe3F958803h9OnTwnfffSe4uLgI3377rVhm2bJlgqenp7Bx40bh+PHjwvTp040Ojezfv79w8OBBYd++fUL37t07/JBTfXPnzhU6d+4sDu395ZdfBF9fX+Hpp58Wy/A+N01hYaFw9OhR4ejRowIA4d133xWOHj0qXLhwQRAEy9zXgoICwd/fX7jvvvuEtLQ0Ye3atYKLiwuH9jbHhx9+KISGhgqOjo7CkCFDhAMHDti6Sm0GAKOvr776SixTWloq/Otf/xK8vLwEFxcX4bbbbhNycnIk1zl//rwwZcoUwdnZWfD19RWeeOIJoaKiwsrfpm2pG0Z4ny3nt99+E/r06SMoFAohKipK+PTTTyXHdTqd8OKLLwr+/v6CQqEQJkyYIGRkZEjKXLt2TZg1a5bg5uYmeHh4CPfff79QWFhoza/RqqnVamHx4sVCaGio4OTkJHTt2lV4/vnnJUNFeZ+bZvfu3Ub/Xp47d64gCJa7r8eOHRNGjhwpKBQKoXPnzsKyZcuaXXeZIOhNe0dERERkZR2yzwgRERG1HgwjREREZFMMI0RERGRTDCNERERkUwwjREREZFMMI0RERGRTDCNERERkUwwjREREZFMMI0RERGRTDCNERERkUwwjREREZFMMI0RERGRT/w8MxdxcwVvhBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#%pip install matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "#showPlot(plot_losses)\n",
    "plt.plot(plot_losses)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d8f1d8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder1 = torch.load(\"./weights/encoder/model_weights_60000.pth\")\n",
    "attn_decoder1 = torch.load(\"./weights/decoder/model_weights_60000.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "be0c1585",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(encoder, decoder, sentence, max_length=MAX_LENGTH):\n",
    "    with torch.no_grad():\n",
    "        input_tensor = tensorFromSentence(ru_tokenizer, sentence)\n",
    "        input_length = input_tensor.size()[0]\n",
    "        encoder_hidden = encoder.initHidden()\n",
    "        #print(input_tensor)\n",
    "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "        for ei in range(input_length):\n",
    "            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n",
    "                                                     encoder_hidden)\n",
    "            encoder_outputs[ei] += encoder_output[0, 0]\n",
    "        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS\n",
    "\n",
    "        decoder_hidden = encoder_hidden\n",
    "\n",
    "        decoded_words = []\n",
    "        decoder_attentions = torch.zeros(max_length, max_length)\n",
    "\n",
    "        for di in range(max_length):\n",
    "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
    "                decoder_input, decoder_hidden, encoder_outputs)\n",
    "            decoder_attentions[di] = decoder_attention.data\n",
    "            topv, topi = decoder_output.data.topk(1)\n",
    "            #print(topv.item())\n",
    "            if topi.item() == EOS_token:\n",
    "                decoded_words.append('<EOS>')\n",
    "                break\n",
    "            else:\n",
    "                decoded_words.append(en_tokenizer.id_to_token(topi.item()))\n",
    "\n",
    "            decoder_input = topi.squeeze().detach()\n",
    "\n",
    "        return decoded_words, decoder_attentions[:di + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "63fe6084",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateRandomly(encoder, decoder, n=10):\n",
    "    for i in range(n):\n",
    "        #pair = random.choice(val_pairs)\n",
    "        #pair = random.choice(pairs)\n",
    "        n = random.randint(0, len(val_input_strings))\n",
    "        print('>', val_input_strings[n])\n",
    "        print('=', val_target_strings[n])\n",
    "        output_words, attentions = evaluate(encoder, decoder, val_input_strings[n])\n",
    "        output_sentence = ' '.join(output_words)\n",
    "        print('<', output_sentence)\n",
    "        print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "78b2d78f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> она любит синие платья .\n",
      "= she likes blue dresses .\n",
      "< she likes blue dress . . <EOS>\n",
      "\n",
      "> ты не смотришь телевизор ?\n",
      "= don t you watch tv ?\n",
      "< don t you watch tv ? <EOS>\n",
      "\n",
      "> ты должен только дождаться ее возвращения .\n",
      "= you have only to wait for her return .\n",
      "< you must only to wait to to return . <EOS>\n",
      "\n",
      "> с самого утра идет снег .\n",
      "= it s been snowing since this morning .\n",
      "< it s been been snow since morning morning . <EOS>\n",
      "\n",
      "> я знаю несколько канадцев .\n",
      "= i know several canadians .\n",
      "< i know a canadians . <EOS>\n",
      "\n",
      "> взобраться на эту крутую гору физически невозможно .\n",
      "= climbing that steep mountain is a physical impossibility .\n",
      "< climb mountain climb to mountain mountain mountain ##ical ##ical . <EOS>\n",
      "\n",
      "> можно мне одолжить ваш словарь ?\n",
      "= may i borrow your dictionary ?\n",
      "< could i borrow your dictionary ? <EOS>\n",
      "\n",
      "> какие у нас шансы ?\n",
      "= what are our chances ?\n",
      "< what are our chance ##s ? <EOS>\n",
      "\n",
      "> спасибо что дали мне знать .\n",
      "= thanks for letting me know .\n",
      "< thanks for let ##ting me know . <EOS>\n",
      "\n",
      "> я попробовал но у меня ничего не вышло .\n",
      "= i tried but i didn t succeed .\n",
      "< i tried but but didn t <EOS>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluateRandomly(encoder1, attn_decoder1,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "1c2a352f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> она решила выити за него .\n",
      "= she decided to marry him .\n",
      "< she decided to go him . <EOS>\n",
      "\n",
      "> я лучше сегодня останусь дома .\n",
      "= i d rather stay at home today .\n",
      "< i d better home home today . <EOS>\n",
      "\n",
      "> было время обеда .\n",
      "= it was time for lunch .\n",
      "< it was a lunch . <EOS>\n",
      "\n",
      "> я не могу сделать это сеичас .\n",
      "= i can t do it now .\n",
      "< i can t do it now . <EOS>\n",
      "\n",
      "> проверьте все еще раз .\n",
      "= check everything one more time .\n",
      "< check everything again . <EOS>\n",
      "\n",
      "> мы оба пошли на жертвы .\n",
      "= we ve both made sacrifices .\n",
      "< we re both went the the ##s . . . <EOS>\n",
      "\n",
      "> ты себя выдашь .\n",
      "= you ll give yourself away .\n",
      "< you ll give yourself . <EOS>\n",
      "\n",
      "> оно у меня .\n",
      "= i have it .\n",
      "< it have it . <EOS>\n",
      "\n",
      "> я должен знать .\n",
      "= i ve got to know .\n",
      "< i have to know . <EOS>\n",
      "\n",
      "> уже одиннадцать часов .\n",
      "= it is already eleven .\n",
      "< it is already eleven clock . . <EOS>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluateRandomly(encoder1, attn_decoder1,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c9b7deed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def showAttention(input_sentence, output_words, attentions):\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    cax = ax.matshow(attentions.numpy(), cmap='bone')\n",
    "    fig.colorbar(cax)\n",
    "\n",
    "    # Set up axes\n",
    "    ax.set_xticklabels([''] + input_sentence.split(' ') +\n",
    "                       ['<EOS>'], rotation=90)\n",
    "    ax.set_yticklabels([''] + output_words)\n",
    "\n",
    "    # Show label at every tick\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def evaluateAndShowAttention(input_sentence):\n",
    "    input_sentence = input_sentence.lower()\n",
    "    output_words, attentions = evaluate(\n",
    "        encoder1, attn_decoder1, input_sentence)\n",
    "    print('input =', input_sentence)\n",
    "    print('output =', ' '.join(output_words))\n",
    "    showAttention(input_sentence, output_words, attentions)\n",
    "\n",
    "def translate(input_sentence):\n",
    "    input_sentence = input_sentence.lower()\n",
    "    output_words, attentions = evaluate(\n",
    "        encoder1, attn_decoder1, input_sentence)\n",
    "    print('input = ', input_sentence)\n",
    "    print('output =', ' '.join(output_words))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "3c0306c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  i am playing\n",
      "output = the in ##le the the the the <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"I am playing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f38c33b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я чувствую себя хорошо\n",
      "output = i feel fine well . <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я чувствую себя хорошо\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "999c859e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  она смотрит телевизор\n",
      "output = she is watching tv tv <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"Она смотрит телевизор\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0f3f155b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я смотрю телевизор\n",
      "output = i watch television television <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я смотрю телевизор\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "7c7e9d22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я еду на автобусе\n",
      "output = i m going to bus . <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я еду на автобусе\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "0938fbf6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я еду на машине\n",
      "output = i m on my car <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я еду на машине\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c4d258e4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  ты смотришь телевизор\n",
      "output = you watch tv tv <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"ты смотришь телевизор\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "cea91e00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я играю на гитаре\n",
      "output = i play the guitar <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"Я играю на гитаре\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "30a5ec53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  у нас нет времени ждать\n",
      "output = we don no have time to wait <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"У нас нет времени ждать\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "3760f6da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я играю на трубе\n",
      "output = i play tru ##et ##et ##et <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"Я играю на трубе\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "a3ab9331",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  куда вы идете\n",
      "output = where you going going <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"Куда вы идете\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "f8ad5dd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я не сплю\n",
      "output = i m awake . <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я не сплю\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "ae5e4ed9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я сплю\n",
      "output = i sleep sleep <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я сплю\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "46445155",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я сплю очень хорошо\n",
      "output = i sleep very well . <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я сплю очень хорошо\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "b7693628",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я сплю очень плохо\n",
      "output = i sleep very very bad . <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я сплю очень плохо\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "eb98b785",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  я ем\n",
      "output = i m eating . <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"я ем\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "7e1ba9d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input =  тойота\n",
      "output = it s <EOS>\n"
     ]
    }
   ],
   "source": [
    "translate(\"Тойота\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "ffc5cffb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "def save_tuple_to_file(filename, data):\n",
    "    with open(filename, 'wb') as f:\n",
    "        pickle.dump(data, f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "f36f9286",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_tuple_to_file(\"val_input_strings\", val_input_strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "2c946781",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_tuple_to_file(\"val_target_strings\", val_input_strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "d7dee273",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_tuple_to_file(\"train_input_strings\", train_input_strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "91b99022",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_tuple_to_file(\"train_target_strings\", train_target_strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37c0ab92",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}