Tokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "import os\n",
    "import pathlib\n",
    "import re\n",
    "import string\n",
    "import sys\n",
    "import tempfile\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import tensorflow_datasets as tfds\n",
    "import tensorflow_text as text\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.get_logger().setLevel('ERROR')\n",
    "pwd = pathlib.Path.cwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import yaml\n",
    "from dataset import DataLoader\n",
    "from dataset import DataPreprocessing\n",
    "from dataset import TokenizerWrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "def config_loader(filepath):\n",
    "    with open (filepath, 'r') as file_descriptor:\n",
    "        config = yaml.safe_load(file_descriptor)\n",
    "    return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = config_loader('config/en_de.yaml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang_src, lang_dest = config['language']['src'],config['language']['dest'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['europarl-v7.es-en.en',\n",
       " 'europarl-v7.es-en.es',\n",
       " 'europarl-v7.de-en.en',\n",
       " 'europarl-v7.cs-en.en',\n",
       " 'questions_easy.csv',\n",
       " 'europarl-v7.fr-en.fr',\n",
       " 'europarl-v7.fr-en.en',\n",
       " 'europarl-v7.de-en.de',\n",
       " 'europarl-v7.cs-en.cs']"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'training'\n",
    "os.listdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader = DataLoader(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded dataset at directory:\n",
      "training/europarl-v7.de-en.en\n"
     ]
    }
   ],
   "source": [
    "src, dest = data_loader.loadData(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_train = tf.convert_to_tensor(src[:50000])\n",
    "dest_train = tf.convert_to_tensor(dest[:50000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_val = tf.convert_to_tensor(src[50001:60001])\n",
    "dest_val = tf.convert_to_tensor(dest[50001:60001])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([50000])"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "src_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([10000])"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "src_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True,\n",
    "                               as_supervised=True)\n",
    "train_examples, val_examples = examples['train'], examples['validation']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51785"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1193"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(examples['validation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensorflow.python.data.ops.dataset_ops.PrefetchDataset"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensorflow.python.framework.ops.EagerTensor"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(src_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_preprocessing = DataPreprocessing(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_set = data_preprocessing.createDataFrameSrcDest(src[:50000],dest[:50000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set = data_preprocessing.createDataFrameSrcDest(src[50001:60001],dest[50001:60001])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>en</th>\n",
       "      <th>de</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Resumption of the session</td>\n",
       "      <td>Wiederaufnahme der Sitzungsperiode</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I declare resumed the session of the European ...</td>\n",
       "      <td>Ich erkläre die am Freitag, dem 17. Dezember u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Although, as you will have seen, the dreaded '...</td>\n",
       "      <td>Wie Sie feststellen konnten, ist der gefürchte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You have requested a debate on this subject in...</td>\n",
       "      <td>Im Parlament besteht der Wunsch nach einer Aus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>In the meantime, I should like to observe a mi...</td>\n",
       "      <td>Heute möchte ich Sie bitten - das ist auch der...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  en  \\\n",
       "0                          Resumption of the session   \n",
       "1  I declare resumed the session of the European ...   \n",
       "2  Although, as you will have seen, the dreaded '...   \n",
       "3  You have requested a debate on this subject in...   \n",
       "4  In the meantime, I should like to observe a mi...   \n",
       "\n",
       "                                                  de  \n",
       "0                 Wiederaufnahme der Sitzungsperiode  \n",
       "1  Ich erkläre die am Freitag, dem 17. Dezember u...  \n",
       "2  Wie Sie feststellen konnten, ist der gefürchte...  \n",
       "3  Im Parlament besteht der Wunsch nach einer Aus...  \n",
       "4  Heute möchte ich Sie bitten - das ist auch der...  "
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = train_set.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_set = test_set.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>en</th>\n",
       "      <th>de</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Furthermore, most of the amendments, which I a...</td>\n",
       "      <td>Die Mehrzahl der Änderungsanträge, denen ich n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>That is an issue which I have referred to before.</td>\n",
       "      <td>Darauf bin ich bereits früher eingegangen.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I know it is a controversial issue and I look ...</td>\n",
       "      <td>Ich weiß, daß in diesem Punkt Uneinigkeit herr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>In substance, therefore, the Commission and Pa...</td>\n",
       "      <td>Sowohl Kommission als auch Parlament haben ein...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I should like especially to thank Mrs Roth-Beh...</td>\n",
       "      <td>Vor allem möchte ich Frau Roth-Behrendt und He...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>It falls within the competence of the Presiden...</td>\n",
       "      <td>Dies fällt in den Zuständigkeitsbereich des Pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>First Part</td>\n",
       "      <td>Teil I</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>Question No 29 by (H-0452/00):</td>\n",
       "      <td>Anfrage Nr. 29 von (H-0452/00):</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>Subject: Political intervention in Georgia's p...</td>\n",
       "      <td>Betrifft: Politische Einschaltung der EU in di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>Its attempts are being hindered by the problem...</td>\n",
       "      <td>Die Abspaltung Süd-Ossetiens und Abchasiens so...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     en  \\\n",
       "0     Furthermore, most of the amendments, which I a...   \n",
       "1     That is an issue which I have referred to before.   \n",
       "2     I know it is a controversial issue and I look ...   \n",
       "3     In substance, therefore, the Commission and Pa...   \n",
       "4     I should like especially to thank Mrs Roth-Beh...   \n",
       "...                                                 ...   \n",
       "9995  It falls within the competence of the Presiden...   \n",
       "9996                                         First Part   \n",
       "9997                     Question No 29 by (H-0452/00):   \n",
       "9998  Subject: Political intervention in Georgia's p...   \n",
       "9999  Its attempts are being hindered by the problem...   \n",
       "\n",
       "                                                     de  \n",
       "0     Die Mehrzahl der Änderungsanträge, denen ich n...  \n",
       "1            Darauf bin ich bereits früher eingegangen.  \n",
       "2     Ich weiß, daß in diesem Punkt Uneinigkeit herr...  \n",
       "3     Sowohl Kommission als auch Parlament haben ein...  \n",
       "4     Vor allem möchte ich Frau Roth-Behrendt und He...  \n",
       "...                                                 ...  \n",
       "9995  Dies fällt in den Zuständigkeitsbereich des Pr...  \n",
       "9996                                             Teil I  \n",
       "9997                    Anfrage Nr. 29 von (H-0452/00):  \n",
       "9998  Betrifft: Politische Einschaltung der EU in di...  \n",
       "9999  Die Abspaltung Süd-Ossetiens und Abchasiens so...  \n",
       "\n",
       "[10000 rows x 2 columns]"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>en</th>\n",
       "      <th>de</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Resumption of the session</td>\n",
       "      <td>Wiederaufnahme der Sitzungsperiode</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I declare resumed the session of the European ...</td>\n",
       "      <td>Ich erkläre die am Freitag, dem 17. Dezember u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Although, as you will have seen, the dreaded '...</td>\n",
       "      <td>Wie Sie feststellen konnten, ist der gefürchte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You have requested a debate on this subject in...</td>\n",
       "      <td>Im Parlament besteht der Wunsch nach einer Aus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>In the meantime, I should like to observe a mi...</td>\n",
       "      <td>Heute möchte ich Sie bitten - das ist auch der...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>Contrary to the Commission' s proposal, the re...</td>\n",
       "      <td>Im Bericht werden entgegen dem Vorschlag der K...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>This in itself is acceptable, but we must bear...</td>\n",
       "      <td>Das ist an sich akzeptabel, dennoch darf nicht...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>Hopefully, better tests than the current ones ...</td>\n",
       "      <td>Bleibt zu hoffen, daß die Tests auch bald bess...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>Mr President, I am pleased to outline our posi...</td>\n",
       "      <td>Herr Präsident, ich möchte Ihnen im folgenden ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>I wish to begin by thanking you for the excell...</td>\n",
       "      <td>Zunächst darf ich Ihnen für die ausgezeichnete...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      en  \\\n",
       "0                              Resumption of the session   \n",
       "1      I declare resumed the session of the European ...   \n",
       "2      Although, as you will have seen, the dreaded '...   \n",
       "3      You have requested a debate on this subject in...   \n",
       "4      In the meantime, I should like to observe a mi...   \n",
       "...                                                  ...   \n",
       "49995  Contrary to the Commission' s proposal, the re...   \n",
       "49996  This in itself is acceptable, but we must bear...   \n",
       "49997  Hopefully, better tests than the current ones ...   \n",
       "49998  Mr President, I am pleased to outline our posi...   \n",
       "49999  I wish to begin by thanking you for the excell...   \n",
       "\n",
       "                                                      de  \n",
       "0                     Wiederaufnahme der Sitzungsperiode  \n",
       "1      Ich erkläre die am Freitag, dem 17. Dezember u...  \n",
       "2      Wie Sie feststellen konnten, ist der gefürchte...  \n",
       "3      Im Parlament besteht der Wunsch nach einer Aus...  \n",
       "4      Heute möchte ich Sie bitten - das ist auch der...  \n",
       "...                                                  ...  \n",
       "49995  Im Bericht werden entgegen dem Vorschlag der K...  \n",
       "49996  Das ist an sich akzeptabel, dennoch darf nicht...  \n",
       "49997  Bleibt zu hoffen, daß die Tests auch bald bess...  \n",
       "49998  Herr Präsident, ich möchte Ihnen im folgenden ...  \n",
       "49999  Zunächst darf ich Ihnen für die ausgezeichnete...  \n",
       "\n",
       "[50000 rows x 2 columns]"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                Resumption of the session\n",
       "1        I declare resumed the session of the European ...\n",
       "2        Although, as you will have seen, the dreaded '...\n",
       "3        You have requested a debate on this subject in...\n",
       "4        In the meantime, I should like to observe a mi...\n",
       "                               ...                        \n",
       "49995    Contrary to the Commission' s proposal, the re...\n",
       "49996    This in itself is acceptable, but we must bear...\n",
       "49997    Hopefully, better tests than the current ones ...\n",
       "49998    Mr President, I am pleased to outline our posi...\n",
       "49999    I wish to begin by thanking you for the excell...\n",
       "Name: en, Length: 50000, dtype: object"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_set['en']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       Die Mehrzahl der Änderungsanträge, denen ich n...\n",
       "1              Darauf bin ich bereits früher eingegangen.\n",
       "2       Ich weiß, daß in diesem Punkt Uneinigkeit herr...\n",
       "3       Sowohl Kommission als auch Parlament haben ein...\n",
       "4       Vor allem möchte ich Frau Roth-Behrendt und He...\n",
       "                              ...                        \n",
       "9995    Dies fällt in den Zuständigkeitsbereich des Pr...\n",
       "9996                                               Teil I\n",
       "9997                      Anfrage Nr. 29 von (H-0452/00):\n",
       "9998    Betrifft: Politische Einschaltung der EU in di...\n",
       "9999    Die Abspaltung Süd-Ossetiens und Abchasiens so...\n",
       "Name: de, Length: 10000, dtype: object"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set['de']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_set = tf.data.Dataset.from_tensor_slices(train_set['de'])\n",
    "dest_set = tf.data.Dataset.from_tensor_slices(train_set['en'])\n",
    "training_set = tf.data.Dataset.zip((src_set,dest_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_set = tf.data.Dataset.from_tensor_slices(test_set['de'])\n",
    "dest_set = tf.data.Dataset.from_tensor_slices(test_set['en'])\n",
    "testing_set = tf.data.Dataset.zip((src_set,dest_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English:  Resumption of the session\n",
      "\n",
      "German:    Wiederaufnahme der Sitzungsperiode\n",
      "\n",
      "English:  I declare resumed the session of the European Parliament adjourned on Friday 17 December 1999, and I would like once again to wish you a happy new year in the hope that you enjoyed a pleasant festive period.\n",
      "\n",
      "German:    Ich erkläre die am Freitag, dem 17. Dezember unterbrochene Sitzungsperiode des Europäischen Parlaments für wiederaufgenommen, wünsche Ihnen nochmals alles Gute zum Jahreswechsel und hoffe, daß Sie schöne Ferien hatten.\n",
      "\n",
      "English:  Although, as you will have seen, the dreaded 'millennium bug' failed to materialise, still the people in a number of countries suffered a series of natural disasters that truly were dreadful.\n",
      "\n",
      "German:    Wie Sie feststellen konnten, ist der gefürchtete \"Millenium-Bug \" nicht eingetreten. Doch sind Bürger einiger unserer Mitgliedstaaten Opfer von schrecklichen Naturkatastrophen geworden.\n",
      "\n",
      "English:  You have requested a debate on this subject in the course of the next few days, during this part-session.\n",
      "\n",
      "German:    Im Parlament besteht der Wunsch nach einer Aussprache im Verlauf dieser Sitzungsperiode in den nächsten Tagen.\n",
      "\n",
      "English:  In the meantime, I should like to observe a minute' s silence, as a number of Members have requested, on behalf of all the victims concerned, particularly those of the terrible storms, in the various countries of the European Union.\n",
      "\n",
      "German:    Heute möchte ich Sie bitten - das ist auch der Wunsch einiger Kolleginnen und Kollegen -, allen Opfern der Stürme, insbesondere in den verschiedenen Ländern der Europäischen Union, in einer Schweigeminute zu gedenken.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for en, de in train_examples.take(5):\n",
    "    print(\"English: \", en.numpy().decode('utf-8'))\n",
    "    print()\n",
    "    print(\"German:   \", de.numpy().decode('utf-8'))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensorflow.python.data.ops.dataset_ops.TensorSliceDataset"
      ]
     },
     "execution_count": 197,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_en = training_set.map(lambda de, en: en)\n",
    "train_de = testing_set.map(lambda de, en: de)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_text.tools.wordpiece_vocab import bert_vocab_from_dataset as bert_vocab\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_tokenizer_params=dict(lower_case=True)\n",
    "reserved_tokens=[\"[PAD]\", \"[UNK]\", \"[START]\", \"[END]\"]\n",
    "\n",
    "bert_vocab_args = dict(\n",
    "    # The target vocabulary size\n",
    "    vocab_size = 8000,\n",
    "    # Reserved tokens that must be included in the vocabulary\n",
    "    reserved_tokens=reserved_tokens,\n",
    "    # Arguments for `text.BertTokenizer`\n",
    "    bert_tokenizer_params=bert_tokenizer_params,\n",
    "    # Arguments for `wordpiece_vocab.wordpiece_tokenizer_learner_lib.learn`\n",
    "    learn_params={},\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 3s, sys: 518 ms, total: 2min 3s\n",
      "Wall time: 2min 3s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "de_vocab = bert_vocab.bert_vocab_from_dataset(\n",
    "    train_de.batch(1000).prefetch(2),\n",
    "    **bert_vocab_args\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[PAD]', '[UNK]', '[START]', '[END]', '!', '\"', '%', \"'\", '(', ')']\n",
      "['dieser', '##r', 'europaischen', '##t', 'bei', 'so', 'mochte', 'aber', 'wenn', 'bericht']\n",
      "['somit', 'sorge', 'tat', 'usa', 'vier', 'wettbewerbsfahigkeit', 'wirtschaftliche', '##ausschuss', '##eren', '##fen']\n",
      "['##-', '##.', '##/', '##:', '##;', '##?', '##[', '##]', '##j', '##q']\n"
     ]
    }
   ],
   "source": [
    "print(de_vocab[:10])\n",
    "print(de_vocab[100:110])\n",
    "print(de_vocab[1000:1010])\n",
    "print(de_vocab[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_vocab_file(filepath, vocab):\n",
    "    with open(filepath, 'w') as f:\n",
    "        for token in vocab:\n",
    "              print(token, file=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_vocab_file('de_vocab.txt', de_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 7s, sys: 1.4 s, total: 1min 9s\n",
      "Wall time: 1min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "en_vocab = bert_vocab.bert_vocab_from_dataset(\n",
    "    train_en.batch(1000).prefetch(2),\n",
    "    **bert_vocab_args\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['[PAD]', '[UNK]', '[START]', '[END]', '!', '\"', '$', '%', '&', \"'\"]\n",
      "['president', 'there', 'you', 'parliament', 'been', 'union', 'these', 'can', 'or', 'if']\n",
      "['date', 'fishing', 'milk', 'possibility', 'scientific', 'setting', '##ment', 'applies', 'b5', 'reached']\n",
      "['##[', '##]', '##j', '##q', '##£', '##·', '##æ', '##ø', '##μ', '##⁄']\n"
     ]
    }
   ],
   "source": [
    "print(en_vocab[:10])\n",
    "print(en_vocab[100:110])\n",
    "print(en_vocab[1000:1010])\n",
    "print(en_vocab[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_vocab_file('en_vocab.txt', en_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {},
   "outputs": [],
   "source": [
    "de_tokenizer = text.BertTokenizer('de_vocab.txt', **bert_tokenizer_params)\n",
    "en_tokenizer = text.BertTokenizer('en_vocab.txt', **bert_tokenizer_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'Resumption of the session'\n",
      "b'I declare resumed the session of the European Parliament adjourned on Friday 17 December 1999, and I would like once again to wish you a happy new year in the hope that you enjoyed a pleasant festive period.'\n",
      "b\"Although, as you will have seen, the dreaded 'millennium bug' failed to materialise, still the people in a number of countries suffered a series of natural disasters that truly were dreadful.\"\n"
     ]
    }
   ],
   "source": [
    "for pt_examples, en_examples in training_set.batch(3).take(1):\n",
    "      for ex in en_examples:\n",
    "        print(ex.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5212, 65, 64, 1010]\n",
      "[40, 2724, 2744, 64, 1010, 65, 64, 85, 103, 4190, 74, 1704, 2105, 1241, 285, 13, 67, 40, 89, 113, 381, 301, 66, 412, 102, 32, 1413, 136, 222, 68, 64, 262, 70, 102, 4604, 32, 47, 1727, 6021, 37, 1394, 1100, 532, 15]\n",
      "[433, 13, 79, 102, 83, 81, 691, 13, 64, 35, 1489, 5730, 9, 3317, 33, 6849, 9, 1772, 66, 1890, 1261, 13, 252, 64, 139, 68, 32, 273, 65, 131, 2300, 32, 1625, 65, 1036, 1547, 70, 1712, 175, 5131, 15]\n"
     ]
    }
   ],
   "source": [
    "# Tokenize the examples -> (batch, word, word-piece)\n",
    "token_batch = en_tokenizer.tokenize(en_examples)\n",
    "# Merge the word and word-piece axes -> (batch, tokens)\n",
    "token_batch = token_batch.merge_dims(-2,-1)\n",
    "\n",
    "for ex in token_batch.to_list():\n",
    "      print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
       "array([b'resumption of the session',\n",
       "       b'i declare resumed the session of the european parliament adjourned on friday 17 december 1999 , and i would like once again to wish you a happy new year in the hope that you enjoyed a p ##le ##asant f ##est ##ive period .',\n",
       "       b\"although , as you will have seen , the d ##re ##aded ' millennium b ##ug ' failed to material ##ise , still the people in a number of countries suffered a series of natural disasters that truly were dreadful .\"],\n",
       "      dtype=object)>"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lookup each token id in the vocabulary.\n",
    "txt_tokens = tf.gather(en_vocab, token_batch)\n",
    "# Join with spaces.\n",
    "tf.strings.reduce_join(txt_tokens, separator=' ', axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
       "array([b'resumption of the session',\n",
       "       b'i declare resumed the session of the european parliament adjourned on friday 17 december 1999 , and i would like once again to wish you a happy new year in the hope that you enjoyed a pleasant festive period .',\n",
       "       b\"although , as you will have seen , the dreaded ' millennium bug ' failed to materialise , still the people in a number of countries suffered a series of natural disasters that truly were dreadful .\"],\n",
       "      dtype=object)>"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = en_tokenizer.detokenize(token_batch)\n",
    "tf.strings.reduce_join(words, separator=' ', axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [],
   "source": [
    "START = tf.argmax(tf.constant(reserved_tokens) == \"[START]\")\n",
    "END = tf.argmax(tf.constant(reserved_tokens) == \"[END]\")\n",
    "\n",
    "def add_start_end(ragged):\n",
    "  count = ragged.bounding_shape()[0]\n",
    "  starts = tf.fill([count,1], START)\n",
    "  ends = tf.fill([count,1], END)\n",
    "  return tf.concat([starts, ragged, ends], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=string, numpy=\n",
       "array([b'[START] resumption of the session [END]',\n",
       "       b'[START] i declare resumed the session of the european parliament adjourned on friday 17 december 1999 , and i would like once again to wish you a happy new year in the hope that you enjoyed a pleasant festive period . [END]',\n",
       "       b\"[START] although , as you will have seen , the dreaded ' millennium bug ' failed to materialise , still the people in a number of countries suffered a series of natural disasters that truly were dreadful . [END]\"],\n",
       "      dtype=object)>"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = en_tokenizer.detokenize(add_start_end(token_batch))\n",
    "tf.strings.reduce_join(words, separator=' ', axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleanup_text(reserved_tokens, token_txt):\n",
    "  # Drop the reserved tokens, except for \"[UNK]\".\n",
    "  bad_tokens = [re.escape(tok) for tok in reserved_tokens if tok != \"[UNK]\"]\n",
    "  bad_token_re = \"|\".join(bad_tokens)\n",
    "\n",
    "  bad_cells = tf.strings.regex_full_match(token_txt, bad_token_re)\n",
    "  result = tf.ragged.boolean_mask(token_txt, ~bad_cells)\n",
    "\n",
    "  # Join them into strings.\n",
    "  result = tf.strings.reduce_join(result, separator=' ', axis=-1)\n",
    "\n",
    "  return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([b'Resumption of the session',\n",
       "       b'I declare resumed the session of the European Parliament adjourned on Friday 17 December 1999, and I would like once again to wish you a happy new year in the hope that you enjoyed a pleasant festive period.',\n",
       "       b\"Although, as you will have seen, the dreaded 'millennium bug' failed to materialise, still the people in a number of countries suffered a series of natural disasters that truly were dreadful.\"],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 309,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_examples.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[b'resumption', b'of', b'the', b'session'],\n",
       " [b'i', b'declare', b'resumed', b'the', b'session', b'of', b'the',\n",
       "  b'european', b'parliament', b'adjourned', b'on', b'friday', b'17',\n",
       "  b'december', b'1999', b',', b'and', b'i', b'would', b'like', b'once',\n",
       "  b'again', b'to', b'wish', b'you', b'a', b'happy', b'new', b'year', b'in',\n",
       "  b'the', b'hope', b'that', b'you', b'enjoyed', b'a', b'pleasant',\n",
       "  b'festive', b'period', b'.']                                             ,\n",
       " [b'although', b',', b'as', b'you', b'will', b'have', b'seen', b',', b'the',\n",
       "  b'dreaded', b\"'\", b'millennium', b'bug', b\"'\", b'failed', b'to',\n",
       "  b'materialise', b',', b'still', b'the', b'people', b'in', b'a', b'number',\n",
       "  b'of', b'countries', b'suffered', b'a', b'series', b'of', b'natural',\n",
       "  b'disasters', b'that', b'truly', b'were', b'dreadful', b'.']              ]>"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_batch = en_tokenizer.tokenize(en_examples).merge_dims(-2,-1)\n",
    "words = en_tokenizer.detokenize(token_batch)\n",
    "words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([b'resumption of the session',\n",
       "       b'i declare resumed the session of the european parliament adjourned on friday 17 december 1999 , and i would like once again to wish you a happy new year in the hope that you enjoyed a pleasant festive period .',\n",
       "       b\"although , as you will have seen , the dreaded ' millennium bug ' failed to materialise , still the people in a number of countries suffered a series of natural disasters that truly were dreadful .\"],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 311,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleanup_text(reserved_tokens, words).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomTokenizer(tf.Module):\n",
    "  def __init__(self, reserved_tokens, vocab_path):\n",
    "    self.tokenizer = text.BertTokenizer(vocab_path, lower_case=True)\n",
    "    self._reserved_tokens = reserved_tokens\n",
    "    self._vocab_path = tf.saved_model.Asset(vocab_path)\n",
    "\n",
    "    vocab = pathlib.Path(vocab_path).read_text().splitlines()\n",
    "    self.vocab = tf.Variable(vocab)\n",
    "\n",
    "    ## Create the signatures for export:   \n",
    "\n",
    "    # Include a tokenize signature for a batch of strings. \n",
    "    self.tokenize.get_concrete_function(\n",
    "        tf.TensorSpec(shape=[None], dtype=tf.string))\n",
    "\n",
    "    # Include `detokenize` and `lookup` signatures for:\n",
    "    #   * `Tensors` with shapes [tokens] and [batch, tokens]\n",
    "    #   * `RaggedTensors` with shape [batch, tokens]\n",
    "    self.detokenize.get_concrete_function(\n",
    "        tf.TensorSpec(shape=[None, None], dtype=tf.int64))\n",
    "    self.detokenize.get_concrete_function(\n",
    "          tf.RaggedTensorSpec(shape=[None, None], dtype=tf.int64))\n",
    "\n",
    "    self.lookup.get_concrete_function(\n",
    "        tf.TensorSpec(shape=[None, None], dtype=tf.int64))\n",
    "    self.lookup.get_concrete_function(\n",
    "          tf.RaggedTensorSpec(shape=[None, None], dtype=tf.int64))\n",
    "\n",
    "    # These `get_*` methods take no arguments\n",
    "    self.get_vocab_size.get_concrete_function()\n",
    "    self.get_vocab_path.get_concrete_function()\n",
    "    self.get_reserved_tokens.get_concrete_function()\n",
    "\n",
    "  @tf.function\n",
    "  def tokenize(self, strings):\n",
    "    enc = self.tokenizer.tokenize(strings)\n",
    "    # Merge the `word` and `word-piece` axes.\n",
    "    enc = enc.merge_dims(-2,-1)\n",
    "    enc = add_start_end(enc)\n",
    "    return enc\n",
    "\n",
    "  @tf.function\n",
    "  def detokenize(self, tokenized):\n",
    "    words = self.tokenizer.detokenize(tokenized)\n",
    "    return cleanup_text(self._reserved_tokens, words)\n",
    "\n",
    "  @tf.function\n",
    "  def lookup(self, token_ids):\n",
    "    return tf.gather(self.vocab, token_ids)\n",
    "\n",
    "  @tf.function\n",
    "  def get_vocab_size(self):\n",
    "    return tf.shape(self.vocab)[0]\n",
    "\n",
    "  @tf.function\n",
    "  def get_vocab_path(self):\n",
    "    return self._vocab_path\n",
    "\n",
    "  @tf.function\n",
    "  def get_reserved_tokens(self):\n",
    "    return tf.constant(self._reserved_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizers = tf.Module()\n",
    "tokenizers.pt = CustomTokenizer(reserved_tokens, 'de_vocab.txt')\n",
    "tokenizers.en = CustomTokenizer(reserved_tokens, 'en_vocab.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'translator_en_de_converter'\n",
    "tf.saved_model.save(tokenizers, model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7178"
      ]
     },
     "execution_count": 316,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reloaded_tokenizers = tf.saved_model.load(model_name)\n",
    "reloaded_tokenizers.en.get_vocab_size().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   2,   72,   69,   32, 2425,    3]])"
      ]
     },
     "execution_count": 320,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = reloaded_tokenizers.en.tokenize(['This is a test'])\n",
    "tokens.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.RaggedTensor [[b'[START]', b'this', b'is', b'a', b'test', b'[END]']]>"
      ]
     },
     "execution_count": 321,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_tokens = reloaded_tokenizers.en.lookup(tokens)\n",
    "text_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is a test\n"
     ]
    }
   ],
   "source": [
    "round_trip = reloaded_tokenizers.en.detokenize(tokens)\n",
    "\n",
    "print(round_trip.numpy()[0].decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  adding: translator_en_de_converter/ (stored 0%)\r\n",
      "  adding: translator_en_de_converter/saved_model.pb (deflated 91%)\r\n",
      "  adding: translator_en_de_converter/assets/ (stored 0%)\r\n",
      "  adding: translator_en_de_converter/assets/en_vocab.txt (deflated 56%)\r\n",
      "  adding: translator_en_de_converter/assets/de_vocab.txt (deflated 59%)\r\n",
      "  adding: translator_en_de_converter/variables/ (stored 0%)\r\n",
      "  adding: translator_en_de_converter/variables/variables.index (deflated 33%)\r\n",
      "  adding: translator_en_de_converter/variables/variables.data-00000-of-00001 (deflated 52%)\r\n"
     ]
    }
   ],
   "source": [
    "!zip -r 'translator_en_de_converter'.zip 'translator_en_de_converter'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "184K\tted_hrlr_translate_pt_en_converter.zip\r\n",
      "148K\ttranslator_en_de_converter.zip\r\n"
     ]
    }
   ],
   "source": [
    "!du -h *.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading package lists... Done\n",
      "Building dependency tree       \n",
      "Reading state information... Done\n",
      "The following NEW packages will be installed:\n",
      "  zip\n",
      "0 upgraded, 1 newly installed, 0 to remove and 5 not upgraded.\n",
      "Need to get 167 kB of archives.\n",
      "After this operation, 638 kB of additional disk space will be used.\n",
      "Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 zip amd64 3.0-11build1 [167 kB]\n",
      "Fetched 167 kB in 1s (334 kB/s)\n",
      "debconf: delaying package configuration, since apt-utils is not installed\n",
      "Selecting previously unselected package zip.\n",
      "(Reading database ... 20632 files and directories currently installed.)\n",
      "Preparing to unpack .../zip_3.0-11build1_amd64.deb ...\n",
      "Unpacking zip (3.0-11build1) ...\n",
      "Setting up zip (3.0-11build1) ...\n"
     ]
    }
   ],
   "source": [
    "!apt-get install zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}