neural-machine-translation/39.lstm-birnn-seq2seq-luong-bahdanau-stack-beam-dropout-l2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len from: 500, len to: 500\n"
     ]
    }
   ],
   "source": [
    "with open('english-train', 'r') as fopen:\n",
    "    text_from = fopen.read().lower().split('\\n')[:-1]\n",
    "with open('vietnam-train', 'r') as fopen:\n",
    "    text_to = fopen.read().lower().split('\\n')[:-1]\n",
    "print('len from: %d, len to: %d'%(len(text_from), len(text_to)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 1935\n",
      "Most common words [(',', 564), ('.', 477), ('the', 368), ('and', 286), ('to', 242), ('of', 220)]\n",
      "Sample data [482, 483, 78, 6, 137, 484, 10, 226, 787, 14] ['rachel', 'pike', ':', 'the', 'science', 'behind', 'a', 'climate', 'headline', 'in']\n"
     ]
    }
   ],
   "source": [
    "concat_from = ' '.join(text_from).split()\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab to size: 1461\n",
      "Most common words [(',', 472), ('.', 430), ('tôi', 283), ('và', 230), ('có', 199), ('chúng', 196)]\n",
      "Sample data [84, 22, 668, 73, 10, 389, 110, 34, 81, 299] ['khoa', 'học', 'đằng', 'sau', 'một', 'tiêu', 'đề', 'về', 'khí', 'hậu']\n"
     ]
    }
   ],
   "source": [
    "concat_to = ' '.join(text_to).split()\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab to size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(text_to)):\n",
    "    text_to[i] += ' EOS'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Chatbot:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 from_dict_size, to_dict_size, batch_size,\n",
    "                 grad_clip=5.0, beam_width=5, force_teaching_ratio=0.5):\n",
    "        \n",
    "        def cells(size, reuse=False):\n",
    "            return tf.contrib.rnn.DropoutWrapper(\n",
    "                tf.nn.rnn_cell.LSTMCell(size, initializer=tf.orthogonal_initializer(),reuse=reuse),\n",
    "                input_keep_prob=0.8,\n",
    "                output_keep_prob=0.8,\n",
    "                state_keep_prob=0.8)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        decoder_embeddings = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        self.encoder_out = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        def bahdanau(size):\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units = size, \n",
    "                                                                    memory = self.encoder_out)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(cell = cells(size), \n",
    "                                                        attention_mechanism = attention_mechanism,\n",
    "                                                        attention_layer_size = size)\n",
    "        \n",
    "        def luong(size):\n",
    "            attention_mechanism = tf.contrib.seq2seq.LuongAttention(num_units = size, \n",
    "                                                                    memory = self.encoder_out)\n",
    "            return tf.contrib.seq2seq.AttentionWrapper(cell = cells(size), \n",
    "                                                        attention_mechanism = attention_mechanism,\n",
    "                                                        attention_layer_size = size)\n",
    "        \n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = bahdanau(size_layer//2),\n",
    "                cell_bw = luong(size_layer//2),\n",
    "                inputs = self.encoder_out,\n",
    "                sequence_length = self.X_seq_len,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_%d'%(n))\n",
    "            encoder_embedded = tf.concat((out_fw, out_bw), 2)\n",
    "            \n",
    "        bi_state_c = tf.concat((state_fw[0].c, state_bw[0].c), -1)\n",
    "        bi_state_h = tf.concat((state_fw[0].h, state_bw[0].h), -1)\n",
    "        bi_lstm_state = tf.nn.rnn_cell.LSTMStateTuple(c=bi_state_c, h=bi_state_h)\n",
    "        encoder_state = tuple([bi_lstm_state] * num_layers)\n",
    "        \n",
    "        dense = tf.layers.Dense(to_dict_size)\n",
    "        \n",
    "        with tf.variable_scope('decode'):\n",
    "            attention_mechanism = tf.contrib.seq2seq.LuongAttention(\n",
    "                num_units = size_layer, \n",
    "                memory = self.encoder_out,\n",
    "                memory_sequence_length = self.X_seq_len)\n",
    "            luong_cells = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = tf.nn.rnn_cell.MultiRNNCell([cells(size_layer) for _ in range(num_layers)]),\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                num_units = size_layer, \n",
    "                memory = self.encoder_out,\n",
    "                memory_sequence_length = self.X_seq_len)\n",
    "            bahdanau_cells = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = tf.nn.rnn_cell.MultiRNNCell([cells(size_layer) for _ in range(num_layers)]),\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "            decoder_cells = tf.nn.rnn_cell.MultiRNNCell([luong_cells, bahdanau_cells])\n",
    "            main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "            decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "            training_helper = tf.contrib.seq2seq.ScheduledEmbeddingTrainingHelper(\n",
    "            inputs = tf.nn.embedding_lookup(decoder_embeddings, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                embedding = decoder_embeddings,\n",
    "                sampling_probability = 1 - force_teaching_ratio,\n",
    "                time_major = False)\n",
    "            training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = training_helper,\n",
    "                initial_state = decoder_cells.zero_state(batch_size, tf.float32),\n",
    "                output_layer = tf.layers.Dense(to_dict_size))\n",
    "            training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "            self.training_logits = training_decoder_output.rnn_output\n",
    "            \n",
    "        with tf.variable_scope('decode', reuse=True):\n",
    "            encoder_out_tiled = tf.contrib.seq2seq.tile_batch(self.encoder_out, beam_width)\n",
    "            encoder_state_tiled = tf.contrib.seq2seq.tile_batch(encoder_state, beam_width)\n",
    "            X_seq_len_tiled = tf.contrib.seq2seq.tile_batch(self.X_seq_len, beam_width)\n",
    "            attention_mechanism = tf.contrib.seq2seq.LuongAttention(\n",
    "                num_units = size_layer, \n",
    "                memory = encoder_out_tiled,\n",
    "                memory_sequence_length = X_seq_len_tiled)\n",
    "            luong_cells = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = tf.nn.rnn_cell.MultiRNNCell([cells(size_layer,reuse=True) for _ in range(num_layers)]),\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "            attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "                num_units = size_layer, \n",
    "                memory = encoder_out_tiled,\n",
    "                memory_sequence_length = X_seq_len_tiled)\n",
    "            bahdanau_cells = tf.contrib.seq2seq.AttentionWrapper(\n",
    "                cell = tf.nn.rnn_cell.MultiRNNCell([cells(size_layer,reuse=True) for _ in range(num_layers)]),\n",
    "                attention_mechanism = attention_mechanism,\n",
    "                attention_layer_size = size_layer)\n",
    "            decoder_cells = tf.nn.rnn_cell.MultiRNNCell([luong_cells, bahdanau_cells])\n",
    "            predicting_decoder = tf.contrib.seq2seq.BeamSearchDecoder(\n",
    "                cell = decoder_cells,\n",
    "                embedding = decoder_embeddings,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS,\n",
    "                initial_state = decoder_cells.zero_state(batch_size * beam_width, tf.float32),\n",
    "                beam_width = beam_width,\n",
    "                output_layer = tf.layers.Dense(to_dict_size, _reuse=True),\n",
    "                length_penalty_weight = 0.0)\n",
    "            predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = False,\n",
    "                maximum_iterations = 2 * tf.reduce_max(self.X_seq_len))\n",
    "            self.predicting_ids = predicting_decoder_output.predicted_ids[:, :, 0]\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        l2 = sum(1e-5 * tf.nn.l2_loss(tf_var) for tf_var in tf.trainable_variables())\n",
    "        self.cost += l2\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 512\n",
    "num_layers = 2\n",
    "embedded_size = 256\n",
    "learning_rate = 1e-2\n",
    "batch_size = 16\n",
    "epoch = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Chatbot(size_layer, num_layers, embedded_size, len(dictionary_from), \n",
    "                len(dictionary_to), batch_size, learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(text_from, dictionary_from)\n",
    "Y = str_idx(text_to, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 9.510825, avg accuracy: 0.022027\n",
      "epoch: 2, avg loss: 7.445881, avg accuracy: 0.038613\n",
      "epoch: 3, avg loss: 6.863027, avg accuracy: 0.041087\n",
      "epoch: 4, avg loss: 6.399978, avg accuracy: 0.042979\n",
      "epoch: 5, avg loss: 6.248574, avg accuracy: 0.049162\n",
      "epoch: 6, avg loss: 6.391312, avg accuracy: 0.049706\n",
      "epoch: 7, avg loss: 6.616857, avg accuracy: 0.045438\n",
      "epoch: 8, avg loss: 6.609593, avg accuracy: 0.045675\n",
      "epoch: 9, avg loss: 6.377094, avg accuracy: 0.051762\n",
      "epoch: 10, avg loss: 6.144921, avg accuracy: 0.053860\n",
      "epoch: 11, avg loss: 5.762990, avg accuracy: 0.056989\n",
      "epoch: 12, avg loss: 5.557138, avg accuracy: 0.054019\n",
      "epoch: 13, avg loss: 5.407917, avg accuracy: 0.055601\n",
      "epoch: 14, avg loss: 5.249044, avg accuracy: 0.063694\n",
      "epoch: 15, avg loss: 5.209891, avg accuracy: 0.062668\n",
      "epoch: 16, avg loss: 5.200728, avg accuracy: 0.059336\n",
      "epoch: 17, avg loss: 5.219026, avg accuracy: 0.061041\n",
      "epoch: 18, avg loss: 5.233603, avg accuracy: 0.061750\n",
      "epoch: 19, avg loss: 5.289140, avg accuracy: 0.064320\n",
      "epoch: 20, avg loss: 5.292430, avg accuracy: 0.063309\n",
      "epoch: 21, avg loss: 5.330809, avg accuracy: 0.067276\n",
      "epoch: 22, avg loss: 5.271437, avg accuracy: 0.064013\n",
      "epoch: 23, avg loss: 5.333670, avg accuracy: 0.063765\n",
      "epoch: 24, avg loss: 5.283467, avg accuracy: 0.065662\n",
      "epoch: 25, avg loss: 5.218686, avg accuracy: 0.064164\n",
      "epoch: 26, avg loss: 5.265750, avg accuracy: 0.068734\n",
      "epoch: 27, avg loss: 5.202237, avg accuracy: 0.066134\n",
      "epoch: 28, avg loss: 5.127524, avg accuracy: 0.069533\n",
      "epoch: 29, avg loss: 5.069793, avg accuracy: 0.069069\n",
      "epoch: 30, avg loss: 5.056033, avg accuracy: 0.066305\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    for k in range(0, len(text_to), batch_size):\n",
    "        index = min(k+batch_size, len(text_to))\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index ], PAD)\n",
    "        predicted, accuracy,loss, _ = sess.run([model.predicting_ids, \n",
    "                                                model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "    total_loss /= (len(text_to) / batch_size)\n",
    "    total_accuracy /= (len(text_to) / batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "QUESTION: or , if you had to choose between the last two , which one would you choose ?\n",
      "REAL ANSWER: sau khi thôi không làm đau mọi người nữa , tôi sẽ hỏi họ bạn có đau không ? đau như thế nào ?\n",
      "PREDICTED ANSWER: bạn \n",
      "\n",
      "row 2\n",
      "QUESTION: i kept on doing this for a while .\n",
      "REAL ANSWER: hoặc nếu được chọn giữa 2 kiểu đau cuối , bạn sẽ chọn cái nào ?\n",
      "PREDICTED ANSWER: nếu ? 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 , 2 2 2 2 ? 2 2 ? ? ? ? 2 2 2 ? 2 ? 2 ? ? ? ? ? ? 2 2 ? ? ? ? ? \n",
      "\n",
      "row 3\n",
      "QUESTION: and then , like all good academic projects , i got more funding .\n",
      "REAL ANSWER: tôi tiếp tục làm thí nghiệm này 1 thời gian\n",
      "PREDICTED ANSWER: thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời thời gian gian gian gian gian gian gian gian gian gian gian gian gian tục gian gian gian gian gian gian gian gian thời gian gian gian gian \n",
      "\n",
      "row 4\n",
      "QUESTION: i moved to sounds , electrical shocks -- i even had a pain suit that i could get people to feel much more pain .\n",
      "REAL ANSWER: và sau đó , giống các đề tài nghiên cứu hay khác , tôi nhận thêm nguồn tài trợ .\n",
      "PREDICTED ANSWER: và \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('QUESTION:',' '.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL ANSWER:',' '.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED ANSWER:',' '.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}