neural-machine-translation/44.memory-network-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len from: 500, len to: 500\n"
     ]
    }
   ],
   "source": [
    "with open('english-train', 'r') as fopen:\n",
    "    text_from = fopen.read().lower().split('\\n')[:-1]\n",
    "with open('vietnam-train', 'r') as fopen:\n",
    "    text_to = fopen.read().lower().split('\\n')[:-1]\n",
    "print('len from: %d, len to: %d'%(len(text_from), len(text_to)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 1935\n",
      "Most common words [(',', 564), ('.', 477), ('the', 368), ('and', 286), ('to', 242), ('of', 220)]\n",
      "Sample data [482, 483, 78, 6, 137, 484, 10, 226, 787, 14] ['rachel', 'pike', ':', 'the', 'science', 'behind', 'a', 'climate', 'headline', 'in']\n"
     ]
    }
   ],
   "source": [
    "concat_from = ' '.join(text_from).split()\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab to size: 1461\n",
      "Most common words [(',', 472), ('.', 430), ('tôi', 283), ('và', 230), ('có', 199), ('chúng', 196)]\n",
      "Sample data [84, 22, 668, 73, 10, 389, 110, 34, 81, 299] ['khoa', 'học', 'đằng', 'sau', 'một', 'tiêu', 'đề', 'về', 'khí', 'hậu']\n"
     ]
    }
   ],
   "source": [
    "concat_to = ' '.join(text_to).split()\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab to size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(text_to)):\n",
    "    text_to[i] += ' EOS'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X\n",
    "\n",
    "def pad_sentence_batch(sentence_batch, pad_int, maxlen):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = maxlen\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(maxlen)\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(text_from, dictionary_from)\n",
    "Y = str_idx(text_to, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen_question = max([len(x) for x in X]) * 2\n",
    "maxlen_answer = max([len(y) for y in Y]) * 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hop_forward(memory_o, memory_i, response_proj, inputs_len, questions_len):\n",
    "    match = memory_i\n",
    "    match = pre_softmax_masking(match, inputs_len)\n",
    "    match = tf.nn.softmax(match)\n",
    "    match = post_softmax_masking(match, questions_len)\n",
    "    response = tf.multiply(match, memory_o)\n",
    "    return response_proj(response)\n",
    "\n",
    "\n",
    "def pre_softmax_masking(x, seq_len):\n",
    "    paddings = tf.fill(tf.shape(x), float('-inf'))\n",
    "    T = tf.shape(x)[1]\n",
    "    max_seq_len = tf.shape(x)[2]\n",
    "    masks = tf.sequence_mask(seq_len, max_seq_len, dtype = tf.float32)\n",
    "    masks = tf.tile(tf.expand_dims(masks, 1), [1, T, 1])\n",
    "    return tf.where(tf.equal(masks, 0), paddings, x)\n",
    "\n",
    "\n",
    "def post_softmax_masking(x, seq_len):\n",
    "    T = tf.shape(x)[2]\n",
    "    max_seq_len = tf.shape(x)[1]\n",
    "    masks = tf.sequence_mask(seq_len, max_seq_len, dtype = tf.float32)\n",
    "    masks = tf.tile(tf.expand_dims(masks, -1), [1, 1, T])\n",
    "    return x * masks\n",
    "\n",
    "\n",
    "def shift_right(x):\n",
    "    batch_size = tf.shape(x)[0]\n",
    "    start = tf.to_int32(tf.fill([batch_size, 1], GO))\n",
    "    return tf.concat([start, x[:, :-1]], 1)\n",
    "\n",
    "\n",
    "def embed_seq(x, vocab_size, zero_pad = True):\n",
    "    lookup_table = tf.get_variable(\n",
    "        'lookup_table', [vocab_size, size_layer], tf.float32\n",
    "    )\n",
    "    if zero_pad:\n",
    "        lookup_table = tf.concat(\n",
    "            (tf.zeros([1, size_layer]), lookup_table[1:, :]), axis = 0\n",
    "        )\n",
    "    return tf.nn.embedding_lookup(lookup_table, x)\n",
    "\n",
    "\n",
    "def position_encoding(sentence_size, embedding_size):\n",
    "    encoding = np.ones((embedding_size, sentence_size), dtype = np.float32)\n",
    "    ls = sentence_size + 1\n",
    "    le = embedding_size + 1\n",
    "    for i in range(1, le):\n",
    "        for j in range(1, ls):\n",
    "            encoding[i - 1, j - 1] = (i - (le - 1) / 2) * (j - (ls - 1) / 2)\n",
    "    encoding = 1 + 4 * encoding / embedding_size / sentence_size\n",
    "    return np.transpose(encoding)\n",
    "\n",
    "def quest_mem(x, vocab_size, max_quest_len):\n",
    "    x = embed_seq(x, vocab_size)\n",
    "    pos = position_encoding(max_quest_len, size_layer)\n",
    "    return x * pos\n",
    "\n",
    "class QA:\n",
    "    def __init__(self, vocab_size_from, vocab_size_to, size_layer, learning_rate, n_hops = 3):\n",
    "        self.X = tf.placeholder(tf.int32,[None,None])\n",
    "        self.Y = tf.placeholder(tf.int32,[None,None])\n",
    "        self.X_seq_len = tf.fill([tf.shape(self.X)[0]],maxlen_question)\n",
    "        self.Y_seq_len = tf.fill([tf.shape(self.X)[0]],maxlen_answer)\n",
    "        max_quest_len = maxlen_question\n",
    "        max_answer_len = maxlen_answer\n",
    "        \n",
    "        lookup_table = tf.get_variable('lookup_table', [vocab_size_from, size_layer], tf.float32)\n",
    "        \n",
    "        with tf.variable_scope('memory_o'):\n",
    "            memory_o = quest_mem(self.X, vocab_size_from, max_quest_len)\n",
    "        \n",
    "        with tf.variable_scope('memory_i'):\n",
    "            memory_i = quest_mem(self.X, vocab_size_from, max_quest_len)\n",
    "        \n",
    "        with tf.variable_scope('interaction'):\n",
    "            response_proj = tf.layers.Dense(size_layer)\n",
    "            for _ in range(n_hops):\n",
    "                answer = hop_forward(memory_o,\n",
    "                                     memory_i,\n",
    "                                     response_proj,\n",
    "                                     self.X_seq_len,\n",
    "                                     self.X_seq_len)\n",
    "                memory_i = answer\n",
    "        \n",
    "        embedding = tf.Variable(tf.random_uniform([vocab_size_to, size_layer], -1, 1))\n",
    "        cell = tf.nn.rnn_cell.LSTMCell(size_layer)\n",
    "        vocab_proj = tf.layers.Dense(vocab_size_to)\n",
    "        state_proj = tf.layers.Dense(size_layer)\n",
    "        init_state = state_proj(tf.layers.flatten(answer))\n",
    "        \n",
    "        helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "            inputs = tf.nn.embedding_lookup(embedding, shift_right(self.Y)),\n",
    "            sequence_length = tf.to_int32(self.Y_seq_len))\n",
    "        encoder_state = tf.nn.rnn_cell.LSTMStateTuple(c=init_state, h=init_state)\n",
    "        decoder = tf.contrib.seq2seq.BasicDecoder(cell = cell,\n",
    "                                                  helper = helper,\n",
    "                                                  initial_state = encoder_state,\n",
    "                                                  output_layer = vocab_proj)\n",
    "        decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder = decoder,\n",
    "                                                                maximum_iterations = max_answer_len)\n",
    "        \n",
    "        helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(embedding = embedding,\n",
    "                                                          start_tokens = tf.tile(\n",
    "                                                              tf.constant([GO], \n",
    "                                                                          dtype=tf.int32), \n",
    "                                                              [tf.shape(init_state)[0]]),\n",
    "                                                          end_token = EOS)\n",
    "        decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "            cell = cell,\n",
    "            helper = helper,\n",
    "            initial_state = encoder_state,\n",
    "            output_layer = vocab_proj)\n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            decoder = decoder,\n",
    "            maximum_iterations = max_answer_len)\n",
    "        self.training_logits = decoder_output.rnn_output\n",
    "        self.predicting_ids = predicting_decoder_output.sample_id\n",
    "        self.logits = decoder_output.sample_id\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, max_answer_len, dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 20\n",
    "batch_size = 16\n",
    "size_layer = 256\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = QA(len(dictionary_from), len(dictionary_to), size_layer, 1e-3)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 1.657692, avg accuracy: 0.883773\n",
      "epoch: 2, avg loss: 0.694845, avg accuracy: 0.914418\n",
      "epoch: 3, avg loss: 0.670171, avg accuracy: 0.919145\n",
      "epoch: 4, avg loss: 0.655215, avg accuracy: 0.920982\n",
      "epoch: 5, avg loss: 0.644285, avg accuracy: 0.922491\n",
      "epoch: 6, avg loss: 0.634069, avg accuracy: 0.923673\n",
      "epoch: 7, avg loss: 0.623932, avg accuracy: 0.923918\n",
      "epoch: 8, avg loss: 0.613115, avg accuracy: 0.924627\n",
      "epoch: 9, avg loss: 0.600960, avg accuracy: 0.925236\n",
      "epoch: 10, avg loss: 0.587711, avg accuracy: 0.925800\n",
      "epoch: 11, avg loss: 0.573947, avg accuracy: 0.926691\n",
      "epoch: 12, avg loss: 0.558712, avg accuracy: 0.928582\n",
      "epoch: 13, avg loss: 0.542557, avg accuracy: 0.930164\n",
      "epoch: 14, avg loss: 0.525535, avg accuracy: 0.932000\n",
      "epoch: 15, avg loss: 0.509237, avg accuracy: 0.933582\n",
      "epoch: 16, avg loss: 0.493271, avg accuracy: 0.935100\n",
      "epoch: 17, avg loss: 0.475748, avg accuracy: 0.937173\n",
      "epoch: 18, avg loss: 0.457230, avg accuracy: 0.938718\n",
      "epoch: 19, avg loss: 0.439197, avg accuracy: 0.940700\n",
      "epoch: 20, avg loss: 0.420498, avg accuracy: 0.942591\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    for k in range(0, len(text_to), batch_size):\n",
    "        index = min(k+batch_size, len(text_to))\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: index], PAD, maxlen_question)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index], PAD, maxlen_answer)\n",
    "        predicted, accuracy,loss, _ = sess.run([model.predicting_ids, \n",
    "                                                model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "    total_loss /= (len(text_to) / batch_size)\n",
    "    total_accuracy /= (len(text_to) / batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "QUESTION: or , if you had to choose between the last two , which one would you choose ?\n",
      "REAL ANSWER: sau khi thôi không làm đau mọi người nữa , tôi sẽ hỏi họ bạn có đau không ? đau như thế nào ?\n",
      "PREDICTED ANSWER: và tôi đã làm ra một chiếc của tôi đã cho các phép và bạn có thể làm cho những gì bạn có thể làm được . \n",
      "\n",
      "row 2\n",
      "QUESTION: i kept on doing this for a while .\n",
      "REAL ANSWER: hoặc nếu được chọn giữa 2 kiểu đau cuối , bạn sẽ chọn cái nào ?\n",
      "PREDICTED ANSWER: và tôi đã được làm việc với tôi . \n",
      "\n",
      "row 3\n",
      "QUESTION: and then , like all good academic projects , i got more funding .\n",
      "REAL ANSWER: tôi tiếp tục làm thí nghiệm này 1 thời gian\n",
      "PREDICTED ANSWER: tôi không phải là một phòng thí nghiệm này . \n",
      "\n",
      "row 4\n",
      "QUESTION: i moved to sounds , electrical shocks -- i even had a pain suit that i could get people to feel much more pain .\n",
      "REAL ANSWER: và sau đó , giống các đề tài nghiên cứu hay khác , tôi nhận thêm nguồn tài trợ .\n",
      "PREDICTED ANSWER: và tôi đã làm ra một chiếc của tôi đã nói , nhưng một nhà khoa học và tôi đã làm cho một cách mà chúng tôi đã làm việc với những người như thế này , và những gì không phải là một cách , chúng tôi có thể làm cho một cách của chúng tôi , và chúng tôi đã là một cách , chúng tôi đã làm việc với những người như thế này , và những người có thể tưởng tượng , và chúng tôi đã làm việc với những người , và chúng tôi đã nói , chúng tôi đã phải là một cách , chúng tôi đã làm việc với những người như bạn có thể không thể làm được . \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('QUESTION:',' '.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL ANSWER:',' '.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED ANSWER:',' '.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}