neural-machine-translation/48.conv-encoder-conv-decoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len from: 500, len to: 500\n"
     ]
    }
   ],
   "source": [
    "with open('english-train', 'r') as fopen:\n",
    "    text_from = fopen.read().lower().split('\\n')[:-1]\n",
    "with open('vietnam-train', 'r') as fopen:\n",
    "    text_to = fopen.read().lower().split('\\n')[:-1]\n",
    "print('len from: %d, len to: %d'%(len(text_from), len(text_to)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 1935\n",
      "Most common words [(',', 564), ('.', 477), ('the', 368), ('and', 286), ('to', 242), ('of', 220)]\n",
      "Sample data [482, 483, 78, 6, 137, 484, 10, 226, 787, 14] ['rachel', 'pike', ':', 'the', 'science', 'behind', 'a', 'climate', 'headline', 'in']\n"
     ]
    }
   ],
   "source": [
    "concat_from = ' '.join(text_from).split()\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab to size: 1461\n",
      "Most common words [(',', 472), ('.', 430), ('tôi', 283), ('và', 230), ('có', 199), ('chúng', 196)]\n",
      "Sample data [84, 22, 668, 73, 10, 389, 110, 34, 81, 299] ['khoa', 'học', 'đằng', 'sau', 'một', 'tiêu', 'đề', 'về', 'khí', 'hậu']\n"
     ]
    }
   ],
   "source": [
    "concat_to = ' '.join(text_to).split()\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab to size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(text_to)):\n",
    "    text_to[i] += ' EOS'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X\n",
    "\n",
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(text_from, dictionary_from)\n",
    "Y = str_idx(text_to, dictionary_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_size = 256\n",
    "n_hidden = 256\n",
    "n_layers = 4\n",
    "n_attn_heads = 16\n",
    "learning_rate = 1e-4\n",
    "batch_size = 16\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encoder_block(inp, n_hidden, filter_size):\n",
    "    inp = tf.expand_dims(inp, 2)\n",
    "    inp = tf.pad(inp, [[0, 0], [(filter_size[0]-1)//2, (filter_size[0]-1)//2], [0, 0], [0, 0]])\n",
    "    conv = tf.layers.conv2d(inp, n_hidden, filter_size, padding=\"VALID\", activation=None)\n",
    "    conv = tf.squeeze(conv, 2)\n",
    "    return conv\n",
    "\n",
    "def decoder_block(inp, n_hidden, filter_size):\n",
    "    inp = tf.expand_dims(inp, 2)\n",
    "    inp = tf.pad(inp, [[0, 0], [filter_size[0]-1, 0], [0, 0], [0, 0]])\n",
    "    conv = tf.layers.conv2d(inp, n_hidden, filter_size, padding=\"VALID\", activation=None)\n",
    "    conv = tf.squeeze(conv, 2)\n",
    "    return conv\n",
    "\n",
    "def glu(x):\n",
    "    return tf.multiply(x[:, :, :tf.shape(x)[2]//2], tf.sigmoid(x[:, :, tf.shape(x)[2]//2:]))\n",
    "\n",
    "def layer(inp, conv_block, kernel_width, n_hidden, residual=None):\n",
    "    z = conv_block(inp, n_hidden, (kernel_width, 1))\n",
    "    return glu(z) + (residual if residual is not None else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Chatbot:\n",
    "    def __init__(self):\n",
    "\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype = tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype = tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([len(dictionary_from), emb_size], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([len(dictionary_to), emb_size], -1, 1))\n",
    "        \n",
    "        def forward(x, y,reuse=False):\n",
    "            with tf.variable_scope('forward',reuse=reuse):\n",
    "                encoder_embedded = tf.nn.embedding_lookup(encoder_embedding, x)\n",
    "                decoder_embedded = tf.nn.embedding_lookup(decoder_embedding, y)\n",
    "\n",
    "                e = tf.identity(encoder_embedded)\n",
    "\n",
    "                for i in range(n_layers):\n",
    "                    z = layer(encoder_embedded, encoder_block, 3, n_hidden * 2, encoder_embedded)\n",
    "                    encoder_embedded = z\n",
    "\n",
    "                encoder_output, output_memory = z, z + e\n",
    "                g = tf.identity(decoder_embedded)\n",
    "\n",
    "                for i in range(n_layers):\n",
    "                    attn_res = h = layer(decoder_embedded, decoder_block, 3, n_hidden * 2, \n",
    "                                         residual=tf.zeros_like(decoder_embedded))\n",
    "                    C = []\n",
    "                    for j in range(n_attn_heads):\n",
    "                        h_ = tf.layers.dense(h, n_hidden//n_attn_heads)\n",
    "                        g_ = tf.layers.dense(g, n_hidden//n_attn_heads)\n",
    "                        zu_ = tf.layers.dense(encoder_output, n_hidden//n_attn_heads)\n",
    "                        ze_ = tf.layers.dense(output_memory, n_hidden//n_attn_heads)\n",
    "\n",
    "                        d = tf.layers.dense(h_, n_hidden//n_attn_heads) + g_\n",
    "                        dz = tf.matmul(d, tf.transpose(zu_, [0, 2, 1]))\n",
    "                        a = tf.nn.softmax(dz)\n",
    "                        c_ = tf.matmul(a, ze_)\n",
    "                        C.append(c_)\n",
    "\n",
    "                    c = tf.concat(C, 2)\n",
    "                    h = tf.layers.dense(attn_res + c, n_hidden)\n",
    "                    decoder_embedded = h\n",
    "\n",
    "                decoder_output = tf.sigmoid(h)\n",
    "                return tf.layers.dense(decoder_output, len(dictionary_to))\n",
    "        self.training_logits = forward(self.X, decoder_input)\n",
    "        self.logits = forward(self.X, self.Y, reuse=True)\n",
    "        self.k = tf.placeholder(dtype = tf.int32)\n",
    "        p = tf.nn.softmax(self.logits)\n",
    "        self.topk_logprobs, self.topk_ids = tf.nn.top_k(tf.log(p), self.k)\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Chatbot()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 7.362195, avg accuracy: 0.002599\n",
      "epoch: 2, avg loss: 6.876383, avg accuracy: 0.035388\n",
      "epoch: 3, avg loss: 6.452302, avg accuracy: 0.040414\n",
      "epoch: 4, avg loss: 6.195236, avg accuracy: 0.050942\n",
      "epoch: 5, avg loss: 6.053087, avg accuracy: 0.079203\n",
      "epoch: 6, avg loss: 5.970512, avg accuracy: 0.090725\n",
      "epoch: 7, avg loss: 5.907717, avg accuracy: 0.099582\n",
      "epoch: 8, avg loss: 5.851829, avg accuracy: 0.107494\n",
      "epoch: 9, avg loss: 5.796880, avg accuracy: 0.113903\n",
      "epoch: 10, avg loss: 5.744820, avg accuracy: 0.124458\n",
      "epoch: 11, avg loss: 5.684102, avg accuracy: 0.128438\n",
      "epoch: 12, avg loss: 5.622067, avg accuracy: 0.133466\n",
      "epoch: 13, avg loss: 5.562336, avg accuracy: 0.139069\n",
      "epoch: 14, avg loss: 5.497786, avg accuracy: 0.142818\n",
      "epoch: 15, avg loss: 5.434413, avg accuracy: 0.149519\n",
      "epoch: 16, avg loss: 5.365533, avg accuracy: 0.154012\n",
      "epoch: 17, avg loss: 5.300494, avg accuracy: 0.159030\n",
      "epoch: 18, avg loss: 5.232462, avg accuracy: 0.165978\n",
      "epoch: 19, avg loss: 5.165458, avg accuracy: 0.172521\n",
      "epoch: 20, avg loss: 5.096660, avg accuracy: 0.177135\n"
     ]
    }
   ],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    for k in range(0, len(text_to), batch_size):\n",
    "        index = min(k+batch_size, len(text_to))\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index ], PAD)\n",
    "        accuracy,loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "    total_loss /= (len(text_to) / batch_size)\n",
    "    total_accuracy /= (len(text_to) / batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Hypothesis:\n",
    "    def __init__(self, log_prob, seq):\n",
    "        self.log_prob = log_prob\n",
    "        self.seq = seq\n",
    "\n",
    "    @property\n",
    "    def step(self):\n",
    "        return len(self.seq) - 1\n",
    "\n",
    "\n",
    "def beam_search(\n",
    "    batch_x,\n",
    "    beam_size,\n",
    "    num_ans = 5,\n",
    "    normalize_by_len = 1.0,\n",
    "):\n",
    "    assert 0 <= normalize_by_len <= 1\n",
    "    batch_size = len(batch_x)\n",
    "    max_len = len(batch_x[0]) * 2\n",
    "    dec_inputs = np.ones((batch_size, 2), dtype=np.int32)\n",
    "    answers = [[] for i in range(batch_size)]\n",
    "    H = [[] for i in range(batch_size)]\n",
    "    \n",
    "    tkl, tkid = sess.run([model.topk_logprobs, \n",
    "                          model.topk_ids],\n",
    "                         feed_dict = {model.X: batch_x,\n",
    "                                     model.Y: dec_inputs,\n",
    "                                     model.k: beam_size})\n",
    "    for i in range(batch_size):\n",
    "        for j, log_prob in enumerate(tkl[i, 0]):\n",
    "            if tkid[i, 0, j] != EOS:\n",
    "                h = Hypothesis(log_prob, [1, tkid[i, 0, j]])\n",
    "                H[i].append(h)\n",
    "        H[i].sort(key=lambda h: h.log_prob)\n",
    "    \n",
    "    done = [False] * batch_size\n",
    "    while not all(done):\n",
    "        tkl_beam = []\n",
    "        tkid_beam = []\n",
    "        dec_inputs_beam = []\n",
    "        steps_beam = []\n",
    "        for i in range(beam_size):\n",
    "            steps = [1] * batch_size\n",
    "            prev_log_probs = np.zeros(batch_size, dtype=np.float32)\n",
    "            dec_inputs = np.ones((batch_size, max_len), dtype=np.int32)\n",
    "            for j, h in enumerate(H):\n",
    "                while h:\n",
    "                    hi = h.pop()\n",
    "                    lp, step, candidate_seq = hi.log_prob, hi.step, hi.seq\n",
    "                    if candidate_seq[-1] != EOS:\n",
    "                        dec_inputs[j, :len(candidate_seq)] = candidate_seq\n",
    "                        steps[j] = step\n",
    "                        prev_log_probs[j] = lp\n",
    "                        break\n",
    "                    else:\n",
    "                        answers[j].append((lp, candidate_seq))\n",
    "            max_step = max(steps)\n",
    "            dec_inputs = dec_inputs[:, :max_step + 2]\n",
    "            tkl, tkid = sess.run([model.topk_logprobs, \n",
    "                          model.topk_ids],\n",
    "                         feed_dict = {model.X: batch_x,\n",
    "                                     model.Y: dec_inputs,\n",
    "                                     model.k: beam_size})\n",
    "            \n",
    "            tkl_beam.append(tkl + prev_log_probs[:, None, None])\n",
    "            tkid_beam.append(tkid)\n",
    "            dec_inputs_beam.append(dec_inputs.copy())\n",
    "            steps_beam.append(steps)\n",
    "            \n",
    "        for i in range(beam_size):\n",
    "            tkl = tkl_beam[i]\n",
    "            tkid = tkid_beam[i]\n",
    "            dec_inputs = dec_inputs_beam[i]\n",
    "            steps = steps_beam[i]\n",
    "            for j in range(batch_size):\n",
    "                step = steps[j]\n",
    "                for k in range(tkid.shape[2]):\n",
    "                    extended_seq = np.hstack((dec_inputs[j, :step+1], [tkid[j, step, k]]))\n",
    "                    log_prob = tkl[j, step, k]\n",
    "                    if len(extended_seq) <= max_len and log_prob > -10:\n",
    "                        h = Hypothesis(log_prob, extended_seq)\n",
    "                        H[j].append(h)\n",
    "                H[j].sort(key=lambda h: h.log_prob / (h.step**normalize_by_len))\n",
    "            \n",
    "        for i in range(batch_size):\n",
    "            done[i] = (len(answers[i]) >= num_ans) or (not H[i]) or (len(H[i]) > 100)\n",
    "            \n",
    "    return answers    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "beamed = beam_search(batch_x, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "beamed = [i for i in beamed if len(i)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted = [max(b, key = lambda t: t[0])[1] for b in beamed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row 1\n",
      "QUESTION: or , if you had to choose between the last two , which one would you choose ?\n",
      "REAL ANSWER: sau khi thôi không làm đau mọi người nữa , tôi sẽ hỏi họ bạn có đau không ? đau như thế nào ?\n",
      "PREDICTED ANSWER: tôi . \n",
      "\n",
      "row 2\n",
      "QUESTION: i kept on doing this for a while .\n",
      "REAL ANSWER: hoặc nếu được chọn giữa 2 kiểu đau cuối , bạn sẽ chọn cái nào ?\n",
      "PREDICTED ANSWER: tôi \n",
      "\n",
      "row 3\n",
      "QUESTION: and then , like all good academic projects , i got more funding .\n",
      "REAL ANSWER: tôi tiếp tục làm thí nghiệm này 1 thời gian\n",
      "PREDICTED ANSWER: tôi \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(predicted)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('QUESTION:',' '.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL ANSWER:',' '.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED ANSWER:',' '.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}