names-vae-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "names = set()\n",
    "for filename in ['male.txt', 'female.txt']:\n",
    "    for line in open(os.path.join('../data/names', filename)):\n",
    "        if len(line.strip()):\n",
    "            names.add(line.strip().lower())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7583 names\n",
      "longest: 54\n",
      "99th percentile longest: 9\n"
     ]
    }
   ],
   "source": [
    "print len(names), 'names'\n",
    "print 'longest:', max(map(len, names))\n",
    "by_len = sorted(names, key=len)\n",
    "print '99th percentile longest:', len(by_len[int(len(names) * 0.95)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13  0 19  4 26 27 27 27 27 27]\n"
     ]
    }
   ],
   "source": [
    "chars = list('abcdefghijklmnopqrstuvwxyz') + ['<END>', '<NULL>']\n",
    "indices_for_chars = {c: i for i, c in enumerate(chars)}\n",
    "\n",
    "NAME_MAX_LEN = 10 # include the <END> char\n",
    "\n",
    "def name_to_vec(name, maxlen=NAME_MAX_LEN):\n",
    "    v = np.zeros(maxlen, dtype=int)\n",
    "    null_idx = indices_for_chars['<NULL>']\n",
    "    v.fill(null_idx)\n",
    "    for i, c in enumerate(name):\n",
    "        if i >= maxlen: break\n",
    "        n = indices_for_chars.get(c, null_idx)\n",
    "        v[i] = n\n",
    "    v[min(len(name), maxlen-1)] = indices_for_chars['<END>']\n",
    "    return v\n",
    "\n",
    "def vec_to_name(vec):\n",
    "    name = ''\n",
    "    for x in vec:\n",
    "        char = chars[x]\n",
    "        if len(char) == 1:\n",
    "            name += char\n",
    "        elif char == '<END>':\n",
    "            return name\n",
    "    return name\n",
    "\n",
    "print name_to_vec('nate')\n",
    "assert vec_to_name(name_to_vec('nate')) == 'nate'\n",
    "assert vec_to_name(name_to_vec('aaaaaaaaaaaa')) == 'aaaaaaaaa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7583, 10)\n"
     ]
    }
   ],
   "source": [
    "name_vecs = np.array([name_to_vec(n) for n in names])\n",
    "print name_vecs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def weight_var(shape, stddev=0.1, weight_decay=0, name=None):\n",
    "    initial = tf.truncated_normal(shape, stddev=stddev)\n",
    "    v = tf.Variable(initial, name=name)\n",
    "    if weight_decay > 0:\n",
    "        l2 = tf.nn.l2_loss(v) * weight_decay\n",
    "        tf.add_to_collection('losses', l2)\n",
    "    return v\n",
    "\n",
    "def leaky_relu(x, leak=0.2, name=\"lrelu\"):\n",
    "    with tf.variable_scope(name):\n",
    "        f1 = 0.5 * (1 + leak)\n",
    "        f2 = 0.5 * (1 - leak)\n",
    "        return f1 * x + f2 * abs(x)\n",
    "\n",
    "def relu(x):\n",
    "    # return tf.nn.relu(x)\n",
    "    return leaky_relu(x)\n",
    "\n",
    "def create_conv(input, out_channels, patch_size=5, stride=1, batch_norm=False, dropout=False):\n",
    "    in_channels = input.get_shape()[-1].value\n",
    "    w = weight_var([patch_size, patch_size, in_channels, out_channels])\n",
    "    b = weight_var([out_channels], stddev=0)\n",
    "    conv = tf.nn.conv2d(input, w, strides=[1,stride,stride,1], padding='SAME')\n",
    "    if batch_norm: conv = create_batch_norm(conv)\n",
    "    activation = relu(conv + b)\n",
    "    if dropout: activation = create_dropout(activation)\n",
    "    return activation\n",
    "    \n",
    "def text_conv(input, out_channels, patch_size=5, stride=1, dropout=False, pool_size=1):\n",
    "    in_channels = input.get_shape()[-1].value\n",
    "    w = weight_var([patch_size, in_channels, out_channels])\n",
    "    b = weight_var([out_channels], stddev=0)\n",
    "    conv = tf.nn.conv1d(input, w, stride=stride, padding='SAME')\n",
    "    activation = relu(conv + b)\n",
    "    # TODO: max_pooling\n",
    "    if dropout: activation = create_dropout(activation)\n",
    "    return activation\n",
    "\n",
    "def create_dropout(units):\n",
    "    return tf.nn.dropout(units, dropout)\n",
    "\n",
    "def create_fc(input, out_size):\n",
    "    # input_dropped = tf.nn.dropout(input, dropout_keep_prob)\n",
    "    in_size = input.get_shape()[-1].value\n",
    "    w = weight_var([in_size, out_size], weight_decay=0.004)\n",
    "    b = weight_var([out_size], weight_decay=0.004)\n",
    "    x = tf.matmul(input, w)\n",
    "    return relu(x + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "name_placeholder = tf.placeholder(shape=[None, NAME_MAX_LEN], dtype=tf.int32, name='names')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "Z_SIZE = 64\n",
    "\n",
    "def encoder_lstm(names):\n",
    "    with tf.variable_scope('encoder'):\n",
    "        cells = [tf.nn.rnn_cell.LSTMCell(size, state_is_tuple=True) for size in [len(chars), 64]]\n",
    "        lstm = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)\n",
    "        one_hot = tf.one_hot(names, len(chars), dtype=tf.float32)\n",
    "        outputs, state = tf.nn.dynamic_rnn(lstm, one_hot, dtype=tf.float32)\n",
    "        outputs_flat = tf.reshape(outputs, [-1, 64 * NAME_MAX_LEN])\n",
    "        z_mean = create_fc(outputs_flat, Z_SIZE)\n",
    "        z_stddev = create_fc(outputs_flat, Z_SIZE)\n",
    "        return z_mean, z_stddev\n",
    "\n",
    "def encoder_conv(names):\n",
    "    with tf.variable_scope('encoder'):\n",
    "        one_hot = tf.one_hot(names, len(chars), dtype=tf.float32)\n",
    "        conv1 = text_conv(one_hot, 64)\n",
    "        conv2 = text_conv(one_hot, 64)\n",
    "        fc1 = create_fc(tf.reshape(conv2, [-1, NAME_MAX_LEN * 64]), 128)\n",
    "        z_mean = create_fc(fc1, Z_SIZE)\n",
    "        z_stddev = create_fc(fc1, Z_SIZE)\n",
    "        return z_mean, z_stddev\n",
    "    \n",
    "# def generator(noise, name='generator'):\n",
    "#     with tf.variable_scope(name, reuse=None):\n",
    "#         cells = [tf.nn.rnn_cell.LSTMCell(size, state_is_tuple=True) for size in [NOISE_SIZE, 256, len(chars)]]\n",
    "#         lstm = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)\n",
    "#         noise_repeated_over_time = tf.tile(tf.reshape(noise, [-1, 1, NOISE_SIZE]), [1, NAME_MAX_LEN, 1])\n",
    "#         outputs, state = tf.nn.dynamic_rnn(lstm, noise_repeated_over_time, dtype=tf.float32)\n",
    "#         output_chars = tf.reshape(tf.argmax(tf.nn.softmax(outputs), axis=2), [-1, NAME_MAX_LEN])\n",
    "#         output_chars = tf.cast(output_chars, tf.int32)\n",
    "#     return output_chars\n",
    "\n",
    "# generated_names = generator(noise)\n",
    "\n",
    "z_mean, z_stddev = encoder_lstm(name_placeholder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "session = tf.Session()\n",
    "session.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.01720497  0.06152692  0.01489044 -0.00332805  0.11789916 -0.0294213\n",
      "  -0.00113157 -0.01073818  0.0393256   0.07124054  0.08411945 -0.02391899\n",
      "  -0.00480187 -0.00608682 -0.00331387 -0.01766554 -0.00191268  0.09316398\n",
      "   0.00934732 -0.0098442  -0.01982485  0.16940635  0.02345115 -0.01490644\n",
      "  -0.02381536  0.18268323 -0.01417166  0.13008356 -0.01922372  0.14771619\n",
      "  -0.03145097 -0.03391168  0.00539082 -0.01963281  0.00139857 -0.03431068\n",
      "  -0.00906854  0.05548166  0.00848088 -0.01805742  0.03736436  0.00411171\n",
      "   0.09423582  0.08919111 -0.00632786  0.16512939 -0.03048882  0.08582069\n",
      "  -0.01247227  0.15202031  0.09132294  0.1232969  -0.02480896  0.09067419\n",
      "   0.11355287  0.09660275  0.05903743 -0.03926057 -0.02319293 -0.03711216\n",
      "   0.0961521   0.09397506  0.12297711 -0.0115314 ]]\n"
     ]
    }
   ],
   "source": [
    "print session.run(z_mean, feed_dict={name_placeholder: [name_to_vec('nate')]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sample_z(z_mean, z_stddev):\n",
    "    samples = tf.random_normal(tf.shape(z_stddev), 0, 1, dtype=tf.float32)\n",
    "    return z_mean + samples * z_stddev\n",
    "\n",
    "z_vals = sample_z(z_mean, z_stddev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def decoder_lstm(z):\n",
    "    z_repeated_over_time = tf.tile(tf.reshape(z, [-1, 1, Z_SIZE]), [1, NAME_MAX_LEN, 1])\n",
    "    cells = [tf.nn.rnn_cell.LSTMCell(size, state_is_tuple=True) for size in [Z_SIZE, 256, len(chars)]]\n",
    "    lstm = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)\n",
    "    outputs, state = tf.nn.dynamic_rnn(lstm, z_repeated_over_time, dtype=tf.float32)\n",
    "    return outputs\n",
    "\n",
    "z_input = tf.placeholder(tf.float32, [None, Z_SIZE], name='z_input')\n",
    "use_z_input = tf.placeholder(tf.int32, shape=[], name=\"use_z_input_condition\")\n",
    "decoder_input = tf.cond(use_z_input > 0, lambda: z_input, lambda: z_vals)\n",
    "\n",
    "decoded = decoder_lstm(decoder_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "diff_loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(decoded, name_placeholder))\n",
    "kl_divergence = tf.reduce_mean(0.5 * tf.reduce_sum(tf.square(z_mean) + tf.square(z_stddev) - tf.log(tf.square(z_stddev)) - 1, 1))\n",
    "loss = diff_loss + kl_divergence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "decoded_vecs = tf.argmax(decoded, axis=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "learn_rate = tf.placeholder(tf.float32, name='learning_rate')\n",
    "optimizer = tf.train.AdamOptimizer(learn_rate)\n",
    "global_step = tf.contrib.framework.get_or_create_global_step()\n",
    "train_step = optimizer.minimize(loss, global_step=global_step)\n",
    "session.run(tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Will not save progress\n"
     ]
    }
   ],
   "source": [
    "save_path = None\n",
    "\n",
    "session = tf.Session()\n",
    "init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())\n",
    "session.run(init_op)\n",
    "\n",
    "saver = None\n",
    "if save_path:\n",
    "    if not os.path.exists(save_path):\n",
    "        os.mkdir(save_path)\n",
    "    saver = tf.train.Saver()\n",
    "    ckpt = tf.train.get_checkpoint_state(save_path)\n",
    "    if ckpt and ckpt.model_checkpoint_path:\n",
    "        saver.restore(session, ckpt.model_checkpoint_path)\n",
    "        print 'Restored from checkpoint', ckpt.model_checkpoint_path\n",
    "    else:\n",
    "        print 'Did not restore from checkpoint'\n",
    "else:\n",
    "    print 'Will not save progress'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nate -> yyyyymmmmm\n"
     ]
    }
   ],
   "source": [
    "def reconstruct(name):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.array([name_to_vec(name)]),\n",
    "        z_input: np.zeros((64, Z_SIZE)),\n",
    "        use_z_input: 0,\n",
    "        learn_rate: 0.01\n",
    "    }\n",
    "    output_ = session.run(decoded_vecs, feed_dict=feed_dict)\n",
    "    return vec_to_name(output_[0])\n",
    "\n",
    "for name in ['nate']:\n",
    "    print name, '->', reconstruct(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 600; loss: 1566.54138184\n",
      "nate -> aaaa\n",
      "will -> aaaa\n",
      "joe -> aaa\n",
      "justin -> aaaaaa\n",
      " example encoding: bell -> aaaa\n",
      "Step: 1200; loss: 1542.34008789\n",
      "nate -> aaaa\n",
      "will -> aaaa\n",
      "joe -> aaa\n",
      "justin -> aaaaaa\n",
      " example encoding: kingston -> aaaaaeeea\n",
      "Step: 1800; loss: 1530.46826172\n",
      "nate -> aaaa\n",
      "will -> aaaa\n",
      "joe -> aaa\n",
      "justin -> aaaaaaa\n",
      " example encoding: erasmus -> aaaaaee\n",
      "Step: 2400; loss: 1498.74853516\n",
      "nate -> aaaa\n",
      "will -> aaaa\n",
      "joe -> aaa\n",
      "justin -> aaaaaa\n",
      " example encoding: nona -> aaaa\n",
      "Step: 3000; loss: 1488.49914551\n",
      "nate -> aaaa\n",
      "will -> aaaa\n",
      "joe -> aaa\n",
      "justin -> aaaaaa\n",
      " example encoding: les -> aaa\n",
      "Step: 3600; loss: 1483.14575195\n",
      "nate -> aaaa\n",
      "will -> aaaa\n",
      "joe -> aaa\n",
      "justin -> aaaaaaa\n",
      " example encoding: berrie -> caaaaa\n",
      "Step: 4200; loss: 1403.80224609\n",
      "nate -> aaaa\n",
      "will -> aaaa\n",
      "joe -> aaa\n",
      "justin -> aaaaaa\n",
      " example encoding: valida -> aaaaaa\n",
      "Step: 4800; loss: 1425.92895508\n",
      "nate -> baea\n",
      "will -> aaaa\n",
      "joe -> jaa\n",
      "justin -> laeeeeh\n",
      " example encoding: augusto -> baaeeea\n",
      "Step: 5400; loss: 1374.92651367\n",
      "nate -> aeee\n",
      "will -> leee\n",
      "joe -> jae\n",
      "justin -> aeeeeea\n",
      " example encoding: erinna -> aaeeea\n",
      "Step: 6000; loss: 1384.16186523\n",
      "nate -> aeee\n",
      "will -> leee\n",
      "joe -> aoe\n",
      "justin -> aeeeeea\n",
      " example encoding: maxi -> meee\n",
      "Step: 6600; loss: 1375.80334473\n",
      "nate -> aeee\n",
      "will -> leee\n",
      "joe -> joa\n",
      "justin -> aaeeeeh\n",
      " example encoding: rosana -> rorsna\n",
      "Step: 7200; loss: 1370.28515625\n",
      "nate -> aeee\n",
      "will -> leee\n",
      "joe -> ll\n",
      "justin -> aaeeeea\n",
      " example encoding: rosamund -> rorrrand\n",
      "Step: 7800; loss: 1360.13366699\n",
      "nate -> aeee\n",
      "will -> llle\n",
      "joe -> lle\n",
      "justin -> aaniita\n",
      " example encoding: gilberta -> gileeeea\n",
      "Step: 8400; loss: 1339.76550293\n",
      "nate -> aaee\n",
      "will -> lill\n",
      "joe -> lle\n",
      "justin -> aasiita\n",
      " example encoding: lorinda -> llrdina\n",
      "Step: 9000; loss: 1311.49194336\n",
      "nate -> nete\n",
      "will -> aill\n",
      "joe -> loe\n",
      "justin -> alsttta\n",
      " example encoding: andria -> aadiia\n",
      "Step: 9600; loss: 1296.42736816\n",
      "nate -> aete\n",
      "will -> aall\n",
      "joe -> loe\n",
      "justin -> aasitt\n",
      " example encoding: conrad -> conddd\n",
      "Step: 10200; loss: 1296.12414551\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> loe\n",
      "justin -> aasitt\n",
      " example encoding: henrietta -> ceeneeeea\n",
      "Step: 10800; loss: 1294.8470459\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> loe\n",
      "justin -> aasiie\n",
      " example encoding: gera -> gera\n",
      "Step: 11400; loss: 1278.14416504\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joa\n",
      "justin -> aasiit\n",
      " example encoding: jeanluc -> jlllllee\n",
      "Step: 12000; loss: 1295.63122559\n",
      "nate -> nate\n",
      "will -> llll\n",
      "joe -> joe\n",
      "justin -> aasiit\n",
      " example encoding: clayton -> llaooon\n",
      "Step: 12600; loss: 1280.59094238\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> loe\n",
      "justin -> aasiin\n",
      " example encoding: jonathon -> aonoooon\n",
      "Step: 13200; loss: 1269.58618164\n",
      "nate -> ante\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> aatiin\n",
      " example encoding: ammamaria -> aaaaaaria\n",
      "Step: 13800; loss: 1263.86474609\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> loe\n",
      "justin -> aasiin\n",
      " example encoding: giordano -> giiooooo\n",
      "Step: 14400; loss: 1252.1829834\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> aasiin\n",
      " example encoding: nathan -> aathnn\n",
      "Step: 15000; loss: 1271.75280762\n",
      "nate -> aate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> aasiin\n",
      " example encoding: xymenes -> aseenee\n",
      "Step: 15600; loss: 1257.65686035\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> jastin\n",
      " example encoding: astrix -> asrria\n",
      "Step: 16200; loss: 1246.94067383\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> jastin\n",
      " example encoding: grace -> rraee\n",
      "Step: 16800; loss: 1247.37451172\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> jastin\n",
      " example encoding: hadria -> carria\n",
      "Step: 17400; loss: 1253.98327637\n",
      "nate -> aate\n",
      "will -> gill\n",
      "joe -> joe\n",
      "justin -> jatiin\n",
      " example encoding: anny -> nnn\n",
      "Step: 18000; loss: 1240.4876709\n",
      "nate -> nate\n",
      "will -> rill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: jacquenet -> jaauuenee\n",
      "Step: 18600; loss: 1238.52282715\n",
      "nate -> aate\n",
      "will -> gill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: lora -> lora\n",
      "Step: 19200; loss: 1234.86303711\n",
      "nate -> tate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: wileen -> bileen\n",
      "Step: 19800; loss: 1202.43530273\n",
      "nate -> aate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: francisca -> rraniiisa\n",
      "Step: 20400; loss: 1203.64355469\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> jee\n",
      "justin -> justin\n",
      " example encoding: mamie -> maiie\n",
      "Step: 21000; loss: 1208.98059082\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: rafaelia -> raaaelia\n",
      "Step: 21600; loss: 1223.97961426\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: glynda -> glanda\n",
      "Step: 22200; loss: 1198.84033203\n",
      "nate -> nate\n",
      "will -> lila\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: sallyann -> callaann\n",
      "Step: 22800; loss: 1185.20288086\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: geeta -> geeta\n",
      "Step: 23400; loss: 1207.625\n",
      "nate -> aate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ketty -> ketty\n",
      "Step: 24000; loss: 1215.12585449\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: giffer -> giieer\n",
      "Step: 24600; loss: 1202.33520508\n",
      "nate -> aate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: staford -> chaoord\n",
      "Step: 25200; loss: 1193.60205078\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: antoinett -> nntonnett\n",
      "Step: 25800; loss: 1219.94213867\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: stefanie -> shenanie\n",
      "Step: 26400; loss: 1202.78466797\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: thanks -> aathanss\n",
      "Step: 27000; loss: 1202.0657959\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: rockwell -> roaaeela\n",
      "Step: 27600; loss: 1192.6114502\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: henrik -> henria\n",
      "Step: 28200; loss: 1193.44726562\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: robena -> roeena\n",
      "Step: 28800; loss: 1197.33190918\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: wilona -> iilona\n",
      "Step: 29400; loss: 1193.41967773\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: simonette -> cioonette\n",
      "Step: 30000; loss: 1180.97851562\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: corrina -> corrina\n",
      "Step: 30600; loss: 1192.38757324\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: antonino -> nntonooo\n",
      "Step: 31200; loss: 1185.03857422\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ginny -> ginny\n",
      "Step: 31800; loss: 1187.16137695\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: danya -> daana\n",
      "Step: 32400; loss: 1184.9675293\n",
      "nate -> aate\n",
      "will -> lall\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: delphinia -> delhhinia\n",
      "Step: 33000; loss: 1182.5970459\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: thornie -> thornie\n",
      "Step: 33600; loss: 1190.53320312\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: parsifal -> carsiial\n",
      "Step: 34200; loss: 1196.43603516\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justina\n",
      " example encoding: paolo -> caolo\n",
      "Step: 34800; loss: 1170.7956543\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: enid -> enid\n",
      "Step: 35400; loss: 1185.70568848\n",
      "nate -> ntte\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: giovanna -> gionanna\n",
      "Step: 36000; loss: 1180.06262207\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: renell -> renell\n",
      "Step: 36600; loss: 1192.91943359\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> jultin\n",
      " example encoding: toinette -> tonnetta\n",
      "Step: 37200; loss: 1194.66088867\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: lorianna -> lorianna\n",
      "Step: 37800; loss: 1190.51342773\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: dwane -> daane\n",
      "Step: 38400; loss: 1178.07043457\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: dedie -> dedie\n",
      "Step: 39000; loss: 1177.6217041\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: sheelah -> sheelah\n",
      "Step: 39600; loss: 1182.10241699\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: zorine -> rorine\n",
      "Step: 40200; loss: 1185.53051758\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: maurice -> mauriee\n",
      "Step: 40800; loss: 1187.05786133\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: bengt -> benta\n",
      "Step: 41400; loss: 1183.35192871\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: janeczka -> janeaaaa\n",
      "Step: 42000; loss: 1176.70080566\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: olle -> olee\n",
      "Step: 42600; loss: 1179.60974121\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> juttin\n",
      " example encoding: davin -> daain\n",
      "Step: 43200; loss: 1176.16894531\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: lissa -> lissa\n",
      "Step: 43800; loss: 1164.1328125\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: onida -> onida\n",
      "Step: 44400; loss: 1184.61767578\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: jackie -> jaiiie\n",
      "Step: 45000; loss: 1174.43371582\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: jennee -> jennee\n",
      "Step: 45600; loss: 1171.35693359\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: jessamyn -> jessaaan\n",
      "Step: 46200; loss: 1172.82104492\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> jlstin\n",
      " example encoding: blondell -> blondell\n",
      "Step: 46800; loss: 1183.60388184\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: bea -> bea\n",
      "Step: 47400; loss: 1178.00085449\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: sammy -> saaay\n",
      "Step: 48000; loss: 1180.01538086\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: adrian -> adrian\n",
      "Step: 48600; loss: 1185.46838379\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: caterina -> caterina\n",
      "Step: 49200; loss: 1181.41357422\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: essie -> essie\n",
      "Step: 49800; loss: 1186.88415527\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: krystle -> krsstle\n",
      "Step: 50400; loss: 1174.67663574\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: berri -> berri\n",
      "Step: 51000; loss: 1173.48925781\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: knox -> taoo\n",
      "Step: 51600; loss: 1188.91931152\n",
      "nate -> natt\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: imojean -> iooeean\n",
      "Step: 52200; loss: 1173.68603516\n",
      "nate -> nate\n",
      "will -> gill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: lucita -> luiita\n",
      "Step: 52800; loss: 1176.19458008\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ferne -> eerne\n",
      "Step: 53400; loss: 1182.43884277\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: raine -> raine\n",
      "Step: 54000; loss: 1179.16149902\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: fraser -> araser\n",
      "Step: 54600; loss: 1172.39794922\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: neal -> neal\n",
      "Step: 55200; loss: 1189.60168457\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: nevin -> neiin\n",
      "Step: 55800; loss: 1175.77294922\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: mohan -> mohan\n",
      "Step: 56400; loss: 1168.72351074\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: erna -> erna\n",
      "Step: 57000; loss: 1167.50646973\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> juttin\n",
      " example encoding: tommy -> toaay\n",
      "Step: 57600; loss: 1171.39135742\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: andrey -> andrey\n",
      "Step: 58200; loss: 1167.69897461\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: charlotta -> charlotta\n",
      "Step: 58800; loss: 1163.73620605\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: foster -> ooster\n",
      "Step: 59400; loss: 1181.50964355\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: normie -> noraie\n",
      "Step: 60000; loss: 1175.44836426\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: bethanne -> bethanne\n",
      "Step: 60600; loss: 1177.16052246\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: florencia -> roorenaia\n",
      "Step: 61200; loss: 1162.63684082\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: sueelle -> seelelle\n",
      "Step: 61800; loss: 1175.44970703\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: consuelo -> consoolo\n",
      "Step: 62400; loss: 1173.84448242\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: franky -> aranay\n",
      "Step: 63000; loss: 1172.1529541\n",
      "nate -> nate\n",
      "will -> ball\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ericha -> erihha\n",
      "Step: 63600; loss: 1166.73144531\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joey\n",
      "justin -> justin\n",
      " example encoding: simeon -> sieeon\n",
      "Step: 64200; loss: 1162.90380859\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> juttin\n",
      " example encoding: magdalen -> maddalen\n",
      "Step: 64800; loss: 1167.88452148\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: chadwick -> shaddiaa\n",
      "Step: 65400; loss: 1169.45031738\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ninnette -> ninnette\n",
      "Step: 66000; loss: 1176.68237305\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justina\n",
      " example encoding: vilhelm -> lilhela\n",
      "Step: 66600; loss: 1171.09411621\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: chelsae -> shelsae\n",
      "Step: 67200; loss: 1161.66540527\n",
      "nate -> natt\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: nelia -> nelia\n",
      "Step: 67800; loss: 1171.83288574\n",
      "nate -> natt\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: barrett -> barrett\n",
      "Step: 68400; loss: 1177.1574707\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: antoine -> antoine\n",
      "Step: 69000; loss: 1168.15319824\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ingelbert -> inaeleert\n",
      "Step: 69600; loss: 1159.06762695\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: aleta -> aleta\n",
      "Step: 70200; loss: 1159.34301758\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: roda -> roda\n",
      "Step: 70800; loss: 1167.28735352\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: steward -> steaard\n",
      "Step: 71400; loss: 1174.77844238\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: mitchell -> mithhell\n",
      "Step: 72000; loss: 1172.22436523\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justina\n",
      " example encoding: brandice -> brandiae\n",
      "Step: 72600; loss: 1156.74755859\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: jolie -> jolie\n",
      "Step: 73200; loss: 1164.37695312\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: paton -> paton\n",
      "Step: 73800; loss: 1171.8795166\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: frederick -> drederiaa\n",
      "Step: 74400; loss: 1169.64147949\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: lenny -> lenny\n",
      "Step: 75000; loss: 1179.61352539\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: suzzy -> sussy\n",
      "Step: 75600; loss: 1171.31213379\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: johan -> johan\n",
      "Step: 76200; loss: 1161.00927734\n",
      "nate -> nate\n",
      "will -> gill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: jennilee -> jennilee\n",
      "Step: 76800; loss: 1168.26953125\n",
      "nate -> nate\n",
      "will -> gill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: rochester -> rohhester\n",
      "Step: 77400; loss: 1166.3807373\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ada -> ada\n",
      "Step: 78000; loss: 1172.23632812\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: sim -> sia\n",
      "Step: 78600; loss: 1177.46289062\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: meaghan -> meahhan\n",
      "Step: 79200; loss: 1181.03527832\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: danielle -> danielle\n",
      "Step: 79800; loss: 1174.9934082\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: rupert -> rurert\n",
      "Step: 80400; loss: 1160.05273438\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: mariya -> mariaa\n",
      "Step: 81000; loss: 1170.50268555\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: allsun -> allsun\n",
      "Step: 81600; loss: 1174.43127441\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: hercules -> heruules\n",
      "Step: 82200; loss: 1175.26379395\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: karisa -> karisa\n",
      "Step: 82800; loss: 1171.79956055\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: eduard -> eduard\n",
      "Step: 83400; loss: 1170.56848145\n",
      "nate -> nate\n",
      "will -> rill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: avivah -> aaihah\n",
      "Step: 84000; loss: 1169.05664062\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: lurline -> lurline\n",
      "Step: 84600; loss: 1159.45178223\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: deeanne -> deeanne\n",
      "Step: 85200; loss: 1168.95910645\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: gillie -> gillie\n",
      "Step: 85800; loss: 1161.6652832\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> juttin\n",
      " example encoding: nichol -> nihhol\n",
      "Step: 86400; loss: 1170.35827637\n",
      "nate -> aate\n",
      "will -> gill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: blancha -> blanhha\n",
      "Step: 87000; loss: 1155.03564453\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: conni -> conni\n",
      "Step: 87600; loss: 1169.54833984\n",
      "nate -> nate\n",
      "will -> gill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: veda -> eeda\n",
      "Step: 88200; loss: 1159.03588867\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: teddi -> teddi\n",
      "Step: 88800; loss: 1163.88452148\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: si -> si\n",
      "Step: 89400; loss: 1170.44396973\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: rianon -> rianon\n",
      "Step: 90000; loss: 1165.98388672\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: korella -> korella\n",
      "Step: 90600; loss: 1164.71105957\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: nonie -> nonie\n",
      "Step: 91200; loss: 1172.96435547\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: aime -> aiee\n",
      "Step: 91800; loss: 1174.26257324\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: johnathan -> johaathan\n",
      "Step: 92400; loss: 1161.16101074\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: wallache -> lallaehe\n",
      "Step: 93000; loss: 1158.5802002\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: essa -> essa\n",
      "Step: 93600; loss: 1162.91369629\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: priscilla -> prisiilla\n",
      "Step: 94200; loss: 1163.82556152\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: shelagh -> shelaah\n",
      "Step: 94800; loss: 1160.95983887\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: raine -> raine\n",
      "Step: 95400; loss: 1172.72558594\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: grace -> graee\n",
      "Step: 96000; loss: 1169.47619629\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: phillipe -> philliee\n",
      "Step: 96600; loss: 1163.29052734\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: gabriele -> gairiele\n",
      "Step: 97200; loss: 1165.25793457\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: yehudi -> eeuudi\n",
      "Step: 97800; loss: 1167.47106934\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: bobbie -> boieie\n",
      "Step: 98400; loss: 1170.30285645\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: lynsey -> lynsey\n",
      "Step: 99000; loss: 1169.04748535\n",
      "nate -> nate\n",
      "will -> bill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: sonni -> sonni\n",
      "Step: 99600; loss: 1161.7232666\n",
      "nate -> aate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: alma -> alaa\n",
      "Step: 100200; loss: 1154.94445801\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: ariella -> ariella\n",
      "Step: 100800; loss: 1153.51831055\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: biddy -> biddy\n",
      "Step: 101400; loss: 1159.71911621\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justina\n",
      " example encoding: leoline -> leoline\n",
      "Step: 102000; loss: 1153.48547363\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: brice -> briee\n",
      "Step: 112200; loss: 1162.00390625\n",
      "nate -> nate\n",
      "will -> iill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: sheelagh -> sheelaah\n",
      "Step: 112800; loss: 1159.01794434\n",
      "nate -> nate\n",
      "will -> lill\n",
      "joe -> joe\n",
      "justin -> justin\n",
      " example encoding: anne -> anne\n"
     ]
    }
   ],
   "source": [
    "train = True\n",
    "\n",
    "test_names = ['nate', 'will', 'joe', 'justin']\n",
    "\n",
    "while train:\n",
    "    names = name_vecs[np.random.randint(name_vecs.shape[0], size=64), :]\n",
    "    feed_dict = {\n",
    "        name_placeholder: names,\n",
    "        z_input: np.zeros((64, Z_SIZE)),\n",
    "        use_z_input: 0,\n",
    "        learn_rate: 0.001\n",
    "    }\n",
    "    _, loss_, step_ = session.run([train_step, loss, global_step], feed_dict=feed_dict)\n",
    "    if step_ % 600 == 0:\n",
    "        output_ = session.run(decoded_vecs, feed_dict=feed_dict)\n",
    "        print \"Step: {0}; loss: {1}\".format(step_, loss_)\n",
    "        for n in test_names:\n",
    "            print n, '->', reconstruct(n)\n",
    "        # print names[0]\n",
    "        # print output_[0]\n",
    "        print \" example encoding: {} -> {}\".format(vec_to_name(names[0]), vec_to_name(output_[0]))\n",
    "        if step_ % 600 == 0:\n",
    "            if saver:\n",
    "                saver.save(session, save_path + '/model.ckpt', global_step=step_)\n",
    "                print 'Saved'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# reconstruct is pretty good at reconstructing american names, even long ones:\n",
    "for name in ['nate', 'will', 'chen', 'atty', 'arielle', 'nathaniel', 'kimberly', 'erica', 'zoe']:\n",
    "    print name, '->', reconstruct(name)\n",
    "\n",
    "# although notably, it's bad at reconstructing names with less-frequent letters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# it's decent at some english words that 'sound' like ames:\n",
    "for name in ['word', 'happy', 'winter', 'candle', 'cherish']:\n",
    "    print name, '->', reconstruct(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# it's worse at more longer, more \"wordy\" names:\n",
    "for name in ['embedding', 'automobile', 'air', 'larynx']:\n",
    "    print name, '->', reconstruct(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# predictably, it's terrible at things that aren't even pronouncable strings of letters:\n",
    "for name in ['ufhoe', 'xyzy', 'ihwrfoecoei']:\n",
    "    print name, '->', reconstruct(name)\n",
    "# it even seems to try to turn some into slightly more name-like strings:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# so reconstruction quality seems like a pretty good of how \"name-ish\" a word is\n",
    "# want to give your kid a cool, ~original~ name no one has, but that sounds good?\n",
    "# what good english words sound like names, but aren't?\n",
    "\n",
    "# first, let's build a 'nameliness' score:\n",
    "def nameliness(word):\n",
    "    r = reconstruct(word)\n",
    "    return sum([1 if a == b else 0 for a, b in zip(word, r)]) / float(len(word))\n",
    "\n",
    "for name in ['nate', 'july', 'fridge', 'gienigoe', 'chzsiucf', 'xyxyzzy']:\n",
    "    print name, ':', nameliness(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# let's grab the top 10k english words, remove the things that are names, and see which can be reconstructed best:\n",
    "# source: https://github.com/first20hours/google-10000-english\n",
    "\n",
    "top_words = list(word.strip() for word in open('../data/google-10000-english.txt'))\n",
    "top_words = list(word for word in top_words if word not in names)\n",
    "print len(top_words)\n",
    "top_words = top_words[:1000] # this is actually kinda slow, so let's stick with the top 1k\n",
    "nameliness_scores = {word: nameliness(word) for word in top_words}\n",
    "print [w for w in top_words if nameliness_scores[w] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# let's build a big lookup table of all the names and their embeddings:\n",
    "def make_batches(list, size=128):\n",
    "    batches = []\n",
    "    while len(list):\n",
    "        batches.append(list[:min(len(list), size)])\n",
    "        list = list[len(batches[-1]):]\n",
    "    return batches\n",
    "\n",
    "embeddings = {}\n",
    "\n",
    "for batch in make_batches(list(names)):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.array([name_to_vec(name) for name in batch]),\n",
    "        z_input: np.zeros((len(batch), Z_SIZE)),\n",
    "        use_z_input: 0\n",
    "    }\n",
    "    output_ = session.run(z_mean, feed_dict=feed_dict)\n",
    "    for name, vec in zip(batch, output_):\n",
    "        embeddings[name] = vec\n",
    "    # print 'processed {}/{}'.format(len(embeddings), len(names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def embed(name):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.array([name_to_vec(name)]),\n",
    "        z_input: np.zeros((1, Z_SIZE)),\n",
    "        use_z_input: 0\n",
    "    }\n",
    "    output_ = session.run(z_mean, feed_dict=feed_dict)\n",
    "    return output_[0]\n",
    "\n",
    "def nearest(embedding):\n",
    "    def distance(name):\n",
    "        return np.linalg.norm(embedding - embeddings[name])\n",
    "    return min(embeddings.iterkeys(), key=distance)\n",
    "\n",
    "def unembed(embedding):\n",
    "    feed_dict = {\n",
    "        name_placeholder: np.zeros((1, NAME_MAX_LEN)),\n",
    "        z_input: np.array([embedding]),\n",
    "        use_z_input: 1\n",
    "    }\n",
    "    output_ = session.run(decoded_vecs, feed_dict=feed_dict)\n",
    "    return vec_to_name(output_[0])\n",
    "\n",
    "assert unembed(embed('nate')) == 'nate'\n",
    "\n",
    "# print nearest(embed('nate'))\n",
    "for name in ['nate', 'yikes', 'panda', 'ixzhxzi', 'justxn']:\n",
    "    print name, 'is closest to', nearest(embed(name))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# what happens if we try to interpolate names?\n",
    "def blend_names(name1, name2):\n",
    "    e1 = embed(name1)\n",
    "    e2 = embed(name2)\n",
    "    for i in range(11):\n",
    "        blend = i / 10.0\n",
    "        print unembed(e1 * (1 - blend) + e2 * blend)\n",
    "\n",
    "blend_names('amy', 'francisco')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "blend_names('nathaniel', 'chen')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "blend_names('nathaniel', 'leinahtan')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print nearest(np.zeros(Z_SIZE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# what if we multiply names?:\n",
    "for name in ['nate', 'willy', 'sam', 'polly', 'jacob']:\n",
    "    print name, '* 2 =', nearest(embed(name) * 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# what's the opposite of a name?\n",
    "for name in ['nancy', 'barry', 'chance', 'rachel', 'gloria']:\n",
    "    print '-' + name, '=', nearest(-embed(name))\n",
    "# weird that rachel's opposite's opposite isn't rachel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# can we do addition and subtraction?\n",
    "print nearest(embed('alberta') - embed('albert') + embed('robert'))\n",
    "print nearest(embed('alberta') - embed('albert') + embed('justin'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# what if, rather than looking for the nearest neighbors, we generate names from these arithmetic operations?\n",
    "print unembed(embed('alberta') - embed('albert') + embed('robert'))\n",
    "print unembed(embed('alberta') - embed('albert') + embed('justin'))\n",
    "print unembed(embed('alberta') - embed('albert') + embed('joseph'))\n",
    "\n",
    "print unembed(embed('alberta') - embed('albert') + embed('nate')) # doesn't work so well with names ending in vowels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# let's generate some random names:\n",
    "def generate():\n",
    "    return unembed(np.random.normal(size=Z_SIZE))\n",
    "for _ in range(10):\n",
    "    print generate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# what if we train a GAN to mimick the latent vectors produced by real names?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}