TextGenFu.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "\n",
    "# method for generating text\n",
    "def generate_text(model, length, vocab_size, ix_to_char):\n",
    "\t# starting with random character\n",
    "\tix = [np.random.randint(vocab_size)]\n",
    "\ty_char = [ix_to_char[ix[-1]]]\n",
    "\tX = np.zeros((1, length, vocab_size))\n",
    "\tfor i in range(length):\n",
    "\t\t# appending the last predicted character to sequence\n",
    "\t\tX[0, i, :][ix[-1]] = 1\n",
    "\t\tprint(ix_to_char[ix[-1]], end=\"\")\n",
    "\t\tix = np.argmax(model.predict(X[:, :i+1, :])[0], 1)\n",
    "\t\ty_char.append(ix_to_char[ix[-1]])\n",
    "\treturn ('').join(y_char)\n",
    "\n",
    "# method for preparing the training data\n",
    "def load_data(data_dir, seq_length):\n",
    "\tdata = open(data_dir, 'r').read()\n",
    "\tchars = list(set(data))\n",
    "\tVOCAB_SIZE = len(chars)\n",
    "\n",
    "\tprint('Data length: {} characters'.format(len(data)))\n",
    "\tprint('Vocabulary size: {} characters'.format(VOCAB_SIZE))\n",
    "\n",
    "\tix_to_char = {ix:char for ix, char in enumerate(chars)}\n",
    "\tchar_to_ix = {char:ix for ix, char in enumerate(chars)}\n",
    "\tprint(type(seq_length))\n",
    "\tprint(type(VOCAB_SIZE))\n",
    "\tX = np.zeros((int(len(data))//int(seq_length), int(seq_length), int(VOCAB_SIZE)))\n",
    "\ty = np.zeros((int(len(data))//int(seq_length), int(seq_length), int(VOCAB_SIZE)))\n",
    "\tfor i in range(0, len(data)//seq_length):\n",
    "\t\tX_sequence = data[i*seq_length:(i+1)*seq_length]\n",
    "\t\tX_sequence_ix = [char_to_ix[value] for value in X_sequence]\n",
    "\t\tinput_sequence = np.zeros((seq_length, VOCAB_SIZE))\n",
    "\t\tfor j in range(seq_length):\n",
    "\t\t\tinput_sequence[j][X_sequence_ix[j]] = 1.\n",
    "\t\t\tX[i] = input_sequence\n",
    "\n",
    "\t\ty_sequence = data[i*seq_length+1:(i+1)*seq_length+1]\n",
    "\t\ty_sequence_ix = [char_to_ix[value] for value in y_sequence]\n",
    "\t\ttarget_sequence = np.zeros((seq_length, VOCAB_SIZE))\n",
    "\t\tfor j in range(seq_length):\n",
    "\t\t\ttarget_sequence[j][y_sequence_ix[j]] = 1.\n",
    "\t\t\ty[i] = target_sequence\n",
    "\treturn X, y, VOCAB_SIZE, ix_to_char\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data length: 1069888 characters\n",
      "Vocabulary size: 76 characters\n",
      "<class 'int'>\n",
      "<class 'int'>\n",
      "]\n",
      ";DDDDE'''ii887777TTTTTTTTTToooDDDDDDDEPP5''''CCCC333-BByyytt3393yyy33yBdy3yy3dy3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3yy33yyd3yy3y3ydy3y\n",
      "\n",
      "Epoch: 0\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/samuelochsner/anaconda3/lib/python3.6/site-packages/keras/models.py:942: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  warnings.warn('The `nb_epoch` argument in `fit` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 312s 15ms/step - loss: 2.2167\n",
      "V the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the LORD the\n",
      "\n",
      "Epoch: 1\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 310s 14ms/step - loss: 1.4023\n",
      "on the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and the children of Israel the son of Egypt, and th\n",
      "\n",
      "Epoch: 2\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 299s 14ms/step - loss: 1.1951\n",
      "zere the sons of Jerubuned and the sons of Jephunneh went out of the sacrifices of the children of Israel, and the sons of Jephunneh went out of the sacrifices of the children of Israel, and the sons of Jephunneh went out of the sacrifices of the children of Israel, and the sons of Jephunneh went out of the sacrifices of the children of Israel, and the sons of Jephunneh went out of the sacrifices of the children of Israel, and the sons of Jephunneh went out of the sacrifices of the children of I\n",
      "\n",
      "Epoch: 3\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 301s 14ms/step - loss: 1.0946\n",
      "I said, The LORD said unto Moses, Go that thou mayest be a stranger that were numbered of them, and shalt thou be a stranger of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the court of the cour\n",
      "\n",
      "Epoch: 4\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 302s 14ms/step - loss: 1.0270\n",
      "Canaan, and said, The man that is in the seventh day the sons of Joseph had said, Wherefore the LORD thy God shall be for a sigh of the first year, for a burnt offering and the shekels of the same day the seventh day the sons of Moses, and the sons of Aaron, and the family of the\n",
      "Amorites, and the children of Israel, and said, Wherefore the men of Israel shall be unclean until the\n",
      "even.\n",
      "\n",
      "15:17 And the priest shall sprinkle them upon the holy place which the LORD shall speak unto them and said un\n",
      "\n",
      "Epoch: 5\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 307s 14ms/step - loss: 0.9731\n",
      "1:19 And the LORD said unto Moses, Go in the land of Egypt the son of Amminadab, and the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Amminadab, the son of Ammihud, the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephunneh the son of Jephu\n",
      "\n",
      "Epoch: 6\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 388s 18ms/step - loss: 0.9266\n",
      "e the son of Aaron the son of Aaron, took their sinter shall the priest, and the priest shall pronounce him unclean until the even.\n",
      "\n",
      "11:18 And the priest shall pronounce him unclean until the even.\n",
      "\n",
      "11:18 And the priest shall pronounce him unclean until the even.\n",
      "\n",
      "11:18 And the priest shall pronounce him unclean until the even.\n",
      "\n",
      "11:18 And the priest shall pronounce him unclean until the even.\n",
      "\n",
      "11:18 And the priest shall pronounce him unclean until the even.\n",
      "\n",
      "11:18 And the priest shall pronounce \n",
      "\n",
      "Epoch: 7\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 376s 18ms/step - loss: 0.8838\n",
      "ve the LORD thy God giveth thee for an inheritance, and the stranger that shall be a statute for ever unto the LORD.\n",
      "\n",
      "24:13 And the LORD said unto Moses, Go in unto the LORD, the LORD shall send the LORD thy God, and will give thee thee unto the priest to the LORD thy God, and the stranger that sojourneth among them.\n",
      "\n",
      "20:1 And the LORD said unto Moses, Go in unto the LORD thy God, that they may serve me and the children of Israel took the camp of Bethuel the son of Jephunneh, the family of the K\n",
      "\n",
      "Epoch: 8\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 362s 17ms/step - loss: 0.8430\n",
      "- the words of this law were save your enemies to be consecrated them.\n",
      "\n",
      "28:17 And he said unto him, What men are the service of the tabernacle of the congregation, and all the people that were with him, and his sons shall put the two side of the garments with the breadth of it, and a cubit and a hand of the tribe of the children of Israel, and say unto them, When ye be the way of the wilderness of Sinai, and the children of Israel went out from the land of Egypt, and the priest shall pronounce h\n",
      "\n",
      "Epoch: 9\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 317s 15ms/step - loss: 0.8035\n",
      ") And the LORD spake unto Moses, saying, 1:2 Take the LORD thy God hath commanded thee, that ye may be clean from me the LORD thy God, that I may deliver thee into the land which the LORD thy God giveth thee for an inheritance among their houses the sons of Ahisaram, the family of the Azramites: of Arnon, the family of the Amorites: of Arnon, the family of the Amorites: of Arnon, the family of the Amorites: of Arnon, the family of the Amorites: of Arnon, the family of the Amorites: of Arnon, the\n",
      "\n",
      "Epoch: 10\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 298s 14ms/step - loss: 0.7654\n",
      "y the cattle of the LORD thy God is a mercy seat of the same, according to their families, the city of the house of their fathers, according to the number of the names, from twenty years old and upward, all that were able to go\n",
      "forth to war; 1:15 Those that were numbered of them, were threescore and ten thousand and four hundred.\n",
      "\n",
      "2:20 Of the children of Ephraim shall be the strangers that sojourn among you, and consumed them there was no water for the sons of Joseph, the son of Jephunneh the LO\n",
      "\n",
      "Epoch: 11\n",
      "\n",
      "Epoch 1/1\n",
      "21397/21397 [==============================] - 330s 15ms/step - loss: 0.7276\n",
      "f the LORD said unto Moses, Stretch out thine hand upon the head of the goats for a sin offering: 7:55 And for a sacrifice of peace offerings, two oxen, five rams, five he goats, five lambs of the first year without blemish: 29:20 And the priest shall take the brasen cornen conten ceraen in the hand of the children of Israel, that they should be a priest unto the LORD thy God, thou shalt not come nigh unto thee, that thou mayest bring the children of Israel shall be the length thereof, and a cub\n",
      "\n",
      "Epoch: 12\n",
      "\n",
      "Epoch 1/1\n",
      "15650/21397 [====================>.........] - ETA: 1:27 - loss: 0.6849"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import time\n",
    "import csv\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Activation, Dropout\n",
    "from keras.layers.recurrent import LSTM, SimpleRNN\n",
    "from keras.layers.wrappers import TimeDistributed\n",
    "#import argparse\n",
    "\n",
    "# Parsing arguments for Network definition\n",
    "#ap = argparse.ArgumentParser()\n",
    "#ap.add_argument('-data_dir', default=)\n",
    "#ap.add_argument('-batch_size', type=int, default=50)\n",
    "#ap.add_argument('-layer_num', type=int, default=2)\n",
    "#ap.add_argument('-seq_length', type=int, default=50)\n",
    "#ap.add_argument('-hidden_dim', type=int, default=500)\n",
    "#ap.add_argument('-generate_length', type=int, default=500)\n",
    "   #ap.add_argument('-nb_epoch', type=int, default=20)\n",
    "#ap.add_argument('-mode', default='train')\n",
    "#ap.add_argument('-weights', default='')\n",
    "#args = vars(ap.parse_args())\n",
    "\n",
    "DATA_DIR = './data/test.txt'\n",
    "BATCH_SIZE = 50\n",
    "HIDDEN_DIM = 500\n",
    "SEQ_LENGTH = int(50)\n",
    "WEIGHTS = ''\n",
    "MODE = 'train'\n",
    "GENERATE_LENGTH = 500\n",
    "LAYER_NUM = 2\n",
    "\n",
    "# Creating training data\n",
    "X, y, VOCAB_SIZE, ix_to_char = load_data(DATA_DIR, SEQ_LENGTH)\n",
    "\n",
    "# Creating and compiling the Network\n",
    "model = Sequential()\n",
    "model.add(LSTM(HIDDEN_DIM, input_shape=(None, VOCAB_SIZE), return_sequences=True))\n",
    "for i in range(LAYER_NUM - 1):\n",
    "  model.add(LSTM(HIDDEN_DIM, return_sequences=True))\n",
    "model.add(TimeDistributed(Dense(VOCAB_SIZE)))\n",
    "model.add(Activation('softmax'))\n",
    "model.compile(loss=\"categorical_crossentropy\", optimizer=\"rmsprop\")\n",
    "\n",
    "# Generate some sample before training to know how bad it is!\n",
    "generate_text(model, GENERATE_LENGTH, VOCAB_SIZE, ix_to_char)\n",
    "\n",
    "if not WEIGHTS == '':\n",
    "  model.load_weights(WEIGHTS)\n",
    "  nb_epoch = int(WEIGHTS[WEIGHTS.rfind('_') + 1:WEIGHTS.find('.')])\n",
    "else:\n",
    "  nb_epoch = 0\n",
    "\n",
    "# Training if there is no trained weights specified\n",
    "if MODE == 'train' or WEIGHTS == '':\n",
    "  while True:\n",
    "    print('\\n\\nEpoch: {}\\n'.format(nb_epoch))\n",
    "    model.fit(X, y, batch_size=BATCH_SIZE, verbose=1, nb_epoch=1)\n",
    "    nb_epoch += 1\n",
    "    generate_text(model, GENERATE_LENGTH, VOCAB_SIZE, ix_to_char)\n",
    "    if nb_epoch % 10 == 0:\n",
    "      model.save_weights('checkpoint_layer_{}_hidden_{}_epoch_{}.hdf5'.format(LAYER_NUM, HIDDEN_DIM, nb_epoch))\n",
    "\n",
    "# Else, loading the trained weights and performing generation only\n",
    "elif WEIGHTS == '':\n",
    "  # Loading the trained weights\n",
    "  model.load_weights(WEIGHTS)\n",
    "  generate_text(model, GENERATE_LENGTH, VOCAB_SIZE, ix_to_char)\n",
    "  print('\\n\\n')\n",
    "else:\n",
    "  print('\\n\\nNothing to do!')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}