Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text corpus to generative text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Embedding, LSTM, Dense, Dropout\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import Sequential\n",
    "import keras.utils as ku \n",
    "import numpy as np\n",
    "\n",
    "from keras.utils import plot_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochSize = 6\n",
    "hidden_size = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dataset_preparation(data):\n",
    "    \n",
    "    # splits corpus on return - change if you want a different split\n",
    "    corpus = data.lower().split(\"^\") \n",
    "    \n",
    "    tokenizer.fit_on_texts(corpus)\n",
    "    total_words = len(tokenizer.word_index) + 1\n",
    "    \n",
    "    input_sequences = []\n",
    "    for line in corpus:\n",
    "        token_list = tokenizer.texts_to_sequences([line])[0]\n",
    "        for i in range(1, len(token_list)):\n",
    "            n_gram_sequence = token_list[:i+1]\n",
    "            input_sequences.append(n_gram_sequence)\n",
    "            \n",
    "    max_sequence_len = max([len(x) for x in input_sequences])\n",
    "    input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))\n",
    "    \n",
    "    predictors, label = input_sequences[:,:-1],input_sequences[:,-1]\n",
    "    label = ku.to_categorical(label, num_classes=total_words)\n",
    "    \n",
    "    return predictors, label, max_sequence_len, total_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(predictors, label, max_sequence_len, total_words):\n",
    "    \n",
    "    #model structure\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(total_words, hidden_size, input_length=max_sequence_len-1, name='embeddingINPUT'))\n",
    "\n",
    "    # 2 hidden layers of LSTM with size: hidden_size\n",
    "    model.add(LSTM(hidden_size, use_bias=False, dropout=0.1, return_sequences = True, name='hiddenLSTM1'))\n",
    "    model.add(LSTM(hidden_size, use_bias=False, name='hiddenLSTM2'))\n",
    "    \n",
    "    model.add(Dense(total_words, activation='softmax', name='denseOUTPUT'))\n",
    "\n",
    "    \n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "    earlystop = EarlyStopping(monitor='loss', min_delta=0, patience=5, verbose=0, mode='auto')\n",
    "    \n",
    "    model.fit(predictors, label, epochs=epochSize, verbose=1, callbacks=[earlystop])\n",
    "    return model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_text(seed_text, next_words, max_sequence_len):\n",
    "    for _ in range(next_words):\n",
    "        token_list = tokenizer.texts_to_sequences([seed_text])[0]\n",
    "        token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')\n",
    "        predicted = model.predict_classes(token_list, verbose=0)\n",
    "\n",
    "        output_word = \"\"\n",
    "        for word, index in tokenizer.word_index.items():\n",
    "            if index == predicted:\n",
    "                output_word = word\n",
    "                break\n",
    "        seed_text += \" \" + output_word\n",
    "    return seed_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = open('data.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictors, label, max_sequence_len, total_words = dataset_preparation(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n"
     ]
    }
   ],
   "source": [
    "model = create_model(predictors, label, max_sequence_len, total_words)\n",
    "#plot_model(model, to_file='model.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = generate_text(\"The\", 30, max_sequence_len)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot_model(model, to_file='model.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "So alignment highly demonstrate that the potential to improve outcomes enhance\n"
     ]
    }
   ],
   "source": [
    "output = generate_text(\"So\", 10, max_sequence_len)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AI and social good ai offers great potential for promoting the public good for example in the realms of education housing public health and sustainability we see great value in collaborating with public and private organizations including academia scientific societies ngos social entrepreneurs and interested private citizens to promote discussions and catalyze efforts to address society’s most pressing challenges some of these projects may address deep societal challenges and will be moonshots – ambitious big bets that could have far reaching impacts others may be creative ideas that could quickly produce positive results by harnessing ai advances are widely shared and\n"
     ]
    }
   ],
   "source": [
    "output = generate_text(\"AI\", 100, max_sequence_len)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "496\n"
     ]
    }
   ],
   "source": [
    "print(max_sequence_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Goals and developing is their pricing by humans if we are providing general purpose tools integrating tools for customers or developing custom solutions for this are the benefits and potential risks of their products and the actions they have taken to deliver benefits and avoid minimise or mitigate the risks they must ensure that processes are in place to address the concerns and complaints of users and other parties and that these are transparent we believe that effective communication when coupled with a principled approach to ethical considerations is a competitive advantage and will lead to progress even when hard moral\n"
     ]
    }
   ],
   "source": [
    "output = generate_text(\"Goals\", 100, max_sequence_len)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}