ml-ideas-generator-on-pretrained.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('all_ml_ideas.csv', header=None, names=['idea'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['idea'] = df['idea'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>predicting freeway congestion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>supervised learning of query term relevant pro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>robotic arm control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>network intrusion detection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multi-website name coreference resolution</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                idea\n",
       "0                      predicting freeway congestion\n",
       "1  supervised learning of query term relevant pro...\n",
       "2                                robotic arm control\n",
       "3                        network intrusion detection\n",
       "4          multi-website name coreference resolution"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_pct = 0.05 #validation percent\n",
    "df = df.iloc[np.random.permutation(len(df))]\n",
    "cut = int(valid_pct * len(df)) + 1\n",
    "train_df, valid_df = df[cut:], df[:cut]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1449"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1376"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "73"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(valid_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "nan_rows = df[df['idea'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [idea]\n",
       "Index: []"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nan_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = TextLMDataBunch.from_df('data', train_df, valid_df, text_cols='idea')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data_clas = TextClasDataBunch.from_df('data', train_df, valid_df, text_cols='text', label_cols='author', vocab=data_lm.train_ds.vocab, bs=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:06 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>6.141744</th>\n",
       "    <th>4.721056</th>\n",
       "    <th>0.357955</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = language_model_learner(data_lm, pretrained_model=URLs.WT103, drop_mult=0.5)\n",
    "learn.fit_one_cycle(1, 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:31 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>5.398848</th>\n",
       "    <th>4.056406</th>\n",
       "    <th>0.396307</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(1, 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd=1e-7\n",
    "lr=1e-3\n",
    "lrs = lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 32:00 <p><table style='width:300px; margin-bottom:10px'>\n",
       "  <tr>\n",
       "    <th>epoch</th>\n",
       "    <th>train_loss</th>\n",
       "    <th>valid_loss</th>\n",
       "    <th>accuracy</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>1</th>\n",
       "    <th>4.579183</th>\n",
       "    <th>3.815258</th>\n",
       "    <th>0.396307</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>2</th>\n",
       "    <th>4.418990</th>\n",
       "    <th>3.777051</th>\n",
       "    <th>0.383523</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>3</th>\n",
       "    <th>4.262864</th>\n",
       "    <th>3.499419</th>\n",
       "    <th>0.413352</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>4</th>\n",
       "    <th>4.161328</th>\n",
       "    <th>3.427666</th>\n",
       "    <th>0.421875</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>5</th>\n",
       "    <th>4.073437</th>\n",
       "    <th>3.516651</th>\n",
       "    <th>0.410511</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>6</th>\n",
       "    <th>3.998420</th>\n",
       "    <th>3.429785</th>\n",
       "    <th>0.414773</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>7</th>\n",
       "    <th>3.928021</th>\n",
       "    <th>3.367995</th>\n",
       "    <th>0.428977</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>8</th>\n",
       "    <th>3.867587</th>\n",
       "    <th>3.359958</th>\n",
       "    <th>0.427557</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>9</th>\n",
       "    <th>3.810681</th>\n",
       "    <th>3.377237</th>\n",
       "    <th>0.437500</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>10</th>\n",
       "    <th>3.755445</th>\n",
       "    <th>3.365312</th>\n",
       "    <th>0.430398</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>11</th>\n",
       "    <th>3.704025</th>\n",
       "    <th>3.320275</th>\n",
       "    <th>0.426136</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>12</th>\n",
       "    <th>3.652749</th>\n",
       "    <th>3.308072</th>\n",
       "    <th>0.430398</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>13</th>\n",
       "    <th>3.603297</th>\n",
       "    <th>3.322817</th>\n",
       "    <th>0.424716</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>14</th>\n",
       "    <th>3.557924</th>\n",
       "    <th>3.288016</th>\n",
       "    <th>0.430398</th>\n",
       "  </tr>\n",
       "  <tr>\n",
       "    <th>15</th>\n",
       "    <th>3.511880</th>\n",
       "    <th>3.283366</th>\n",
       "    <th>0.431818</th>\n",
       "  </tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(15,lrs, wd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_of_ideas = 100\n",
    "ideas_counter = 0\n",
    "all_ideas = []\n",
    "\n",
    "for i in range(100):\n",
    "    idea = learn.predict(\"xxbos xxfld 1\", n_words=20, temperature=0.8)\n",
    "    ideas = idea.split(\"xxbos xxfld 1\")\n",
    "    ideas = ideas[1:-1]\n",
    "    \n",
    "    for idea in ideas:\n",
    "        idea = idea.replace(\"xxbos xxfld 1 \",\"\").strip()\n",
    "        if(idea):\n",
    "            all_ideas.append(idea)\n",
    "            ideas_counter = ideas_counter+1\n",
    "            \n",
    "    if ideas_counter > number_of_ideas:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['the problem is right : grasping and extracting the face of pose',\n",
       " 'applying machine learning to text treatment',\n",
       " 'machine learning techniques for learning through machine learning',\n",
       " 'a machine learning approach to predicting career success from a single object',\n",
       " 'using machine learning to predict the outcome of a machine learning approach',\n",
       " 'based on stock prices',\n",
       " 'identifying stock price models for machine learning techniques',\n",
       " 'a study in time travel time series analysis of musical features',\n",
       " 'vectors in the amazon impact network',\n",
       " 'classification of web articles in facebook',\n",
       " 'dynamic signal processing in post - secondary order data',\n",
       " 'copy selection with machine learning techniques',\n",
       " 'interpretation of user classification',\n",
       " 'the application of deep learning to fairness in using a semantic framework',\n",
       " \"creating a different entity 's portfolio\",\n",
       " 'using supervised learning of blind data',\n",
       " 'system classification for driving automatic vehicle design and classification with gene expression',\n",
       " 'based on public documents from text expression',\n",
       " 'semantic learning for music',\n",
       " 'machine learning for cancer prediction',\n",
       " 'learning static variations with deep learning for learning options',\n",
       " 'image classification for svm',\n",
       " 'satellite imagery classification',\n",
       " 'making decision selection from a single object',\n",
       " 'object detection using product preferences',\n",
       " 'speech detection with deep learning',\n",
       " 'genomic data based on stock trading',\n",
       " 'learning to predict approach to handwriting',\n",
       " 'classification of musical features from the composer data',\n",
       " 'semantic social network and smartphone features',\n",
       " 'machine learning techniques',\n",
       " 'using real - time information to predict the popularity of the market',\n",
       " 'video game classification',\n",
       " 'a learning task for time series players',\n",
       " 'using a single machine learning approach for a single learning approach to learning to identify other environments',\n",
       " 'multiple - genre classification of fraud \\n prediction for a mass neural network',\n",
       " 'learning of human activity recognition from analysis of text',\n",
       " \"an nba player 's approach to learning and character forecasting through video game ecg\",\n",
       " 'playing a vocal instrument in local mri learning',\n",
       " 'real - time music recordings',\n",
       " 'finding new artistic and artistic features in music videos',\n",
       " 'an analysis of musical genres',\n",
       " 'predicting a single image - specific musical style',\n",
       " 'a cost approach to crime prediction',\n",
       " 'automatic user prediction and automated review recognition',\n",
       " 'food processing via machine learning',\n",
       " 'human activity recognition using multi - label fantasy',\n",
       " 'predicting a match in the keystroke poker',\n",
       " 'estimation of game types',\n",
       " 'ai identification of deep learning in locomotion monitoring using neural networks',\n",
       " 'the value of collaborative attention projecting for real - time playing',\n",
       " 'the sea level and low speed : the two waves',\n",
       " 'learning to predict the price of beer and personal genomes',\n",
       " 'trading and removing a novel image from the text',\n",
       " 'real - time news user identification on google gestures',\n",
       " 'removing and re - learning to play game and lyrics',\n",
       " 'rapid - mass dynamics with acoustic images',\n",
       " 'real - time music direction',\n",
       " \"what 's your right ?\",\n",
       " 'exploring event and music',\n",
       " 'human activity prediction using machine learning',\n",
       " 'model of architecture in california',\n",
       " 'vs light crime',\n",
       " 'adaptive learning for image recognition',\n",
       " 'predicting the approach of human activity using machine learning',\n",
       " 'the win given trajectories',\n",
       " 'a machine learning approach to online design',\n",
       " 'a massive based multi - layer feature unsupervised approach for multi - agent music',\n",
       " 'can you learn from a single hand',\n",
       " 'reaction with the media',\n",
       " 'measurement of time to order over time',\n",
       " 'how people can stop : learning the objects of blood and blood',\n",
       " 'machine learning for autonomous vehicles',\n",
       " 'vehicle types in neural networks',\n",
       " 'building a model for what does it store ?',\n",
       " 'for enhanced identification of machine learning techniques',\n",
       " \"exploring new york city 's public image through machine learning\",\n",
       " 'a novel approach to career image recognition',\n",
       " 'in general game playing',\n",
       " 'structure classification for adaptation of text',\n",
       " 'a variance learning approach for speech recognition',\n",
       " 'the optimization of a non - peer temporal layer',\n",
       " \"a distinguishing feature of a song 's legal expression\",\n",
       " 'learning to sound in english : learning to learn using word learning',\n",
       " 'information sharing with adaptive neural networks',\n",
       " 'playing the game with multi - touch neural networks',\n",
       " 'recursive estimation of dynamic and static images',\n",
       " 'predicting the quality of the net - style result in the media',\n",
       " 'the character of the sea snake robot',\n",
       " 'predicting the stock market price of machine learning',\n",
       " 'using inverted nucleotide data to predict the price of convolutional protein models',\n",
       " 'search engine',\n",
       " 'using twitter data to predict prices in high - cost trading',\n",
       " 'a machine learning approach',\n",
       " 'creating a new approach to building a deep learning approach',\n",
       " 'fingerprint learning component',\n",
       " 'machine learning techniques for functional change learning for the building of new york city college football networks',\n",
       " 'predicting cancer risk of breast cancer risk',\n",
       " 'cancer diagnosis and prediction',\n",
       " 'stock market classification',\n",
       " 'identifying the outcome of the news media']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_ideas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('ml_ft_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_pickle('data/ml_train_df.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_df.to_pickle('data/ml_valid_df.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}