similar_intersection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(pd.read_csv(\"./data/minimal.50d.3f.csv\")).set_index(\"0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "words = df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def find_most_similar(word, n=5):\n",
    "    dotted = dfn.dot(dfn.loc[word])\n",
    "    return dotted.reindex(dotted.abs().sort_values(ascending = False).index).head(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lens = (df**2).sum(axis=1)\n",
    "dfn = df.div(np.sqrt(lens), axis='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "simil1 = find_most_similar(\"duck\", 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "simil2 = find_most_similar(\"animal\", 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>rabbit</th>\n",
       "      <td>0.745679</td>\n",
       "      <td>0.576384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pig</th>\n",
       "      <td>0.739566</td>\n",
       "      <td>0.735967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat</th>\n",
       "      <td>0.703084</td>\n",
       "      <td>0.684202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dog</th>\n",
       "      <td>0.693232</td>\n",
       "      <td>0.725183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>goat</th>\n",
       "      <td>0.681624</td>\n",
       "      <td>0.566893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>monkey</th>\n",
       "      <td>0.677947</td>\n",
       "      <td>0.637051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fish</th>\n",
       "      <td>0.676006</td>\n",
       "      <td>0.728696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bite</th>\n",
       "      <td>0.673226</td>\n",
       "      <td>0.592544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>snake</th>\n",
       "      <td>0.667508</td>\n",
       "      <td>0.567578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rat</th>\n",
       "      <td>0.661683</td>\n",
       "      <td>0.597974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eaten</th>\n",
       "      <td>0.657430</td>\n",
       "      <td>0.559729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>meat</th>\n",
       "      <td>0.656034</td>\n",
       "      <td>0.675625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shark</th>\n",
       "      <td>0.653056</td>\n",
       "      <td>0.646871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bird</th>\n",
       "      <td>0.643410</td>\n",
       "      <td>0.800332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wild</th>\n",
       "      <td>0.630795</td>\n",
       "      <td>0.657390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>whale</th>\n",
       "      <td>0.629061</td>\n",
       "      <td>0.687003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eating</th>\n",
       "      <td>0.619378</td>\n",
       "      <td>0.714795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eat</th>\n",
       "      <td>0.612636</td>\n",
       "      <td>0.584034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bites</th>\n",
       "      <td>0.606200</td>\n",
       "      <td>0.541364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>frog</th>\n",
       "      <td>0.604737</td>\n",
       "      <td>0.535857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cow</th>\n",
       "      <td>0.589635</td>\n",
       "      <td>0.722633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mouth</th>\n",
       "      <td>0.587042</td>\n",
       "      <td>0.573372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pigs</th>\n",
       "      <td>0.579878</td>\n",
       "      <td>0.725978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alligator</th>\n",
       "      <td>0.578257</td>\n",
       "      <td>0.538110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>droppings</th>\n",
       "      <td>0.577192</td>\n",
       "      <td>0.573154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>elephant</th>\n",
       "      <td>0.576257</td>\n",
       "      <td>0.723775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>turtle</th>\n",
       "      <td>0.573822</td>\n",
       "      <td>0.544670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cats</th>\n",
       "      <td>0.573498</td>\n",
       "      <td>0.681642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pet</th>\n",
       "      <td>0.568641</td>\n",
       "      <td>0.747253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>birds</th>\n",
       "      <td>0.561430</td>\n",
       "      <td>0.719805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mushroom</th>\n",
       "      <td>0.558664</td>\n",
       "      <td>0.538530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eggs</th>\n",
       "      <td>0.558094</td>\n",
       "      <td>0.575842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hog</th>\n",
       "      <td>0.556495</td>\n",
       "      <td>0.542437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>snakes</th>\n",
       "      <td>0.547561</td>\n",
       "      <td>0.586354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dogs</th>\n",
       "      <td>0.546733</td>\n",
       "      <td>0.758686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mouse</th>\n",
       "      <td>0.545863</td>\n",
       "      <td>0.594525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ostrich</th>\n",
       "      <td>0.539546</td>\n",
       "      <td>0.538319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deer</th>\n",
       "      <td>0.536599</td>\n",
       "      <td>0.546471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hunts</th>\n",
       "      <td>0.532684</td>\n",
       "      <td>0.542579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0         1\n",
       "0                            \n",
       "rabbit     0.745679  0.576384\n",
       "pig        0.739566  0.735967\n",
       "cat        0.703084  0.684202\n",
       "dog        0.693232  0.725183\n",
       "goat       0.681624  0.566893\n",
       "monkey     0.677947  0.637051\n",
       "fish       0.676006  0.728696\n",
       "bite       0.673226  0.592544\n",
       "snake      0.667508  0.567578\n",
       "rat        0.661683  0.597974\n",
       "eaten      0.657430  0.559729\n",
       "meat       0.656034  0.675625\n",
       "shark      0.653056  0.646871\n",
       "bird       0.643410  0.800332\n",
       "wild       0.630795  0.657390\n",
       "whale      0.629061  0.687003\n",
       "eating     0.619378  0.714795\n",
       "eat        0.612636  0.584034\n",
       "bites      0.606200  0.541364\n",
       "frog       0.604737  0.535857\n",
       "cow        0.589635  0.722633\n",
       "mouth      0.587042  0.573372\n",
       "pigs       0.579878  0.725978\n",
       "alligator  0.578257  0.538110\n",
       "droppings  0.577192  0.573154\n",
       "elephant   0.576257  0.723775\n",
       "turtle     0.573822  0.544670\n",
       "cats       0.573498  0.681642\n",
       "pet        0.568641  0.747253\n",
       "birds      0.561430  0.719805\n",
       "mushroom   0.558664  0.538530\n",
       "eggs       0.558094  0.575842\n",
       "hog        0.556495  0.542437\n",
       "snakes     0.547561  0.586354\n",
       "dogs       0.546733  0.758686\n",
       "mouse      0.545863  0.594525\n",
       "ostrich    0.539546  0.538319\n",
       "deer       0.536599  0.546471\n",
       "hunts      0.532684  0.542579"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([simil1, simil2], join='inner', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0\n",
       "duck         1.000000\n",
       "crab         0.775715\n",
       "lobster      0.761955\n",
       "lame         0.747115\n",
       "rabbit       0.745679\n",
       "pig          0.739566\n",
       "goose        0.735945\n",
       "chicken      0.725280\n",
       "grilled      0.722480\n",
       "fried        0.712456\n",
       "shrimp       0.707160\n",
       "cat          0.703084\n",
       "dog          0.693232\n",
       "goat         0.681624\n",
       "monkey       0.677947\n",
       "confit       0.677671\n",
       "fish         0.676006\n",
       "bite         0.673226\n",
       "broiled      0.670938\n",
       "boar         0.670674\n",
       "lamb         0.668923\n",
       "snake        0.667508\n",
       "rat          0.661683\n",
       "tails        0.658660\n",
       "sandwich     0.658085\n",
       "eaten        0.657430\n",
       "meat         0.656034\n",
       "salmon       0.655644\n",
       "crabs        0.655578\n",
       "roast        0.654873\n",
       "               ...   \n",
       "crispy       0.540386\n",
       "retriever    0.539618\n",
       "ostrich      0.539546\n",
       "mullet       0.539332\n",
       "halibut      0.538470\n",
       "wag          0.537798\n",
       "candy        0.537739\n",
       "appetizer    0.537733\n",
       "oysters      0.537285\n",
       "crayfish     0.537130\n",
       "zfc         -0.536828\n",
       "deer         0.536599\n",
       "worm         0.535902\n",
       "badger       0.535853\n",
       "lunch        0.535725\n",
       "ears         0.535670\n",
       "bourdelle   -0.535635\n",
       "shaped       0.535565\n",
       "punch        0.534654\n",
       "beaks        0.534613\n",
       "dinner       0.534534\n",
       "lavine      -0.534397\n",
       "wagging      0.534372\n",
       "raccoon      0.534007\n",
       "steaks       0.533863\n",
       "jussieu     -0.533810\n",
       "hairy        0.533809\n",
       "hook         0.533270\n",
       "curried      0.533255\n",
       "hunts        0.532684\n",
       "Length: 200, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simil1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}