results.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "final.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UYbtEG-INE1y",
        "outputId": "5befbcdc-5765-4110-aeb2-09256abe6cf6"
      },
      "source": [
        "from google.colab import drive\n",
        "# drive.mount('/content/gdrive')\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qgSsaAiOMbAP",
        "outputId": "5680462f-9c5e-47d5-b908-47d837b4c311"
      },
      "source": [
        "!pip install jsonpickle"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting jsonpickle\n",
            "  Downloading https://files.pythonhosted.org/packages/bb/1a/f2db026d4d682303793559f1c2bb425ba3ec0d6fd7ac63397790443f2461/jsonpickle-2.0.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from jsonpickle) (3.10.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->jsonpickle) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->jsonpickle) (3.4.1)\n",
            "Installing collected packages: jsonpickle\n",
            "Successfully installed jsonpickle-2.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oEiIKqLR9L3E",
        "outputId": "0c3074f4-c0c9-4d94-d649-426953381ec5"
      },
      "source": [
        "!pip install rake_nltk"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting rake_nltk\n",
            "  Downloading https://files.pythonhosted.org/packages/8e/c4/b4ff57e541ac5624ad4b20b89c2bafd4e98f29fd83139f3a81858bdb3815/rake_nltk-1.0.4.tar.gz\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from rake_nltk) (3.2.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk->rake_nltk) (1.15.0)\n",
            "Building wheels for collected packages: rake-nltk\n",
            "  Building wheel for rake-nltk (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for rake-nltk: filename=rake_nltk-1.0.4-py2.py3-none-any.whl size=7819 sha256=8cd513c47dfe4887fe867c94adc425ba34080859ec4ac535e4926032dd5b33b4\n",
            "  Stored in directory: /root/.cache/pip/wheels/ef/92/fc/271b3709e71a96ffe934b27818946b795ac6b9b8ff8682483f\n",
            "Successfully built rake-nltk\n",
            "Installing collected packages: rake-nltk\n",
            "Successfully installed rake-nltk-1.0.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ENJRWMWpTawV"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import glob\n",
        "import json\n",
        "import jsonpickle \n",
        "import math\n",
        "import nltk\n",
        "import pandas as pd\n",
        "import pathlib\n",
        "import numpy as np # linear algebra\n",
        "from itertools import combinations \n",
        "from collections import defaultdict \n",
        "from nltk.corpus import wordnet as wn\n",
        "from nltk import word_tokenize\n",
        "from scipy import spatial\n",
        "from nltk.metrics import edit_distance\n",
        "from collections import defaultdict "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F7oGf1zrMhuR",
        "outputId": "8f6eb133-a224-47c3-e15b-2f4c223cf745"
      },
      "source": [
        "nltk.download('stopwords')  \n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xpLlCERGyRFz"
      },
      "source": [
        "#Node structure for graph\n",
        "class Node:\n",
        "\n",
        "    def __init__(self,src,dest,wt):\n",
        "        self.src = src\n",
        "        self.dest = dest\n",
        "        self.wt = wt"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4T4SZl5iyXYe"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import glob\n",
        "import json\n",
        "import jsonpickle \n",
        "import math\n",
        "import nltk\n",
        "import pandas as pd\n",
        "import pathlib\n",
        "import numpy as np # linear algebra\n",
        "from itertools import combinations \n",
        "from collections import defaultdict \n",
        "from nltk.corpus import wordnet as wn\n",
        "from nltk import word_tokenize\n",
        "from scipy import spatial\n",
        "from nltk.metrics import edit_distance\n",
        "from collections import defaultdict \n",
        "# from WordNet import WordNet\n",
        "# from Node import Node\n",
        "\n",
        "#Class to represent an un-directed graph using adjacency list representation \n",
        "class Graph: \n",
        "   \n",
        "    def __init__(self,vertices): \n",
        "        self.V = vertices #No. of vertices \n",
        "        self.V_org = vertices \n",
        "        self.graph = defaultdict(list) # default dictionary to store graph \n",
        "        \n",
        "        \n",
        "    # function to add an edge to graph \n",
        "    def addEdge(self,u,v,w): \n",
        "        self.graph[u].append(Node(u,v,w))\n",
        "        self.graph[v].append(Node(v,u,w))\n",
        "\n",
        "        \n",
        "    #function to print graph\n",
        "    def printGraph(self):\n",
        "        s = \"\"\n",
        "        for i in self.graph:\n",
        "            s = s + str(i) + \" is connected to \"\n",
        "            print(str(i) + \" is connected to \")\n",
        "            for node in self.graph[i]:\n",
        "                s = s + str(node.dest) + \"(Weight = \" + str(node.wt) + \")\" + \" \"\n",
        "                print(str(node.dest) + \"(Weight = \" + str(node.wt) + \")\" + \" \")\n",
        "            s = s + \"\\n\"\n",
        "            print(\"\\n\")\n",
        "        return s\n",
        "\n",
        "    \n",
        "    #function to get BFS results for a given node till the given level\n",
        "    def BFS(self, s, max_levels):\n",
        "        visited = set()\n",
        " \n",
        "        queue = []\n",
        " \n",
        "        queue.append(s)\n",
        "        visited.add(s)\n",
        "        level = 0\n",
        "        result = {}\n",
        "        while queue:\n",
        "            aux = []\n",
        "            result[level] = []\n",
        "            \n",
        "            while queue:\n",
        "                s = queue.pop(0)\n",
        "                visited.add(s)\n",
        "                result[level].append(s)\n",
        "                for node in self.graph[s]:\n",
        "                    if node.dest not in visited:\n",
        "                        aux.append(node.dest)\n",
        "                        visited.add(node.dest)\n",
        "            level += 1\n",
        "            if level > max_levels:\n",
        "                break\n",
        "            for node in aux:\n",
        "                queue.append(node)\n",
        "            \n",
        "        return result   \n",
        "\n",
        "    def BFSi(self, s, max_levels):\n",
        "        visited = set()\n",
        "         \n",
        "        queue = []\n",
        "        wordNet = WordNet()\n",
        "        queue.append((s,0,0,1))\n",
        "        visited.add(s)\n",
        "        level = 0\n",
        "        result = {}\n",
        "        while queue:\n",
        "            aux = []\n",
        "            result[level] = []\n",
        "            \n",
        "            while queue:\n",
        "                s = queue.pop(0)\n",
        "                visited.add(s[0])\n",
        "                result[level].append(s)\n",
        "                for node in self.graph[s[0]]:\n",
        "                    if node.dest not in visited:\n",
        "                        \n",
        "#                         Wordnet Similarity\n",
        "                        q1 = wordNet.clean_sentence(s[0])\n",
        "                        q2 = wordNet.clean_sentence(node.dest)\n",
        "                        sim = 0\n",
        "                        sim = wordNet.semanticSimilarity(q1, q2)\n",
        "\n",
        "                        sumOfCooccurence = 0\n",
        "                        for chi in self.graph[node.dest]:\n",
        "                            if chi.dest in visited:\n",
        "                                sumOfCooccurence += chi.wt\n",
        "                        aux.append((node.dest, sumOfCooccurence, level+1, sim))        \n",
        "                        visited.add(node.dest)\n",
        "            level += 1\n",
        "            if level > max_levels:\n",
        "                break\n",
        "            for node in aux:\n",
        "                queue.append(node)\n",
        "            solution = []\n",
        "            for key in result:\n",
        "                for tup in result[key]:\n",
        "                    if tup[2] != 0 and ((tup[1] + np.exp(tup[3]))/np.exp(tup[2])) >= 2:\n",
        "                        solution.append( ( tup[0], (tup[1] + np.exp(tup[3]))/np.exp(tup[2]) ) )\n",
        "        return solution            \n",
        "    \n",
        "    def exportNetwork(self, filename = \"output\"):\n",
        "        filename += \".json\"\n",
        "        obj = jsonpickle.encode(self.graph)\n",
        "        with open(filename, \"w\") as outfile: \n",
        "            json.dump(obj, outfile)\n",
        "\n",
        "    def importNetwork(self, filename = \"output\"):\n",
        "        filename += \".json\"\n",
        "        with open(filename) as json_file:\n",
        "            data = json.load(json_file)\n",
        "            self.graph = jsonpickle.decode(data)\n",
        "            self.V = len(self.graph)\n",
        "            self.V_org = len(self.graph)"
      ],
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N78rp3h5y3CJ"
      },
      "source": [
        "import pandas as pd\n",
        "import pathlib\n",
        "\n",
        "class DataHandler:\n",
        "    def __init__(self, path):\n",
        "        self.type = pathlib.Path(path).suffix\n",
        "        self.dataset_location = path\n",
        "        \n",
        "        #Use the pandas dataframe to get columns \n",
        "        if self.type == '.csv':\n",
        "            self.df = pd.read_csv(path)\n",
        "            self.df = self.df[self.df['Domain'] == \"CS \"]\n",
        "        elif self.type == '.xlsx':\n",
        "            self.df = pd.read_excel(path)\n",
        "            self.df = self.df[self.df['Domain'] == \"CS \"]\n",
        "\n",
        "    def getDataframe(self):\n",
        "        return self.df"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HTydGKAYy6f9"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import glob\n",
        "import json\n",
        "import jsonpickle \n",
        "import math\n",
        "import nltk\n",
        "import pandas as pd\n",
        "import pathlib\n",
        "import numpy as np # linear algebra\n",
        "from itertools import combinations \n",
        "from collections import defaultdict \n",
        "from nltk.corpus import wordnet as wn\n",
        "from nltk import word_tokenize\n",
        "from scipy import spatial\n",
        "from nltk.metrics import edit_distance\n",
        "from collections import defaultdict \n",
        "# from DataHandler import DataHandler\n",
        "# from Graph import Graph\n",
        "\n",
        "class PredictPaper:\n",
        "    def __init__(self, dataPath, graphPath):\n",
        "        self.dh = DataHandler(dataPath)\n",
        "        self.dataset = self.dh.getDataframe()\n",
        "        self.dataset = self.dataset.reset_index()\n",
        "        self.g = Graph(10000)\n",
        "        self.g.importNetwork(graphPath)\n",
        "        self.querySet = []\n",
        "        self.result = {}\n",
        "\n",
        "\n",
        "    def predictUtil(self, queryKeyList):\n",
        "        for queryKey in queryKeyList:\n",
        "            queryKey = queryKey.strip().lower()\n",
        "            queryDict = self.g.BFS(queryKey, 3)\n",
        "            # print(queryDict)\n",
        "            self.querySet.extend([queryKey])\n",
        "            self.querySet.extend([item for sublist in queryDict.values() for item in sublist])\n",
        "            self.querySet = list(set(self.querySet))\n",
        "        # print(self.querySet)\n",
        "            # if queryKey not in self.querySet:\n",
        "            #   self.querySet.append(queryKey)\n",
        "            # for item in queryDict:\n",
        "            #   if item[0] not in self.querySet:\n",
        "            #     self.querySet.append(item[0])\n",
        "        # print(self.querySet)\n",
        "        return self.querySet\n",
        "\n",
        "\n",
        "    # Gives sorted dictionary result of papers and number of matching keywords \n",
        "    # between input keywords and rake extracted keywords from abstract\n",
        "    def predict(self, queryKeyList, commonWordsCount):\n",
        "        self.querySet = self.predictUtil(queryKeyList)\n",
        "        ran = len(self.dataset['Rake keywords'])\n",
        "        for i in range(2000):\n",
        "            val = self.dataset['Rake keywords'][i]\n",
        "            keywords = val.split(\";\")\n",
        "            com = self.calcCommon(keywords)\n",
        "            if com >= commonWordsCount:\n",
        "                self.result[i] = com\n",
        "        self.result = dict(sorted(self.result.items(), key = lambda kv:(kv[1], kv[0]), reverse = True))\n",
        "        for key in self.result:\n",
        "            self.result[key] = [self.result[key], self.dataset['area'][key].strip().lower()]\n",
        "        return self.result\n",
        "\n",
        "    def calcCommon(self, keywords):\n",
        "        cou = 0\n",
        "        for w1 in self.querySet:\n",
        "            for w2 in keywords:\n",
        "                if(w1.strip().lower() == w2.strip().lower()):\n",
        "                    cou+=1\n",
        "        return cou"
      ],
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydf8fhpm77E9"
      },
      "source": [
        "from rake_nltk import Metric, Rake\n",
        "\n",
        "class KeywordExtractor:\n",
        "    def __init__(self):\n",
        "        self.extractedKeywords = []\n",
        "\n",
        "    def rakealgo(self, abstract):\n",
        "        r = Rake(min_length=1, max_length=2)\n",
        "        r.extract_keywords_from_text(abstract)\n",
        "        result = r.get_ranked_phrases()\n",
        "        topResult = result[:10]\n",
        "        self.extractedKeywords.append(topResult)\n",
        "        return self.extractedKeywords"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhgF9w1A9XdM"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import glob\n",
        "import json\n",
        "import jsonpickle \n",
        "import math\n",
        "import nltk\n",
        "import pandas as pd\n",
        "import pathlib\n",
        "import numpy as np # linear algebra\n",
        "from itertools import combinations \n",
        "from collections import defaultdict \n",
        "from nltk.corpus import wordnet as wn\n",
        "from nltk import word_tokenize\n",
        "from scipy import spatial\n",
        "from nltk.metrics import edit_distance\n",
        "from collections import defaultdict \n",
        "\n",
        "\n",
        "class Lesk(object):\n",
        "\n",
        "    def __init__(self, sentence):\n",
        "        self.sentence = sentence\n",
        "        self.meanings = {}\n",
        "        for word in sentence:\n",
        "            self.meanings[word] = ''\n",
        "\n",
        "    def getSenses(self, word):\n",
        "        # print word\n",
        "        return wn.synsets(word.lower())\n",
        "\n",
        "    def getGloss(self, senses):\n",
        "\n",
        "        gloss = {}\n",
        "\n",
        "        for sense in senses:\n",
        "            gloss[sense.name()] = []\n",
        "\n",
        "        for sense in senses:\n",
        "            gloss[sense.name()] += word_tokenize(sense.definition())\n",
        "\n",
        "        return gloss\n",
        "\n",
        "    def getAll(self, word):\n",
        "        senses = self.getSenses(word)\n",
        "\n",
        "        if senses == []:\n",
        "            return {word.lower(): senses}\n",
        "\n",
        "        return self.getGloss(senses)\n",
        "\n",
        "    def Score(self, set1, set2):\n",
        "        # Base\n",
        "        overlap = 0\n",
        "\n",
        "        # Step\n",
        "        for word in set1:\n",
        "            if word in set2:\n",
        "                overlap += 1\n",
        "\n",
        "        return overlap\n",
        "\n",
        "    def overlapScore(self, word1, word2):\n",
        "\n",
        "        gloss_set1 = self.getAll(word1)\n",
        "        if self.meanings[word2] == '':\n",
        "            gloss_set2 = self.getAll(word2)\n",
        "        else:\n",
        "            # print 'here'\n",
        "            gloss_set2 = self.getGloss([wn.synset(self.meanings[word2])])\n",
        "\n",
        "        # print gloss_set2\n",
        "\n",
        "        score = {}\n",
        "        for i in gloss_set1.keys():\n",
        "            score[i] = 0\n",
        "            for j in gloss_set2.keys():\n",
        "                score[i] += self.Score(gloss_set1[i], gloss_set2[j])\n",
        "\n",
        "        bestSense = None\n",
        "        max_score = 0\n",
        "        for i in gloss_set1.keys():\n",
        "            if score[i] > max_score:\n",
        "                max_score = score[i]\n",
        "                bestSense = i\n",
        "\n",
        "        return bestSense, max_score\n",
        "\n",
        "    def lesk(self, word, sentence):\n",
        "        maxOverlap = 0\n",
        "        context = sentence\n",
        "        word_sense = []\n",
        "        meaning = {}\n",
        "\n",
        "        senses = self.getSenses(word)\n",
        "\n",
        "        for sense in senses:\n",
        "            meaning[sense.name()] = 0\n",
        "\n",
        "        for word_context in context:\n",
        "            if not word == word_context:\n",
        "                score = self.overlapScore(word, word_context)\n",
        "                if score[0] == None:\n",
        "                    continue\n",
        "                meaning[score[0]] += score[1]\n",
        "\n",
        "        if senses == []:\n",
        "            return word, None, None\n",
        "\n",
        "        self.meanings[word] = max(meaning.keys(), key=lambda x: meaning[x])\n",
        "\n",
        "        return word, self.meanings[word], wn.synset(self.meanings[word]).definition()"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jpqEBXx2MOOz"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import glob\n",
        "import json\n",
        "import math\n",
        "import nltk\n",
        "import pandas as pd\n",
        "import pathlib\n",
        "import numpy as np # linear algebra\n",
        "from itertools import combinations \n",
        "from collections import defaultdict \n",
        "from nltk.corpus import wordnet as wn\n",
        "from nltk import word_tokenize\n",
        "from scipy import spatial\n",
        "from nltk.metrics import edit_distance\n",
        "from collections import defaultdict \n",
        "# from Lesk import Lesk\n",
        "\n",
        "class WordNet:\n",
        "     \n",
        "    def __init__(self): \n",
        "        self.STOP_WORDS = nltk.corpus.stopwords.words()\n",
        "        \n",
        "    def tokenize(self, q1, q2):\n",
        "        return word_tokenize(q1), word_tokenize(q2)\n",
        "\n",
        "\n",
        "    def posTag(self, q1, q2):\n",
        "        return nltk.pos_tag(q1), nltk.pos_tag(q2)\n",
        "\n",
        "\n",
        "    def stemmer(self, tag_q1, tag_q2):\n",
        "        stem_q1 = []\n",
        "        stem_q2 = []\n",
        "\n",
        "        for token in tag_q1:\n",
        "            stem_q1.append(stem(token))\n",
        "\n",
        "        for token in tag_q2:\n",
        "            stem_q2.append(stem(token))\n",
        "\n",
        "        return stem_q1, stem_q2\n",
        "    \n",
        "    def path(self, set1, set2):\n",
        "        return wn.path_similarity(set1, set2)\n",
        "\n",
        "\n",
        "    def wup(self, set1, set2):\n",
        "        return wn.wup_similarity(set1, set2)\n",
        "\n",
        "\n",
        "    def edit(self, word1, word2):\n",
        "        if float(edit_distance(word1, word2)) == 0.0:\n",
        "            return 0.0\n",
        "        return 1.0 / float(edit_distance(word1, word2))\n",
        "\n",
        "    def computePath(self, q1, q2):\n",
        "\n",
        "        R = np.zeros((len(q1), len(q2)))\n",
        "\n",
        "        for i in range(len(q1)):\n",
        "            for j in range(len(q2)):\n",
        "                if q1[i][1] == None or q2[j][1] == None:\n",
        "                    sim = self.edit(q1[i][0], q2[j][0])\n",
        "                else:\n",
        "                    sim = self.path(wn.synset(q1[i][1]), wn.synset(q2[j][1]))\n",
        "\n",
        "                if sim == None:\n",
        "                    sim = self.edit(q1[i][0], q2[j][0])\n",
        "\n",
        "                R[i, j] = sim\n",
        "\n",
        "        # print R\n",
        "\n",
        "        return R\n",
        "\n",
        "    def computeWup(self, q1, q2):\n",
        "        \n",
        "        R = np.zeros((len(q1), len(q2)))\n",
        "\n",
        "        for i in range(len(q1)):\n",
        "            for j in range(len(q2)):\n",
        "                if q1[i][1] == None or q2[j][1] == None:\n",
        "                    sim = self.edit(q1[i][0], q2[j][0])\n",
        "                else:\n",
        "                    sim = self.wup(wn.synset(q1[i][1]), wn.synset(q2[j][1]))\n",
        "\n",
        "                if sim == None:\n",
        "                    sim = self.edit(q1[i][0], q2[j][0])\n",
        "\n",
        "                R[i, j] = sim\n",
        "\n",
        "        # print R\n",
        "\n",
        "        return R\n",
        "\n",
        "    def overallSim(self, q1, q2, R):\n",
        "\n",
        "        sum_X = 0.0\n",
        "        sum_Y = 0.0\n",
        "\n",
        "        for i in range(len(q1)):\n",
        "            max_i = 0.0\n",
        "            for j in range(len(q2)):\n",
        "                if R[i, j] > max_i:\n",
        "                    max_i = R[i, j]\n",
        "            sum_X += max_i\n",
        "\n",
        "        for i in range(len(q1)):\n",
        "            max_j = 0.0\n",
        "            for j in range(len(q2)):\n",
        "                if R[i, j] > max_j:\n",
        "                    max_j = R[i, j]\n",
        "            sum_Y += max_j\n",
        "\n",
        "        if (float(len(q1)) + float(len(q2))) == 0.0:\n",
        "            return 0.0\n",
        "\n",
        "        overall = (sum_X + sum_Y) / (2 * (float(len(q1)) + float(len(q2))))\n",
        "\n",
        "        return overall\n",
        "\n",
        "    def clean_sentence(self, val):\n",
        "        \"remove chars that are not letters or numbers, downcase, then remove stop words\"\n",
        "        regex = re.compile('([^\\s\\w]|_)+')\n",
        "        sentence = regex.sub('', val).lower()\n",
        "        sentence = sentence.split(\" \")\n",
        "\n",
        "        for word in list(sentence):\n",
        "            if word in self.STOP_WORDS:\n",
        "                sentence.remove(word)\n",
        "\n",
        "        sentence = \" \".join(sentence)\n",
        "        return sentence\n",
        "    \n",
        "    def semanticSimilarity(self, q1, q2):\n",
        "        tokens_q1, tokens_q2 = self.tokenize(q1, q2)\n",
        "        # stem_q1, stem_q2 = stemmer(tokens_q1, tokens_q2)\n",
        "        tag_q1, tag_q2 = self.posTag(tokens_q1, tokens_q2)\n",
        "\n",
        "        sentence = []\n",
        "        for i, word in enumerate(tag_q1):\n",
        "            if 'NN' in word[1] or 'JJ' in word[1] or 'VB' in word[1]:\n",
        "                sentence.append(word[0])\n",
        "\n",
        "        sense1 = Lesk(sentence)\n",
        "        sentence1Means = []\n",
        "        for word in sentence:\n",
        "            sentence1Means.append(sense1.lesk(word, sentence))\n",
        "\n",
        "        sentence = []\n",
        "        for i, word in enumerate(tag_q2):\n",
        "            if 'NN' in word[1] or 'JJ' in word[1] or 'VB' in word[1]:\n",
        "                sentence.append(word[0])\n",
        "\n",
        "        sense2 = Lesk(sentence)\n",
        "        sentence2Means = []\n",
        "        for word in sentence:\n",
        "            sentence2Means.append(sense2.lesk(word, sentence))\n",
        "        # for i, word in enumerate(sentence1Means):\n",
        "        #     print sentence1Means[i][0], sentence2Means[i][0]\n",
        "\n",
        "        R1 = self.computePath(sentence1Means, sentence2Means)\n",
        "        R2 = self.computeWup(sentence1Means, sentence2Means)\n",
        "\n",
        "        R = (R1 + R2) / 2\n",
        "\n",
        "        # print R\n",
        "\n",
        "        return self.overallSim(sentence1Means, sentence2Means, R)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmcfoBTPOn6F",
        "outputId": "7da39642-20e5-4119-8314-311e0e625b3f"
      },
      "source": [
        "# from PredictPaper import PredictPaper\n",
        "\n",
        "# Driver Function \n",
        "if __name__ == \"__main__\": \n",
        "    dataPath = \"drive/My Drive/Colab Notebooks/Data.xlsx\"\n",
        "    graphPath = \"drive/My Drive/Colab Notebooks/rake/output2k\"\n",
        "    predPap = PredictPaper(dataPath, graphPath)\n",
        "    inputKey  = ['machine learning', 'deep learning']\n",
        "    commonWords = 1\n",
        "    result = predPap.predict(inputKey, commonWords)\n",
        "    print(result)"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{1582: [5, 'computer vision'], 1257: [5, 'computer vision'], 819: [5, 'computer vision'], 1606: [4, 'computer vision'], 1253: [4, 'computer vision'], 1214: [4, 'relational databases'], 814: [4, 'parallel computing'], 789: [4, 'parallel computing'], 393: [4, 'relational databases'], 1915: [3, 'computer vision'], 1884: [3, 'computer vision'], 1668: [3, 'relational databases'], 1596: [3, 'computer vision'], 1442: [3, 'relational databases'], 1402: [3, 'relational databases'], 1360: [3, 'distributed computing'], 1324: [3, 'relational databases'], 1255: [3, 'machine learning'], 1220: [3, 'relational databases'], 1219: [3, 'structured storage'], 1218: [3, 'relational databases'], 1216: [3, 'distributed computing'], 1204: [3, 'data structures'], 1010: [3, 'computer programming'], 1008: [3, 'computer programming'], 928: [3, 'distributed computing'], 901: [3, 'distributed computing'], 792: [3, 'operating systems'], 785: [3, 'relational databases'], 783: [3, 'distributed computing'], 781: [3, 'relational databases'], 780: [3, 'relational databases'], 526: [3, 'relational databases'], 445: [3, 'computer vision'], 270: [3, 'relational databases'], 130: [3, 'relational databases'], 115: [3, 'relational databases'], 10: [3, 'computer vision'], 1961: [2, 'machine learning'], 1945: [2, 'relational databases'], 1919: [2, 'computer vision'], 1918: [2, 'computer vision'], 1917: [2, 'computer vision'], 1912: [2, 'computer vision'], 1899: [2, 'relational databases'], 1870: [2, 'computer vision'], 1869: [2, 'computer vision'], 1848: [2, 'distributed computing'], 1839: [2, 'image processing'], 1802: [2, 'relational databases'], 1801: [2, 'data structures'], 1794: [2, 'algorithm design'], 1791: [2, 'image processing'], 1778: [2, 'data structures'], 1715: [2, 'computer vision'], 1692: [2, 'relational databases'], 1690: [2, 'data structures'], 1655: [2, 'relational databases'], 1594: [2, 'computer vision'], 1591: [2, 'computer vision'], 1532: [2, 'computer vision'], 1531: [2, 'computer vision'], 1529: [2, 'computer vision'], 1482: [2, 'computer graphics'], 1448: [2, 'data structures'], 1417: [2, 'computer vision'], 1404: [2, 'relational databases'], 1361: [2, 'distributed computing'], 1351: [2, 'computer vision'], 1347: [2, 'machine learning'], 1314: [2, 'computer vision'], 1308: [2, 'computer vision'], 1304: [2, 'computer vision'], 1284: [2, 'computer vision'], 1281: [2, 'computer vision'], 1268: [2, 'computer vision'], 1254: [2, 'computer vision'], 1251: [2, 'computer vision'], 1250: [2, 'machine learning'], 1238: [2, 'relational databases'], 1234: [2, 'relational databases'], 1226: [2, 'data structures'], 1224: [2, 'data structures'], 1223: [2, 'computer programming'], 1221: [2, 'data structures'], 1212: [2, 'relational databases'], 1210: [2, 'relational databases'], 1205: [2, 'network security'], 1203: [2, 'machine learning'], 1202: [2, 'relational databases'], 1185: [2, 'relational databases'], 1139: [2, 'computer vision'], 1069: [2, 'data structures'], 1043: [2, 'computer programming'], 1034: [2, 'computer vision'], 974: [2, 'image processing'], 930: [2, 'machine learning'], 929: [2, 'parallel computing'], 927: [2, 'machine learning'], 925: [2, 'data structures'], 908: [2, 'network security'], 900: [2, 'distributed computing'], 897: [2, 'distributed computing'], 886: [2, 'computer vision'], 864: [2, 'machine learning'], 831: [2, 'data structures'], 798: [2, 'structured storage'], 790: [2, 'distributed computing'], 788: [2, 'distributed computing'], 787: [2, 'data structures'], 786: [2, 'distributed computing'], 784: [2, 'distributed computing'], 778: [2, 'computer graphics'], 771: [2, 'software engineering'], 605: [2, 'data structures'], 583: [2, 'relational databases'], 561: [2, 'computer vision'], 530: [2, 'relational databases'], 306: [2, 'network security'], 272: [2, 'relational databases'], 266: [2, 'relational databases'], 251: [2, 'computer vision'], 248: [2, 'computer vision'], 236: [2, 'computer vision'], 215: [2, 'machine learning'], 177: [2, 'computer vision'], 172: [2, 'relational databases'], 97: [2, 'relational databases'], 89: [2, 'parallel computing'], 68: [2, 'operating systems'], 1982: [1, 'distributed computing'], 1981: [1, 'data structures'], 1980: [1, 'machine learning'], 1978: [1, 'structured storage'], 1977: [1, 'relational databases'], 1976: [1, 'computer vision'], 1975: [1, 'computer vision'], 1974: [1, 'computer vision'], 1973: [1, 'computer programming'], 1972: [1, 'computer graphics'], 1971: [1, 'computer graphics'], 1964: [1, 'relational databases'], 1956: [1, 'data structures'], 1954: [1, 'computer programming'], 1944: [1, 'machine learning'], 1943: [1, 'distributed computing'], 1927: [1, 'computer graphics'], 1926: [1, 'cryptography'], 1914: [1, 'computer vision'], 1913: [1, 'computer graphics'], 1906: [1, 'computer vision'], 1897: [1, 'relational databases'], 1886: [1, 'network security'], 1885: [1, 'parallel computing'], 1867: [1, 'computer graphics'], 1866: [1, 'computer vision'], 1865: [1, 'computer vision'], 1864: [1, 'computer vision'], 1863: [1, 'computer vision'], 1861: [1, 'relational databases'], 1860: [1, 'bioinformatics'], 1858: [1, 'computer vision'], 1850: [1, 'distributed computing'], 1846: [1, 'network security'], 1836: [1, 'data structures'], 1820: [1, 'computer graphics'], 1817: [1, 'network security'], 1808: [1, 'computer programming'], 1805: [1, 'operating systems'], 1803: [1, 'computer vision'], 1800: [1, 'computer vision'], 1799: [1, 'distributed computing'], 1793: [1, 'distributed computing'], 1792: [1, 'computer vision'], 1787: [1, 'relational databases'], 1779: [1, 'data structures'], 1777: [1, 'computer graphics'], 1768: [1, 'relational databases'], 1754: [1, 'machine learning'], 1742: [1, 'computer graphics'], 1733: [1, 'computer programming'], 1704: [1, 'computer vision'], 1698: [1, 'data structures'], 1686: [1, 'network security'], 1671: [1, 'data structures'], 1664: [1, 'computer graphics'], 1660: [1, 'structured storage'], 1636: [1, 'algorithm design'], 1621: [1, 'computer vision'], 1618: [1, 'network security'], 1615: [1, 'computer vision'], 1613: [1, 'software engineering'], 1610: [1, 'computer graphics'], 1607: [1, 'operating systems'], 1599: [1, 'machine learning'], 1598: [1, 'distributed computing'], 1587: [1, 'distributed computing'], 1583: [1, 'parallel computing'], 1580: [1, 'computer graphics'], 1577: [1, 'computer vision'], 1569: [1, 'machine learning'], 1568: [1, 'computer vision'], 1566: [1, 'relational databases'], 1556: [1, 'machine learning'], 1554: [1, 'machine learning'], 1552: [1, 'machine learning'], 1551: [1, 'computer programming'], 1521: [1, 'network security'], 1491: [1, 'structured storage'], 1490: [1, 'computer programming'], 1480: [1, 'data structures'], 1468: [1, 'computer graphics'], 1459: [1, 'distributed computing'], 1452: [1, 'algorithm design'], 1449: [1, 'software engineering'], 1441: [1, 'computer programming'], 1439: [1, 'computer vision'], 1424: [1, 'structured storage'], 1420: [1, 'relational databases'], 1419: [1, 'network security'], 1413: [1, 'network security'], 1407: [1, 'network security'], 1403: [1, 'computer vision'], 1397: [1, 'computer vision'], 1390: [1, 'computer vision'], 1389: [1, 'relational databases'], 1376: [1, 'computer graphics'], 1370: [1, 'distributed computing'], 1363: [1, 'distributed computing'], 1362: [1, 'distributed computing'], 1358: [1, 'computer vision'], 1349: [1, 'machine learning'], 1339: [1, 'algorithm design'], 1333: [1, 'computer graphics'], 1329: [1, 'computer programming'], 1323: [1, 'relational databases'], 1309: [1, 'machine learning'], 1293: [1, 'data structures'], 1289: [1, 'computer graphics'], 1288: [1, 'computer programming'], 1267: [1, 'computer vision'], 1263: [1, 'computer programming'], 1256: [1, 'computer vision'], 1252: [1, 'image processing'], 1249: [1, 'computer vision'], 1244: [1, 'network security'], 1243: [1, 'structured storage'], 1242: [1, 'relational databases'], 1240: [1, 'machine learning'], 1237: [1, 'relational databases'], 1236: [1, 'relational databases'], 1235: [1, 'relational databases'], 1233: [1, 'relational databases'], 1231: [1, 'relational databases'], 1228: [1, 'relational databases'], 1227: [1, 'software engineering'], 1225: [1, 'data structures'], 1222: [1, 'data structures'], 1215: [1, 'cryptography'], 1213: [1, 'network security'], 1209: [1, 'relational databases'], 1208: [1, 'relational databases'], 1207: [1, 'relational databases'], 1206: [1, 'relational databases'], 1201: [1, 'distributed computing'], 1199: [1, 'algorithm design'], 1183: [1, 'algorithm design'], 1177: [1, 'distributed computing'], 1172: [1, 'distributed computing'], 1169: [1, 'data structures'], 1164: [1, 'distributed computing'], 1161: [1, 'computer graphics'], 1157: [1, 'network security'], 1138: [1, 'algorithm design'], 1137: [1, 'computer programming'], 1125: [1, 'computer graphics'], 1122: [1, 'relational databases'], 1079: [1, 'software engineering'], 1073: [1, 'relational databases'], 1068: [1, 'computer graphics'], 1053: [1, 'relational databases'], 1051: [1, 'data structures'], 1044: [1, 'computer programming'], 1042: [1, 'computer graphics'], 1039: [1, 'computer graphics'], 1031: [1, 'algorithm design'], 1030: [1, 'computer programming'], 1024: [1, 'computer programming'], 1022: [1, 'computer programming'], 1020: [1, 'computer programming'], 1017: [1, 'computer programming'], 1015: [1, 'computer programming'], 1013: [1, 'computer programming'], 1012: [1, 'computer programming'], 1005: [1, 'computer programming'], 999: [1, 'network security'], 998: [1, 'network security'], 994: [1, 'computer graphics'], 993: [1, 'computer graphics'], 992: [1, 'computer graphics'], 991: [1, 'computer graphics'], 988: [1, 'computer programming'], 984: [1, 'data structures'], 981: [1, 'algorithm design'], 976: [1, 'structured storage'], 975: [1, 'machine learning'], 970: [1, 'software engineering'], 969: [1, 'network security'], 967: [1, 'distributed computing'], 966: [1, 'algorithm design'], 962: [1, 'distributed computing'], 955: [1, 'parallel computing'], 943: [1, 'computer graphics'], 937: [1, 'algorithm design'], 926: [1, 'relational databases'], 922: [1, 'image processing'], 920: [1, 'operating systems'], 917: [1, 'relational databases'], 916: [1, 'algorithm design'], 914: [1, 'network security'], 913: [1, 'distributed computing'], 912: [1, 'distributed computing'], 911: [1, 'network security'], 909: [1, 'distributed computing'], 907: [1, 'distributed computing'], 906: [1, 'distributed computing'], 905: [1, 'network security'], 904: [1, 'distributed computing'], 903: [1, 'distributed computing'], 902: [1, 'distributed computing'], 899: [1, 'network security'], 896: [1, 'distributed computing'], 895: [1, 'distributed computing'], 888: [1, 'computer vision'], 884: [1, 'algorithm design'], 883: [1, 'computer graphics'], 865: [1, 'machine learning'], 857: [1, 'bioinformatics'], 842: [1, 'distributed computing'], 830: [1, 'relational databases'], 829: [1, 'computer vision'], 827: [1, 'computer vision'], 825: [1, 'software engineering'], 824: [1, 'machine learning'], 797: [1, 'computer programming'], 794: [1, 'computer graphics'], 791: [1, 'relational databases'], 782: [1, 'relational databases'], 779: [1, 'distributed computing'], 765: [1, 'cryptography'], 763: [1, 'algorithm design'], 755: [1, 'computer programming'], 734: [1, 'bioinformatics'], 724: [1, 'bioinformatics'], 720: [1, 'relational databases'], 719: [1, 'data structures'], 698: [1, 'computer programming'], 693: [1, 'data structures'], 678: [1, 'software engineering'], 677: [1, 'relational databases'], 646: [1, 'distributed computing'], 632: [1, 'bioinformatics'], 631: [1, 'data structures'], 619: [1, 'bioinformatics'], 610: [1, 'machine learning'], 606: [1, 'parallel computing'], 600: [1, 'bioinformatics'], 599: [1, 'machine learning'], 590: [1, 'parallel computing'], 572: [1, 'distributed computing'], 566: [1, 'computer vision'], 560: [1, 'machine learning'], 537: [1, 'network security'], 531: [1, 'computer graphics'], 519: [1, 'computer graphics'], 511: [1, 'distributed computing'], 506: [1, 'parallel computing'], 500: [1, 'network security'], 499: [1, 'network security'], 496: [1, 'machine learning'], 488: [1, 'distributed computing'], 479: [1, 'data structures'], 477: [1, 'computer vision'], 475: [1, 'computer vision'], 473: [1, 'operating systems'], 463: [1, 'distributed computing'], 460: [1, 'network security'], 455: [1, 'machine learning'], 454: [1, 'computer vision'], 452: [1, 'image processing'], 451: [1, 'distributed computing'], 450: [1, 'computer graphics'], 443: [1, 'machine learning'], 435: [1, 'computer graphics'], 426: [1, 'software engineering'], 417: [1, 'data structures'], 414: [1, 'distributed computing'], 408: [1, 'distributed computing'], 382: [1, 'relational databases'], 372: [1, 'computer graphics'], 359: [1, 'network security'], 340: [1, 'relational databases'], 339: [1, 'relational databases'], 333: [1, 'computer vision'], 331: [1, 'computer vision'], 330: [1, 'relational databases'], 323: [1, 'distributed computing'], 322: [1, 'distributed computing'], 321: [1, 'distributed computing'], 319: [1, 'relational databases'], 315: [1, 'data structures'], 311: [1, 'algorithm design'], 300: [1, 'machine learning'], 284: [1, 'computer vision'], 269: [1, 'computer graphics'], 267: [1, 'computer graphics'], 255: [1, 'computer programming'], 250: [1, 'computer vision'], 243: [1, 'computer graphics'], 242: [1, 'machine learning'], 241: [1, 'image processing'], 224: [1, 'computer vision'], 212: [1, 'computer vision'], 210: [1, 'machine learning'], 181: [1, 'network security'], 153: [1, 'relational databases'], 137: [1, 'computer vision'], 134: [1, 'data structures'], 129: [1, 'distributed computing'], 100: [1, 'algorithm design'], 77: [1, 'computer graphics'], 73: [1, 'network security'], 65: [1, 'computer graphics'], 60: [1, 'relational databases'], 44: [1, 'relational databases'], 42: [1, 'computer vision'], 39: [1, 'relational databases'], 37: [1, 'data structures'], 25: [1, 'machine learning'], 16: [1, 'computer graphics'], 15: [1, 'computer graphics'], 14: [1, 'computer graphics'], 11: [1, 'computer graphics'], 6: [1, 'network security'], 5: [1, 'data structures'], 3: [1, 'operating systems']}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UQNxpD8nSDbA"
      },
      "source": [
        "3 common keywords\n",
        "\n",
        "0. {911: [5, 'network security'], 1618: [4, 'network security'], 1413: [4, 'network security'], 1361: [4, 'distributed computing'], 1214: [4, 'relational databases'], 1213: [4, 'network security'], 1017: [4, 'computer programming'], 1013: [4, 'computer programming'], 902: [4, 'distributed computing'], 901: [4, 'distributed computing'], 900: [4, 'distributed computing'], 814: [4, 'parallel computing'], 789: [4, 'parallel computing'], 537: [4, 'network security'], 393: [4, 'relational databases'], 130: [4, 'relational databases'], 1982: [3, 'distributed computing'], 1794: [3, 'algorithm design'], 1792: [3, 'computer vision'], 1668: [3, 'relational databases'], 1521: [3, 'network security'], 1442: [3, 'relational databases'], 1419: [3, 'network security'], 1407: [3, 'network security'], 1402: [3, 'relational databases'], 1360: [3, 'distributed computing'], 1324: [3, 'relational databases'], 1226: [3, 'data structures'], 1223: [3, 'computer programming'], 1222: [3, 'data structures'], 1220: [3, 'relational databases'], 1219: [3, 'structured storage'], 1218: [3, 'relational databases'], 1216: [3, 'distributed computing'], 1210: [3, 'relational databases'], 1204: [3, 'data structures'], 1139: [3, 'computer vision'], 1048: [3, 'network security'], 1027: [3, 'computer programming'], 1022: [3, 'computer programming'], 1018: [3, 'computer programming'], 1016: [3, 'computer programming'], 1009: [3, 'computer programming'], 998: [3, 'network security'], 926: [3, 'relational databases'], 912: [3, 'distributed computing'], 909: [3, 'distributed computing'], 908: [3, 'network security'], 907: [3, 'distributed computing'], 905: [3, 'network security'], 899: [3, 'network security'], 895: [3, 'distributed computing'], 792: [3, 'operating systems'], 785: [3, 'relational databases'], 783: [3, 'distributed computing'], 781: [3, 'relational databases'], 780: [3, 'relational databases'], 536: [3, 'network security'], 526: [3, 'relational databases'], 500: [3, 'network security'], 270: [3, 'relational databases'], 266: [3, 'relational databases'], 172: [3, 'relational databases'], 115: [3, 'relational databases']}\n",
        "\n",
        "1. {911: [5, 'network security'], 902: [4, 'distributed computing'], 1521: [3, 'network security'], 1419: [3, 'network security'], 1361: [3, 'distributed computing'], 1213: [3, 'network security'], 998: [3, 'network security'], 912: [3, 'distributed computing'], 909: [3, 'distributed computing'], 907: [3, 'distributed computing'], 905: [3, 'network security'], 900: [3, 'distributed computing'], 895: [3, 'distributed computing'], 537: [3, 'network security']}\n",
        "\n",
        "2. {911: [4, 'network security'], 902: [4, 'distributed computing'], 900: [4, 'distributed computing'], 912: [3, 'distributed computing'], 909: [3, 'distributed computing'], 907: [3, 'distributed computing'], 905: [3, 'network security'], 895: [3, 'distributed computing']}\n",
        "\n",
        "3. {911: [5, 'network security'], 902: [4, 'distributed computing'], 900: [4, 'distributed computing'], 1521: [3, 'network security'], 1419: [3, 'network security'], 1361: [3, 'distributed computing'], 1213: [3, 'network security'], 998: [3, 'network security'], 912: [3, 'distributed computing'], 909: [3, 'distributed computing'], 907: [3, 'distributed computing'], 905: [3, 'network security'], 895: [3, 'distributed computing'], 537: [3, 'network security']}\n",
        "\n",
        "4. {911: [5, 'network security'], 902: [4, 'distributed computing'], 900: [4, 'distributed computing'], 814: [4, 'parallel computing'], 1521: [3, 'network security'], 1419: [3, 'network security'], 1407: [3, 'network security'], 1361: [3, 'distributed computing'], 1360: [3, 'distributed computing'], 1218: [3, 'relational databases'], 1216: [3, 'distributed computing'], 1214: [3, 'relational databases'], 1213: [3, 'network security'], 1210: [3, 'relational databases'], 1048: [3, 'network security'], 998: [3, 'network security'], 912: [3, 'distributed computing'], 909: [3, 'distributed computing'], 907: [3, 'distributed computing'], 905: [3, 'network security'], 901: [3, 'distributed computing'], 895: [3, 'distributed computing'], 792: [3, 'operating systems'], 789: [3, 'parallel computing'], 785: [3, 'relational databases'], 783: [3, 'distributed computing'], 781: [3, 'relational databases'], 537: [3, 'network security'], 393: [3, 'relational databases'], 270: [3, 'relational databases']}"
      ],
      "execution_count": 93,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x6yI5V_qT_l3"
      },
      "source": [
        "1 common keywords\n",
        "\n",
        "0. {1582: [5, 'computer vision'], 1257: [5, 'computer vision'], 819: [5, 'computer vision'], 1606: [4, 'computer vision'], 1253: [4, 'computer vision'], 1214: [4, 'relational databases'], 814: [4, 'parallel computing'], 789: [4, 'parallel computing'], 393: [4, 'relational databases'], 1915: [3, 'computer vision'], 1884: [3, 'computer vision'], 1668: [3, 'relational databases'], 1596: [3, 'computer vision'], 1442: [3, 'relational databases'], 1402: [3, 'relational databases'], 1360: [3, 'distributed computing'], 1324: [3, 'relational databases'], 1255: [3, 'machine learning'], 1220: [3, 'relational databases'], 1219: [3, 'structured storage'], 1218: [3, 'relational databases'], 1216: [3, 'distributed computing'], 1204: [3, 'data structures'], 1010: [3, 'computer programming'], 1008: [3, 'computer programming'], 928: [3, 'distributed computing'], 901: [3, 'distributed computing'], 792: [3, 'operating systems'], 785: [3, 'relational databases'], 783: [3, 'distributed computing'], 781: [3, 'relational databases'], 780: [3, 'relational databases'], 526: [3, 'relational databases'], 445: [3, 'computer vision'], 270: [3, 'relational databases'], 130: [3, 'relational databases'], 115: [3, 'relational databases'], 10: [3, 'computer vision'], 1961: [2, 'machine learning'], 1945: [2, 'relational databases'], 1919: [2, 'computer vision'], 1918: [2, 'computer vision'], 1917: [2, 'computer vision'], 1912: [2, 'computer vision'], 1899: [2, 'relational databases'], 1870: [2, 'computer vision'], 1869: [2, 'computer vision'], 1848: [2, 'distributed computing'], 1839: [2, 'image processing'], 1802: [2, 'relational databases'], 1801: [2, 'data structures'], 1794: [2, 'algorithm design'], 1791: [2, 'image processing'], 1778: [2, 'data structures'], 1715: [2, 'computer vision'], 1692: [2, 'relational databases'], 1690: [2, 'data structures'], 1655: [2, 'relational databases'], 1594: [2, 'computer vision'], 1591: [2, 'computer vision'], 1532: [2, 'computer vision'], 1531: [2, 'computer vision'], 1529: [2, 'computer vision'], 1482: [2, 'computer graphics'], 1448: [2, 'data structures'], 1417: [2, 'computer vision'], 1404: [2, 'relational databases'], 1361: [2, 'distributed computing'], 1351: [2, 'computer vision'], 1347: [2, 'machine learning'], 1314: [2, 'computer vision'], 1308: [2, 'computer vision'], 1304: [2, 'computer vision'], 1284: [2, 'computer vision'], 1281: [2, 'computer vision'], 1268: [2, 'computer vision'], 1254: [2, 'computer vision'], 1251: [2, 'computer vision'], 1250: [2, 'machine learning'], 1238: [2, 'relational databases'], 1234: [2, 'relational databases'], 1226: [2, 'data structures'], 1224: [2, 'data structures'], 1223: [2, 'computer programming'], 1221: [2, 'data structures'], 1212: [2, 'relational databases'], 1210: [2, 'relational databases'], 1205: [2, 'network security'], 1203: [2, 'machine learning'], 1202: [2, 'relational databases'], 1185: [2, 'relational databases'], 1139: [2, 'computer vision'], 1069: [2, 'data structures'], 1043: [2, 'computer programming'], 1034: [2, 'computer vision'], 974: [2, 'image processing'], 930: [2, 'machine learning'], 929: [2, 'parallel computing'], 927: [2, 'machine learning'], 925: [2, 'data structures'], 908: [2, 'network security'], 900: [2, 'distributed computing'], 897: [2, 'distributed computing'], 886: [2, 'computer vision'], 864: [2, 'machine learning'], 831: [2, 'data structures'], 798: [2, 'structured storage'], 790: [2, 'distributed computing'], 788: [2, 'distributed computing'], 787: [2, 'data structures'], 786: [2, 'distributed computing'], 784: [2, 'distributed computing'], 778: [2, 'computer graphics'], 771: [2, 'software engineering'], 605: [2, 'data structures'], 583: [2, 'relational databases'], 561: [2, 'computer vision'], 530: [2, 'relational databases'], 306: [2, 'network security'], 272: [2, 'relational databases'], 266: [2, 'relational databases'], 251: [2, 'computer vision'], 248: [2, 'computer vision'], 236: [2, 'computer vision'], 215: [2, 'machine learning'], 177: [2, 'computer vision'], 172: [2, 'relational databases'], 97: [2, 'relational databases'], 89: [2, 'parallel computing'], 68: [2, 'operating systems'], 1982: [1, 'distributed computing'], 1981: [1, 'data structures'], 1980: [1, 'machine learning'], 1978: [1, 'structured storage'], 1977: [1, 'relational databases'], 1976: [1, 'computer vision'], 1975: [1, 'computer vision'], 1974: [1, 'computer vision'], 1973: [1, 'computer programming'], 1972: [1, 'computer graphics'], 1971: [1, 'computer graphics'], 1964: [1, 'relational databases'], 1956: [1, 'data structures'], 1954: [1, 'computer programming'], 1944: [1, 'machine learning'], 1943: [1, 'distributed computing'], 1927: [1, 'computer graphics'], 1926: [1, 'cryptography'], 1914: [1, 'computer vision'], 1913: [1, 'computer graphics'], 1906: [1, 'computer vision'], 1897: [1, 'relational databases'], 1886: [1, 'network security'], 1885: [1, 'parallel computing'], 1867: [1, 'computer graphics'], 1866: [1, 'computer vision'], 1865: [1, 'computer vision'], 1864: [1, 'computer vision'], 1863: [1, 'computer vision'], 1861: [1, 'relational databases'], 1860: [1, 'bioinformatics'], 1858: [1, 'computer vision'], 1850: [1, 'distributed computing'], 1846: [1, 'network security'], 1836: [1, 'data structures'], 1820: [1, 'computer graphics'], 1817: [1, 'network security'], 1808: [1, 'computer programming'], 1805: [1, 'operating systems'], 1803: [1, 'computer vision'], 1800: [1, 'computer vision'], 1799: [1, 'distributed computing'], 1793: [1, 'distributed computing'], 1792: [1, 'computer vision'], 1787: [1, 'relational databases'], 1779: [1, 'data structures'], 1777: [1, 'computer graphics'], 1768: [1, 'relational databases'], 1754: [1, 'machine learning'], 1742: [1, 'computer graphics'], 1733: [1, 'computer programming'], 1704: [1, 'computer vision'], 1698: [1, 'data structures'], 1686: [1, 'network security'], 1671: [1, 'data structures'], 1664: [1, 'computer graphics'], 1660: [1, 'structured storage'], 1636: [1, 'algorithm design'], 1621: [1, 'computer vision'], 1618: [1, 'network security'], 1615: [1, 'computer vision'], 1613: [1, 'software engineering'], 1610: [1, 'computer graphics'], 1607: [1, 'operating systems'], 1599: [1, 'machine learning'], 1598: [1, 'distributed computing'], 1587: [1, 'distributed computing'], 1583: [1, 'parallel computing'], 1580: [1, 'computer graphics'], 1577: [1, 'computer vision'], 1569: [1, 'machine learning'], 1568: [1, 'computer vision'], 1566: [1, 'relational databases'], 1556: [1, 'machine learning'], 1554: [1, 'machine learning'], 1552: [1, 'machine learning'], 1551: [1, 'computer programming'], 1521: [1, 'network security'], 1491: [1, 'structured storage'], 1490: [1, 'computer programming'], 1480: [1, 'data structures'], 1468: [1, 'computer graphics'], 1459: [1, 'distributed computing'], 1452: [1, 'algorithm design'], 1449: [1, 'software engineering'], 1441: [1, 'computer programming'], 1439: [1, 'computer vision'], 1424: [1, 'structured storage'], 1420: [1, 'relational databases'], 1419: [1, 'network security'], 1413: [1, 'network security'], 1407: [1, 'network security'], 1403: [1, 'computer vision'], 1397: [1, 'computer vision'], 1390: [1, 'computer vision'], 1389: [1, 'relational databases'], 1376: [1, 'computer graphics'], 1370: [1, 'distributed computing'], 1363: [1, 'distributed computing'], 1362: [1, 'distributed computing'], 1358: [1, 'computer vision'], 1349: [1, 'machine learning'], 1339: [1, 'algorithm design'], 1333: [1, 'computer graphics'], 1329: [1, 'computer programming'], 1323: [1, 'relational databases'], 1309: [1, 'machine learning'], 1293: [1, 'data structures'], 1289: [1, 'computer graphics'], 1288: [1, 'computer programming'], 1267: [1, 'computer vision'], 1263: [1, 'computer programming'], 1256: [1, 'computer vision'], 1252: [1, 'image processing'], 1249: [1, 'computer vision'], 1244: [1, 'network security'], 1243: [1, 'structured storage'], 1242: [1, 'relational databases'], 1240: [1, 'machine learning'], 1237: [1, 'relational databases'], 1236: [1, 'relational databases'], 1235: [1, 'relational databases'], 1233: [1, 'relational databases'], 1231: [1, 'relational databases'], 1228: [1, 'relational databases'], 1227: [1, 'software engineering'], 1225: [1, 'data structures'], 1222: [1, 'data structures'], 1215: [1, 'cryptography'], 1213: [1, 'network security'], 1209: [1, 'relational databases'], 1208: [1, 'relational databases'], 1207: [1, 'relational databases'], 1206: [1, 'relational databases'], 1201: [1, 'distributed computing'], 1199: [1, 'algorithm design'], 1183: [1, 'algorithm design'], 1177: [1, 'distributed computing'], 1172: [1, 'distributed computing'], 1169: [1, 'data structures'], 1164: [1, 'distributed computing'], 1161: [1, 'computer graphics'], 1157: [1, 'network security'], 1138: [1, 'algorithm design'], 1137: [1, 'computer programming'], 1125: [1, 'computer graphics'], 1122: [1, 'relational databases'], 1079: [1, 'software engineering'], 1073: [1, 'relational databases'], 1068: [1, 'computer graphics'], 1053: [1, 'relational databases'], 1051: [1, 'data structures'], 1044: [1, 'computer programming'], 1042: [1, 'computer graphics'], 1039: [1, 'computer graphics'], 1031: [1, 'algorithm design'], 1030: [1, 'computer programming'], 1024: [1, 'computer programming'], 1022: [1, 'computer programming'], 1020: [1, 'computer programming'], 1017: [1, 'computer programming'], 1015: [1, 'computer programming'], 1013: [1, 'computer programming'], 1012: [1, 'computer programming'], 1005: [1, 'computer programming'], 999: [1, 'network security'], 998: [1, 'network security'], 994: [1, 'computer graphics'], 993: [1, 'computer graphics'], 992: [1, 'computer graphics'], 991: [1, 'computer graphics'], 988: [1, 'computer programming'], 984: [1, 'data structures'], 981: [1, 'algorithm design'], 976: [1, 'structured storage'], 975: [1, 'machine learning'], 970: [1, 'software engineering'], 969: [1, 'network security'], 967: [1, 'distributed computing'], 966: [1, 'algorithm design'], 962: [1, 'distributed computing'], 955: [1, 'parallel computing'], 943: [1, 'computer graphics'], 937: [1, 'algorithm design'], 926: [1, 'relational databases'], 922: [1, 'image processing'], 920: [1, 'operating systems'], 917: [1, 'relational databases'], 916: [1, 'algorithm design'], 914: [1, 'network security'], 913: [1, 'distributed computing'], 912: [1, 'distributed computing'], 911: [1, 'network security'], 909: [1, 'distributed computing'], 907: [1, 'distributed computing'], 906: [1, 'distributed computing'], 905: [1, 'network security'], 904: [1, 'distributed computing'], 903: [1, 'distributed computing'], 902: [1, 'distributed computing'], 899: [1, 'network security'], 896: [1, 'distributed computing'], 895: [1, 'distributed computing'], 888: [1, 'computer vision'], 884: [1, 'algorithm design'], 883: [1, 'computer graphics'], 865: [1, 'machine learning'], 857: [1, 'bioinformatics'], 842: [1, 'distributed computing'], 830: [1, 'relational databases'], 829: [1, 'computer vision'], 827: [1, 'computer vision'], 825: [1, 'software engineering'], 824: [1, 'machine learning'], 797: [1, 'computer programming'], 794: [1, 'computer graphics'], 791: [1, 'relational databases'], 782: [1, 'relational databases'], 779: [1, 'distributed computing'], 765: [1, 'cryptography'], 763: [1, 'algorithm design'], 755: [1, 'computer programming'], 734: [1, 'bioinformatics'], 724: [1, 'bioinformatics'], 720: [1, 'relational databases'], 719: [1, 'data structures'], 698: [1, 'computer programming'], 693: [1, 'data structures'], 678: [1, 'software engineering'], 677: [1, 'relational databases'], 646: [1, 'distributed computing'], 632: [1, 'bioinformatics'], 631: [1, 'data structures'], 619: [1, 'bioinformatics'], 610: [1, 'machine learning'], 606: [1, 'parallel computing'], 600: [1, 'bioinformatics'], 599: [1, 'machine learning'], 590: [1, 'parallel computing'], 572: [1, 'distributed computing'], 566: [1, 'computer vision'], 560: [1, 'machine learning'], 537: [1, 'network security'], 531: [1, 'computer graphics'], 519: [1, 'computer graphics'], 511: [1, 'distributed computing'], 506: [1, 'parallel computing'], 500: [1, 'network security'], 499: [1, 'network security'], 496: [1, 'machine learning'], 488: [1, 'distributed computing'], 479: [1, 'data structures'], 477: [1, 'computer vision'], 475: [1, 'computer vision'], 473: [1, 'operating systems'], 463: [1, 'distributed computing'], 460: [1, 'network security'], 455: [1, 'machine learning'], 454: [1, 'computer vision'], 452: [1, 'image processing'], 451: [1, 'distributed computing'], 450: [1, 'computer graphics'], 443: [1, 'machine learning'], 435: [1, 'computer graphics'], 426: [1, 'software engineering'], 417: [1, 'data structures'], 414: [1, 'distributed computing'], 408: [1, 'distributed computing'], 382: [1, 'relational databases'], 372: [1, 'computer graphics'], 359: [1, 'network security'], 340: [1, 'relational databases'], 339: [1, 'relational databases'], 333: [1, 'computer vision'], 331: [1, 'computer vision'], 330: [1, 'relational databases'], 323: [1, 'distributed computing'], 322: [1, 'distributed computing'], 321: [1, 'distributed computing'], 319: [1, 'relational databases'], 315: [1, 'data structures'], 311: [1, 'algorithm design'], 300: [1, 'machine learning'], 284: [1, 'computer vision'], 269: [1, 'computer graphics'], 267: [1, 'computer graphics'], 255: [1, 'computer programming'], 250: [1, 'computer vision'], 243: [1, 'computer graphics'], 242: [1, 'machine learning'], 241: [1, 'image processing'], 224: [1, 'computer vision'], 212: [1, 'computer vision'], 210: [1, 'machine learning'], 181: [1, 'network security'], 153: [1, 'relational databases'], 137: [1, 'computer vision'], 134: [1, 'data structures'], 129: [1, 'distributed computing'], 100: [1, 'algorithm design'], 77: [1, 'computer graphics'], 73: [1, 'network security'], 65: [1, 'computer graphics'], 60: [1, 'relational databases'], 44: [1, 'relational databases'], 42: [1, 'computer vision'], 39: [1, 'relational databases'], 37: [1, 'data structures'], 25: [1, 'machine learning'], 16: [1, 'computer graphics'], 15: [1, 'computer graphics'], 14: [1, 'computer graphics'], 11: [1, 'computer graphics'], 6: [1, 'network security'], 5: [1, 'data structures'], 3: [1, 'operating systems']}\n",
        "\n",
        "1. {1944: [1, 'machine learning'], 1884: [1, 'computer vision'], 1866: [1, 'computer vision'], 1754: [1, 'machine learning'], 1598: [1, 'distributed computing'], 1569: [1, 'machine learning'], 1554: [1, 'machine learning'], 1552: [1, 'machine learning'], 1390: [1, 'computer vision'], 1347: [1, 'machine learning'], 1257: [1, 'computer vision'], 1255: [1, 'machine learning'], 1254: [1, 'computer vision'], 1253: [1, 'computer vision'], 1252: [1, 'image processing'], 1240: [1, 'machine learning'], 1204: [1, 'data structures'], 1139: [1, 'computer vision'], 975: [1, 'machine learning'], 962: [1, 'distributed computing'], 930: [1, 'machine learning'], 927: [1, 'machine learning'], 824: [1, 'machine learning'], 496: [1, 'machine learning'], 300: [1, 'machine learning'], 242: [1, 'machine learning'], 215: [1, 'machine learning'], 25: [1, 'machine learning']}\n",
        "\n",
        "2. {1944: [1, 'machine learning'], 1884: [1, 'computer vision'], 1866: [1, 'computer vision'], 1754: [1, 'machine learning'], 1598: [1, 'distributed computing'], 1569: [1, 'machine learning'], 1554: [1, 'machine learning'], 1552: [1, 'machine learning'], 1390: [1, 'computer vision'], 1347: [1, 'machine learning'], 1257: [1, 'computer vision'], 1255: [1, 'machine learning'], 1254: [1, 'computer vision'], 1253: [1, 'computer vision'], 1252: [1, 'image processing'], 1240: [1, 'machine learning'], 1204: [1, 'data structures'], 1139: [1, 'computer vision'], 975: [1, 'machine learning'], 962: [1, 'distributed computing'], 930: [1, 'machine learning'], 927: [1, 'machine learning'], 824: [1, 'machine learning'], 771: [1, 'software engineering'], 496: [1, 'machine learning'], 300: [1, 'machine learning'], 242: [1, 'machine learning'], 215: [1, 'machine learning'], 68: [1, 'operating systems'], 25: [1, 'machine learning']}\n",
        "\n",
        "3. {1944: [1, 'machine learning'], 1884: [1, 'computer vision'], 1866: [1, 'computer vision'], 1754: [1, 'machine learning'], 1598: [1, 'distributed computing'], 1569: [1, 'machine learning'], 1554: [1, 'machine learning'], 1552: [1, 'machine learning'], 1390: [1, 'computer vision'], 1347: [1, 'machine learning'], 1257: [1, 'computer vision'], 1255: [1, 'machine learning'], 1254: [1, 'computer vision'], 1253: [1, 'computer vision'], 1252: [1, 'image processing'], 1240: [1, 'machine learning'], 1204: [1, 'data structures'], 1139: [1, 'computer vision'], 975: [1, 'machine learning'], 962: [1, 'distributed computing'], 930: [1, 'machine learning'], 927: [1, 'machine learning'], 824: [1, 'machine learning'], 496: [1, 'machine learning'], 300: [1, 'machine learning'], 242: [1, 'machine learning'], 215: [1, 'machine learning'], 25: [1, 'machine learning']}\n",
        "\n",
        "4. {1257: [3, 'computer vision'], 1347: [2, 'machine learning'], 1204: [2, 'data structures'], 927: [2, 'machine learning'], 819: [2, 'computer vision'], 215: [2, 'machine learning'], 1980: [1, 'machine learning'], 1944: [1, 'machine learning'], 1885: [1, 'parallel computing'], 1884: [1, 'computer vision'], 1866: [1, 'computer vision'], 1778: [1, 'data structures'], 1754: [1, 'machine learning'], 1598: [1, 'distributed computing'], 1582: [1, 'computer vision'], 1569: [1, 'machine learning'], 1554: [1, 'machine learning'], 1552: [1, 'machine learning'], 1490: [1, 'computer programming'], 1390: [1, 'computer vision'], 1324: [1, 'relational databases'], 1255: [1, 'machine learning'], 1254: [1, 'computer vision'], 1253: [1, 'computer vision'], 1252: [1, 'image processing'], 1240: [1, 'machine learning'], 1203: [1, 'machine learning'], 1139: [1, 'computer vision'], 975: [1, 'machine learning'], 962: [1, 'distributed computing'], 955: [1, 'parallel computing'], 930: [1, 'machine learning'], 928: [1, 'distributed computing'], 920: [1, 'operating systems'], 824: [1, 'machine learning'], 797: [1, 'computer programming'], 780: [1, 'relational databases'], 771: [1, 'software engineering'], 496: [1, 'machine learning'], 460: [1, 'network security'], 300: [1, 'machine learning'], 242: [1, 'machine learning'], 68: [1, 'operating systems'], 25: [1, 'machine learning']}"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}