build_keywordAuthor_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "import json\n",
    "import networkx as nx\n",
    "from tqdm import tqdm\n",
    "from itertools import chain\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.express as px\n",
    "import community as community_louvain\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "def flatten_chain(matrix):\n",
    "     return list(chain.from_iterable(matrix))\n",
    "\n",
    "def append_row(df, row):\n",
    "    return pd.concat([\n",
    "                df, \n",
    "                pd.DataFrame([row], columns=row.index)]\n",
    "           ).reset_index(drop=True)\n",
    "\n",
    "def div_dict(my_dict):\n",
    "    sum_p = sum(my_dict.values())\n",
    "    for i in my_dict:\n",
    "        my_dict[i] = float(my_dict[i]/sum_p)\n",
    "    return my_dict \n",
    "\n",
    "def dict_max(my_dict,edge_val,else_string):\n",
    "    maxval = np.max(list(my_dict.values()))\n",
    "    maxkey = max(my_dict, key=my_dict.get)\n",
    "    if maxval > edge_val:\n",
    "        return(maxkey)\n",
    "    else:\n",
    "        return(else_string)\n",
    "    \n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "s = set(stopwords.words('english'))\n",
    "\n",
    "folder = './scopus_publication_files_16042024'\n",
    "files = os.listdir(folder)\n",
    "file = files[0]\n",
    "\n",
    "author_folder = './scopus_author_files_16042024'\n",
    "author_files = os.listdir(author_folder)\n",
    "\n",
    "df_subjectAreas = pd.read_xml('scopus_subject_classification.xml')\n",
    "df_subjectAreas.set_index('code',inplace=True)\n",
    "df_subject_areas_color = pd.DataFrame({'subject':list(df_subjectAreas.groupby('subject-classification').sum().index), 'color':['#5FBC52','#C1A765','#56B280','#EB704F','#FF8001','#00C400','#FF3300','#9933FF','#BC7BED','#535AE7','#FAA72E','#DFC717','#E94727','#00FF99','#CF63B0','#5B99F3','#4ABFD6','#3DE3D3','#D143E9','#A6A6A6','#CF63B0','#DB53A7','#FF5B9D','#4641FF','#C46DE7','#9358F2','#CCCCFF']})\n",
    "\n",
    "df_names = pd.read_csv('people_in_S4S_pureFiltered_withAuthorIDs.csv',index_col=[0],keep_default_na=False).set_index('scopusID')\n",
    "\n",
    "df_keysPerSubject = df_subjectAreas\n",
    "df_keysPerSubject[\"count_author\"] = [0] * len(df_keysPerSubject)\n",
    "df_keysPerSubject[\"count_index\"] = [0] * len(df_keysPerSubject)\n",
    "df_keysPerSubject[\"count_paper\"] = [0] * len(df_keysPerSubject)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 144/144 [00:04<00:00, 33.10it/s]\n"
     ]
    }
   ],
   "source": [
    "authors = []\n",
    "df_authors = pd.DataFrame(columns=['scopusID', 'firstname','lastname','citationCount','documentCount','publicationRange','current_affiliation_institute','current_affiliation_department','current_affiliation_country','current_affiliation_city','current_affiliation_lat','current_affiliation_lon'])\n",
    "# df_co_authors = pd.DataFrame(columns=['scopusId', 'firstname','lastname','citationCount','documentCount','publicationRange','current_affiliation_institute','current_affiliation_department','current_affiliation_country','current_affiliation_city','current_affiliation_lat','current_affiliation_lon'])\n",
    "\n",
    "for file in tqdm(author_files):\n",
    "    with open(author_folder + '/' + file, encoding='utf-16') as fd:\n",
    "        author_dict = json.load(fd)\n",
    "    df_authors = append_row(df_authors, pd.Series(author_dict).rename({'scopusid':'scopusId'}))\n",
    "\n",
    "df_authors['fullname'] = df_authors['firstname'] + df_authors['lastname']\n",
    "df_authors.set_index('fullname',inplace=True)\n",
    "\n",
    "df_authors['keywords'] = [[]]*len(df_authors)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2198/2198 [00:39<00:00, 55.56it/s]\n"
     ]
    }
   ],
   "source": [
    "keyword_list = []\n",
    "author_list = []\n",
    "author_keywords = {}\n",
    "df_papers = pd.DataFrame(columns=['eid', 'doi', 'authorCount', 'releaseDate', 'citationCount',\n",
    "       'authorNames', 'authorScopusIds', 'refCount', 'authorKeywords',\n",
    "       'idxterms', 'subjectAreasDetail', 'subjectAreas'])\n",
    "\n",
    "keyword_dict = {}\n",
    "\n",
    "subject_list = []\n",
    "for file in tqdm(files):\n",
    "    with open(folder + '/' +file, encoding='utf16') as fd:\n",
    "        paper_dict = json.load(fd)\n",
    "    # del paper_dict['references']\n",
    "    df_papers = append_row(df_papers, pd.Series(paper_dict))\n",
    "    \n",
    "    paper_keywords = [] \n",
    "    #filtering keywords for stopwords (e.g. the netherlands vs netherlands)\n",
    "    filtered_keywords = []\n",
    "    if 'authorKeywords' in paper_dict.keys():\n",
    "        for word in paper_dict['authorKeywords']: \n",
    "            word = word.lower()\n",
    "            term = ''\n",
    "            for word_part in word.split(' '):\n",
    "                if word_part not in s: \n",
    "                    term += word_part + ' '\n",
    "            term = term.strip()\n",
    "            filtered_keywords.append(term)\n",
    "\n",
    "        paper_keywords.append(filtered_keywords)\n",
    "    \n",
    "\n",
    "    #index terms\n",
    "    if 'idxterms' in paper_dict.keys():\n",
    "        if paper_dict['idxterms'] != None:\n",
    "            idxterms = paper_dict['idxterms']\n",
    "            filtered_idxterms = []\n",
    "            for word in idxterms: \n",
    "                word = word.lower()\n",
    "                if word not in s: \n",
    "                    filtered_idxterms.append(word) \n",
    "        paper_keywords.append(filtered_idxterms)\n",
    "\n",
    "    paper_keywords = pd.Series(flatten_chain(paper_keywords))\n",
    "    paper_keywords = paper_keywords.str.lower()\n",
    "    paper_keywords.drop_duplicates(inplace=True)\n",
    "\n",
    "\n",
    "    keyword_list.append(paper_keywords.to_list())\n",
    "    # if paper_dict['authorNames'] != None:\n",
    "    #     author_list.append(paper_dict['authorNames'])\n",
    "\n",
    "    i=0\n",
    "    author_sublist = []\n",
    "    if paper_dict['authorNames'] != None:\n",
    "        for author, scopusId in zip(paper_dict['authorNames'], paper_dict['authorScopusIds']):\n",
    "            if scopusId in df_names.index.to_list():\n",
    "                name = df_names['fullname'].loc[scopusId].strip()\n",
    "                author_sublist.append(name)\n",
    "                if name not in author_keywords.keys():\n",
    "                    author_keywords[name] = []\n",
    "                    author_keywords[name].append(list(paper_keywords))\n",
    "                else:\n",
    "                    author_keywords[name].append(list(paper_keywords))\n",
    "\n",
    "\n",
    "    for key in paper_keywords:\n",
    "\n",
    "        if key in keyword_dict.keys():\n",
    "            pass\n",
    "        else:\n",
    "            keyword_dict[key] = {}\n",
    "            keyword_dict[key]['subject_areas_code'] = []\n",
    "            keyword_dict[key]['subject_areas_detailed'] = []\n",
    "            keyword_dict[key]['total_citations'] = 0\n",
    "            keyword_dict[key]['num_of_papers'] = 0\n",
    "            keyword_dict[key]['authors'] = []\n",
    "            keyword_dict[key]['scopusIds'] = []\n",
    "            keyword_dict[key]['type'] = []\n",
    "            keyword_dict[key]['doi'] = []\n",
    "            keyword_dict[key]['title'] = []\n",
    "            keyword_dict[key]['releaseDate'] = []\n",
    "\n",
    "        keyword_dict[key]['type'].append('author' if i < len(filtered_keywords) else 'index')\n",
    "\n",
    "        keyword_dict[key]['doi'].append(paper_dict['doi'])\n",
    "        keyword_dict[key]['title'].append(paper_dict['title'])\n",
    "        keyword_dict[key]['releaseDate'].append(paper_dict['releaseDate'])\n",
    "\n",
    "        if 'subjectAreas' in paper_dict.keys():\n",
    "            for subj in paper_dict['subjectAreas']:\n",
    "                subj = int(subj)\n",
    "                subject_list.append(subj)\n",
    "                keyword_dict[key]['subject_areas_code'].append(subj)\n",
    "                # keyword_dict[key]['subject_areas_detailed'].append(df_subjectAreas.loc[subj][\"detail\"])\n",
    "\n",
    "                df_keysPerSubject[\"count_author\"] .loc[subj]+= len(filtered_keywords)\n",
    "                df_keysPerSubject[\"count_index\"].loc[subj] += len(filtered_idxterms)\n",
    "                df_keysPerSubject[\"count_paper\"].loc[subj] += 1\n",
    "\n",
    "\n",
    "        if paper_dict['authorNames'] != None:\n",
    "            for author, scopusId in zip(paper_dict['authorNames'],paper_dict['authorScopusIds']):\n",
    "                if scopusId in df_names.index.to_list():\n",
    "                    keyword_dict[key]['authors'].append(df_names['fullname'].loc[scopusId].strip())  \n",
    "                    keyword_dict[key]['scopusIds'].append(scopusId)  \n",
    "                    # author_sublist.append(df_names['fullname'].loc[scopusId].strip())\n",
    "\n",
    "        keyword_dict[key]['total_citations'] += int(paper_dict['citationCount']) if paper_dict['citationCount'] != None else 0\n",
    "        keyword_dict[key]['num_of_papers'] += 1\n",
    "        i+=1\n",
    "    author_list.append(author_sublist)\n",
    "        \n",
    "keyword_list_flattened = flatten_chain(keyword_list)\n",
    "df_keywords = pd.DataFrame.from_dict(keyword_dict,orient='index')\n",
    "\n",
    "df_keywords['authors'] = df_keywords['authors'].apply(lambda x: Counter(x))\n",
    "df_keywords['author_names'] = df_keywords['authors'].apply(lambda x: list(x.keys()))\n",
    "df_keywords['num_of_authors'] = df_keywords['authors'].apply(lambda x: len(x))\n",
    "\n",
    "df_keywords['scopusIds'] = df_keywords['scopusIds'].apply(lambda x: Counter(x))\n",
    "\n",
    "# df_keywords['authors'] = df_keywords['authors'].apply(lambda x: str(x).replace('{','').replace('}','').replace('Counter','').replace('(','').replace(')','').replace(\"'\",''))\n",
    "# df_keywords['scopusIds'] = df_keywords['scopusIds'].apply(lambda x: str(x).replace('{','').replace('}','').replace('Counter','').replace('(','').replace(')','').replace(\"'\",''))\n",
    "\n",
    "# df_keywords['papers_per_author'] = df_keywords['num_of_authors']/df_keywords['num_of_papers']\n",
    "# df_keywords['relevance'] = df_keywords['num_of_authors']/max(df_keywords['num_of_authors'])   / df_keywords['num_of_papers']/max(df_keywords['num_of_papers'])\n",
    "\n",
    "# df_keywords['subjects_detail_counted'] = df_keywords['subject_areas_detailed'].apply(lambda x: Counter(x))\n",
    "\n",
    "df_keywords['type'] = df_keywords['type'].apply(lambda x: Counter(x))\n",
    "\n",
    "\n",
    "df_keywords['main_type'] = df_keywords[\"type\"].apply(lambda x: div_dict(dict(x)))\n",
    "df_keywords['main_type'] = df_keywords['main_type'].apply(lambda x: dict_max(x,0.5,'None'))\n",
    "\n",
    "# df_keywords['type'] = df_keywords['type'].apply(lambda x: str(x).replace('{','').replace('}','').replace('Counter','').replace('(','').replace(')','').replace(\"'\",''))\n",
    "\n",
    "\n",
    "df_authors['keywords'] = author_keywords\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_papers['releaseDate_formatted'] = pd.to_datetime(df_papers['releaseDate'], format='mixed', dayfirst=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_authors['documentCount'] = df_authors['publishedArticles'].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2507"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df_authors['documentCount'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/8791 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8791/8791 [00:14<00:00, 616.27it/s]\n"
     ]
    }
   ],
   "source": [
    "counted_keywords = pd.DataFrame.from_dict(dict(Counter(keyword_list_flattened)),orient='index',columns=['counted'])\n",
    "\n",
    "keywords_below = counted_keywords[counted_keywords['counted'] < 2].index.to_list()\n",
    "keyword_list_reduced = keyword_list.copy()\n",
    "\n",
    "for key in tqdm(keywords_below):\n",
    "    for i in range(len(keyword_list_reduced)):\n",
    "        try:\n",
    "            keyword_list_reduced[i].remove(key)\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "keyword_list_reduced_flattened = flatten_chain(keyword_list_reduced)\n",
    "df_keywords = df_keywords.join(counted_keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "keyword_nodes = list(df_keywords[df_keywords['counted'] >= 2].index)\n",
    "author_nodes = df_keywords[df_keywords['counted'] >= 2]['authors'].apply(lambda x: list(x.keys()))\n",
    "authors = []\n",
    "for i in author_nodes.index:\n",
    "    for auth in author_nodes.loc[i]:\n",
    "        if auth in authors:\n",
    "            pass\n",
    "        else:\n",
    "            authors.append(auth)\n",
    "author_nodes = authors\n",
    "\n",
    "nodes = keyword_nodes + author_nodes\n",
    "\n",
    "\n",
    "\n",
    "df_keyword_nodes = pd.DataFrame({'node_id':keyword_nodes, 'type':['keyword']*len(keyword_nodes)})\n",
    "df_author_nodes = pd.DataFrame({'node_id':author_nodes, 'type':['author']*len(author_nodes)})\n",
    "\n",
    "df_nodes = pd.concat([df_keyword_nodes,df_author_nodes])\n",
    "df_nodes.set_index('node_id',inplace=True, drop=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2152/2152 [00:11<00:00, 188.48it/s]\n",
      "100%|██████████| 144/144 [00:08<00:00, 17.15it/s]\n"
     ]
    }
   ],
   "source": [
    "adj_matrix = np.zeros(shape=(len(nodes),len(nodes)))\n",
    "adj_matrix = pd.DataFrame(adj_matrix, index=nodes, columns=nodes)\n",
    "\n",
    "i=0\n",
    "for key in tqdm(nodes):\n",
    "\n",
    "    if key in keyword_nodes:\n",
    "        for sublist in keyword_list_reduced:\n",
    "            if key in sublist:\n",
    "                for second_key in sublist:\n",
    "                        adj_matrix[key][second_key] += 1\n",
    "                        \n",
    "    elif key in author_nodes:\n",
    "        for sublist in author_list:\n",
    "            if key in sublist:\n",
    "                for second_key in sublist:\n",
    "                        try: #this is strange\n",
    "                            adj_matrix[key][second_key] += 1\n",
    "                        except:\n",
    "                            pass\n",
    "\n",
    "\n",
    "for name in tqdm(df_authors.index):\n",
    "    try:\n",
    "        counted = Counter(flatten_chain(df_authors['keywords'].loc[name]))\n",
    "    except:\n",
    "        continue\n",
    "    for key,val in counted.items():\n",
    "        try:\n",
    "            adj_matrix[name][key] = val\n",
    "            adj_matrix[key][name] = val\n",
    "        except:\n",
    "            continue\n",
    "\n",
    "np.fill_diagonal(adj_matrix.values, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "adj_matrix_authors = adj_matrix.loc[author_nodes[0]:,:author_nodes[0]]\n",
    "df_edges_authors = adj_matrix_authors.rename_axis('source')\\\n",
    "  .reset_index()\\\n",
    "  .melt('source', value_name='weight', var_name='target')\\\n",
    "  .query('source != target')\\\n",
    "  .reset_index(drop=True)\n",
    "df_edges_authors = df_edges_authors[df_edges_authors['weight']>0].reset_index()\n",
    "df_edges_authors.drop('index',axis=1,inplace=True)\n",
    "df_edges_authors.set_index('target',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "adj_matrix_tmps = adj_matrix.loc[author_nodes[0]:,author_nodes[0]:]\n",
    "df_edges_tmp = adj_matrix_tmps.rename_axis('source')\\\n",
    "  .reset_index()\\\n",
    "  .melt('source', value_name='weight', var_name='target')\\\n",
    "  .query('source != target')\\\n",
    "  .reset_index(drop=True)\n",
    "df_edges_tmp = df_edges_tmp[df_edges_tmp['weight']>0].reset_index()\n",
    "df_edges_tmp.drop('index',axis=1,inplace=True)\n",
    "# df_edges_tmp.set_index('target',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "680.0"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(df_edges_tmp['weight'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_edges = adj_matrix.rename_axis('source')\\\n",
    "  .reset_index()\\\n",
    "  .melt('source', value_name='weight', var_name='target')\\\n",
    "  .query('source != target')\\\n",
    "  .reset_index(drop=True)\n",
    "df_edges = df_edges[df_edges['weight']>0].reset_index()\n",
    "df_edges.drop('index',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/40774 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40774/40774 [00:02<00:00, 17935.91it/s]\n"
     ]
    }
   ],
   "source": [
    "grouped_source_length = df_edges.groupby(['source']).size()\n",
    "grouped_target_length = df_edges.groupby(['target']).size()\n",
    "\n",
    "added_jaccard = []\n",
    "for i in tqdm(range(len(df_edges))):\n",
    "    added_jaccard.append(df_edges['weight'].loc[i]/grouped_source_length.loc[df_edges['source'].loc[i]] + df_edges['weight'].loc[i]/grouped_source_length.loc[df_edges['target'].loc[i]])\n",
    "df_edges['added_jaccard'] = added_jaccard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/40774 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40774/40774 [00:15<00:00, 2558.13it/s]\n",
      "100%|██████████| 2152/2152 [00:08<00:00, 240.29it/s]\n"
     ]
    }
   ],
   "source": [
    "grouped_sum = df_edges.groupby(['source']).sum()['added_jaccard']\n",
    "grouped_length = df_edges.groupby(['source']).size()\n",
    "\n",
    "i=0\n",
    "df_edges['added_jaccard_sum'] = [0]*len(df_edges)\n",
    "df_edges['length'] = [0]*len(df_edges)\n",
    "\n",
    "for key in tqdm(df_edges['source']):\n",
    "    df_edges['added_jaccard_sum'].loc[i] = grouped_sum[key]\n",
    "    df_edges['length'].loc[i] = grouped_length[key]\n",
    "    i+=1\n",
    "\n",
    "df_edges['border_length'] = df_edges['length'].apply(lambda x: int(x) - int(x*0.3))\n",
    "\n",
    "indices = []\n",
    "for key in tqdm(list(grouped_length.index)):\n",
    "    temp = df_edges[df_edges['source'] == key]\n",
    "    if grouped_length[key] < 2:\n",
    "        pass\n",
    "    else:\n",
    "        temp.sort_values('added_jaccard',inplace=True)\n",
    "    indices.append(list(temp.iloc[:int(temp['border_length'].mean())].index))\n",
    "    \n",
    "indices = flatten_chain(indices)\n",
    "df_edges_reduced = df_edges.drop(indices)\n",
    "# # df_edges_reduced = df_edges.drop(df_edges.loc[(df_edges['added_jaccard'] < df_edges['added_jaccard_sum'] * 0.2)].index)\n",
    "\n",
    "del df_edges_reduced[\"weight\"]\n",
    "df_edges_reduced.rename({\"added_jaccard\":'weight'}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df_edges[\"weight\"]\n",
    "df_edges.rename({\"added_jaccard\":'weight'}, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names_copy = df_names.reset_index(drop=False)\n",
    "df_names_copy.set_index('fullname',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fullname\n",
       "Andries Hof        12238851300\n",
       "Ine Dorresteijn    55225962900\n",
       "Jana Eichel        55796642600\n",
       "Name: scopusID, dtype: object"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_names_copy.loc[list(df_edges_authors.loc['land use']['source'])]['scopusID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Key\n",
      "Chain\n",
      "Key+Chain\n"
     ]
    }
   ],
   "source": [
    "t = 'key chain'\n",
    "key=''\n",
    "for part in t.split(' '):\n",
    "        part = part[0].upper() + part[1:]\n",
    "        print(part)\n",
    "        key += part + '+'\n",
    "\n",
    "key = key[:-1]\n",
    "print(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target\n",
       "land use        Andries Hof\n",
       "land use    Ine Dorresteijn\n",
       "land use        Jana Eichel\n",
       "Name: source, dtype: object"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_edges_authors.loc['land use']['source']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(df_edges_authors.loc['integrated assessment modelling']['source'], pd.Series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'integrated assessment modelling'"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keyword_nodes[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f2(x):\n",
    "    if isinstance(df_edges_authors.loc[x]['source'],  pd.Series) == True:\n",
    "        y = df_names_copy.loc[list(df_edges_authors.loc[x]['source'])]['scopusID'].to_list()\n",
    "    else:\n",
    "        y = []\n",
    "        tmp = df_names_copy.loc[df_edges_authors.loc[x]['source']]['scopusID']#\n",
    "        y.append(tmp)\n",
    "    return y\n",
    "    \n",
    "\n",
    "def query_keyword(key, list1):\n",
    "    #list1 : list of author scopus ids\n",
    "    #key: keyword\n",
    "    key1=''\n",
    "    if key != '':\n",
    "        for part in key.split(' '):\n",
    "            part = part[0].upper() + part[1:]\n",
    "            key1 += part + '+'\n",
    "\n",
    "        key1 = key1[:-1]\n",
    "    else:\n",
    "        key1 = ''\n",
    "    string = f\"https://www.scopus.com/results/results.uri?sort=plf-f&src=s&sid=f5247eb0cdb48d4556ef7a2ddb1892f5&sot=a&sdt=cl&cluster=scopubyr%2C%222020%22%2Ct%2C%222021%22%2Ct%2C%222022%22%2Ct%2C%222023%22%2Ct%2C%222024%22%2Ct%2Bscoexactkeywords%2C%22{key1}%22%2Ct&sl=40&s=\"\n",
    "    for auth in list1:\n",
    "        string += f\"AU-ID%28{auth}%29+OR+\"\n",
    "    string = string[:-4]\n",
    "    return string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%2c%222024%22%2ct\n",
    "%2c%222023%22%2ct\n",
    "%2c%222022%22%2ct\n",
    "%2c%222021%22%2ct\n",
    "%2c%222020%22%2ct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_nodes['View_author_profile'] = ['--']*len(df_nodes)\n",
    "df_nodes['View_author_profile'].loc[author_nodes] = df_authors['scopusID'].loc[author_nodes].apply(lambda x: f'https://www.scopus.com/authid/detail.uri?authorId={x}')\n",
    "\n",
    "\n",
    "df_nodes['View_publications'] = ['--']*len(df_nodes)\n",
    "df_nodes['View_publications'].loc[author_nodes] = df_authors['scopusID'].loc[author_nodes].apply(lambda x: f'https://www.scopus.com/results/results.uri?sort=plf-f&src=s&nlo=&nlr=&nls=&sid=38e25376e6563d5f8d3298de27536bcc&sot=aut&sdt=cl&cluster=scopubyr%2c%222024%22%2ct%2c%222023%22%2ct%2c%222022%22%2ct%2c%222021%22%2ct%2c%222020%22%2ct&sl=17&s=AU-ID%28{x}%29&origin=resultslist&zone=leftSideBar&editSaveSearch=&txGid=1ca2a667cd972701106c4e8248d84d66')\n",
    "\n",
    "test  = pd.Series(keyword_nodes).apply(lambda x:f2(x))\n",
    "urls = []\n",
    "i=0\n",
    "for keyword in keyword_nodes:\n",
    "    urls.append(query_keyword(keyword, test[i]))\n",
    "    i+=1\n",
    "df_nodes['View_publications'].loc[keyword_nodes] = urls \n",
    "\n",
    "\n",
    "\n",
    "def f1(d1):  \n",
    "    try:\n",
    "        v = list(d1.values())\n",
    "        k = list(d1.keys())\n",
    "        maxkey = int(k[v.index(max(v))])\n",
    "    except:\n",
    "        maxkey=1000\n",
    "    return maxkey\n",
    "\n",
    "main_subject_detailed_authors = df_subjectAreas.loc[list(df_authors['subjectAreaCount_detailed'].loc[author_nodes].apply(lambda x: f1(x)))]['detail']\n",
    "main_subject_general_authors = df_subjectAreas.loc[list(df_authors['subjectAreaCount_detailed'].loc[author_nodes].apply(lambda x: f1(x)))]['subject-classification']\n",
    "\n",
    "main_subject_detailed_keywords = df_subjectAreas.loc[list(df_keywords['subject_areas_code'].loc[keyword_nodes].apply(lambda x: f1(Counter(x))))]['detail']\n",
    "main_subject_general_keywords = df_subjectAreas.loc[list(df_keywords['subject_areas_code'].loc[keyword_nodes].apply(lambda x: f1(Counter(x))))]['subject-classification']\n",
    "\n",
    "\n",
    "df_nodes['main_subject'] = [0]*len(df_nodes)\n",
    "df_nodes['main_subject'].loc[author_nodes] = list(main_subject_general_authors)\n",
    "df_nodes['main_subject'].loc[keyword_nodes] = list(main_subject_general_keywords)\n",
    "\n",
    "df_nodes['detailed_main_subject'] = [0]*len(df_nodes)\n",
    "df_nodes['detailed_main_subject'].loc[author_nodes] = list(main_subject_detailed_authors)\n",
    "df_nodes['detailed_main_subject'].loc[keyword_nodes] = list(main_subject_detailed_keywords)\n",
    "\n",
    "df_nodes['total_citations'] = [0]*len(df_nodes)\n",
    "df_nodes['total_citations'].loc[keyword_nodes] = df_keywords['total_citations'].loc[keyword_nodes]\n",
    "df_nodes['total_citations'].loc[author_nodes] = df_authors['citationCount'].loc[author_nodes]\n",
    "\n",
    "df_nodes['num_of_papers'] = [0]*len(df_nodes)\n",
    "df_nodes['num_of_papers'].loc[keyword_nodes] = df_keywords['num_of_papers'].loc[keyword_nodes]\n",
    "df_nodes['num_of_papers'].loc[author_nodes] = df_authors['publishedArticles'].apply(lambda x: len(x)).loc[author_nodes]\n",
    "\n",
    "df_nodes['num_of_authors'] = [0]*len(df_nodes)\n",
    "df_nodes['num_of_authors'].loc[keyword_nodes] = df_keywords['num_of_authors'].loc[keyword_nodes]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "network_graph = nx.from_pandas_edgelist(df_edges, source='source', target='target', edge_attr=['weight'])\n",
    "network_graph.add_nodes_from((n, dict(d)) for n, d in df_nodes.iterrows())\n",
    "\n",
    "#community detection using louvain algorithm\n",
    "partion = community_louvain.best_partition(network_graph)\n",
    "nx.set_node_attributes(network_graph, partion, 'community')\n",
    "\n",
    "nx.write_gexf(network_graph,'network_testing_18042024.gexf')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "newenv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}