Graph analytics

analytics/Network_Analysis/betweenness_centrality.py

@@ -0,0 +1,54 @@
+import sqlite3
+import networkx as nx
+from prettytable import PrettyTable
+import time
+from constants import DB_LOC, CONSTRUCT_GRAPH
+
+# Define function to retrieve edge data from SQLite database
+def get_edge_data():
+    conn = sqlite3.connect(DB_LOC)
+    cursor = conn.cursor()
+    query = CONSTRUCT_GRAPH
+    cursor.execute(query)
+    items = cursor.fetchall()
+    conn.close()
+    return items
+
+# Define function to insert top node data into a table format in a md file
+def insert_into_md(top_nodes):
+    myTable = PrettyTable(["Sno","Top 5000 valued packages", "Betweenness Centrality"])
+    i=1
+    for node, value in top_nodes:
+        myTable.add_row([i , node , value])
+        i+=1
+    myTable.add_row([ '-' , '-','-'])
+
+
+    data = myTable.get_string()
+    with open('plots/between_centrality.md', 'w') as f:
+            f.write(data)
+
+# Define function to create directed graph and calculate Betweenness centrality metric for each node
+def get_top_nodes(items):
+    G = nx.DiGraph()
+    G.add_edges_from(items)
+
+    bet = nx.betweenness_centrality(G,k=500)
+
+    top_nodes = sorted(bet.items(), key=lambda x: x[1], reverse=True)[:5000]
+    return top_nodes
+
+# Main function to call other functions
+def main():
+    t_in = time.time()
+    print("program running")
+
+    items = get_edge_data()
+    top_nodes = get_top_nodes(items)
+    insert_into_md(top_nodes)
+
+    t_out = time.time()
+    print('Program run time in seconds:', t_out - t_in, '(s)')
+
+if __name__ == '__main__':
+    main()

analytics/Network_Analysis/centrality.py

@@ -0,0 +1,69 @@
+import sqlite3
+import networkx as nx
+from prettytable import PrettyTable
+import time
+from constants import DB_LOC, CONSTRUCT_GRAPH
+
+# Define function to retrieve edge data from SQLite database
+def get_edge_data():
+    conn = sqlite3.connect(DB_LOC)
+    cursor = conn.cursor()
+    query = CONSTRUCT_GRAPH
+    cursor.execute(query)
+    items = cursor.fetchall()
+    conn.close()
+    return items
+
+# Define function to insert top node data into a table format in a md file
+def insert_into_md(values):
+    with open('plots/centrality.md', 'w') as f:
+        f.write('| Package | Coreness | Degree Centrality | Eigenvector Centrality | Betweenness Centrality | Closeness Centrality | Page Rank |\n')
+        f.write('| --- | --- | --- | --- | --- | --- | --- |\n')
+        for value in values:
+            f.write(f'| {value[0]} | {value[1]} | {value[2]} | {value[3]} | {value[4]} | {value[5]} | {value[6]} |\n')
+
+
+# Define function to create directed graph and calculate Eigenvector Centrality metric for each node
+def get_top_nodes(items):
+    G = nx.DiGraph()
+    G.add_edges_from(items)
+    # calculate degree centrality
+    degree_centrality = nx.degree_centrality(G)
+
+    # calculate eigenvector centrality
+    eigenvector_centrality = nx.eigenvector_centrality(G)
+
+    # calculate PageRank
+    pagerank = nx.pagerank(G)
+
+    # calculate betweenness centrality
+    betweenness_centrality = nx.betweenness_centrality(G,k=500)
+
+    # calculate closeness centrality
+    closeness_centrality = nx.closeness_centrality(G)
+
+    G.remove_edges_from(nx.selfloop_edges(G))
+    # calculate coreness
+    coreness = nx.core_number(G)
+
+    # create a list of tuples with the node, coreness, degree centrality, eigenvector centrality, betweenness centrality, closeness centrality and PageRank values
+    values = [(node, coreness[node], degree_centrality[node], eigenvector_centrality[node] , betweenness_centrality[node], closeness_centrality[node], pagerank[node]) for node in G.nodes()]
+
+    # sort the list by coreness
+    values.sort(key=lambda x: x[1], reverse=True)
+    return values
+
+# Main function to call other functions
+def main():
+    t_in = time.time()
+    print("program running")
+
+    items = get_edge_data()
+    values = get_top_nodes(items)
+    insert_into_md(values)
+
+    t_out = time.time()
+    print('Program run time in seconds:', t_out - t_in, '(s)')
+
+if __name__ == '__main__':
+    main()

analytics/Network_Analysis/closeness_centrality.py

@@ -0,0 +1,54 @@
+import sqlite3
+import networkx as nx
+from prettytable import PrettyTable
+import time
+from constants import DB_LOC, CONSTRUCT_GRAPH
+
+# Define function to retrieve edge data from SQLite database
+def get_edge_data():
+    conn = sqlite3.connect(DB_LOC)
+    cursor = conn.cursor()
+    query = CONSTRUCT_GRAPH
+    cursor.execute(query)
+    items = cursor.fetchall()
+    conn.close()
+    return items
+
+# Define function to insert top node data into a table format in a md file
+def insert_into_md(top_nodes):
+    myTable = PrettyTable(["Sno","Top 5000 valued packages", "Closeness Centrality"])
+    i=1
+    for node, value in top_nodes:
+        myTable.add_row([i , node , value])
+        i+=1
+    myTable.add_row([ '-' , '-','-'])
+
+
+    data = myTable.get_string()
+    with open('plots/closeness_centrality.md', 'w') as f:
+            f.write(data)
+
+# Define function to create directed graph and calculate Closeness centrality metric for each node
+def get_top_nodes(items):
+    G = nx.DiGraph()
+    G.add_edges_from(items)
+
+    close = nx.closeness_centrality(G,k=500)
+
+    top_nodes = sorted(close.items(), key=lambda x: x[1], reverse=True)[:5000]
+    return top_nodes
+
+# Main function to call other functions
+def main():
+    t_in = time.time()
+    print("program running")
+
+    items = get_edge_data()
+    top_nodes = get_top_nodes(items)
+    insert_into_md(top_nodes)
+
+    t_out = time.time()
+    print('Program run time in seconds:', t_out - t_in, '(s)')
+
+if __name__ == '__main__':
+    main()

analytics/Network_Analysis/core_minres.py

@@ -0,0 +1,56 @@
+import sqlite3
+import networkx as nx
+from prettytable import PrettyTable
+import cpnet
+import time
+from constants import DB_LOC, CONSTRUCT_GRAPH
+# Define function to retrieve edge data from SQLite database
+def get_edge_data():
+    conn = sqlite3.connect(DB_LOC)
+    cursor = conn.cursor()
+    query = CONSTRUCT_GRAPH
+    cursor.execute(query)
+    items = cursor.fetchall()
+    conn.close()
+    return items
+
+# Define function to create directed graph and calculate MINRES metric for each node
+def get_top_nodes(items):
+    G = nx.DiGraph()
+    G.add_edges_from(items)
+
+    alg = cpnet.MINRES()
+    alg.detect(G)
+    pair_id = alg.get_pair_id()
+    coreness = alg.get_coreness()
+
+    top_nodes = sorted(coreness.items(), key=lambda x: x[1], reverse=True)[:500]
+    return top_nodes
+
+# Define function to insert top node data into a table format in a md file
+def insert_into_md(top_nodes):
+    myTable = PrettyTable(["Sno","Top 500 coreness value packages", "CP-MINRES"])
+    i=0
+    for node, value in top_nodes:
+        myTable.add_row([i , node , value])
+        i+=1
+    myTable.add_row([ '-' , '-','-'])
+
+    data = myTable.get_string()
+    with open('plots/minres.md', 'w') as f:
+            f.write(data)
+
+# Main function to call other functions
+def main():
+    t_in = time.time()
+    print("program running")
+
+    items = get_edge_data()
+    top_nodes = get_top_nodes(items)
+    insert_into_md(top_nodes)
+
+    t_out = time.time()
+    print('Program run time in seconds:', t_out - t_in, '(s)')
+
+if __name__ == '__main__':
+    main()

analytics/Network_Analysis/coreness.py

@@ -0,0 +1,55 @@
+import sqlite3
+import networkx as nx
+from prettytable import PrettyTable
+import time
+from constants import DB_LOC, CONSTRUCT_GRAPH
+
+# Define function to retrieve edge data from SQLite database
+def get_edge_data():
+    conn = sqlite3.connect(DB_LOC)
+    cursor = conn.cursor()
+    query = CONSTRUCT_GRAPH
+    cursor.execute(query)
+    items = cursor.fetchall()
+    conn.close()
+    return items
+
+# Define function to insert top node data into a table format in a md file
+def insert_into_md(top_nodes):
+    myTable = PrettyTable(["Sno","Top 5000 valued packages", "Coreness"])
+    i=1
+    for node, value in top_nodes:
+        myTable.add_row([i , node , value])
+        i+=1
+    myTable.add_row([ '-' , '-','-'])
+
+
+    data = myTable.get_string()
+    with open('plots/coreness.md', 'w') as f:
+            f.write(data)
+
+# Define function to create directed graph and calculate Coreness metric for each node
+def get_top_nodes(items):
+    G = nx.DiGraph()
+    G.add_edges_from(items)
+    # compute the core number of each node
+    G.remove_edges_from(nx.selfloop_edges(G))
+    core= nx.core_number(G)
+
+    top_nodes = sorted(core.items(), key=lambda x: x[1], reverse=True)[:50000]
+    return top_nodes
+
+# Main function to call other functions
+def main():
+    t_in = time.time()
+    print("program running")
+
+    items = get_edge_data()
+    top_nodes = get_top_nodes(items)
+    insert_into_md(top_nodes)
+
+    t_out = time.time()
+    print('Program run time in seconds:', t_out - t_in, '(s)')
+
+if __name__ == '__main__':
+    main()

analytics/Network_Analysis/degree_centrality.py

@@ -0,0 +1,53 @@
+import sqlite3
+import networkx as nx
+from prettytable import PrettyTable
+import time
+from constants import DB_LOC, CONSTRUCT_GRAPH
+
+# Define function to retrieve edge data from SQLite database
+def get_edge_data():
+    conn = sqlite3.connect(DB_LOC)
+    cursor = conn.cursor()
+    query = CONSTRUCT_GRAPH
+    cursor.execute(query)
+    items = cursor.fetchall()
+    conn.close()
+    return items
+
+# Define function to insert top node data into a table format in a md file
+def insert_into_md(top_nodes):
+    myTable = PrettyTable(["Sno","Top 5000 valued packages", "Degree Centrality"])
+    i=1
+    for node, value in top_nodes:
+        myTable.add_row([i , node , value])
+        i+=1
+    myTable.add_row([ '-' , '-','-'])
+
+
+    data = myTable.get_string()
+    with open('plots/degree_centrality.md', 'w') as f:
+            f.write(data)
+
+# Define function to create directed graph and calculate Degree centrality metric for each node
+def get_top_nodes(items):
+    G = nx.DiGraph()
+    G.add_edges_from(items)
+    deg = nx.degree_centrality(G)
+
+    top_nodes = sorted(deg.items(), key=lambda x: x[1], reverse=True)[:5000]
+    return top_nodes
+
+# Main function to call other functions
+def main():
+    t_in = time.time()
+    print("program running")
+
+    items = get_edge_data()
+    top_nodes = get_top_nodes(items)
+    insert_into_md(top_nodes)
+
+    t_out = time.time()
+    print('Program run time in seconds:', t_out - t_in, '(s)')
+
+if __name__ == '__main__':
+    main()