Simplify and improve stats

bin/extract_galaxy_tools.py

@@ -59,45 +59,36 @@ def get_last_url_position(toot_id: str) -> str:
     """
 
     if "/" in toot_id:
-        toot_id = toot_id.split("/")[-2]
+        toot_id = toot_id.split("/")[-1]
     return toot_id
 
 
-def add_tool_stats_to_tools(tools_df: pd.DataFrame, tool_stats_path: Path, column_name: str) -> pd.DataFrame:
+def get_tool_stats_from_stats_file(tool_stats_df: pd.DataFrame, tool_ids: List[str]) -> int:
     """
-    Adds the usage statistics to the community tool table
+    Computes a count for tool stats based on the tool id. The counts for local and toolshed installed tools are
+    aggregated. All tool versions are also aggregated.
 
-    :param tool_stats_path: path to the table with
-        the tool stats (csv,
-        must include "tool_name" and "count")
-    :param tools_path: path to the table with
-        the tools (csv,
-        must include "Galaxy wrapper id")
-    :param output_path: path to store the new table
-    :param column_name: column to add for the tool stats,
-        different columns could be added for the main servers
-    """
 
-    # parse csvs
-    tool_stats_df = pd.read_csv(tool_stats_path)
+    :param tools_stats_df: df with tools stats in the form `toolshed.g2.bx.psu.edu/repos/iuc/snpsift/snpSift_filter,3394539`
+    :tool_ids: tool ids to get statistics for and aggregate
+    """
 
     # extract tool id
     tool_stats_df["Galaxy wrapper id"] = tool_stats_df["tool_name"].apply(get_last_url_position)
+    # print(tool_stats_df["Galaxy wrapper id"].to_list())
 
-    # group local and toolshed tools into one entry
-    grouped_tool_stats_tools = tool_stats_df.groupby("Galaxy wrapper id", as_index=False)["count"].sum()
-
-    # keep all rows of the tools table (how='right'), also for those where no stats are available
-    community_tool_stats = pd.merge(grouped_tool_stats_tools, tools_df, how="right", on="Galaxy wrapper id")
-    community_tool_stats.rename(columns={"count": column_name}, inplace=True)
+    agg_count = 0
+    for tool_id in tool_ids:
+        if tool_id in tool_stats_df["Galaxy wrapper id"].to_list():
 
-    return community_tool_stats
+            # get stats of the tool for all versions
+            counts = tool_stats_df.loc[(tool_stats_df["Galaxy wrapper id"] == tool_id), "count"]
+            agg_versions = counts.sum()
 
+            # aggregate all counts for all tools in the suite
+            agg_count += agg_versions
 
-def add_usage_stats_for_all_server(tools_df: pd.DataFrame) -> pd.DataFrame:
-    for column, path in GALAXY_TOOL_STATS.items():
-        tools_df = add_tool_stats_to_tools(tools_df, path, column)
-    return tools_df
+    return int(agg_count)
 
 
 def read_file(filepath: Optional[str]) -> List[str]:
@@ -558,8 +549,8 @@ def export_tools_to_tsv(
         # the Galaxy tools need to be formatted for the add_instances_to_table to work
         df["Galaxy tool ids"] = format_list_column(df["Galaxy tool ids"])
 
-    if add_usage_stats:
-        df = add_usage_stats_for_all_server(df)
+    # if add_usage_stats:
+    #     df = add_usage_stats_for_all_server(df)
 
     df.to_csv(output_fp, sep="\t", index=False)
 
@@ -761,6 +752,11 @@ def reduce_ontology_terms(terms: List, ontology: Any) -> List:
                     url = row["url"]
                     tool[f"Tools available on {name}"] = check_tools_on_servers(tool["Galaxy tool ids"], url)
 
+            # add tool stats
+            for name, path in GALAXY_TOOL_STATS.items():
+                tool_stats_df = pd.read_csv(path)
+                tool[name] = get_tool_stats_from_stats_file(tool_stats_df, tool["Galaxy tool ids"])
+
         export_tools_to_json(tools, args.all_tools_json)
         export_tools_to_tsv(tools, args.all_tools, format_list_col=True, add_usage_stats=True)
 

bin/tests/test_tool_stats.py

@@ -0,0 +1,58 @@
+import os
+import sys
+from typing import List
+
+import pandas as pd
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+
+from extract_galaxy_tools import GALAXY_TOOL_STATS
+
+
+def get_last_url_position(toot_id: str) -> str:
+    """
+    Returns the last url position of the toot_id, if the value is not a
+    url it returns the toot_id. So works for local and toolshed
+    installed tools.
+
+    :param tool_id: galaxy tool id
+    """
+
+    if "/" in toot_id:
+        toot_id = toot_id.split("/")[-1]
+    return toot_id
+
+
+def get_tool_stats_from_stats_file(tool_stats_df: pd.DataFrame, tool_ids: List[str]) -> int:
+    """
+    Adds the usage statistics to the community tool table
+
+    :param tools_stats_df: df with tools stats in the form `toolshed.g2.bx.psu.edu/repos/iuc/snpsift/snpSift_filter,3394539`
+    :tool_ids: tool ids to get statistics for and aggregate
+    """
+
+    # extract tool id
+    tool_stats_df["Galaxy wrapper id"] = tool_stats_df["tool_name"].apply(get_last_url_position)
+    # print(tool_stats_df["Galaxy wrapper id"].to_list())
+
+    agg_count = 0
+    for tool_id in tool_ids:
+        if tool_id in tool_stats_df["Galaxy wrapper id"].to_list():
+
+            # get stats of the tool for all versions
+            counts = tool_stats_df.loc[(tool_stats_df["Galaxy wrapper id"] == tool_id), "count"]
+            agg_versions = counts.sum()
+
+            # aggregate all counts for all tools in the suite
+            agg_count += agg_versions
+
+    return int(agg_count)
+
+
+tools_stats_df = pd.read_csv(GALAXY_TOOL_STATS["Total tool usage (usegalaxy.eu)"])
+
+tool_ids = ["nextclade"]
+
+counts = get_tool_stats_from_stats_file(tools_stats_df, tool_ids)
+
+print(counts)