Added test database for parsers

analytics/metadata_parse.py

@@ -0,0 +1,48 @@
+import json
+import numpy as np
+from collections import defaultdict, Counter
+
+# Load data from the 'data.json' file
+with open('data.json', 'r') as json_file:
+    data = json.load(json_file)
+
+# Create a dictionary to store JSON output
+output_data = {}
+
+# Initialize a dictionary to store MIME-type statistics for each tag
+mime_type_stats = defaultdict(lambda: defaultdict(Counter))
+
+# Iterate through the repositories
+for repo_name, tags in data.items():
+    # Initialize dictionaries for repo metrics
+    repo_metrics = {}
+
+    # Extract and process the data for each tag
+    for tag_name, files in tags.items():
+        line_count_list = [int(file_data["line_count"]) for file_data in files.values()]
+        mean_line_count = np.mean(line_count_list)
+        median_line_count = np.median(line_count_list)
+        std_dev_line_count = np.std(line_count_list, ddof=0)  # Use ddof=0 for population standard deviation
+
+        # Create a dictionary for metrics per tag
+        tag_metrics = {
+            "mean": mean_line_count,
+            "median": median_line_count,
+            "std_dev": std_dev_line_count
+        }
+        repo_metrics[tag_name] = tag_metrics
+
+        # Collect MIME-type data per tag
+        mime_counter = Counter(file_data["mime_type"] for file_data in files.values())
+        mime_type_stats[repo_name][tag_name] = mime_counter
+
+    # Add the repo metrics to the JSON output
+    output_data[repo_name] = repo_metrics
+
+# Output data to JSON
+with open("output_data.json", "w") as json_file:
+    json.dump(output_data, json_file, indent=2)
+
+# Output MIME-type data to JSON for each tag
+with open("mime_type_data.json", "w") as json_file:
+    json.dump(mime_type_stats, json_file, indent=2)

analytics/mime_type_data.json

@@ -0,0 +1,292 @@
+{
+  "claws-mail-plugins": {
+    "debian/3.11.1-3+deb8u1": {
+      "text/x-shellscript": 31,
+      "text/x-m4": 32,
+      "text/x-makefile": 103,
+      "image/png": 3,
+      "text/plain": 586,
+      "text/xml": 93,
+      "application/pdf": 5,
+      "text/html": 5,
+      "application/postscript": 5,
+      "text/troff": 1,
+      "inode/x-empty": 1,
+      "image/x-xpmi": 162,
+      "application/pgp-keys": 1,
+      "text/x-diff": 8,
+      "text/x-po": 26,
+      "application/x-gettext-translation": 26,
+      "text/x-c": 728,
+      "image/vnd.microsoft.icon": 1,
+      "text/x-c++": 1,
+      "text/x-perl": 31,
+      "text/x-python": 9,
+      "application/csv": 3,
+      "application/octet-stream": 2,
+      "application/x-git": 1
+    },
+    "3.14.1-3": {
+      "text/plain": 373,
+      "image/x-xpmi": 1,
+      "text/xml": 28,
+      "text/x-makefile": 1,
+      "application/pgp-keys": 1,
+      "text/x-diff": 2,
+      "application/octet-stream": 2,
+      "application/x-git": 1,
+      "text/x-perl": 1,
+      "text/x-shellscript": 11
+    },
+    "debian/3.17.8-1": {
+      "text/x-shellscript": 32,
+      "text/x-m4": 37,
+      "text/x-makefile": 93,
+      "image/png": 3,
+      "text/plain": 555,
+      "text/xml": 40,
+      "application/pdf": 3,
+      "text/html": 3,
+      "application/postscript": 3,
+      "text/troff": 1,
+      "image/x-xpmi": 197,
+      "application/pgp-keys": 1,
+      "text/x-diff": 2,
+      "text/x-po": 25,
+      "application/x-gettext-translation": 24,
+      "text/x-c": 805,
+      "image/vnd.microsoft.icon": 1,
+      "text/x-c++": 47,
+      "text/x-perl": 23,
+      "text/x-python": 9,
+      "application/octet-stream": 2,
+      "application/x-git": 1
+    },
+    "debian/3.17.3-2": {
+      "text/x-shellscript": 32,
+      "text/x-m4": 37,
+      "text/x-makefile": 88,
+      "image/png": 3,
+      "text/plain": 522,
+      "text/xml": 68,
+      "application/pdf": 3,
+      "text/html": 3,
+      "application/postscript": 3,
+      "text/troff": 1,
+      "image/x-xpmi": 197,
+      "application/pgp-keys": 1,
+      "text/x-diff": 3,
+      "text/x-po": 24,
+      "application/x-gettext-translation": 23,
+      "text/x-c": 754,
+      "image/vnd.microsoft.icon": 1,
+      "text/x-c++": 1,
+      "text/x-perl": 24,
+      "text/x-python": 9,
+      "application/octet-stream": 2,
+      "application/x-git": 1
+    }
+  },
+  "icinga2-bin": {
+    "debian/2.10.3-2": {
+      "text/plain": 521,
+      "text/x-c": 364,
+      "text/x-python": 6,
+      "image/bmp": 1,
+      "image/x-ms-bmp": 1,
+      "text/x-c++": 332,
+      "text/xml": 13,
+      "image/vnd.microsoft.icon": 2,
+      "image/png": 47,
+      "text/x-shellscript": 36,
+      "text/troff": 1,
+      "inode/x-empty": 1,
+      "text/html": 2,
+      "text/x-po": 15,
+      "text/x-diff": 3,
+      "application/octet-stream": 2,
+      "application/x-git": 1,
+      "text/x-perl": 1
+    },
+    "debian/2.12.3-1": {
+      "text/plain": 511,
+      "text/x-c": 346,
+      "image/x-ms-bmp": 2,
+      "text/x-c++": 348,
+      "text/xml": 13,
+      "image/vnd.microsoft.icon": 2,
+      "image/png": 56,
+      "application/octet-stream": 3,
+      "text/x-shellscript": 33,
+      "text/x-python": 3,
+      "text/troff": 1,
+      "inode/x-empty": 1,
+      "text/x-po": 15,
+      "text/x-diff": 5,
+      "application/x-git": 1,
+      "text/x-perl": 1
+    },
+    "debian/2.6.0-2": {
+      "text/plain": 517,
+      "text/x-python": 17,
+      "text/x-c": 378,
+      "image/bmp": 1,
+      "image/x-ms-bmp": 1,
+      "text/x-c++": 331,
+      "text/xml": 14,
+      "image/vnd.microsoft.icon": 3,
+      "image/png": 40,
+      "text/x-shellscript": 55,
+      "text/troff": 1,
+      "image/x-icns": 1,
+      "image/x-xpmi": 1,
+      "inode/x-empty": 1,
+      "text/html": 2,
+      "text/x-po": 15,
+      "text/x-diff": 3,
+      "application/json": 1,
+      "application/octet-stream": 2,
+      "application/x-git": 1,
+      "text/x-perl": 1,
+      "text/x-objective-c": 1
+    }
+  },
+  "libavc1394-dev": {
+    "debian/0.5.4-5": {
+      "text/x-shellscript": 17,
+      "text/x-makefile": 7,
+      "text/plain": 48,
+      "text/x-m4": 1,
+      "text/x-c": 18,
+      "text/x-diff": 3,
+      "text/troff": 3,
+      "application/octet-stream": 2,
+      "application/x-git": 1,
+      "text/x-perl": 1
+    }
+  },
+  "golly": {
+    "debian/3.2-2": {
+      "text/plain": 464,
+      "text/x-shellscript": 22,
+      "image/png": 71,
+      "text/xml": 28,
+      "text/html": 129,
+      "text/x-java": 12,
+      "text/x-c": 146,
+      "application/zip": 7,
+      "text/x-objective-c": 24,
+      "audio/x-aiff": 1,
+      "application/json": 1,
+      "text/x-makefile": 4,
+      "text/x-c++": 60,
+      "text/x-m4": 4,
+      "image/x-icns": 3,
+      "image/vnd.microsoft.icon": 5,
+      "image/x-xpmi": 51,
+      "text/x-python": 58,
+      "audio/x-wav": 1,
+      "image/gif": 3,
+      "application/gzip": 24,
+      "application/octet-stream": 9,
+      "text/troff": 1,
+      "text/x-diff": 2,
+      "application/x-git": 1,
+      "text/x-perl": 2
+    },
+    "debian/3.3-1": {
+      "text/plain": 470,
+      "text/x-shellscript": 20,
+      "image/png": 71,
+      "text/xml": 28,
+      "text/html": 133,
+      "text/x-java": 12,
+      "text/x-c": 147,
+      "application/zip": 11,
+      "text/x-objective-c": 24,
+      "audio/x-aiff": 1,
+      "application/json": 1,
+      "text/x-makefile": 4,
+      "text/x-c++": 59,
+      "text/x-m4": 4,
+      "image/x-icns": 3,
+      "image/vnd.microsoft.icon": 5,
+      "image/x-xpmi": 51,
+      "text/x-python": 58,
+      "audio/x-wav": 1,
+      "image/gif": 3,
+      "application/gzip": 27,
+      "application/octet-stream": 9,
+      "text/troff": 1,
+      "text/x-diff": 5,
+      "application/x-git": 1,
+      "text/x-perl": 2
+    },
+    "debian/2.8-1": {
+      "text/plain": 423,
+      "text/x-shellscript": 22,
+      "text/xml": 29,
+      "image/png": 52,
+      "text/html": 125,
+      "text/x-diff": 16,
+      "text/x-c": 144,
+      "application/zip": 3,
+      "text/x-objective-c": 24,
+      "audio/x-aiff": 1,
+      "text/x-makefile": 4,
+      "text/x-c++": 57,
+      "text/x-m4": 4,
+      "image/x-icns": 3,
+      "image/vnd.microsoft.icon": 5,
+      "image/x-xpmi": 51,
+      "text/x-python": 58,
+      "audio/x-wav": 1,
+      "image/gif": 3,
+      "application/gzip": 18,
+      "text/x-java": 1,
+      "text/troff": 1,
+      "application/octet-stream": 2,
+      "application/x-git": 1,
+      "text/x-perl": 2
+    }
+  },
+  "libghc-dependent-sum-template-prof": {
+    "haskell-dependent-sum-template_v0.0.0.5-3": {
+      "text/plain": 20913,
+      "text/troff": 27,
+      "text/x-shellscript": 39,
+      "text/x-diff": 173,
+      "inode/x-empty": 4,
+      "image/png": 2,
+      "text/html": 2,
+      "text/xml": 2,
+      "application/octet-stream": 3,
+      "text/x-lisp": 1,
+      "text/x-perl": 3,
+      "application/pgp-keys": 1,
+      "application/gzip": 1,
+      "image/x-xpmi": 1,
+      "application/x-git": 1
+    }
+  },
+  "libgeoip2-perl": {
+    "archive/debian/2.006002-1": {
+      "text/plain": 119,
+      "text/x-perl": 3,
+      "image/x-tga": 8,
+      "application/json": 11,
+      "application/octet-stream": 32,
+      "application/x-git": 1,
+      "text/x-shellscript": 11
+    },
+    "debian/2.006001-2": {
+      "text/plain": 120,
+      "text/x-perl": 3,
+      "image/x-tga": 8,
+      "application/json": 11,
+      "application/octet-stream": 32,
+      "application/x-git": 1,
+      "text/x-shellscript": 11
+    }
+  }
+}