feat: add statistics about total number of repos handled to the dashb…

…oard
eclipse-csi · Feb 28, 2024 · e75a508 · e75a508
1 parent 3d5346a
commit e75a508
Show file tree

Hide file tree

Showing 7 changed files with 67 additions and 10 deletions.
diff --git a/otterdog/webapp/db/__init__.py b/otterdog/webapp/db/__init__.py
@@ -43,6 +43,7 @@ async def init_mongo_database(mongo: Mongo) -> None:
         ConfigurationModel,
         InstallationModel,
         PullRequestModel,
+        StatisticsModel,
         TaskModel,
     )
 
@@ -52,5 +53,6 @@ async def init_mongo_database(mongo: Mongo) -> None:
             TaskModel,
             ConfigurationModel,
             PullRequestModel,
+            StatisticsModel,
         ]  # type: ignore
     )
diff --git a/otterdog/webapp/db/models.py b/otterdog/webapp/db/models.py
@@ -109,3 +109,9 @@ class PullRequestModel(Model):
             ),
         ],
     }
+
+
+class StatisticsModel(Model):
+    project_name: str = Field(primary_field=True)
+    github_id: str = Field(index=True)
+    num_repos: int
diff --git a/otterdog/webapp/db/service.py b/otterdog/webapp/db/service.py
@@ -29,6 +29,7 @@
     InstallationStatus,
     PullRequestModel,
     PullRequestStatus,
+    StatisticsModel,
     TaskModel,
     TaskStatus,
 )
@@ -400,3 +401,23 @@ async def get_merged_pull_requests_paged(params: dict[str, str]) -> tuple[list[P
         ),
         await mongo.odm.count(PullRequestModel, *queries),
     )
+
+
+async def save_statistics(model: StatisticsModel) -> None:
+    await mongo.odm.save(model)
+
+
+async def get_statistics() -> tuple[int, int]:
+    pipeline = [
+        {
+            "$group": {
+                "_id": None,
+                "num_projects": {"$sum": 1},
+                "num_repos": {"$sum": "$num_repos"},
+            },
+        }
+    ]
+
+    collection = mongo.odm.get_collection(StatisticsModel)
+    stats = await collection.aggregate(pipeline).next()
+    return stats["num_projects"], stats["num_repos"]
diff --git a/otterdog/webapp/home/routes.py b/otterdog/webapp/home/routes.py
@@ -20,6 +20,7 @@
     get_merged_pull_requests_count,
     get_open_or_incomplete_pull_requests,
     get_open_or_incomplete_pull_requests_count,
+    get_statistics,
     get_tasks,
 )
 from otterdog.webapp.tasks.fetch_all_pull_requests import FetchAllPullRequestsTask
@@ -38,12 +39,14 @@ async def index():
     installations = await get_installations()
     configurations = await get_configurations()
     configurations_by_key = associate_by_key(configurations, lambda x: x.github_id)
+    statistics = await get_statistics()
     return await render_home_template(
         "index.html",
         open_pull_request_count=await get_open_or_incomplete_pull_requests_count(),
         merged_pull_request_count=await get_merged_pull_requests_count(),
         installations=installations,
         configurations=configurations_by_key,
+        total_repository_count=statistics[1],
     )
 
 

diff --git a/otterdog/webapp/tasks/__init__.py b/otterdog/webapp/tasks/__init__.py
@@ -6,9 +6,9 @@
 #  SPDX-License-Identifier: EPL-2.0
 #  *******************************************************************************
 
+import contextlib
 from abc import ABC, abstractmethod
 from collections.abc import AsyncIterator
-from contextlib import asynccontextmanager
 from functools import cached_property
 from logging import Logger, getLogger
 from typing import Generic, Protocol, TypeVar
@@ -103,7 +103,7 @@ async def rest_api(self) -> RestApi:
 
     # Ignore pycharm warning:
     # https://youtrack.jetbrains.com/issue/PY-66517/False-unexpected-argument-with-asynccontextmanager-defined-as-a-method
-    @asynccontextmanager
+    @contextlib.asynccontextmanager
     async def get_organization_config(
         self,
         initialize_template: bool = True,

diff --git a/otterdog/webapp/tasks/fetch_config.py b/otterdog/webapp/tasks/fetch_config.py
@@ -8,9 +8,10 @@
 
 from dataclasses import dataclass
 
+from otterdog.models.github_organization import GitHubOrganization
 from otterdog.utils import jsonnet_evaluate_file
-from otterdog.webapp.db.models import ConfigurationModel, TaskModel
-from otterdog.webapp.db.service import save_config
+from otterdog.webapp.db.models import ConfigurationModel, StatisticsModel, TaskModel
+from otterdog.webapp.db.service import save_config, save_statistics
 from otterdog.webapp.tasks import InstallationBasedTask, Task
 from otterdog.webapp.utils import fetch_config_from_github
 
@@ -48,6 +49,7 @@ async def _execute(self) -> None:
                 config_file,
             )
 
+            # save configuration
             config_data = jsonnet_evaluate_file(config_file)
             config = ConfigurationModel(  # type: ignore
                 github_id=self.org_id,
@@ -57,5 +59,14 @@ async def _execute(self) -> None:
             )
             await save_config(config)
 
+            # save statistics
+            github_organization = GitHubOrganization.from_model_data(config_data)
+            statistics = StatisticsModel(  # type: ignore
+                project_name=org_config.name,
+                github_id=self.org_id,
+                num_repos=len(github_organization.repositories),
+            )
+            await save_statistics(statistics)
+
     def __repr__(self) -> str:
         return f"FetchConfigTask(repo={self.org_id}/{self.repo_name})"
diff --git a/otterdog/webapp/templates/home/index.html b/otterdog/webapp/templates/home/index.html
@@ -46,23 +46,23 @@ <h1 class="m-0 text-dark">Dashboard</h1>
       <div class="container-fluid">
         <!-- Small boxes (Stat box) -->
         <div class="row">
-          <div class="col-lg-3 col-6">
+          <div class="col-lg-2">
             <!-- small box -->
             <div class="small-box bg-info">
               <div class="inner">
                 <h3>{{ installations|length }}</h3>
                 <p>GitHub Organizations</p>
               </div>
               <div class="icon">
-                <i class="fab fa-github"></i>
+                <i class="fas fa-sitemap"></i>
               </div>
               <a href="/admin/organizations" class="small-box-footer">More info <i class="fas fa-arrow-circle-right"></i></a>
             </div>
           </div>
           <!-- ./col -->
-          <div class="col-lg-3 col-6">
+          <div class="col-lg-2">
             <!-- small box -->
-            <div class="small-box bg-info">
+            <div class="small-box bg-warning">
               <div class="inner">
                 <h3>{{ open_pull_request_count }}</h3>
                 <p>Open Pull Requests</p>
@@ -74,20 +74,34 @@ <h3>{{ open_pull_request_count }}</h3>
             </div>
           </div>
           <!-- ./col -->
-          <div class="col-lg-3 col-6">
+          <div class="col-lg-2">
             <!-- small box -->
             <div class="small-box bg-success">
               <div class="inner">
                 <h3>{{ merged_pull_request_count }}</h3>
                 <p>Merged Pull Requests</p>
               </div>
               <div class="icon">
-                <i class="fas fa-code-pull-request"></i>
+                <i class="fas fa-code-merge"></i>
               </div>
               <a href="/admin/pullrequests" class="small-box-footer">More info <i class="fas fa-arrow-circle-right"></i></a>
             </div>
           </div>
           <!-- ./col -->
+          <div class="col-lg-2">
+            <!-- small box -->
+            <div class="small-box bg-info">
+              <div class="inner">
+                <h3>{{ total_repository_count }}</h3>
+                <p>Total Repository Count</p>
+              </div>
+              <div class="icon">
+                <i class="fab fa-github"></i>
+              </div>
+<!--              <a href="/statistics" class="small-box-footer">More info <i class="fas fa-arrow-circle-right"></i></a>-->
+            </div>
+          </div>
+          <!-- ./col -->
         </div>
         <!-- ./row -->