add contributors + fix image links

projectIndexer.py

@@ -18,15 +18,16 @@ def cli():
 @click.option('--github-token', default=None, help='Github token (default is in .env file)')
 @click.option('--output-repos', '-o', default='data/repos.json', help='build file (default is repos.json)')
 @click.option('--output-readme', '-r', default='data/readme.json', help='build file (default is readme.json)')
+@click.option('--output-contributors', '-r', default='data/contributors.json', help='build file (default is contributors.json)')
 @click.option('--verbose', default=False, is_flag=True, help='Verbose build')
-def fetch_github(github_token, output_repos, output_readme, verbose):
+def fetch_github(github_token, output_repos, output_readme, output_contributors, verbose):
     fetch_data = FetchData(github_token)
     start = time()
     repos = fetch_data.fetch_repos()
     if repos is None:
         logger.error("No repos fetched")
         return
-    repos_count = fetch_data.save_to_file(repos, output_repos, output_readme, verbose)
+    repos_count = fetch_data.save_to_file(repos, output_repos, output_readme, output_contributors, verbose)
     logger.info(f"Fetched {repos_count} repos in {time() - start:.2f} seconds")
 
 @click.command(help='List repos from file - for debugging')
@@ -46,14 +47,15 @@ def list(github_token, input_file):
 @click.option('--fetch-directly', default=False, is_flag=True, help='Fetch repos directly from github')
 @click.option('--input-repos', default='data/repos.json', help='Input file (default is repos.json)')
 @click.option('--input-readme', default='data/readme.json', help='Input file (default is readme.json)')
+@click.option('--input-contributors', default='data/contributors.json', help='Input file (default is contributors.json)')
 @click.option('--build-dir', '-o', default='build', help='build directory (default is build)')
 @click.option('--template-dir', '-t', default='templates', help="Template directory (default is 'templates')")
 @click.option('--static-dir', default='static', help="Static directory (default is 'static')")
 @click.option('--project-dir', default='projects', help="Project directory (default is 'projects')")
 @click.option('--hide-private', default=False, is_flag=True, help="Hide private repos")
 @click.option('--verbose', default=False, is_flag=True, help='Verbose build')
 @click.option('--compile-tailwind', default=False, is_flag=True, help='Compile tailwind (requires npx + tailwindcss)')
-def generate(github_token: str, fetch_directly: bool, input_repos: str, input_readme:str, build_dir: str, template_dir: str, static_dir: str, project_dir: str, hide_private: bool, verbose: bool, compile_tailwind: bool):
+def generate(github_token: str, fetch_directly: bool, input_repos: str, input_readme:str, input_contributors:str, build_dir: str, template_dir: str, static_dir: str, project_dir: str, hide_private: bool, verbose: bool, compile_tailwind: bool):
     print(f"Generating web to {build_dir} directory")
     start = time()
     fetch_data = FetchData(github_token)
@@ -66,8 +68,9 @@ def generate(github_token: str, fetch_directly: bool, input_repos: str, input_re
     else:
         repos = fetch_data.load_repo(input_repos)
         readme = fetch_data.load_readme(input_readme)
+        contributors = fetch_data.load_contributors(input_contributors)
 
-    if not repos or not readme:
+    if not repos or not readme or not contributors:
         raise Exception("No repos loaded")
 
     # sort repos by time of last push (newest to oldest)
@@ -80,7 +83,7 @@ def generate(github_token: str, fetch_directly: bool, input_repos: str, input_re
     for repo, t in sorted_repos:
         repos.append(repo)
 
-    generate_web = GenerateWeb(repos, readme, build_dir, path.abspath(template_dir), static_dir, project_dir, hide_private, verbose, compile_tailwind)
+    generate_web = GenerateWeb(repos, readme, contributors, build_dir, path.abspath(template_dir), static_dir, project_dir, hide_private, verbose, compile_tailwind)
 
     generate_web.generate()
     print(f"Generated web to {build_dir} directory in {time() - start:.2f} seconds")

src/fetch_data.py

@@ -26,9 +26,14 @@ def fetch_readme(self, repo: Repository.Repository) -> str:
             return repo.get_readme().decoded_content.decode("utf-8")
         except UnknownObjectException:
             return ""
+
+    def fetch_contributors(self, repo: Repository.Repository, contributors_limit:int = 15) -> PaginatedList:
+        try:
+            return sorted(repo.get_contributors(), key=lambda x: x.contributions, reverse=True)[:contributors_limit]
+        except UnknownObjectException:
+            return []
 
-
-    def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data/repos.json', file_readme:str= 'data/readme.json', verbose=False) -> int:
+    def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data/repos.json', file_readme:str= 'data/readme.json', file_contributors:str= 'data/contributors.json', verbose=False) -> int:
         """
         Save repos to file
         :param file_repos:
@@ -42,6 +47,7 @@ def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data
 
         data_repo = []
         data_readme = {}
+        data_contrib = {}
         index = 0
         for repo in repos:
             if verbose:
@@ -50,13 +56,18 @@ def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data
 
             data_readme[repo.full_name] = self.fetch_readme(repo)
 
+            contributors = self.fetch_contributors(repo)
+            data_contrib[repo.full_name] = [[contributor.html_url, contributor.avatar_url, contributor.name, contributor.login, contributor.contributions,] for contributor in contributors]
             index += 1
         with open(file_repos, 'w') as json_file:
             json.dump(data_repo, json_file)
 
         with open(file_readme, 'w') as json_file:
             json.dump(data_readme, json_file)
 
+        with open(file_contributors, 'w') as json_file:
+            json.dump(data_contrib, json_file)
+
         if verbose:
             logger.info(f"Saved {index} repos to {file_repos}")
 
@@ -81,6 +92,15 @@ def load_readme(self, file_readme: str="data/readme.json") ->  dict[str, str]:
         except FileNotFoundError:
             logger.error(f"File {file_readme} not found")
             return {}
+
+    def load_contributors(self, file_contrib: str="data/contributors.json") -> dict[str, list[str]]:
+        # return {repo, [name, login, url_to_img]}
+        try:
+            with open(file_contrib, 'r') as json_file:
+                return json.load(json_file)
+        except FileNotFoundError:
+            logger.error(f"File {file_contrib} not found")
+            return {}
 
 if __name__ == "__main__":
     fetch_data = FetchData(environ.get('MY_GITHUB_TOKEN'))

src/generate_web.py

@@ -25,6 +25,7 @@ def __init__(
             self,
             repos: list[Repository.Repository],
             readme: dict[str, str],
+            contributors: dict[list],
             build_dir: str = 'build',
             template_dir: path = 'templates',
             static_dir: path = 'static',
@@ -39,6 +40,7 @@ def __init__(
             self.repos = repos
 
         self.readme = readme
+        self.contributors = contributors
         self.static_dir = static_dir
         self.template_dir = template_dir
         self.project_dir = project_dir
@@ -114,14 +116,17 @@ def generate_repos_list(self):
         self.render_page('repos.html', self.paths.get("Repos").get("path"), repos=self.repos)
 
     def generate_repos_detail(self):
-        for repo in self.repos:
+        repo_count = len(self.repos)
+        for i, repo in enumerate(self.repos):
+            print(f"Generating {repo.name} {i}/{repo_count}")
             readme_md = self.readme.get(repo.full_name, "No readme found")
             readme_fixed_images = fix_readme_relative_images(readme_md, repo.full_name, repo.default_branch)
             readme_fixed_images = readme_fixed_images.replace(':\n- ', ':\n\n- ') # fix markdown lists
             readme_fixed_images = readme_fixed_images.replace('- ', '- ● ') # add the dot before each element of the list
             readme_html = markdown(readme_fixed_images, extensions=['fenced_code'])
             path_repo = self.paths.get("Repo").get("path").format(repo.name)
-            self.render_page('repoDetail.html', path_repo, repo=repo, readme=readme_html)
+
+            self.render_page('repoDetail.html', path_repo, repo=repo, readme=readme_html, repo_contrib = self.contributors[repo.full_name])
 
     def generate_demo(self):
         self.render_page('demo.html', self.paths.get("Demo").get("path"))

src/web_helpers.py

@@ -52,10 +52,14 @@ def replace_url(img_url, alt_text=""):
             new_url = base_url + img_url
         elif img_url.startswith("./"):
             new_url = f"{base_url}/{img_url[2:]}"
-        else:
+        elif img_url.startswith("https://github.com"):
+            new_url = base_url + img_url.split("/blob/master")[-1]
+        elif img_url.startswith("http"):
             # The URL is already absolute, or it's a type of relative URL
-            # that we aren't handling.
             new_url = img_url
+        else:
+            new_url = base_url + "/"+ img_url
+
         return f"![{alt_text}]({new_url})" if alt_text else new_url
 
     # Replace Markdown image URLs

templates/repoDetail.html

@@ -1,7 +1,7 @@
 {% set title = repo.name %}
 {% extends "base.html" %}
 {% block content %}
-<div class="ml-2 mx-2 mt-2 ">
+<div class="mx-2 mt-2 ">
     {# Repo info #}
     <div class="flex flex-col md:flex-row gap-2 h-auto md:items-stretch">
         <div class="p-4 flex flex-col w-full min-w-0 md:w-3/6 dark:bg-gray-800 bg-white rounded-lg shadow-md">
@@ -52,16 +52,15 @@ <h2 class="dark:text-white text-xl font-bold text-gray-800 mb-1">{{ repo.name }}
     </div>
 
     {# Contributors #}
-    <div class="mt-2 flex flex-col md:flex-row gap-2 h-auto">
-        {% for contrib in repo.get_contributors() %}
-        <a class="p-4 flex flex-row w-full min-w-0 md:w-1/4 dark:bg-gray-800 bg-white rounded-lg shadow-md"
-            href="{{contrib.html_url}}">
-            <img class="w-20 aspect-square rounded-3xl" src="{{contrib.avatar_url}}">
+    <div class="mt-2 grid grid-cols-1 md:grid-cols-3 gap-2 h-auto">
+        {% for contrib in repo_contrib %}
+        <a class="p-4 flex flex-row dark:bg-gray-800 bg-white rounded-lg shadow-md" href="{{ contrib[0] }}">
+            <img class="w-20 aspect-square rounded-3xl" src="{{ contrib[1] }}">
             <div class="ml-3 flex flex-col">
-                <p class="dark:text-white text-xl font-bold text-gray-800">{{ contrib.name }}</p>
-                <p class="ml-[1px] leading-none text-sm dark:text-gray-300 text-gray-600">{{ contrib.login }}</p>
-                <p class="leading-10 dark:text-gray-300 text-gray-600">{{ contrib.contributions }} {% if
-                    contrib.contributions
+                <p class="dark:text-white text-xl font-bold text-gray-800">{{ contrib[2] }}</p>
+                <p class="ml-[1px] leading-none text-sm dark:text-gray-300 text-gray-600">{{ contrib[3]}}</p>
+                <p class="leading-10 dark:text-gray-300 text-gray-600">{{ contrib[4] }} {% if
+                    contrib[3]
                     == 1 %}commit{% else
                     %}commits{% endif %}</p>
             </div>