GSoC 2024: GitHub Discussion UI at Home, Discussion and Contributor's route

lib/discussion.ts

@@ -70,12 +70,12 @@ export async function fetchGithubDiscussion(
 export async function checkAnsweredByUser(
   github: string,
   number: string,
-  repoName: string,
+  repository: string,
 ) {
   const org = env.NEXT_PUBLIC_GITHUB_ORG;
 
   const dicussion: Dicussion = await octokit.graphql(`query {
-    repository(owner: "${org}", name: "${repoName}") {
+    repository(owner: "${org}", name: "${repository}") {
       discussion (number: ${number}) {
         answer {
           author {
@@ -106,7 +106,7 @@ export async function getGithubDiscussions(githubHandle: string) {
         const isAnswered = await checkAnsweredByUser(
           githubHandle,
           discussion.link?.split("/").pop() ?? "",
-          discussion.repoName,
+          discussion.repository,
         );
         title = isAnswered ? "Discussion Answered" : "Commented on discussion";
         activityType = isAnswered

schemas/discussion-data.yaml

@@ -10,7 +10,7 @@ required:
   - url
   - category
   - time
-  - repoName
+  - repository
   - participants
 
 properties:
@@ -40,6 +40,6 @@ properties:
       type: array
       items:
         type: string
-    repoName:
+    repository:
       type: string
 

scraper/src/github-scraper/discussion.ts

@@ -1,5 +1,5 @@
 import { octokit } from "./config.js";
-import { Discussion, ParsedDiscussion } from "./types.js";
+import { Discussion, ParsedDiscussion, Repository } from "./types.js";
 import { saveDiscussionData } from "./utils.js";
 
 const query = `query($org: String!, $cursor: String) {
@@ -50,44 +50,42 @@ async function fetchGitHubDiscussions(
   org: string,
   endDate: Date,
   startDate: Date,
-  cursor = null,
 ) {
-  const variables = {
-    org,
-    cursor,
-  };
-  for await (const response of octokit.graphql.paginate.iterator(
-    query,
-    variables,
-  )) {
-    const repositories = await response.organization.repositories.edges;
-    type repo = (typeof repositories)[0];
+  const iterator = octokit.graphql.paginate.iterator(query, { org });
+
+  let Discussions: { repository: string; discussion: Discussion }[] = [];
+
+  for await (const response of iterator) {
+    const repositories: Repository[] = response.organization.repositories.edges;
+
     for (const repo of repositories) {
-      const discussions = await repo.node.discussions.edges.map(
-        (discussion: repo) => ({
-          repoName: repo.node.name,
-          discussion: discussion.node,
-        }),
-      );
-      const discussionsWithinDateRange = await discussions.find((d: repo) => {
+      const discussions = repo.node.discussions.edges.map((discussion) => ({
+        repository: repo.node.name,
+        discussion: discussion.node,
+      }));
+      const discussionsWithinDateRange = discussions.find((d) => {
         const createdAt = new Date(d.discussion.createdAt);
         const updatedAt = new Date(d.discussion.updatedAt);
 
         return (
-          createdAt >= new Date(startDate) || updatedAt >= new Date(startDate)
+          // When created or updated date will be lower than the start date and greater than end date then return true
+          (createdAt <= new Date(startDate) &&
+            createdAt >= new Date(endDate)) ||
+          (updatedAt <= new Date(startDate) && updatedAt >= new Date(endDate))
         );
       });
       if (discussionsWithinDateRange) {
-        return discussions;
+        return Discussions;
       }
+      Discussions = Discussions.concat(discussions);
     }
   }
 
-  return null;
+  return Discussions;
 }
 
 async function parseDiscussionData(
-  allDiscussions: { repoName: string; discussion: Discussion }[],
+  allDiscussions: { repository: string; discussion: Discussion }[],
   endDate: Date,
   startDate: Date,
 ) {
@@ -119,7 +117,7 @@ async function parseDiscussionData(
           emoji: d.discussion.category.emojiHTML.replace(/<\/?div>/g, ""),
         },
         participants: participants || [],
-        repoName: d.repoName,
+        repository: d.repository,
       };
     },
   );
@@ -138,9 +136,11 @@ export async function scrapeDiscussions(
       endDate,
       startDate,
     );
-    const parsedDiscussions =
-      allDiscussions &&
-      (await parseDiscussionData(allDiscussions, endDate, startDate));
+    const parsedDiscussions = await parseDiscussionData(
+      allDiscussions,
+      endDate,
+      startDate,
+    );
     await saveDiscussionData(parsedDiscussions, dataDir);
   } catch (error: any) {
     throw new Error(`Error fetching discussions: ${error.message}`);

scraper/src/github-scraper/index.ts

@@ -48,7 +48,6 @@ const scrapeGitHub = async (
   console.log("Scraping completed");
 };
 
-// Type Done and check done
 const main = async () => {
   // Extract command line arguments (skip the first two default arguments)
   const args: string[] = process.argv.slice(2);

scraper/src/github-scraper/types.ts

@@ -197,6 +197,17 @@ export type Discussion = {
   updatedAt: string;
 };
 
+export type Repository = {
+  node: {
+    name: string;
+    discussions: {
+      edges: {
+        node: Discussion;
+      }[];
+    };
+  };
+};
+
 export type ParsedDiscussion = {
   source?: string;
   title: string;
@@ -211,5 +222,5 @@ export type ParsedDiscussion = {
     emoji: string;
   };
   participants?: string[];
-  repoName: string;
+  repository: string;
 };

scraper/src/github-scraper/utils.ts

@@ -205,16 +205,18 @@ export async function saveDiscussionData(
   dataDir: string,
 ) {
   // check data dir present or not and file is present or not if not then create it
+  await mkdir(dataDir + "/discussions", { recursive: true });
+  if (discussions.length === 0) {
+    return;
+  }
   const discussionsDir = path.join(dataDir, "discussions");
-  await mkdir(discussionsDir, { recursive: true });
-
   const file = path.join(discussionsDir, "discussions.json");
   try {
     // Try reading the file
     const response = await readFile(file);
     const oldData = JSON.parse(response.toString());
-    const newData = await mergeDiscussions(oldData, discussions);
-    const jsonData = JSON.stringify(newData, null, 2);
+    const mergedData = await mergeDiscussions(oldData, discussions);
+    const jsonData = JSON.stringify(mergedData, null, 2);
     await writeFile(file, jsonData);
   } catch (err) {
     // File doesn't exist, create it with initial data

tests/github-discussion-schema.test.mjs

@@ -26,13 +26,11 @@ const filesInDir = fs
 
 filesInDir.forEach((file) => {
   const content = fs.readFileSync(join(GH_DATA, file)).toString();
-  if (content !== "null") {
-    const data = JSON.parse(stripJsonComments(content));
+  const data = JSON.parse(stripJsonComments(content));
 
-    describe(`Validate '${file}'`, function () {
-      it("should be properly validated by the json schema", () => {
-        expect(data).to.be.jsonSchema(schema);
-      });
+  describe(`Validate '${file}'`, function () {
+    it("should be properly validated by the json schema", () => {
+      expect(data).to.be.jsonSchema(schema);
     });
-  }
+  });
 });