Migrate scraper to typescript and adds support for GitHub discussions

* GSoC_Week_1:Refactor present scrapper into typescript and addition in type.ts and gh_events.ts * GSoC_Week_1:Refactor present scrapper into typescript and addition in type.ts and gh_events.ts * Delete newGt.ts * remove unwated changes * Scraper githu.ts divided into different files for better understanding * fix scraper setup * Integrate Github Discussion in scraper and update scraper-dry-run workflow * Removing dry-run work flow error (date-fns) * Fixing scraper-dry-run date error by puting null as a default value for date * Fixing scraper-dry-run failing * Fixing scraper-dry-run failing * Fixing scraper-dry-run failing * Fixing scraper-dry-run failing (Genrate markdown files) * Fixing scraper-dry-run failing (Genrate markdown files) * Fixing scraper-dry-run failing (Genrate markdown files) * Fixing scraper-dry-run failing (Genrate markdown files) * Suggested cahnges done * Update test schema for discussion * resolve-dry-run error with pnpm * Revert accidental cahnges in scraper0dry-run.yaml * Revert accidental cahnges in scraperdry-run.yaml * dotenv used in generateNewContrbutors.js * remove: dotenv used in generateNewContrbutors.js * Fix path for data repository to solve dry-run error * update pnpm-lock.yaml * Store seprately all discussion in discussion folder * Update discussion schema * Update discussion schema * Update scraper-dry-run.yaml * Update scraper-dry-run.yaml and fix some typos * Update scraper-dry-run.yaml * Remove casting in fetchEvents.ts * fix type error * Fix type errors * Modify types and remove all types error from scraper * Description added to discussion scraper * Description added to discussion scraper * Discussion UI created at home, disucssions and cotrnbutors profile route * uncomment in scraper * Update Github Dicussions to Discussion * Point mechanism for discussions and responsiveness added * Site map updated for gh-discussion * type error fix in api.ts and modify logic of leaderboard for discussions * prose-h2 added to fix markdown bug * Modified suggested changes still one type error remaining * Modified suggested changes still one type error remaining * Chages done as per review * fix type error in api.ts mismatch in return type of discussion * Add suspense boundary for discussions filter * fix open in github button responsiveness issue * fix incorrect roots and remove unused imports * Enable empathy badge and merge discussions with old data * requested changes are done participants logic in progress * remove unnecessary `useMemo` * Implement logic to fetch discussion within daterange (updated or created) and merge with old data * Removing previous logic of fetch participants * Remove unwanted changes * Handle nullable values while scraping discussion * Modify testing if the discussion.json is empty or notpresent * Fixing testing logic * Change in discussion scraper logic and modify reposName to repository as requested in review * fix naming convetntions and discussion-schema-testing when there is no discussion dir present * Update tests/github-discussion-schema.test.mjs * remove unused packages, scripts and upgraded package versions * remove unused methods * move markdown render to server side * check if discussions dir. exists before reading * support for scraper workflow to run on non-main branches * update scraper action workflow * update data repo dir. in scraper workflow * update data dir. in scraper workflow --------- Co-authored-by: rithviknishad <[email protected]> Co-authored-by: Rithvik Nishad <[email protected]>
ohcnetwork · Jul 25, 2024 · 2a56d9b · 2a56d9b
1 parent a090186
commit 2a56d9b
Show file tree

Hide file tree

Showing 50 changed files with 5,277 additions and 3,497 deletions.
diff --git a/.env b/.env
@@ -26,4 +26,4 @@ DATA_SOURCE="https://github.com/coronasafe/leaderboard-data.git"
 # SLACK_EOD_BOT_SIGNING_SECRET=<Slack EOD Bot App's Signing Secret>
 
 ## -- Features -- ##
-NEXT_PUBLIC_FEATURES=Leaderboard,Contributors,Feed,Releases,Projects
+NEXT_PUBLIC_FEATURES=Leaderboard,Contributors,Feed,Releases,Projects,Discussions
diff --git a/.github/workflows/scraper-dry-run.yaml b/.github/workflows/scraper-dry-run.yaml
@@ -20,16 +20,25 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: setup python
-        uses: actions/setup-python@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
         with:
-          python-version: "3.10"
+          node-version: "20.14.0"
+
+      - name: Install pnpm
+        run: npm install -g pnpm
 
       - name: Install dependencies
-        run: pip install -r scraper/requirements.txt
+        run: pnpm install --frozen-lockfile
+        working-directory: scraper
+
+      - name: Build the project
+        run: pnpm build
+        working-directory: scraper
 
       - name: Scrape data from GitHub
-        run: python scraper/src/github.py ${{ github.repository_owner }} data/github -l DEBUG
+        run: pnpm start ${{ github.repository_owner }} ../data/github 
+        working-directory: scraper
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -39,40 +48,26 @@ jobs:
         run: node scripts/generateNewContributors.js
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
+          
       - uses: actions/upload-artifact@v4
         with:
           name: output
           retention-days: 5
           path: |
             data
             contributors
-
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9
-          run_install: false
-
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20
-          cache: "pnpm"
-
+            
       - name: Get pnpm store directory
         shell: bash
         run: |
           echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
-
       - uses: actions/cache@v4
         name: Setup pnpm cache
         with:
           path: ${{ env.STORE_PATH }}
           key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
           restore-keys: |
             ${{ runner.os }}-pnpm-store-
-
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 

diff --git a/.github/workflows/scraper.yaml b/.github/workflows/scraper.yaml
@@ -4,6 +4,11 @@ on:
       slack-eod-channel:
         required: false
         type: string
+      leaderboard_ref:
+        required: false
+        type: string
+        default: main
+        description: A specific branch of scraper that is to be used.
     secrets:
       GIT_ACCESS_TOKEN:
         required: true
@@ -22,35 +27,44 @@ jobs:
         with:
           repository: coronasafe/leaderboard
           path: ./leaderboard
+          ref: ${{ inputs.leaderboard_ref }}
           sparse-checkout: |
             scraper
             scripts
 
-      - name: setup python
-        uses: actions/setup-python@v3
+      - uses: actions/setup-node@v3
         with:
-          python-version: "3.10"
+          node-version: "20.14.0"
+      - run: npm install -g pnpm
 
-      - name: Install dependencies
-        run: pip install -r scraper/requirements.txt
+      - run: pnpm install --frozen-lockfile
+        working-directory: scraper
 
-      - name: Scrape GitHub data
-        uses: nick-fields/retry@v2
-        with:
-          timeout_minutes: 10
-          max_attempts: 3
-          command: python scraper/src/github.py ${{ github.repository_owner }} data/github -l DEBUG
-          on_retry_command: git checkout .
+      - run: pnpm build
+        working-directory: scraper
+
+      - name: Scrape data from GitHub
+        run: pnpm start ${{ github.repository_owner }} ../../data/github
+        working-directory: scraper
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - name: Install dependencies
+        run: pip install -r scraper/requirements.txt
+
       - name: Scrape EOD updates from Slack Channel
         if: ${{ env.SLACK_API_TOKEN && env.SLACK_CHANNEL }}
         run: python scraper/src/slack.py --lookback_days=2
         env:
           SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
           SLACK_CHANNEL: ${{ inputs.slack-eod-channel }}
+
       - run: mkdir -p contributors
+
       - name: Generate markdown files for new contributors
         run: node scripts/generateNewContributors.js
         env:

diff --git a/.gitignore b/.gitignore
@@ -7,7 +7,6 @@
 .pnp.js
 bun.lockb
 package-lock.json
-pnpm-lock.yaml
 
 # testing
 /coverage

diff --git a/app/api/leaderboard/functions.ts b/app/api/leaderboard/functions.ts
@@ -26,6 +26,11 @@ export const getLeaderboardData = async (
     .filter(
       (contributor) => roles.length == 0 || roles.includes(contributor.role),
     )
+    .filter((contributor) => {
+      if (sortBy) {
+        return contributor.summary[sortBy] ?? 0 > 0;
+      }
+    })
     .sort((a, b) => {
       if (sortBy === "pr_stale") {
         return b.activityData.pr_stale - a.activityData.pr_stale;
@@ -55,7 +60,7 @@ export const getLeaderboardData = async (
       },
       highlights: {
         ...contributor.summary,
-        pr_stale: contributor.activityData.pr_stale,
+        pr_stale: contributor.activityData.pr_stale ?? 0,
       },
     };
   });

diff --git a/app/contributors/[slug]/page.tsx b/app/contributors/[slug]/page.tsx
@@ -11,7 +11,7 @@ import BadgeIcons from "../../../components/contributors/BadgeIcons";
 import GithubActivity from "../../../components/contributors/GithubActivity";
 import GraduateAttributeBadge from "../../../components/contributors/GraduateAttributeBadge";
 import InfoCard from "../../../components/contributors/InfoCard";
-import React, { Suspense } from "react";
+import React from "react";
 import clsx from "clsx";
 import { formatDuration, parseDateRangeSearchParam } from "@/lib/utils";
 import Markdown from "@/components/Markdown";

diff --git a/app/contributors/sitemap.ts b/app/contributors/sitemap.ts
@@ -1,5 +1,5 @@
 import { sitemapEntry } from "@/app/sitemap";
-import { getContributors, getContributorsSlugs } from "@/lib/api";
+import { getContributors } from "@/lib/api";
 import { MetadataRoute } from "next";
 
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {

diff --git a/app/discussions/layout.tsx b/app/discussions/layout.tsx
@@ -0,0 +1,35 @@
+import { Metadata } from "next";
+import { env } from "@/env.mjs";
+import { notFound } from "next/navigation";
+import { featureIsEnabled } from "@/lib/utils";
+import FilterDiscussions from "../../components/discussions/FilterDiscussions";
+import { categories } from "../../lib/discussion";
+import DiscussionLeaderboard from "../../components/discussions/DiscussionLeaderboard";
+import { Suspense } from "react";
+
+export const metadata: Metadata = {
+  title: `Disucssions | ${env.NEXT_PUBLIC_PAGE_TITLE}`,
+};
+
+export default function DiscussionsLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  if (!featureIsEnabled("Discussions")) return notFound();
+
+  return (
+    <div className="mx-auto max-w-6xl p-5">
+      <div className="items-center gap-5 pb-8 lg:mt-10 lg:flex">
+        <h1 className="text-3xl sm:text-4xl">Disucssions</h1>
+        <Suspense fallback={<></>}>
+          <FilterDiscussions categories={categories} />
+        </Suspense>
+      </div>
+      <div className="flex w-full flex-col-reverse gap-3 lg:flex lg:flex-row">
+        {children}
+        <DiscussionLeaderboard />
+      </div>
+    </div>
+  );
+}
diff --git a/app/discussions/page.tsx b/app/discussions/page.tsx
@@ -0,0 +1,19 @@
+import { fetchGithubDiscussion } from "../../lib/discussion";
+import GithubDiscussions from "../../components/discussions/GithubDiscussions";
+
+interface Params {
+  searchParams: { [key: string]: string };
+}
+
+export default async function Page({ searchParams }: Params) {
+  const discussions = await fetchGithubDiscussion();
+
+  return (
+    discussions && (
+      <GithubDiscussions
+        discussions={discussions}
+        searchParams={searchParams}
+      />
+    )
+  );
+}
diff --git a/app/issues/page.tsx b/app/issues/page.tsx
@@ -1,9 +1,3 @@
-import { env } from "@/env.mjs";
-import octokit from "@/lib/octokit";
-import Image from "next/image";
-import Link from "next/link";
-import { GoIssueOpened } from "react-icons/go";
-import Markdown from "@/components/Markdown";
 import ActiveProjects from "../projects/ActiveProjects";
 
 const page = async () => {

diff --git a/app/leaderboard/page.tsx b/app/leaderboard/page.tsx
@@ -8,7 +8,7 @@ export default async function LeaderboardPage({ searchParams }: PageProps) {
   const keyString = `search=${searchParams?.search}`;
   return (
     <section className="border-t border-secondary-300 bg-background text-foreground dark:border-secondary-700">
-      <div className="mx-auto max-w-6xl">
+      <div className="mx-auto max-w-7xl">
         <Searchbar searchParams={searchParams} />
         <div className="mx-4 border-secondary-600 xl:mx-0">
           <Suspense

diff --git a/app/page.tsx b/app/page.tsx
@@ -11,6 +11,8 @@ import { env } from "@/env.mjs";
 import CommunityEngagemet from "@/app/CommunityEngagementSummary";
 import { differenceInWeeks, parseISO } from "date-fns";
 import { featureIsEnabled, formatDate } from "@/lib/utils";
+import { fetchGithubDiscussion } from "../lib/discussion";
+import GithubDiscussion from "@/components/discussions/GithubDiscussion";
 
 export default async function Home() {
   const contributors = (await getContributors())
@@ -21,7 +23,7 @@ export default async function Home() {
           .includes(contributor.role) ?? true,
     )
     .sort((a, b) => b.weekSummary.points - a.weekSummary.points);
-
+  const discussions = await fetchGithubDiscussion(5);
   const startDate = parseISO(env.NEXT_PUBLIC_ORG_START_DATE);
 
   return (
@@ -95,7 +97,33 @@ export default async function Home() {
                     </div>
                   </div>
                 )}
-
+                {discussions && (
+                  <div className="mx-auto">
+                    <div className="space-y-12">
+                      <div className="flex items-center justify-between pr-2">
+                        <h2 className="text-3xl font-bold tracking-tight sm:text-4xl">
+                          Discussions
+                        </h2>
+                        <Link
+                          href="/discussions"
+                          className="flex items-center gap-1 rounded p-2 text-secondary-400 underline underline-offset-2 transition-all duration-200 ease-in-out hover:gap-2 hover:text-primary-200"
+                        >
+                          More
+                          <MdOutlineArrowForwardIos />
+                        </Link>
+                      </div>
+                      {discussions.map((discussion, index) => {
+                        return (
+                          <GithubDiscussion
+                            key={index}
+                            discussion={discussion}
+                            minimal
+                          />
+                        );
+                      })}
+                    </div>
+                  </div>
+                )}
                 {featureIsEnabled("Projects") && (
                   <div className="mx-auto">
                     <div className="space-y-12">

diff --git a/app/sitemap.ts b/app/sitemap.ts
@@ -24,6 +24,7 @@ export default function sitemap(): MetadataRoute.Sitemap {
     entry("/releases"),
     entry("/leaderboard"),
     entry("/issues"),
+    entry("/discussions"),
     entry("/feed", { changeFrequency: "always", priority: 0.7 }),
   ];
 }
diff --git a/components/Markdown.tsx b/components/Markdown.tsx
@@ -4,22 +4,23 @@ import remarkGfm from "remark-gfm";
 import remarkRehype from "remark-rehype";
 import rehypeStringify from "rehype-stringify";
 import clsx from "clsx";
+
 export default async function Markdown(props: {
   children: string;
   className?: string;
 }) {
-  const result = await unified()
+  const processedMarkdown = unified()
     .use(remarkParse)
     .use(remarkGfm)
     .use(remarkRehype)
     .use(rehypeStringify)
-    .process(props.children || "");
+    .processSync(props.children || "");
 
   return (
-    <div className="prose font-inter text-sm leading-relaxed dark:prose-invert sm:text-base xl:text-left">
+    <div className="prose font-inter text-sm leading-relaxed dark:prose-invert prose-h2:mt-3 sm:text-base xl:text-left">
       <div
         className={clsx(props.className ?? "")}
-        dangerouslySetInnerHTML={{ __html: result.toString() }}
+        dangerouslySetInnerHTML={{ __html: processedMarkdown.toString() }}
       />
     </div>
   );

diff --git a/components/Select.tsx b/components/Select.tsx
@@ -108,7 +108,7 @@ export function Select({
           <>
             {showSelectionsAs ? (
               showSelectionsAs == "tags" ? (
-                <div className="left-[-2.5rem] top-9 flex w-full flex-wrap gap-0.5">
+                <div className="-left-10 top-9 flex w-full flex-wrap gap-0.5">
                   {Array.isArray(value) &&
                   value?.length > 0 &&
                   value.length !== options.length ? (
@@ -129,7 +129,7 @@ export function Select({
                   )}
                 </div>
               ) : (
-                <div className="left-[-2.5rem] top-9 flex gap-0.5">
+                <div className="-left-10 top-9 flex gap-0.5">
                   {Array.isArray(value) &&
                   value.length > 0 &&
                   value.length !== options.length ? (