Refactor: Modularize scraper pipeline into GitHubService and ScoringEngine

components/home-dashboard.tsx

@@ -31,7 +31,7 @@ import ActiveContributors from "@/components/Leaderboard/stats-card/active-contr
 import { PaginatedActivitySection } from "@/components/PaginatedActivitySection";
 import { ActivityGroup, MonthBuckets } from "@/lib/db";
 import { Config } from "@/types/config";
-import { RepoStats } from "@/scripts/generateLeaderboard";
+import { RepoStats } from "@/scripts/types";
 
 // --- Types ---
 type ViewState = "overview" | "repositories";

lib/db.ts

@@ -1,6 +1,6 @@
 // lib/db.ts — temporary stub (no DB)
 
-import { RepoStats, UserEntry } from "@/scripts/generateLeaderboard";
+import { RepoStats, UserEntry } from "@/scripts/types";
 import fs from "fs";
 import path from "path";
 import { differenceInDays } from "date-fns";
@@ -26,11 +26,6 @@ export type ActivityGroup = {
   activities: ActivityItem[];
 };
 
-type RecentActivitiesJSON = {
-  updatedAt: number;
-  entries: UserEntry[];
-  groups: ActivityGroup[];
-};
 
 export type MonthBuckets = {
   w1: number;
@@ -42,58 +37,11 @@ export type MonthBuckets = {
 // Helper function to extract repository name from GitHub URL
 function extractRepoFromUrl(url: string | null | undefined): string | null {
   if (!url || typeof url !== 'string') return null;
-  
+
   const match = url.match(/github\.com\/[^/]+\/([^/]+)/);
   return match && match[1] !== undefined ? match[1] : null;
 }
 
-// Used by app/page.tsx
-// export async function getRecentActivitiesGroupedByType(valid: "week" | "month" | "year"): Promise<ActivityGroup[]> {
-//   const filePath = path.join(
-//     process.cwd(),
-//     "public",
-//     "leaderboard",
-//     `${valid}.json`
-//   );
-
-//   let activityGroups: ActivityGroup[] = [];
-
-//   if (fs.existsSync(filePath)) {
-//     const file = fs.readFileSync(filePath, "utf-8");
-//     const data: RecentActivitiesJSON = JSON.parse(file);
-
-//     const groupsFromEntries: ActivityGroup[] =
-//       Object.entries(
-//         data.entries.reduce((acc, user) => {
-//           for (const [type, meta] of Object.entries(
-//             user.activity_breakdown
-//           )) {
-//             if (!acc[type]) {
-//               acc[type] = {
-//                 activity_definition: type,
-//                 activity_name: type,
-//                 activities: [],
-//               };
-//             }
-
-//             acc[type].activities.push({
-//               slug: `${user.username}-${type}`,
-//               contributor: user.username,
-//               contributor_name: user.name,
-//               contributor_avatar_url: user.avatar_url,
-//               occured_at: data.updatedAt,
-//               points: meta.points,
-//             });
-//           }
-//           return acc;
-//         }, {} as Record<string, ActivityGroup>)
-//       ).map(([, group]) => group);
-
-//     activityGroups = groupsFromEntries;
-//   }
-
-//   return activityGroups;
-// }
 
 export async function getRecentActivitiesGroupedByType(
   valid: "week" | "month" | "2month" | "year"
@@ -114,35 +62,35 @@ export async function getRecentActivitiesGroupedByType(
 
   const groups = new Map<string, ActivityGroup>();
 
- for (const user of data.entries) {
-  if (!user.activities) continue;
+  for (const user of data.entries) {
+    if (!user.activities) continue;
 
-  for (const act of user.activities) {
-    const type = act.type;
+    for (const act of user.activities) {
+      const type = act.type;
+
+      if (!groups.has(type)) {
+        groups.set(type, {
+          activity_definition: type,
+          activity_name: type,
+          activity_description: null,
+          activities: [],
+        });
+      }
 
-    if (!groups.has(type)) {
-      groups.set(type, {
-        activity_definition: type,
-        activity_name: type,
-        activity_description: null,
-        activities: [],
+      groups.get(type)!.activities.push({
+        slug: `${user.username}-${type}-${act.occured_at}-${groups.get(type)!.activities.length}`,
+        contributor: user.username,
+        contributor_name: user.name,
+        contributor_avatar_url: user.avatar_url,
+        contributor_role: user.role,
+        occured_at: act.occured_at,
+        title: act.title ?? null,
+        link: act.link ?? null,
+        repo: extractRepoFromUrl(act.link ?? null),
+        points: act.points ?? 0,
       });
     }
-
-    groups.get(type)!.activities.push({
-      slug: `${user.username}-${type}-${act.occured_at}-${groups.get(type)!.activities.length}`,
-      contributor: user.username,
-      contributor_name: user.name,
-      contributor_avatar_url: user.avatar_url,
-      contributor_role: (user.role ?? null) as string | null,
-      occured_at: act.occured_at,
-      title: act.title ?? null,     // ✅ REAL title
-      link: act.link ?? null,       // ✅ REAL GitHub link
-      repo: extractRepoFromUrl(act.link ?? null), // ✅ Extract repo name
-      points: act.points ?? 0,
-    });
   }
-}
 
 
   // newest first
@@ -163,14 +111,14 @@ export async function getRecentActivitiesGroupedByType(
 
 export async function getUpdatedTime() {
   const publicPath = path.join(process.cwd(), "public", "leaderboard");
-  if(!fs.existsSync(publicPath)) return null;
+  if (!fs.existsSync(publicPath)) return null;
   const files = fs.readdirSync(publicPath).filter(
     (file) => file.endsWith(".json") && file !== "recent-activities.json"
   );
 
   let latestUpdatedAt = 0;
-  for(const file of files){
-    try{
+  for (const file of files) {
+    try {
       const filePath = path.join(publicPath, file);
       const data = JSON.parse(fs.readFileSync(filePath, "utf-8"));
       if (data.updatedAt && data.updatedAt > latestUpdatedAt) {
@@ -243,7 +191,7 @@ export async function getReposOverview(): Promise<RepoStats[]> {
   );
 
   if (!fs.existsSync(filePath)) return [];
-  
+
   try {
     const file = fs.readFileSync(filePath, "utf-8");
     const data = JSON.parse(file);