Reducing tech debt

backend/.mocharc.yaml

@@ -0,0 +1,4 @@
+require: 'ts-node/register'
+extension:
+  - 'ts'
+spec: 'test/**/*.spec.ts'

backend/package.json

@@ -8,16 +8,16 @@
     "dev-start": "nodemon --exec ts-node src/server.ts",
     "build": "tsc && npm run copy-sql",
     "gcp-build": "npm install --save-dev && npm run build",
-    "copy-sql": "cp ./src/db/db_init.sql ./dist/db/db_init.sql",
+    "copy-sql": "cp ./src/db/db_init.sql ./dist/src/db/db_init.sql",
     "tsconfig": "tsc --init",
-    "test": "tsc && jest --passWithNoTests"
+    "test": "mocha"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
     "@google-cloud/connect-firestore": "^2.0.2",
-    "@google-cloud/firestore": "^6.7.0",
+    "@google-cloud/firestore": "^4.15.1",
     "@google-cloud/storage": "^6.12.0",
     "@types/cors": "^2.8.13",
     "@types/cron": "^2.0.1",
@@ -44,18 +44,24 @@
     "passport": "^0.6.0",
     "passport-google-oauth20": "^2.0.0",
     "pg": "^8.11.1",
-    "sanitize-filename": "^1.6.3",
-    "ts-node": "^10.9.1"
+    "sanitize-filename": "^1.6.3"
   },
   "devDependencies": {
+    "@types/chai": "^4.3.5",
     "@types/cors": "^2.8.13",
     "@types/cron": "^2.0.1",
     "@types/express": "^4.17.17",
     "@types/express-session": "^1.17.7",
     "@types/luxon": "^3.3.1",
+    "@types/mocha": "^10.0.1",
     "@types/multer": "^1.4.7",
     "@types/pg": "^8.10.2",
+    "@types/sinon": "^10.0.16",
+    "chai": "^4.3.8",
     "jest": "^29.5.0",
+    "mocha": "^10.2.0",
+    "sinon": "^15.2.0",
+    "ts-node": "^10.9.1",
     "typescript": "^5.1.6"
   }
 }

backend/src/api/leetcode.ts

@@ -1,16 +1,18 @@
 import axios from "axios";
 import { isValidUsername } from "../utils/utils";
 
-// Function to make GraphQL requests for a specific user ID
-export async function getSubmitStats(userId: string) {
-  if (!isValidUsername(userId)) {
-    console.log("invalid username");
-    return;
-  }
-  try {
-    // Make the GraphQL request
-    const response = await axios.post("https://leetcode.com/graphql", {
-      query: `
+
+export class LeetCodeApi {
+  // Function to make GraphQL requests for a specific user ID
+  async getSubmitStats(userId: string) {
+    if (!isValidUsername(userId)) {
+      console.log("invalid username");
+      return;
+    }
+    try {
+      // Make the GraphQL request
+      const response = await axios.post("https://leetcode.com/graphql", {
+        query: `
         { 
           matchedUser(username: "${userId}") 
           {
@@ -27,28 +29,28 @@ export async function getSubmitStats(userId: string) {
           }
       }
         `,
-    });
+      });
 
-    if (response.data.data.matchedUser === null) {
-      throw new Error(`Leetcode username ${userId} was not found`);
-    }
+      if (response.data.data.matchedUser === null) {
+        throw new Error(`Leetcode username ${userId} was not found`);
+      }
 
-    return response.data.data.matchedUser;
-  } catch (error) {
-    console.error(`Error fetching data for user ID ${userId}:`, error);
-    throw error;
+      return response.data.data.matchedUser;
+    } catch (error) {
+      console.error(`Error fetching data for user ID ${userId}:`, error);
+      throw error;
+    }
   }
-}
 
-export async function getLatestAcceptedSubmits(
-  userId: string,
-  limit: number = 10
-) {
-  if (!isValidUsername(userId)) return;
-  try {
-    // Make the GraphQL request
-    const response = await axios.post("https://leetcode.com/graphql", {
-      query: `
+  async getLatestAcceptedSubmits(
+    userId: string,
+    limit: number = 10
+  ) {
+    if (!isValidUsername(userId)) return;
+    try {
+      // Make the GraphQL request
+      const response = await axios.post("https://leetcode.com/graphql", {
+        query: `
       {     
         recentAcSubmissionList(username: "${userId}", limit: ${limit}) {
           title
@@ -58,9 +60,12 @@ export async function getLatestAcceptedSubmits(
           }
       }
       `,
-    });
-    return response.data.data.recentAcSubmissionList;
-  } catch (error) {
-    console.error(`Error fetching data for user ID ${userId}:`, error);
+      });
+      return response.data.data.recentAcSubmissionList;
+    } catch (error) {
+      console.error(`Error fetching data for user ID ${userId}:`, error);
+    }
   }
 }
+
+export const leetcodeApi = new LeetCodeApi();

backend/src/api/vjudge.ts

@@ -3,19 +3,22 @@ import { isValidUsername } from '../utils/utils'
 import { UserNameNotFoundError } from '../errors/username-not-found-error';
 import { ExternalApiError } from '../errors/external-api-error';
 
+export class VjudgeApi {
+    async getSubmissionStats(username: string) {
+        if (!isValidUsername(username)) throw new Error("Invalid username format");
 
-export async function getSubmissionStats(username: string) {
-    if(!isValidUsername(username)) throw new Error("Invalid username format");
-
-    try {
-        const response = await axios.get(`https://vjudge.net/user/solveDetail/${username}`);
-        return response.data; 
-    } catch(err) {
-        if(err instanceof AxiosError){
-            const response = err.response!!;
-            if(response.status === 404) throw new UserNameNotFoundError(username, "vjudge");
-            else throw new ExternalApiError("vjudge", response.status);
+        try {
+            const response = await axios.get(`https://vjudge.net/user/solveDetail/${username}`);
+            return response.data;
+        } catch (err) {
+            if (err instanceof AxiosError) {
+                const response = err.response!!;
+                if (response.status === 404) throw new UserNameNotFoundError(username, "vjudge");
+                else throw new ExternalApiError("vjudge", response.status);
+            }
+            else throw err;
         }
-        else throw err;
     }
-}
+}
+
+export const vjudgeApi = new VjudgeApi();

backend/src/job/LeetcodeUpdateCollectingJob.ts

@@ -1,25 +1,26 @@
-import { updatesCollectorService } from "../services/UpdatesCollectorService";
+import { leetcodeUpdatesCollectorService } from "../services/updates-collection/LeetcodeUpdatesCollectorService";
 import { userRepository } from "../repository/UserRepository";
 
 // Job responsible for collecting updates from Leetcode and updating the db
 export class LeetcodeUpdateCollectingJob {
   async run() {
-    const userData = await userRepository.getAllUsernames();
+    const mongoIDs = await userRepository.getAllMongoIds();
 
     // we are querying external APIs, so we cannot do any load we want
     // Introduce batches of size <= 10 that we will process once in 0.2 seconds;
     const batchSize = 10;
     const delayBetweenBatches = 200;
-    const totalBatches = Math.ceil(userData.length / batchSize);
+    const totalBatches = Math.ceil(mongoIDs.length / batchSize);
 
     for (let i = 0; i < totalBatches; i++) {
       const startIndex = i * batchSize;
-      const endIndex = Math.min(startIndex + batchSize, userData.length);
-      const batch = userData.slice(startIndex, endIndex);
+      const endIndex = Math.min(startIndex + batchSize, mongoIDs.length);
+      const batch = mongoIDs.slice(startIndex, endIndex);
 
       // Process the current batch of users concurrently using Promise.all
-      await Promise.all(batch.map(u =>
-        updatesCollectorService.getAndStoreLeetcodeUpdates(u.username!!)));
+      await Promise.all(batch.map(id => {
+        leetcodeUpdatesCollectorService.getAndStoreUpdates(id);
+      }));
 
       // Introduce a delay between batches (in milliseconds)
       if (i < totalBatches - 1) {

backend/src/job/VjudgeUpdateCollectingJob.ts

@@ -1,26 +1,26 @@
-import { updatesCollectorService } from "../services/UpdatesCollectorService";
+import { vjudgeUpdatesCollectorService } from "../services/updates-collection/VjudgeUpdatesCollectorService";
 import { userRepository } from "../repository/UserRepository";
 
 // Job responsible for collecting updates from VJudge and updating our db
 export class VjudgeUpdateCollectingJob {
   async run() {
-    const userData = await userRepository.getAllUsernames();
+    const mongoIDs = await userRepository.getAllMongoIds();
 
     // we are querying external APIs, so we cannot do any load we want
     // VJudge is most likely not as robust as LeetCode so reduce the load compared to Leetcode
     // Introduce batches of size <= 5 that we will process once in 0.5 seconds;
     const batchSize = 5;
     const delayBetweenBatches = 200;
-    const totalBatches = Math.ceil(userData.length / batchSize);
+    const totalBatches = Math.ceil(mongoIDs.length / batchSize);
 
     for (let i = 0; i < totalBatches; i++) {
       const startIndex = i * batchSize;
-      const endIndex = Math.min(startIndex + batchSize, userData.length);
-      const batch = userData.slice(startIndex, endIndex);
+      const endIndex = Math.min(startIndex + batchSize, mongoIDs.length);
+      const batch = mongoIDs.slice(startIndex, endIndex);
 
       // Process the current batch of users concurrently
-      await Promise.all(batch.map(u =>
-        updatesCollectorService.getAndStoreVjudgeUpdates(u.username!!)));
+      await Promise.all(batch.map(id =>
+        vjudgeUpdatesCollectorService.getAndStoreUpdates(id)));
 
       // Wait <delay> time before processing next batch;
       if (i < totalBatches - 1) {

backend/src/model/UpdateEventData.ts

@@ -1,4 +1,5 @@
 export interface UpdateEventData {
+    id: string,
     username: string;
     platform: string;
     problemTitle: string;

backend/src/routes/test/trigger-requests-route.ts

@@ -1,23 +1,24 @@
 import express, { Request, Response } from 'express'
 
-import { getSubmitStats } from '../../api/leetcode'
+import { leetcodeApi } from '../../api/leetcode'
 import { validateUsername, handleValidationErrors } from '../../utils/middleware'
 import { getUserIDs } from '../../model/users';
-import { getSubmissionStats } from '../../api/vjudge';
+import { vjudgeApi } from '../../api/vjudge';
 import { UserNameNotFoundError } from '../../errors/username-not-found-error';
 import { ExternalApiError } from '../../errors/external-api-error';
-import { updatesCollectorService } from '../../services/UpdatesCollectorService';
 import { EventRepository } from '../../repository/EventRepository';
 import { isValidUsername } from '../../utils/utils';
 import { UserRepository } from '../../repository/UserRepository';
+import { allUpdatesCollectorService } from '../../services/updates-collection/AllUpdatesCollectorService';
+import { User } from '../../model/schemas/userSchema';
 
 const router = express.Router()
 
 router.get('/', async (req: Request, res: Response) => {
   try {
     const userIds = getUserIDs();
-    const promises: Promise<ReturnType<typeof getSubmitStats>>[] = Array.from(userIds).map(async (id) => {
-      const cur = await getSubmitStats(id);
+    const promises: Promise<ReturnType<typeof leetcodeApi.getSubmitStats>>[] = Array.from(userIds).map(async (id) => {
+      const cur = await leetcodeApi.getSubmitStats(id);
       console.log("cur:", cur);
       return cur;
     });
@@ -40,7 +41,7 @@ router.get('/leetcode/:userId', [
 ], async (req: Request, res: Response) => {
   const { userId } = req.params;
 
-  const data = await getSubmitStats(userId);
+  const data = await leetcodeApi.getSubmitStats(userId);
   console.log("data:", data);
 
   res.send(data);
@@ -55,7 +56,7 @@ router.get('/vjudge/:username', [
   const { username } = req.params;
 
   try {
-    const data = await getSubmissionStats(username);
+    const data = await vjudgeApi.getSubmissionStats(username);
     console.log("data:", data);
 
     res.send(data);
@@ -70,33 +71,6 @@ router.get('/vjudge/:username', [
   }
 });
 
-router.get('/updatesList/:username', [
-  // Sanitize the userId variable
-  validateUsername('username'),
-  handleValidationErrors
-], async (req: Request, res: Response) => {
-  const { username } = req.params;
-
-  if (!username) {
-    res.status(400).send();
-  }
-  else {
-    try {
-      const updates = await updatesCollectorService.getUpdates(username);
-
-      res.json(updates).send();
-    }
-    catch (err) {
-      if (err instanceof UserNameNotFoundError) {
-        res.status(404).send("user not found");
-      }
-      else {
-        res.status(500).send("Oops! Something went wrong");
-      }
-    }
-  }
-});
-
 router.get('/feed', [
   handleValidationErrors
 ], async (req: Request, res: Response) => {

backend/src/routes/users-route.ts

@@ -5,8 +5,8 @@ import { getUserIDs, addUserID } from "../model/users";
 import { User } from "../model/schemas/userSchema";
 import { UserNameNotFoundError } from "../errors/username-not-found-error";
 import { UsernameAlreadyTakenError } from "../errors/username-already-taken-error";
-import { getSubmissionStats } from "../api/vjudge";
-import { getLatestAcceptedSubmits, getSubmitStats } from "../api/leetcode";
+import { vjudgeApi } from "../api/vjudge";
+import { leetcodeApi} from "../api/leetcode";
 import { imageRepository } from "../repository/ImageRepository";
 import {
   notificationRepository,
@@ -47,7 +47,7 @@ router.get(
       res.status(404).send("Leetcode username not found");
     } else {
       const leetcodeUsername = userData.leetcode.username;
-      const submitStats = await getLatestAcceptedSubmits(leetcodeUsername);
+      const submitStats = await leetcodeApi.getLatestAcceptedSubmits(leetcodeUsername);
 
       if (!submitStats) {
         res.status(404).send("User not found on leetcode");
@@ -165,7 +165,7 @@ router.put(
                   );
                 }
 
-                const vjudgeStats = await getSubmissionStats(
+                const vjudgeStats = await vjudgeApi.getSubmissionStats(
                   data.vjudgeUsername
                 );
                 // console.log(vjudgeStats);
@@ -188,7 +188,7 @@ router.put(
                     "leetcode"
                   );
                 }
-                const leetcodeStats = await getSubmitStats(
+                const leetcodeStats = await leetcodeApi.getSubmitStats(
                   data.leetcodeUsername
                 );
                 console.log("this is LC data: " + leetcodeStats);