Skip to content

Commit f20ce1b

Browse files
shaygekoshaygeko
authored
Merge pull request #28 from thecodegram/shaygeko/refactor_and_tests
Reducing tech debt
2 parents 70df2e7 + 27cff4f commit f20ce1b

17 files changed

+2172
-1728
lines changed

‎backend/.mocharc.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
require: 'ts-node/register'
2+
extension:
3+
- 'ts'
4+
spec: 'test/**/*.spec.ts'

‎backend/package-lock.json

Lines changed: 1739 additions & 1442 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎backend/package.json

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,16 @@
88
"dev-start": "nodemon --exec ts-node src/server.ts",
99
"build": "tsc && npm run copy-sql",
1010
"gcp-build": "npm install --save-dev && npm run build",
11-
"copy-sql": "cp ./src/db/db_init.sql ./dist/db/db_init.sql",
11+
"copy-sql": "cp ./src/db/db_init.sql ./dist/src/db/db_init.sql",
1212
"tsconfig": "tsc --init",
13-
"test": "tsc && jest --passWithNoTests"
13+
"test": "mocha"
1414
},
1515
"keywords": [],
1616
"author": "",
1717
"license": "ISC",
1818
"dependencies": {
1919
"@google-cloud/connect-firestore": "^2.0.2",
20-
"@google-cloud/firestore": "^6.7.0",
20+
"@google-cloud/firestore": "^4.15.1",
2121
"@google-cloud/storage": "^6.12.0",
2222
"@types/cors": "^2.8.13",
2323
"@types/cron": "^2.0.1",
@@ -44,18 +44,24 @@
4444
"passport": "^0.6.0",
4545
"passport-google-oauth20": "^2.0.0",
4646
"pg": "^8.11.1",
47-
"sanitize-filename": "^1.6.3",
48-
"ts-node": "^10.9.1"
47+
"sanitize-filename": "^1.6.3"
4948
},
5049
"devDependencies": {
50+
"@types/chai": "^4.3.5",
5151
"@types/cors": "^2.8.13",
5252
"@types/cron": "^2.0.1",
5353
"@types/express": "^4.17.17",
5454
"@types/express-session": "^1.17.7",
5555
"@types/luxon": "^3.3.1",
56+
"@types/mocha": "^10.0.1",
5657
"@types/multer": "^1.4.7",
5758
"@types/pg": "^8.10.2",
59+
"@types/sinon": "^10.0.16",
60+
"chai": "^4.3.8",
5861
"jest": "^29.5.0",
62+
"mocha": "^10.2.0",
63+
"sinon": "^15.2.0",
64+
"ts-node": "^10.9.1",
5965
"typescript": "^5.1.6"
6066
}
6167
}

‎backend/src/api/leetcode.ts

Lines changed: 37 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
import axios from "axios";
22
import { isValidUsername } from "../utils/utils";
33

4-
// Function to make GraphQL requests for a specific user ID
5-
export async function getSubmitStats(userId: string) {
6-
if (!isValidUsername(userId)) {
7-
console.log("invalid username");
8-
return;
9-
}
10-
try {
11-
// Make the GraphQL request
12-
const response = await axios.post("https://leetcode.com/graphql", {
13-
query: `
4+
5+
export class LeetCodeApi {
6+
// Function to make GraphQL requests for a specific user ID
7+
async getSubmitStats(userId: string) {
8+
if (!isValidUsername(userId)) {
9+
console.log("invalid username");
10+
return;
11+
}
12+
try {
13+
// Make the GraphQL request
14+
const response = await axios.post("https://leetcode.com/graphql", {
15+
query: `
1416
{
1517
matchedUser(username: "${userId}")
1618
{
@@ -27,28 +29,28 @@ export async function getSubmitStats(userId: string) {
2729
}
2830
}
2931
`,
30-
});
32+
});
3133

32-
if (response.data.data.matchedUser === null) {
33-
throw new Error(`Leetcode username ${userId} was not found`);
34-
}
34+
if (response.data.data.matchedUser === null) {
35+
throw new Error(`Leetcode username ${userId} was not found`);
36+
}
3537

36-
return response.data.data.matchedUser;
37-
} catch (error) {
38-
console.error(`Error fetching data for user ID ${userId}:`, error);
39-
throw error;
38+
return response.data.data.matchedUser;
39+
} catch (error) {
40+
console.error(`Error fetching data for user ID ${userId}:`, error);
41+
throw error;
42+
}
4043
}
41-
}
4244

43-
export async function getLatestAcceptedSubmits(
44-
userId: string,
45-
limit: number = 10
46-
) {
47-
if (!isValidUsername(userId)) return;
48-
try {
49-
// Make the GraphQL request
50-
const response = await axios.post("https://leetcode.com/graphql", {
51-
query: `
45+
async getLatestAcceptedSubmits(
46+
userId: string,
47+
limit: number = 10
48+
) {
49+
if (!isValidUsername(userId)) return;
50+
try {
51+
// Make the GraphQL request
52+
const response = await axios.post("https://leetcode.com/graphql", {
53+
query: `
5254
{
5355
recentAcSubmissionList(username: "${userId}", limit: ${limit}) {
5456
title
@@ -58,9 +60,12 @@ export async function getLatestAcceptedSubmits(
5860
}
5961
}
6062
`,
61-
});
62-
return response.data.data.recentAcSubmissionList;
63-
} catch (error) {
64-
console.error(`Error fetching data for user ID ${userId}:`, error);
63+
});
64+
return response.data.data.recentAcSubmissionList;
65+
} catch (error) {
66+
console.error(`Error fetching data for user ID ${userId}:`, error);
67+
}
6568
}
6669
}
70+
71+
export const leetcodeApi = new LeetCodeApi();

‎backend/src/api/vjudge.ts

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,22 @@ import { isValidUsername } from '../utils/utils'
33
import { UserNameNotFoundError } from '../errors/username-not-found-error';
44
import { ExternalApiError } from '../errors/external-api-error';
55

6+
export class VjudgeApi {
7+
async getSubmissionStats(username: string) {
8+
if (!isValidUsername(username)) throw new Error("Invalid username format");
69

7-
export async function getSubmissionStats(username: string) {
8-
if(!isValidUsername(username)) throw new Error("Invalid username format");
9-
10-
try {
11-
const response = await axios.get(`https://vjudge.net/user/solveDetail/${username}`);
12-
return response.data;
13-
} catch(err) {
14-
if(err instanceof AxiosError){
15-
const response = err.response!!;
16-
if(response.status === 404) throw new UserNameNotFoundError(username, "vjudge");
17-
else throw new ExternalApiError("vjudge", response.status);
10+
try {
11+
const response = await axios.get(`https://vjudge.net/user/solveDetail/${username}`);
12+
return response.data;
13+
} catch (err) {
14+
if (err instanceof AxiosError) {
15+
const response = err.response!!;
16+
if (response.status === 404) throw new UserNameNotFoundError(username, "vjudge");
17+
else throw new ExternalApiError("vjudge", response.status);
18+
}
19+
else throw err;
1820
}
19-
else throw err;
2021
}
21-
}
22+
}
23+
24+
export const vjudgeApi = new VjudgeApi();

‎backend/src/job/LeetcodeUpdateCollectingJob.ts

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,26 @@
1-
import { updatesCollectorService } from "../services/UpdatesCollectorService";
1+
import { leetcodeUpdatesCollectorService } from "../services/updates-collection/LeetcodeUpdatesCollectorService";
22
import { userRepository } from "../repository/UserRepository";
33

44
// Job responsible for collecting updates from Leetcode and updating the db
55
export class LeetcodeUpdateCollectingJob {
66
async run() {
7-
const userData = await userRepository.getAllUsernames();
7+
const mongoIDs = await userRepository.getAllMongoIds();
88

99
// we are querying external APIs, so we cannot do any load we want
1010
// Introduce batches of size <= 10 that we will process once in 0.2 seconds;
1111
const batchSize = 10;
1212
const delayBetweenBatches = 200;
13-
const totalBatches = Math.ceil(userData.length / batchSize);
13+
const totalBatches = Math.ceil(mongoIDs.length / batchSize);
1414

1515
for (let i = 0; i < totalBatches; i++) {
1616
const startIndex = i * batchSize;
17-
const endIndex = Math.min(startIndex + batchSize, userData.length);
18-
const batch = userData.slice(startIndex, endIndex);
17+
const endIndex = Math.min(startIndex + batchSize, mongoIDs.length);
18+
const batch = mongoIDs.slice(startIndex, endIndex);
1919

2020
// Process the current batch of users concurrently using Promise.all
21-
await Promise.all(batch.map(u =>
22-
updatesCollectorService.getAndStoreLeetcodeUpdates(u.username!!)));
21+
await Promise.all(batch.map(id => {
22+
leetcodeUpdatesCollectorService.getAndStoreUpdates(id);
23+
}));
2324

2425
// Introduce a delay between batches (in milliseconds)
2526
if (i < totalBatches - 1) {

‎backend/src/job/VjudgeUpdateCollectingJob.ts

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,26 @@
1-
import { updatesCollectorService } from "../services/UpdatesCollectorService";
1+
import { vjudgeUpdatesCollectorService } from "../services/updates-collection/VjudgeUpdatesCollectorService";
22
import { userRepository } from "../repository/UserRepository";
33

44
// Job responsible for collecting updates from VJudge and updating our db
55
export class VjudgeUpdateCollectingJob {
66
async run() {
7-
const userData = await userRepository.getAllUsernames();
7+
const mongoIDs = await userRepository.getAllMongoIds();
88

99
// we are querying external APIs, so we cannot do any load we want
1010
// VJudge is most likely not as robust as LeetCode so reduce the load compared to Leetcode
1111
// Introduce batches of size <= 5 that we will process once in 0.5 seconds;
1212
const batchSize = 5;
1313
const delayBetweenBatches = 200;
14-
const totalBatches = Math.ceil(userData.length / batchSize);
14+
const totalBatches = Math.ceil(mongoIDs.length / batchSize);
1515

1616
for (let i = 0; i < totalBatches; i++) {
1717
const startIndex = i * batchSize;
18-
const endIndex = Math.min(startIndex + batchSize, userData.length);
19-
const batch = userData.slice(startIndex, endIndex);
18+
const endIndex = Math.min(startIndex + batchSize, mongoIDs.length);
19+
const batch = mongoIDs.slice(startIndex, endIndex);
2020

2121
// Process the current batch of users concurrently
22-
await Promise.all(batch.map(u =>
23-
updatesCollectorService.getAndStoreVjudgeUpdates(u.username!!)));
22+
await Promise.all(batch.map(id =>
23+
vjudgeUpdatesCollectorService.getAndStoreUpdates(id)));
2424

2525
// Wait <delay> time before processing next batch;
2626
if (i < totalBatches - 1) {

‎backend/src/model/UpdateEventData.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export interface UpdateEventData {
2+
id: string,
23
username: string;
34
platform: string;
45
problemTitle: string;

‎backend/src/routes/test/trigger-requests-route.ts

Lines changed: 8 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,24 @@
11
import express, { Request, Response } from 'express'
22

3-
import { getSubmitStats } from '../../api/leetcode'
3+
import { leetcodeApi } from '../../api/leetcode'
44
import { validateUsername, handleValidationErrors } from '../../utils/middleware'
55
import { getUserIDs } from '../../model/users';
6-
import { getSubmissionStats } from '../../api/vjudge';
6+
import { vjudgeApi } from '../../api/vjudge';
77
import { UserNameNotFoundError } from '../../errors/username-not-found-error';
88
import { ExternalApiError } from '../../errors/external-api-error';
9-
import { updatesCollectorService } from '../../services/UpdatesCollectorService';
109
import { EventRepository } from '../../repository/EventRepository';
1110
import { isValidUsername } from '../../utils/utils';
1211
import { UserRepository } from '../../repository/UserRepository';
12+
import { allUpdatesCollectorService } from '../../services/updates-collection/AllUpdatesCollectorService';
13+
import { User } from '../../model/schemas/userSchema';
1314

1415
const router = express.Router()
1516

1617
router.get('/', async (req: Request, res: Response) => {
1718
try {
1819
const userIds = getUserIDs();
19-
const promises: Promise<ReturnType<typeof getSubmitStats>>[] = Array.from(userIds).map(async (id) => {
20-
const cur = await getSubmitStats(id);
20+
const promises: Promise<ReturnType<typeof leetcodeApi.getSubmitStats>>[] = Array.from(userIds).map(async (id) => {
21+
const cur = await leetcodeApi.getSubmitStats(id);
2122
console.log("cur:", cur);
2223
return cur;
2324
});
@@ -40,7 +41,7 @@ router.get('/leetcode/:userId', [
4041
], async (req: Request, res: Response) => {
4142
const { userId } = req.params;
4243

43-
const data = await getSubmitStats(userId);
44+
const data = await leetcodeApi.getSubmitStats(userId);
4445
console.log("data:", data);
4546

4647
res.send(data);
@@ -55,7 +56,7 @@ router.get('/vjudge/:username', [
5556
const { username } = req.params;
5657

5758
try {
58-
const data = await getSubmissionStats(username);
59+
const data = await vjudgeApi.getSubmissionStats(username);
5960
console.log("data:", data);
6061

6162
res.send(data);
@@ -70,33 +71,6 @@ router.get('/vjudge/:username', [
7071
}
7172
});
7273

73-
router.get('/updatesList/:username', [
74-
// Sanitize the userId variable
75-
validateUsername('username'),
76-
handleValidationErrors
77-
], async (req: Request, res: Response) => {
78-
const { username } = req.params;
79-
80-
if (!username) {
81-
res.status(400).send();
82-
}
83-
else {
84-
try {
85-
const updates = await updatesCollectorService.getUpdates(username);
86-
87-
res.json(updates).send();
88-
}
89-
catch (err) {
90-
if (err instanceof UserNameNotFoundError) {
91-
res.status(404).send("user not found");
92-
}
93-
else {
94-
res.status(500).send("Oops! Something went wrong");
95-
}
96-
}
97-
}
98-
});
99-
10074
router.get('/feed', [
10175
handleValidationErrors
10276
], async (req: Request, res: Response) => {

‎backend/src/routes/users-route.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ import { getUserIDs, addUserID } from "../model/users";
55
import { User } from "../model/schemas/userSchema";
66
import { UserNameNotFoundError } from "../errors/username-not-found-error";
77
import { UsernameAlreadyTakenError } from "../errors/username-already-taken-error";
8-
import { getSubmissionStats } from "../api/vjudge";
9-
import { getLatestAcceptedSubmits, getSubmitStats } from "../api/leetcode";
8+
import { vjudgeApi } from "../api/vjudge";
9+
import { leetcodeApi} from "../api/leetcode";
1010
import { imageRepository } from "../repository/ImageRepository";
1111
import {
1212
notificationRepository,
@@ -47,7 +47,7 @@ router.get(
4747
res.status(404).send("Leetcode username not found");
4848
} else {
4949
const leetcodeUsername = userData.leetcode.username;
50-
const submitStats = await getLatestAcceptedSubmits(leetcodeUsername);
50+
const submitStats = await leetcodeApi.getLatestAcceptedSubmits(leetcodeUsername);
5151

5252
if (!submitStats) {
5353
res.status(404).send("User not found on leetcode");
@@ -165,7 +165,7 @@ router.put(
165165
);
166166
}
167167

168-
const vjudgeStats = await getSubmissionStats(
168+
const vjudgeStats = await vjudgeApi.getSubmissionStats(
169169
data.vjudgeUsername
170170
);
171171
// console.log(vjudgeStats);
@@ -188,7 +188,7 @@ router.put(
188188
"leetcode"
189189
);
190190
}
191-
const leetcodeStats = await getSubmitStats(
191+
const leetcodeStats = await leetcodeApi.getSubmitStats(
192192
data.leetcodeUsername
193193
);
194194
console.log("this is LC data: " + leetcodeStats);

0 commit comments

Comments
 (0)