Feat/scholar sync (#142)

* initialize scholar module

* wip: create service

* base structure

* wip: repository and interface

* wip

* feat: SyncApiKeyAuth

* wip: working on scholardb sync

* feat: department update

* wip

* feat: scholarDB update API

* feat: Add sync slack logs

* refactor: APIKeyAuth set authentication/authorization instead of isPublic

* refactor: Remove redundant secret in body

* fix: Add missing SkackNotiService

* feat: sync examtime

* feat: sync classtimes

* refactor: syncTime method to handle both examtime
and classtime syncing

* refactor: Divide sync services

* feat: Sync taken lectures

* feat: Create schema for saving taken_lecture raw data

* fix: sync baseline with prod database

* feat: Save raw taken lectures

* feat: repopulate taken lectures of new user from rawTakenLecture data

* chore: Add needed env vars to .env.example

* fix: Allow bigger request for sync requests

* fix: Remove Error being logged when jwt auth fails

* fix: Nested object becomes empty, english_lec can be ""

* chore: console.log related

* fix: Fix or skip tests

* chore: update jest

* fix: Maark lectures not in API as deleted

* enhance: syncScholarDB log and result

* test: syncScholarDB

* test: classtime sync

* fix: Time not saved correctly

* test: examtime sync

* fix: professor id wrongly connected

* test: enhance syncScholarDB test

* test: syncTakenLecture

* fix sync bugs

* fix: enlarge type, name_en fields length in schema

* fix: handle multiple users with same student_id

* fix: timezone related issue

env/.env.example

@@ -7,4 +7,6 @@ EXPIRES_IN=3600
 REFRESH_EXPIRES_IN=2592000
 SSO_IS_BETA = false
 SSO_CLIENT_ID = ""
-SSO_SECRET_KEY = ""
+SSO_SECRET_KEY = ""
+SYNC_SECRET = ""
+SLACK_KEY = ""

package.json

@@ -30,9 +30,9 @@
     "client:generate": "npx prisma generate --schema src/prisma/schema.prisma",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "lint:check": "eslint .",
-    "test": "jest",
-    "test:watch": "jest --watch",
-    "test:cov": "jest --coverage",
+    "test": "dotenv -e env/.env.test -- jest --maxWorkers 1",
+    "test:watch": "dotenv -e env/.env.test -- jest --watch",
+    "test:cov": "dotenv -e env/.env.test -- jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
     "test:e2e": "jest --config ./test/jest-e2e.json",
     "prepare": "husky install",
@@ -51,6 +51,7 @@
     "@nestjs/platform-express": "^9.0.0",
     "@nestjs/swagger": "^7.1.1",
     "@prisma/client": "^5.18.0",
+    "@slack/web-api": "^7.7.0",
     "@types/cookie-parser": "^1.4.3",
     "@types/express-session": "^1.17.7",
     "@types/morgan": "^1.9.4",
@@ -87,7 +88,7 @@
     "@types/csurf": "^1.11.5",
     "@types/express": "^4.17.13",
     "@types/inquirer": "^9.0.7",
-    "@types/jest": "28.1.4",
+    "@types/jest": "^29.5.14",
     "@types/node": "^16.18.23",
     "@types/supertest": "^6.0.2",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
@@ -96,13 +97,13 @@
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
     "husky": "^8.0.0",
-    "jest": "28.1.2",
+    "jest": "^29.7.0",
     "lint-staged": "^13.2.3",
     "prettier": "^2.3.2",
     "prisma": "^5.18.0",
     "source-map-support": "^0.5.20",
     "supertest": "^7.0.0",
-    "ts-jest": "^28.0.5",
+    "ts-jest": "^29.2.5",
     "ts-loader": "^9.2.3",
     "ts-node": "^10.9.1",
     "tsconfig-paths": "4.0.0",

src/app.controller.spec.ts

@@ -1,14 +1,13 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { AppController } from './app.controller';
-import { AppService } from './app.service';
+import { AppModule } from './app.module';
 
 describe('AppController', () => {
   let appController: AppController;
 
   beforeEach(async () => {
     const app: TestingModule = await Test.createTestingModule({
-      controllers: [AppController],
-      providers: [AppService],
+      imports: [AppModule],
     }).compile();
 
     appController = app.get<AppController>(AppController);

src/app.module.ts

@@ -1,9 +1,15 @@
+import { ClsPluginTransactional } from '@nestjs-cls/transactional';
+import { TransactionalAdapterPrisma } from '@nestjs-cls/transactional-adapter-prisma';
 import { Module } from '@nestjs/common';
 import { APP_GUARD } from '@nestjs/core';
 import { JwtService } from '@nestjs/jwt';
+import { PrismaService } from '@src/prisma/prisma.service';
+import { ClsModule } from 'nestjs-cls';
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
+import { AuthConfig } from './modules/auth/auth.config';
 import { AuthModule } from './modules/auth/auth.module';
+import { AuthGuard } from './modules/auth/guard/auth.guard';
 import { JwtCookieGuard } from './modules/auth/guard/jwt-cookie.guard';
 import { MockAuthGuard } from './modules/auth/guard/mock-auth-guard';
 import { CoursesModule } from './modules/courses/courses.module';
@@ -16,19 +22,14 @@ import { RatesModule } from './modules/rates/rates.module';
 import { ReviewsModule } from './modules/reviews/reviews.module';
 import { SemestersModule } from './modules/semesters/semesters.module';
 import { SessionModule } from './modules/session/session.module';
+import { ShareModule } from './modules/share/share.module';
 import { StatusModule } from './modules/status/status.module';
+import { SyncModule } from './modules/sync/sync.module';
 import { TimetablesModule } from './modules/timetables/timetables.module';
 import { TracksModule } from './modules/tracks/tracks.module';
 import { UserModule } from './modules/user/user.module';
 import { WishlistModule } from './modules/wishlist/wishlist.module';
 import { PrismaModule } from './prisma/prisma.module';
-import { ShareModule } from './modules/share/share.module';
-import { AuthConfig } from './modules/auth/auth.config';
-import { AuthGuard } from './modules/auth/guard/auth.guard';
-import { ClsModule } from 'nestjs-cls';
-import { ClsPluginTransactional } from '@nestjs-cls/transactional';
-import { PrismaService } from '@src/prisma/prisma.service';
-import { TransactionalAdapterPrisma } from '@nestjs-cls/transactional-adapter-prisma';
 
 @Module({
   imports: [
@@ -50,6 +51,7 @@ import { TransactionalAdapterPrisma } from '@nestjs-cls/transactional-adapter-pr
     PlannersModule,
     TracksModule,
     ShareModule,
+    SyncModule,
     ClsModule.forRoot({
       global: true,
       middleware: { mount: true },

src/bootstrap/bootstrap.ts

@@ -2,6 +2,7 @@ import { ValidationPipe, VersioningType } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import cookieParser from 'cookie-parser';
 import csrf from 'csurf';
+import { json } from 'express';
 import session from 'express-session';
 import { AppModule } from '../app.module';
 import settings from '../settings';
@@ -69,6 +70,9 @@ async function bootstrap() {
     }),
   );
 
+  app.use('/sync', json({ limit: '50mb' }));
+  app.use(json({ limit: '100kb' }));
+
   app.enableShutdownHooks();
   return app.listen(8000);
 }

src/common/decorators/sync-api-key-auth.decorator.ts

@@ -0,0 +1,4 @@
+import { SetMetadata } from '@nestjs/common';
+
+export const USE_SYNC_API_KEY = 'useSyncAPIKey';
+export const SyncApiKeyAuth = () => SetMetadata(USE_SYNC_API_KEY, true);

src/common/entities/EProfessor.ts

@@ -0,0 +1,8 @@
+import { Prisma } from '@prisma/client';
+
+export namespace EProfessor {
+  export const Basic = Prisma.validator<Prisma.subject_professorDefaultArgs>()(
+    {},
+  );
+  export type Basic = Prisma.subject_professorGetPayload<typeof Basic>;
+}

src/common/entities/ETakenLecture.ts

@@ -0,0 +1,12 @@
+import { Prisma } from '@prisma/client';
+
+export namespace ETakenLecture {
+  export const Basic =
+    Prisma.validator<Prisma.session_userprofile_taken_lecturesDefaultArgs>()(
+      {},
+    );
+
+  export type Basic = Prisma.session_userprofile_taken_lecturesGetPayload<
+    typeof Basic
+  >;
+}

src/common/entities/EUserProfile.ts

@@ -0,0 +1,13 @@
+import { Prisma } from '@prisma/client';
+import { ETakenLecture } from './ETakenLecture';
+
+export namespace EUserProfile {
+  export const WithTakenLectures =
+    Prisma.validator<Prisma.session_userprofileDefaultArgs>()({
+      include: { taken_lectures: ETakenLecture.Basic },
+    });
+
+  export type WithTakenLectures = Prisma.session_userprofileGetPayload<
+    typeof WithTakenLectures
+  >;
+}