feat: queue processing (#26)

* feat: added bull queue lessons processing

* ci: who uses these anyways 🧌

* feat: added redis unix socket support

.env.example

@@ -3,6 +3,11 @@ UNTIS_USERNAME=
 UNTIS_PASSWORD=
 UNTIS_BASEURL=
 
+BULL_REDIS_HOST=localhost
+BULL_REDIS_PORT=6379
+# or
+#BULL_REDIS_PATH=/run/redis/redis.sock
+
 #MAINTENANCE_TITLE=
 #MAINTENANCE_DESCRIPTION=
 #MAINTENANCE_LOCATION=

.github/workflows/node-ci.yaml

@@ -6,8 +6,7 @@ jobs:
     strategy:
       matrix:
         node-version: [16.x, 18.x]
-        os: [ubuntu-latest, windows-latest, macos-latest]
-    runs-on: ${{matrix.os}}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3

default.nix

@@ -13,7 +13,7 @@ mkYarnPackage rec {
 
   offlineCache = fetchYarnDeps {
     yarnLock = src + "/yarn.lock";
-    hash = "sha256-NHghkf5Nziyz3M7E4941sV5JFqY7RYMTlZqYsQPZLpU=";
+    hash = "sha256-mToEmc4cNd2fsyT/DUkzMiO4BYUDw6aexbcXOVfB2ds=";
   };
 
   nativeBuildInputs = [makeWrapper];
@@ -30,6 +30,8 @@ mkYarnPackage rec {
       --add-flags "$out/libexec/untis-ics-sync/deps/untis-ics-sync/dist/main.js"
   '';
 
+  dontStrip = true;
+
   meta = with lib; {
     description = "Serves a calendar API (ICS) for events provided from Untis";
     homepage = "https://github.com/bddvlpr/untis-ics-sync";

module.nix

@@ -40,12 +40,20 @@ in {
       serviceConfig = {
         Restart = "on-failure";
         ExecStart = "${lib.getExe cfg.package}";
+        Environment = [
+          "BULL_REDIS_PATH=${config.services.redis.servers.untis-ics-sync.unixSocket}"
+        ];
         EnvironmentFile = cfg.envFile;
         User = cfg.user;
         Group = cfg.group;
       };
     };
 
+    services.redis.servers.untis-ics-sync = {
+      enable = true;
+      user = cfg.user;
+    };
+
     users = {
       users.untis-ics-sync = lib.mkIf (cfg.user == "untis-ics-sync") {
         isSystemUser = true;

package.json

@@ -20,13 +20,15 @@
     "test:e2e": "jest --config ./test/jest-e2e.json"
   },
   "dependencies": {
+    "@nestjs/bull": "^10.0.1",
     "@nestjs/cache-manager": "^2.1.0",
     "@nestjs/common": "^9.0.0",
     "@nestjs/config": "^3.0.1",
     "@nestjs/core": "^9.0.0",
     "@nestjs/platform-express": "^9.0.0",
     "@nestjs/swagger": "^7.1.10",
     "@willsoto/nestjs-prometheus": "^6.0.0",
+    "bull": "^4.11.5",
     "cache-manager": "^5.2.3",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",

src/app.module.ts

@@ -1,15 +1,27 @@
 import { Module } from '@nestjs/common';
 import { UntisModule } from './untis/untis.module';
 import { ClassesModule } from './classes/classes.module';
-import { ConfigModule } from '@nestjs/config';
+import { ConfigModule, ConfigService } from '@nestjs/config';
 import { SubjectsModule } from './subjects/subjects.module';
 import { LessonsModule } from './lessons/lessons.module';
 import { HolidaysModule } from './holidays/holidays.module';
 import { MetricsModule } from './metrics/metrics.module';
+import { BullModule } from '@nestjs/bull';
 
 @Module({
   imports: [
     ConfigModule.forRoot({ isGlobal: true }),
+    BullModule.forRootAsync({
+      imports: [ConfigModule],
+      inject: [ConfigService],
+      useFactory: (configService: ConfigService) => ({
+        redis: {
+          host: configService.get<string>('BULL_REDIS_HOST'),
+          port: configService.get<number>('BULL_REDIS_PORT'),
+          path: configService.get<string>('BULL_REDIS_PATH'),
+        },
+      }),
+    }),
     UntisModule,
     ClassesModule,
     SubjectsModule,

src/lessons/lessons.controller.ts

@@ -1,3 +1,4 @@
+import { InjectQueue } from '@nestjs/bull';
 import {
   Controller,
   DefaultValuePipe,
@@ -17,17 +18,16 @@ import {
   ApiQuery,
   ApiTags,
 } from '@nestjs/swagger';
-import { UntisService } from 'src/untis/untis.service';
+import { Queue } from 'bull';
 import { GetLessonDto } from './dto/get-lesson.dto';
-import { LessonsService } from './lessons.service';
+import { FetchData, FetchIcsData } from './lessons.processor';
 
 @ApiTags('lessons')
 @Controller('lessons')
 export class LessonsController {
   constructor(
     private readonly configService: ConfigService,
-    private readonly untisService: UntisService,
-    private readonly lessonsService: LessonsService,
+    @InjectQueue('lessons-queue') private readonly lessonsQueue: Queue,
   ) {}
 
   @ApiOkResponse({
@@ -39,11 +39,13 @@ export class LessonsController {
   })
   @Get(':classId')
   async getLessonsForClass(@Param('classId', ParseIntPipe) classId: number) {
-    const lessons = await this.untisService.fetchTimetable(
-      this.configService.get<number>('LESSONS_TIMETABLE_BEFORE', 7),
-      this.configService.get<number>('LESSONS_TIMETABLE_AFTER', 7),
+    const job = this.lessonsQueue.add('fetch', {
+      before: this.configService.get<number>('LESSONS_TIMETABLE_BEFORE', 7),
+      after: this.configService.get<number>('LESSONS_TIMETABLE_AFTER', 14),
       classId,
-    );
+    } as FetchData);
+
+    const lessons = (await job).finished();
     if (!lessons)
       throw new HttpException(
         'No lessons found, does class exist?',
@@ -76,22 +78,23 @@ export class LessonsController {
     @Query('offset', new DefaultValuePipe(0), ParseIntPipe)
     offset?: number,
   ) {
-    const lessons = await this.untisService.fetchTimetable(
-      this.configService.get<number>('LESSONS_TIMETABLE_BEFORE', 7),
-      this.configService.get<number>('LESSONS_TIMETABLE_AFTER', 7),
+    const job = this.lessonsQueue.add('fetch-ics', {
+      before: this.configService.get<number>('LESSONS_TIMETABLE_BEFORE', 7),
+      after: this.configService.get<number>('LESSONS_TIMETABLE_AFTER', 14),
       classId,
-    );
-    if (!lessons)
+      includedSubjects,
+      excludedSubjects,
+      alarms,
+      offset,
+    } as FetchIcsData);
+
+    const ics = await (await job).finished();
+    if (!ics)
       throw new HttpException(
         'No lessons found, does class exist?',
         HttpStatus.NOT_FOUND,
       );
 
-    return this.lessonsService.convertToEvents(lessons, {
-      includedSubjects,
-      excludedSubjects,
-      alarms,
-      offset,
-    });
+    return ics;
   }
 }

src/lessons/lessons.module.ts

@@ -2,10 +2,20 @@ import { Module } from '@nestjs/common';
 import { LessonsService } from './lessons.service';
 import { LessonsController } from './lessons.controller';
 import { UntisModule } from 'src/untis/untis.module';
+import { BullModule } from '@nestjs/bull';
+import { LessonsProcessor } from './lessons.processor';
 
 @Module({
-  providers: [LessonsService],
+  providers: [LessonsService, LessonsProcessor],
   controllers: [LessonsController],
-  imports: [UntisModule],
+  imports: [
+    UntisModule,
+    BullModule.registerQueue({
+      name: 'lessons-queue',
+      settings: {
+        stalledInterval: 5,
+      },
+    }),
+  ],
 })
 export class LessonsModule {}

src/lessons/lessons.processor.ts

@@ -0,0 +1,67 @@
+import { Process, Processor } from '@nestjs/bull';
+import { Job } from 'bull';
+import { UntisService } from 'src/untis/untis.service';
+import { LessonsService } from './lessons.service';
+
+export interface FetchData {
+  before: number;
+  after: number;
+  classId: number;
+}
+
+export interface FetchIcsData {
+  before: number;
+  after: number;
+  classId: number;
+  includedSubjects?: number[];
+  excludedSubjects?: number[];
+  alarms?: number[];
+  offset?: number;
+}
+
+@Processor('lessons-queue')
+export class LessonsProcessor {
+  constructor(
+    private readonly untisService: UntisService,
+    private readonly lessonsService: LessonsService,
+  ) {}
+
+  @Process('fetch')
+  async handleFetch(job: Job<FetchData>) {
+    const { before, after, classId } = job.data;
+    const lessons = await this.untisService.fetchTimetable(
+      before,
+      after,
+      classId,
+    );
+    await job.progress(100);
+    return lessons;
+  }
+
+  @Process('fetch-ics')
+  async handleFetchIcs(job: Job<FetchIcsData>) {
+    const {
+      before,
+      after,
+      classId,
+      includedSubjects,
+      excludedSubjects,
+      alarms,
+      offset,
+    } = job.data;
+    const lessons = await this.untisService.fetchTimetable(
+      before,
+      after,
+      classId,
+    );
+    await job.progress(50);
+    const ics = await this.lessonsService.convertToEvents(lessons, {
+      includedSubjects,
+      excludedSubjects,
+      alarms,
+      offset,
+    });
+    await job.progress(100);
+    return ics;
+  }
+}