feat: update

packages/canyon-backend/package.json

@@ -24,6 +24,7 @@
     "@nestjs/mongoose": "^10.0.1",
     "@nestjs/passport": "^10.0.2",
     "@nestjs/platform-express": "^10.2.7",
+    "@nestjs/schedule": "^4.0.0",
     "@nestjs/serve-static": "^4.0.0",
     "@prisma/client": "^5.5.2",
     "axios": "^1.6.0",
@@ -32,6 +33,7 @@
     "class-validator": "^0.14.0",
     "cookie": "^0.5.0",
     "dayjs": "^1.11.10",
+    "diff": "^5.1.0",
     "express": "^4.18.2",
     "graphql": "^16.8.1",
     "graphql-query-complexity": "^0.12.0",

packages/canyon-backend/prisma/schema.prisma

@@ -86,3 +86,18 @@ model Codechange {
 
   @@map("codechange")
 }
+
+// status notstarted, running, success, failed, canceled
+model Task {
+  id        Int      @id @default(autoincrement())
+  name      String
+  createdAt DateTime @default(now()) @map("created_at") @db.Timestamp(3)
+  updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamp(3)
+  status    String
+  reportID  String   @map("report_id")
+  commitSha String   @map("commit_sha")
+  projectID String   @map("project_id")
+  result    Json
+
+  @@map("task")
+}

packages/canyon-backend/src/adapter/coverage-data.adapter.ts

@@ -1,6 +1,6 @@
 import axios from 'axios';
 import * as process from 'process';
-import { compressedData, decompressedData } from '../zstd';
+import { compressedData, decompressedData } from '../utils/zstd';
 export function getSpecificCoverageData(coverageDataId: string) {
   return axios
     .get(`${process.env['COVERAGE_DATA_URL']}/coverage-data/${coverageDataId}`)

packages/canyon-backend/src/coverage/coverage.controller.ts

@@ -5,18 +5,25 @@ import {
   Controller,
   Request,
   Get,
+  Query,
 } from '@nestjs/common';
 import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard';
 import { CoverageClientDto } from './dto/coverage-client.dto';
 import { DashboardService } from './services/dashboard.service';
 import { CoverageService } from './services/coverage.service';
 import { CoverageClientService } from './services/coverage-client.service';
+import { TriggerAggCoverageService } from './services/trigger-agg-coverage.service';
+// import {RetrieveCoverageTreeSummaryService} from "./services/retrieve-coverage-tree-summary.service";
+import { ListAggStatusService } from './services/list-agg-status.service';
 
 @Controller('')
 export class CoverageController {
   constructor(
     private readonly dashboardService: DashboardService, // private readonly pubsub: PubSubService,
     private readonly coverageClientService: CoverageClientService,
+    private readonly triggerAggCoverageService: TriggerAggCoverageService,
+    // private readonly retrieveCoverageTreeSummaryService: RetrieveCoverageTreeSummaryService,
+    private readonly listAggStatusService: ListAggStatusService,
   ) {}
 
   @UseGuards(JwtAuthGuard)
@@ -32,4 +39,31 @@ export class CoverageController {
   async dashboard(): Promise<any> {
     return this.dashboardService.invoke();
   }
+
+  // 触发覆盖率聚合方法
+  // 传 reportId 和 reporterId 都可以
+  @Post('coverage/triggeragg')
+  triggeragg(@Body() params: { reportID: string }) {
+    return this.triggerAggCoverageService.invoke({
+      reportID: params.reportID,
+    });
+  }
+
+  // 获取聚合状态
+  @Get('coverage/aggstatus')
+  listAggStatus(@Query() params: { reportID: string }) {
+    return this.listAggStatusService.invoke({
+      report_id: params.reportID,
+    });
+  }
+
+  // // 获取概览
+  // @Get('coverage/treesummary')
+  // retrieveCoverageTreeSummary(
+  //   @Query() params: { reportId?: string; report_id?: string },
+  // ) {
+  //   return this.retrieveCoverageTreeSummaryService.invoke({
+  //     report_id: params.reportId || params.report_id,
+  //   });
+  // }
 }

packages/canyon-backend/src/coverage/services/coverage.service.ts

@@ -14,7 +14,7 @@ import {
 } from '../../adapter/coverage-data.adapter';
 import { getFileInfo } from '../../adapter/gitlab.adapter';
 import { GitlabFileInfo } from '../models/gitlab-file-info.model';
-import { decompressedData } from '../../zstd';
+// import { decompressedData } from '../../zstd';
 import { Codechange } from '../models/codechange.model';
 @Injectable()
 export class CoverageService {

packages/canyon-backend/src/coverage/services/list-agg-status.service.ts

@@ -0,0 +1,43 @@
+import { Injectable } from '@nestjs/common';
+import { PrismaService } from '../../prisma/prisma.service';
+function checkTasksStatus(tasks) {
+  return tasks.some((task) => ['running', 'notstarted'].includes(task.status));
+}
+function checkTasksStatusSuccess(tasks) {
+  return tasks.every((task) => task.status === 'success');
+}
+@Injectable()
+export class ListAggStatusService {
+  constructor(private readonly prisma: PrismaService) {}
+
+  async invoke(params) {
+    const tasks = await this.prisma.task.findMany({
+      where: {
+        reportID: params.reportID,
+      },
+    });
+    if (tasks.length === 0) {
+      return {
+        code: 0,
+        msg: '未查询到reportId',
+        data: [],
+      };
+    } else if (checkTasksStatus(tasks)) {
+      return {
+        code: 1,
+        msg: '聚合中',
+        data: [],
+      };
+    } else if (checkTasksStatusSuccess(tasks)) {
+      return {
+        code: 2,
+        msg: '聚合完成',
+      };
+    } else {
+      return {
+        code: 3,
+        msg: '聚合失败',
+      };
+    }
+  }
+}

packages/canyon-backend/src/coverage/services/trigger-agg-coverage.service.ts

@@ -0,0 +1,87 @@
+import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
+import { PrismaService } from '../../prisma/prisma.service';
+import axios from 'axios';
+
+function checkTasksStatus(tasks) {
+  return tasks.some((task) => {
+    return ['running', 'notstarted'].includes(task.status);
+  });
+}
+@Injectable()
+export class TriggerAggCoverageService {
+  constructor(private readonly prisma: PrismaService) {}
+
+  async invoke(params) {
+    const { reportID } = params;
+    const tasks = await this.prisma.task.findMany({
+      where: {
+        reportID,
+      },
+    });
+    const coverages = await this.prisma.coverage.findMany({
+      where: {
+        reportID: reportID,
+      },
+    });
+    if (coverages.length === 0) {
+      return {
+        msg: '未查询到reportId',
+        data: [],
+        code: 0,
+      };
+    }
+
+    // 如果有聚合在跑，就提示在聚合
+    if (tasks.length > 0 && checkTasksStatus(tasks)) {
+      return {
+        msg: '报告聚合中',
+        data: [],
+        code: -1,
+      };
+    } else {
+      // 如果这个 reportID聚合完成了，就删除旧的聚合任务，创建新的聚合任务
+      for (let i = 0; i < tasks.length; i++) {
+        await this.prisma.task.deleteMany({
+          where: {
+            id: tasks[i].id,
+          },
+        });
+        console.log('删除旧的聚合任务' + tasks[i].id);
+      }
+
+      // ********** 重要 **********
+      // 关键
+      // ********** 重要 **********
+
+      const commitsAssociatedWithReport = await this.prisma.coverage.findMany({
+        where: {
+          reportID,
+        },
+        distinct: ['commitSha'], // 使用 distinct 来确保返回唯一的 commitSha
+        select: {
+          commitSha: true,
+          projectID: true, // 添加 projectID 到返回结果中
+        },
+      });
+      for (let i = 0; i < commitsAssociatedWithReport.length; i++) {
+        const { commitSha, projectID } = commitsAssociatedWithReport[i];
+        await this.prisma.task.create({
+          data: {
+            name: `Coverage Agg Task ${reportID},commitSha:${commitSha},projectID:${projectID}`,
+            status: 'notstarted',
+            reportID,
+            commitSha: '',
+            projectID: '',
+            result: {},
+          },
+        });
+      }
+
+      return {
+        msg: '聚合中',
+        data: [],
+        code: 1,
+      };
+    }
+  }
+}

packages/canyon-backend/src/tasks/services/aggregation-coverage.service.ts

@@ -0,0 +1,97 @@
+import { PrismaService } from '../../prisma/prisma.service';
+import {
+  createNewCoverageData,
+  getSpecificCoverageData,
+} from '../../adapter/coverage-data.adapter';
+import { genSummaryMapByCoverageMap, mergeCoverage } from '@canyon/data';
+
+export class AggregationCoverageService {
+  constructor(private readonly prisma: PrismaService) {}
+  async invoke(covType, commitSha, reportID) {
+    const coverages = await this.prisma.coverage.findMany({
+      where: {
+        covType: covType === 'agg' ? 'normal' : 'agg',
+        reportID: covType === 'agg' ? reportID : null,
+        commitSha,
+      },
+    });
+
+    let mainCov = {};
+
+    for (let i = 0; i < coverages.length; i++) {
+      const singleCov = await getSpecificCoverageData(coverages[i].relationID);
+      mainCov = mergeCoverage(mainCov, singleCov);
+    }
+
+    const mainCovCoverageData = await createNewCoverageData(mainCov);
+
+    // 删除老的
+    await this.prisma.coverage.deleteMany({
+      where: {
+        reportID: covType === 'agg' ? reportID : null,
+        covType: covType,
+        commitSha,
+      },
+    });
+
+    // 只有是agg的时候要删除就的agg的summary
+    if (covType === 'agg') {
+      await this.prisma.summary.deleteMany({
+        where: {
+          reportID: reportID,
+        },
+      });
+    }
+
+    const { compareTarget, projectID, reporter } =
+      await this.prisma.coverage.findFirst({
+        where: {
+          commitSha,
+          covType: 'normal',
+        },
+        orderBy: {
+          createdAt: 'desc', // 按照 createdAt 字段的降序排列（最新的在前面）
+        },
+      });
+
+    const codechanges = await this.prisma.codechange.findMany({
+      where: {
+        commitSha,
+        compareTarget,
+      },
+    });
+
+    const coverageSummaryMap = genSummaryMapByCoverageMap(mainCov, codechanges);
+    for (const coverageSummaryMapKey in coverageSummaryMap) {
+      // 落库数据
+      const { total, skipped, covered } = coverageSummaryMap[
+        coverageSummaryMapKey
+      ] as any;
+      console.log(coverageSummaryMapKey, total, skipped, covered);
+      await this.prisma.summary.create({
+        data: {
+          reportID: reportID,
+          metricType: coverageSummaryMapKey,
+          commitSha: commitSha,
+          total,
+          skipped,
+          covered,
+        },
+      });
+    }
+
+    await this.prisma.coverage.create({
+      data: {
+        compareTarget,
+        commitSha,
+        reportID: covType === 'agg' ? reportID : '',
+        projectID,
+        relationID: mainCovCoverageData.insertedId,
+        covType: covType,
+        reporter: reporter,
+        key: '',
+        instrumentCwd: '',
+      },
+    });
+  }
+}

packages/canyon-backend/src/tasks/services/cleaning-up-outdated-data.service.ts

@@ -0,0 +1,21 @@
+import { Cron } from '@nestjs/schedule';
+import { PrismaService } from '../../prisma/prisma.service';
+import { Injectable } from '@nestjs/common';
+
+@Injectable()
+export class CleaningUpOutdatedDataService {
+  constructor(private readonly prisma: PrismaService) {}
+  // 每过一天清理一下coverage集合，删除超过一周的normal数据
+  @Cron('30 2 * * *') // 每天凌晨2点半执行
+  async cleaningUpOutdatedData() {
+    const coverageDeleteManyRes = await this.prisma.coverage.deleteMany({
+      where: {
+        covType: 'normal',
+        createdAt: {
+          lt: new Date(new Date().valueOf() - 7 * 24 * 60 * 60 * 1000),
+        },
+      },
+    });
+    console.log(coverageDeleteManyRes, 'coverageDeleteManyRes');
+  }
+}