diff --git a/packages/canyon-backend/package.json b/packages/canyon-backend/package.json index 7ac3cb99..0f8d014e 100644 --- a/packages/canyon-backend/package.json +++ b/packages/canyon-backend/package.json @@ -24,6 +24,7 @@ "@nestjs/mongoose": "^10.0.1", "@nestjs/passport": "^10.0.2", "@nestjs/platform-express": "^10.2.7", + "@nestjs/schedule": "^4.0.0", "@nestjs/serve-static": "^4.0.0", "@prisma/client": "^5.5.2", "axios": "^1.6.0", @@ -32,6 +33,7 @@ "class-validator": "^0.14.0", "cookie": "^0.5.0", "dayjs": "^1.11.10", + "diff": "^5.1.0", "express": "^4.18.2", "graphql": "^16.8.1", "graphql-query-complexity": "^0.12.0", diff --git a/packages/canyon-backend/prisma/schema.prisma b/packages/canyon-backend/prisma/schema.prisma index 7ff90c03..7133dd3e 100755 --- a/packages/canyon-backend/prisma/schema.prisma +++ b/packages/canyon-backend/prisma/schema.prisma @@ -86,3 +86,18 @@ model Codechange { @@map("codechange") } + +// status notstarted, running, success, failed, canceled +model Task { + id Int @id @default(autoincrement()) + name String + createdAt DateTime @default(now()) @map("created_at") @db.Timestamp(3) + updatedAt DateTime @updatedAt @map("updated_at") @db.Timestamp(3) + status String + reportID String @map("report_id") + commitSha String @map("commit_sha") + projectID String @map("project_id") + result Json + + @@map("task") +} diff --git a/packages/canyon-backend/src/adapter/coverage-data.adapter.ts b/packages/canyon-backend/src/adapter/coverage-data.adapter.ts index 235a7a64..1e1b3845 100644 --- a/packages/canyon-backend/src/adapter/coverage-data.adapter.ts +++ b/packages/canyon-backend/src/adapter/coverage-data.adapter.ts @@ -1,6 +1,6 @@ import axios from 'axios'; import * as process from 'process'; -import { compressedData, decompressedData } from '../zstd'; +import { compressedData, decompressedData } from '../utils/zstd'; export function getSpecificCoverageData(coverageDataId: string) { return axios .get(`${process.env['COVERAGE_DATA_URL']}/coverage-data/${coverageDataId}`) diff --git a/packages/canyon-backend/src/coverage/coverage.controller.ts b/packages/canyon-backend/src/coverage/coverage.controller.ts index 9ff11a7e..9aba3a66 100644 --- a/packages/canyon-backend/src/coverage/coverage.controller.ts +++ b/packages/canyon-backend/src/coverage/coverage.controller.ts @@ -5,18 +5,25 @@ import { Controller, Request, Get, + Query, } from '@nestjs/common'; import { JwtAuthGuard } from '../auth/guards/jwt-auth.guard'; import { CoverageClientDto } from './dto/coverage-client.dto'; import { DashboardService } from './services/dashboard.service'; import { CoverageService } from './services/coverage.service'; import { CoverageClientService } from './services/coverage-client.service'; +import { TriggerAggCoverageService } from './services/trigger-agg-coverage.service'; +// import {RetrieveCoverageTreeSummaryService} from "./services/retrieve-coverage-tree-summary.service"; +import { ListAggStatusService } from './services/list-agg-status.service'; @Controller('') export class CoverageController { constructor( private readonly dashboardService: DashboardService, // private readonly pubsub: PubSubService, private readonly coverageClientService: CoverageClientService, + private readonly triggerAggCoverageService: TriggerAggCoverageService, + // private readonly retrieveCoverageTreeSummaryService: RetrieveCoverageTreeSummaryService, + private readonly listAggStatusService: ListAggStatusService, ) {} @UseGuards(JwtAuthGuard) @@ -32,4 +39,31 @@ export class CoverageController { async dashboard(): Promise { return this.dashboardService.invoke(); } + + // 触发覆盖率聚合方法 + // 传 reportId 和 reporterId 都可以 + @Post('coverage/triggeragg') + triggeragg(@Body() params: { reportID: string }) { + return this.triggerAggCoverageService.invoke({ + reportID: params.reportID, + }); + } + + // 获取聚合状态 + @Get('coverage/aggstatus') + listAggStatus(@Query() params: { reportID: string }) { + return this.listAggStatusService.invoke({ + report_id: params.reportID, + }); + } + + // // 获取概览 + // @Get('coverage/treesummary') + // retrieveCoverageTreeSummary( + // @Query() params: { reportId?: string; report_id?: string }, + // ) { + // return this.retrieveCoverageTreeSummaryService.invoke({ + // report_id: params.reportId || params.report_id, + // }); + // } } diff --git a/packages/canyon-backend/src/coverage/services/coverage.service.ts b/packages/canyon-backend/src/coverage/services/coverage.service.ts index e7105f6e..8772a57f 100755 --- a/packages/canyon-backend/src/coverage/services/coverage.service.ts +++ b/packages/canyon-backend/src/coverage/services/coverage.service.ts @@ -14,7 +14,7 @@ import { } from '../../adapter/coverage-data.adapter'; import { getFileInfo } from '../../adapter/gitlab.adapter'; import { GitlabFileInfo } from '../models/gitlab-file-info.model'; -import { decompressedData } from '../../zstd'; +// import { decompressedData } from '../../zstd'; import { Codechange } from '../models/codechange.model'; @Injectable() export class CoverageService { diff --git a/packages/canyon-backend/src/coverage/services/list-agg-status.service.ts b/packages/canyon-backend/src/coverage/services/list-agg-status.service.ts new file mode 100644 index 00000000..05e9ad5a --- /dev/null +++ b/packages/canyon-backend/src/coverage/services/list-agg-status.service.ts @@ -0,0 +1,43 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '../../prisma/prisma.service'; +function checkTasksStatus(tasks) { + return tasks.some((task) => ['running', 'notstarted'].includes(task.status)); +} +function checkTasksStatusSuccess(tasks) { + return tasks.every((task) => task.status === 'success'); +} +@Injectable() +export class ListAggStatusService { + constructor(private readonly prisma: PrismaService) {} + + async invoke(params) { + const tasks = await this.prisma.task.findMany({ + where: { + reportID: params.reportID, + }, + }); + if (tasks.length === 0) { + return { + code: 0, + msg: '未查询到reportId', + data: [], + }; + } else if (checkTasksStatus(tasks)) { + return { + code: 1, + msg: '聚合中', + data: [], + }; + } else if (checkTasksStatusSuccess(tasks)) { + return { + code: 2, + msg: '聚合完成', + }; + } else { + return { + code: 3, + msg: '聚合失败', + }; + } + } +} diff --git a/packages/canyon-backend/src/coverage/services/trigger-agg-coverage.service.ts b/packages/canyon-backend/src/coverage/services/trigger-agg-coverage.service.ts new file mode 100644 index 00000000..fe2f7f2d --- /dev/null +++ b/packages/canyon-backend/src/coverage/services/trigger-agg-coverage.service.ts @@ -0,0 +1,87 @@ +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { PrismaService } from '../../prisma/prisma.service'; +import axios from 'axios'; + +function checkTasksStatus(tasks) { + return tasks.some((task) => { + return ['running', 'notstarted'].includes(task.status); + }); +} +@Injectable() +export class TriggerAggCoverageService { + constructor(private readonly prisma: PrismaService) {} + + async invoke(params) { + const { reportID } = params; + const tasks = await this.prisma.task.findMany({ + where: { + reportID, + }, + }); + const coverages = await this.prisma.coverage.findMany({ + where: { + reportID: reportID, + }, + }); + if (coverages.length === 0) { + return { + msg: '未查询到reportId', + data: [], + code: 0, + }; + } + + // 如果有聚合在跑,就提示在聚合 + if (tasks.length > 0 && checkTasksStatus(tasks)) { + return { + msg: '报告聚合中', + data: [], + code: -1, + }; + } else { + // 如果这个 reportID聚合完成了,就删除旧的聚合任务,创建新的聚合任务 + for (let i = 0; i < tasks.length; i++) { + await this.prisma.task.deleteMany({ + where: { + id: tasks[i].id, + }, + }); + console.log('删除旧的聚合任务' + tasks[i].id); + } + + // ********** 重要 ********** + // 关键 + // ********** 重要 ********** + + const commitsAssociatedWithReport = await this.prisma.coverage.findMany({ + where: { + reportID, + }, + distinct: ['commitSha'], // 使用 distinct 来确保返回唯一的 commitSha + select: { + commitSha: true, + projectID: true, // 添加 projectID 到返回结果中 + }, + }); + for (let i = 0; i < commitsAssociatedWithReport.length; i++) { + const { commitSha, projectID } = commitsAssociatedWithReport[i]; + await this.prisma.task.create({ + data: { + name: `Coverage Agg Task ${reportID},commitSha:${commitSha},projectID:${projectID}`, + status: 'notstarted', + reportID, + commitSha: '', + projectID: '', + result: {}, + }, + }); + } + + return { + msg: '聚合中', + data: [], + code: 1, + }; + } + } +} diff --git a/packages/canyon-backend/src/tasks/README.md b/packages/canyon-backend/src/tasks/README.md new file mode 100644 index 00000000..e69de29b diff --git a/packages/canyon-backend/src/tasks/services/aggregation-coverage.service.ts b/packages/canyon-backend/src/tasks/services/aggregation-coverage.service.ts new file mode 100644 index 00000000..83b102e1 --- /dev/null +++ b/packages/canyon-backend/src/tasks/services/aggregation-coverage.service.ts @@ -0,0 +1,97 @@ +import { PrismaService } from '../../prisma/prisma.service'; +import { + createNewCoverageData, + getSpecificCoverageData, +} from '../../adapter/coverage-data.adapter'; +import { genSummaryMapByCoverageMap, mergeCoverage } from '@canyon/data'; + +export class AggregationCoverageService { + constructor(private readonly prisma: PrismaService) {} + async invoke(covType, commitSha, reportID) { + const coverages = await this.prisma.coverage.findMany({ + where: { + covType: covType === 'agg' ? 'normal' : 'agg', + reportID: covType === 'agg' ? reportID : null, + commitSha, + }, + }); + + let mainCov = {}; + + for (let i = 0; i < coverages.length; i++) { + const singleCov = await getSpecificCoverageData(coverages[i].relationID); + mainCov = mergeCoverage(mainCov, singleCov); + } + + const mainCovCoverageData = await createNewCoverageData(mainCov); + + // 删除老的 + await this.prisma.coverage.deleteMany({ + where: { + reportID: covType === 'agg' ? reportID : null, + covType: covType, + commitSha, + }, + }); + + // 只有是agg的时候要删除就的agg的summary + if (covType === 'agg') { + await this.prisma.summary.deleteMany({ + where: { + reportID: reportID, + }, + }); + } + + const { compareTarget, projectID, reporter } = + await this.prisma.coverage.findFirst({ + where: { + commitSha, + covType: 'normal', + }, + orderBy: { + createdAt: 'desc', // 按照 createdAt 字段的降序排列(最新的在前面) + }, + }); + + const codechanges = await this.prisma.codechange.findMany({ + where: { + commitSha, + compareTarget, + }, + }); + + const coverageSummaryMap = genSummaryMapByCoverageMap(mainCov, codechanges); + for (const coverageSummaryMapKey in coverageSummaryMap) { + // 落库数据 + const { total, skipped, covered } = coverageSummaryMap[ + coverageSummaryMapKey + ] as any; + console.log(coverageSummaryMapKey, total, skipped, covered); + await this.prisma.summary.create({ + data: { + reportID: reportID, + metricType: coverageSummaryMapKey, + commitSha: commitSha, + total, + skipped, + covered, + }, + }); + } + + await this.prisma.coverage.create({ + data: { + compareTarget, + commitSha, + reportID: covType === 'agg' ? reportID : '', + projectID, + relationID: mainCovCoverageData.insertedId, + covType: covType, + reporter: reporter, + key: '', + instrumentCwd: '', + }, + }); + } +} diff --git a/packages/canyon-backend/src/tasks/services/cleaning-up-outdated-data.service.ts b/packages/canyon-backend/src/tasks/services/cleaning-up-outdated-data.service.ts new file mode 100644 index 00000000..b6069683 --- /dev/null +++ b/packages/canyon-backend/src/tasks/services/cleaning-up-outdated-data.service.ts @@ -0,0 +1,21 @@ +import { Cron } from '@nestjs/schedule'; +import { PrismaService } from '../../prisma/prisma.service'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class CleaningUpOutdatedDataService { + constructor(private readonly prisma: PrismaService) {} + // 每过一天清理一下coverage集合,删除超过一周的normal数据 + @Cron('30 2 * * *') // 每天凌晨2点半执行 + async cleaningUpOutdatedData() { + const coverageDeleteManyRes = await this.prisma.coverage.deleteMany({ + where: { + covType: 'normal', + createdAt: { + lt: new Date(new Date().valueOf() - 7 * 24 * 60 * 60 * 1000), + }, + }, + }); + console.log(coverageDeleteManyRes, 'coverageDeleteManyRes'); + } +} diff --git a/packages/canyon-backend/src/tasks/services/coverage-agg-task.service.ts b/packages/canyon-backend/src/tasks/services/coverage-agg-task.service.ts new file mode 100644 index 00000000..35ee28f9 --- /dev/null +++ b/packages/canyon-backend/src/tasks/services/coverage-agg-task.service.ts @@ -0,0 +1,96 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { PrismaService } from '../../prisma/prisma.service'; +import CanyonUtil from 'canyon-util'; +import { Cron, Interval } from '@nestjs/schedule'; +import * as process from 'process'; +import { diffLine } from '../../utils/diffline'; +import { + createNewCoverageData, + getSpecificCoverageData, +} from '../../adapter/coverage-data.adapter'; +import { PullChangeCodeAndInsertDbService } from './pull-change-code-and-insert-db.service'; +import { AggregationCoverageService } from './aggregation-coverage.service'; + +interface Coverage { + id: string; + key: string; + compareTarget: string; + commitSha: string; + projectID: number; + instrumentCwd: string; + reporter: number; + reportID: string; + covType: string; + relationID: string; + createdAt: Date; +} + +/* +*********** +* *********** +* *********** +* *********** +极其重要的方法 +* 这是一个覆盖率聚合任务调度器,它5秒钟轮训一次数据库,查询状态为notstarted并且类型为coverageagg的任务, +* 每次只查询一个,查询到一个以后建立一个事务,确保在分布式的服务中只会执行一次任务。 +* 查询到任务后会立马更新数据库该任务状态,通过事务确保原子性,之后会将该覆盖率聚合任务的data参数拿出来,里面含有 +* reportID关键字段,我们会通过该字段到coverage表中查询需要聚合的行。 +* 为了确保任务顺利执行,我们将覆盖率数据逐一取出来聚合,聚合完成后再删除原本的数据 +* *********** +* *********** +* *********** +* *********** + */ +@Injectable() +export class CoverageAggTaskService { + constructor( + private readonly prisma: PrismaService, + private readonly pullChangeCodeAndInsertDbService: PullChangeCodeAndInsertDbService, + private readonly aggregationCoverageService: AggregationCoverageService, + ) {} + // 5-10秒轮询一次task表,查询状态为notstarted并且类型为coverageagg的任务,每次只查询一个,查询到一个以后建立一个事务,确保在分布式的服务中只会执行一次任务。 + @Interval(5000 * (Math.random() + 1)) + async coverageAgg() { + // 1.开事务,确保原子性 + let notstartedTask = null; + await this.prisma.$transaction(async (tx) => { + // 在事务中执行数据库操作 + notstartedTask = await tx.task.findFirst({ + where: { + status: 'notstarted', + }, + }); + if (notstartedTask) { + await tx.task.update({ + where: { id: notstartedTask.id }, + data: { status: 'running' }, + }); + } + }); + if (notstartedTask) { + const { projectID, commitSha, reportID } = notstartedTask; + const { reporter, compareTarget } = await this.prisma.coverage.findFirst({ + where: { + commitSha, + projectID, + covType: 'agg', + }, + orderBy: { + createdAt: 'desc', + }, + }); + await this.pullChangeCodeAndInsertDbService.invoke( + projectID, + commitSha, + compareTarget, + reporter, + ); + await this.aggregationCoverageService.invoke('agg', commitSha, reportID); + await this.aggregationCoverageService.invoke('all', commitSha, reportID); + await this.prisma.task.update({ + where: { id: notstartedTask.id }, + data: { status: 'success' }, + }); + } + } +} diff --git a/packages/canyon-backend/src/tasks/services/pull-change-code-and-insert-db.service.ts b/packages/canyon-backend/src/tasks/services/pull-change-code-and-insert-db.service.ts new file mode 100644 index 00000000..38b19518 --- /dev/null +++ b/packages/canyon-backend/src/tasks/services/pull-change-code-and-insert-db.service.ts @@ -0,0 +1,38 @@ +import { PrismaService } from '../../prisma/prisma.service'; +import { diffLine } from '../../utils/diffline'; + +export class PullChangeCodeAndInsertDbService { + constructor(private readonly prisma: PrismaService) {} + async invoke(projectID, commitSha, compareTarget, accessToken) { + const codechanges = await this.prisma.codechange.findMany({ + where: { + projectID, + commitSha, + compareTarget, + }, + }); + if (codechanges.length === 0) { + const diffLineData = await diffLine({ + repoID: projectID, + baseCommitSha: compareTarget, + compareCommitSha: commitSha, + token: accessToken, + gitlabUrl: process.env.GITLAB_URL, + }); + const data = diffLineData.map(({ path, additions, deletions }) => { + return { + projectID, + commitSha, + compareTarget, + path, + additions, + deletions, + }; + }); + + await this.prisma.codechange.createMany({ + data: data, + }); + } + } +} diff --git a/packages/canyon-backend/src/tasks/services/rerun.service.ts b/packages/canyon-backend/src/tasks/services/rerun.service.ts new file mode 100644 index 00000000..624beb4a --- /dev/null +++ b/packages/canyon-backend/src/tasks/services/rerun.service.ts @@ -0,0 +1,29 @@ +import { Interval } from '@nestjs/schedule'; +import { PrismaService } from '../../prisma/prisma.service'; +import { Injectable } from '@nestjs/common'; +@Injectable() +export class RerunService { + constructor(private readonly prisma: PrismaService) {} + // 定时任务,每隔一段时间查询一下还在 running 的任务,并且重置一下状态,时间为5-10分钟之间 + @Interval(5 * 60 * 1000 * (Math.random() + 1)) //添加随机数,防止分布式服务同时执行 + async rerun() { + const tasks = await this.prisma.task.findMany({ + where: { + status: 'running', + }, + }); + + for (let i = 0; i < tasks.length; i++) { + if (new Date().valueOf() - tasks[i].createdAt.valueOf() > 5 * 60 * 1000) { + await this.prisma.task.update({ + where: { + id: tasks[i].id, + }, + data: { + status: 'notstarted', + }, + }); + } + } + } +} diff --git a/packages/canyon-backend/src/tasks/tasks.module.ts b/packages/canyon-backend/src/tasks/tasks.module.ts new file mode 100755 index 00000000..e43b7444 --- /dev/null +++ b/packages/canyon-backend/src/tasks/tasks.module.ts @@ -0,0 +1,18 @@ +import { Module } from '@nestjs/common'; +// import { TasksService } from './tasks.service'; +import { PrismaModule } from '../prisma/prisma.module'; +import { CoverageAggTaskService } from './services/coverage-agg-task.service'; +import { CleaningUpOutdatedDataService } from './services/cleaning-up-outdated-data.service'; +import { RerunService } from './services/rerun.service'; +// import { DatabaseModule } from '../database/database.module'; +// import { coverageProviders } from '../coverage/providers/coverage.providers'; + +@Module({ + imports: [PrismaModule], + providers: [ + CoverageAggTaskService, + RerunService, + CleaningUpOutdatedDataService, + ], +}) +export class TasksModule {} diff --git a/packages/canyon-backend/src/utils/coverage.ts b/packages/canyon-backend/src/utils/coverage.ts new file mode 100644 index 00000000..4dde5cba --- /dev/null +++ b/packages/canyon-backend/src/utils/coverage.ts @@ -0,0 +1,53 @@ +function getLineCoverage(data: any) { + const statementMap = data.statementMap; + const statements = data.s; + const lineMap = Object.create(null); + Object.entries(statements).forEach(([st, count]: any) => { + if (!statementMap[st]) { + return; + } + const { line } = statementMap[st].start; + const prevVal = lineMap[line]; + if (prevVal === undefined || prevVal < count) { + lineMap[line] = count; + } + }); + return lineMap; +} + +export function calculateNewLineCoverageForSingleFile(coverage, newLine) { + const lineStats = getLineCoverage(coverage); + const rows = []; + Object.entries(lineStats).forEach(([lineNumber, count]) => { + if (newLine.new_lines.includes(Number(lineNumber))) { + rows.push([lineNumber, count]); + } + }); + return { + total: newLine.new_lines.length, + covered: newLine.new_lines.length - rows.filter((i) => !i[1]).length, + skipped: 0, + }; +} + +export function calculateAllNewLineCoverageForProject(coverages, new_lines) { + const rows = []; + + for (let i = 0; i < new_lines.length; i++) { + const newLine = new_lines[i]; + const coverage = coverages.find((c) => c.path === newLine.path); + if (coverage) { + rows.push(calculateNewLineCoverageForSingleFile(coverage, newLine)); + } + } + return rows.reduce( + (acc, cur) => { + return { + total: acc.total + cur.total, + covered: acc.covered + cur.covered, + skipped: acc.skipped + cur.skipped, + }; + }, + { total: 0, covered: 0, skipped: 0 }, + ); +} diff --git a/packages/canyon-backend/src/utils/diffline.ts b/packages/canyon-backend/src/utils/diffline.ts new file mode 100644 index 00000000..330fcaec --- /dev/null +++ b/packages/canyon-backend/src/utils/diffline.ts @@ -0,0 +1,179 @@ +import * as Diff from 'diff'; +import { Change } from 'diff'; +interface DiffLine { + repoID: string; + baseCommitSha?: string; + compareCommitSha: string; + includesFileExtensions?: string[]; + gitlabUrl?: string; + token?: string; +} + +function calculateNewRows( + a: string, + b: string, +): { additions: number[]; deletions: number[] } { + const diffResult: Change[] = Diff.diffLines(a, b); + function generateArray(startValue: number, length: number) { + return Array.from({ length }, (_, index) => startValue - index).reverse(); + } + function sumToIndex(arr: number[], index: number) { + return arr.slice(0, index + 1).reduce((sum, value) => sum + value, 0); + } + const additionsDiffResult = diffResult.filter((i) => !i.removed); + const additions: any = []; + additionsDiffResult.forEach((i, index) => { + if (i.added) { + additions.push( + generateArray( + sumToIndex( + additionsDiffResult.map((i) => i.count || 0), + index, + ), + i.count || 0, + ), + ); + } + }); + + const deletionsDiffResult = diffResult.filter((i) => i.removed); + const deletions: any = []; + deletionsDiffResult.forEach((i, index) => { + if (i.removed) { + deletions.push( + generateArray( + sumToIndex( + deletionsDiffResult.map((i) => i.count || 0), + index, + ), + i.count || 0, + ), + ); + } + }); + return { + additions: additions.flat(Infinity), + deletions: deletions.flat(Infinity), + }; +} + +function getDecode(str: string) { + return decodeURIComponent( + atob(str) + .split('') + .map(function (c) { + return '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2); + }) + .join(''), + ); +} +export async function diffLine({ + repoID, + baseCommitSha = undefined, + compareCommitSha, + includesFileExtensions = ['ts', 'tsx', 'jsx', 'vue', 'js'], + gitlabUrl = 'https://gitlab.com', + token = 'default_token', +}: DiffLine): Promise< + { path: string; additions: number[]; deletions: number[] }[] +> { + const gitlabApiUrlFile = `${gitlabUrl}/api/v4/projects/${repoID}/repository/files`; + const gitlabApiUrlCommit = `${gitlabUrl}/api/v4/projects/${repoID}/repository/commits/${compareCommitSha}`; + + const gitlabApiUrlCommitResponse = await fetch(gitlabApiUrlCommit, { + headers: { + Authorization: 'Bearer ' + token, // 在请求头中使用 GitLab API token + }, + }) + .then((res) => res.json()) + .then((data) => { + return { + parent_ids: data.parent_ids || [], + stats: data.stats, + }; + }); + + const result = []; + // 只关心 50000 行以内的更改 + if ( + gitlabApiUrlCommitResponse.parent_ids.length > 0 && + gitlabApiUrlCommitResponse.stats.additions < 50000 + ) { + // 声明realBaseCommitSha,如果baseCommitSha存在,则使用baseCommitSha,否则使用gitlabApiUrlCommitResponse.parent_ids[0] + const realBaseCommitSha = + baseCommitSha || gitlabApiUrlCommitResponse.parent_ids[0]; + const gitDiffs = await fetch( + `${gitlabUrl}/api/v4/projects/${repoID}/repository/compare?from=${realBaseCommitSha}&to=${compareCommitSha}`, + { + headers: { + Authorization: 'Bearer ' + token, // 在请求头中使用 GitLab API token + }, + }, + ) + .then((res) => res.json()) + .then((response) => { + return (response.diffs || []).map( + ({ + old_path, + new_path, + a_mode, + b_mode, + new_file, + renamed_file, + deleted_file, + }) => { + return { + old_path, + new_path, + a_mode, + b_mode, + new_file, + renamed_file, + deleted_file, + }; + }, + ); + }); + + // const includesFileExtensions = /\.tsx?$|\.jsx?$|\.vue$|\.js$/i; + + const isMatchingExtension = ( + includesFileExtensions: string[], + pathname: string, + ) => includesFileExtensions.some((ext) => pathname.endsWith('.' + ext)); + + const gitDiffsFiltered = gitDiffs.filter((gitDiff) => + isMatchingExtension(includesFileExtensions, gitDiff.new_path), + ); + + for (let i = 0; i < gitDiffsFiltered.length; i++) { + const contents = await Promise.all( + [realBaseCommitSha, compareCommitSha].map((c) => { + return fetch( + `${gitlabApiUrlFile}/${encodeURIComponent( + gitDiffsFiltered[i].new_path, + )}?ref=${c}`, + { + headers: { + Authorization: 'Bearer ' + token, // 在请求头中使用 GitLab API token + }, + method: 'GET', + }, + ) + .then((res) => res.json()) + .then((r) => { + return getDecode(r.content); + }) + .catch(() => { + return ''; + }); + }), + ); + result.push({ + path: gitDiffsFiltered[i].new_path, + ...calculateNewRows(contents[0], contents[1]), + }); + } + } + return result; +} diff --git a/packages/canyon-backend/src/utils/utils.ts b/packages/canyon-backend/src/utils/utils.ts new file mode 100644 index 00000000..426a6bed --- /dev/null +++ b/packages/canyon-backend/src/utils/utils.ts @@ -0,0 +1,23 @@ +export function camelizeJson(obj) { + if (typeof obj === 'object') { + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + obj[i] = camelizeJson(obj[i]); + } + } else { + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + const camelKey = key.replace(/_([a-z])/g, function (match, p1) { + return p1.toUpperCase(); + }); + obj[camelKey] = camelizeJson(obj[key]); + if (camelKey !== key) { + delete obj[key]; + } + } + } + } + } + + return obj; +} diff --git a/packages/canyon-backend/src/zstd/index.ts b/packages/canyon-backend/src/utils/zstd.ts similarity index 100% rename from packages/canyon-backend/src/zstd/index.ts rename to packages/canyon-backend/src/utils/zstd.ts diff --git a/packages/canyon-data/src/index.ts b/packages/canyon-data/src/index.ts index b365d28f..bcd6d399 100644 --- a/packages/canyon-data/src/index.ts +++ b/packages/canyon-data/src/index.ts @@ -82,6 +82,12 @@ export function calculateNewLineCoverageForSingleFile(coverage:FileCoverageData, }; } +export function mergeCoverage(first:any,second:any) { + const map = libCoverage.createCoverageMap(first); + map.merge(second); + return map.toJSON(); +} + export const genSummaryMapByCoverageMap = (coverageMapData: CoverageMapData,codeChanges?:CodeChange[]):CoverageSummaryMap => { const summaryMap: any = {}; const m = libCoverage.createCoverageMap(coverageMapData);