diff --git a/.env.example b/.env.example index 76abb0d5..1fd2c50e 100755 --- a/.env.example +++ b/.env.example @@ -3,7 +3,6 @@ SYSTEM_QUESTION_LINK= #-----------------------Backend Config------------------------------# # DB Config DATABASE_URL=postgress://canyon:canyon@localhost:5432/canyon -MONGODB_URL=mongodb://xxxxx # Gitlab Auth Config GITLAB_URL="https://gitlab.com" diff --git a/packages/canyon-backend/.prettierrc b/packages/canyon-backend/.prettierrc new file mode 100755 index 00000000..dcb72794 --- /dev/null +++ b/packages/canyon-backend/.prettierrc @@ -0,0 +1,4 @@ +{ + "singleQuote": true, + "trailingComma": "all" +} \ No newline at end of file diff --git a/packages/canyon-backend/package.json b/packages/canyon-backend/package.json index 0c56a1aa..7b6e62c8 100755 --- a/packages/canyon-backend/package.json +++ b/packages/canyon-backend/package.json @@ -1,6 +1,6 @@ { "name": "canyon-backend", - "version": "0.0.2", + "version": "0.0.1", "description": "", "private": true, "scripts": { @@ -11,14 +11,12 @@ }, "dependencies": { "@canyon/data": "workspace:^", - "@mongodb-js/zstd": "^1.2.0", "@nestjs/apollo": "^12.1.0", "@nestjs/common": "^10.3.9", "@nestjs/config": "^3.2.2", "@nestjs/core": "^10.3.9", "@nestjs/graphql": "^12.1.1", "@nestjs/jwt": "^10.2.0", - "@nestjs/mongoose": "^10.0.6", "@nestjs/passport": "^10.0.3", "@nestjs/platform-express": "^10.3.9", "@nestjs/schedule": "^4.0.2", @@ -28,7 +26,6 @@ "axios": "^1.7.2", "better-sqlite3": "^9.6.0", "body-parser": "^1.20.2", - "camaro": "^6.2.3", "canyon-data": "^0.1.1-alpha.5", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", @@ -38,14 +35,11 @@ "dotenv": "^16.4.5", "express": "^4.19.2", "graphql": "^16.9.0", - "graphql-query-complexity": "^0.12.0", - "graphql-subscriptions": "^2.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", "istanbul-lib-source-maps": "^4.0.1", "istanbul-reports": "^3.1.7", "lodash": "^4.17.21", - "mongoose": "^8.4.4", "passport-jwt": "^4.0.1", "passport-local": "^1.0.0", "prisma": "^5.16.1", diff --git a/packages/canyon-backend/src/app.module.ts b/packages/canyon-backend/src/app.module.ts index 3302754b..0bb6fc05 100755 --- a/packages/canyon-backend/src/app.module.ts +++ b/packages/canyon-backend/src/app.module.ts @@ -1,37 +1,29 @@ -import { Module } from '@nestjs/common'; -import { AppController } from './app.controller'; -import { AppService } from './app.service'; -// import { ConfigModule } from '@nestjs/config'; -import { PrismaModule } from './prisma/prisma.module'; -import { CoverageModule } from './coverage/coverage.module'; -import { ScheduleModule } from '@nestjs/schedule'; -import { AuthModule } from './auth/auth.module'; -import { UserModule } from './user/user.module'; -import { GraphQLModule } from '@nestjs/graphql'; -import { ApolloDriver, ApolloDriverConfig } from '@nestjs/apollo'; -import { ProjectModule } from './project/project.module'; -import { CodechangeModule } from './codechange/codechange.module'; -import { SourcecodeController } from './sourcecode/sourcecode.controller'; -import { SourcecodeService } from './sourcecode/sourcecode.service'; -import { SourcecodeModule } from './sourcecode/sourcecode.module'; -import { join } from 'path'; -import { ServeStaticModule } from '@nestjs/serve-static'; -import { UploadModule } from './upload/upload.module'; -import { MongooseModule } from '@nestjs/mongoose'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { CoveragediskEntity } from './coverage/entity/coveragedisk.entity'; -import {TaskModule} from "./task/task.module"; -// import * as path from 'node:path'; +import { Module } from "@nestjs/common"; +import { AppController } from "./app.controller"; +import { AppService } from "./app.service"; +import { PrismaModule } from "./prisma/prisma.module"; +import { CoverageModule } from "./coverage/coverage.module"; +import { ScheduleModule } from "@nestjs/schedule"; +import { AuthModule } from "./auth/auth.module"; +import { UserModule } from "./user/user.module"; +import { GraphQLModule } from "@nestjs/graphql"; +import { ApolloDriver, ApolloDriverConfig } from "@nestjs/apollo"; +import { ProjectModule } from "./project/project.module"; +import { CodechangeModule } from "./codechange/codechange.module"; +import { SourcecodeController } from "./sourcecode/sourcecode.controller"; +import { SourcecodeService } from "./sourcecode/sourcecode.service"; +import { SourcecodeModule } from "./sourcecode/sourcecode.module"; +import { join } from "path"; +import { ServeStaticModule } from "@nestjs/serve-static"; +import { UploadModule } from "./upload/upload.module"; +import { TypeOrmModule } from "@nestjs/typeorm"; +import { CoveragediskEntity } from "./coverage/entity/coveragedisk.entity"; @Module({ imports: [ - // ConfigModule.forRoot({ - // envFilePath: path.resolve(__dirname, '../../../.env'), - // }), - MongooseModule.forRoot(process.env.MONGODB_URL), TypeOrmModule.forRoot({ - type: 'better-sqlite3', - database: 'db/sql', + type: "better-sqlite3", + database: "db/sql", synchronize: true, entities: [CoveragediskEntity], }), @@ -44,13 +36,12 @@ import {TaskModule} from "./task/task.module"; CodechangeModule, SourcecodeModule, UploadModule, - TaskModule, ServeStaticModule.forRoot({ - rootPath: join(__dirname, '../../canyon-platform', 'dist'), - exclude: ['/graphql/(.*)'], + rootPath: join(__dirname, "../../canyon-platform", "dist"), + exclude: ["/graphql/(.*)"], }), GraphQLModule.forRoot({ - autoSchemaFile: 'schema.gql', + autoSchemaFile: "schema.gql", driver: ApolloDriver, }), ], diff --git a/packages/canyon-backend/src/coverage/coverage.module.ts b/packages/canyon-backend/src/coverage/coverage.module.ts index c5c3a146..02675428 100755 --- a/packages/canyon-backend/src/coverage/coverage.module.ts +++ b/packages/canyon-backend/src/coverage/coverage.module.ts @@ -1,54 +1,32 @@ -import { Module } from '@nestjs/common'; -import { RetrieveCoverageTreeSummaryService } from './services/retrieve-coverage-tree-summary.service'; -import { CoverageClientService } from './services/coverage-client.service'; -import { PrismaService } from '../prisma/prisma.service'; -import { CoverageController } from './coverage.controller'; -import { CoverageService } from './services/coverage.service'; -import { MongooseModule } from '@nestjs/mongoose'; -import { - CoverageData, - CoverageDataSchema, -} from './schemas/coverage-data.schema'; -import { CoverageDataAdapterService } from './services/common/coverage-data-adapter.service'; -import { ConsumerCoverageService } from './services/core/consumer-coverage.service'; -import { PullChangeCodeAndInsertDbService } from './services/common/pull-change-code-and-insert-db.service'; -import { CoverageLog, CoverageLogSchema } from './schemas/coverage-log.schema'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { CoveragediskEntity } from './entity/coveragedisk.entity'; -import { CoveragediskService } from './services/core/coveragedisk.service'; -import { TestExcludeService } from './services/common/test-exclude.service'; -import { PullFilePathAndInsertDbService } from './services/common/pull-file-path-and-insert-db.service'; -import {CoverageReportsService} from "./services/coverage-reports.service"; +import { Module } from "@nestjs/common"; +import { RetrieveCoverageTreeSummaryService } from "./services/retrieve-coverage-tree-summary.service"; +import { CoverageClientService } from "./services/coverage-client.service"; +import { PrismaService } from "../prisma/prisma.service"; +import { CoverageController } from "./coverage.controller"; +import { CoverageService } from "./services/coverage.service"; +import { ConsumerCoverageService } from "./services/core/consumer-coverage.service"; +import { PullChangeCodeAndInsertDbService } from "./services/common/pull-change-code-and-insert-db.service"; +import { TypeOrmModule } from "@nestjs/typeorm"; +import { CoveragediskEntity } from "./entity/coveragedisk.entity"; +import { CoveragediskService } from "./services/core/coveragedisk.service"; +import { TestExcludeService } from "./services/common/test-exclude.service"; +import { PullFilePathAndInsertDbService } from "./services/common/pull-file-path-and-insert-db.service"; +import { CoverageReportsService } from "./services/coverage-reports.service"; @Module({ - imports: [ - MongooseModule.forFeature([ - { - name: CoverageData.name, - schema: CoverageDataSchema, - collection: 'canyon_coverage_data', - }, - { - name: CoverageLog.name, - schema: CoverageLogSchema, - collection: 'canyon_coverage_log', - } - ]), - TypeOrmModule.forFeature([CoveragediskEntity]), - ], + imports: [TypeOrmModule.forFeature([CoveragediskEntity])], controllers: [CoverageController], providers: [ PrismaService, CoverageClientService, RetrieveCoverageTreeSummaryService, CoverageService, - CoverageDataAdapterService, ConsumerCoverageService, PullChangeCodeAndInsertDbService, PullFilePathAndInsertDbService, CoveragediskService, TestExcludeService, - CoverageReportsService + CoverageReportsService, ], }) export class CoverageModule {} diff --git a/packages/canyon-backend/src/coverage/schemas/coverage-data.schema.ts b/packages/canyon-backend/src/coverage/schemas/coverage-data.schema.ts deleted file mode 100644 index ff43badf..00000000 --- a/packages/canyon-backend/src/coverage/schemas/coverage-data.schema.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose'; -// 定义 'ant' 类型 -// 定义 antType 类型 -type CoverageType = { - [key: string]: object; -}; - -@Schema() -export class CoverageData { - @Prop() - coverageID: string; - - @Prop({ type: Object }) - coverage: CoverageType; - - @Prop() - v: string; - - @Prop() - createdAt: Date; -} - -export const CoverageDataSchema = SchemaFactory.createForClass(CoverageData); diff --git a/packages/canyon-backend/src/coverage/schemas/coverage-log.schema.ts b/packages/canyon-backend/src/coverage/schemas/coverage-log.schema.ts deleted file mode 100644 index 0e27e8a0..00000000 --- a/packages/canyon-backend/src/coverage/schemas/coverage-log.schema.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Prop, Schema, SchemaFactory } from '@nestjs/mongoose'; -// 定义 'ant' 类型 -// 定义 antType 类型 -// type CoverageType = { -// [key: string]: object; -// }; - -@Schema() -export class CoverageLog { - @Prop() - key: string; - - @Prop() - sha: string; - - @Prop() - projectID: string; - - @Prop() - reportID: string; - - @Prop() - instrumentCwd: string; - - @Prop() - covType: string; - - @Prop() - covOrigin: string; - - @Prop({ type: Object }) - tag: any; - - @Prop() - createdAt: Date; -} - -export const CoverageLogSchema = SchemaFactory.createForClass(CoverageLog); diff --git a/packages/canyon-backend/src/coverage/services/common/coverage-data-adapter.service.ts b/packages/canyon-backend/src/coverage/services/common/coverage-data-adapter.service.ts deleted file mode 100755 index 95341a25..00000000 --- a/packages/canyon-backend/src/coverage/services/common/coverage-data-adapter.service.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; -import { PrismaService } from 'src/prisma/prisma.service'; -import { InjectModel } from '@nestjs/mongoose'; -import { CoverageData } from '../../schemas/coverage-data.schema'; -import { Model } from 'mongoose'; -import * as mongoose from 'mongoose'; -// import { splitJSONIntoQuarters, validateObject } from '../../utils/coverage'; -import { compressedData, decompressedData } from '../../../utils/zstd'; -@Injectable() -export class CoverageDataAdapterService { - constructor( - private readonly prisma: PrismaService, - @InjectModel(CoverageData.name) - private coverageDataModel: Model, - ) {} - // crud - // 1/4 格式化，防止空key - async create(coverage, coverageID) { - return this.coverageDataModel - .create({ - coverage: await compressedData(JSON.stringify(coverage)), - coverageID, - v: '20240412', - createdAt: new Date(), - }) - .then((r) => String(r._id)); - } - async update(relationID, coverage) { - return this.coverageDataModel.updateOne( - { - _id: new mongoose.Types.ObjectId(relationID), - }, - { - coverage: await compressedData(JSON.stringify(coverage)), - createdAt: new Date(), - }, - ); - } - retrieve(relationID) { - return this.coverageDataModel - .findOne({ - _id: new mongoose.Types.ObjectId(relationID), - }) - .then((r) => { - return decompressedData(r.coverage) - }) - .then((r) => JSON.parse(r)); - } -} diff --git a/packages/canyon-backend/src/coverage/services/core/consumer-coverage.service.ts b/packages/canyon-backend/src/coverage/services/core/consumer-coverage.service.ts index 3a075c5c..cfe4c7c0 100644 --- a/packages/canyon-backend/src/coverage/services/core/consumer-coverage.service.ts +++ b/packages/canyon-backend/src/coverage/services/core/consumer-coverage.service.ts @@ -1,28 +1,21 @@ -import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; -import { PrismaService } from '../../../prisma/prisma.service'; -import { - genSummaryMapByCoverageMap, - getSummaryByPath, - // mergeCoverageMap, -} from '@canyon/data'; -import {mergeCoverageMap} from "../../../utils/coverage"; -import { percent, removeNullKeys } from '../../../utils/utils'; -import { CoverageDataAdapterService } from '../common/coverage-data-adapter.service'; -import { PullChangeCodeAndInsertDbService } from '../common/pull-change-code-and-insert-db.service'; -import { logger } from '../../../logger'; -import { CoveragediskService } from './coveragedisk.service'; -import { TestExcludeService } from '../common/test-exclude.service'; -import { formatReportObject } from '../../../utils/coverage'; -import { resolveProjectID } from '../../../utils'; -import { PullFilePathAndInsertDbService } from '../common/pull-file-path-and-insert-db.service'; -// import {merge_coverage_json_str} from 'canyon-data' +import { Injectable } from "@nestjs/common"; +import { PrismaService } from "../../../prisma/prisma.service"; +import { genSummaryMapByCoverageMap, getSummaryByPath } from "@canyon/data"; +import { mergeCoverageMap } from "../../../utils/coverage"; +import { percent, removeNullKeys } from "../../../utils/utils"; +import { PullChangeCodeAndInsertDbService } from "../common/pull-change-code-and-insert-db.service"; +import { logger } from "../../../logger"; +import { CoveragediskService } from "./coveragedisk.service"; +import { TestExcludeService } from "../common/test-exclude.service"; +import { resolveProjectID } from "../../../utils"; +import { PullFilePathAndInsertDbService } from "../common/pull-file-path-and-insert-db.service"; + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @Injectable() export class ConsumerCoverageService { constructor( private readonly prisma: PrismaService, - private readonly coverageDataAdapterService: CoverageDataAdapterService, private readonly pullChangeCodeAndInsertDbService: PullChangeCodeAndInsertDbService, private readonly coveragediskService: CoveragediskService, private readonly testExcludeService: TestExcludeService, @@ -37,7 +30,7 @@ export class ConsumerCoverageService { await this.coveragediskService.getQueueWithSameShaAndProjectID(); // 如果存在 if (queueDataToBeConsumed) { - const _lockName = 'consumer_coverage'; + const _lockName = "consumer_coverage"; const lockName = `${_lockName}_${queueDataToBeConsumed.projectID}_${queueDataToBeConsumed.sha}`; const lockAcquired = await this.acquireLock(lockName, 1000 * 60 * 2); @@ -54,15 +47,16 @@ export class ConsumerCoverageService { // queueDataToBeConsumed, // project?.instrumentCwd, // ); - const dataFormatAndCheckQueueDataToBeConsumed = queueDataToBeConsumed; + const dataFormatAndCheckQueueDataToBeConsumed = + queueDataToBeConsumed; try { await this.consume( dataFormatAndCheckQueueDataToBeConsumed, - 'agg', + "agg", ); await this.consume( dataFormatAndCheckQueueDataToBeConsumed, - 'all', + "all", ); // 这里有需要补偿变更行覆盖率，因为一些懒加载的原因，可能导致每个agg类型的变更行数量不一样。所以每次更新后，需要更新一下所有agg的变更行覆盖率summary await this.compensationChangeLineCoverageSummary( @@ -93,7 +87,7 @@ export class ConsumerCoverageService { sha: queueDataToBeConsumed.sha, projectID: queueDataToBeConsumed.projectID, covType: covType, - reportID: covType === 'agg' ? queueDataToBeConsumed.reportID : null, + reportID: covType === "agg" ? queueDataToBeConsumed.reportID : null, }), }); // 异步拉取文件路径，不影响主消费任务，耗时比较长 @@ -115,14 +109,13 @@ export class ConsumerCoverageService { }, }); if (coverage) { - // const cov = await this.coverageDataAdapterService.retrieve( - // coverage.relationID, - // ); - const cov = await this.prisma.covHit.findFirst({ - where:{ - id:`__${coverage.id}__` - } - }).then(res=>JSON.parse(res.mapJsonStr)) + const cov = await this.prisma.covHit + .findFirst({ + where: { + id: `__${coverage.id}__`, + }, + }) + .then((res) => JSON.parse(res.mapJsonStr)); // const newcoverage = JSON.parse(merge_coverage_json_str(JSON.stringify(queueDataToBeConsumed.coverage), JSON.stringify(cov))); const newcoverage = mergeCoverageMap(queueDataToBeConsumed.coverage, cov); @@ -133,7 +126,7 @@ export class ConsumerCoverageService { }, data: { summary: getSummaryByPath( - '~', + "~", genSummaryMapByCoverageMap( await this.testExcludeService.invoke( queueDataToBeConsumed.projectID, @@ -146,27 +139,22 @@ export class ConsumerCoverageService { compareTarget: queueDataToBeConsumed.compareTarget, }, }); // 更新时间 - // const r = await this.coverageDataAdapterService.update( - // coverage.relationID, - // newcoverage, - // ); - // return r; return this.prisma.covHit.update({ - where:{ - id:`__${coverage.id}__` + where: { + id: `__${coverage.id}__`, }, - data:{ - mapJsonStr:JSON.stringify(newcoverage) - } - }) + data: { + mapJsonStr: JSON.stringify(newcoverage), + }, + }); } else { // 创建新的agg const newAgg = await this.prisma.coverage.create({ data: { covType: covType, - relationID: '', + relationID: "", summary: getSummaryByPath( - '~', + "~", genSummaryMapByCoverageMap( await this.testExcludeService.invoke( queueDataToBeConsumed.projectID, @@ -188,16 +176,12 @@ export class ConsumerCoverageService { tag: queueDataToBeConsumed.tag, }, }); - // const newcd = await this.coverageDataAdapterService.create( - // queueDataToBeConsumed.coverage, - // newAgg.id, - // ); return await this.prisma.covHit.create({ - data:{ - id:`__${newAgg.id}__`, - mapJsonStr:JSON.stringify(queueDataToBeConsumed.coverage) - } - }) + data: { + id: `__${newAgg.id}__`, + mapJsonStr: JSON.stringify(queueDataToBeConsumed.coverage), + }, + }); // return this.prisma.coverage.update({ // where: { // id: newAgg.id, @@ -215,13 +199,13 @@ export class ConsumerCoverageService { resolveProjectID(coverage.projectID), coverage.sha, coverage.compareTarget, - 'accessToken', + "accessToken", this.prisma, ); } catch (e) { logger({ - type: 'error', - title: 'pullChangeCode', + type: "error", + title: "pullChangeCode", message: String(e), }); } @@ -270,7 +254,7 @@ export class ConsumerCoverageService { return true; // 锁获取成功 } } catch (error) { - console.error('Error acquiring lock:', error); + console.error("Error acquiring lock:", error); return false; // 锁获取失败 } } @@ -288,7 +272,7 @@ export class ConsumerCoverageService { where: { projectID: queueDataToBeConsumed.projectID, sha: queueDataToBeConsumed.sha, - covType: 'all', + covType: "all", }, }); // 变更行数量0，不需要补偿 @@ -298,7 +282,7 @@ export class ConsumerCoverageService { where: { projectID: queueDataToBeConsumed.projectID, sha: queueDataToBeConsumed.sha, - covType: 'agg', + covType: "agg", }, }); // 补偿函数 diff --git a/packages/canyon-backend/src/coverage/services/coverage-client.service.ts b/packages/canyon-backend/src/coverage/services/coverage-client.service.ts index cbf21577..61cd280c 100755 --- a/packages/canyon-backend/src/coverage/services/coverage-client.service.ts +++ b/packages/canyon-backend/src/coverage/services/coverage-client.service.ts @@ -1,12 +1,9 @@ -import { Injectable, UnauthorizedException } from '@nestjs/common'; -import { PrismaService } from '../../prisma/prisma.service'; -import { CoverageClientDto } from '../dto/coverage-client.dto'; -import { Coverage } from '@prisma/client'; -import { InjectModel } from '@nestjs/mongoose'; -import { Model } from 'mongoose'; -import { CoverageLog } from '../schemas/coverage-log.schema'; -import { CoveragediskService } from './core/coveragedisk.service'; -import {formatReportObject} from "../../utils/coverage"; +import { Injectable, UnauthorizedException } from "@nestjs/common"; +import { PrismaService } from "../../prisma/prisma.service"; +import { CoverageClientDto } from "../dto/coverage-client.dto"; +import { Coverage } from "@prisma/client"; +import { CoveragediskService } from "./core/coveragedisk.service"; +import { formatReportObject } from "../../utils/coverage"; /** * 上传覆盖率，十分重要的服务 */ @@ -25,8 +22,6 @@ function checkUserID(id) { export class CoverageClientService { constructor( private readonly prisma: PrismaService, - @InjectModel(CoverageLog.name) - private coverageLogModel: Model, private coveragediskService: CoveragediskService, ) {} @@ -50,10 +45,10 @@ export class CoverageClientService { }); if (isRepeated) { return { - msg: 'ok', - coverageId: 'isRepeated', - dataFormatAndCheckTime: '', - coverageInsertDbTime: '', + msg: "ok", + coverageId: "isRepeated", + dataFormatAndCheckTime: "", + coverageInsertDbTime: "", }; } } @@ -61,12 +56,12 @@ export class CoverageClientService { coverageClientDto.sha = coverageClientDto.commitSha; // 注意这里还是小驼峰 - if (!coverageClientDto.reportID || coverageClientDto.reportID === '-') { + if (!coverageClientDto.reportID || coverageClientDto.reportID === "-") { coverageClientDto.reportID = coverageClientDto.sha; } if ( !coverageClientDto.compareTarget || - coverageClientDto.compareTarget === '-' + coverageClientDto.compareTarget === "-" ) { coverageClientDto.compareTarget = coverageClientDto.sha; } @@ -79,21 +74,21 @@ export class CoverageClientService { // ****************************************************** const coverageReport = coverageClientDto; const cov: Coverage & { coverage: any } = { - id: '', - key: coverageReport.key || '', + id: "", + key: coverageReport.key || "", sha: coverageReport.sha, - branch: coverageReport.branch || '-', + branch: coverageReport.branch || "-", compareTarget: coverageReport.compareTarget, - provider: 'gitlab', + provider: "gitlab", projectID: coverageReport.projectID, instrumentCwd: coverageReport.instrumentCwd, reporter: currentUser, reportID: coverageReport.reportID, - covType: 'normal', + covType: "normal", // rule: 'auto', //没有就是手工 summary: {}, tag: coverageReport.tags || {}, - relationID: '', + relationID: "", createdAt: new Date(), updatedAt: new Date(), //后加的 @@ -103,66 +98,55 @@ export class CoverageClientService { where: { id: cov.projectID, }, - select:{ - instrumentCwd:true - } + select: { + instrumentCwd: true, + }, }); const dataFormatAndCheckQueueDataToBeConsumed = - await this.dataFormatAndCheck( - cov, - project?.instrumentCwd, - ); + await this.dataFormatAndCheck(cov, project?.instrumentCwd); // 这里要有一个地方分开bfs - const objMap = {} - const objHit = {} + const objMap = {}; + const objHit = {}; - Object.entries(dataFormatAndCheckQueueDataToBeConsumed.coverage).forEach(([path,value]:any)=>{ - objMap[path] = { - fnMap: value.fnMap, - branchMap: value.branchMap, - statementMap: value.statementMap - } - objHit[path] = { - f:value.f, - b:value.b, - s:value.s - } - }) + Object.entries(dataFormatAndCheckQueueDataToBeConsumed.coverage).forEach( + ([path, value]: any) => { + objMap[path] = { + fnMap: value.fnMap, + branchMap: value.branchMap, + statementMap: value.statementMap, + }; + objHit[path] = { + f: value.f, + b: value.b, + s: value.s, + }; + }, + ); await this.coveragediskService.pushQueue({ ...dataFormatAndCheckQueueDataToBeConsumed, - coverage: objHit + coverage: objHit, }); await this.prisma.covMapTest.createMany({ - data: Object.entries(objMap).map(([path,value])=>{ - const {projectID,sha} = dataFormatAndCheckQueueDataToBeConsumed + data: Object.entries(objMap).map(([path, value]) => { + const { projectID, sha } = dataFormatAndCheckQueueDataToBeConsumed; return { id: `__${projectID}__${sha}__${path}__`, mapJsonStr: JSON.stringify(value), //???没删除bfs projectID: projectID, sha: sha, - path: path - } + path: path, + }; }), skipDuplicates: true, - }) - await this.coverageLogModel.create({ - key: cov.key, - sha: cov.sha, - projectID: cov.projectID, - reportID: cov.reportID, - covType: 'normal', - tag: JSON.stringify(cov.tag), - instrumentCwd: cov.instrumentCwd, - createdAt: new Date(), }); return { - msg: 'ok', - coverageId: '', - dataFormatAndCheckTime: '', - coverageInsertDbTime: '', + msg: "ok", + coverageId: "", + dataFormatAndCheckTime: "", + coverageInsertDbTime: "", }; } @@ -171,9 +155,9 @@ export class CoverageClientService { where: { id: { contains: coverageClientDto.projectID, - mode: 'insensitive', // Ignore case sensitivity + mode: "insensitive", // Ignore case sensitivity not: { - contains: '-ut', + contains: "-ut", }, }, }, @@ -230,7 +214,7 @@ export class CoverageClientService { // 针对windows电脑，把反斜杠替换成正斜杠 // 做数据过滤，去除 \u0000 字符 for (const coverageKey in coverage) { - if (!coverageKey.includes('\u0000')) { + if (!coverageKey.includes("\u0000")) { obj[coverageKey] = coverage[coverageKey]; } } diff --git a/packages/canyon-backend/src/coverage/services/coverage-reports.service.ts b/packages/canyon-backend/src/coverage/services/coverage-reports.service.ts index 9048fdf1..aa35f4e6 100644 --- a/packages/canyon-backend/src/coverage/services/coverage-reports.service.ts +++ b/packages/canyon-backend/src/coverage/services/coverage-reports.service.ts @@ -1,83 +1,81 @@ -import { Injectable } from '@nestjs/common'; -import { PrismaService } from 'src/prisma/prisma.service'; -import { CoverageDataAdapterService } from './common/coverage-data-adapter.service'; -import { TestExcludeService } from './common/test-exclude.service'; +import { Injectable } from "@nestjs/common"; +import { PrismaService } from "src/prisma/prisma.service"; import * as dayjs from "dayjs"; -const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @Injectable() export class CoverageReportsService { - constructor( - private readonly prisma: PrismaService, - private readonly coverageDataAdapterService: CoverageDataAdapterService, - private readonly testExcludeService: TestExcludeService, - ) {} + constructor(private readonly prisma: PrismaService) {} - async invoke( - {bu,start,end} - ) { + async invoke({ bu, start, end }) { const projects = await this.prisma.project.findMany({ - where:{ - bu:bu, - pathWithNamespace:{ - not:{ - contains:'canyon' - } - } + where: { + bu: bu, + pathWithNamespace: { + not: { + contains: "canyon", + }, + }, }, - select:{ - id:true, - name:true, - pathWithNamespace:true, - description:true - } - }) + select: { + id: true, + name: true, + pathWithNamespace: true, + description: true, + }, + }); // await sleep(1000) - const covs =await this.prisma.coverage.findMany({ - where:{ - covType:'all', - projectID:{ - in:projects.map(item => item.id) + const covs = await this.prisma.coverage.findMany({ + where: { + covType: "all", + projectID: { + in: projects.map((item) => item.id), }, // updatedAt:{} // 根据updatedAt的一个范围 - updatedAt:{ + updatedAt: { gte: dayjs(start).toDate(), - lte: dayjs(end).toDate() - } - } - }) - const obj = {} + lte: dayjs(end).toDate(), + }, + }, + }); + const obj = {}; for (let i = 0; i < covs.length; i++) { - const projectID = `${covs[i].projectID.split('-')[1]}-${covs[i].projectID.includes('-ut')?'ut':'auto'}` + const projectID = `${covs[i].projectID.split("-")[1]}-${covs[i].projectID.includes("-ut") ? "ut" : "auto"}`; if (obj[projectID] === undefined) { obj[projectID] = { - maxCoverage:covs[i].summary['statements']['pct'], - projectID:covs[i].projectID.split('-')[1] - } + maxCoverage: covs[i].summary["statements"]["pct"], + projectID: covs[i].projectID.split("-")[1], + }; } else { - obj[projectID].maxCoverage = Math.max(obj[projectID].maxCoverage, covs[i].summary['statements']['pct']) + obj[projectID].maxCoverage = Math.max( + obj[projectID].maxCoverage, + covs[i].summary["statements"]["pct"], + ); } } - const rows = [] - Object.keys(obj).forEach(key => { - const index = rows.findIndex(item => item.projectID === obj[key].projectID) - if (index>-1){ - rows[index][key.includes('-ut')?'ut':'auto']=obj[key].maxCoverage + const rows = []; + Object.keys(obj).forEach((key) => { + const index = rows.findIndex( + (item) => item.projectID === obj[key].projectID, + ); + if (index > -1) { + rows[index][key.includes("-ut") ? "ut" : "auto"] = obj[key].maxCoverage; } else { rows.push({ - projectID:obj[key].projectID, - ut:key.includes('-ut')?obj[key].maxCoverage:0, - auto:key.includes('-auto')?obj[key].maxCoverage:0, - }) + projectID: obj[key].projectID, + ut: key.includes("-ut") ? obj[key].maxCoverage : 0, + auto: key.includes("-auto") ? obj[key].maxCoverage : 0, + }); } - }) + }); for (let i = 0; i < rows.length; i++) { - const project = projects.find(item => item.id.includes(rows[i].projectID)) + const project = projects.find((item) => + item.id.includes(rows[i].projectID), + ); // rows[i].name = project.name - rows[i].pathWithNamespace = project.pathWithNamespace - rows[i].description = project.description + rows[i].pathWithNamespace = project.pathWithNamespace; + rows[i].description = project.description; } - return rows.sort((a,b)=>a.auto-b.auto) + return rows.sort((a, b) => a.auto - b.auto); } } diff --git a/packages/canyon-backend/src/coverage/services/coverage.service.ts b/packages/canyon-backend/src/coverage/services/coverage.service.ts index 24b0acb7..172adc11 100755 --- a/packages/canyon-backend/src/coverage/services/coverage.service.ts +++ b/packages/canyon-backend/src/coverage/services/coverage.service.ts @@ -1,17 +1,14 @@ -import { Injectable } from '@nestjs/common'; -import { PrismaService } from 'src/prisma/prisma.service'; -import { CoverageSummary } from '../models/coverage-summary'; -import { genSummaryMapByCoverageMap } from '@canyon/data'; -import { CoverageDataAdapterService } from './common/coverage-data-adapter.service'; -import { TestExcludeService } from './common/test-exclude.service'; -import {decompressedData} from "../../utils/zstd"; -import {removeNullKeys} from "../../utils/utils"; +import { Injectable } from "@nestjs/common"; +import { PrismaService } from "src/prisma/prisma.service"; +import { CoverageSummary } from "../models/coverage-summary"; +import { genSummaryMapByCoverageMap } from "@canyon/data"; +import { TestExcludeService } from "./common/test-exclude.service"; +import { removeNullKeys } from "../../utils/utils"; @Injectable() export class CoverageService { constructor( private readonly prisma: PrismaService, - private readonly coverageDataAdapterService: CoverageDataAdapterService, private readonly testExcludeService: TestExcludeService, ) {} @@ -24,15 +21,15 @@ export class CoverageService { where: { sha: sha, projectID, - covType: 'agg', - NOT:{ - projectID:{ - contains:'-ut' - } - } + covType: "agg", + NOT: { + projectID: { + contains: "-ut", + }, + }, }, orderBy: { - updatedAt: 'desc', + updatedAt: "desc", }, }); if (coverages.length === 0) { @@ -65,55 +62,67 @@ export class CoverageService { projectID, commitSha, reportID, - filepath + filepath, ); return JSON.stringify(coverageData[filepath]); } // 私有方法 - private async getCoverageDataFromAdapter(projectID, sha, reportID,filepath=null) { + private async getCoverageDataFromAdapter( + projectID, + sha, + reportID, + filepath = null, + ) { const { id } = await this.prisma.coverage.findFirst({ where: { projectID, sha: sha, - covType: reportID === '' ? 'all' : 'agg', - reportID: reportID === '' ? undefined : reportID, + covType: reportID === "" ? "all" : "agg", + reportID: reportID === "" ? undefined : reportID, }, }); - const maps = [this.prisma.covHit.findFirst({ - where:{ - id:`__${id}__` - } - }).then(res=>{ - return JSON.parse(res.mapJsonStr) - }),this.prisma.covMapTest.findMany({ - where:removeNullKeys({ - projectID, - sha, - path:filepath - }) - }).then(res=>{ - return res.reduce((acc,cur)=>{ - acc[cur.path] = JSON.parse(cur.mapJsonStr) - return acc - },{}) - })] + const maps = [ + this.prisma.covHit + .findFirst({ + where: { + id: `__${id}__`, + }, + }) + .then((res) => { + return JSON.parse(res.mapJsonStr); + }), + this.prisma.covMapTest + .findMany({ + where: removeNullKeys({ + projectID, + sha, + path: filepath, + }), + }) + .then((res) => { + return res.reduce((acc, cur) => { + acc[cur.path] = JSON.parse(cur.mapJsonStr); + return acc; + }, {}); + }), + ]; // this.prisma.covMap - const time = Date.now() - const [hit,map] = await Promise.all(maps) - const obj = {} + const time = Date.now(); + const [hit, map] = await Promise.all(maps); + const obj = {}; - Object.entries(hit).forEach(([key,value]:any)=>{ - if (map[key]){ + Object.entries(hit).forEach(([key, value]: any) => { + if (map[key]) { obj[key] = { - path:key, + path: key, ...value, - ...map[key] - } + ...map[key], + }; } - }) + }); - return obj + return obj; } } diff --git a/packages/canyon-backend/src/task/syns.service.ts b/packages/canyon-backend/src/task/syns.service.ts deleted file mode 100644 index a7053fe0..00000000 --- a/packages/canyon-backend/src/task/syns.service.ts +++ /dev/null @@ -1,159 +0,0 @@ -import {Injectable} from "@nestjs/common"; -import {PrismaService} from "../prisma/prisma.service"; -import {Timeout} from "@nestjs/schedule"; -import {CoverageDataAdapterService} from "../coverage/services/common/coverage-data-adapter.service"; -import * as fs from "node:fs"; -import {compressedData} from "../utils/zstd"; - -@Injectable() -export class SynsService { - constructor( - private readonly prisma: PrismaService, - private readonly coverageDataAdapterService: CoverageDataAdapterService, - ) { - } - - // @Timeout(1) - async mainTask() { - // 1.查出07-12 12:00以前的coverage数据 - const ids = await this.prisma.coverage.findMany({ - where:{ - // projectID:{ - // not:{ - // // contains:'-ut' - // } - // }, - // updatedAt: { - // // lt: new Date('2024-07-12 12:00:00'), - // // gte: new Date('2024-07-12 00:00:00') - // } - }, - select:{ - id:true, - relationID:true, - covType:true - } - }); - console.log(ids.length) - // 2.遍历coverage - for (let i = 0; i < ids.length; i++) { - console.log('ids',ids[i].id,i,ids.length) - const {relationID,id:coverageID} = ids[i] - - if (relationID){ - const coverageData = await this.coverageDataAdapterService.retrieve(relationID).catch(err=>{ - return null - }) - if (coverageData){ - - // ****** - // ****** - // ****** - // ****** - // ****** - // ****** - - const coverageClientDto = await this.prisma.coverage.findFirst({ - where:{ - id: coverageID - } - }) - - const { sha, projectID, reportID, branch,compareTarget,reporter } = - coverageClientDto; - - const coverage = coverageData; - - if (JSON.stringify(coverage)!=='{}'){ - - - - - - - - - const fileMapTasks = Object.entries(coverage).map( - async (coverageEntries) => { - const [path, fileCoverage]: any = coverageEntries; - - }, - ); - const time2 = new Date().getTime(); - await Promise.all(fileMapTasks); - - - const rows = Object.entries(coverage).map((coverageEntries)=>{ - const [path, fileCoverage]: any = coverageEntries; - return { - id: `__${projectID}__${sha}__${path}__`, - mapJsonStr: JSON.stringify({ - fnMap: fileCoverage.fnMap||{}, - statementMap: fileCoverage.statementMap||{}, - branchMap: fileCoverage.branchMap||{}, - }), - projectID:projectID, - sha:sha, - path, - } - }) - - - await this.prisma.covMapTest - .createMany({ - data: rows, - skipDuplicates: true, - }) - .then((res) => { - return res; - }) - .catch(() => { - return true; - }); - - console.log('fileMapTasks', new Date().getTime() - time2); - - - - - - const time3 = new Date().getTime(); - - - await this.prisma.covHit - .create({ - data: { - id: `__${ids[i].id}__`, - mapJsonStr: JSON.stringify(Object.entries(coverage).reduce((previousValue, currentValue:any)=>{ - previousValue[currentValue[0]] = { - f: currentValue[1].f||{}, - b: currentValue[1].b||{}, - s: currentValue[1].s||{}, - } - return previousValue - },{})), - }, - }) - .then((res) => { - return res; - }) - .catch((e) => { - return true; - }); - - - console.log('hitTasks', new Date().getTime() - time3); - - - - } else { - console.log('coverage is ',JSON.stringify(coverage)) - } - - } - } else { - console.log('relationID is null') - } - } - } -} diff --git a/packages/canyon-backend/src/task/task.module.ts b/packages/canyon-backend/src/task/task.module.ts deleted file mode 100644 index 536c9fd9..00000000 --- a/packages/canyon-backend/src/task/task.module.ts +++ /dev/null @@ -1,32 +0,0 @@ -// import {PrismaModule} from "../prisma/prisma.module"; -import {Module} from "@nestjs/common"; -import {SynsService} from "./syns.service"; -import {PrismaService} from "../prisma/prisma.service"; -import {CoverageDataAdapterService} from "../coverage/services/common/coverage-data-adapter.service"; -import {MongooseModule} from "@nestjs/mongoose"; -import {CoverageData, CoverageDataSchema} from "../coverage/schemas/coverage-data.schema"; -import {CoverageLog, CoverageLogSchema} from "../coverage/schemas/coverage-log.schema"; - -@Module({ - imports: [ - MongooseModule.forFeature([ - { - name: CoverageData.name, - schema: CoverageDataSchema, - collection: 'canyon_coverage_data', - }, - { - name: CoverageLog.name, - schema: CoverageLogSchema, - collection: 'canyon_coverage_log', - } - ]), - ], - controllers: [], - providers: [ - PrismaService, - SynsService, - CoverageDataAdapterService - ], -}) -export class TaskModule {} diff --git a/packages/canyon-backend/src/utils/jacoco.ts b/packages/canyon-backend/src/utils/jacoco.ts deleted file mode 100644 index b918d21d..00000000 --- a/packages/canyon-backend/src/utils/jacoco.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { transform } from 'camaro'; -const counterTemplate = [ - 'counter', - { - type: '@type', - missed: 'number(@missed)', - covered: 'number(@covered)', - }, -]; - -const template = [ - 'report', - { - report: { - name: '@name', - package: [ - 'package', - { - name: '@name', - class: [ - 'class', - { - name: '@name', - sourcefilename: '@sourcefilename', - counter: counterTemplate, - method: [ - 'method', - { - name: '@name', - desc: '@desc', - line: '@line', - counter: counterTemplate, - }, - ], - }, - ], - sourcefile: [ - 'sourcefile', - { - name: '@name', - line: [ - 'line', - { - nr: 'number(@nr)', - mi: 'number(@mi)', - ci: 'number(@ci)', - mb: 'number(@mb)', - cb: 'number(@cb)', - }, - ], - counter: counterTemplate, - }, - ], - counter: counterTemplate, - }, - ], - counter: counterTemplate, - }, - }, -]; -export const jacocoXml2Json = (xml) => { - return transform(xml, template).then((r) => r[0]); -}; diff --git a/packages/canyon-backend/src/utils/zstd.ts b/packages/canyon-backend/src/utils/zstd.ts deleted file mode 100755 index a0fba354..00000000 --- a/packages/canyon-backend/src/utils/zstd.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { compress, decompress } from '@mongodb-js/zstd'; - -export async function compressedData(str) { - const buffer = Buffer.from(str); - const compressed = await compress(buffer); - return compressed.toString('base64'); -} - -export async function decompressedData(str) { - const decompressed = await decompress(Buffer.from(str, 'base64')); - return Buffer.from(decompressed).toString(); -} diff --git a/packages/canyon-platform/.prettierrc b/packages/canyon-platform/.prettierrc new file mode 100755 index 00000000..e1018d0d --- /dev/null +++ b/packages/canyon-platform/.prettierrc @@ -0,0 +1,12 @@ +{ + "jsxSingleQuote": true, + "semi": true, + "tabWidth": 2, + "printWidth": 100, + "bracketSameLine": false, + "useTabs": false, + "arrowParens": "always", + "endOfLine": "auto", + "singleQuote": true, + "trailingComma": "all" +} diff --git a/packages/canyon-platform/package.json b/packages/canyon-platform/package.json index 87da62d7..bb68e7b8 100644 --- a/packages/canyon-platform/package.json +++ b/packages/canyon-platform/package.json @@ -37,30 +37,21 @@ "react": "^18.3.1", "react-copy-to-clipboard": "^5.1.0", "react-dom": "^18.3.1", - "react-fast-marquee": "^1.6.4", "react-highlight-words": "^0.20.0", - "react-i18next": "^14.1.2", - "swr": "^2.2.5" + "react-i18next": "^14.1.2" }, "devDependencies": { "@graphql-codegen/cli": "^5.0.2", "@graphql-codegen/client-preset": "^4.3.1", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", + "@typescript-eslint/eslint-plugin": "^7.16.0", + "@typescript-eslint/parser": "^7.16.0", "@vitejs/plugin-react": "^4.3.1", + "@vitejs/plugin-react-swc": "^3.7.0", "autoprefixer": "^10.4.19", "babel-plugin-canyon": "^1.8.42", "babel-plugin-istanbul": "^6.1.1", - "postcss": "^8.4.39", - "react-router-dom": "^6.24.0", - "tailwindcss": "^3.4.4", - "typescript": "^5.5.2", - "unplugin-auto-import": "^0.17.6", - "unplugin-auto-import-antd": "^0.0.1", - "vite": "^5.3.2", - "vite-plugin-pages": "^0.32.3", - "@typescript-eslint/eslint-plugin": "^7.16.0", - "@typescript-eslint/parser": "^7.16.0", "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-import": "^2.29.1", @@ -70,6 +61,15 @@ "eslint-plugin-react-hooks": "^4.6.2", "eslint-plugin-react-refresh": "^0.4.7", "eslint-plugin-simple-import-sort": "^12.1.1", - "prettier": "^3.3.2" + "postcss": "^8.4.39", + "prettier": "^3.3.2", + "react-router-dom": "^6.24.0", + "swc-plugin-coverage-instrument": "^0.0.24", + "tailwindcss": "^3.4.4", + "typescript": "^5.5.2", + "unplugin-auto-import": "^0.17.6", + "unplugin-auto-import-antd": "^0.0.1", + "vite": "^5.3.2", + "vite-plugin-pages": "^0.32.3" } } diff --git a/packages/canyon-platform/vite.config.ts b/packages/canyon-platform/vite.config.ts index 89d4de5e..cfefee69 100644 --- a/packages/canyon-platform/vite.config.ts +++ b/packages/canyon-platform/vite.config.ts @@ -1,4 +1,4 @@ -import react from '@vitejs/plugin-react'; +import react from '@vitejs/plugin-react-swc'; import * as path from 'path'; import AutoImport from 'unplugin-auto-import/vite'; import AntdResolver from 'unplugin-auto-import-antd'; @@ -8,20 +8,7 @@ const resolve = (p: string) => path.resolve(__dirname, p); export default defineConfig({ plugins: [ react({ - babel: { - plugins: - process.env.NODE_ENV === 'development' - ? [] - : [ - 'istanbul', - [ - 'canyon', - { - instrumentCwd: resolve('../..'), - }, - ], - ], - }, + plugins: [['swc-plugin-coverage-instrument', {}]], }), AutoImport({ imports: ['react', 'react-i18next', 'react-router-dom'],