chore: daily development

packages/canyon-collect/package.json

@@ -29,8 +29,10 @@
     "@nestjs/typeorm": "^10.0.2",
     "typeorm": "^0.3.20",
     "sqlite3": "^5.1.7",
+    "test-exclude": "^7.0.1",
     "canyon-data": "^2.0.0-beta.6",
-    "canyon-map": "^2.0.0-beta.5"
+    "canyon-map": "^2.0.0-beta.5",
+    "diff": "^7.0.0"
   },
   "devDependencies": {
     "@nestjs/cli": "^10.4.9",
@@ -53,6 +55,7 @@
     "ts-loader": "^9.5.1",
     "ts-node": "^10.9.2",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.7.2"
+    "typescript": "^5.7.2",
+    "@types/test-exclude": "^6.0.2"
   }
 }

packages/canyon-collect/src/apps/collect/collect.module.ts

@@ -9,6 +9,8 @@ import { TypeOrmModule } from "@nestjs/typeorm";
 import { PrismaService } from "../../prisma/prisma.service";
 import { CoveragediskService } from "./services/core/coveragedisk.service";
 import { ConsumerCoverageService } from "./services/core/consumer-coverage.service";
+import { PullChangeCodeAndInsertDbService } from "./services/common/pull-change-code-and-insert-db.service";
+import { TestExcludeService } from "./services/common/test-exclude.service";
 
 @Module({
     imports: [TypeOrmModule.forFeature([CoveragediskEntity])],
@@ -19,6 +21,8 @@ import { ConsumerCoverageService } from "./services/core/consumer-coverage.servi
         CoverageMapClientService,
         ConsumerCoverageService,
         CoveragediskService,
+        PullChangeCodeAndInsertDbService,
+        TestExcludeService,
     ],
 })
 export class CollectModule {

packages/canyon-collect/src/apps/collect/services/common/pull-change-code-and-insert-db.service.ts

@@ -0,0 +1,43 @@
+// import { diffLine } from "../../../utils/diffline";
+
+import { diffLine } from "../../../../utils/diffline";
+
+export class PullChangeCodeAndInsertDbService {
+    async invoke(projectID, commitSha, compareTarget, accessToken, prisma) {
+        const codechanges = await prisma.codechange.findMany({
+            where: {
+                projectID,
+                sha: commitSha,
+                compareTarget,
+            },
+        });
+        const gitProvider = await prisma.gitProvider.findFirst({
+            where: {
+                disabled: false,
+            },
+        });
+        if (codechanges.length === 0) {
+            const diffLineData = await diffLine({
+                repoID: projectID,
+                baseCommitSha: compareTarget,
+                compareCommitSha: commitSha,
+                token: gitProvider?.privateToken,
+                gitlabUrl: gitProvider?.url,
+            });
+            const data = diffLineData.map(({ path, additions, deletions }) => {
+                return {
+                    projectID,
+                    sha: commitSha,
+                    compareTarget,
+                    path,
+                    additions,
+                    deletions,
+                };
+            });
+
+            await prisma.codechange.createMany({
+                data: data,
+            });
+        }
+    }
+}

packages/canyon-collect/src/apps/collect/services/common/test-exclude.service.ts

@@ -0,0 +1,53 @@
+import { Injectable } from "@nestjs/common";
+// import { PrismaService } from "../../../prisma/prisma.service";
+import * as TestExclude from "test-exclude";
+import { PrismaService } from "../../../../prisma/prisma.service";
+@Injectable()
+export class TestExcludeService {
+    constructor(private readonly prisma: PrismaService) {}
+
+    async invoke(projectID, coverage) {
+        const project = await this.prisma.project.findFirst({
+            where: {
+                id: projectID,
+            },
+        });
+
+        let matchRule: any = {}; // Default value
+
+        try {
+            // Attempt to parse project?.coverage
+            matchRule = JSON.parse(project?.coverage || "{}");
+        } catch (error) {
+            // console.error('Error parsing coverage:', error);
+            // Log the error or handle it as needed
+            // You can also return an empty object or any default value
+        }
+        const exclude = new TestExclude({
+            cwd: "",
+            include: matchRule.include,
+            exclude: matchRule.exclude || [],
+            extension: matchRule.extensions || [
+                ".js",
+                ".cjs",
+                ".mjs",
+                ".ts",
+                ".tsx",
+                ".jsx",
+                ".vue",
+            ],
+        });
+
+        const filterCoverage = {};
+
+        for (const filterCoverageKey of Object.keys(coverage)) {
+            // TODO 当过滤条件特别多的时候，性能会很差，大概能达到3s的计算时间，所以得在消费的时候就落库概览数据，summarys
+            if (exclude.shouldInstrument(filterCoverageKey)) {
+                filterCoverage[filterCoverageKey] = coverage[filterCoverageKey];
+            }
+        }
+        return Object.keys(filterCoverage).length > 0
+            ? filterCoverage
+            : coverage;
+    }
+}

packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts

@@ -7,7 +7,11 @@ import {
 
 import { CoveragediskService } from "./coveragedisk.service";
 import { PrismaService } from "../../../../prisma/prisma.service";
-import { removeNullKeys } from "../../../../utils/utils";
+import {
+    removeNullKeys,
+    resolveProjectID,
+    summaryToDbSummary,
+} from "../../../../utils/utils";
 import { compressedData, decompressedData } from "../../../../utils/zstd";
 import { coverageObj } from "../../models/coverage.model";
 import {
@@ -17,6 +21,9 @@ import {
 import { IstanbulHitMapSchema } from "../../../../zod/istanbul.zod";
 import { remapCoverageWithInstrumentCwd } from "canyon-map";
 import { convertDataFromCoverageMapDatabase } from "../../../../utils/coverage";
+import { logger } from "../../../../logger";
+import { PullChangeCodeAndInsertDbService } from "../common/pull-change-code-and-insert-db.service";
+import { TestExcludeService } from "../common/test-exclude.service";
 
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
@@ -31,8 +38,8 @@ export class ConsumerCoverageService {
         private readonly prisma: PrismaService,
         // private readonly pullChangeCodeAndInsertDbService: PullChangeCodeAndInsertDbService,
         private readonly coveragediskService: CoveragediskService,
-        // private readonly testExcludeService: TestExcludeService,
-        // private readonly pullFilePathAndInsertDbService: PullFilePathAndInsertDbService,
+        private readonly pullChangeCodeAndInsertDbService: PullChangeCodeAndInsertDbService,
+        private readonly testExcludeService: TestExcludeService,
     ) {}
 
     async invoke() {
@@ -130,7 +137,18 @@ export class ConsumerCoverageService {
                 return convertDataFromCoverageMapDatabase(coverageMaps);
             });
 
-        const codechanges = [];
+        // 拉取变更代码
+        await this.pullChangeCode(queueDataToBeConsumed);
+        // 判断是否需要拉取变更代码，对比sha和compareTarget
+        const codechanges =
+            queueDataToBeConsumed.sha === queueDataToBeConsumed.compareTarget
+                ? []
+                : await this.prisma.codechange.findMany({
+                      where: {
+                          sha: queueDataToBeConsumed.sha,
+                          compareTarget: queueDataToBeConsumed.compareTarget,
+                      },
+                  });
 
         // TODO cov应该是全量的，应该是find出来的hit，因为已经合并过了，避免重复
 
@@ -199,16 +217,7 @@ export class ConsumerCoverageService {
                     ...coverageObj,
                     hit: compressedHit,
                     covType: covType,
-                    // newlinesCovered: sum.newlines.covered,
-                    // newlinesTotal: sum.newlines.total,
-                    statementsCovered: sum.statements.covered,
-                    statementsTotal: sum.statements.total,
-                    // functionsCovered: sum.functions.covered,
-                    // functionsTotal: sum.functions.total,
-                    // branchesCovered: sum.branches.covered,
-                    // branchesTotal: sum.branches.total,
-                    // linesCovered: sum.lines.covered,
-                    // linesTotal: sum.lines.total,
+                    ...summaryToDbSummary(sum),
                     summary: summaryZstd,
                     //以下都读的是queueDataToBeConsumed
                     // key: queueDataToBeConsumed.key,
@@ -230,7 +239,25 @@ export class ConsumerCoverageService {
             });
         }
     }
-    // async pullChangeCode(coverage) {}
+    async pullChangeCode(coverage) {
+        if (coverage.sha !== coverage.compareTarget) {
+            try {
+                await this.pullChangeCodeAndInsertDbService.invoke(
+                    resolveProjectID(coverage.projectID),
+                    coverage.sha,
+                    coverage.compareTarget,
+                    "accessToken",
+                    this.prisma,
+                );
+            } catch (e) {
+                logger({
+                    type: "error",
+                    title: "pullChangeCode",
+                    message: String(e),
+                });
+            }
+        }
+    }
 
     async acquireLock(lockName: string, lockTimeout: number): Promise<boolean> {
         const now = new Date();

packages/canyon-collect/src/apps/collect/services/coverage-map-client.service.ts

@@ -14,6 +14,7 @@ import {
 } from "../../../zod/istanbul.zod";
 import { remapCoverageWithInstrumentCwd } from "canyon-map";
 import { compressedData } from "../../../utils/zstd";
+import { summaryToDbSummary } from "../../../utils/utils";
 
 function getNewPathByOldPath(covMap, path) {
     // @ts-ignore
@@ -79,8 +80,7 @@ export class CoverageMapClientService {
                     branch: branch,
                     summary: summary,
                     hit: hit,
-                    statementsCovered: 0,
-                    statementsTotal: overallSummary.statements.total,
+                    ...summaryToDbSummary(overallSummary),
                     reportID: sha,
                     compareTarget: compareTarget || sha, // 默认是自己
                 },