From 2a4a683512328e3eda793afbd138cb986181e367 Mon Sep 17 00:00:00 2001 From: zhangtao25 Date: Mon, 23 Dec 2024 14:29:31 +0800 Subject: [PATCH] chore: daily development --- packages/canyon-collect/package.json | 7 +- .../src/apps/collect/collect.module.ts | 4 + .../pull-change-code-and-insert-db.service.ts | 43 ++++ .../services/common/test-exclude.service.ts | 53 +++++ .../core/consumer-coverage.service.ts | 57 ++++-- .../services/coverage-map-client.service.ts | 4 +- packages/canyon-collect/src/utils/diffline.ts | 185 ++++++++++++++++++ packages/canyon-collect/src/utils/utils.ts | 19 ++ 8 files changed, 353 insertions(+), 19 deletions(-) create mode 100644 packages/canyon-collect/src/apps/collect/services/common/pull-change-code-and-insert-db.service.ts create mode 100644 packages/canyon-collect/src/apps/collect/services/common/test-exclude.service.ts create mode 100755 packages/canyon-collect/src/utils/diffline.ts diff --git a/packages/canyon-collect/package.json b/packages/canyon-collect/package.json index 10d2a5d1..06817f0e 100644 --- a/packages/canyon-collect/package.json +++ b/packages/canyon-collect/package.json @@ -29,8 +29,10 @@ "@nestjs/typeorm": "^10.0.2", "typeorm": "^0.3.20", "sqlite3": "^5.1.7", + "test-exclude": "^7.0.1", "canyon-data": "^2.0.0-beta.6", - "canyon-map": "^2.0.0-beta.5" + "canyon-map": "^2.0.0-beta.5", + "diff": "^7.0.0" }, "devDependencies": { "@nestjs/cli": "^10.4.9", @@ -53,6 +55,7 @@ "ts-loader": "^9.5.1", "ts-node": "^10.9.2", "tsconfig-paths": "^4.2.0", - "typescript": "^5.7.2" + "typescript": "^5.7.2", + "@types/test-exclude": "^6.0.2" } } diff --git a/packages/canyon-collect/src/apps/collect/collect.module.ts b/packages/canyon-collect/src/apps/collect/collect.module.ts index 2d18e47d..e67150fa 100644 --- a/packages/canyon-collect/src/apps/collect/collect.module.ts +++ b/packages/canyon-collect/src/apps/collect/collect.module.ts @@ -9,6 +9,8 @@ import { TypeOrmModule } from "@nestjs/typeorm"; import { PrismaService } from "../../prisma/prisma.service"; import { CoveragediskService } from "./services/core/coveragedisk.service"; import { ConsumerCoverageService } from "./services/core/consumer-coverage.service"; +import { PullChangeCodeAndInsertDbService } from "./services/common/pull-change-code-and-insert-db.service"; +import { TestExcludeService } from "./services/common/test-exclude.service"; @Module({ imports: [TypeOrmModule.forFeature([CoveragediskEntity])], @@ -19,6 +21,8 @@ import { ConsumerCoverageService } from "./services/core/consumer-coverage.servi CoverageMapClientService, ConsumerCoverageService, CoveragediskService, + PullChangeCodeAndInsertDbService, + TestExcludeService, ], }) export class CollectModule { diff --git a/packages/canyon-collect/src/apps/collect/services/common/pull-change-code-and-insert-db.service.ts b/packages/canyon-collect/src/apps/collect/services/common/pull-change-code-and-insert-db.service.ts new file mode 100644 index 00000000..7148151a --- /dev/null +++ b/packages/canyon-collect/src/apps/collect/services/common/pull-change-code-and-insert-db.service.ts @@ -0,0 +1,43 @@ +// import { diffLine } from "../../../utils/diffline"; + +import { diffLine } from "../../../../utils/diffline"; + +export class PullChangeCodeAndInsertDbService { + async invoke(projectID, commitSha, compareTarget, accessToken, prisma) { + const codechanges = await prisma.codechange.findMany({ + where: { + projectID, + sha: commitSha, + compareTarget, + }, + }); + const gitProvider = await prisma.gitProvider.findFirst({ + where: { + disabled: false, + }, + }); + if (codechanges.length === 0) { + const diffLineData = await diffLine({ + repoID: projectID, + baseCommitSha: compareTarget, + compareCommitSha: commitSha, + token: gitProvider?.privateToken, + gitlabUrl: gitProvider?.url, + }); + const data = diffLineData.map(({ path, additions, deletions }) => { + return { + projectID, + sha: commitSha, + compareTarget, + path, + additions, + deletions, + }; + }); + + await prisma.codechange.createMany({ + data: data, + }); + } + } +} diff --git a/packages/canyon-collect/src/apps/collect/services/common/test-exclude.service.ts b/packages/canyon-collect/src/apps/collect/services/common/test-exclude.service.ts new file mode 100644 index 00000000..64485851 --- /dev/null +++ b/packages/canyon-collect/src/apps/collect/services/common/test-exclude.service.ts @@ -0,0 +1,53 @@ +import { Injectable } from "@nestjs/common"; +// import { PrismaService } from "../../../prisma/prisma.service"; +import * as TestExclude from "test-exclude"; +import { PrismaService } from "../../../../prisma/prisma.service"; +@Injectable() +export class TestExcludeService { + constructor(private readonly prisma: PrismaService) {} + + async invoke(projectID, coverage) { + const project = await this.prisma.project.findFirst({ + where: { + id: projectID, + }, + }); + + let matchRule: any = {}; // Default value + + try { + // Attempt to parse project?.coverage + matchRule = JSON.parse(project?.coverage || "{}"); + } catch (error) { + // console.error('Error parsing coverage:', error); + // Log the error or handle it as needed + // You can also return an empty object or any default value + } + const exclude = new TestExclude({ + cwd: "", + include: matchRule.include, + exclude: matchRule.exclude || [], + extension: matchRule.extensions || [ + ".js", + ".cjs", + ".mjs", + ".ts", + ".tsx", + ".jsx", + ".vue", + ], + }); + + const filterCoverage = {}; + + for (const filterCoverageKey of Object.keys(coverage)) { + // TODO 当过滤条件特别多的时候，性能会很差，大概能达到3s的计算时间，所以得在消费的时候就落库概览数据，summarys + if (exclude.shouldInstrument(filterCoverageKey)) { + filterCoverage[filterCoverageKey] = coverage[filterCoverageKey]; + } + } + return Object.keys(filterCoverage).length > 0 + ? filterCoverage + : coverage; + } +} diff --git a/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts b/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts index d673bb1e..f3efa9fa 100644 --- a/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts +++ b/packages/canyon-collect/src/apps/collect/services/core/consumer-coverage.service.ts @@ -7,7 +7,11 @@ import { import { CoveragediskService } from "./coveragedisk.service"; import { PrismaService } from "../../../../prisma/prisma.service"; -import { removeNullKeys } from "../../../../utils/utils"; +import { + removeNullKeys, + resolveProjectID, + summaryToDbSummary, +} from "../../../../utils/utils"; import { compressedData, decompressedData } from "../../../../utils/zstd"; import { coverageObj } from "../../models/coverage.model"; import { @@ -17,6 +21,9 @@ import { import { IstanbulHitMapSchema } from "../../../../zod/istanbul.zod"; import { remapCoverageWithInstrumentCwd } from "canyon-map"; import { convertDataFromCoverageMapDatabase } from "../../../../utils/coverage"; +import { logger } from "../../../../logger"; +import { PullChangeCodeAndInsertDbService } from "../common/pull-change-code-and-insert-db.service"; +import { TestExcludeService } from "../common/test-exclude.service"; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -31,8 +38,8 @@ export class ConsumerCoverageService { private readonly prisma: PrismaService, // private readonly pullChangeCodeAndInsertDbService: PullChangeCodeAndInsertDbService, private readonly coveragediskService: CoveragediskService, - // private readonly testExcludeService: TestExcludeService, - // private readonly pullFilePathAndInsertDbService: PullFilePathAndInsertDbService, + private readonly pullChangeCodeAndInsertDbService: PullChangeCodeAndInsertDbService, + private readonly testExcludeService: TestExcludeService, ) {} async invoke() { @@ -130,7 +137,18 @@ export class ConsumerCoverageService { return convertDataFromCoverageMapDatabase(coverageMaps); }); - const codechanges = []; + // 拉取变更代码 + await this.pullChangeCode(queueDataToBeConsumed); + // 判断是否需要拉取变更代码，对比sha和compareTarget + const codechanges = + queueDataToBeConsumed.sha === queueDataToBeConsumed.compareTarget + ? [] + : await this.prisma.codechange.findMany({ + where: { + sha: queueDataToBeConsumed.sha, + compareTarget: queueDataToBeConsumed.compareTarget, + }, + }); // TODO cov应该是全量的，应该是find出来的hit，因为已经合并过了，避免重复 @@ -199,16 +217,7 @@ export class ConsumerCoverageService { ...coverageObj, hit: compressedHit, covType: covType, - // newlinesCovered: sum.newlines.covered, - // newlinesTotal: sum.newlines.total, - statementsCovered: sum.statements.covered, - statementsTotal: sum.statements.total, - // functionsCovered: sum.functions.covered, - // functionsTotal: sum.functions.total, - // branchesCovered: sum.branches.covered, - // branchesTotal: sum.branches.total, - // linesCovered: sum.lines.covered, - // linesTotal: sum.lines.total, + ...summaryToDbSummary(sum), summary: summaryZstd, //以下都读的是queueDataToBeConsumed // key: queueDataToBeConsumed.key, @@ -230,7 +239,25 @@ export class ConsumerCoverageService { }); } } - // async pullChangeCode(coverage) {} + async pullChangeCode(coverage) { + if (coverage.sha !== coverage.compareTarget) { + try { + await this.pullChangeCodeAndInsertDbService.invoke( + resolveProjectID(coverage.projectID), + coverage.sha, + coverage.compareTarget, + "accessToken", + this.prisma, + ); + } catch (e) { + logger({ + type: "error", + title: "pullChangeCode", + message: String(e), + }); + } + } + } async acquireLock(lockName: string, lockTimeout: number): Promise { const now = new Date(); diff --git a/packages/canyon-collect/src/apps/collect/services/coverage-map-client.service.ts b/packages/canyon-collect/src/apps/collect/services/coverage-map-client.service.ts index 2d51044b..db2d1f1d 100755 --- a/packages/canyon-collect/src/apps/collect/services/coverage-map-client.service.ts +++ b/packages/canyon-collect/src/apps/collect/services/coverage-map-client.service.ts @@ -14,6 +14,7 @@ import { } from "../../../zod/istanbul.zod"; import { remapCoverageWithInstrumentCwd } from "canyon-map"; import { compressedData } from "../../../utils/zstd"; +import { summaryToDbSummary } from "../../../utils/utils"; function getNewPathByOldPath(covMap, path) { // @ts-ignore @@ -79,8 +80,7 @@ export class CoverageMapClientService { branch: branch, summary: summary, hit: hit, - statementsCovered: 0, - statementsTotal: overallSummary.statements.total, + ...summaryToDbSummary(overallSummary), reportID: sha, compareTarget: compareTarget || sha, // 默认是自己 }, diff --git a/packages/canyon-collect/src/utils/diffline.ts b/packages/canyon-collect/src/utils/diffline.ts new file mode 100755 index 00000000..818209db --- /dev/null +++ b/packages/canyon-collect/src/utils/diffline.ts @@ -0,0 +1,185 @@ +import * as Diff from "diff"; +import { Change } from "diff"; +interface DiffLine { + repoID: string; + baseCommitSha?: string; + compareCommitSha: string; + includesFileExtensions?: string[]; + gitlabUrl?: string; + token?: string; +} + +function calculateNewRows( + a: string, + b: string, +): { additions: number[]; deletions: number[] } { + const diffResult: Change[] = Diff.diffLines(a, b); + function generateArray(startValue: number, length: number) { + return Array.from( + { length }, + (_, index) => startValue - index, + ).reverse(); + } + function sumToIndex(arr: number[], index: number) { + return arr.slice(0, index + 1).reduce((sum, value) => sum + value, 0); + } + const additionsDiffResult = diffResult.filter((i) => !i.removed); + const additions: any = []; + additionsDiffResult.forEach((i, index) => { + if (i.added) { + additions.push( + generateArray( + sumToIndex( + additionsDiffResult.map((i) => i.count || 0), + index, + ), + i.count || 0, + ), + ); + } + }); + + const deletionsDiffResult = diffResult.filter((i) => i.removed); + const deletions: any = []; + deletionsDiffResult.forEach((i, index) => { + if (i.removed) { + deletions.push( + generateArray( + sumToIndex( + deletionsDiffResult.map((i) => i.count || 0), + index, + ), + i.count || 0, + ), + ); + } + }); + return { + additions: additions.flat(Infinity), + deletions: deletions.flat(Infinity), + }; +} + +function getDecode(str: string) { + return decodeURIComponent( + atob(str) + .split("") + .map(function (c) { + return "%" + ("00" + c.charCodeAt(0).toString(16)).slice(-2); + }) + .join(""), + ); +} +export async function diffLine({ + repoID, + baseCommitSha = undefined, + compareCommitSha, + includesFileExtensions = ["ts", "tsx", "jsx", "vue", "js"], + gitlabUrl = "https://gitlab.com", + token = "default_token", +}: DiffLine): Promise< + { path: string; additions: number[]; deletions: number[] }[] +> { + const gitlabApiUrlFile = `${gitlabUrl}/api/v4/projects/${repoID}/repository/files`; + const gitlabApiUrlCommit = `${gitlabUrl}/api/v4/projects/${repoID}/repository/commits/${compareCommitSha}`; + + const gitlabApiUrlCommitResponse = await fetch(gitlabApiUrlCommit, { + headers: { + // Authorization: 'Bearer ' + token, // 在请求头中使用 GitLab API token + "private-token": token, + }, + }) + .then((res) => res.json()) + .then((data) => { + return { + parent_ids: data.parent_ids || [], + stats: data.stats, + }; + }); + + const result = []; + // 只关心 50000 行以内的更改 + if ( + gitlabApiUrlCommitResponse.parent_ids.length > 0 && + gitlabApiUrlCommitResponse.stats.additions < 5000000 + ) { + // 声明realBaseCommitSha，如果baseCommitSha存在，则使用baseCommitSha，否则使用gitlabApiUrlCommitResponse.parent_ids[0] + const realBaseCommitSha = + baseCommitSha || gitlabApiUrlCommitResponse.parent_ids[0]; + const gitDiffs = await fetch( + `${gitlabUrl}/api/v4/projects/${repoID}/repository/compare?from=${realBaseCommitSha}&to=${compareCommitSha}`, + { + headers: { + // Authorization: 'Bearer ' + token, // 在请求头中使用 GitLab API token + "private-token": token, + }, + }, + ) + .then((res) => res.json()) + .then((response) => { + return (response.diffs || []).map( + ({ + old_path, + new_path, + a_mode, + b_mode, + new_file, + renamed_file, + deleted_file, + }) => { + return { + old_path, + new_path, + a_mode, + b_mode, + new_file, + renamed_file, + deleted_file, + }; + }, + ); + }); + + // const includesFileExtensions = /\.tsx?$|\.jsx?$|\.vue$|\.js$/i; + + const isMatchingExtension = ( + includesFileExtensions: string[], + pathname: string, + ) => includesFileExtensions.some((ext) => pathname.endsWith("." + ext)); + + const gitDiffsFiltered = gitDiffs.filter((gitDiff) => + isMatchingExtension(includesFileExtensions, gitDiff.new_path), + ); + + for (let i = 0; i < gitDiffsFiltered.length; i++) { + const contents = await Promise.all( + [realBaseCommitSha, compareCommitSha].map((c) => { + return fetch( + `${gitlabApiUrlFile}/${encodeURIComponent( + gitDiffsFiltered[i].new_path, + )}?ref=${c}`, + { + headers: { + // Authorization: 'Bearer ' + token, // 在请求头中使用 GitLab API token + "private-token": token, + }, + method: "GET", + }, + ) + .then((res) => res.json()) + .then((r) => { + return getDecode(r.content); + }) + .catch(() => { + return ""; + }); + }), + ); + result.push({ + path: gitDiffsFiltered[i].new_path, + ...calculateNewRows(contents[0], contents[1]), + }); + } + } + return result; +} diff --git a/packages/canyon-collect/src/utils/utils.ts b/packages/canyon-collect/src/utils/utils.ts index ded0e1ad..0ddeebca 100755 --- a/packages/canyon-collect/src/utils/utils.ts +++ b/packages/canyon-collect/src/utils/utils.ts @@ -43,3 +43,22 @@ export function mapToLowerCamelCase(coverage): any { reporter: coverage.reporter, }; } + +export const summaryToDbSummary = (summary) => { + return { + statementsCovered: summary.statements.covered, + statementsTotal: summary.statements.total, + branchesCovered: summary.branches.covered, + branchesTotal: summary.branches.total, + functionsCovered: summary.functions.covered, + functionsTotal: summary.functions.total, + linesCovered: summary.lines.covered, + linesTotal: summary.lines.total, + newlinesCovered: summary.newlines.covered, + newlinesTotal: summary.newlines.total, + }; +}; + +export function resolveProjectID(projectID) { + return projectID.split("-")[1] || projectID; +}