From b0a8c72a8ef17c7c8e6d418b06296f0263646585 Mon Sep 17 00:00:00 2001 From: Joe Becher Date: Mon, 15 Nov 2021 10:24:29 -0500 Subject: [PATCH 1/6] chore: remove unused var --- test/helpers/web.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/helpers/web.test.ts b/test/helpers/web.test.ts index a05ac351a..515e7ca5d 100644 --- a/test/helpers/web.test.ts +++ b/test/helpers/web.test.ts @@ -259,7 +259,6 @@ describe('displayChangelog()', () => { describe('generateRequestHeadersPOST()', () => { const args: UploaderArgs = { ...createEmptyArgs() } - const source = args.source || '' it('should return return the correct url when args.upstream is not set', () => { args.upstream = '' From 35824398650981d9b08ff7a2a9bd2124af6f7512 Mon Sep 17 00:00:00 2001 From: Joe Becher Date: Wed, 17 Nov 2021 11:17:24 -0500 Subject: [PATCH 2/6] fix: always use blocklist Closes #504 Renames blocklist --- npm-shrinkwrap.json | 28 ++++++++++++++ package.json | 2 + src/helpers/files.ts | 34 ++++++++++++++--- src/helpers/logger.ts | 27 +++++++++++++ src/index.ts | 14 ++++--- test/fixtures/coverage-summary.json | 3 ++ test/fixtures/other/coverage-summary.json | 1 + test/fixtures/other/coverage.xml | 1 + test/helpers/files.test.ts | 46 +++++++++++++++++++++++ test/index.test.ts | 3 ++ 10 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/coverage-summary.json create mode 100644 test/fixtures/other/coverage-summary.json create mode 100644 test/fixtures/other/coverage.xml diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 3ddaecb33..cb150bb58 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -9,11 +9,13 @@ "version": "0.1.12", "license": "ISC", "dependencies": { + "@types/micromatch": "4.0.2", "fast-glob": "3.2.7", "glob": "7.2.0", "https-proxy-agent": "5.0.0", "js-yaml": "4.1.0", "line-reader": "0.4.0", + "micromatch": "4.0.4", "node-fetch": "2.6.6", "snake-case": "3.0.4", "validator": "13.7.0", @@ -1564,6 +1566,11 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/braces": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.1.tgz", + "integrity": "sha512-+euflG6ygo4bn0JHtn4pYqcXwRtLvElQ7/nnjDu7iYG56H0+OhCd7d6Ug0IE3WcFpZozBKW2+80FUbv5QGk5AQ==" + }, "node_modules/@types/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", @@ -1629,6 +1636,14 @@ "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", "dev": true }, + "node_modules/@types/micromatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.2.tgz", + "integrity": "sha512-oqXqVb0ci19GtH0vOA/U2TmHTcRY9kuZl4mqUxe0QmJAlIW13kzhuK5pi1i9+ngav8FjpSb9FVS/GE00GLX1VA==", + "dependencies": { + "@types/braces": "*" + } + }, "node_modules/@types/minimatch": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", @@ -9889,6 +9904,11 @@ "@babel/types": "^7.3.0" } }, + "@types/braces": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.1.tgz", + "integrity": "sha512-+euflG6ygo4bn0JHtn4pYqcXwRtLvElQ7/nnjDu7iYG56H0+OhCd7d6Ug0IE3WcFpZozBKW2+80FUbv5QGk5AQ==" + }, "@types/glob": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", @@ -9954,6 +9974,14 @@ "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==", "dev": true }, + "@types/micromatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.2.tgz", + "integrity": "sha512-oqXqVb0ci19GtH0vOA/U2TmHTcRY9kuZl4mqUxe0QmJAlIW13kzhuK5pi1i9+ngav8FjpSb9FVS/GE00GLX1VA==", + "requires": { + "@types/braces": "*" + } + }, "@types/minimatch": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", diff --git a/package.json b/package.json index bdf89e547..81e043d29 100644 --- a/package.json +++ b/package.json @@ -31,11 +31,13 @@ }, "homepage": "https://github.com/codecov/uploader#readme", "dependencies": { + "@types/micromatch": "4.0.2", "fast-glob": "3.2.7", "glob": "7.2.0", "https-proxy-agent": "5.0.0", "js-yaml": "4.1.0", "line-reader": "0.4.0", + "micromatch": "4.0.4", "node-fetch": "2.6.6", "snake-case": "3.0.4", "validator": "13.7.0", diff --git a/src/helpers/files.ts b/src/helpers/files.ts index 70bafcb66..74e159c0c 100644 --- a/src/helpers/files.ts +++ b/src/helpers/files.ts @@ -4,7 +4,8 @@ import fs from 'fs' import { readFile } from 'fs/promises' import { posix as path } from 'path' import { UploaderArgs } from '../types' -import { logError, verbose } from './logger' +import { logError, UploadLogger, verbose } from './logger' +import micromatch from "micromatch"; export const MARKER_NETWORK_END = '\n<<<<<< network\n' export const MARKER_FILE_END = '<<<<<< EOF\n' @@ -25,7 +26,7 @@ export async function getFileListing( return getAllFiles(projectRoot, projectRoot, args).join('\n') } -export function manualBlacklist(): string[] { +function manualBlocklist(): string[] { // TODO: honor the .gitignore file instead of a hard-coded list return [ '.DS_Store', @@ -41,7 +42,7 @@ export function manualBlacklist(): string[] { ] } -export function globBlacklist(): string[] { +function globBlocklist(): string[] { // TODO: honor the .gitignore file instead of a hard-coded list return [ '__pycache__', @@ -205,7 +206,7 @@ export async function getCoverageFiles( }), { cwd: projectRoot, dot: true, - ignore: [...manualBlacklist(), ...globBlacklist()], + ignore: getBlocklist(), }) } @@ -246,7 +247,7 @@ export function getAllFiles( files = glob .sync(['**/*', '**/.[!.]*'], { cwd: dirPath, - ignore: manualBlacklist().map(globstar), + ignore: manualBlocklist().map(globstar), }) } else { files = stdout.split(/[\r\n]+/) @@ -332,3 +333,26 @@ export function removeFile(projectRoot: string, filePath: string): void { } }) } +export function getBlocklist() { + return [...manualBlocklist(), ...globBlocklist()] +} + +export function cleanCoverageFilePaths(projectRoot: string, paths: string[], ignoreGlobs: string[]) { + UploadLogger.verbose(`Preparing to clean the following coverage paths: ${paths.toString()}`) + const coverageFilePaths = [... new Set(paths.filter(file => { + return fileExists(projectRoot, file) + }))] + + if (coverageFilePaths.length === 0) { + throw new Error('Error while cleaning paths. No paths matched existing files!') + } + + const ignoredFiles = micromatch(coverageFilePaths, ignoreGlobs) + + const filesAfterCheckingIgnore = coverageFilePaths.filter(path => { + return !ignoredFiles.includes(path) + }) + + return filesAfterCheckingIgnore +} + diff --git a/src/helpers/logger.ts b/src/helpers/logger.ts index dbcd63221..faa805040 100644 --- a/src/helpers/logger.ts +++ b/src/helpers/logger.ts @@ -41,3 +41,30 @@ export function logError(message: string): void { export function info(message: string): void { console.log(`[${_getTimestamp()}] ['info'] ${message}`) } + +export class UploadLogger { + private static _instance: UploadLogger + logLevel = 'info' + + private constructor() { + // Intentionally empty + } + + static init() { + if (!UploadLogger._instance) { + UploadLogger._instance = new UploadLogger() + } + } + + static setLogLevel(level: string) { + UploadLogger.init() + UploadLogger._instance.logLevel = level + } + + static verbose(message: string) { + UploadLogger.init() + if (UploadLogger._instance.logLevel === 'verbose') { + console.log(`[${_getTimestamp()}] ['verbose'] ${message}`) + } + } +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 3013a25b1..8b0b51ee0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,13 +5,14 @@ import { version } from '../package.json' import * as validateHelpers from './helpers/validate' import { detectProvider } from './helpers/provider' import * as webHelpers from './helpers/web' -import { info, logError, verbose } from './helpers/logger' +import { info, logError, UploadLogger, verbose } from './helpers/logger' import { getToken } from './helpers/token' import { + cleanCoverageFilePaths, coverageFilePatterns, fetchGitRoot, - fileExists, fileHeader, + getBlocklist, getCoverageFiles, getFileListing, getFilePath, @@ -21,6 +22,7 @@ import { readCoverageFile, removeFile, } from './helpers/files' +import { arch } from 'os' /** * @@ -77,6 +79,10 @@ export async function main( args: UploaderArgs, ): Promise> { + if (args.verbose) { + UploadLogger.setLogLevel('verbose') + } + // Did user asking for changelog? if (args.changelog) { webHelpers.displayChangelog() @@ -170,9 +176,7 @@ export async function main( )) // Remove invalid and duplicate file paths - coverageFilePaths = [... new Set(coverageFilePaths.filter(file => { - return fileExists(args.dir || projectRoot, file) - }))] + coverageFilePaths = cleanCoverageFilePaths(args.dir || projectRoot, coverageFilePaths, getBlocklist()) if (coverageFilePaths.length > 0) { info(`=> Found ${coverageFilePaths.length} possible coverage files:\n ` + diff --git a/test/fixtures/coverage-summary.json b/test/fixtures/coverage-summary.json new file mode 100644 index 000000000..a02d891b7 --- /dev/null +++ b/test/fixtures/coverage-summary.json @@ -0,0 +1,3 @@ +{ + "I am": "bad coverage" +} diff --git a/test/fixtures/other/coverage-summary.json b/test/fixtures/other/coverage-summary.json new file mode 100644 index 000000000..df40430a4 --- /dev/null +++ b/test/fixtures/other/coverage-summary.json @@ -0,0 +1 @@ +{ "Hello": "!" } diff --git a/test/fixtures/other/coverage.xml b/test/fixtures/other/coverage.xml new file mode 100644 index 000000000..cadc237e2 --- /dev/null +++ b/test/fixtures/other/coverage.xml @@ -0,0 +1 @@ +Boo! \ No newline at end of file diff --git a/test/helpers/files.test.ts b/test/helpers/files.test.ts index c55c611b4..23fc1ab1a 100644 --- a/test/helpers/files.test.ts +++ b/test/helpers/files.test.ts @@ -3,6 +3,7 @@ import fs from 'fs' import childProcess from 'child_process' import * as fileHelpers from '../../src/helpers/files' import mock from 'mock-fs' +import path from 'path/posix' describe('File Helpers', () => { afterEach(() => { @@ -281,6 +282,51 @@ describe('File Helpers', () => { }) }) + describe("cleanCoverageFilePaths()", () => { + it("works", async () => { + const paths = await fileHelpers.getCoverageFiles( + '.', + fileHelpers.coverageFilePatterns(), + ) + const ignoreGlobs = fileHelpers.getBlocklist() + + expect(() => fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ignoreGlobs)).not.toThrow() + }) + + it("returns the input array when passed an empty ignore array", async() => { + const paths = await fileHelpers.getCoverageFiles( + '.', + fileHelpers.coverageFilePatterns(), + ) + expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, [])).toEqual(paths) + }) + + it("ignores an ignore filename", async() => { + const paths = await fileHelpers.getCoverageFiles( + '.', + fileHelpers.coverageFilePatterns(), + ) + expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ["coverage-summary.json"])).not.toContain(expect.stringContaining('coverage.txt')) + }) + + it("ignores an ignore filename glob", async() => { + const paths = await fileHelpers.getCoverageFiles( + '.', + fileHelpers.coverageFilePatterns(), + ) + const foo = expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ["**/coverage*"])).not.toContainEqual(expect.stringMatching('coverage-summary.json')) + console.log(foo) + }) + + it("ignores an ignore filename globstar", async() => { + const paths = await fileHelpers.getCoverageFiles( + '.', + fileHelpers.coverageFilePatterns(), + ) + expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ["**/other/*"])).not.toContainEqual(expect.stringMatching('other')) + }) + }) + it('can remove a file', () => { const fn = jest.spyOn(fs, 'unlink').mockImplementation(() => null) fileHelpers.removeFile('.', 'coverage.xml') diff --git a/test/index.test.ts b/test/index.test.ts index d9f6040e2..386602529 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -5,6 +5,7 @@ import * as app from '../src' import { version } from '../package.json' import nock from 'nock' import fs from 'fs' +import { UploadLogger } from '../src/helpers/logger' // Backup the env const realEnv = { ...process.env } @@ -12,6 +13,8 @@ const realEnv = { ...process.env } describe('Uploader Core', () => { const env = process.env + UploadLogger.setLogLevel('verbose') + beforeEach(() => { // https://bensmithgall.com/blog/jest-mock-trick if this works! const mockExit = jest.fn() From b4856b97a1a63d78f379858d3a8b9322eeacccdf Mon Sep 17 00:00:00 2001 From: Joe Becher Date: Wed, 17 Nov 2021 13:36:09 -0500 Subject: [PATCH 3/6] chore: remove unused import --- src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 8b0b51ee0..10c3c483e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,7 +22,6 @@ import { readCoverageFile, removeFile, } from './helpers/files' -import { arch } from 'os' /** * From fc999c0efe1ddfb784c6b8ac2f5209d38826a208 Mon Sep 17 00:00:00 2001 From: Joe Becher Date: Wed, 17 Nov 2021 13:37:30 -0500 Subject: [PATCH 4/6] chore: remove unused import --- test/helpers/files.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/helpers/files.test.ts b/test/helpers/files.test.ts index 23fc1ab1a..151153854 100644 --- a/test/helpers/files.test.ts +++ b/test/helpers/files.test.ts @@ -3,7 +3,6 @@ import fs from 'fs' import childProcess from 'child_process' import * as fileHelpers from '../../src/helpers/files' import mock from 'mock-fs' -import path from 'path/posix' describe('File Helpers', () => { afterEach(() => { From 4d75a02830a0ff386b081b993cee8098266d4fb9 Mon Sep 17 00:00:00 2001 From: Joe Becher Date: Wed, 17 Nov 2021 13:39:16 -0500 Subject: [PATCH 5/6] chore: add return type --- src/helpers/files.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/files.ts b/src/helpers/files.ts index 74e159c0c..25c7df877 100644 --- a/src/helpers/files.ts +++ b/src/helpers/files.ts @@ -333,7 +333,7 @@ export function removeFile(projectRoot: string, filePath: string): void { } }) } -export function getBlocklist() { +export function getBlocklist(): string[] { return [...manualBlocklist(), ...globBlocklist()] } From 1bba9e81e80a6af392524d25fa3930638405d3af Mon Sep 17 00:00:00 2001 From: Joe Becher Date: Wed, 17 Nov 2021 13:40:02 -0500 Subject: [PATCH 6/6] chore: add return type --- src/helpers/files.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/helpers/files.ts b/src/helpers/files.ts index 25c7df877..82c63e486 100644 --- a/src/helpers/files.ts +++ b/src/helpers/files.ts @@ -337,7 +337,7 @@ export function getBlocklist(): string[] { return [...manualBlocklist(), ...globBlocklist()] } -export function cleanCoverageFilePaths(projectRoot: string, paths: string[], ignoreGlobs: string[]) { +export function cleanCoverageFilePaths(projectRoot: string, paths: string[], ignoreGlobs: string[]): string[] { UploadLogger.verbose(`Preparing to clean the following coverage paths: ${paths.toString()}`) const coverageFilePaths = [... new Set(paths.filter(file => { return fileExists(projectRoot, file)