Merge pull request #512 from codecov/CESD-492-always-use-block-list

[CESD-492] always use block list

package.json

@@ -31,11 +31,13 @@
   },
   "homepage": "https://github.com/codecov/uploader#readme",
   "dependencies": {
+    "@types/micromatch": "4.0.2",
     "fast-glob": "3.2.7",
     "glob": "7.2.0",
     "https-proxy-agent": "5.0.0",
     "js-yaml": "4.1.0",
     "line-reader": "0.4.0",
+    "micromatch": "4.0.4",
     "node-fetch": "2.6.6",
     "snake-case": "3.0.4",
     "validator": "13.7.0",

src/helpers/files.ts

@@ -4,7 +4,8 @@ import fs from 'fs'
 import { readFile } from 'fs/promises'
 import { posix as path } from 'path'
 import { UploaderArgs } from '../types'
-import { logError, verbose } from './logger'
+import { logError, UploadLogger, verbose } from './logger'
+import micromatch from "micromatch";
 
 export const MARKER_NETWORK_END = '\n<<<<<< network\n'
 export const MARKER_FILE_END = '<<<<<< EOF\n'
@@ -25,7 +26,7 @@ export async function getFileListing(
   return getAllFiles(projectRoot, projectRoot, args).join('\n')
 }
 
-export function manualBlacklist(): string[] {
+function manualBlocklist(): string[] {
   // TODO: honor the .gitignore file instead of a hard-coded list
   return [
     '.DS_Store',
@@ -41,7 +42,7 @@ export function manualBlacklist(): string[] {
   ]
 }
 
-export function globBlacklist(): string[] {
+function globBlocklist(): string[] {
   // TODO: honor the .gitignore file instead of a hard-coded list
   return [
     '__pycache__',
@@ -205,7 +206,7 @@ export async function getCoverageFiles(
   }), {
     cwd: projectRoot,
     dot: true,
-    ignore: [...manualBlacklist(), ...globBlacklist()],
+    ignore: getBlocklist(),
   })
 }
 
@@ -246,7 +247,7 @@ export function getAllFiles(
     files = glob
       .sync(['**/*', '**/.[!.]*'], {
         cwd: dirPath,
-        ignore: manualBlacklist().map(globstar),
+        ignore: manualBlocklist().map(globstar),
       })
   } else {
     files = stdout.split(/[\r\n]+/)
@@ -332,3 +333,26 @@ export function removeFile(projectRoot: string, filePath: string): void {
     }
   })
 }
+export function getBlocklist(): string[] {
+  return [...manualBlocklist(), ...globBlocklist()]
+}
+
+export function cleanCoverageFilePaths(projectRoot: string, paths: string[], ignoreGlobs: string[]): string[] {
+  UploadLogger.verbose(`Preparing to clean the following coverage paths: ${paths.toString()}`)
+  const coverageFilePaths = [... new Set(paths.filter(file => {
+    return fileExists(projectRoot, file)
+  }))]
+
+  if (coverageFilePaths.length === 0) {
+    throw new Error('Error while cleaning paths. No paths matched existing files!')  
+  }
+
+  const ignoredFiles = micromatch(coverageFilePaths, ignoreGlobs)
+
+  const filesAfterCheckingIgnore = coverageFilePaths.filter(path => {
+    return !ignoredFiles.includes(path)
+  })
+
+  return filesAfterCheckingIgnore
+}
+

src/helpers/logger.ts

@@ -41,3 +41,30 @@ export function logError(message: string): void {
 export function info(message: string): void {
   console.log(`[${_getTimestamp()}] ['info'] ${message}`)
 }
+
+export class UploadLogger {
+  private static _instance: UploadLogger
+  logLevel = 'info'
+
+  private constructor() {
+    // Intentionally empty
+  }
+
+  static init() {
+    if (!UploadLogger._instance) {
+      UploadLogger._instance = new UploadLogger()
+    }
+  }
+
+  static setLogLevel(level: string) {
+    UploadLogger.init()
+    UploadLogger._instance.logLevel = level
+  }
+
+  static verbose(message: string) {
+    UploadLogger.init()
+    if (UploadLogger._instance.logLevel === 'verbose') {
+      console.log(`[${_getTimestamp()}] ['verbose'] ${message}`)
+    }
+  }
+}

src/index.ts

@@ -5,13 +5,14 @@ import { version } from '../package.json'
 import * as validateHelpers from './helpers/validate'
 import { detectProvider } from './helpers/provider'
 import * as webHelpers from './helpers/web'
-import { info, logError, verbose } from './helpers/logger'
+import { info, logError, UploadLogger, verbose } from './helpers/logger'
 import { getToken } from './helpers/token'
 import {
+  cleanCoverageFilePaths,
   coverageFilePatterns,
   fetchGitRoot,
-  fileExists,
   fileHeader,
+  getBlocklist,
   getCoverageFiles,
   getFileListing,
   getFilePath,
@@ -77,6 +78,10 @@ export async function main(
   args: UploaderArgs,
 ): Promise<void | Record<string, unknown>> {
 
+  if (args.verbose) {
+    UploadLogger.setLogLevel('verbose')
+  }
+
   // Did user asking for changelog?
   if (args.changelog) {
     webHelpers.displayChangelog()
@@ -170,9 +175,7 @@ export async function main(
   ))
 
   // Remove invalid and duplicate file paths
-  coverageFilePaths = [... new Set(coverageFilePaths.filter(file => {
-    return fileExists(args.dir || projectRoot, file)
-  }))]
+  coverageFilePaths = cleanCoverageFilePaths(args.dir || projectRoot, coverageFilePaths, getBlocklist())
 
   if (coverageFilePaths.length > 0) {
     info(`=> Found ${coverageFilePaths.length} possible coverage files:\n  ` +

test/fixtures/coverage-summary.json

@@ -0,0 +1,3 @@
+{
+  "I am": "bad coverage"
+}

test/fixtures/other/coverage-summary.json

@@ -0,0 +1 @@
+{ "Hello": "!" }

test/fixtures/other/coverage.xml

@@ -0,0 +1 @@
+Boo!

test/helpers/files.test.ts

@@ -281,6 +281,51 @@ describe('File Helpers', () => {
     })
   })
 
+  describe("cleanCoverageFilePaths()", () => {
+    it("works", async () => {
+      const paths = await fileHelpers.getCoverageFiles(
+        '.',
+        fileHelpers.coverageFilePatterns(),
+      )
+      const ignoreGlobs = fileHelpers.getBlocklist()
+
+      expect(() => fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ignoreGlobs)).not.toThrow()
+    })
+
+    it("returns the input array when passed an empty ignore array", async() => {
+      const paths = await fileHelpers.getCoverageFiles(
+        '.',
+        fileHelpers.coverageFilePatterns(),
+      )
+      expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, [])).toEqual(paths)
+    })
+
+    it("ignores an ignore filename", async() => {
+      const paths = await fileHelpers.getCoverageFiles(
+        '.',
+        fileHelpers.coverageFilePatterns(),
+      )
+      expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ["coverage-summary.json"])).not.toContain(expect.stringContaining('coverage.txt'))
+    })
+
+    it("ignores an ignore filename glob", async() => {
+      const paths = await fileHelpers.getCoverageFiles(
+        '.',
+        fileHelpers.coverageFilePatterns(),
+      )
+      const foo = expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ["**/coverage*"])).not.toContainEqual(expect.stringMatching('coverage-summary.json'))
+      console.log(foo)
+    })
+
+    it("ignores an ignore filename globstar", async() => {
+      const paths = await fileHelpers.getCoverageFiles(
+        '.',
+        fileHelpers.coverageFilePatterns(),
+      )
+      expect(fileHelpers.cleanCoverageFilePaths(process.cwd(), paths, ["**/other/*"])).not.toContainEqual(expect.stringMatching('other'))
+    })
+  })
+
   it('can remove a file', () => {
     const fn = jest.spyOn(fs, 'unlink').mockImplementation(() => null)
     fileHelpers.removeFile('.', 'coverage.xml')

test/helpers/web.test.ts

@@ -259,7 +259,6 @@ describe('displayChangelog()', () => {
 
 describe('generateRequestHeadersPOST()', () => {
   const args: UploaderArgs = { ...createEmptyArgs() }
-  const source = args.source || ''
 
   it('should return return the correct url when args.upstream is not set', () => {
     args.upstream = ''

test/index.test.ts

@@ -5,13 +5,16 @@ import * as app from '../src'
 import { version } from '../package.json'
 import nock from 'nock'
 import fs from 'fs'
+import { UploadLogger } from '../src/helpers/logger'
 
 // Backup the env
 const realEnv = { ...process.env }
 
 describe('Uploader Core', () => {
   const env = process.env
 
+  UploadLogger.setLogLevel('verbose')
+
   beforeEach(() => {
     // https://bensmithgall.com/blog/jest-mock-trick if this works!
     const mockExit = jest.fn()