diff --git a/README.md b/README.md index 962b2d68..b71a06ea 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,11 @@ In this repository you'll find: - [Configuring experiments](#configuring-experiments) - [Configuring assignees and reviewers](#configuring-assignees-and-reviewers) - [Unsupported features and configurations](#unsupported-features-and-configurations) - * [Extension Task](#extension-task) + * [Dependabot Task](#dependabot-task) + [dependabot@V2](#dependabotv2) + [dependabot@V1](#dependabotv1) - * [Updater Docker image](#updater-docker-image) - * [Server](#server) + * [Dependabot Updater Docker image](#dependabot-updater-docker-image) + * [Dependabot Server](#dependabot-server) - [Migration Guide](#migration-guide) - [Contributing](#contributing) * [Reporting issues and feature requests](#reporting-issues-and-feature-requests) @@ -194,13 +194,13 @@ Reviewers can be any of the following values: - Team name ## Unsupported features and configurations -We aim to support all [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file), but there are some limitations for: +We aim to support all [official configuration options](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file), but there are some limitations: -### Extension Task +### Dependabot Task #### `dependabot@V2` - [`schedule`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) is ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. -- [Security-only updates](https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#overriding-the-default-behavior-with-a-configuration-file) (`open-pull-requests-limit: 0`) are not supported. _(coming soon)_ +- [`securityAdvisoriesFile`](#configuring-security-advisories-and-known-vulnerabilities) task input is not yet supported. #### `dependabot@V1` - [`schedule`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#scheduleinterval) is ignored, use [pipeline scheduled triggers](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml#scheduled-triggers) instead. @@ -210,11 +210,11 @@ We aim to support all [official configuration options](https://docs.github.com/e - [`assignees`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#assignees) and [`reviewers`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#reviewers) must be a list of user guids or email addresses; group/team names are not supported. - Private feed/registry authentication may not work with all package ecyosystems. Support is _slightly_ improved when task input `useUpdateScriptVNext: true` is set, but not still not fully supported. See [problems with authentication](https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317) for more. -### Updater Docker image +### Dependabot Updater Docker Image - `DEPENDABOT_ASSIGNEES` and `DEPENDABOT_REVIEWERS` must be a list of user guids; email addresses and group/team names are not supported. - Private feed/registry authentication may not work with all package ecyosystems. See [problems with authentication](https://github.com/tinglesoftware/dependabot-azure-devops/discussions/1317) for more. -### Server +### Dependabot Server - [`directories`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#directories) are not supported. - [`groups`](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#groups) are not supported. diff --git a/docs/migrations/v1-to-v2.md b/docs/migrations/v1-to-v2.md index e28a8e67..620e7242 100644 --- a/docs/migrations/v1-to-v2.md +++ b/docs/migrations/v1-to-v2.md @@ -20,15 +20,16 @@ The task now uses [Dependabot CLI](https://github.com/dependabot/cli) to perform > [!WARNING] > **It is strongly recommended that you complete (or abandon) all active Depedabot pull requests created in V1 before migrating to V2.** Due to changes in Dependabot dependency metadata, V2 pull requests are not compatible with V1 (and vice versa). Migrating to V2 before completing existing pull requests will lead to duplication of pull requests. +### Security-only updates +Security-only updates (i.e. `open-pull-requests-limit: 0`) incur a slight performance overhead due to limitations in Dependabot CLI, detailed in [dependabot/cli#360](https://github.com/dependabot/cli/issues/360). To work around this, vulnerable dependencies will first be discovered using an "ignore everything" update job; After which, security advisories for the discovered dependencies will be checked against the [GitHub Advisory Database](https://github.com/advisories) before finally performing the requested security-only update job. + +Currently the [`securityAdvisoriesFile`](../../README.md#configuring-security-advisories-and-known-vulnerabilities) task input is not supported, but is expected to be supported in the near future. + ### New pipeline agent requirements; "Go" must be installed Dependabot CLI requires [Go](https://go.dev/doc/install) (1.22+) and [Docker](https://docs.docker.com/engine/install/) (with Linux containers). If you use [Microsoft-hosted agents](https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#software), we recommend using the [ubuntu-latest](https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md) image, which meets all task requirements. For self-hosted agents, you will need to install Go 1.22+. -### Security-only updates and "fixed vulnerabilities" are not implemented (yet) -Using configuration `open-pull-requests-limit: 0` will cause a "not implemented" error. This is [current limitation of V2](../../README.md#unsupported-features-and-configurations). A solution is still under development and is expected to be resolved before general availability. -See: https://github.com/dependabot/cli/issues/360 for more technical details. - ### Task Input `updaterOptions` has been renamed to `experiments` Renamed to match Dependabot Core/CLI terminology. The input value remains unchanged. See [configuring experiments](../../README.md#configuring-experiments) for more details. @@ -70,13 +71,9 @@ The following environment variables have been removed entirely; the feature is n ## Todo before general availability Before removing the preview flag from V2 `task.json`, we need to: - - [x] Open an issue in Dependabot-CLI, enquire how security-advisories are expected to be provided **before** knowing the list of dependencies. (https://github.com/dependabot/cli/issues/360) - - [ ] Convert GitHub security advisory client in `vulnerabilities.rb` to TypeScript code - - [ ] Implement `security-advisories` config once the answer the above is known - - [x] Review `task.json`, add documentation for new V2 inputs - - [x] Update `\docs\extension.md` with V2 docs - - [x] Update `\extension\README.MD` with V2 docs - - [x] Update `\README.MD` with V2 docs - - [ ] Do a general code tidy-up pass (check all "TODO" comments) + - [ ] Add "superseded by X" close reason when PR is closed during a PR update + - [ ] Add documentation for required permissions and PAT scopes + - [ ] Add support for 'securityAdvisoriesFile' task input - [ ] Add unit tests for V2 utils scripts - - [ ] Investigate https://zod.dev/ \ No newline at end of file + - [ ] General code tidy-up (check all "TODO" comments) + - [ ] Investigate https://zod.dev/ diff --git a/extension/README.md b/extension/README.md index 4b744019..2fa046a8 100644 --- a/extension/README.md +++ b/extension/README.md @@ -64,7 +64,7 @@ Dependabot uses Docker containers, which may take time to install if not already |azureDevOpsAccessToken|**_Optional_**. The Personal Access Token for accessing Azure DevOps. Supply a value here to avoid using permissions for the Build Service either because you cannot change its permissions or because you prefer that the Pull Requests be done by a different user. When not provided, the current authentication scope is used. In either case, be use the following permissions are granted:
- Code (Full)
- Pull Requests Threads (Read & Write).
See the [documentation](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page#create-a-pat) to know more about creating a Personal Access Token.
Use this in place of `azureDevOpsServiceConnection` such as when it is not possible to create a service connection.| |gitHubConnection|**_Optional_**. The GitHub service connection for authenticating requests against GitHub repositories. This is useful to avoid rate limiting errors. The token must include permissions to read public repositories. See the [GitHub docs](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) for more on Personal Access Tokens and [Azure DevOps docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/library/service-endpoints?view=azure-devops&tabs=yaml#sep-github) for the GitHub service connection.| |gitHubAccessToken|**_Optional_**. The raw GitHub PAT for authenticating requests against GitHub repositories. Use this in place of `gitHubConnection` such as when it is not possible to create a service connection.| -|storeDependencyList|**_Optional_**. Determines if the last know dependency list information should be stored in the parent DevOps project properties. If enabled, the authenticated user must have the "Project & Team (Write)" permission for the project. Enabling this option improves performance when doing security-only updates. Defaults to `false`.| +|storeDependencyList|**_Optional_**. Determines if the last know dependency list information should be stored in the parent DevOps project properties. If enabled, the authenticated user must have the "Project & Team (Write)" permission for the project. Defaults to `false`.| |targetRepositoryName|**_Optional_**. The name of the repository to target for processing. If this value is not supplied then the Build Repository Name is used. Supplying this value allows creation of a single pipeline that runs Dependabot against multiple repositories by running a `dependabot` task for each repository to update.| |targetUpdateIds|**_Optional_**. A semicolon (`;`) delimited list of update identifiers run. Index are zero-based and in the order written in the configuration file. When not present, all the updates are run. This is meant to be used in scenarios where you want to run updates a different times from the same configuration file given you cannot schedule them independently in the pipeline.| |experiments|**_Optional_**. Comma separated list of Dependabot experiments; available options depend on the ecosystem. Example: `tidy=true,vendor=true,goprivate=*`. See: [Configuring experiments](https://github.com/tinglesoftware/dependabot-azure-devops/#configuring-experiments)| diff --git a/extension/package-lock.json b/extension/package-lock.json index e284c4d2..0c661a7a 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -12,13 +12,15 @@ "axios": "1.7.8", "azure-devops-node-api": "14.1.0", "azure-pipelines-task-lib": "4.17.3", - "js-yaml": "4.1.0" + "js-yaml": "4.1.0", + "semver": "7.6.3" }, "devDependencies": { "@types/jest": "29.5.14", "@types/js-yaml": "4.0.9", "@types/node": "22.10.0", "@types/q": "1.5.8", + "@types/semver": "7.5.8", "jest": "29.7.0", "ts-jest": "29.2.5", "ts-node": "10.9.2", @@ -1163,6 +1165,12 @@ "integrity": "sha512-hroOstUScF6zhIi+5+x0dzqrHA1EJi+Irri6b1fxolMTqqHIV/Cg77EtnQcZqZCu8hR3mX2BzIxN4/GzI68Kfw==", "dev": true }, + "node_modules/@types/semver": { + "version": "7.5.8", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", + "integrity": "sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==", + "dev": true + }, "node_modules/@types/stack-utils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", @@ -1341,6 +1349,14 @@ "uuid": "^3.0.1" } }, + "node_modules/azure-pipelines-task-lib/node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "bin": { + "semver": "bin/semver" + } + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -2475,18 +2491,6 @@ "node": ">=10" } }, - "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", - "dev": true, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/istanbul-lib-report": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", @@ -3020,18 +3024,6 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, - "node_modules/jest-snapshot/node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", - "dev": true, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/jest-util": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", @@ -3246,18 +3238,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/make-dir/node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", - "dev": true, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -3732,11 +3712,14 @@ } }, "node_modules/semver": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "bin": { - "semver": "bin/semver" + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" } }, "node_modules/set-function-length": { @@ -4059,19 +4042,6 @@ } } }, - "node_modules/ts-jest/node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/ts-node": { "version": "10.9.2", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", diff --git a/extension/package.json b/extension/package.json index f3ad99d8..17213e26 100644 --- a/extension/package.json +++ b/extension/package.json @@ -31,13 +31,15 @@ "axios": "1.7.8", "azure-devops-node-api": "14.1.0", "azure-pipelines-task-lib": "4.17.3", - "js-yaml": "4.1.0" + "js-yaml": "4.1.0", + "semver": "7.6.3" }, "devDependencies": { "@types/jest": "29.5.14", "@types/js-yaml": "4.0.9", "@types/node": "22.10.0", "@types/q": "1.5.8", + "@types/semver": "7.5.8", "jest": "29.7.0", "ts-jest": "29.2.5", "ts-node": "10.9.2", diff --git a/extension/tasks/dependabotV2/index.ts b/extension/tasks/dependabotV2/index.ts index c809197b..868f5a1a 100644 --- a/extension/tasks/dependabotV2/index.ts +++ b/extension/tasks/dependabotV2/index.ts @@ -1,17 +1,20 @@ import { debug, error, setResult, TaskResult, warning, which } from 'azure-pipelines-task-lib/task'; import { AzureDevOpsWebApiClient } from './utils/azure-devops/AzureDevOpsWebApiClient'; -import { setSecrets } from './utils/azure-devops/formattingCommands'; +import { section, setSecrets } from './utils/azure-devops/formattingCommands'; import { DependabotCli } from './utils/dependabot-cli/DependabotCli'; import { DependabotJobBuilder } from './utils/dependabot-cli/DependabotJobBuilder'; import { DependabotOutputProcessor, - parseProjectDependencyListProperty, parsePullRequestProperties, } from './utils/dependabot-cli/DependabotOutputProcessor'; import { IDependabotUpdateOperationResult } from './utils/dependabot-cli/interfaces/IDependabotUpdateOperationResult'; import { IDependabotUpdate } from './utils/dependabot/interfaces/IDependabotConfig'; import parseDependabotConfigFile from './utils/dependabot/parseConfigFile'; import parseTaskInputConfiguration from './utils/getSharedVariables'; +import { GitHubGraphClient } from './utils/github/GitHubGraphClient'; +import { IPackage } from './utils/github/IPackage'; +import { ISecurityVulnerability } from './utils/github/ISecurityVulnerability'; +import { getGhsaPackageEcosystemFromDependabotPackageEcosystem } from './utils/github/PackageEcosystem'; async function run() { let dependabot: DependabotCli = undefined; @@ -51,6 +54,14 @@ async function run() { throw new Error('Failed to parse dependabot.yaml configuration file from the target repository'); } + // Print a warning about the required workarounds for security-only updates, if any update is configured as such + // TODO: If and when Dependabot supports a better way to do security-only updates, remove this. + if (dependabotConfig.updates?.some((u) => u['open-pull-requests-limit'] === 0)) { + warning( + 'Security-only updates incur a slight performance overhead due to limitations in Dependabot CLI. For more info, see: https://github.com/tinglesoftware/dependabot-azure-devops/blob/main/docs/migrations/v1-to-v2.md#security-only-updates', + ); + } + // Initialise the DevOps API clients // There are two clients; one for authoring pull requests and one for auto-approving pull requests (if configured) const prAuthorClient = new AzureDevOpsWebApiClient( @@ -116,15 +127,6 @@ async function run() { const updateId = updates.indexOf(update).toString(); const packageEcosystem = update['package-ecosystem']; - // Parse the last dependency list snapshot (if any) from the project properties. - // This is required when doing a security-only update as dependabot requires the list of vulnerable dependencies to be updated. - // Automatic discovery of vulnerable dependencies during a security-only update is not currently supported by dependabot-updater. - const dependencyList = parseProjectDependencyListProperty( - await prAuthorClient.getProjectProperties(taskInputs.projectId), - taskInputs.repository, - packageEcosystem, - ); - // Parse the Dependabot metadata for the existing pull requests that are related to this update // Dependabot will use this to determine if we need to create new pull requests or update/close existing ones const existingPullRequestsForPackageEcosystem = parsePullRequestProperties( @@ -135,25 +137,73 @@ async function run() { existingPullRequestsForPackageEcosystem, ).map(([id, deps]) => deps); + // If this is a security-only update (i.e. 'open-pull-requests-limit: 0'), then we first need to discover the dependencies + // that need updating and check each one for vulnerabilities. This is because Dependabot requires the list of vulnerable dependencies + // to be supplied in the job definition of security-only update job, it will not automatically discover them like a versioned update does. + // https://docs.github.com/en/code-security/dependabot/dependabot-security-updates/configuring-dependabot-security-updates#overriding-the-default-behavior-with-a-configuration-file + let securityVulnerabilities: ISecurityVulnerability[] = []; + let dependencyNamesToUpdate: string[] = []; + const securityUpdatesOnly = update['open-pull-requests-limit'] === 0; + if (securityUpdatesOnly) { + // Run an update job to discover all dependencies + const discoveredDependencyListOutputs = await dependabot.update( + DependabotJobBuilder.listAllDependenciesJob(taskInputs, updateId, update, dependabotConfig.registries), + dependabotUpdaterOptions, + ); + + // Get the list of vulnerabilities that apply to the discovered dependencies + section(`GHSA dependency vulnerability check`); + const ghsaClient = new GitHubGraphClient(taskInputs.githubAccessToken); + const packagesToCheckForVulnerabilities: IPackage[] = discoveredDependencyListOutputs + ?.find((x) => x.output.type == 'update_dependency_list') + ?.output?.data?.dependencies?.map((d) => ({ name: d.name, version: d.version })); + if (packagesToCheckForVulnerabilities?.length) { + console.info( + `Detected ${packagesToCheckForVulnerabilities.length} dependencies; Checking for vulnerabilities...`, + ); + securityVulnerabilities = await ghsaClient.getSecurityVulnerabilitiesAsync( + getGhsaPackageEcosystemFromDependabotPackageEcosystem(packageEcosystem), + packagesToCheckForVulnerabilities || [], + ); + + // Only update dependencies that have vulnerabilities + dependencyNamesToUpdate = Array.from(new Set(securityVulnerabilities.map((v) => v.package.name))); + console.info( + `Detected ${securityVulnerabilities.length} vulnerabilities affecting ${dependencyNamesToUpdate.length} dependencies`, + ); + if (dependencyNamesToUpdate.length) { + console.log(dependencyNamesToUpdate); + } + } else { + console.info('No vulnerabilities detected in any dependencies'); + } + } + // Run an update job for "all dependencies"; this will create new pull requests for dependencies that need updating const openPullRequestsLimit = update['open-pull-requests-limit']; const openPullRequestsCount = Object.entries(existingPullRequestsForPackageEcosystem).length; const hasReachedOpenPullRequestLimit = openPullRequestsLimit > 0 && openPullRequestsCount >= openPullRequestsLimit; if (!hasReachedOpenPullRequestLimit) { - failedTasks += handleUpdateOperationResults( - await dependabot.update( - DependabotJobBuilder.newUpdateAllJob( - taskInputs, - updateId, - update, - dependabotConfig.registries, - dependencyList?.['dependencies'], - existingPullRequestDependenciesForPackageEcosystem, + const dependenciesHaveVulnerabilities = dependencyNamesToUpdate.length && securityVulnerabilities.length; + if (!securityUpdatesOnly || dependenciesHaveVulnerabilities) { + failedTasks += handleUpdateOperationResults( + await dependabot.update( + DependabotJobBuilder.updateAllDependenciesJob( + taskInputs, + updateId, + update, + dependabotConfig.registries, + dependencyNamesToUpdate, + existingPullRequestDependenciesForPackageEcosystem, + securityVulnerabilities, + ), + dependabotUpdaterOptions, ), - dependabotUpdaterOptions, - ), - ); + ); + } else { + console.info('Nothing to update; dependencies are not affected by any known vulnerability'); + } } else { warning( `Skipping update for ${packageEcosystem} packages as the open pull requests limit (${openPullRequestsLimit}) has already been reached`, @@ -167,13 +217,14 @@ async function run() { for (const pullRequestId in existingPullRequestsForPackageEcosystem) { failedTasks += handleUpdateOperationResults( await dependabot.update( - DependabotJobBuilder.newUpdatePullRequestJob( + DependabotJobBuilder.updatePullRequestJob( taskInputs, pullRequestId, update, dependabotConfig.registries, existingPullRequestDependenciesForPackageEcosystem, existingPullRequestsForPackageEcosystem[pullRequestId], + securityVulnerabilities, ), dependabotUpdaterOptions, ), diff --git a/extension/tasks/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts b/extension/tasks/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts index e310986a..e9d250bc 100644 --- a/extension/tasks/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts +++ b/extension/tasks/dependabotV2/utils/azure-devops/AzureDevOpsWebApiClient.ts @@ -418,7 +418,7 @@ export class AzureDevOpsWebApiClient { comment: pullRequest.mergeStatus === PullRequestAsyncStatus.Conflicts ? 'Resolve merge conflicts' - : `Rebase with '${targetBranchName}'`, + : `Rebase '${sourceBranchName}' onto '${targetBranchName}'`, author: pr.author, changes: pr.changes.map((change) => { return { diff --git a/extension/tasks/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts b/extension/tasks/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts index b4da7a6a..b6ff1dbc 100644 --- a/extension/tasks/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts +++ b/extension/tasks/dependabotV2/utils/dependabot-cli/DependabotJobBuilder.ts @@ -1,4 +1,3 @@ -import { warning } from 'azure-pipelines-task-lib'; import { IDependabotAllowCondition, IDependabotGroup, @@ -6,32 +5,61 @@ import { IDependabotUpdate, } from '../dependabot/interfaces/IDependabotConfig'; import { ISharedVariables } from '../getSharedVariables'; +import { ISecurityVulnerability } from '../github/ISecurityVulnerability'; import { IDependabotUpdateOperation } from './interfaces/IDependabotUpdateOperation'; /** * Wrapper class for building dependabot update job objects */ export class DependabotJobBuilder { + /** + * Create a dependabot update job that updates nothing, but will discover the dependency list for a package ecyosystem + * @param taskInputs + * @param update + * @param registries + * @returns + */ + public static listAllDependenciesJob( + taskInputs: ISharedVariables, + id: string, + update: IDependabotUpdate, + registries: Record, + ): IDependabotUpdateOperation { + return { + config: update, + job: { + 'id': `discover-${id}-${update['package-ecosystem']}-dependency-list`, + 'package-manager': update['package-ecosystem'], + 'ignore-conditions': [{ 'dependency-name': '*' }], + 'source': mapSourceFromDependabotConfigToJobConfig(taskInputs, update), + 'experiments': taskInputs.experiments, + 'debug': taskInputs.debug, + }, + credentials: mapRegistryCredentialsFromDependabotConfigToJobConfig(taskInputs, registries), + }; + } + /** * Create a dependabot update job that updates all dependencies for a package ecyosystem * @param taskInputs * @param update * @param registries - * @param dependencyList + * @param dependencyNamesToUpdate * @param existingPullRequests + * @param securityVulnerabilities * @returns */ - public static newUpdateAllJob( + public static updateAllDependenciesJob( taskInputs: ISharedVariables, id: string, update: IDependabotUpdate, registries: Record, - dependencyList: any[], - existingPullRequests: any[], + dependencyNamesToUpdate?: string[], + existingPullRequests?: any[], + securityVulnerabilities?: ISecurityVulnerability[], ): IDependabotUpdateOperation { const packageEcosystem = update['package-ecosystem']; const securityUpdatesOnly = update['open-pull-requests-limit'] == 0; - const updateDependencyNames = securityUpdatesOnly ? mapDependenciesForSecurityUpdate(dependencyList) : undefined; return buildUpdateJobConfig( `update-${id}-${packageEcosystem}-${securityUpdatesOnly ? 'security-only' : 'all'}`, taskInputs, @@ -39,8 +67,11 @@ export class DependabotJobBuilder { registries, false, undefined, - updateDependencyNames, + securityUpdatesOnly + ? dependencyNamesToUpdate?.filter((d) => securityVulnerabilities?.find((v) => v.package.name == d)) + : dependencyNamesToUpdate, existingPullRequests, + securityVulnerabilities, ); } @@ -51,18 +82,20 @@ export class DependabotJobBuilder { * @param registries * @param existingPullRequests * @param pullRequestToUpdate + * @param securityVulnerabilities * @returns */ - public static newUpdatePullRequestJob( + public static updatePullRequestJob( taskInputs: ISharedVariables, id: string, update: IDependabotUpdate, registries: Record, existingPullRequests: any[], pullRequestToUpdate: any, + securityVulnerabilities?: ISecurityVulnerability[], ): IDependabotUpdateOperation { const dependencyGroupName = pullRequestToUpdate['dependency-group-name']; - const dependencies = (dependencyGroupName ? pullRequestToUpdate['dependencies'] : pullRequestToUpdate)?.map( + const dependencyNames = (dependencyGroupName ? pullRequestToUpdate['dependencies'] : pullRequestToUpdate)?.map( (d) => d['dependency-name'], ); return buildUpdateJobConfig( @@ -72,8 +105,9 @@ export class DependabotJobBuilder { registries, true, dependencyGroupName, - dependencies, + dependencyNames, existingPullRequests, + securityVulnerabilities?.filter((v) => dependencyNames.includes(v.package.name)), ); } } @@ -83,10 +117,11 @@ function buildUpdateJobConfig( taskInputs: ISharedVariables, update: IDependabotUpdate, registries: Record, - updatingPullRequest: boolean, - updateDependencyGroupName: string | undefined, - updateDependencyNames: string[] | undefined, - existingPullRequests: any[], + updatingPullRequest?: boolean | undefined, + updateDependencyGroupName?: string | undefined, + updateDependencyNames?: string[] | undefined, + existingPullRequests?: any[], + securityVulnerabilities?: ISecurityVulnerability[], ) { return { config: update, @@ -94,26 +129,17 @@ function buildUpdateJobConfig( 'id': id, 'package-manager': update['package-ecosystem'], 'update-subdependencies': true, // TODO: add config for this? - 'updating-a-pull-request': updatingPullRequest, + 'updating-a-pull-request': updatingPullRequest || false, 'dependency-group-to-refresh': updateDependencyGroupName, 'dependency-groups': mapGroupsFromDependabotConfigToJobConfig(update.groups), 'dependencies': updateDependencyNames, 'allowed-updates': mapAllowedUpdatesFromDependabotConfigToJobConfig(update.allow), 'ignore-conditions': mapIgnoreConditionsFromDependabotConfigToJobConfig(update.ignore), 'security-updates-only': update['open-pull-requests-limit'] == 0, - 'security-advisories': [], // TODO: add config for this! - 'source': { - 'provider': 'azure', - 'api-endpoint': taskInputs.apiEndpointUrl, - 'hostname': taskInputs.hostname, - 'repo': `${taskInputs.organization}/${taskInputs.project}/_git/${taskInputs.repository}`, - 'branch': update['target-branch'], - 'commit': undefined, // use latest commit of target branch - 'directory': update.directory, - 'directories': update.directories, - }, - 'existing-pull-requests': existingPullRequests.filter((pr) => !pr['dependency-group-name']), - 'existing-group-pull-requests': existingPullRequests.filter((pr) => pr['dependency-group-name']), + 'security-advisories': mapSecurityAdvisories(securityVulnerabilities), + 'source': mapSourceFromDependabotConfigToJobConfig(taskInputs, update), + 'existing-pull-requests': existingPullRequests?.filter((pr) => !pr['dependency-group-name']), + 'existing-group-pull-requests': existingPullRequests?.filter((pr) => pr['dependency-group-name']), 'commit-message-options': update['commit-message'] === undefined ? undefined @@ -136,25 +162,17 @@ function buildUpdateJobConfig( }; } -function mapDependenciesForSecurityUpdate(dependencyList: any[]): string[] { - if (!dependencyList || dependencyList.length == 0) { - // This happens when no previous dependency list snapshot exists yet; - // TODO: Find a way to discover dependencies for a first-time security-only update (no existing dependency list snapshot). - // It would be nice if we could use dependabot-cli for this (e.g. `dependabot --discover-only`), but this is not supported currently. - // TODO: Open a issue in dependabot-cli project, ask how we should handle this scenario. - warning( - 'Security updates can only be performed if there is a previous dependency list snapshot available, but there is none as you have not completed a successful update job yet. ' + - 'Dependabot does not currently support discovering vulnerable dependencies during security-only updates and it is likely that this update operation will fail.', - ); - - // Attempt to do a security update for "all dependencies"; it will probably fail this is not supported in dependabot-updater yet, but it is best we can do... - return []; - } - - // Return only dependencies that are vulnerable, ignore the rest - const dependencyNames = dependencyList.map((dependency) => dependency['name']); - const dependencyVulnerabilities = {}; // TODO: getGitHubSecurityAdvisoriesForDependencies(dependencyNames); - return dependencyNames.filter((dependency) => dependencyVulnerabilities[dependency]?.length > 0); +function mapSourceFromDependabotConfigToJobConfig(taskInputs: ISharedVariables, update: IDependabotUpdate): any { + return { + 'provider': 'azure', + 'api-endpoint': taskInputs.apiEndpointUrl, + 'hostname': taskInputs.hostname, + 'repo': `${taskInputs.organization}/${taskInputs.project}/_git/${taskInputs.repository}`, + 'branch': update['target-branch'], + 'commit': undefined, // use latest commit of target branch + 'directory': update.directory, + 'directories': update.directories, + }; } function mapGroupsFromDependabotConfigToJobConfig(dependencyGroups: Record): any[] { @@ -205,6 +223,31 @@ function mapIgnoreConditionsFromDependabotConfigToJobConfig(ignoreConditions: ID }); } +function mapSecurityAdvisories(securityVulnerabilities: ISecurityVulnerability[]): any[] { + if (!securityVulnerabilities) { + return undefined; + } + + // A single security advisory can cause a vulnerability in multiple versions of a package. + // We need to map each unique security advisory to a list of affected-versions and patched-versions. + const vulnerabilitiesGroupedByPackageNameAndAdvisoryId = new Map(); + for (const vuln of securityVulnerabilities) { + const key = `${vuln.package.name}/${vuln.advisory.identifiers.map((i) => `${i.type}:${i.value}`).join('/')}`; + if (!vulnerabilitiesGroupedByPackageNameAndAdvisoryId.has(key)) { + vulnerabilitiesGroupedByPackageNameAndAdvisoryId.set(key, []); + } + vulnerabilitiesGroupedByPackageNameAndAdvisoryId.get(key).push(vuln); + } + return Array.from(vulnerabilitiesGroupedByPackageNameAndAdvisoryId.values()).map((vulns) => { + return { + 'dependency-name': vulns[0].package.name, + 'affected-versions': vulns.map((v) => v.vulnerableVersionRange).filter((v) => v && v.length > 0), + 'patched-versions': vulns.map((v) => v.firstPatchedVersion).filter((v) => v && v.length > 0), + 'unaffected-versions': [], + }; + }); +} + function mapVersionStrategyToRequirementsUpdateStrategy(versioningStrategy: string): string | undefined { if (!versioningStrategy) { return undefined; diff --git a/extension/tasks/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts b/extension/tasks/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts index fa8b18e8..81bdbb5e 100644 --- a/extension/tasks/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts +++ b/extension/tasks/dependabotV2/utils/dependabot-cli/interfaces/IDependabotUpdateJobConfig.ts @@ -34,7 +34,7 @@ export interface IDependabotUpdateJobConfig { 'updated-at'?: string; 'version-requirement'?: string; }[]; - 'security-updates-only': boolean; + 'security-updates-only'?: boolean; 'security-advisories'?: { 'dependency-name': string; 'affected-versions': string[]; diff --git a/extension/tasks/dependabotV2/utils/github/GitHubGraphClient.ts b/extension/tasks/dependabotV2/utils/github/GitHubGraphClient.ts new file mode 100644 index 00000000..d669bd54 --- /dev/null +++ b/extension/tasks/dependabotV2/utils/github/GitHubGraphClient.ts @@ -0,0 +1,204 @@ +import axios from 'axios'; +import * as semver from 'semver'; + +import { warning } from 'azure-pipelines-task-lib/task'; + +import { IPackage } from './IPackage'; +import { ISecurityVulnerability } from './ISecurityVulnerability'; +import { PackageEcosystem } from './PackageEcosystem'; + +const GHSA_GRAPHQL_API = 'https://api.github.com/graphql'; + +const GHSA_SECURITY_VULNERABILITIES_QUERY = ` + query($ecosystem: SecurityAdvisoryEcosystem, $package: String) { + securityVulnerabilities(first: 100, ecosystem: $ecosystem, package: $package) { + nodes { + advisory { + identifiers { + type, + value + }, + severity, + summary, + description, + references { + url + } + cvss { + score + vectorString + } + epss { + percentage + percentile + } + cwes (first: 100) { + nodes { + cweId + name + description + } + } + publishedAt + updatedAt + withdrawnAt + permalink + } + vulnerableVersionRange + firstPatchedVersion { + identifier + } + } + } + } +`; + +/** + * GitHub GraphQL client + */ +export class GitHubGraphClient { + private readonly accessToken: string; + + constructor(accessToken: string) { + this.accessToken = accessToken; + } + + /** + * Get the list of security vulnerabilities for a given package ecosystem and list of packages + * @param packageEcosystem + * @param packages + */ + public async getSecurityVulnerabilitiesAsync( + packageEcosystem: PackageEcosystem, + packages: IPackage[], + ): Promise { + // GitHub API doesn't support querying multiple package at once, so we need to make a request for each package individually. + // To speed up the process, we can make the requests in parallel, 100 at a time. We batch the requests to avoid hitting the rate limit too quickly. + // https://docs.github.com/en/graphql/overview/rate-limits-and-node-limits-for-the-graphql-api + const securityVulnerabilities = await this.batchGraphQueryAsync( + 100, + packages, + async (pkg) => { + const variables = { + ecosystem: packageEcosystem, + package: pkg.name, + }; + const response = await axios.post( + GHSA_GRAPHQL_API, + JSON.stringify({ + query: GHSA_SECURITY_VULNERABILITIES_QUERY, + variables: variables, + }), + { + headers: { + 'Authorization': `Bearer ${this.accessToken}`, + 'Content-Type': 'application/json', + }, + }, + ); + if (response.status < 200 || response.status > 299) { + throw new Error(`GHSA GraphQL request failed with response: ${response.status} ${response.statusText}`); + } + const errors = response.data?.errors; + if (errors) { + throw new Error(`GHSA GraphQL request failed with errors: ${JSON.stringify(errors)}`); + } + + const vulnerabilities = response.data?.data?.securityVulnerabilities?.nodes; + return vulnerabilities + ?.filter((v: any) => v?.advisory) + ?.map((v: any) => { + return { + ecosystem: packageEcosystem, + package: pkg, + advisory: { + identifiers: v.advisory.identifiers?.map((i: any) => { + return { + type: i.type, + value: i.value, + }; + }), + severity: v.advisory.severity, + summary: v.advisory.summary, + description: v.advisory.description, + references: v.advisory.references?.map((r: any) => r.url), + cvss: !v.advisory.cvss + ? undefined + : { + score: v.advisory.cvss.score, + vectorString: v.advisory.cvss.vectorString, + }, + epss: !v.advisory.epss + ? undefined + : { + percentage: v.advisory.epss.percentage, + percentile: v.advisory.epss.percentile, + }, + cwes: v.advisory.cwes?.nodes?.map((c: any) => { + return { + id: c.cweId, + name: c.name, + description: c.description, + }; + }), + publishedAt: v.advisory.publishedAt, + updatedAt: v.advisory.updatedAt, + withdrawnAt: v.advisory.withdrawnAt, + permalink: v.advisory.permalink, + }, + vulnerableVersionRange: v.vulnerableVersionRange, + firstPatchedVersion: v.firstPatchedVersion?.identifier, + }; + }); + }, + ); + + // Filter out vulnerabilities that have been withdrawn or that are not relevant the current version of the package + const affectedVulnerabilities = securityVulnerabilities + .filter((v) => !v.advisory.withdrawnAt) + .filter((v) => { + const pkg = v.package; + if (!pkg || !pkg.version || !v.vulnerableVersionRange) { + return false; + } + + /** + * The vulnerable version range follows a basic syntax with a few forms: + * `= 0.2.0` denotes a single vulnerable version + * `<= 1.0.8` denotes a version range up to and including the specified version + * `< 0.1.11` denotes a version range up to, but excluding, the specified version + * `>= 4.3.0, < 4.3.5` denotes a version range with a known minimum and maximum version + * `>= 0.0.1` denotes a version range with a known minimum, but no known maximum + */ + const versionRangeRequirements = v.vulnerableVersionRange.split(',').map((v) => v.trim()); + return versionRangeRequirements.every((r) => pkg.version && semver.satisfies(pkg.version, r)); + }); + + return affectedVulnerabilities; + } + + /** + * Batch requests in parallel to speed up the process when we are forced to do a N+1 query + * @param batchSize + * @param items + * @param action + * @returns + */ + private async batchGraphQueryAsync(batchSize: number, items: T1[], action: (item: T1) => Promise) { + const results: T2[] = []; + for (let i = 0; i < items.length; i += batchSize) { + const batch = items.slice(i, i + batchSize); + if (batch?.length) { + try { + const batchResults = await Promise.all(batch.map(action)); + if (batchResults?.length) { + results.push(...batchResults.flat()); + } + } catch (error) { + warning(`Request batch [${i}-${i + batchSize}] failed; The data may be incomplete. ${error}`); + } + } + } + return results; + } +} diff --git a/extension/tasks/dependabotV2/utils/github/IPackage.ts b/extension/tasks/dependabotV2/utils/github/IPackage.ts new file mode 100644 index 00000000..af519517 --- /dev/null +++ b/extension/tasks/dependabotV2/utils/github/IPackage.ts @@ -0,0 +1,4 @@ +export interface IPackage { + name: string; + version?: string; +} diff --git a/extension/tasks/dependabotV2/utils/github/ISecurityAdvisory.ts b/extension/tasks/dependabotV2/utils/github/ISecurityAdvisory.ts new file mode 100644 index 00000000..96e0c43a --- /dev/null +++ b/extension/tasks/dependabotV2/utils/github/ISecurityAdvisory.ts @@ -0,0 +1,39 @@ +export interface ISecurityAdvisory { + identifiers: { + type: SecurityAdvisoryIdentifierType | string; + value: string; + }[]; + severity: SecurityAdvisorySeverity; + summary: string; + description: string; + references: string[]; + cvss: { + score: number; + vectorString: string; + }; + epss: { + percentage: number; + percentile: number; + }; + cwes: { + id: string; + name: string; + description: string; + }[]; + publishedAt: string; + updatedAt: string; + withdrawnAt: string; + permalink: string; +} + +export enum SecurityAdvisoryIdentifierType { + Cve = 'CVE', + Ghsa = 'GHSA', +} + +export enum SecurityAdvisorySeverity { + Low = 'LOW', + Moderate = 'MODERATE', + High = 'HIGH', + Critical = 'CRITICAL', +} diff --git a/extension/tasks/dependabotV2/utils/github/ISecurityVulnerability.ts b/extension/tasks/dependabotV2/utils/github/ISecurityVulnerability.ts new file mode 100644 index 00000000..e6640425 --- /dev/null +++ b/extension/tasks/dependabotV2/utils/github/ISecurityVulnerability.ts @@ -0,0 +1,11 @@ +import { IPackage } from './IPackage'; +import { ISecurityAdvisory } from './ISecurityAdvisory'; +import { PackageEcosystem } from './PackageEcosystem'; + +export interface ISecurityVulnerability { + ecosystem: PackageEcosystem; + package: IPackage; + advisory: ISecurityAdvisory; + vulnerableVersionRange: string; + firstPatchedVersion: string; +} diff --git a/extension/tasks/dependabotV2/utils/github/PackageEcosystem.ts b/extension/tasks/dependabotV2/utils/github/PackageEcosystem.ts new file mode 100644 index 00000000..8b129543 --- /dev/null +++ b/extension/tasks/dependabotV2/utils/github/PackageEcosystem.ts @@ -0,0 +1,47 @@ +export enum PackageEcosystem { + Composer = 'COMPOSER', + Erlang = 'ERLANG', + Actions = 'ACTIONS', + Go = 'GO', + Maven = 'MAVEN', + Npm = 'NPM', + Nuget = 'NUGET', + Pip = 'PIP', + Pub = 'PUB', + Rubygems = 'RUBYGEMS', + Rust = 'RUST', + Swift = 'SWIFT', +} + +export function getGhsaPackageEcosystemFromDependabotPackageEcosystem( + dependabotPackageEcosystem: string, +): PackageEcosystem { + switch (dependabotPackageEcosystem) { + case 'composer': + return PackageEcosystem.Composer; + case 'elm': + return PackageEcosystem.Erlang; + case 'github_actions': + return PackageEcosystem.Actions; + case 'go_modules': + return PackageEcosystem.Go; + case 'maven': + return PackageEcosystem.Maven; + case 'npm_and_yarn': + return PackageEcosystem.Npm; + case 'nuget': + return PackageEcosystem.Nuget; + case 'pip': + return PackageEcosystem.Pip; + case 'pub': + return PackageEcosystem.Pub; + case 'bundler': + return PackageEcosystem.Rubygems; + case 'cargo': + return PackageEcosystem.Rust; + case 'swift': + return PackageEcosystem.Swift; + default: + throw new Error(`Unknown dependabot package ecosystem: ${dependabotPackageEcosystem}`); + } +}