From 903bcbd5346754c60010c77ef7791a4981780f1a Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 26 Nov 2024 11:14:58 +0100 Subject: [PATCH 1/5] GRADLE: DROP THE -WEERROR COMPLIE OPTION Signed-off-by: Frank Viernau --- buildSrc/src/main/kotlin/ort-kotlin-conventions.gradle.kts | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/kotlin/ort-kotlin-conventions.gradle.kts b/buildSrc/src/main/kotlin/ort-kotlin-conventions.gradle.kts index 62353ce48c235..169b5b956445e 100644 --- a/buildSrc/src/main/kotlin/ort-kotlin-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/ort-kotlin-conventions.gradle.kts @@ -185,7 +185,7 @@ tasks.withType().configureEach { ) compilerOptions { - allWarningsAsErrors = true + allWarningsAsErrors = false freeCompilerArgs = listOf("-Xconsistent-data-class-copy-visibility") jvmTarget = maxKotlinJvmTarget optIn = optInRequirements diff --git a/gradle.properties b/gradle.properties index 316d6a7ec186e..2196b6b1777f2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,7 +19,7 @@ org.gradle.caching = true org.gradle.configuration-cache = true org.gradle.configuration-cache.parallel = true org.gradle.jvmargs = -Xmx2g -XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 -org.gradle.kotlin.dsl.allWarningsAsErrors = true +org.gradle.kotlin.dsl.allWarningsAsErrors = false org.gradle.parallel = true kotlin.code.style = official From a243bbd1b9e53d002f3f4063bc95d1918b103774 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Fri, 13 Dec 2024 16:19:05 +0100 Subject: [PATCH 2/5] feat(advisor): Add BlackDuck as advisor Signed-off-by: Frank Viernau --- .../kotlin/ort-base-conventions.gradle.kts | 9 ++ gradle/libs.versions.toml | 4 + plugins/advisors/black-duck/build.gradle.kts | 56 ++++++++ ...rieve-package-findings-expected-result.yml | 29 ++++ .../src/funTest/kotlin/BlackDuckFunTest.kt | 96 ++++++++++++++ .../black-duck/src/main/kotlin/BlackDuck.kt | 123 +++++++++++++++++ .../src/main/kotlin/BlackDuckConfiguration.kt | 39 ++++++ .../main/kotlin/ExtendedComponentService.kt | 124 ++++++++++++++++++ .../black-duck/src/main/kotlin/Purl.kt | 52 ++++++++ 9 files changed, 532 insertions(+) create mode 100644 plugins/advisors/black-duck/build.gradle.kts create mode 100644 plugins/advisors/black-duck/src/funTest/assets/retrieve-package-findings-expected-result.yml create mode 100644 plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt create mode 100644 plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt create mode 100644 plugins/advisors/black-duck/src/main/kotlin/BlackDuckConfiguration.kt create mode 100644 plugins/advisors/black-duck/src/main/kotlin/ExtendedComponentService.kt create mode 100644 plugins/advisors/black-duck/src/main/kotlin/Purl.kt diff --git a/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts b/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts index fd9f955f3d454..d7fa66e581d22 100644 --- a/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/ort-base-conventions.gradle.kts @@ -45,6 +45,15 @@ repositories { includeGroup("org.gradle") } } + + maven { + // com.blackducksoftware.bdio:bdio2 + url = uri("https://sig-repo.synopsys.com/bds-bdio-release") + } + + maven { // com.blackducksoftware.magpie:magpie + url = uri("https://repo.blackduck.com/bds-integrations-release") + } } tasks.withType().configureEach { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9653102a0557d..f2eeb80511619 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,6 +15,8 @@ versionsPlugin = "0.51.0" aeSecurity = "0.132.0" asciidoctorj = "3.0.0" asciidoctorjPdf = "2.3.19" +blackduckCommon = "66.2.19" +blackduckCommonApi = "2023.10.0.4" clikt = "5.0.2" commonsCompress = "1.27.1" cyclonedx = "10.0.0" @@ -89,6 +91,8 @@ aeSecurity = { module = "org.metaeffekt.core:ae-security", version.ref = "aeSecu asciidoctorj = { module = "org.asciidoctor:asciidoctorj", version.ref = "asciidoctorj" } asciidoctorj-pdf = { module = "org.asciidoctor:asciidoctorj-pdf", version.ref = "asciidoctorjPdf" } awsS3 = { module = "software.amazon.awssdk:s3", version.ref = "s3" } +blackduck-common = { module = "com.synopsys.integration:blackduck-common", version.ref = "blackduckCommon" } +blackduck-common-api = { module = "com.synopsys.integration:blackduck-common-api", version.ref = "blackduckCommonApi" } clikt = { module = "com.github.ajalt.clikt:clikt", version.ref = "clikt" } commonsCompress = { module = "org.apache.commons:commons-compress", version.ref = "commonsCompress" } cyclonedx = { module = "org.cyclonedx:cyclonedx-core-java", version.ref = "cyclonedx" } diff --git a/plugins/advisors/black-duck/build.gradle.kts b/plugins/advisors/black-duck/build.gradle.kts new file mode 100644 index 0000000000000..a410b8663bfe4 --- /dev/null +++ b/plugins/advisors/black-duck/build.gradle.kts @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2020 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +plugins { + // Apply precompiled plugins. + id("ort-plugin-conventions") + + // Apply third-party plugins. + alias(libs.plugins.kotlinSerialization) +} + +repositories { + mavenCentral() + maven { + // com.blackducksoftware.bdio:bdio2 + url = uri("https://sig-repo.synopsys.com/bds-bdio-release") + } + + maven { // com.blackducksoftware.magpie:magpie + url = uri("https://repo.blackduck.com/bds-integrations-release") + } +} + +dependencies { + api(projects.advisor) + api(projects.model) + + implementation(projects.utils.ortUtils) + + implementation(libs.blackduck.common) + implementation(libs.blackduck.common.api) + implementation(libs.bundles.ks3) + implementation(libs.kotlinx.serialization.core) + implementation(libs.kotlinx.serialization.json) + + implementation(projects.utils.commonUtils) + implementation(projects.utils.ortUtils) + + ksp(projects.advisor) +} diff --git a/plugins/advisors/black-duck/src/funTest/assets/retrieve-package-findings-expected-result.yml b/plugins/advisors/black-duck/src/funTest/assets/retrieve-package-findings-expected-result.yml new file mode 100644 index 0000000000000..035fc6ef967b7 --- /dev/null +++ b/plugins/advisors/black-duck/src/funTest/assets/retrieve-package-findings-expected-result.yml @@ -0,0 +1,29 @@ +--- +Crate::sys-info:0.7.0: + advisor: + name: "BlackDuck" + capabilities: + - "VULNERABILITIES" + summary: + start_time: "1970-01-01T00:00:00Z" + end_time: "1970-01-01T00:00:00Z" + vulnerabilities: + - id: "CVE-2020-36434" + description: "An issue was discovered in the sys-info crate before 0.8.0 for Rust.\ + \ sys_info::disk_info calls can trigger a double free." + references: + - url: "https://zeiss.app.blackduck.com/api/vulnerabilities/CVE-2020-36434" + scoring_system: "CVSS:3.1" + severity: "CRITICAL" + score: 9.8 + vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + - url: "https://zeiss.app.blackduck.com/api/cwes/CWE-415" + scoring_system: "CVSS:3.1" + severity: "CRITICAL" + score: 9.8 + vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" + - url: "https://zeiss.app.blackduck.com/api/vulnerabilities/BDSA-2020-4804" + scoring_system: "CVSS:3.1" + severity: "CRITICAL" + score: 9.8 + vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" diff --git a/plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt b/plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt new file mode 100644 index 0000000000000..e1987c3fb2794 --- /dev/null +++ b/plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.advisors.blackduck + +import io.kotest.core.spec.style.WordSpec +import io.kotest.inspectors.forAll +import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder +import io.kotest.matchers.shouldBe +import io.kotest.matchers.shouldNot + +import java.time.Instant + +import org.ossreviewtoolkit.advisor.normalizeVulnerabilityData +import org.ossreviewtoolkit.model.AdvisorResult +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.readValue +import org.ossreviewtoolkit.utils.test.getAssetFile +import org.ossreviewtoolkit.utils.test.identifierToPackage + +class BlackDuckFunTest : WordSpec({ + "retrievePackageFindings()" should { + "return the vulnerabilities for the supported ecosystems" { + val blackDuck = createBlackDuck() + val packages = setOf( + // TODO: Add hackage / pod + "Crate::sys-info:0.7.0", + "Gem::rack:2.0.4", + "Maven:com.jfinal:jfinal:1.4", + "NPM::rebber:1.0.0", + "NuGet::Bunkum:4.0.0", + "Pub::http:0.13.1", + "PyPI::django:3.2" + ).mapTo(mutableSetOf()) { + identifierToPackage(it) + } + + val packageFindings = blackDuck.retrievePackageFindings(packages).mapKeys { it.key.id.toCoordinates() } + + packageFindings.keys shouldContainExactlyInAnyOrder packages.map { it.id.toCoordinates() } + packageFindings.keys.forAll { id -> + packageFindings.getValue(id).vulnerabilities shouldNot beEmpty() + } + } + + "return the expected result for the given package(s)" { + val expectedResult = getAssetFile("retrieve-package-findings-expected-result.yml") + .readValue>() + val blackDuck = createBlackDuck() + val packages = setOf( + // Package using CVSS 3.1 vector: + "Crate::sys-info:0.7.0" + // Todo: Add a package using CVSS 2 vector: + ).mapTo(mutableSetOf()) { + identifierToPackage(it) + } + + val packageFindings = blackDuck.retrievePackageFindings(packages).mapKeys { it.key.id } + + packageFindings.patchTimes() shouldBe expectedResult.patchTimes() + } + } +}) + +private fun createBlackDuck(): BlackDuck = + BlackDuckFactory.create( + serverUrl = System.getenv("BLACK_DUCK_SERVER_URL"), + apiToken = System.getenv("BLACK_DUCK_API_TOKEN") + ) + +private fun Map.patchTimes(): Map = + mapValues { (_, advisorResult) -> + advisorResult.normalizeVulnerabilityData().copy( + summary = advisorResult.summary.copy( + startTime = Instant.EPOCH, + endTime = Instant.EPOCH + ) + ) + } diff --git a/plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt b/plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt new file mode 100644 index 0000000000000..c7066ae243ee0 --- /dev/null +++ b/plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.advisors.blackduck + +import com.synopsys.integration.blackduck.api.generated.view.VulnerabilityView + +import java.time.Instant + +import org.apache.logging.log4j.kotlin.logger + +import org.ossreviewtoolkit.advisor.AdviceProvider +import org.ossreviewtoolkit.advisor.AdviceProviderFactory +import org.ossreviewtoolkit.model.AdvisorCapability +import org.ossreviewtoolkit.model.AdvisorDetails +import org.ossreviewtoolkit.model.AdvisorResult +import org.ossreviewtoolkit.model.AdvisorSummary +import org.ossreviewtoolkit.model.Package +import org.ossreviewtoolkit.model.vulnerabilities.Cvss2Rating +import org.ossreviewtoolkit.model.vulnerabilities.Vulnerability +import org.ossreviewtoolkit.model.vulnerabilities.VulnerabilityReference +import org.ossreviewtoolkit.plugins.api.OrtPlugin +import org.ossreviewtoolkit.plugins.api.PluginDescriptor +import org.ossreviewtoolkit.utils.common.alsoIfNull +import org.ossreviewtoolkit.utils.common.enumSetOf + +@OrtPlugin( + id = "BlackDuck", + displayName = "BlackDuck", + description = "An advisor that retrieves vulnerability information from a BlackDuck instance.", + factory = AdviceProviderFactory::class +) +class BlackDuck(override val descriptor: PluginDescriptor, config: BlackDuckConfiguration) : AdviceProvider { + override val details = AdvisorDetails(descriptor.id, enumSetOf(AdvisorCapability.VULNERABILITIES)) + + private val componentService = ExtendedComponentService.create(config.serverUrl, config.apiToken) + + override suspend fun retrievePackageFindings(packages: Set): Map { + val startTime = Instant.now() + + // TODO: run in parallel. + val result = packages.map { pkg -> + val vulnerabilities = getVulnerabilitiesByPurl(pkg).orEmpty() + + pkg to AdvisorResult( + advisor = details, + summary = AdvisorSummary( + startTime = startTime, + endTime = Instant.now() + ), + vulnerabilities = vulnerabilities + ) + }.toList().toMap() + + return result + } + + private fun getVulnerabilitiesByPurl(pkg: Package): List? { + logger.info { "Get vulnerabilities for ${pkg.id.toCoordinates()} by purl: '${pkg.purl}'." } + + val purl = pkg.purl.takeIf { Purl.isValid(it) } ?: run { + logger.warn { "Skipping invalid purl '$this'." } + return null + } + + val searchResults = componentService.searchKbComponentsByPurl(purl) + val originViews = searchResults.mapNotNull { searchResult -> + componentService.getOriginView(searchResult).alsoIfNull { + // A purl matches on the granularity of a component variant / origin. So, This should not happen. + logger.warn { "Could details for variant '${searchResult.variant}' matched by '${pkg.purl}'." } + } + } + + val vulnerabilities = originViews.flatMap { componentService.getVulnerabilities(it) }.distinctBy { it.name } + + logger.info { + "Found ${vulnerabilities.size} vulnerabilities by purl $purl for package ${pkg.id.toCoordinates()}'" + } + + return vulnerabilities.map { it.toOrtVulnerability() } + } +} + +private fun VulnerabilityView.toOrtVulnerability(): Vulnerability { + val referenceUris = mutableListOf(meta.href.uri()).apply { + meta.links.mapTo(this) { it.href.uri() } + } + + val references = referenceUris.map { uri -> + val cvssVector = cvss3?.vector ?: cvss2?.vector + val scoringSystem = cvssVector?.substringBefore('/', Cvss2Rating.PREFIXES.first()) + + VulnerabilityReference( + url = uri, + scoringSystem = scoringSystem, + severity = severity.toString(), + score = overallScore.toFloat(), + vector = cvssVector + ) + } + + return Vulnerability( + id = name, + description = description, + references = references + ) +} diff --git a/plugins/advisors/black-duck/src/main/kotlin/BlackDuckConfiguration.kt b/plugins/advisors/black-duck/src/main/kotlin/BlackDuckConfiguration.kt new file mode 100644 index 0000000000000..9140f2c7d3226 --- /dev/null +++ b/plugins/advisors/black-duck/src/main/kotlin/BlackDuckConfiguration.kt @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.advisors.blackduck + +import org.ossreviewtoolkit.plugins.api.OrtPluginOption + +/** + * The configuration for the BlackDuck vulnerability provider. + */ +data class BlackDuckConfiguration( + /** + * The base URL of the BlackDuck REST API. + */ + @OrtPluginOption() + val serverUrl: String, + + /** + * The API token to use for authentication. + */ + @OrtPluginOption() + val apiToken: String +) diff --git a/plugins/advisors/black-duck/src/main/kotlin/ExtendedComponentService.kt b/plugins/advisors/black-duck/src/main/kotlin/ExtendedComponentService.kt new file mode 100644 index 0000000000000..c69dec37fbe3b --- /dev/null +++ b/plugins/advisors/black-duck/src/main/kotlin/ExtendedComponentService.kt @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.advisors.blackduck + +import com.synopsys.integration.blackduck.api.core.BlackDuckPath +import com.synopsys.integration.blackduck.api.core.response.LinkMultipleResponses +import com.synopsys.integration.blackduck.api.generated.discovery.ApiDiscovery +import com.synopsys.integration.blackduck.api.generated.response.ComponentsView +import com.synopsys.integration.blackduck.api.generated.view.ComponentView +import com.synopsys.integration.blackduck.api.generated.view.OriginView +import com.synopsys.integration.blackduck.api.generated.view.VulnerabilityView +import com.synopsys.integration.blackduck.configuration.BlackDuckServerConfigBuilder +import com.synopsys.integration.blackduck.configuration.BlackDuckServerConfigKeys.KEYS +import com.synopsys.integration.blackduck.http.BlackDuckRequestBuilder +import com.synopsys.integration.blackduck.service.BlackDuckApiClient +import com.synopsys.integration.blackduck.service.BlackDuckServicesFactory +import com.synopsys.integration.blackduck.service.dataservice.ComponentService +import com.synopsys.integration.log.IntLogger +import com.synopsys.integration.log.LogLevel +import com.synopsys.integration.log.PrintStreamIntLogger +import com.synopsys.integration.rest.HttpUrl +import com.synopsys.integration.util.IntEnvironmentVariables + +import java.util.Optional +import java.util.concurrent.Executors + +import org.apache.commons.lang3.StringUtils + +// Parameter for BlackDuck services factory, see also +// https://github.com/blackducksoftware/blackduck-common/blob/67.0.2/src/main/java/com/blackduck/integration/blackduck/service/BlackDuckServicesFactory.java#L82-L84 +private const val BLACK_DUCK_SERVICES_THREAD_POOL_SIZE = 30 + +private val KB_COMPONENTS_SEARCH_PATH = BlackDuckPath( + "/api/search/kb-purl-component", + // Use ComponentsView even though SearchKbPurlComponentView is probably the class dedicated to this result, + // to avoid any conversion to the needed ComponentsView. + ComponentsView::class.java, + true +) + +internal class ExtendedComponentService( + blackDuckApiClient: BlackDuckApiClient, + apiDiscovery: ApiDiscovery, + logger: IntLogger +) : ComponentService(blackDuckApiClient, apiDiscovery, logger) { + companion object { + fun create(serverUrl: String, apiToken: String): ExtendedComponentService { + val logger = PrintStreamIntLogger(System.out, LogLevel.INFO) // TODO: Handle logging. + val factory = createBlackDuckServicesFactory(serverUrl, apiToken, logger) + return ExtendedComponentService(factory.blackDuckApiClient, factory.apiDiscovery, factory.logger) + } + } + + fun searchKbComponentsByPurl(purl: String): List { + // See https://community.blackduck.com/s/article/Searching-Black-Duck-KnowledgeBase-using-Package-URLs. + val responses = apiDiscovery.metaMultipleResponses(KB_COMPONENTS_SEARCH_PATH) + + val request = BlackDuckRequestBuilder() + .commonGet() + .addQueryParameter("purl", purl) + .buildBlackDuckRequest(responses) + + return blackDuckApiClient.getAllResponses(request) + } + + override fun getComponentView(searchResult: ComponentsView): Optional { + // Override the super function to fix it. The super function accidentally uses the wrong URL, pointing the + // ComponentVersionView as opposed to the ComponentView. + if (StringUtils.isNotBlank(searchResult.component)) { + val url = HttpUrl(searchResult.component) + return Optional.ofNullable(this.blackDuckApiClient.getResponse(url, ComponentView::class.java)) + } else { + return Optional.empty() + } + } + + fun getOriginView(searchResult: ComponentsView): OriginView? { + if (searchResult.variant.isNullOrBlank()) return null + + val url = HttpUrl(searchResult.variant) + return blackDuckApiClient.getResponse(url, OriginView::class.java) + } + + fun getVulnerabilities(originView: OriginView): List { + val link = LinkMultipleResponses("vulnerabilities", VulnerabilityView::class.java) + val metaVulnerabilitiesLinked = originView.metaMultipleResponses(link) + + return blackDuckApiClient.getAllResponses(metaVulnerabilitiesLinked) + } +} + +private fun createBlackDuckServicesFactory( + serverUrl: String, + apiToken: String, + logger: IntLogger +): BlackDuckServicesFactory { + val serverConfig = BlackDuckServerConfigBuilder(KEYS.apiToken).apply { + url = serverUrl + this.apiToken = apiToken + }.build() + + val httpClient = serverConfig.createBlackDuckHttpClient(logger) + val environmentVariables = IntEnvironmentVariables.empty() + val executorService = Executors.newFixedThreadPool(BLACK_DUCK_SERVICES_THREAD_POOL_SIZE) + + return BlackDuckServicesFactory(environmentVariables, executorService, logger, httpClient) +} diff --git a/plugins/advisors/black-duck/src/main/kotlin/Purl.kt b/plugins/advisors/black-duck/src/main/kotlin/Purl.kt new file mode 100644 index 0000000000000..dd341e110eff8 --- /dev/null +++ b/plugins/advisors/black-duck/src/main/kotlin/Purl.kt @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.advisors.blackduck + +import org.ossreviewtoolkit.utils.common.withoutPrefix + +// TODO: Check if this code should go to common PURL code. +internal data class Purl( + val type: String, + val namespace: String?, + val name: String, + val version: String? +) { + companion object { + fun parse(s: String): Purl? { + // TODO: Adhere to decoding / encoding. + var remaining = s.withoutPrefix("pkg:") + ?.substringBefore("?") // drop qualifiers + ?: return null + + val type = remaining.substringBefore("/") + remaining = remaining.withoutPrefix("$type/")!! + + val version = remaining.substringAfter("@", "").takeUnless { it.isBlank() } + remaining = remaining.substringBefore("@") + + val name = remaining.substringAfterLast("/") + val namespace = remaining.substringBeforeLast("/", "").takeUnless { it.isBlank() } + + return Purl(type, namespace, name, version) + } + + fun isValid(s: String): Boolean = !parse(s)?.name.orEmpty().isNullOrBlank() + } +} From 60754398dd8cbe5a796b8561f611772ba89092b8 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Mon, 16 Dec 2024 17:11:12 +0100 Subject: [PATCH 3/5] feat(model): Add the attribute `blackDuckOrigin` The property allows to specify the origin (BlackDuck terminology) corresponding to this package. Knowing the origin is necessary in order to retrieve any information about the package from BlackDuck, for example known security vulnerabilities. Normally, for a couple of ecosystems, it is possible to determine the origin automaticall based on the purl. However, this does not always work: 1. BlackDuck does not know the origin but a similar one from a different external namesace. For example, the database contains entries for NuGet release but not for the GitHub release of a particular package. 2. TBC Signed-off-by: Frank Viernau --- .../main/kotlin/BlackDuckOriginReference.kt | 57 +++++++++++++++++++ model/src/main/kotlin/Package.kt | 8 ++- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 model/src/main/kotlin/BlackDuckOriginReference.kt diff --git a/model/src/main/kotlin/BlackDuckOriginReference.kt b/model/src/main/kotlin/BlackDuckOriginReference.kt new file mode 100644 index 0000000000000..2c957bb49e411 --- /dev/null +++ b/model/src/main/kotlin/BlackDuckOriginReference.kt @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2017 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model + +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonValue + +/** + * A reference to a BlackDuck Origin, see also + * https://community.blackduck.com/s/article/What-is-an-Origin-and-Origin-ID-in-Blackduck. + * Note: While the term Origin is still current, the properties `originId` and `originType` haven been deprecated in + * favor of `externalNamespace` and `externalId`. + */ +data class BlackDuckOriginReference( + /** + * The namespace such as 'maven', 'pypi' or 'github'. + */ + val externalNamespace: String, + + /** + * The component's identifier within the external namespace. + */ + val externalId: String +) { + @JsonValue + fun toCoordinates() = listOf(externalNamespace, externalId).joinToString(":") + + companion object { + @JsonCreator + fun parse(coordinates: String): BlackDuckOriginReference { + val index = coordinates.indexOf(':') + require(index != -1) + + return BlackDuckOriginReference( + externalNamespace = coordinates.substring(0, index), + externalId = coordinates.substring(index + 1, coordinates.length) + ) + } + } +} diff --git a/model/src/main/kotlin/Package.kt b/model/src/main/kotlin/Package.kt index 7f57b0254cd38..6a79c47f3549e 100644 --- a/model/src/main/kotlin/Package.kt +++ b/model/src/main/kotlin/Package.kt @@ -137,7 +137,13 @@ data class Package( * default is used. If not null, this must not be empty and not contain any duplicates. */ @JsonInclude(JsonInclude.Include.NON_NULL) - val sourceCodeOrigins: List? = null + val sourceCodeOrigins: List? = null, + + /** + * The BlackDuck Origin (component) belonging to this package. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + val blackDuckOrigin: BlackDuckOriginReference? = null ) { companion object { /** From f2c994f3368693eb1c6dfa96acf704127778b73a Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Mon, 16 Dec 2024 17:31:36 +0100 Subject: [PATCH 4/5] feat(model): Allow setting `blackDuckOrigin` via package curations Signed-off-by: Frank Viernau --- model/src/main/kotlin/PackageCurationData.kt | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/model/src/main/kotlin/PackageCurationData.kt b/model/src/main/kotlin/PackageCurationData.kt index 67a29ea907fe6..848d32e399f3c 100644 --- a/model/src/main/kotlin/PackageCurationData.kt +++ b/model/src/main/kotlin/PackageCurationData.kt @@ -107,7 +107,13 @@ data class PackageCurationData( * duplicates. */ @JsonInclude(JsonInclude.Include.NON_NULL) - val sourceCodeOrigins: List? = null + val sourceCodeOrigins: List? = null, + + /** + * The BlackDuck Origin (component) belonging to this package. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + val blackDuckOrigin: BlackDuckOriginReference? = null ) { init { declaredLicenseMapping.forEach { (key, value) -> @@ -156,7 +162,8 @@ data class PackageCurationData( vcsProcessed = vcsProcessed, isMetadataOnly = isMetadataOnly ?: original.isMetadataOnly, isModified = isModified ?: original.isModified, - sourceCodeOrigins = sourceCodeOrigins ?: original.sourceCodeOrigins + sourceCodeOrigins = sourceCodeOrigins ?: original.sourceCodeOrigins, + blackDuckOrigin = blackDuckOrigin ?: original.blackDuckOrigin ) val declaredLicenseMappingDiff = buildMap { @@ -197,6 +204,7 @@ data class PackageCurationData( @Suppress("UnsafeCallOnNullableType") (value ?: otherValue)!! }, - sourceCodeOrigins = sourceCodeOrigins ?: other.sourceCodeOrigins + sourceCodeOrigins = sourceCodeOrigins ?: other.sourceCodeOrigins, + blackDuckOrigin = blackDuckOrigin ?: other.blackDuckOrigin ) } From 57f5accff7ab6256fb22e1235f2caf1c59e6fb3e Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 17 Dec 2024 09:56:01 +0100 Subject: [PATCH 5/5] WIP! feat(black-duck): Allow overriding querying by purl If the package has the BlackDuck origin speciified as external ID, use that reference to determine the corresponding origin for which to query the vulnerabilities. Note: This is necessary in the following case TBD Signed-off-by: Frank Viernau --- .../src/funTest/kotlin/BlackDuckFunTest.kt | 40 +++++++++++++++++ .../black-duck/src/main/kotlin/BlackDuck.kt | 44 ++++++++++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt b/plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt index e1987c3fb2794..ee454430de58d 100644 --- a/plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt +++ b/plugins/advisors/black-duck/src/funTest/kotlin/BlackDuckFunTest.kt @@ -22,7 +22,10 @@ package org.ossreviewtoolkit.plugins.advisors.blackduck import io.kotest.core.spec.style.WordSpec import io.kotest.inspectors.forAll import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.shouldContain import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder +import io.kotest.matchers.collections.shouldNotContain +import io.kotest.matchers.should import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNot @@ -30,6 +33,7 @@ import java.time.Instant import org.ossreviewtoolkit.advisor.normalizeVulnerabilityData import org.ossreviewtoolkit.model.AdvisorResult +import org.ossreviewtoolkit.model.BlackDuckOriginReference import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.readValue import org.ossreviewtoolkit.utils.test.getAssetFile @@ -76,6 +80,42 @@ class BlackDuckFunTest : WordSpec({ packageFindings.patchTimes() shouldBe expectedResult.patchTimes() } + + "return the vulnerabilities for the BlackDuck origin if specified, instead of for the purl" { + val blackDuck = createBlackDuck() + val pkg = identifierToPackage("Crate::sys-info:0.7.0").copy( + blackDuckOrigin = BlackDuckOriginReference( + externalNamespace = "gitlab", + externalId = "libtiff/libtiff:v4.5.0" + ) + ) + + val advisorResult = blackDuck.retrievePackageFindings(setOf(pkg)).getValue(pkg) + + with(advisorResult.vulnerabilities.map { it.id }) { + // Vulnerability in libtiff: + this shouldContain "CVE-2024-7006" + // Vulnerability in sysinfo, see also 'retrieve-package-findings-expected-result.yml'. + this shouldNotContain "CVE-2020-36434" + } + } + + "return no vulnerabilities if the reference refers to a component, but not to a specific origin" { + val blackDuck = createBlackDuck() + val pkg = identifierToPackage("PyPI::django:3.2").copy( + blackDuckOrigin = BlackDuckOriginReference( + externalNamespace = "gitlab", + externalId = "libtiff/libtiff" + ) + ) + + val advisorResult = blackDuck.retrievePackageFindings(setOf(pkg)).getValue(pkg) + + with(advisorResult) { + vulnerabilities should beEmpty() + // TODO: Report an issue + } + } } }) diff --git a/plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt b/plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt index c7066ae243ee0..6ccf8ea40f167 100644 --- a/plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt +++ b/plugins/advisors/black-duck/src/main/kotlin/BlackDuck.kt @@ -19,6 +19,8 @@ package org.ossreviewtoolkit.plugins.advisors.blackduck +import com.synopsys.integration.bdio.model.Forge +import com.synopsys.integration.bdio.model.externalid.ExternalId import com.synopsys.integration.blackduck.api.generated.view.VulnerabilityView import java.time.Instant @@ -40,6 +42,8 @@ import org.ossreviewtoolkit.plugins.api.PluginDescriptor import org.ossreviewtoolkit.utils.common.alsoIfNull import org.ossreviewtoolkit.utils.common.enumSetOf +// TODO: Add handling for request execution exceptions. + @OrtPlugin( id = "BlackDuck", displayName = "BlackDuck", @@ -56,7 +60,11 @@ class BlackDuck(override val descriptor: PluginDescriptor, config: BlackDuckConf // TODO: run in parallel. val result = packages.map { pkg -> - val vulnerabilities = getVulnerabilitiesByPurl(pkg).orEmpty() + val vulnerabilities = if (pkg.blackDuckOrigin != null) { + getVulnerabilitiesByExternalId(pkg).orEmpty() + } else { + getVulnerabilitiesByPurl(pkg).orEmpty() + } pkg to AdvisorResult( advisor = details, @@ -71,6 +79,37 @@ class BlackDuck(override val descriptor: PluginDescriptor, config: BlackDuckConf return result } + private fun getVulnerabilitiesByExternalId(pkg: Package): List? { + val ref = pkg.blackDuckOrigin!! + logger.info { "Get vulnerabilities for ${pkg.id.toCoordinates()} by external ID: '${ref.toCoordinates()}'." } + + val forge = Forge.getKnownForges()[ref.externalNamespace] ?: run { + logger.error("Unknown forge: '${ref.externalNamespace}") + return null + } + + val externalId = ExternalId.createFromExternalId(forge, ref.externalId, null, null) + + val searchResults = componentService.getAllSearchResults(externalId) + logger.info { "Found ${searchResults.size} search results for external ID: '${ref.toCoordinates()}'." } + + val originViews = searchResults.mapNotNull { searchResult -> + componentService.getOriginView(searchResult).alsoIfNull { + // A purl matches on the granularity of a component variant / origin. So, This should not happen. + logger.warn { "Could not get origin details for '${searchResult.variant}' matched by '${pkg.purl}'." } + } + } + + val vulnerabilities = originViews.flatMap { componentService.getVulnerabilities(it) }.distinctBy { it.name } + + logger.info { + "Found ${vulnerabilities.size} vulnerabilities by ${ref.toCoordinates()} for package " + + "${pkg.id.toCoordinates()}'" + } + + return vulnerabilities.map { it.toOrtVulnerability() } + } + private fun getVulnerabilitiesByPurl(pkg: Package): List? { logger.info { "Get vulnerabilities for ${pkg.id.toCoordinates()} by purl: '${pkg.purl}'." } @@ -80,10 +119,11 @@ class BlackDuck(override val descriptor: PluginDescriptor, config: BlackDuckConf } val searchResults = componentService.searchKbComponentsByPurl(purl) + val originViews = searchResults.mapNotNull { searchResult -> componentService.getOriginView(searchResult).alsoIfNull { // A purl matches on the granularity of a component variant / origin. So, This should not happen. - logger.warn { "Could details for variant '${searchResult.variant}' matched by '${pkg.purl}'." } + logger.warn { "Could not get origin details for '${searchResult.variant}' matched by '${pkg.purl}'." } } }