From 5d749947322fa0e57a31eb8b5fbf7b1ef83badb1 Mon Sep 17 00:00:00 2001 From: Nicolas Nobelis Date: Fri, 15 Mar 2024 17:25:42 +0100 Subject: [PATCH] feat(fossid-webapp): Retain snippet choice state in FossID When a file has been marked as identified in the current run or a previous one, it is not pending anymore, thus its snippets won't be listed. Consequently, proper license findings cannot be created because the license and the line range information is not stored nor returned by FossID. It is also important to retain which snippet was chosen, as it will allow to deal with deletion of snippet choices (which will be added in a future commit). This commit solves the issue by storing the snippet artifact and version to an identification bound to the scan. Identically, the license and source location is stored as JSON in a comment attached to the scan. Then, when the ORT FossID scanner runs, this information is loaded to create a proper license finding out of it. Signed-off-by: Nicolas Nobelis --- .../scanners/fossid/src/main/kotlin/FossId.kt | 87 +++++++++++++++- .../src/main/kotlin/FossIdScanResults.kt | 50 ++++++++-- .../fossid/src/main/kotlin/OrtComment.kt | 48 +++++++++ .../test/kotlin/FossIdSnippetChoiceTest.kt | 98 ++++++++++++++++++- .../fossid/src/test/kotlin/TestUtils.kt | 11 ++- 5 files changed, 277 insertions(+), 17 deletions(-) create mode 100644 plugins/scanners/fossid/src/main/kotlin/OrtComment.kt diff --git a/plugins/scanners/fossid/src/main/kotlin/FossId.kt b/plugins/scanners/fossid/src/main/kotlin/FossId.kt index 92e793058a747..1edc9562196ed 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossId.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossId.kt @@ -38,6 +38,8 @@ import kotlinx.coroutines.withTimeoutOrNull import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.clients.fossid.FossIdRestService +import org.ossreviewtoolkit.clients.fossid.addComponentIdentification +import org.ossreviewtoolkit.clients.fossid.addFileComment import org.ossreviewtoolkit.clients.fossid.checkDownloadStatus import org.ossreviewtoolkit.clients.fossid.checkResponse import org.ossreviewtoolkit.clients.fossid.createIgnoreRule @@ -79,6 +81,7 @@ import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.model.config.snippet.SnippetChoice import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason import org.ossreviewtoolkit.model.createAndLogIssue +import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.scanner.PackageScannerWrapper import org.ossreviewtoolkit.scanner.ProvenanceScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext @@ -116,6 +119,12 @@ class FossId internal constructor( @JvmStatic private val GIT_FETCH_DONE_REGEX = Regex("-> FETCH_HEAD(?: Already up to date.)*$") + /** + * A regular expression to extract the artifact and version from a Purl returned by FossID. + */ + @JvmStatic + private val SNIPPET_PURL_REGEX = Regex("^.*/(?[^@]+)@(?.+)") + @JvmStatic private val WAIT_DELAY = 10.seconds @@ -905,7 +914,13 @@ class FossId internal constructor( val snippetLicenseFindings = mutableSetOf() val snippetFindings = mapSnippetFindings(rawResults, issues, snippetChoices, snippetLicenseFindings) - markFilesWithChosenSnippetsAsIdentified(scanCode, snippetChoices, snippetFindings, rawResults.listPendingFiles) + markFilesWithChosenSnippetsAsIdentified( + scanCode, + snippetChoices, + snippetFindings, + rawResults.listPendingFiles, + snippetLicenseFindings + ) val ignoredFiles = rawResults.listIgnoredFiles.associateBy { it.path } @@ -931,15 +946,20 @@ class FossId internal constructor( } /** - * Mark all the files having a snippet choice as identified, only if they have no non-chosen source location - * remaining. + * Mark all the files in [snippetChoices] as identified, only after searching in [snippetFindings] that they have no + * non-chosen source location remaining. Only files in [listPendingFiles] are marked. + * Files marked as identified have a license identification and a source location (stored in a comment), using + * [licenseFindings] as reference. */ private fun markFilesWithChosenSnippetsAsIdentified( scanCode: String, snippetChoices: List = emptyList(), snippetFindings: Set, - pendingFiles: List + pendingFiles: List, + licenseFindings: Set ) { + val licenseFindingsByPath = licenseFindings.groupBy { it.location.path } + runBlocking(Dispatchers.IO) { val candidatePathsToMark = snippetChoices.groupBy({ it.given.sourceLocation.path }) { it.choice.reason @@ -962,6 +982,65 @@ class FossId internal constructor( requests += async { service.markAsIdentified(config.user, config.apiKey, scanCode, path, false) } + + val filteredSnippetChoicesByPath = snippetChoices.filter { + it.given.sourceLocation.path == path + } + + val relevantSnippetChoices = filteredSnippetChoicesByPath.filter { + it.choice.reason == SnippetChoiceReason.ORIGINAL_FINDING + } + + relevantSnippetChoices.forEach { filteredSnippetChoice -> + val match = SNIPPET_PURL_REGEX.matchEntire(filteredSnippetChoice.choice.purl.orEmpty()) + match?.also { + val artifact = match.groups["artifact"]?.value.orEmpty() + val version = match.groups["version"]?.value.orEmpty() + val location = filteredSnippetChoice.given.sourceLocation + + requests += async { + logger.info { + "Adding component identification '$artifact/$version' to '$path' " + + "at ${location.startLine}-${location.endLine}." + } + + service.addComponentIdentification( + config.user, + config.apiKey, + scanCode, + path, + artifact, + version, + false + ) + } + } + } + + // The chosen snippet source location lines can neither be stored in the scan nor the file, so + // it is stored in a comment attached to the identified file instead. + val licenseFindingsByLicense = licenseFindingsByPath[path]?.groupBy({ it.license.toString() }) { + it.location + }.orEmpty() + + val relevantChoicesCount = relevantSnippetChoices.size + val notRelevantChoicesCount = filteredSnippetChoicesByPath.count { + it.choice.reason == SnippetChoiceReason.NO_RELEVANT_FINDING + } + val payload = OrtCommentPayload( + licenseFindingsByLicense, + relevantChoicesCount, + notRelevantChoicesCount + ) + val comment = OrtComment(payload) + val jsonComment = jsonMapper.writeValueAsString(comment) + requests += async { + logger.info { + "Adding file comment to '$path' with relevant count $relevantChoicesCount and not " + + "relevant count $notRelevantChoicesCount." + } + service.addFileComment(config.user, config.apiKey, scanCode, path, jsonComment) + } } } } diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt index 104f54b2b9701..b7385f718f294 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt @@ -44,6 +44,7 @@ import org.ossreviewtoolkit.model.TextLocation import org.ossreviewtoolkit.model.config.snippet.SnippetChoice import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason import org.ossreviewtoolkit.model.createAndLogIssue +import org.ossreviewtoolkit.model.jsonMapper import org.ossreviewtoolkit.model.mapLicense import org.ossreviewtoolkit.model.utils.PurlType import org.ossreviewtoolkit.utils.common.alsoIfNull @@ -51,6 +52,7 @@ import org.ossreviewtoolkit.utils.common.collapseToRanges import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.prettyPrintRanges import org.ossreviewtoolkit.utils.ort.DeclaredLicenseProcessor +import org.ossreviewtoolkit.utils.ort.ORT_NAME import org.ossreviewtoolkit.utils.spdx.SpdxConstants import org.ossreviewtoolkit.utils.spdx.toSpdx @@ -90,6 +92,20 @@ internal fun List.mapSummary( val files = filterNot { it.getFileName() in ignoredFiles } files.forEach { summarizable -> val summary = summarizable.toSummary() + var fileComment: OrtComment? = null + + if (summarizable is MarkedAsIdentifiedFile) { + summarizable.comments.values.firstOrNull { + it.comment.contains(ORT_NAME) + }?.also { + runCatching { + fileComment = jsonMapper.readValue(it.comment, OrtComment::class.java) + }.onFailure { + logger.error { "Cannot deserialize comment for ${summary.path}: ${it.message}." } + } + } + } + val defaultLocation = TextLocation(summary.path, TextLocation.UNKNOWN_LINE, TextLocation.UNKNOWN_LINE) summary.licences.forEach { licenseAddedInTheUI -> @@ -98,6 +114,14 @@ internal fun List.mapSummary( } } + fileComment?.ort?.licenses?.forEach { (licenseInORTComment, locations) -> + locations.forEach { location -> + mapLicense(licenseInORTComment, location, issues, detectedLicenseMapping)?.let { + licenseFindings += it + } + } + } + summarizable.getCopyright().let { if (it.isNotEmpty()) { copyrightFindings += CopyrightFinding(it, defaultLocation) @@ -262,15 +286,23 @@ internal fun mapSnippetFindings( findings.map { SnippetFinding(it.key, it.value) } }.toSet().also { remainingSnippetChoices.forEach { snippetChoice -> - val message = "The configuration contains a snippet choice for the snippet ${snippetChoice.choice.purl} " + - "at ${snippetChoice.given.sourceLocation.prettyPrint()}, but the FossID result contains no such " + - "snippet." - logger.warn(message) - issues += Issue( - source = "FossId", - message = message, - severity = Severity.WARNING - ) + // The issue is created only if the chosen snippet does not correspond to a file marked by a previous run. + val isNotOldMarkedAsIdentifiedFile = rawResults.markedAsIdentifiedFiles.none { markedFile -> + markedFile.file.path == snippetChoice.given.sourceLocation.path + } + + if (isNotOldMarkedAsIdentifiedFile) { + val message = + "The configuration contains a snippet choice for the snippet ${snippetChoice.choice.purl} at " + + "${snippetChoice.given.sourceLocation.prettyPrint()}, but the FossID result contains no such " + + "snippet." + logger.warn(message) + issues += Issue( + source = "FossId", + message = message, + severity = Severity.WARNING + ) + } } } } diff --git a/plugins/scanners/fossid/src/main/kotlin/OrtComment.kt b/plugins/scanners/fossid/src/main/kotlin/OrtComment.kt new file mode 100644 index 0000000000000..46dc1d3cb789a --- /dev/null +++ b/plugins/scanners/fossid/src/main/kotlin/OrtComment.kt @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.fossid + +import org.ossreviewtoolkit.model.TextLocation + +/** + * A class representing a comment generated by ORT and attached to a file marked as identified in FossID. + * This comment contains a mapping between license identifiers and their corresponding source locations. + * This comment is serialized as JSON in the marked as identified file's comment, hence the property being named + * [OrtComment.ort]. + */ +data class OrtComment(val ort: OrtCommentPayload) + +/** + * The payload of an [OrtComment]. + */ +data class OrtCommentPayload( + /** + * The license of chosen snippets mapped to their source location. + */ + val licenses: Map>, + /** + * The amount of chosen snippets for this file. + */ + val snippetChoicesCount: Int, + /** + * The amount of not relevant snippets for this file. + */ + val notRelevantSnippetsCount: Int +) diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt index 13569aeda15b6..17e3b6c869b0b 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetChoiceTest.kt @@ -38,6 +38,8 @@ import org.ossreviewtoolkit.clients.fossid.FossIdRestService import org.ossreviewtoolkit.clients.fossid.FossIdServiceWithVersion import org.ossreviewtoolkit.clients.fossid.PolymorphicList import org.ossreviewtoolkit.clients.fossid.PolymorphicResponseBody +import org.ossreviewtoolkit.clients.fossid.addComponentIdentification +import org.ossreviewtoolkit.clients.fossid.addFileComment import org.ossreviewtoolkit.clients.fossid.listIdentifiedFiles import org.ossreviewtoolkit.clients.fossid.listIgnoredFiles import org.ossreviewtoolkit.clients.fossid.listMarkedAsIdentifiedFiles @@ -59,6 +61,8 @@ import org.ossreviewtoolkit.model.config.snippet.Given import org.ossreviewtoolkit.model.config.snippet.Provenance import org.ossreviewtoolkit.model.config.snippet.SnippetChoice import org.ossreviewtoolkit.model.config.snippet.SnippetChoiceReason +import org.ossreviewtoolkit.model.jsonMapper +import org.ossreviewtoolkit.utils.ort.ORT_NAME import org.ossreviewtoolkit.utils.spdx.toSpdx /** A sample file in the results. **/ @@ -260,9 +264,10 @@ class FossIdSnippetChoiceTest : WordSpec({ ) .expectMarkAsIdentified(scanCode, FILE_1) + val choiceLocation = TextLocation(FILE_1, 10, 20) val snippetChoices = createSnippetChoices( vcsInfo.url, - createSnippetChoice(TextLocation(FILE_1, 10, 20), PURL_1, comment = "") + createSnippetChoice(choiceLocation, PURL_1, comment = "") ) val fossId = createFossId(config) @@ -271,6 +276,18 @@ class FossIdSnippetChoiceTest : WordSpec({ summary.snippetFindings should beEmpty() coVerify { service.markAsIdentified(USER, API_KEY, scanCode, FILE_1, any()) + service.addComponentIdentification( + user = USER, + apiKey = API_KEY, + scanCode = scanCode, + path = FILE_1, + componentName = "fakepackage1", + componentVersion = "1.0.0", + isDirectory = false + ) + val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0) + val comment = jsonMapper.writeValueAsString(OrtComment(payload)) + service.addFileComment(USER, API_KEY, scanCode, FILE_1, comment) } } @@ -496,6 +513,80 @@ class FossIdSnippetChoiceTest : WordSpec({ "${choiceLocation.prettyPrint()}, but the FossID result contains no such snippet." } } + + "add the license of already marked as identified file with a snippet choice to the license findings" { + val projectCode = projectCode(PROJECT) + val scanCode = scanCode(PROJECT, null) + val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true) + val vcsInfo = createVcsInfo() + val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode) + val pkgId = createIdentifier(index = 42) + + val choiceLocation = TextLocation(FILE_1, 10, 20) + val payload = OrtCommentPayload(mapOf("MIT" to listOf(choiceLocation)), 1, 0) + val comment = jsonMapper.writeValueAsString(mapOf(ORT_NAME to payload)) + val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1, comment) + FossIdRestService.create(config.serverUrl) + .expectProjectRequest(projectCode) + .expectListScans(projectCode, listOf(scan)) + .expectCheckScanStatus(scanCode, ScanStatus.FINISHED) + .expectCreateScan(projectCode, scanCode, vcsInfo, "") + .expectDownload(scanCode) + .mockFiles( + scanCode, + markedFiles = listOf(markedAsIdentifiedFile) + ) + + val snippetChoices = createSnippetChoices( + vcsInfo.url, + createSnippetChoice(choiceLocation, PURL_1, "") + ) + val fossId = createFossId(config) + + val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = snippetChoices).summary + + summary.licenseFindings shouldHaveSize 1 + summary.licenseFindings.first().apply { + license shouldBe "MIT".toSpdx() + location shouldBe choiceLocation + } + summary.issues.filter { it.severity > Severity.HINT } should beEmpty() + summary.snippetFindings should beEmpty() + } + + "add the license of marked as identified files that have been manually marked in the UI (legacy behavior)" { + val projectCode = projectCode(PROJECT) + val scanCode = scanCode(PROJECT, null) + val config = createConfig(deltaScans = false, fetchSnippetMatchedLines = true) + val vcsInfo = createVcsInfo() + val scan = createScan(vcsInfo.url, "${vcsInfo.revision}_other", scanCode) + val pkgId = createIdentifier(index = 42) + + val markedAsIdentifiedFile = createMarkAsIdentifiedFile("MIT", FILE_1) + FossIdRestService.create(config.serverUrl) + .expectProjectRequest(projectCode) + .expectListScans(projectCode, listOf(scan)) + .expectCheckScanStatus(scanCode, ScanStatus.FINISHED) + .expectCreateScan(projectCode, scanCode, vcsInfo, "") + .expectDownload(scanCode) + .mockFiles( + scanCode, + markedFiles = listOf(markedAsIdentifiedFile) + ) + + val fossId = createFossId(config) + + val summary = fossId.scan(createPackage(pkgId, vcsInfo), snippetChoices = emptyList()).summary + + summary.licenseFindings shouldHaveSize 1 + summary.licenseFindings.first().apply { + license shouldBe "MIT".toSpdx() + val defaultLocation = TextLocation(FILE_1, -1, -1) + location shouldBe defaultLocation + } + summary.issues.filter { it.severity > Severity.HINT } should beEmpty() + summary.snippetFindings should beEmpty() + } } }) @@ -552,6 +643,11 @@ fun FossIdServiceWithVersion.mockFiles( fun FossIdServiceWithVersion.expectMarkAsIdentified(scanCode: String, path: String): FossIdServiceWithVersion { coEvery { markAsIdentified(USER, API_KEY, scanCode, path, any()) } returns EntityResponseBody(status = 1) + coEvery { + addComponentIdentification(USER, API_KEY, scanCode, path, any(), any(), false) + } returns EntityResponseBody(status = 1) + coEvery { addFileComment(USER, API_KEY, scanCode, path, any()) } returns + EntityResponseBody(status = 1) return this } diff --git a/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt b/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt index 7cb3cc1ae7f76..1425943d2fa3a 100644 --- a/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt +++ b/plugins/scanners/fossid/src/test/kotlin/TestUtils.kt @@ -51,6 +51,7 @@ import org.ossreviewtoolkit.clients.fossid.model.Scan import org.ossreviewtoolkit.clients.fossid.model.identification.common.LicenseMatchType import org.ossreviewtoolkit.clients.fossid.model.identification.identifiedFiles.IdentifiedFile import org.ossreviewtoolkit.clients.fossid.model.identification.ignored.IgnoredFile +import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.Comment import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.File import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.License import org.ossreviewtoolkit.clients.fossid.model.identification.markedAsIdentified.LicenseFile @@ -345,7 +346,11 @@ private fun createMarkedIdentifiedFile(index: Int): MarkedAsIdentifiedFile { /** * Create a [MarkedAsIdentifiedFile] with the give [license] and [path]. */ -internal fun createMarkAsIdentifiedFile(license: String, path: String): MarkedAsIdentifiedFile { +internal fun createMarkAsIdentifiedFile( + license: String, + path: String, + comment: String? = null +): MarkedAsIdentifiedFile { val fileLicense = License( id = 1, type = LicenseMatchType.FILE, @@ -368,7 +373,7 @@ internal fun createMarkAsIdentifiedFile(license: String, path: String): MarkedAs return MarkedAsIdentifiedFile( comment = "comment", - comments = emptyMap(), + comments = if (comment == null) emptyMap() else mapOf(1 to Comment(1, 1, comment)), identificationId = 1, identificationCopyright = "copyright", isDistributed = 1, @@ -381,7 +386,7 @@ internal fun createMarkAsIdentifiedFile(license: String, path: String): MarkedAs sha1 = "fileSha1", sha256 = "fileSha256", size = 0, - licenses = mutableMapOf(1 to fileLicense) + licenses = if (comment != null) null else mutableMapOf(1 to fileLicense) ) } }