diff --git a/model/src/main/kotlin/SnippetFinding.kt b/model/src/main/kotlin/SnippetFinding.kt index 3514b2e402ce2..9fc15b2900256 100644 --- a/model/src/main/kotlin/SnippetFinding.kt +++ b/model/src/main/kotlin/SnippetFinding.kt @@ -20,8 +20,8 @@ package org.ossreviewtoolkit.model /** - * A class representing a snippet finding for a source file. A snippet finding is a code snippet from another origin, - * matching the code being scanned. + * A class representing the snippet findings for a source file location. A snippet finding is a code snippet from + * another origin, matching the code being scanned. * It is meant to be reviewed by an operator as it could be a false positive. */ data class SnippetFinding( @@ -31,7 +31,7 @@ data class SnippetFinding( val sourceLocation: TextLocation, /** - * The corresponding snippet. + * The corresponding snippets. */ - val snippet: Snippet + val snippets: Set ) diff --git a/model/src/main/kotlin/utils/SortedSetConverters.kt b/model/src/main/kotlin/utils/SortedSetConverters.kt index 3477115e8e2f4..89390874c2638 100644 --- a/model/src/main/kotlin/utils/SortedSetConverters.kt +++ b/model/src/main/kotlin/utils/SortedSetConverters.kt @@ -89,8 +89,7 @@ class ScopeSortedSetConverter : StdConverter, SortedSet>() { } class SnippetFindingSortedSetConverter : StdConverter, SortedSet>() { - override fun convert(value: Set) = - value.toSortedSet(compareBy { it.sourceLocation }.thenByDescending { it.snippet.purl }) + override fun convert(value: Set) = value.toSortedSet(compareBy { it.sourceLocation }) } private fun Provenance.getSortKey(): String = diff --git a/plugins/reporters/freemarker/src/main/kotlin/FreemarkerTemplateProcessor.kt b/plugins/reporters/freemarker/src/main/kotlin/FreemarkerTemplateProcessor.kt index eebdc0ba4e987..fa5470ce9c6bc 100644 --- a/plugins/reporters/freemarker/src/main/kotlin/FreemarkerTemplateProcessor.kt +++ b/plugins/reporters/freemarker/src/main/kotlin/FreemarkerTemplateProcessor.kt @@ -323,7 +323,7 @@ class FreemarkerTemplateProcessor( */ @Suppress("UNUSED") // This function is used in the templates. fun collectLicenses(snippetFindings: Collection): Set = - snippetFindings.mapTo(mutableSetOf()) { it.snippet.licenses.toString() } + snippetFindings.flatMapTo(mutableSetOf()) { it.snippets.map { snippet -> snippet.licenses.toString() } } /** * Return a flag indicating that issues have been encountered during the run of an advisor with the given diff --git a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt index 14f78bd8fb2ba..2a25181de7437 100644 --- a/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt +++ b/plugins/scanners/fossid/src/main/kotlin/FossIdScanResults.kt @@ -188,7 +188,7 @@ internal fun mapSnippetFindings(rawResults: RawResults, issues: MutableList finding.value.map { SnippetFinding(finding.key, it) } } + findings.map { SnippetFinding(it.key, it.value) } }.toSet() } diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdLicenseMappingTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdLicenseMappingTest.kt index 33b85dd97ec98..70dffcfe885c1 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdLicenseMappingTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdLicenseMappingTest.kt @@ -67,7 +67,9 @@ class FossIdLicenseMappingTest : WordSpec({ issues should beEmpty() findings should haveSize(1) findings.first() shouldNotBeNull { - snippet.licenses.toString() shouldBe "Apache-2.0" + snippets.first() shouldNotBeNull { + licenses.toString() shouldBe "Apache-2.0" + } } } @@ -80,7 +82,9 @@ class FossIdLicenseMappingTest : WordSpec({ issues should beEmpty() findings should haveSize(1) findings.first() shouldNotBeNull { - snippet.licenses.toString() shouldBe "Apache-2.0" + snippets.first() shouldNotBeNull { + licenses.toString() shouldBe "Apache-2.0" + } } } @@ -98,7 +102,9 @@ class FossIdLicenseMappingTest : WordSpec({ } findings should haveSize(1) findings.first() shouldNotBeNull { - snippet.licenses shouldBe SpdxConstants.NOASSERTION.toSpdx() + snippets.first() shouldNotBeNull { + licenses shouldBe SpdxConstants.NOASSERTION.toSpdx() + } } } } diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetMappingTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetMappingTest.kt new file mode 100644 index 0000000000000..3a90f9c589c8a --- /dev/null +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdSnippetMappingTest.kt @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2023 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.fossid + +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.collections.beEmpty +import io.kotest.matchers.collections.containExactly +import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.should +import io.kotest.matchers.shouldBe + +import org.ossreviewtoolkit.clients.fossid.PolymorphicList +import org.ossreviewtoolkit.clients.fossid.model.result.MatchType +import org.ossreviewtoolkit.clients.fossid.model.result.MatchedLines +import org.ossreviewtoolkit.clients.fossid.model.result.Snippet +import org.ossreviewtoolkit.model.Issue +import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.utils.test.shouldNotBeNull + +class FossIdSnippetMappingTest : WordSpec({ + "mapSnippetFindings" should { + "group snippets by source file location" { + val issues = mutableListOf() + val listSnippets = mapOf( + "src/main/java/Tokenizer.java" to setOf( + createSnippet( + 1, + MatchType.FULL, + "pkg:github/vdurmont/semver4j@3.1.0", + "MIT", + "src/main/java/com/vdurmont/semver4j/Tokenizer.java" + ), + createSnippet( + 2, + MatchType.FULL, + "pkg:maven/com.vdurmont/semver4j@3.1.0", + "MIT", + "com/vdurmont/semver4j/Tokenizer.java" + ) + ), + "src/main/java/com/vdurmont/semver4j/Requirement.java" to setOf( + createSnippet( + 3, + MatchType.PARTIAL, + "pkg:github/vdurmont/semver4j@3.1.0", + "MIT", + "com/vdurmont/semver4j/Requirement.java" + ) + ) + ) + val localFile = ((1..24) + (45..675)).toPolymorphicList() + val remoteFile = (1..655).toPolymorphicList() + // There is no matched line information for full match snippets. + val snippetMatchedLines = mapOf(3 to MatchedLines(localFile, remoteFile)) + val rawResults = RawResults( + emptyList(), + emptyList(), + emptyList(), + emptyList(), + listSnippets, + snippetMatchedLines + ) + + val mappedSnippets = mapSnippetFindings(rawResults, issues) + + issues should beEmpty() + mappedSnippets shouldHaveSize 3 + mappedSnippets.first().apply { + sourceLocation shouldBe TextLocation("src/main/java/Tokenizer.java", TextLocation.UNKNOWN_LINE) + snippets shouldHaveSize 2 + snippets.map { it.purl } should containExactly( + "pkg:github/vdurmont/semver4j@3.1.0", + "pkg:maven/com.vdurmont/semver4j@3.1.0" + ) + } + mappedSnippets.elementAtOrNull(1) shouldNotBeNull { + sourceLocation shouldBe TextLocation("src/main/java/com/vdurmont/semver4j/Requirement.java", 1, 24) + snippets.map { it.purl } should containExactly("pkg:github/vdurmont/semver4j@3.1.0") + } + mappedSnippets.elementAtOrNull(2) shouldNotBeNull { + sourceLocation shouldBe TextLocation("src/main/java/com/vdurmont/semver4j/Requirement.java", 45, 675) + snippets.map { it.purl } should containExactly("pkg:github/vdurmont/semver4j@3.1.0") + } + } + } +}) + +private fun createSnippet(id: Int, matchType: MatchType, purl: String, license: String, file: String) = + Snippet( + id = id, + created = "", + scanId = 1, + scanFileId = 1, + fileId = 1, + matchType = matchType, + reason = null, + author = null, + artifact = null, + version = null, + purl = purl, + artifactLicense = license, + artifactLicenseCategory = null, + releaseDate = null, + mirror = null, + file = file, + fileLicense = null, + url = "", + hits = null, + size = null, + updated = null, + cpe = null, + score = "1.0", + matchFileId = null, + classification = null, + highlighting = null + ) + +private fun IntRange.toPolymorphicList() = toList().toPolymorphicList() +private fun List.toPolymorphicList() = PolymorphicList(this) diff --git a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt index eae9dcae55bca..ea53898302419 100644 --- a/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt +++ b/plugins/scanners/fossid/src/test/kotlin/FossIdTest.kt @@ -109,7 +109,6 @@ import org.ossreviewtoolkit.plugins.scanners.fossid.FossId.Companion.SERVER_URL_ import org.ossreviewtoolkit.plugins.scanners.fossid.FossId.Companion.convertGitUrlToProjectName import org.ossreviewtoolkit.scanner.ScanContext import org.ossreviewtoolkit.utils.spdx.SpdxExpression -import org.ossreviewtoolkit.utils.test.shouldNotBeNull @Suppress("LargeClass") class FossIdTest : WordSpec({ @@ -361,7 +360,7 @@ class FossIdTest : WordSpec({ summary.issues.map { it.copy(timestamp = Instant.EPOCH) } shouldBe expectedIssues } - "report pending files as snippets" { + "report snippets of pending files" { val projectCode = projectCode(PROJECT) val scanCode = scanCode(PROJECT, null) val config = createConfig(deltaScans = false) @@ -382,9 +381,9 @@ class FossIdTest : WordSpec({ val summary = fossId.scan(createPackage(pkgId, vcsInfo)).summary val expectedPendingFile = (1..5).map(::createPendingFile).toSet() - val expectedSnippetFindings = (1..5).map(::createSnippetFindings).flatten() + val expectedSnippetFindings = (1..5).map(::createSnippetFindings) - summary.snippetFindings shouldHaveSize expectedPendingFile.size * 5 + summary.snippetFindings shouldHaveSize expectedPendingFile.size summary.snippetFindings.map { it.sourceLocation.path }.toSet() shouldBe expectedPendingFile summary.snippetFindings shouldBe expectedSnippetFindings } @@ -409,14 +408,36 @@ class FossIdTest : WordSpec({ val summary = fossId.scan(createPackage(pkgId, vcsInfo)).summary - summary.snippetFindings shouldHaveSize 1 - summary.snippetFindings.first() shouldNotBeNull { - sourceLocation.startLine shouldBe(1) - sourceLocation.endLine shouldBe(3) - snippet.location.startLine shouldBe 11 - snippet.location.endLine shouldBe 12 - snippet.additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SOURCE] shouldBe "1-3, 21-22, 36" - snippet.additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SNIPPET] shouldBe "11-12" + summary.snippetFindings shouldHaveSize 3 + summary.snippetFindings.first().apply { + sourceLocation shouldBe TextLocation("/pending/file/1", 1, 3) + snippets shouldHaveSize 1 + snippets.first().apply { + location.startLine shouldBe 11 + location.endLine shouldBe 12 + additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SOURCE] shouldBe "1-3, 21-22, 36" + additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SNIPPET] shouldBe "11-12" + } + } + summary.snippetFindings.elementAt(1).apply { + sourceLocation shouldBe TextLocation("/pending/file/1", 21, 22) + snippets shouldHaveSize 1 + snippets.first().apply { + location.startLine shouldBe 11 + location.endLine shouldBe 12 + additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SOURCE] shouldBe "1-3, 21-22, 36" + additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SNIPPET] shouldBe "11-12" + } + } + summary.snippetFindings.last().apply { + sourceLocation shouldBe TextLocation("/pending/file/1", 36) + snippets shouldHaveSize 1 + snippets.first().apply { + location.startLine shouldBe 11 + location.endLine shouldBe 12 + additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SOURCE] shouldBe "1-3, 21-22, 36" + additionalData[FossId.SNIPPET_DATA_MATCHED_LINE_SNIPPET] shouldBe "11-12" + } } } @@ -1410,10 +1431,10 @@ private fun createSnippet(index: Int): Snippet = /** * Generate a ORT snippet finding based on the given [index]. */ -private fun createSnippetFindings(index: Int): Set = - (1..5).map { snippetIndex -> - SnippetFinding( - TextLocation("/pending/file/$index", TextLocation.UNKNOWN_LINE), +private fun createSnippetFindings(index: Int): SnippetFinding = + SnippetFinding( + TextLocation("/pending/file/$index", TextLocation.UNKNOWN_LINE), + (1..5).map { snippetIndex -> OrtSnippet( snippetIndex.toFloat(), TextLocation("file$snippetIndex", TextLocation.UNKNOWN_LINE), @@ -1427,8 +1448,8 @@ private fun createSnippetFindings(index: Int): Set = FossId.SNIPPET_DATA_MATCH_TYPE to MatchType.PARTIAL.toString() ) ) - ) - }.toSet() + }.toSet() + ) /** * Prepare this service mock to answer a request for a project with the given [projectCode]. Return a response with diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt index 8daae2dc41d8e..4e4ac4c940dcb 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt @@ -59,7 +59,8 @@ internal fun generateSummary(startTime: Instant, endTime: Instant, result: FullS val snippets = getSnippets(scanResponse) snippets.forEach { - snippetFindings += SnippetFinding(sourceLocation, it) + // TODO: Aggregate the snippet by source file location. + snippetFindings += SnippetFinding(sourceLocation, setOf(it)) } } } diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt index 2bd3709c5bcdc..34de6507d53c7 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt @@ -115,19 +115,21 @@ class ScanOssResultParserTest : WordSpec({ summary.snippetFindings.shouldContainExactly( SnippetFinding( TextLocation("src/main/java/com/vdurmont/semver4j/Requirement.java", 1, 710), - Snippet( - 98.0f, - TextLocation( - "https://osskb.org/api/file_contents/6ff2427335b985212c9b79dfa795799f", - 1, - 710 - ), - RepositoryProvenance( - VcsInfo(VcsType.UNKNOWN, "https://github.com/vdurmont/semver4j", ""), - "." - ), - "pkg:github/vdurmont/semver4j", - SpdxExpression.parse("CC-BY-SA-2.0") + setOf( + Snippet( + 98.0f, + TextLocation( + "https://osskb.org/api/file_contents/6ff2427335b985212c9b79dfa795799f", + 1, + 710 + ), + RepositoryProvenance( + VcsInfo(VcsType.UNKNOWN, "https://github.com/vdurmont/semver4j", ""), + "." + ), + "pkg:github/vdurmont/semver4j", + SpdxExpression.parse("CC-BY-SA-2.0") + ) ) ) ) diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt index 70cc8661240a0..c67ec3c339552 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssScannerDirectoryTest.kt @@ -111,18 +111,20 @@ class ScanOssScannerDirectoryTest : StringSpec({ snippetFindings.shouldContainExactly( SnippetFinding( TextLocation("utils/src/main/kotlin/ArchiveUtils.kt", 1, 240), - Snippet( - 99.0f, - TextLocation( - "https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095", - 128, - 367 - ), - RepositoryProvenance( - VcsInfo(VcsType.UNKNOWN, "https://github.com/scanoss/ort", ""), "." - ), - "pkg:github/scanoss/ort", - SpdxExpression.parse("Apache-2.0") + setOf( + Snippet( + 99.0f, + TextLocation( + "https://osskb.org/api/file_contents/871fb0c5188c2f620d9b997e225b0095", + 128, + 367 + ), + RepositoryProvenance( + VcsInfo(VcsType.UNKNOWN, "https://github.com/scanoss/ort", ""), "." + ), + "pkg:github/scanoss/ort", + SpdxExpression.parse("Apache-2.0") + ) ) ) )