diff --git a/clients/scanoss/build.gradle.kts b/clients/scanoss/build.gradle.kts deleted file mode 100644 index 9bd0774d86eca..0000000000000 --- a/clients/scanoss/build.gradle.kts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -plugins { - // Apply precompiled plugins. - id("ort-library-conventions") - - // Apply third-party plugins. - alias(libs.plugins.kotlinSerialization) -} - -dependencies { - api(libs.okhttp) - api(libs.retrofit) - - implementation(libs.kotlinx.serialization.core) - implementation(libs.kotlinx.serialization.json) - implementation(libs.retrofit.converter.kotlinxSerialization) - - testImplementation(libs.wiremock) -} diff --git a/clients/scanoss/src/main/kotlin/ScanOssService.kt b/clients/scanoss/src/main/kotlin/ScanOssService.kt deleted file mode 100644 index d50a4c614f686..0000000000000 --- a/clients/scanoss/src/main/kotlin/ScanOssService.kt +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.clients.scanoss - -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonNamingStrategy - -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.MultipartBody -import okhttp3.OkHttpClient - -import org.ossreviewtoolkit.clients.scanoss.model.ScanResponse - -import retrofit2.Retrofit -import retrofit2.converter.kotlinx.serialization.asConverterFactory -import retrofit2.http.Multipart -import retrofit2.http.POST -import retrofit2.http.Part - -typealias FullScanResponse = Map> - -/** - * The API definition for SCANOSS, see https://docs.osskb.org/. - */ -interface ScanOssService { - companion object { - /** - * The default API URL. - */ - const val DEFAULT_API_URL = "https://api.osskb.org/" - - /** - * The JSON (de-)serialization object used by this service. - */ - val JSON = Json { - ignoreUnknownKeys = true - namingStrategy = JsonNamingStrategy.SnakeCase - } - - /** - * Create a new service instance that connects to the [url] specified and uses the optionally provided [client]. - */ - fun create(url: String? = null, client: OkHttpClient? = null): ScanOssService { - val contentType = "application/json".toMediaType() - val retrofit = Retrofit.Builder() - .apply { if (client != null) client(client) } - .baseUrl(url ?: DEFAULT_API_URL) - .addConverterFactory(JSON.asConverterFactory(contentType)) - .build() - - return retrofit.create(ScanOssService::class.java) - } - } - - /** - * Perform a scan using the streaming API based on the given winnowing fingerprint [file]. - * - * TODO: Implement support for scanning with SBOM. - */ - @Multipart - @POST("scan/direct") - suspend fun scan(@Part file: MultipartBody.Part): FullScanResponse -} diff --git a/clients/scanoss/src/main/kotlin/model/BooleanSerializer.kt b/clients/scanoss/src/main/kotlin/model/BooleanSerializer.kt deleted file mode 100644 index 4641176cc99bb..0000000000000 --- a/clients/scanoss/src/main/kotlin/model/BooleanSerializer.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.clients.scanoss.model - -import kotlinx.serialization.KSerializer -import kotlinx.serialization.Serializer -import kotlinx.serialization.encoding.Decoder -import kotlinx.serialization.encoding.Encoder - -/** - * SCANOSS returns booleans as "yes" or "no". Therefore, we need a special deserializer for it. - */ -@Serializer(Boolean::class) -object BooleanSerializer : KSerializer { - override fun serialize(encoder: Encoder, value: Boolean) = encoder.encodeString(if (value) "yes" else "no") - override fun deserialize(decoder: Decoder): Boolean = decoder.decodeString() == "yes" -} diff --git a/clients/scanoss/src/main/kotlin/model/Model.kt b/clients/scanoss/src/main/kotlin/model/Model.kt deleted file mode 100644 index 09b7808e18bc4..0000000000000 --- a/clients/scanoss/src/main/kotlin/model/Model.kt +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.clients.scanoss.model - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.JsonNames - -/** - * Information about the engine running on the server providing the response. Note that different engine versions may be - * used within a single response as the request is distributed across random backends. - */ -@Serializable -data class Server( - val version: String, - val kbVersion: KnowledgeBaseVersion? = null -) - -/** - * Information about the version of the knowledge base running on the server providing the response. - */ -@Serializable -data class KnowledgeBaseVersion( - val monthly: String, - val daily: String -) - -/** - * The "licenses" section of the raw report. - */ -@Serializable -data class License( - /** Name of the license */ - val name: String, - - /** Location where the license was mined from. */ - val source: Source, - - /** Are there patent hints for this license.*/ - @Serializable(BooleanSerializer::class) - val patentHints: Boolean? = null, - - /** Is this considered a copyleft license or not. */ - @Serializable(BooleanSerializer::class) - val copyleft: Boolean? = null, - - /** URL of the OSADL checklist for this license. */ - val checklistUrl: String? = null, - - val incompatibleWith: String? = null, - - /** Date the OSADL data was last updated. */ - val osadlUpdated: String? = null -) - -/** - * The "Dependency" section of the raw report. - */ -@Serializable -data class Dependency( - val vendor: String, - val component: String, - val version: String, - val source: String -) - -/** - * The "copyrights" section of the raw report. - */ -@Serializable -data class Copyright( - /** The copyright found. */ - val name: String, - - /** - * Location where the copyright was mined from. Reuse the [Source] enum for simplicity even if the - * [Source.FILE_SPDX_TAG] value is not supported here. - */ - val source: Source -) - -@Serializable -enum class Source { - /** A component level declaration was found in the component’s repository for the matched file. */ - @SerialName("component_declared") - COMPONENT_DECLARED, - - /** Minr detected a license text in the file header. */ - @SerialName("file_header") - FILE_HEADER, - - /** The matched file contains a SPDX-License-Identifier tag in its header. */ - @SerialName("file_spdx_tag") - FILE_SPDX_TAG, - - /** Minr detected a license in the LICENSE file in the component of the matched file. */ - @SerialName("license_file") - LICENSE_FILE, - - /** Scancode detected a license declaration in the matched file. */ - @SerialName("scancode") - SCANCODE -} - -/** - * The "vulnerabilities" section of the raw report. - */ -@Serializable -data class Vulnerability( - /** Vulnerability identifier (i.e CVE or Github Advisory ID). */ - @JsonNames("ID") - val id: String, - - /** Published CVE Identifier. */ - @JsonNames("CVE") - val cve: String, - - /** Source location for the vulnerability data. */ - val source: VulnerabilitySource, - - /** Severity of the vulnerability */ - val severity: String, - - /** Summary of the issue. */ - val summary: String, - - /** Date first reported. */ - val reported: String, - - /** Version when the issue was introduced. */ - val introduced: String, - - /** Version the issue is patched/resolved in */ - val patched: String -) - -/** - * Source location for the vulnerability data. - */ -@Serializable -enum class VulnerabilitySource { - /** GitHub Advisories. */ - @SerialName("github_advisories") - GITHUB_ADVISORIES, - - /** National Vulnerability Database. */ - @SerialName("nvd") - NVD -} - -/** - * The "quality" section of the raw report. - */ -@Serializable -data class Quality( - /** Rating for that quality score */ - val score: String, - - /** Type of quality score presented. */ - val source: QualityScoreType -) - -/** - * Type of quality score. - */ -@Serializable -enum class QualityScoreType { - /** - * Score from 0 to 5 based on various items : Proper amount of code comment, proper length of file, proper lines - * length, SPDX license identifier tag found and uniform indentation. - */ - @SerialName("best_practices") - BEST_PRACTICES -} - -/** - * The "cryptography" section of the raw report. - */ -@Serializable -data class Cryptography( - /** Algorithm name. */ - val algorithm: String, - - /** Algorithm strength (number of bits). */ - val strength: String -) - -/** - * Type of identification for the scanned file. - */ -@Serializable -enum class IdentificationType { - @SerialName("file") - FILE, - - @SerialName("none") - NONE, - - @SerialName("snippet") - SNIPPET -} - -/** - * Status of the file match. - */ -@Serializable -enum class FileMatchStatus { - @SerialName("identified") - IDENTIFIED, - - @SerialName("pending") - PENDING -} diff --git a/clients/scanoss/src/main/kotlin/model/ScanResponse.kt b/clients/scanoss/src/main/kotlin/model/ScanResponse.kt deleted file mode 100644 index 1c31aec011b70..0000000000000 --- a/clients/scanoss/src/main/kotlin/model/ScanResponse.kt +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.clients.scanoss.model - -import kotlinx.serialization.Serializable - -/** - * Response of the SCANOSS streaming API returned for each file that was requested to scan. - */ -@Serializable -data class ScanResponse( - /** Type of identification for the scanned file. */ - val id: IdentificationType, - - /** Status of the file match. */ - val status: FileMatchStatus? = null, - - /** Matching line numbers of a snippet in the source file. */ - val lines: String? = null, - - /** Matching line numbers for the snippet in the open source file. */ - val ossLines: String? = null, - - /** Percentage of the source file that matches the open source file. */ - val matched: String? = null, - - /** List of possible Package URLs identifying the OSS component. */ - val purl: List? = null, - - /** Vendor for the matched OSS component */ - val vendor: String? = null, - - /** Name for the matched OSS component. */ - val component: String? = null, - - /** Earliest matched OSS component version. */ - val version: String? = null, - - /** Latest matched OSS component version. */ - val latest: String? = null, - - /** URL of the OSS component. */ - val url: String? = null, - - /** Release date of the component. */ - val releaseDate: String? = null, - - /** Name of the matching OSS file. */ - val file: String? = null, - - /** MD5 hash of the project zip that was mined from. */ - val urlHash: String? = null, - - /** Unique hash for the matching OSS file stored in the KB. */ - val fileHash: String? = null, - - /** URL to download the matching OSS file from the KB. */ - val fileUrl: String? = null, - - /** List of declared dependencies for the detected component. */ - val dependencies: List = emptyList(), - - /** List of licenses associated with the OSS component. */ - val licenses: List = emptyList(), - - /** List of copyrights found in the OSS file. */ - val copyrights: List = emptyList(), - - /** List of known vulnerabilities associated with the OSS component. */ - val vulnerabilities: List = emptyList(), - - /** List of quality metrics associated with the OSS component. */ - val quality: List = emptyList(), - - /** List of cryptographic algorithms found in the OSS component. */ - val cryptography: List = emptyList(), - - /** Details about the SCANOSS server used to conduct the scan. */ - val server: Server -) diff --git a/clients/scanoss/src/test/assets/details/mappings/scan_details.json b/clients/scanoss/src/test/assets/details/mappings/scan_details.json deleted file mode 100644 index 79712efe357a3..0000000000000 --- a/clients/scanoss/src/test/assets/details/mappings/scan_details.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "id" : "0a7de786-48f0-4815-b5f9-8d3c0f435bc5", - "name" : "scan_direct", - "request" : { - "url" : "/scan/direct", - "method" : "POST", - "bodyPatterns" : [ { - "anything" : "anything" - } ] - }, - "response" : { - "status" : 200, - "body" : "{\"osskb.c\":[{\"id\":\"snippet\",\"status\":\"pending\",\"lines\":\"1-10\",\"oss_lines\":\"175-184\",\"matched\":\"90%\",\"purl\":[\"pkg:github/unoconv/unoconv\"],\"vendor\":\"unoconv\",\"component\":\"unoconv\",\"version\":\"0.8.2\",\"latest\":\"0.8.2\",\"url\":\"https://github.com/unoconv/unoconv\",\"release_date\":\"\",\"file\":\"unoconv\",\"url_hash\":\"c36074c3996ba9d7d85f4a57787b5645\",\"file_hash\":\"0f55e083dcc72a11334eb1a77137e2c4\",\"file_url\":\"https://osskb.org/api/file_contents/0f55e083dcc72a11334eb1a77137e2c4\",\"dependencies\":[],\"licenses\":[{\"name\":\"GPL-2.0-only\",\"checklist_url\":\"https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt\",\"copyleft\":\"yes\",\"patent_hints\":\"yes\",\"incompatible_with\":\"Apache-1.0,Apache-1.1,Apache-2.0,BSD-4-Clause,BSD-4-Clause-UC,FTL,IJG,OpenSSL,Python-2.0,zlib-acknowledgement,XFree86-1.1\",\"osadl_updated\":\"2022-02-15\",\"source\":\"component_declared\"}],\"copyrights\":[{\"name\":\"Copyright2007-2010DagWieers\",\"source\":\"file_header\"},{\"name\":\"Copyright2007-2010DagWieers\",\"source\":\"scancode\"}],\"vulnerabilities\":[{\"ID\":\"GHSA-27p5-7cw6-m45h\",\"CVE\":\"CVE-2019-17400\",\"severity\":\"HIGH\",\"reported\":\"2021-08-18\",\"introduced\":\"\",\"patched\":\"<0.9.0\",\"summary\":\"Server-SideRequestForgeryinunoconv\",\"source\":\"github_advisories\"}],\"quality\":[{\"score\":\"2/5\",\"source\":\"best_practices\"}],\"cryptography\":[],\"server\":{\"hostname\":\"p9\",\"version\":\"4.3.0\",\"flags\":\"0\",\"elapsed\":\"0.004624s\"}}]}", - "headers" : { - "Server" : "nginx/1.14.2", - "Date" : "Tue, 01 Mar 2022 09:39:58 GMT", - "Content-Type" : "application/json" - } - }, - "uuid" : "0a7de786-48f0-4815-b5f9-8d3c0f435bc5", - "persistent" : true, - "insertionIndex" : 10 -} diff --git a/clients/scanoss/src/test/assets/scan/file.wfp b/clients/scanoss/src/test/assets/scan/file.wfp deleted file mode 100644 index 3585258738d3d..0000000000000 --- a/clients/scanoss/src/test/assets/scan/file.wfp +++ /dev/null @@ -1,6 +0,0 @@ -file=24e35278ad5d4d3babe7379dc34d5bce,439,pasted.wfp -5=369450fc -6=bf7226a9 -8=b04dd861 -9=9727e3cd -11=2152ba16 diff --git a/clients/scanoss/src/test/assets/scan/mappings/scan_direct-0a7de786-48f0-4815-b5f9-8d3c0f435bc5.json b/clients/scanoss/src/test/assets/scan/mappings/scan_direct-0a7de786-48f0-4815-b5f9-8d3c0f435bc5.json deleted file mode 100644 index 77952d8c9d561..0000000000000 --- a/clients/scanoss/src/test/assets/scan/mappings/scan_direct-0a7de786-48f0-4815-b5f9-8d3c0f435bc5.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "id" : "0a7de786-48f0-4815-b5f9-8d3c0f435bc5", - "name" : "scan_direct", - "request" : { - "url" : "/scan/direct", - "method" : "POST", - "bodyPatterns" : [ { - "anything" : "anything" - } ] - }, - "response" : { - "status" : 200, - "body" : "{\"pasted.wfp\": [{\"id\": \"snippet\",\"status\": \"pending\",\"lines\": \"1-11\",\"oss_lines\": \"119-129\",\"matched\": \"90%\",\"purl\": [\"pkg:github/unoconv/unoconv\",\"pkg:deb/unoconv\",\"pkg:pypi/unoconv\"],\"vendor\": \"unoconv\",\"component\": \"unoconv\",\"version\": \"0.6\",\"latest\": \"0.6\",\"url\": \"https://github.com/unoconv/unoconv\",\"release_date\": \"2012-09-10\",\"file\": \"unoconv\",\"url_hash\": \"2b5b8e4c1c62f2b3cba48ceabc1f3671\",\"file_hash\": \"38e743a8566d3df4a2dc4432f8d6b091\",\"source_hash\": \"24e35278ad5d4d3babe7379dc34d5bce\",\"file_url\": \"https://osskb.org/api/file_contents/38e743a8566d3df4a2dc4432f8d6b091\",\"licenses\": [{\"name\": \"GPL-2.0-only\",\"patent_hints\": \"yes\", \"copyleft\": \"yes\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt\", \"incompatible_with\": \"Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, FTL, IJG, OpenSSL, Python-2.0, zlib-acknowledgement, XFree86-1.1\",\"osadl_updated\": \"2022-02-15\",\"source\": \"component_declared\"},{\"name\": \"GPL-1.0-or-later\",\"source\": \"scancode\"},{\"name\": \"GPL-2.0-only\",\"patent_hints\": \"yes\", \"copyleft\": \"yes\", \"checklist_url\": \"https://www.osadl.org/fileadmin/checklists/unreflicenses/GPL-2.0-only.txt\", \"incompatible_with\": \"Apache-1.0, Apache-1.1, Apache-2.0, BSD-4-Clause, BSD-4-Clause-UC, FTL, IJG, OpenSSL, Python-2.0, zlib-acknowledgement, XFree86-1.1\",\"osadl_updated\": \"2022-02-15\",\"source\": \"scancode\"}],\"server\": {\"version\": \"4.4.1\",\"kb_version\": {\"monthly\":\"22.01\", \"daily\":\"22.02.08\"}}}]}\n\r\n", - "headers" : { - "Server" : "nginx/1.14.2", - "Date" : "Tue, 01 Mar 2022 09:39:58 GMT", - "Content-Type" : "application/json" - } - }, - "uuid" : "0a7de786-48f0-4815-b5f9-8d3c0f435bc5", - "persistent" : true, - "insertionIndex" : 10 -} \ No newline at end of file diff --git a/clients/scanoss/src/test/kotlin/ScanOssDetailsTest.kt b/clients/scanoss/src/test/kotlin/ScanOssDetailsTest.kt deleted file mode 100644 index f4a8ba99928cf..0000000000000 --- a/clients/scanoss/src/test/kotlin/ScanOssDetailsTest.kt +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.clients.scanoss - -import com.github.tomakehurst.wiremock.WireMockServer -import com.github.tomakehurst.wiremock.core.WireMockConfiguration - -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.collections.beEmpty -import io.kotest.matchers.collections.shouldHaveSize -import io.kotest.matchers.maps.shouldHaveKey -import io.kotest.matchers.nulls.shouldNotBeNull -import io.kotest.matchers.should -import io.kotest.matchers.shouldBe -import io.kotest.matchers.shouldNot - -import java.io.File - -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.MultipartBody -import okhttp3.RequestBody.Companion.asRequestBody - -import org.ossreviewtoolkit.clients.scanoss.model.Source - -private const val SCANOSS_DETAILS_RESPONSE_FILENAME = "osskb.c" - -class ScanOssDetailsTest : StringSpec({ - val server = WireMockServer( - WireMockConfiguration.options() - .dynamicPort() - .usingFilesUnderDirectory("src/test/assets/details") - ) - lateinit var service: ScanOssService - - val sampleFile = File("src/test/assets/scan/file.wfp").let { file -> - MultipartBody.Part.createFormData( - "file", - file.name, - file.asRequestBody("application/octet-stream".toMediaType()) - ) - } - - beforeSpec { - server.start() - service = ScanOssService.create("http://localhost:${server.port()}") - } - - afterSpec { - server.stop() - } - - beforeTest { - server.resetAll() - } - - "The response details can be parsed" { - val result = service.scan(sampleFile) - result shouldHaveKey SCANOSS_DETAILS_RESPONSE_FILENAME - result[SCANOSS_DETAILS_RESPONSE_FILENAME] shouldNotBeNull { - this shouldNot beEmpty() - first() shouldNotBeNull { - dependencies should beEmpty() - copyrights shouldHaveSize 2 - copyrights.first() shouldNotBeNull { - source shouldBe Source.FILE_HEADER - } - - vulnerabilities shouldHaveSize 1 - vulnerabilities.first() shouldNotBeNull { - id shouldBe "GHSA-27p5-7cw6-m45h" - } - - quality shouldHaveSize 1 - quality.first() shouldNotBeNull { - score shouldBe "2/5" - } - - cryptography should beEmpty() - } - } - } -}) diff --git a/clients/scanoss/src/test/kotlin/ScanOssServiceTest.kt b/clients/scanoss/src/test/kotlin/ScanOssServiceTest.kt deleted file mode 100644 index c76b0cc16aa5f..0000000000000 --- a/clients/scanoss/src/test/kotlin/ScanOssServiceTest.kt +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (C) 2022 The ORT Project Authors (see ) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * License-Filename: LICENSE - */ - -package org.ossreviewtoolkit.clients.scanoss - -import com.github.tomakehurst.wiremock.WireMockServer -import com.github.tomakehurst.wiremock.core.WireMockConfiguration - -import io.kotest.core.spec.style.StringSpec -import io.kotest.matchers.collections.beEmpty -import io.kotest.matchers.collections.shouldHaveSize -import io.kotest.matchers.maps.shouldHaveKey -import io.kotest.matchers.nulls.shouldNotBeNull -import io.kotest.matchers.shouldBe -import io.kotest.matchers.shouldNot - -import java.io.File - -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.MultipartBody -import okhttp3.RequestBody.Companion.asRequestBody - -import org.ossreviewtoolkit.clients.scanoss.model.IdentificationType - -/** - * The default filename in a SCANOSS response. - */ -const val SCANOSS_RESPONSE_FILENAME = "pasted.wfp" - -class ScanOssServiceTest : StringSpec({ - val server = WireMockServer( - WireMockConfiguration.options() - .dynamicPort() - .usingFilesUnderDirectory("src/test/assets/scan") - ) - lateinit var service: ScanOssService - - val sampleFile = File("src/test/assets/scan/file.wfp").let { file -> - MultipartBody.Part.createFormData( - "file", - file.name, - file.asRequestBody("application/octet-stream".toMediaType()) - ) - } - - beforeSpec { - server.start() - service = ScanOssService.create("http://localhost:${server.port()}") - } - - afterSpec { - server.stop() - } - - beforeTest { - server.resetAll() - } - - "A WFP file scan can be requested" { - val result = service.scan(sampleFile) - result shouldHaveKey SCANOSS_RESPONSE_FILENAME - result[SCANOSS_RESPONSE_FILENAME] shouldNotBeNull { - this shouldNot beEmpty() - first() shouldNotBeNull { - id shouldBe IdentificationType.SNIPPET - licenses shouldHaveSize 3 - licenses[0].name shouldBe "GPL-2.0-only" - licenses[1].name shouldBe "GPL-1.0-or-later" - licenses[2].name shouldBe "GPL-2.0-only" - } - } - } - - "A Boolean value from SCANOSS can be deserialized" { - val result = service.scan(sampleFile) - result shouldHaveKey SCANOSS_RESPONSE_FILENAME - result[SCANOSS_RESPONSE_FILENAME] shouldNotBeNull { - this shouldNot beEmpty() - first() shouldNotBeNull { - licenses shouldHaveSize 3 - licenses[0].copyleft shouldBe true - licenses[0].patentHints shouldBe true - licenses[1].copyleft shouldBe null - licenses[1].patentHints shouldBe null - } - } - } -}) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 7afcaa4dd3fba..c756d9f0852be 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -54,7 +54,7 @@ reflections = "0.10.2" retrofit = "2.11.0" s3 = "2.27.1" saxonHe = "12.5" -scanoss = "1.1.6" +scanoss = "0.7.1" semver4j = "5.3.0" slf4j = "2.0.16" springCore = "5.3.37" @@ -163,7 +163,7 @@ retrofit-converter-jackson = { module = "com.squareup.retrofit2:converter-jackso retrofit-converter-kotlinxSerialization = { module = "com.squareup.retrofit2:converter-kotlinx-serialization", version.ref = "retrofit" } retrofit-converter-scalars = { module = "com.squareup.retrofit2:converter-scalars", version.ref = "retrofit" } saxonHe = { module = "net.sf.saxon:Saxon-HE", version.ref = "saxonHe" } -scanoss = { module = "com.scanoss:scanner", version.ref = "scanoss" } +scanoss = { module = "com.scanoss:scanoss", version.ref = "scanoss" } semver4j = { module = "org.semver4j:semver4j", version.ref = "semver4j" } slf4j = { module = "org.slf4j:slf4j-api ", version.ref = "slf4j" } springCore = { module = "org.springframework:spring-core", version.ref = "springCore" } diff --git a/plugins/scanners/scanoss/build.gradle.kts b/plugins/scanners/scanoss/build.gradle.kts index db4f333a1f156..cbf2aee290790 100644 --- a/plugins/scanners/scanoss/build.gradle.kts +++ b/plugins/scanners/scanoss/build.gradle.kts @@ -28,7 +28,6 @@ dependencies { api(projects.model) api(projects.scanner) - implementation(projects.clients.scanossClient) implementation(projects.utils.commonUtils) implementation(projects.utils.spdxUtils) diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt index de86ccdd9c317..aede5691541e9 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOss.kt @@ -19,24 +19,18 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss -import com.scanoss.scanner.BlacklistRules -import com.scanoss.scanner.Scanner -import com.scanoss.scanner.Winnowing +import com.scanoss.Winnowing +import com.scanoss.dto.ScanFileResult +import com.scanoss.rest.ScanApi +import com.scanoss.utils.JsonUtils +import com.scanoss.utils.PackageDetails import java.io.File import java.time.Instant -import java.util.Properties import java.util.UUID -import kotlinx.coroutines.runBlocking - -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.MultipartBody -import okhttp3.RequestBody.Companion.toRequestBody - import org.apache.logging.log4j.kotlin.logger -import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.ScanSummary import org.ossreviewtoolkit.scanner.PathScannerWrapper import org.ossreviewtoolkit.scanner.ScanContext @@ -46,10 +40,6 @@ import org.ossreviewtoolkit.scanner.ScannerWrapperFactory import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.VCS_DIRECTORIES -// An arbitrary name to use for the multipart body being sent. -private const val FAKE_WFP_FILE_NAME = "fake.wfp" -private const val ARG_FIELD_NAME = "file" - class ScanOss internal constructor( override val name: String, config: ScanOssConfig, @@ -63,13 +53,15 @@ class ScanOss internal constructor( ScanOssConfig.create(options, secrets).also { logger.info { "The $type API URL is ${it.apiUrl}." } } } - private val service = ScanOssService.create(config.apiUrl) + private val service = ScanApi.builder() + // As there is only a single endpoint, the SCANOSS API client expects the path to be part of the API URL. + .url(config.apiUrl.removeSuffix("/") + "/scan/direct") + .apiKey(config.apiKey) + .build() override val version: String by lazy { // TODO: Find out the best / cheapest way to query the SCANOSS server for its version. - val pomProperties = "/META-INF/maven/com.scanoss/scanner/pom.properties" - val properties = Scanner::class.java.getResourceAsStream(pomProperties).use { Properties().apply { load(it) } } - properties.getProperty("version") + PackageDetails.getVersion() } override val configuration = "" @@ -96,34 +88,32 @@ class ScanOss internal constructor( val wfpString = buildString { path.walk() .onEnter { it.name !in VCS_DIRECTORIES } - // TODO: Consider not applying the (somewhat arbitrary) blacklist. - .filterNot { it.isDirectory || BlacklistRules.hasBlacklistedExt(it.name) } + .filterNot { it.isDirectory } .forEach { logger.info { "Computing fingerprint for file ${it.absolutePath}..." } append(createWfpForFile(it)) } } - val response = runBlocking { - val wfpBody = wfpString.toRequestBody("application/octet-stream".toMediaType()) - val wfpFile = MultipartBody.Part.createFormData(ARG_FIELD_NAME, FAKE_WFP_FILE_NAME, wfpBody) - - service.scan(wfpFile) - } + val result = service.scan( + wfpString, + context.labels["scanOssContext"], + context.labels["scanOssId"]?.toIntOrNull() ?: Thread.currentThread().id.toInt() + ) // Replace the anonymized UUIDs by their file paths. - val resolvedResponse = response.map { entry -> - val uuid = UUID.fromString(entry.key) + val results = JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(result)).map { + val uuid = UUID.fromString(it.filePath) val fileName = fileNamesAnonymizationMapping[uuid] ?: throw IllegalArgumentException( "The $name server returned UUID '$uuid' which is not present in the mapping." ) - fileName to entry.value - }.toMap() + ScanFileResult(fileName, it.fileDetails) + } val endTime = Instant.now() - return generateSummary(startTime, endTime, resolvedResponse) + return generateSummary(startTime, endTime, results) } internal fun generateRandomUUID() = UUID.randomUUID() @@ -132,7 +122,7 @@ class ScanOss internal constructor( generateRandomUUID().let { uuid -> // TODO: Let's keep the original file extension to give SCANOSS some hint about the mime type. fileNamesAnonymizationMapping[uuid] = file.path - return Winnowing.wfpForFile(uuid.toString(), file.path) + return Winnowing.builder().build().wfpForFile(file.path, uuid.toString()) } } } diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt index 9a465c45ac255..06fbe01a69eda 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssConfig.kt @@ -19,7 +19,6 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss -import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.config.ScannerConfiguration import org.ossreviewtoolkit.utils.common.Options @@ -46,7 +45,9 @@ data class ScanOssConfig( const val API_KEY_PROPERTY = "apiKey" fun create(options: Options, secrets: Options): ScanOssConfig { - val apiUrl = options[API_URL_PROPERTY] ?: ScanOssService.DEFAULT_API_URL + // TODO: Remove the hard-coded default URL once https://github.com/scanoss/scanoss.java/issues/11 is + // resolved. + val apiUrl = options[API_URL_PROPERTY] ?: "https://api.osskb.org/" val apiKey = secrets[API_KEY_PROPERTY].orEmpty() return ScanOssConfig(apiUrl, apiKey) diff --git a/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt b/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt index 4e4ac4c940dcb..9e206a7b547d6 100644 --- a/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt +++ b/plugins/scanners/scanoss/src/main/kotlin/ScanOssResultParser.kt @@ -19,11 +19,11 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss +import com.scanoss.dto.ScanFileDetails +import com.scanoss.dto.ScanFileResult + import java.time.Instant -import org.ossreviewtoolkit.clients.scanoss.FullScanResponse -import org.ossreviewtoolkit.clients.scanoss.model.IdentificationType -import org.ossreviewtoolkit.clients.scanoss.model.ScanResponse import org.ossreviewtoolkit.model.CopyrightFinding import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.RepositoryProvenance @@ -40,23 +40,23 @@ import org.ossreviewtoolkit.utils.spdx.SpdxExpression * Generate a summary from the given SCANOSS [result], using [startTime], [endTime] as metadata. This variant can be * used if the result is not read from a local file. */ -internal fun generateSummary(startTime: Instant, endTime: Instant, result: FullScanResponse): ScanSummary { +internal fun generateSummary(startTime: Instant, endTime: Instant, results: List): ScanSummary { val licenseFindings = mutableSetOf() val copyrightFindings = mutableSetOf() val snippetFindings = mutableSetOf() - result.forEach { (_, scanResponses) -> - scanResponses.forEach { scanResponse -> - if (scanResponse.id == IdentificationType.FILE) { - licenseFindings += getLicenseFindings(scanResponse) - copyrightFindings += getCopyrightFindings(scanResponse) + results.forEach { result -> + result.fileDetails.forEach { details -> + if (details.id == "file") { + licenseFindings += getLicenseFindings(details) + copyrightFindings += getCopyrightFindings(details) } - if (scanResponse.id == IdentificationType.SNIPPET) { - val file = requireNotNull(scanResponse.file) - val lines = requireNotNull(scanResponse.lines) + if (details.id == "snippet") { + val file = requireNotNull(details.file) + val lines = requireNotNull(details.lines) val sourceLocation = convertLines(file, lines) - val snippets = getSnippets(scanResponse) + val snippets = getSnippets(details) snippets.forEach { // TODO: Aggregate the snippet by source file location. @@ -76,13 +76,13 @@ internal fun generateSummary(startTime: Instant, endTime: Instant, result: FullS } /** - * Get the license findings from the given [scanResponse]. + * Get the license findings from the given [details]. */ -private fun getLicenseFindings(scanResponse: ScanResponse): List { - val path = scanResponse.file ?: return emptyList() - val score = scanResponse.matched?.removeSuffix("%")?.toFloatOrNull() +private fun getLicenseFindings(details: ScanFileDetails): List { + val path = details.file ?: return emptyList() + val score = details.matched?.removeSuffix("%")?.toFloatOrNull() - return scanResponse.licenses.map { license -> + return details.licenseDetails.orEmpty().map { license -> val licenseExpression = runCatching { SpdxExpression.parse(license.name) }.getOrNull() val validatedLicense = when { @@ -104,12 +104,12 @@ private fun getLicenseFindings(scanResponse: ScanResponse): List } /** - * Get the copyright findings from the given [scanResponse]. + * Get the copyright findings from the given [details]. */ -private fun getCopyrightFindings(scanResponse: ScanResponse): List { - val path = scanResponse.file ?: return emptyList() +private fun getCopyrightFindings(details: ScanFileDetails): List { + val path = details.file ?: return emptyList() - return scanResponse.copyrights.map { copyright -> + return details.copyrightDetails.orEmpty().map { copyright -> CopyrightFinding( statement = copyright.name, location = TextLocation( @@ -122,17 +122,17 @@ private fun getCopyrightFindings(scanResponse: ScanResponse): List { - val matched = requireNotNull(scanResponse.matched) - val fileUrl = requireNotNull(scanResponse.fileUrl) - val ossLines = requireNotNull(scanResponse.ossLines) - val url = requireNotNull(scanResponse.url) - val purls = requireNotNull(scanResponse.purl) - - val licenses = scanResponse.licenses.map { license -> +private fun getSnippets(details: ScanFileDetails): Set { + val matched = requireNotNull(details.matched) + val fileUrl = requireNotNull(details.fileUrl) + val ossLines = requireNotNull(details.ossLines) + val url = requireNotNull(details.url) + val purls = requireNotNull(details.purls) + + val licenses = details.licenseDetails.orEmpty().map { license -> SpdxExpression.parse(license.name) }.toSet() diff --git a/plugins/scanners/scanoss/src/test/assets/.gitattributes b/plugins/scanners/scanoss/src/test/assets/.gitattributes index a6a530baeebdd..f923d7ac6bf07 100644 --- a/plugins/scanners/scanoss/src/test/assets/.gitattributes +++ b/plugins/scanners/scanoss/src/test/assets/.gitattributes @@ -1,3 +1,4 @@ # Use Unix line endings for scan results which are misused for also testing the calculation of the SPDX package verification code. *.json text eol=lf filesToScan/*.kt text eol=lf +wfp/*.license text eol=lf diff --git a/plugins/scanners/scanoss/src/test/assets/wfp/Apache-2.0.license b/plugins/scanners/scanoss/src/test/assets/wfp/Apache-2.0.license new file mode 100644 index 0000000000000..8dada3edaf50d --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/wfp/Apache-2.0.license @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/scanners/scanoss/src/test/assets/wfp/CC-BY-3.0.license b/plugins/scanners/scanoss/src/test/assets/wfp/CC-BY-3.0.license new file mode 100644 index 0000000000000..1a16e05564d2a --- /dev/null +++ b/plugins/scanners/scanoss/src/test/assets/wfp/CC-BY-3.0.license @@ -0,0 +1,319 @@ +Creative Commons Legal Code + +Attribution 3.0 Unported + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR + DAMAGES RESULTING FROM ITS USE. + +License + +THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE +COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY +COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS +AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. + +BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE +TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY +BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS +CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND +CONDITIONS. + +1. Definitions + + a. "Adaptation" means a work based upon the Work, or upon the Work and + other pre-existing works, such as a translation, adaptation, + derivative work, arrangement of music or other alterations of a + literary or artistic work, or phonogram or performance and includes + cinematographic adaptations or any other form in which the Work may be + recast, transformed, or adapted including in any form recognizably + derived from the original, except that a work that constitutes a + Collection will not be considered an Adaptation for the purpose of + this License. For the avoidance of doubt, where the Work is a musical + work, performance or phonogram, the synchronization of the Work in + timed-relation with a moving image ("synching") will be considered an + Adaptation for the purpose of this License. + b. "Collection" means a collection of literary or artistic works, such as + encyclopedias and anthologies, or performances, phonograms or + broadcasts, or other works or subject matter other than works listed + in Section 1(f) below, which, by reason of the selection and + arrangement of their contents, constitute intellectual creations, in + which the Work is included in its entirety in unmodified form along + with one or more other contributions, each constituting separate and + independent works in themselves, which together are assembled into a + collective whole. A work that constitutes a Collection will not be + considered an Adaptation (as defined above) for the purposes of this + License. + c. "Distribute" means to make available to the public the original and + copies of the Work or Adaptation, as appropriate, through sale or + other transfer of ownership. + d. "Licensor" means the individual, individuals, entity or entities that + offer(s) the Work under the terms of this License. + e. "Original Author" means, in the case of a literary or artistic work, + the individual, individuals, entity or entities who created the Work + or if no individual or entity can be identified, the publisher; and in + addition (i) in the case of a performance the actors, singers, + musicians, dancers, and other persons who act, sing, deliver, declaim, + play in, interpret or otherwise perform literary or artistic works or + expressions of folklore; (ii) in the case of a phonogram the producer + being the person or legal entity who first fixes the sounds of a + performance or other sounds; and, (iii) in the case of broadcasts, the + organization that transmits the broadcast. + f. "Work" means the literary and/or artistic work offered under the terms + of this License including without limitation any production in the + literary, scientific and artistic domain, whatever may be the mode or + form of its expression including digital form, such as a book, + pamphlet and other writing; a lecture, address, sermon or other work + of the same nature; a dramatic or dramatico-musical work; a + choreographic work or entertainment in dumb show; a musical + composition with or without words; a cinematographic work to which are + assimilated works expressed by a process analogous to cinematography; + a work of drawing, painting, architecture, sculpture, engraving or + lithography; a photographic work to which are assimilated works + expressed by a process analogous to photography; a work of applied + art; an illustration, map, plan, sketch or three-dimensional work + relative to geography, topography, architecture or science; a + performance; a broadcast; a phonogram; a compilation of data to the + extent it is protected as a copyrightable work; or a work performed by + a variety or circus performer to the extent it is not otherwise + considered a literary or artistic work. + g. "You" means an individual or entity exercising rights under this + License who has not previously violated the terms of this License with + respect to the Work, or who has received express permission from the + Licensor to exercise rights under this License despite a previous + violation. + h. "Publicly Perform" means to perform public recitations of the Work and + to communicate to the public those public recitations, by any means or + process, including by wire or wireless means or public digital + performances; to make available to the public Works in such a way that + members of the public may access these Works from a place and at a + place individually chosen by them; to perform the Work to the public + by any means or process and the communication to the public of the + performances of the Work, including by public digital performance; to + broadcast and rebroadcast the Work by any means including signs, + sounds or images. + i. "Reproduce" means to make copies of the Work by any means including + without limitation by sound or visual recordings and the right of + fixation and reproducing fixations of the Work, including storage of a + protected performance or phonogram in digital form or other electronic + medium. + +2. Fair Dealing Rights. Nothing in this License is intended to reduce, +limit, or restrict any uses free from copyright or rights arising from +limitations or exceptions that are provided for in connection with the +copyright protection under copyright law or other applicable laws. + +3. License Grant. Subject to the terms and conditions of this License, +Licensor hereby grants You a worldwide, royalty-free, non-exclusive, +perpetual (for the duration of the applicable copyright) license to +exercise the rights in the Work as stated below: + + a. to Reproduce the Work, to incorporate the Work into one or more + Collections, and to Reproduce the Work as incorporated in the + Collections; + b. to create and Reproduce Adaptations provided that any such Adaptation, + including any translation in any medium, takes reasonable steps to + clearly label, demarcate or otherwise identify that changes were made + to the original Work. For example, a translation could be marked "The + original work was translated from English to Spanish," or a + modification could indicate "The original work has been modified."; + c. to Distribute and Publicly Perform the Work including as incorporated + in Collections; and, + d. to Distribute and Publicly Perform Adaptations. + e. For the avoidance of doubt: + + i. Non-waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme cannot be waived, the Licensor + reserves the exclusive right to collect such royalties for any + exercise by You of the rights granted under this License; + ii. Waivable Compulsory License Schemes. In those jurisdictions in + which the right to collect royalties through any statutory or + compulsory licensing scheme can be waived, the Licensor waives the + exclusive right to collect such royalties for any exercise by You + of the rights granted under this License; and, + iii. Voluntary License Schemes. The Licensor waives the right to + collect royalties, whether individually or, in the event that the + Licensor is a member of a collecting society that administers + voluntary licensing schemes, via that society, from any exercise + by You of the rights granted under this License. + +The above rights may be exercised in all media and formats whether now +known or hereafter devised. The above rights include the right to make +such modifications as are technically necessary to exercise the rights in +other media and formats. Subject to Section 8(f), all rights not expressly +granted by Licensor are hereby reserved. + +4. Restrictions. The license granted in Section 3 above is expressly made +subject to and limited by the following restrictions: + + a. You may Distribute or Publicly Perform the Work only under the terms + of this License. You must include a copy of, or the Uniform Resource + Identifier (URI) for, this License with every copy of the Work You + Distribute or Publicly Perform. You may not offer or impose any terms + on the Work that restrict the terms of this License or the ability of + the recipient of the Work to exercise the rights granted to that + recipient under the terms of the License. You may not sublicense the + Work. You must keep intact all notices that refer to this License and + to the disclaimer of warranties with every copy of the Work You + Distribute or Publicly Perform. When You Distribute or Publicly + Perform the Work, You may not impose any effective technological + measures on the Work that restrict the ability of a recipient of the + Work from You to exercise the rights granted to that recipient under + the terms of the License. This Section 4(a) applies to the Work as + incorporated in a Collection, but this does not require the Collection + apart from the Work itself to be made subject to the terms of this + License. If You create a Collection, upon notice from any Licensor You + must, to the extent practicable, remove from the Collection any credit + as required by Section 4(b), as requested. If You create an + Adaptation, upon notice from any Licensor You must, to the extent + practicable, remove from the Adaptation any credit as required by + Section 4(b), as requested. + b. If You Distribute, or Publicly Perform the Work or any Adaptations or + Collections, You must, unless a request has been made pursuant to + Section 4(a), keep intact all copyright notices for the Work and + provide, reasonable to the medium or means You are utilizing: (i) the + name of the Original Author (or pseudonym, if applicable) if supplied, + and/or if the Original Author and/or Licensor designate another party + or parties (e.g., a sponsor institute, publishing entity, journal) for + attribution ("Attribution Parties") in Licensor's copyright notice, + terms of service or by other reasonable means, the name of such party + or parties; (ii) the title of the Work if supplied; (iii) to the + extent reasonably practicable, the URI, if any, that Licensor + specifies to be associated with the Work, unless such URI does not + refer to the copyright notice or licensing information for the Work; + and (iv) , consistent with Section 3(b), in the case of an Adaptation, + a credit identifying the use of the Work in the Adaptation (e.g., + "French translation of the Work by Original Author," or "Screenplay + based on original Work by Original Author"). The credit required by + this Section 4 (b) may be implemented in any reasonable manner; + provided, however, that in the case of a Adaptation or Collection, at + a minimum such credit will appear, if a credit for all contributing + authors of the Adaptation or Collection appears, then as part of these + credits and in a manner at least as prominent as the credits for the + other contributing authors. For the avoidance of doubt, You may only + use the credit required by this Section for the purpose of attribution + in the manner set out above and, by exercising Your rights under this + License, You may not implicitly or explicitly assert or imply any + connection with, sponsorship or endorsement by the Original Author, + Licensor and/or Attribution Parties, as appropriate, of You or Your + use of the Work, without the separate, express prior written + permission of the Original Author, Licensor and/or Attribution + Parties. + c. Except as otherwise agreed in writing by the Licensor or as may be + otherwise permitted by applicable law, if You Reproduce, Distribute or + Publicly Perform the Work either by itself or as part of any + Adaptations or Collections, You must not distort, mutilate, modify or + take other derogatory action in relation to the Work which would be + prejudicial to the Original Author's honor or reputation. Licensor + agrees that in those jurisdictions (e.g. Japan), in which any exercise + of the right granted in Section 3(b) of this License (the right to + make Adaptations) would be deemed to be a distortion, mutilation, + modification or other derogatory action prejudicial to the Original + Author's honor and reputation, the Licensor will waive or not assert, + as appropriate, this Section, to the fullest extent permitted by the + applicable national law, to enable You to reasonably exercise Your + right under Section 3(b) of this License (right to make Adaptations) + but not otherwise. + +5. Representations, Warranties and Disclaimer + +UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR +OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY +KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, +INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, +FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF +LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, +WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION +OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. + +6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE +LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR +ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES +ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. Termination + + a. This License and the rights granted hereunder will terminate + automatically upon any breach by You of the terms of this License. + Individuals or entities who have received Adaptations or Collections + from You under this License, however, will not have their licenses + terminated provided such individuals or entities remain in full + compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will + survive any termination of this License. + b. Subject to the above terms and conditions, the license granted here is + perpetual (for the duration of the applicable copyright in the Work). + Notwithstanding the above, Licensor reserves the right to release the + Work under different license terms or to stop distributing the Work at + any time; provided, however that any such election will not serve to + withdraw this License (or any other license that has been, or is + required to be, granted under the terms of this License), and this + License will continue in full force and effect unless terminated as + stated above. + +8. Miscellaneous + + a. Each time You Distribute or Publicly Perform the Work or a Collection, + the Licensor offers to the recipient a license to the Work on the same + terms and conditions as the license granted to You under this License. + b. Each time You Distribute or Publicly Perform an Adaptation, Licensor + offers to the recipient a license to the original Work on the same + terms and conditions as the license granted to You under this License. + c. If any provision of this License is invalid or unenforceable under + applicable law, it shall not affect the validity or enforceability of + the remainder of the terms of this License, and without further action + by the parties to this agreement, such provision shall be reformed to + the minimum extent necessary to make such provision valid and + enforceable. + d. No term or provision of this License shall be deemed waived and no + breach consented to unless such waiver or consent shall be in writing + and signed by the party to be charged with such waiver or consent. + e. This License constitutes the entire agreement between the parties with + respect to the Work licensed here. There are no understandings, + agreements or representations with respect to the Work not specified + here. Licensor shall not be bound by any additional provisions that + may appear in any communication from You. This License may not be + modified without the mutual written agreement of the Licensor and You. + f. The rights granted under, and the subject matter referenced, in this + License were drafted utilizing the terminology of the Berne Convention + for the Protection of Literary and Artistic Works (as amended on + September 28, 1979), the Rome Convention of 1961, the WIPO Copyright + Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 + and the Universal Copyright Convention (as revised on July 24, 1971). + These rights and subject matter take effect in the relevant + jurisdiction in which the License terms are sought to be enforced + according to the corresponding provisions of the implementation of + those treaty provisions in the applicable national law. If the + standard suite of rights granted under applicable copyright law + includes additional rights not granted under this License, such + additional rights are deemed to be included in the License; this + License is not intended to restrict the license of any rights under + applicable law. + + +Creative Commons Notice + + Creative Commons is not a party to this License, and makes no warranty + whatsoever in connection with the Work. Creative Commons will not be + liable to You or any party on any legal theory for any damages + whatsoever, including without limitation any general, special, + incidental or consequential damages arising in connection to this + license. Notwithstanding the foregoing two (2) sentences, if Creative + Commons has expressly identified itself as the Licensor hereunder, it + shall have all rights and obligations of Licensor. + + Except for the limited purpose of indicating to the public that the + Work is licensed under the CCPL, Creative Commons does not authorize + the use by either party of the trademark "Creative Commons" or any + related trademark or logo of Creative Commons without the prior + written consent of Creative Commons. Any permitted use will be in + compliance with Creative Commons' then-current trademark usage + guidelines, as may be published on its website or otherwise made + available upon request from time to time. For the avoidance of doubt, + this trademark restriction does not form part of this License. + + Creative Commons may be contacted at https://creativecommons.org/. diff --git a/plugins/scanners/scanoss/src/test/assets/wfp/sigmod03.pdf b/plugins/scanners/scanoss/src/test/assets/wfp/sigmod03.pdf new file mode 100644 index 0000000000000..0d72d0900a146 Binary files /dev/null and b/plugins/scanners/scanoss/src/test/assets/wfp/sigmod03.pdf differ diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt index 033f84d08b75d..774c51af61771 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssConfigTest.kt @@ -24,12 +24,10 @@ import io.kotest.matchers.should import io.kotest.matchers.shouldBe import io.kotest.matchers.string.beEmpty -import org.ossreviewtoolkit.clients.scanoss.ScanOssService - class ScanOssConfigTest : StringSpec({ "Default values are used" { with(ScanOssConfig.create(emptyMap(), emptyMap())) { - apiUrl shouldBe ScanOssService.DEFAULT_API_URL + apiUrl shouldBe "https://api.osskb.org/" apiKey should beEmpty() } } diff --git a/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt b/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt index 34de6507d53c7..2df7fd37bad66 100644 --- a/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt +++ b/plugins/scanners/scanoss/src/test/kotlin/ScanOssResultParserTest.kt @@ -19,6 +19,8 @@ package org.ossreviewtoolkit.plugins.scanners.scanoss +import com.scanoss.utils.JsonUtils + import io.kotest.core.spec.style.WordSpec import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.collections.haveSize @@ -30,10 +32,6 @@ import io.kotest.matchers.should import java.io.File import java.time.Instant -import kotlinx.serialization.json.decodeFromStream - -import org.ossreviewtoolkit.clients.scanoss.FullScanResponse -import org.ossreviewtoolkit.clients.scanoss.ScanOssService import org.ossreviewtoolkit.model.CopyrightFinding import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.RepositoryProvenance @@ -47,12 +45,12 @@ import org.ossreviewtoolkit.utils.spdx.SpdxExpression class ScanOssResultParserTest : WordSpec({ "generateSummary()" should { "properly summarize JUnit 4.12 findings" { - val result = File("src/test/assets/scanoss-junit-4.12.json").inputStream().use { - ScanOssService.JSON.decodeFromStream(it) + val results = File("src/test/assets/scanoss-junit-4.12.json").readText().let { + JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(it)) } val time = Instant.now() - val summary = generateSummary(time, time, result) + val summary = generateSummary(time, time, results) summary.licenses.map { it.toString() } should containExactlyInAnyOrder( "Apache-2.0", @@ -85,12 +83,12 @@ class ScanOssResultParserTest : WordSpec({ } "properly summarize Semver4j 3.1.0 with snippet findings" { - val result = File("src/test/assets/scanoss-semver4j-3.1.0-with-snippet.json").inputStream().use { - ScanOssService.JSON.decodeFromStream(it) + val results = File("src/test/assets/scanoss-semver4j-3.1.0-with-snippet.json").readText().let { + JsonUtils.toScanFileResultsFromObject(JsonUtils.toJsonObject(it)) } val time = Instant.now() - val summary = generateSummary(time, time, result) + val summary = generateSummary(time, time, results) summary.licenses.map { it.toString() } should containExactlyInAnyOrder( "Apache-2.0", diff --git a/plugins/scanners/scanoss/src/test/kotlin/WinnowingTest.kt b/plugins/scanners/scanoss/src/test/kotlin/WinnowingTest.kt new file mode 100644 index 0000000000000..14dca61a90bc4 --- /dev/null +++ b/plugins/scanners/scanoss/src/test/kotlin/WinnowingTest.kt @@ -0,0 +1,447 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.scanners.scanoss + +import com.scanoss.Winnowing + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.shouldBe + +import java.io.File + +private val winnowing = Winnowing.builder().build() + +private fun calculate(file: File) = winnowing.wfpForFile(file.path, file.name) + +class WinnowingTest : StringSpec({ + "The fingerprint should be calculated correctly for the Apache-2.0 license text" { + calculate(File("src/test/assets/wfp/Apache-2.0.license")) shouldBe """ + file=e3fc50a88d0a364313df4b21ef20c29e,11357,Apache-2.0.license + 5=33193f1b,1716113f,0c557e09 + 9=643a45ae,2ae8e84a,bacd17ac + 10=c54959e6,c8c4ada9 + 12=095e9c9e,10754b25 + 13=8de4c91b + 15=1906f673,b70c2c99 + 17=82844dcf,9d75ed8c + 18=e09ca533,163fd100,e3ab20a5 + 19=9aa08fa2 + 20=dfb7c8e2 + 21=62a267bf,89999f72 + 23=fa77a2c6 + 24=780b3e93 + 26=f9722399,cad7579b,a73214cc,a3cd5e97 + 28=08f2243a + 30=0fad4d4c,0ed8d3be + 32=dd22b5a4,97fc9200 + 35=cdb44ffc,78a155f0 + 36=4d1ab0e2 + 37=6a9de676,a8ec6cfd + 38=c9befb24 + 40=6e2db6c0 + 41=ebbea868,46493752 + 42=0f422343 + 43=075081b0 + 44=42d0762e,7440b7b9,63517188 + 45=165b2ad4,edf3c34c + 46=e1c66bc4 + 48=622f4a87 + 49=cd74d072,a4b7a885,91d65b18 + 50=78292613 + 51=b4ef74e6 + 52=2aaf3e45,6fe9ce24,f96a3d30 + 53=21c17277 + 54=d40c1035,60aff2a9 + 55=7f733a47,9344b1df,eb9357ec,e51a39b1,09460717 + 56=8ad000d9,c785f8e6,e4532416 + 57=5e702dd7 + 58=2f6e9882,7a964a06,708b34c0 + 59=188207f5,d4c28400 + 60=f4758426,9a99d40c + 62=996ce703,e24cad4b + 63=3fe90b35,85eed91b + 64=9daab9e2 + 66=e944b576,09ec5b5b + 68=1e2e9525,85b8711d + 69=519c4a94,18c48e24,8089bd5b + 70=464418d4 + 71=f59e453b + 73=e7e9e58b,4d46954e,09ec5b5b + 75=1e2e9525,85b8711d + 76=519c4a94,65fed3d8 + 77=d86789c1,c03adbf1 + 78=e8abf2cc,6132f953 + 79=4a3a6437,1d0ec63f + 80=961dfaa7 + 81=88805158,413325d7,9d0a82fa,2715bb0e + 83=459e6ad4,8d4e7642 + 84=3746bff1 + 85=0293b704,e5d56bac,a4ada2fd + 86=fb10d563,e158cdb1 + 87=4c54b951,dffe695b + 90=868d4d34,2031e5cc,fab79c22 + 91=49754210,75c8c3a2 + 94=d4f9c2cf,1e035e8c + 97=0b6589d1,3f5a04e8,ff3a0280 + 100=847627e2,308b3a5b + 101=87eb119e + 102=a83f8a22 + 103=9641afec + 106=c95fb1b1,18ee73d3 + 107=2e03d45a,87eb119e,365a6794 + 108=0b7da30f + 109=63cde0e4,2be61d91,c95fb1b1 + 110=114d09d2 + 112=7a48523d,7115bf81 + 113=b5ee6564,b1de8d55 + 115=6acd7f03,214ef97e,7f260a40 + 116=59c1c016,74a954f8 + 117=2c89593a,7f46ecf0 + 118=2e3a07bf,87f2a81f + 119=e106d187,7a534d67 + 120=f0ee62b1,2201f7a1 + 123=d3ffcd2d,37d5a047 + 124=8739f582 + 125=7c7eb915,69232603 + 126=263d8c6d + 127=8e789e91,51d5f695,66be4909 + 128=af50e92f + 130=a45daefd + 131=5e7a2cb7,3ef85c88 + 132=052ff589 + 133=7fa59123 + 134=4f6b9e7a,e0828558,89c2f58e + 135=b6c0ec4e + 136=a65c75c1,895a98a4 + 139=d2ad211e,8c145b15 + 140=08b6afb1 + 141=475f127b,5ee108a6 + 143=65cb6303 + 144=efcc7681,2205d383,ff5dfccd,17609495 + 146=57b9fa12 + 147=a6ecc461,396fcc7e,22ba3918 + 148=e91d2c20 + 149=485a7102,505a3c9b,a5e0ae3b + 151=161f0f33,b5609d22,c28cfd88 + 153=dbb3d3eb + 154=3ecbfcdc + 155=8bb06c37,e24b9112,d84c0611 + 156=fcfb88a6 + 157=cf4eda1e,769eb0a3,5743ecc8 + 158=986b4ce7 + 159=6d717d98,761da93f,d2c51d9d + 161=5e153bd1,e0d6f354,6a77b096 + 162=b93373f6,767707a3 + 163=d9d542cd + 165=0fd68808,f7e266d8 + 167=c723e5df + 168=b0de22c2,a1a1527c + 169=e46356e3,44c2f10a,fe79d01e,97281b1d + 171=38dfdc96,b23c851e,3049cb0a,44d62a74,e99d13ed + 172=adb77f40,6b56e70e + 173=c324ee9c + 174=74544c0c + 176=c0571d2a + 178=dbc9bd6f,a1108271 + 180=3017532e + 181=cdd249e9 + 182=708e7097,96570df0 + 183=7ac2159e + 184=40cd3f42,c453dac3 + 185=80c8be5e,2564c555 + 186=58537bf5,a556181e + 187=f8726ff7 + 189=096d9c95,b353c2c1 + 191=d1e03b12 + 192=fb0d20c4 + 193=028ead5a,cab55853,d7aead00 + 195=93951dcd + 197=ff5dfccd,064b3e26 + 199=d3f942b8,8fe9d4f5,2a9536b0 + 200=abbf89cb,579ca428 + + """.trimIndent() + } + + "The fingerprint should be calculated correctly for the CC-BY-3.0 license text" { + calculate(File("src/test/assets/wfp/CC-BY-3.0.license")) shouldBe """ + file=6dffb34dbf23fffe10cc646d9c030e14,19467,CC-BY-3.0.license + 5=996e9d8b,1a6dc238 + 6=70a3753f,14d9bf35,0dbf4b11,d58971b7,86fbc9f3 + 8=38a05276,065a11d9 + 9=40be60c9 + 12=209acf93 + 14=7c3a24a3 + 15=c38cf970,466ca7dc + 16=919511d3,8a9102e8,40b284be,d9266ff1,805b32ad + 19=86eacf9b,c81b73e2 + 20=1e033450,5795e718,5aca67b8 + 22=3fd2e8a4,343551a3,64b16514,e3f1b701 + 27=a76c7c2d + 28=bd096f88,5e546ea2 + 29=1437d5f9,325cf2e3 + 30=a733b3e5,25af8372 + 31=be495901 + 32=3c339748,4f4653d0 + 33=6fa5686e,8645a350 + 34=f860eb15,8b9b1694,d893b55f + 36=77a04aee,c8127973 + 37=69082aaf,a6e7784d + 38=6e81d7a2,d893b55f + 39=e16ee1b1 + 40=ba81dae3,43e4637c + 41=04e45076 + 42=311db27a,73be7e0f + 43=0a4ed7ad,d3fd5534 + 44=7e0fd8c0,cca26f8e + 45=2fd3017e + 46=16e05ee5,47fade55 + 47=ce09763d,ff2a6946 + 48=8b9b1694,423df531 + 50=f2718c09 + 51=889fc712,71405538 + 53=37b1c352,caac3e6a,c616211c,341fa48a + 54=cdee6dd5 + 55=315930af,8741ac46 + 57=341fa48a,cea2ca24 + 58=343016f4,be498bd4 + 60=854cc811,6153e6d2,71d402b5,bbcc9383,b132c5be + 61=bdcf408f + 62=6e2805d2 + 63=e6a59edd,af9d954a,cc21cbf4,9397cfc1 + 64=be52a47c + 65=2971bc06 + 66=f2d04fa4,d362e8e2 + 67=6a383757 + 68=2c1eb7a2,e6076e1d,4d95288f + 69=4416e808 + 70=a0f7c877 + 71=d6531bb9 + 72=a0b8ea06,e0b0c0ed + 73=589ba8bb + 75=9923bbfe,b76d2c92,78daf969 + 76=2b9583f1 + 77=5b224371 + 78=548057a9,56f711d8,575d37e2 + 79=65cf1e0b + 80=cf287ae2 + 81=eeab16c0,bb39b03f,c158f555,89894fe3 + 82=ee951a68 + 83=4e2d65ef + 84=04223a61 + 85=98df7fd7,889d0fca + 86=1aa726dd,8196acca,7732fb2e + 88=25b4c65c,438ff750,13226cc8 + 90=2a6d52d6,1536466a,4cd9d222 + 91=0e17a9b4 + 92=f8d4430e,8e2b9262,337ee49e + 93=d8d0241d + 94=ccf7fcd4 + 95=27d161a1,d82c7466,eaf25271 + 96=e82e47bb + 98=1d3562d1 + 99=0f2de545,2ed05c37,cc4fa387 + 100=d7b89c5a,9651375c + 101=8e9d2f99,3857dc7d + 102=e5ec2f61 + 104=fd0e0a0a + 105=71097e12,86031b43 + 106=86f7b777 + 107=8b339abb,767f1ccc + 109=09ec5b5b + 110=4a75259d + 111=5b540506,2f330cd6 + 112=1e8baa7a + 114=8ca02d83,d181a2ee + 115=9cce42a1 + 117=86a1a220,26c8f4ce + 118=e9bb38f0,6c3b792d + 119=80245504,b7c4275d + 120=e512cb20 + 122=7b877508,b7d64ff2 + 123=7f7faa51,3c8a15d6,b945dac8 + 125=3a6885fa + 128=595d9c7d,b1c3131a,b1054311 + 130=176b4d65 + 131=552322c3 + 132=39d3391c,d3c119e3,6dba98be + 133=3612d955,b1054311 + 135=176b4d65,cfc872a6,e1f534e6 + 137=6dba98be + 138=69f8020d + 139=8110b212 + 140=ac47da61,58d0d53e + 142=3f7323d4,dc6e88e2,3612d955 + 144=23847681 + 145=a81007f6,6c109f94 + 146=db901615 + 147=a190783a,469aeb91 + 150=7af883f2,81742c47 + 151=30b7f89d + 153=6a258a8d,6d79dbfb,2b667545 + 154=bd96fe24,bf50ef4b + 156=874b2db0,53a48725,0a36b9f1 + 157=7db83c86,61311f6a + 158=53159e1d,463f1815 + 160=96fc5a94,e7a6c096 + 161=e1407d9a,afc0b125,8cf1412a,6edf7576 + 162=53a48725 + 163=990299a3 + 164=ffbcb502,72b6eee7 + 165=4a3b96da,926c873e,463f1815 + 166=96fc5a94 + 167=91937c8e + 168=74dcd511 + 169=f75176d3,c7c9d5e4,422c8308,ec7707d2 + 170=ecb75d36 + 172=fbdf710e + 173=1838c008 + 174=280c0db6,efed2463 + 175=2b9ae0e2 + 176=cc0d7f48 + 177=c07bb49f,eba7bf30 + 178=621cba11 + 179=a3389cbf + 180=b25341c0,c3c6b3c1 + 181=ca375f87,e79f8103 + 182=bd59471e,ad31744a + 183=21542008 + 184=62e0f9b9 + 185=17756753,1a8e5ecd + 187=ad71497e + 188=3b7d49b7,0f7cedec,32551f09 + 190=2b75c595,d74f5df2,e49a28fc + 191=54c31657 + 192=89456130 + 193=e299a9b1,b483acc3 + 194=d3751394 + 195=9220c7d5,5f22cc60,ab9d29f4 + 196=656ea59e + 197=18e4e674,20c125a0 + 199=5c0e6186 + 200=333fbfdf,e90be40d + 201=4358912a + 202=44779d50,65aa76d8 + 203=3cc50754 + 205=97ba92f3,88978d3b + 206=fcb49efa,0ae679c9,78299c2c + 207=2e19de94,c236b6be + 208=5db5515f + 209=cc0d7f48 + 210=168ba04d,3746c6a4 + 211=d3701d39,19b53075 + 212=6ee318eb,9badb5d4,1bbd31d5 + 213=b5058d7f + 214=38a53b6c + 215=c5d90cde + 216=a7630a58 + 217=d3cf1631,80085603,5e287c2d + 218=fdafa926 + 219=6b19f33f + 220=d9e8eacd,ad4c22f9 + 224=899e2509,5b166640 + 225=de7b5976 + 226=4aef1372,06df53b1,f45611af + 227=6bd3b904,a698df65 + 228=e284984a,d29facaa + 229=a2917cc1,52caa4e1,445f6090 + 230=9f83005f,cc043ccf,d39c8033 + 231=e33f8a80,ad16bc58 + 234=f13448ee,a3b2393e + 235=e60b5450 + 236=8116203f,edb253ba + 237=0bc3223d + 239=2a65df10,b3214933 + 241=0bc1f684 + 242=47ab5082,7324b121,2439b300 + 243=af0be73d + 244=55af4b2b,d284dcc1,c316abc4,5d0233ec + 245=43dc755c + 246=69eb4bf0,35aab8a1 + 248=cdaf5893,ad730d35,d380d0d0,ef5d7c8c + 249=965063b4,adb6a6e6 + 250=63d3be03,f987694f,46317da9 + 251=86de27f3 + 252=b9402f7f,2738bc91,80ecbcba + 254=bd105c74,53238d58 + 255=4ef2cc2d,e4dd3a43,e9368d53,8ef1a17d + 260=edff6af2 + 261=2b667545,b704d0d4,f6121499 + 262=0aa0b189 + 263=f505f5b7,98f19e72 + 264=9607fe02 + 265=d0e120ca + 266=75a37628,539c9984 + 267=63bb8001,d056a503 + 268=82c4bda0,db23ee3b + 269=a794639c + 270=90b2477b + 272=1d7a9b2b + 273=cb585c7c,d337b261 + 274=3bacf32d + 275=3c85e934 + 276=3e9d915f + 277=407e50ab,100b3e9c + 278=d919c919,41f7f6c9 + 279=ffe655c1 + 280=bb75b62b + 281=fd8c0eda,80aed826 + 282=9d2a4ec0,fc329cad + 283=3c9dec21,0837ec68 + 284=e617b189 + 285=297feecf + 286=70c176ac,73da16c5 + 287=7bc01c8f,f6603bb6,2ee66c0e,25e6770b + 288=762ae77c + 289=cf2c5941,ef58f204 + 290=957fe931,573c80e6 + 292=798fd442,4b8de504,7719c7c5 + 293=791f94a8,13eee247,1ecbd858 + 294=12768726 + 298=0d58979d + 300=7a01f934,01bf219a,44c62e84,4f021f05 + 302=aee2f91d,9c543eee + 303=45466d3c,b5bb169a + 304=a288071d + 305=4c51082e + 306=27116cb3 + 307=e33456c6 + 309=8846fe22,adfed4fe,8f5562d9,9f621163 + 310=4179947d + 311=09153697 + 312=db17de6c + 313=67511552,28cf7201,abaa6931 + 314=e590f3ed + 315=37d4e6f9,7a9ff9f5 + 316=541c72ce + 317=2c82086e,9b8228d3 + 319=2d3b840b,cb841dec + + """.trimIndent() + } + + "The fingerprint should be calculated correctly for the Winnowing paper" { + // Note that the ".pdf" file extension is hard-coded to be skipped for snippet caclulation. + calculate(File("src/test/assets/wfp/sigmod03.pdf")) shouldBe """ + file=232512a681d2488f61ade07516f899b5,155473,sigmod03.pdf + + """.trimIndent() + } +}) diff --git a/settings.gradle.kts b/settings.gradle.kts index 390f154f17d57..93d4480b57e13 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -34,7 +34,6 @@ include(":clients:github-graphql") include(":clients:nexus-iq") include(":clients:oss-index") include(":clients:osv") -include(":clients:scanoss") include(":clients:vulnerable-code") include(":detekt-rules") include(":downloader") @@ -58,7 +57,6 @@ project(":clients:github-graphql").name = "github-graphql-client" project(":clients:nexus-iq").name = "nexus-iq-client" project(":clients:oss-index").name = "oss-index-client" project(":clients:osv").name = "osv-client" -project(":clients:scanoss").name = "scanoss-client" project(":clients:vulnerable-code").name = "vulnerable-code-client" project(":utils:common").name = "common-utils"