diff --git a/clients/dos/src/main/kotlin/DosClient.kt b/clients/dos/src/main/kotlin/DosClient.kt index 872b47bbf677b..4cd75464b2cfd 100644 --- a/clients/dos/src/main/kotlin/DosClient.kt +++ b/clients/dos/src/main/kotlin/DosClient.kt @@ -96,22 +96,25 @@ class DosClient(private val service: DosService) { } /** - * Add a new scan job [zipFileKey] and [purls]. Return a [JobResponseBody] or null on error. + * Add a new scan job [zipFileKey] and [packages]. Return a [JobResponseBody] or null on error. */ - suspend fun addScanJob(zipFileKey: String, purls: List): JobResponseBody? { - if (zipFileKey.isEmpty() || purls.isEmpty()) { + suspend fun addScanJob(zipFileKey: String, packages: List): JobResponseBody? { + if (zipFileKey.isEmpty() || packages.isEmpty()) { logger.error { "The ZIP file key and Package URLs are required to add a scan job." } return null } - val requestBody = JobRequestBody(zipFileKey, purls) + val requestBody = JobRequestBody(zipFileKey, packages) val response = service.addScanJob(requestBody) val responseBody = response.body() return if (response.isSuccessful && responseBody != null) { responseBody } else { - logger.error { "Error adding a new scan job for $zipFileKey and $purls: ${response.errorBody()?.string()}" } + logger.error { + "Error adding a new scan job for $zipFileKey and ${packages.map { it.purl }}: " + + "${response.errorBody()?.string()}" + } null } diff --git a/clients/dos/src/main/kotlin/DosModel.kt b/clients/dos/src/main/kotlin/DosModel.kt index 22cf3f332ad67..35d0b1894b14d 100644 --- a/clients/dos/src/main/kotlin/DosModel.kt +++ b/clients/dos/src/main/kotlin/DosModel.kt @@ -46,6 +46,18 @@ data class UploadUrlResponseBody( val message: String? = null ) +@Serializable +data class PackageInfo( + /** The purl for the package (includes the VCS information). */ + val purl: String, + + /** + * The declared license for the package, if any. TODO: Use SpdxExpression type instead of String when the + * SpdxExpression class has been migrated to use kotlinx-serialization. + */ + val declaredLicenseExpressionSPDX: String? +) + @Serializable data class ScanResultsRequestBody( /** @@ -90,8 +102,8 @@ data class JobRequestBody( /** The key of the previously uploaded ZIP file to scan. */ val zipFileKey: String, - /** The list of purls whose packages' source code is contained in the ZIP file. */ - val purls: List + /** The list of packages whose source code is contained in the ZIP file. */ + val packages: List ) @Serializable diff --git a/plugins/scanners/dos/src/main/kotlin/DosScanner.kt b/plugins/scanners/dos/src/main/kotlin/DosScanner.kt index e26767e158d3e..6e31c23bb8397 100644 --- a/plugins/scanners/dos/src/main/kotlin/DosScanner.kt +++ b/plugins/scanners/dos/src/main/kotlin/DosScanner.kt @@ -33,6 +33,7 @@ import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.clients.dos.DosClient import org.ossreviewtoolkit.clients.dos.DosService +import org.ossreviewtoolkit.clients.dos.PackageInfo import org.ossreviewtoolkit.clients.dos.ScanResultsResponseBody import org.ossreviewtoolkit.downloader.DefaultWorkingTreeCache import org.ossreviewtoolkit.model.Issue @@ -104,13 +105,13 @@ class DosScanner internal constructor( return@runBlocking null } - val purls = context.coveredPackages.getDosPurls(provenance) + val packages = context.coveredPackages.getDosPackages(provenance) - logger.info { "Packages requested for scanning: ${purls.joinToString()}" } + logger.info { "Packages requested for scanning: ${packages.joinToString { it.purl }}" } // Ask for scan results from DOS API val existingScanResults = runCatching { - client.getScanResults(purls, config.fetchConcluded) + client.getScanResults(packages.map{it.purl}, config.fetchConcluded) }.onFailure { issues += createAndLogIssue(name, it.collectMessages()) }.onSuccess { @@ -124,7 +125,7 @@ class DosScanner internal constructor( runCatching { downloader.download(provenance) }.mapCatching { sourceDir -> - runBackendScan(purls, sourceDir, startTime, issues) + runBackendScan(packages, sourceDir, startTime, issues) }.onFailure { issues += createAndLogIssue(name, it.collectMessages()) }.getOrNull() @@ -135,7 +136,7 @@ class DosScanner internal constructor( "The job ID must not be null for 'pending' status." } - pollForCompletion(purls.first(), jobId, "Pending scan", startTime, issues) + pollForCompletion(packages.first().purl, jobId, "Pending scan", startTime, issues) } "ready" -> existingScanResults @@ -162,12 +163,12 @@ class DosScanner internal constructor( } internal suspend fun runBackendScan( - purls: List, + packages: List, sourceDir: File, startTime: Instant, issues: MutableList ): ScanResultsResponseBody? { - logger.info { "Initiating a backend scan for $purls." } + logger.info { "Initiating a backend scan for ${packages.map { it.purl }}." } val tmpDir = createOrtTempDir() val zipName = "${sourceDir.name}.zip" @@ -189,18 +190,18 @@ class DosScanner internal constructor( return null } - val jobResponse = client.addScanJob(zipName, purls) + val jobResponse = client.addScanJob(zipName, packages) val id = jobResponse?.scannerJobId if (id == null) { - issues += createAndLogIssue(name, "Failed to add scan job for '$zipName' and $purls.") + issues += createAndLogIssue(name, "Failed to add scan job for '$zipName' and ${packages.map { it.purl }}.") return null } // In case of multiple PURLs, they all point to packages with the same provenance. So if one package scan is // complete, all package scans are complete, which is why it is enough to arbitrarily pool for the first // package here. - return pollForCompletion(purls.first(), id, "New scan", startTime, issues) + return pollForCompletion(packages.first().purl, id, "New scan", startTime, issues) } private suspend fun pollForCompletion( @@ -235,15 +236,25 @@ class DosScanner internal constructor( } } -private fun Collection.getDosPurls(provenance: Provenance = UnknownProvenance): List { +private fun Collection.getDosPackages(provenance: Provenance = UnknownProvenance): List { val extras = provenance.toPurlExtras() return when (provenance) { is RepositoryProvenance -> { // Maintain the VCS path to get the "bookmarking" right for the file tree in the package configuration UI. - map { it.id.toPurl(extras.qualifiers, it.vcsProcessed.path) } + map { + PackageInfo( + purl = it.id.toPurl(extras.qualifiers, it.vcsProcessed.path), + declaredLicenseExpressionSPDX = it.declaredLicensesProcessed.spdxExpression?.toString() + ) + } } - else -> map { it.id.toPurl(extras) } + else -> map { + PackageInfo( + purl = it.id.toPurl(extras), + declaredLicenseExpressionSPDX = it.declaredLicensesProcessed.spdxExpression?.toString() + ) + } } } diff --git a/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt b/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt index 7e85e11591dbe..938295ee438c4 100644 --- a/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt +++ b/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt @@ -40,6 +40,7 @@ import java.time.Instant import kotlinx.serialization.encodeToString import org.ossreviewtoolkit.clients.dos.JSON +import org.ossreviewtoolkit.clients.dos.PackageInfo import org.ossreviewtoolkit.clients.dos.ScanResultsResponseBody import org.ossreviewtoolkit.model.Identifier import org.ossreviewtoolkit.model.Issue @@ -163,7 +164,12 @@ class DosScannerTest : StringSpec({ val issues = mutableListOf() val result = scanner.runBackendScan( - purls = listOf(pkg.purl), + packages = listOf( + PackageInfo( + purl = pkg.purl, + declaredLicenseExpressionSPDX = null + ) + ), sourceDir = tempdir(), startTime = Instant.now(), issues = issues