Skip to content

Commit

Permalink
refactor(dos): Edit job query parameters
Browse files Browse the repository at this point in the history
Add declared licenses of the packages to the request body in order to
save this information about the package on DOS server side (so the
information can be shown in the DOS Clearance UI).

Signed-off-by: Johanna Lamppu <[email protected]>
  • Loading branch information
lamppu authored and sschuberth committed Jul 3, 2024
1 parent 61866be commit e929d4d
Show file tree
Hide file tree
Showing 4 changed files with 53 additions and 21 deletions.
13 changes: 8 additions & 5 deletions clients/dos/src/main/kotlin/DosClient.kt
Original file line number Diff line number Diff line change
Expand Up @@ -96,22 +96,25 @@ class DosClient(private val service: DosService) {
}

/**
* Add a new scan job [zipFileKey] and [purls]. Return a [JobResponseBody] or null on error.
* Add a new scan job [zipFileKey] and [packages]. Return a [JobResponseBody] or null on error.
*/
suspend fun addScanJob(zipFileKey: String, purls: List<String>): JobResponseBody? {
if (zipFileKey.isEmpty() || purls.isEmpty()) {
suspend fun addScanJob(zipFileKey: String, packages: List<PackageInfo>): JobResponseBody? {
if (zipFileKey.isEmpty() || packages.isEmpty()) {
logger.error { "The ZIP file key and Package URLs are required to add a scan job." }
return null
}

val requestBody = JobRequestBody(zipFileKey, purls)
val requestBody = JobRequestBody(zipFileKey, packages)
val response = service.addScanJob(requestBody)
val responseBody = response.body()

return if (response.isSuccessful && responseBody != null) {
responseBody
} else {
logger.error { "Error adding a new scan job for $zipFileKey and $purls: ${response.errorBody()?.string()}" }
logger.error {
"Error adding a new scan job for $zipFileKey and ${packages.map { it.purl }}: " +
"${response.errorBody()?.string()}"
}

null
}
Expand Down
16 changes: 14 additions & 2 deletions clients/dos/src/main/kotlin/DosModel.kt
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,18 @@ data class UploadUrlResponseBody(
val message: String? = null
)

@Serializable
data class PackageInfo(
/** The purl for the package (includes the VCS information). */
val purl: String,

/**
* The declared license for the package, if any. TODO: Use SpdxExpression type instead of String when the
* SpdxExpression class has been migrated to use kotlinx-serialization.
*/
val declaredLicenseExpressionSPDX: String?
)

@Serializable
data class ScanResultsRequestBody(
/**
Expand Down Expand Up @@ -90,8 +102,8 @@ data class JobRequestBody(
/** The key of the previously uploaded ZIP file to scan. */
val zipFileKey: String,

/** The list of purls whose packages' source code is contained in the ZIP file. */
val purls: List<String>
/** The list of packages whose source code is contained in the ZIP file. */
val packages: List<PackageInfo>
)

@Serializable
Expand Down
37 changes: 24 additions & 13 deletions plugins/scanners/dos/src/main/kotlin/DosScanner.kt
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import org.apache.logging.log4j.kotlin.logger

import org.ossreviewtoolkit.clients.dos.DosClient
import org.ossreviewtoolkit.clients.dos.DosService
import org.ossreviewtoolkit.clients.dos.PackageInfo
import org.ossreviewtoolkit.clients.dos.ScanResultsResponseBody
import org.ossreviewtoolkit.downloader.DefaultWorkingTreeCache
import org.ossreviewtoolkit.model.Issue
Expand Down Expand Up @@ -104,13 +105,13 @@ class DosScanner internal constructor(
return@runBlocking null
}

val purls = context.coveredPackages.getDosPurls(provenance)
val packages = context.coveredPackages.getDosPackages(provenance)

logger.info { "Packages requested for scanning: ${purls.joinToString()}" }
logger.info { "Packages requested for scanning: ${packages.joinToString { it.purl }}" }

// Ask for scan results from DOS API
val existingScanResults = runCatching {
client.getScanResults(purls, config.fetchConcluded)
client.getScanResults(packages.map{it.purl}, config.fetchConcluded)
}.onFailure {
issues += createAndLogIssue(name, it.collectMessages())
}.onSuccess {
Expand All @@ -124,7 +125,7 @@ class DosScanner internal constructor(
runCatching {
downloader.download(provenance)
}.mapCatching { sourceDir ->
runBackendScan(purls, sourceDir, startTime, issues)
runBackendScan(packages, sourceDir, startTime, issues)
}.onFailure {
issues += createAndLogIssue(name, it.collectMessages())
}.getOrNull()
Expand All @@ -135,7 +136,7 @@ class DosScanner internal constructor(
"The job ID must not be null for 'pending' status."
}

pollForCompletion(purls.first(), jobId, "Pending scan", startTime, issues)
pollForCompletion(packages.first().purl, jobId, "Pending scan", startTime, issues)
}

"ready" -> existingScanResults
Expand All @@ -162,12 +163,12 @@ class DosScanner internal constructor(
}

internal suspend fun runBackendScan(
purls: List<String>,
packages: List<PackageInfo>,
sourceDir: File,
startTime: Instant,
issues: MutableList<Issue>
): ScanResultsResponseBody? {
logger.info { "Initiating a backend scan for $purls." }
logger.info { "Initiating a backend scan for ${packages.map { it.purl }}." }

val tmpDir = createOrtTempDir()
val zipName = "${sourceDir.name}.zip"
Expand All @@ -189,18 +190,18 @@ class DosScanner internal constructor(
return null
}

val jobResponse = client.addScanJob(zipName, purls)
val jobResponse = client.addScanJob(zipName, packages)
val id = jobResponse?.scannerJobId

if (id == null) {
issues += createAndLogIssue(name, "Failed to add scan job for '$zipName' and $purls.")
issues += createAndLogIssue(name, "Failed to add scan job for '$zipName' and ${packages.map { it.purl }}.")
return null
}

// In case of multiple PURLs, they all point to packages with the same provenance. So if one package scan is
// complete, all package scans are complete, which is why it is enough to arbitrarily pool for the first
// package here.
return pollForCompletion(purls.first(), id, "New scan", startTime, issues)
return pollForCompletion(packages.first().purl, id, "New scan", startTime, issues)
}

private suspend fun pollForCompletion(
Expand Down Expand Up @@ -235,15 +236,25 @@ class DosScanner internal constructor(
}
}

private fun Collection<Package>.getDosPurls(provenance: Provenance = UnknownProvenance): List<String> {
private fun Collection<Package>.getDosPackages(provenance: Provenance = UnknownProvenance): List<PackageInfo> {
val extras = provenance.toPurlExtras()

return when (provenance) {
is RepositoryProvenance -> {
// Maintain the VCS path to get the "bookmarking" right for the file tree in the package configuration UI.
map { it.id.toPurl(extras.qualifiers, it.vcsProcessed.path) }
map {
PackageInfo(
purl = it.id.toPurl(extras.qualifiers, it.vcsProcessed.path),
declaredLicenseExpressionSPDX = it.declaredLicensesProcessed.spdxExpression?.toString()
)
}
}

else -> map { it.id.toPurl(extras) }
else -> map {
PackageInfo(
purl = it.id.toPurl(extras),
declaredLicenseExpressionSPDX = it.declaredLicensesProcessed.spdxExpression?.toString()
)
}
}
}
8 changes: 7 additions & 1 deletion plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import java.time.Instant
import kotlinx.serialization.encodeToString

import org.ossreviewtoolkit.clients.dos.JSON
import org.ossreviewtoolkit.clients.dos.PackageInfo
import org.ossreviewtoolkit.clients.dos.ScanResultsResponseBody
import org.ossreviewtoolkit.model.Identifier
import org.ossreviewtoolkit.model.Issue
Expand Down Expand Up @@ -163,7 +164,12 @@ class DosScannerTest : StringSpec({
val issues = mutableListOf<Issue>()

val result = scanner.runBackendScan(
purls = listOf(pkg.purl),
packages = listOf(
PackageInfo(
purl = pkg.purl,
declaredLicenseExpressionSPDX = null
)
),
sourceDir = tempdir(),
startTime = Instant.now(),
issues = issues
Expand Down

0 comments on commit e929d4d

Please sign in to comment.