refactor(dos): Edit scan results query parameters

When asking for results for multiple packages that are part of the same source, in case DOS finds only some of these packages in the database, it will add new package rows for new purl bookmarks. To ensure that the declared license is saved for these packages as well, they need to be a part of the scan results request body also. As the declared licenses are added in this request as well, this also allows to save the declared licenses for existing packages in the DOS database. Signed-off-by: Johanna Lamppu <[email protected]>
oss-review-toolkit · Jul 3, 2024 · eae8cbe · eae8cbe
1 parent e929d4d
commit eae8cbe
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 18 deletions.
diff --git a/clients/dos/src/main/kotlin/DosClient.kt b/clients/dos/src/main/kotlin/DosClient.kt
@@ -121,23 +121,26 @@ class DosClient(private val service: DosService) {
     }
 
     /**
-     * Get scan results for a list of [purls]. In case multiple purls are provided, it is assumed that they all refer to
-     * the same provenance (like a monorepo). If [fetchConcluded] is true, return concluded licenses instead of detected
-     * licenses. Return either existing results, a "pending" message if the package is currently being scanned, a
-     * "no-results" message if a scan yielded no results, or null on error.
+     * Get scan results for a list of [packages]. In case multiple packages are provided, it is assumed that they all
+     * refer to the same provenance (like a monorepo). If [fetchConcluded] is true, return concluded licenses instead of
+     * detected licenses. Return either existing results, a "pending" message if the package is currently being scanned,
+     * a "no-results" message if a scan yielded no results, or null on error. If only some of the packages exist in DOS
+     * database (identified by purl), new bookmarks for the remaining packages are made (hence the need to provide the
+     * declared licenses for these packages in this request).
      */
-    suspend fun getScanResults(purls: List<String>, fetchConcluded: Boolean): ScanResultsResponseBody? {
-        if (purls.isEmpty()) {
+    suspend fun getScanResults(packages: List<PackageInfo>, fetchConcluded: Boolean): ScanResultsResponseBody? {
+        if (packages.isEmpty()) {
             logger.error { "The list of PURLs to get scan results for must not be empty." }
             return null
         }
 
         val options = ScanResultsRequestBody.ReqOptions(fetchConcluded)
-        val requestBody = ScanResultsRequestBody(purls, options)
+        val requestBody = ScanResultsRequestBody(packages, options)
         val response = service.getScanResults(requestBody)
         val responseBody = response.body()
 
         return if (response.isSuccessful && responseBody != null) {
+            val purls = packages.map { it.purl }
             when (responseBody.state.status) {
                 "no-results" -> logger.info { "No scan results found for $purls." }
                 "pending" -> logger.info { "Scan pending for $purls." }

diff --git a/clients/dos/src/main/kotlin/DosModel.kt b/clients/dos/src/main/kotlin/DosModel.kt
@@ -61,10 +61,11 @@ data class PackageInfo(
 @Serializable
 data class ScanResultsRequestBody(
     /**
-     * The list of purls to get scan results for. In case multiple purls are provided, it is assumed that they all
-     * refer to the same provenance (like a monorepo).
+     * The list of packages to get scan results for. In case multiple packages are provided, it is assumed that they all
+     * refer to the same provenance (like a monorepo). If only some of the packages exist in the DOS database, new purl
+     * bookmarks will be added for the missing packages (hence the need for the declared license here).
      */
-    val purls: List<String>? = null,
+    val packages: List<PackageInfo>,
 
     /** Options fort requesting scan results. */
     val options: ReqOptions? = null

diff --git a/plugins/scanners/dos/src/main/kotlin/DosScanner.kt b/plugins/scanners/dos/src/main/kotlin/DosScanner.kt
@@ -111,7 +111,7 @@ class DosScanner internal constructor(
 
             // Ask for scan results from DOS API
             val existingScanResults = runCatching {
-                client.getScanResults(packages.map{it.purl}, config.fetchConcluded)
+                client.getScanResults(packages, config.fetchConcluded)
             }.onFailure {
                 issues += createAndLogIssue(name, it.collectMessages())
             }.onSuccess {
@@ -136,7 +136,7 @@ class DosScanner internal constructor(
                         "The job ID must not be null for 'pending' status."
                     }
 
-                    pollForCompletion(packages.first().purl, jobId, "Pending scan", startTime, issues)
+                    pollForCompletion(packages.first(), jobId, "Pending scan", startTime, issues)
                 }
 
                 "ready" -> existingScanResults
@@ -201,11 +201,11 @@ class DosScanner internal constructor(
         // In case of multiple PURLs, they all point to packages with the same provenance. So if one package scan is
         // complete, all package scans are complete, which is why it is enough to arbitrarily pool for the first
         // package here.
-        return pollForCompletion(packages.first().purl, id, "New scan", startTime, issues)
+        return pollForCompletion(packages.first(), id, "New scan", startTime, issues)
     }
 
     private suspend fun pollForCompletion(
-        purl: String,
+        pkg: PackageInfo,
         jobId: String,
         logMessagePrefix: String,
         startTime: Instant,
@@ -222,7 +222,7 @@ class DosScanner internal constructor(
             when (jobState.state.status) {
                 "completed" -> {
                     logger.info { "Scan completed" }
-                    return client.getScanResults(listOf(purl), config.fetchConcluded)
+                    return client.getScanResults(listOf(pkg), config.fetchConcluded)
                 }
 
                 "failed" -> {

diff --git a/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt b/plugins/scanners/dos/src/test/kotlin/DosScannerTest.kt
@@ -105,7 +105,15 @@ class DosScannerTest : StringSpec({
                 )
         )
 
-        val status = scanner.client.getScanResults(listOf("purl"), false)?.state?.status
+        val status = scanner.client.getScanResults(
+            listOf(
+                PackageInfo(
+                    purl = "purl",
+                    declaredLicenseExpressionSPDX = null
+                )
+            ),
+            false
+        )?.state?.status
 
         status shouldBe "no-results"
     }
@@ -120,7 +128,15 @@ class DosScannerTest : StringSpec({
                 )
         )
 
-        val response = scanner.client.getScanResults(listOf("purl"), false)
+        val response = scanner.client.getScanResults(
+            listOf(
+                PackageInfo(
+                    purl = "purl",
+                    declaredLicenseExpressionSPDX = null
+                )
+            ),
+            false
+        )
 
         response?.state?.status shouldBe "pending"
         response?.state?.jobId shouldBe "dj34eh4h65"
@@ -136,7 +152,15 @@ class DosScannerTest : StringSpec({
                 )
         )
 
-        val response = scanner.client.getScanResults(listOf("purl"), false)
+        val response = scanner.client.getScanResults(
+            listOf(
+                PackageInfo(
+                    purl = "purl",
+                    declaredLicenseExpressionSPDX = null
+                )
+            ),
+            false
+        )
 
         val actualJson = JSON.encodeToString(response?.results)
         val expectedJson = JSON.decodeFromString<ScanResultsResponseBody>(getResourceAsString("/ready.json")).let {