feat(downloader): Display progress info for parallel downloads in the…

… CLI Limit parallel downloads to 8 by default and display progres information for each download as well as the overall progress, which looks like Verifying ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% eta -:--:-- > Package 'Maven:org.glassfish.jaxb:jaxb-core:3.0.2'... 305/312 > Package 'Maven:io.perfmark:perfmark-api:0.26.0'... 306/312 > Package 'NPM:@VUE:compiler-dom:3.4.31'... 307/312 > Package 'NPM:@jridgewell:sourcemap-codec:1.4.15'... 308/312 > Package 'Maven:jakarta.servlet:jakarta.servlet-api:6.1.0'... 309/312 > Package 'Maven:com.google.code.findbugs:jsr305:3.0.2'... 310/312 > Package 'Maven:org.eclipse.angus:angus-activation:2.0.2'... 311/312 > Package 'Maven:org.semver4j:semver4j:5.3.0'... 312/312 Signed-off-by: Sebastian Schuberth <[email protected]>
oss-review-toolkit · Aug 20, 2024 · c64cc83 · c64cc83
1 parent 9932ab7
commit c64cc83
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 13 deletions.
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -150,6 +150,7 @@ maven-resolver-transport-http = { module = "org.apache.maven.resolver:maven-reso
 maven-resolver-transport-wagon = { module = "org.apache.maven.resolver:maven-resolver-transport-wagon", version.ref = "mavenResolver" }
 mockk = { module = "io.mockk:mockk", version.ref = "mockk" }
 mordant = { module = "com.github.ajalt.mordant:mordant", version.ref = "mordant" }
+mordantCoroutines = { module = "com.github.ajalt.mordant:mordant-coroutines", version.ref = "mordant" }
 okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
 okhttp-loggingInterceptor = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp" }
 postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" }

diff --git a/plugins/commands/downloader/build.gradle.kts b/plugins/commands/downloader/build.gradle.kts
@@ -33,4 +33,6 @@ dependencies {
 
     implementation(libs.clikt)
     implementation(libs.kotlinx.coroutines)
+    implementation(libs.mordant)
+    implementation(libs.mordantCoroutines)
 }
diff --git a/plugins/commands/downloader/src/main/kotlin/DownloaderCommand.kt b/plugins/commands/downloader/src/main/kotlin/DownloaderCommand.kt
@@ -20,6 +20,7 @@
 package org.ossreviewtoolkit.plugins.commands.downloader
 
 import com.github.ajalt.clikt.core.ProgramResult
+import com.github.ajalt.clikt.core.terminal
 import com.github.ajalt.clikt.parameters.groups.default
 import com.github.ajalt.clikt.parameters.groups.mutuallyExclusiveOptions
 import com.github.ajalt.clikt.parameters.groups.required
@@ -34,15 +35,30 @@ import com.github.ajalt.clikt.parameters.options.split
 import com.github.ajalt.clikt.parameters.options.switch
 import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.file
+import com.github.ajalt.mordant.animation.coroutines.animateInCoroutine
+import com.github.ajalt.mordant.animation.progress.MultiProgressBarAnimation
+import com.github.ajalt.mordant.animation.progress.addTask
+import com.github.ajalt.mordant.animation.progress.advance
+import com.github.ajalt.mordant.rendering.TextAlign
 import com.github.ajalt.mordant.rendering.Theme
+import com.github.ajalt.mordant.table.ColumnWidth
+import com.github.ajalt.mordant.widgets.EmptyWidget
+import com.github.ajalt.mordant.widgets.progress.percentage
+import com.github.ajalt.mordant.widgets.progress.progressBar
+import com.github.ajalt.mordant.widgets.progress.progressBarContextLayout
+import com.github.ajalt.mordant.widgets.progress.progressBarLayout
+import com.github.ajalt.mordant.widgets.progress.text
+import com.github.ajalt.mordant.widgets.progress.timeRemaining
 
 import java.io.File
 
 import kotlin.time.measureTime
 
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 
@@ -300,8 +316,7 @@ class DownloaderCommand : OrtCommand(
 
         val packageDownloadDirs = packages.associateWith { outputDir.resolve(it.id.toPath()) }
 
-        @Suppress("ForbiddenMethodCall")
-        runBlocking { downloadAllPackages(packageDownloadDirs, failureMessages) }
+        downloadAllPackages(packageDownloadDirs, failureMessages)
 
         if (archiveMode == ArchiveMode.BUNDLE && !dryRun) {
             val zipFile = outputDir.resolve("archive.zip")
@@ -319,21 +334,44 @@ class DownloaderCommand : OrtCommand(
         }
     }
 
-    private suspend fun downloadAllPackages(
+    @Suppress("ForbiddenMethodCall")
+    private fun downloadAllPackages(
         packageDownloadDirs: Map<Package, File>,
-        failureMessages: MutableList<String>
-    ) {
-        withContext(Dispatchers.IO) {
-            packageDownloadDirs.entries.mapIndexed { index, (pkg, dir) ->
-                async {
-                    val progress = "${index + 1} of ${packageDownloadDirs.size}"
+        failureMessages: MutableList<String>,
+        maxParallelDownloads: Int = 8
+    ) = runBlocking {
+        val parallelDownloads = packageDownloadDirs.size.coerceAtMost(maxParallelDownloads)
+
+        val overallLayout = progressBarLayout(alignColumns = false) {
+            text(if (dryRun) "Verifying" else "Downloading", align = TextAlign.LEFT)
+            progressBar()
+            percentage()
+            timeRemaining()
+        }
+
+        val taskLayout = progressBarContextLayout<Pair<Package, Int>> {
+            text(fps = animationFps, align = TextAlign.LEFT) { "> Package '${context.first.id.toCoordinates()}'..." }
+            cell(width = ColumnWidth.Expand()) { EmptyWidget }
+            text(fps = animationFps, align = TextAlign.RIGHT) { "${context.second.inc()}/${packageDownloadDirs.size}" }
+        }
 
-                    val verb = if (dryRun) "Verifying" else "Starting"
-                    echo("$verb download for '${pkg.id.toCoordinates()}' ($progress).")
+        val progress = MultiProgressBarAnimation(terminal).animateInCoroutine()
+        val overall = progress.addTask(overallLayout, total = packageDownloadDirs.size.toLong())
+        val tasks = List(parallelDownloads) { progress.addTask(taskLayout, context = Package.EMPTY to 0, total = 1) }
 
-                    downloadPackage(pkg, dir, failureMessages).also {
-                        if (!dryRun) echo("Finished download for ${pkg.id.toCoordinates()} ($progress).")
+        launch { progress.execute() }
+
+        @OptIn(ExperimentalCoroutinesApi::class)
+        withContext(Dispatchers.IO.limitedParallelism(parallelDownloads)) {
+            packageDownloadDirs.entries.mapIndexed { index, (pkg, dir) ->
+                async {
+                    with(tasks[index % parallelDownloads]) {
+                        reset { context = pkg to index }
+                        downloadPackage(pkg, dir, failureMessages)
+                        advance()
                     }
+
+                    overall.advance()
                 }
             }.awaitAll()
         }