Display progress info for parallel downloads in the CLI

gradle/libs.versions.toml

@@ -150,6 +150,7 @@ maven-resolver-transport-http = { module = "org.apache.maven.resolver:maven-reso
 maven-resolver-transport-wagon = { module = "org.apache.maven.resolver:maven-resolver-transport-wagon", version.ref = "mavenResolver" }
 mockk = { module = "io.mockk:mockk", version.ref = "mockk" }
 mordant = { module = "com.github.ajalt.mordant:mordant", version.ref = "mordant" }
+mordantCoroutines = { module = "com.github.ajalt.mordant:mordant-coroutines", version.ref = "mordant" }
 okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" }
 okhttp-loggingInterceptor = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp" }
 postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" }

plugins/commands/downloader/build.gradle.kts

@@ -33,4 +33,6 @@ dependencies {
 
     implementation(libs.clikt)
     implementation(libs.kotlinx.coroutines)
+    implementation(libs.mordant)
+    implementation(libs.mordantCoroutines)
 }

plugins/commands/downloader/src/main/kotlin/DownloaderCommand.kt

@@ -20,6 +20,7 @@
 package org.ossreviewtoolkit.plugins.commands.downloader
 
 import com.github.ajalt.clikt.core.ProgramResult
+import com.github.ajalt.clikt.core.terminal
 import com.github.ajalt.clikt.parameters.groups.default
 import com.github.ajalt.clikt.parameters.groups.mutuallyExclusiveOptions
 import com.github.ajalt.clikt.parameters.groups.required
@@ -34,15 +35,30 @@ import com.github.ajalt.clikt.parameters.options.split
 import com.github.ajalt.clikt.parameters.options.switch
 import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.file
+import com.github.ajalt.mordant.animation.coroutines.animateInCoroutine
+import com.github.ajalt.mordant.animation.progress.MultiProgressBarAnimation
+import com.github.ajalt.mordant.animation.progress.addTask
+import com.github.ajalt.mordant.animation.progress.advance
+import com.github.ajalt.mordant.rendering.TextAlign
 import com.github.ajalt.mordant.rendering.Theme
+import com.github.ajalt.mordant.table.ColumnWidth
+import com.github.ajalt.mordant.widgets.EmptyWidget
+import com.github.ajalt.mordant.widgets.progress.percentage
+import com.github.ajalt.mordant.widgets.progress.progressBar
+import com.github.ajalt.mordant.widgets.progress.progressBarContextLayout
+import com.github.ajalt.mordant.widgets.progress.progressBarLayout
+import com.github.ajalt.mordant.widgets.progress.text
+import com.github.ajalt.mordant.widgets.progress.timeRemaining
 
 import java.io.File
 
 import kotlin.time.measureTime
 
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.async
 import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
 
@@ -234,8 +250,10 @@ class DownloaderCommand : OrtCommand(
             throw ProgramResult(0)
         }
 
+        val verb = if (dryRun) "Verifying" else "Downloading"
+
         echo(
-            "Downloading ${packageTypes.joinToString(" and ") { "${it}s" }} from ORT result file at " +
+            "$verb ${packageTypes.joinToString(" and ") { "${it}s" }} from ORT result file at " +
                 "'${ortFile.canonicalPath}'..."
         )
 
@@ -294,12 +312,11 @@ class DownloaderCommand : OrtCommand(
             }
         }
 
-        echo("Downloading ${packages.size} project(s) / package(s) in total.")
+        echo("$verb ${packages.size} project(s) / package(s) in total.")
 
         val packageDownloadDirs = packages.associateWith { outputDir.resolve(it.id.toPath()) }
 
-        @Suppress("ForbiddenMethodCall")
-        runBlocking { downloadAllPackages(packageDownloadDirs, failureMessages) }
+        downloadAllPackages(packageDownloadDirs, failureMessages)
 
         if (archiveMode == ArchiveMode.BUNDLE && !dryRun) {
             val zipFile = outputDir.resolve("archive.zip")
@@ -317,21 +334,44 @@ class DownloaderCommand : OrtCommand(
         }
     }
 
-    private suspend fun downloadAllPackages(
+    @Suppress("ForbiddenMethodCall")
+    private fun downloadAllPackages(
         packageDownloadDirs: Map<Package, File>,
-        failureMessages: MutableList<String>
-    ) {
-        withContext(Dispatchers.IO) {
-            packageDownloadDirs.entries.mapIndexed { index, (pkg, dir) ->
-                async {
-                    val progress = "${index + 1} of ${packageDownloadDirs.size}"
+        failureMessages: MutableList<String>,
+        maxParallelDownloads: Int = 8
+    ) = runBlocking {
+        val parallelDownloads = packageDownloadDirs.size.coerceAtMost(maxParallelDownloads)
+
+        val overallLayout = progressBarLayout(alignColumns = false) {
+            text(if (dryRun) "Verifying" else "Downloading", align = TextAlign.LEFT)
+            progressBar()
+            percentage()
+            timeRemaining()
+        }
+
+        val taskLayout = progressBarContextLayout<Pair<Package, Int>> {
+            text(fps = animationFps, align = TextAlign.LEFT) { "> Package '${context.first.id.toCoordinates()}'..." }
+            cell(width = ColumnWidth.Expand()) { EmptyWidget }
+            text(fps = animationFps, align = TextAlign.RIGHT) { "${context.second.inc()}/${packageDownloadDirs.size}" }
+        }
+
+        val progress = MultiProgressBarAnimation(terminal).animateInCoroutine()
+        val overall = progress.addTask(overallLayout, total = packageDownloadDirs.size.toLong())
+        val tasks = List(parallelDownloads) { progress.addTask(taskLayout, context = Package.EMPTY to 0, total = 1) }
 
-                    val verb = if (dryRun) "Verifying" else "Starting"
-                    echo("$verb download for '${pkg.id.toCoordinates()}' ($progress).")
+        launch { progress.execute() }
 
-                    downloadPackage(pkg, dir, failureMessages).also {
-                        if (!dryRun) echo("Finished download for ${pkg.id.toCoordinates()} ($progress).")
+        @OptIn(ExperimentalCoroutinesApi::class)
+        withContext(Dispatchers.IO.limitedParallelism(parallelDownloads)) {
+            packageDownloadDirs.entries.mapIndexed { index, (pkg, dir) ->
+                async {
+                    with(tasks[index % parallelDownloads]) {
+                        reset { context = pkg to index }
+                        downloadPackage(pkg, dir, failureMessages)
+                        advance()
                     }
+
+                    overall.advance()
                 }
             }.awaitAll()
         }