feat(clients): Add the Double Open Server (DOS) client

This client library project provides the base for communicating with the
DOS package configuration provider and DOS scanner wrapper.

Signed-off-by: Sebastian Schuberth <[email protected]>

clients/dos/build.gradle.kts

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+plugins {
+    // Apply precompiled plugins.
+    id("ort-library-conventions")
+
+    // Apply third-party plugins.
+    alias(libs.plugins.kotlinSerialization)
+}
+
+dependencies {
+    api(libs.okhttp)
+    api(libs.retrofit)
+
+    implementation(projects.utils.commonUtils)
+
+    implementation(libs.bundles.kotlinxSerialization)
+    implementation(libs.log4j.api.kotlin)
+    implementation(libs.okhttp.loggingInterceptor)
+    implementation(libs.retrofit.converter.kotlinxSerialization)
+}

clients/dos/src/main/kotlin/AuthorizationInterceptor.kt

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.clients.dos
+
+import okhttp3.Interceptor
+
+import retrofit2.Invocation
+
+/**
+ * A custom annotation for skipping the authorization interceptor for certain API calls.
+ */
+@Target(AnnotationTarget.FUNCTION)
+@Retention(AnnotationRetention.RUNTIME)
+internal annotation class SkipAuthorization
+
+/**
+ * An HTTP interceptor to conditionally skip authorization.
+ */
+internal class AuthorizationInterceptor(private val token: String) : Interceptor {
+    override fun intercept(chain: Interceptor.Chain): okhttp3.Response {
+        val original = chain.request()
+
+        val skipAuthorization = original.tag(Invocation::class.java)
+            ?.method()
+            ?.isAnnotationPresent(SkipAuthorization::class.java) == true
+
+        if (skipAuthorization) return chain.proceed(original)
+
+        val requestBuilder = original.newBuilder()
+            .header("Authorization", "Bearer $token")
+            .method(original.method, original.body)
+
+        return chain.proceed(requestBuilder.build())
+    }
+}

clients/dos/src/main/kotlin/DosClient.kt

@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.clients.dos
+
+import java.io.File
+
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.RequestBody.Companion.asRequestBody
+
+import org.apache.logging.log4j.kotlin.logger
+
+import org.ossreviewtoolkit.utils.common.formatSizeInMib
+
+/**
+ * A client implementation on top of the DOS service.
+ */
+class DosClient(private val service: DosService) {
+    /**
+     * Get the package configuration for the given [purl]. Return a [PackageConfigurationResponseBody] or null on error.
+     */
+    suspend fun getPackageConfiguration(purl: String): PackageConfigurationResponseBody? {
+        if (purl.isEmpty()) {
+            logger.error { "The Package URL is required to get the package configuration." }
+            return null
+        }
+
+        val requestBody = PackageConfigurationRequestBody(purl)
+        val response = service.getPackageConfiguration(requestBody)
+        val responseBody = response.body()
+
+        return if (response.isSuccessful && responseBody != null) {
+            responseBody
+        } else {
+            logger.error { "Error getting the package configuration for PURL $purl: ${response.errorBody()}" }
+
+            null
+        }
+    }
+
+    /**
+     * Return a pre-signed URL as provided by the DOS API to upload a package to S3 object storage for scanning, or null
+     * if the request did not succeed.
+     */
+    suspend fun getUploadUrl(key: String): String? {
+        if (key.isEmpty()) {
+            logger.error { "Need the name of the zipped packet to upload" }
+            return null
+        }
+
+        val requestBody = UploadUrlRequestBody(key)
+        val response = service.getUploadUrl(requestBody)
+        val responseBody = response.body()
+
+        return if (response.isSuccessful && responseBody != null && responseBody.success) {
+            response.body()?.presignedUrl
+        } else {
+            logger.error { "Unable to get a pre-signed URL for $key: ${response.errorBody()?.string()}" }
+            null
+        }
+    }
+
+    /**
+     * Upload the file at [file] to S3 using the [pre-signed URL][presignedUrl] and return true on success or false
+     * other on failure.
+     */
+    suspend fun uploadFile(file: File, presignedUrl: String): Boolean {
+        logger.info { "Uploading file $file of size ${file.formatSizeInMib} to S3..." }
+
+        val contentType = "application/zip".toMediaType()
+        val response = service.uploadFile(presignedUrl, file.asRequestBody(contentType))
+
+        return if (response.isSuccessful) {
+            logger.info { "Successfully uploaded $file to S3." }
+            true
+        } else {
+            logger.error { "Failed to upload $file to S3: ${response.errorBody()}" }
+            false
+        }
+    }
+
+    /**
+     * Add a new scan job [zipFileKey] and [purls]. Return a [JobResponseBody] or null on error.
+     */
+    suspend fun addScanJob(zipFileKey: String, purls: List<String>): JobResponseBody? {
+        if (zipFileKey.isEmpty() || purls.isEmpty()) {
+            logger.error { "The ZIP file key and Package URLs are required to add a scan job." }
+            return null
+        }
+
+        val requestBody = JobRequestBody(zipFileKey, purls)
+        val response = service.addScanJob(requestBody)
+        val responseBody = response.body()
+
+        return if (response.isSuccessful && responseBody != null) {
+            responseBody
+        } else {
+            logger.error { "Error adding a new scan job for $zipFileKey and $purls: ${response.errorBody()}" }
+
+            null
+        }
+    }
+
+    /**
+     * Get scan results for a list of [purls]. In case multiple PURLs are provided, it is assumed that they all refer to
+     * the same provenance (like a monorepo). If [fetchConcluded] is true, return concluded licenses instead of detected
+     * licenses. Return either existing results, a "pending" message if the package is currently being scanned, a
+     * "no-results" message if a scan yielded no results, or null on error.
+     */
+    suspend fun getScanResults(purls: List<String>, fetchConcluded: Boolean): ScanResultsResponseBody? {
+        if (purls.isEmpty()) {
+            logger.error { "The list of PURLs to get scan results for must not be empty." }
+            return null
+        }
+
+        val options = ScanResultsRequestBody.ReqOptions(fetchConcluded)
+        val requestBody = ScanResultsRequestBody(purls, options)
+        val response = service.getScanResults(requestBody)
+        val responseBody = response.body()
+
+        return if (response.isSuccessful && responseBody != null) {
+            when (responseBody.state.status) {
+                "no-results" -> logger.info { "No scan results found for $purls." }
+                "pending" -> logger.info { "Scan pending for $purls." }
+                "ready" -> {
+                    logger.info { "Scan results ready for $purls." }
+
+                    if (fetchConcluded) {
+                        logger.info { "Returning concluded licenses instead of detected licenses." }
+                    }
+                }
+            }
+
+            responseBody
+        } else {
+            logger.error { "Error getting scan results: ${response.errorBody()}" }
+
+            null
+        }
+    }
+
+    /**
+     * Get the state of the scan job with the given [id]. Return a [JobStateResponseBody] with one of waiting / active /
+     * completed / failed state, or null on error.
+     */
+    suspend fun getScanJobState(id: String): JobStateResponseBody? {
+        if (id.isEmpty()) {
+            logger.error { "Need the job ID to check for job state" }
+            return null
+        }
+
+        val response = service.getScanJobState(id)
+        val responseBody = response.body()
+
+        return if (response.isSuccessful && responseBody != null) {
+            responseBody
+        } else {
+            logger.error { "Error getting the scan state for job $id: ${response.errorBody()}" }
+
+            null
+        }
+    }
+}

clients/dos/src/main/kotlin/DosModel.kt

@@ -0,0 +1,115 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.clients.dos
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+
+/**
+ * The JSON (de-)serialization object used by this service.
+ */
+val JSON = Json { ignoreUnknownKeys = true }
+
+@Serializable
+data class UploadUrlRequestBody(
+    val key: String? = null
+)
+
+@Serializable
+data class UploadUrlResponseBody(
+    val success: Boolean,
+    val presignedUrl: String? = null,
+    val message: String? = null
+)
+
+@Serializable
+data class ScanResultsRequestBody(
+    val purls: List<String>? = null,
+    val options: ReqOptions? = null
+) {
+    @Serializable
+    data class ReqOptions(
+        val fetchConcluded: Boolean? = false
+    )
+}
+
+@Serializable
+data class ScanResultsResponseBody(
+    val state: State,
+    val purls: List<String>? = null,
+    val results: JsonObject? = null
+) {
+    @Serializable
+    data class State(
+        val status: String,
+        val jobId: String? = null
+    )
+}
+
+@Serializable
+data class JobRequestBody(
+    val zipFileKey: String? = null,
+    val purls: List<String>? = null
+)
+
+@Serializable
+data class JobResponseBody(
+    val scannerJobId: String? = null,
+    val message: String? = null
+)
+
+@Serializable
+data class JobStateResponseBody(
+    val state: State,
+    val results: JsonObject? = null
+) {
+    @Serializable
+    data class State(
+        val status: String? = null,
+        val message: String? = null
+    )
+}
+
+@Serializable
+data class PackageConfigurationRequestBody(
+    val purl: String? = null
+)
+
+@Serializable
+data class PackageConfigurationResponseBody(
+    val licenseConclusions: List<LicenseConclusion>,
+    val pathExclusions: List<PathExclusion>
+) {
+    @Serializable
+    data class LicenseConclusion(
+        val path: String,
+        val detectedLicenseExpressionSPDX: String? = null,
+        val concludedLicenseExpressionSPDX: String,
+        val comment: String? = null
+    )
+
+    @Serializable
+    data class PathExclusion(
+        val pattern: String,
+        val reason: String,
+        val comment: String? = null
+    )
+}