From 7a17a71c4d483a54dc4059d310e0c7e88043b7c6 Mon Sep 17 00:00:00 2001 From: Jyrki Keisala Date: Tue, 18 Jun 2024 10:47:29 +0200 Subject: [PATCH] feat(clients): Add the Double Open Server (DOS) client This client library project provides the base for communicating with the DOS package configuration provider and DOS scanner wrapper. Signed-off-by: Sebastian Schuberth --- clients/dos/build.gradle.kts | 38 ++++ .../main/kotlin/AuthorizationInterceptor.kt | 52 +++++ clients/dos/src/main/kotlin/DosClient.kt | 179 ++++++++++++++++++ clients/dos/src/main/kotlin/DosModel.kt | 115 +++++++++++ clients/dos/src/main/kotlin/DosService.kt | 122 ++++++++++++ gradle/libs.versions.toml | 1 + settings.gradle.kts | 2 + utils/ort/src/test/kotlin/UtilsTest.kt | 3 +- 8 files changed, 511 insertions(+), 1 deletion(-) create mode 100644 clients/dos/build.gradle.kts create mode 100644 clients/dos/src/main/kotlin/AuthorizationInterceptor.kt create mode 100644 clients/dos/src/main/kotlin/DosClient.kt create mode 100644 clients/dos/src/main/kotlin/DosModel.kt create mode 100644 clients/dos/src/main/kotlin/DosService.kt diff --git a/clients/dos/build.gradle.kts b/clients/dos/build.gradle.kts new file mode 100644 index 0000000000000..15b643d408e22 --- /dev/null +++ b/clients/dos/build.gradle.kts @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +plugins { + // Apply precompiled plugins. + id("ort-library-conventions") + + // Apply third-party plugins. + alias(libs.plugins.kotlinSerialization) +} + +dependencies { + api(libs.okhttp) + api(libs.retrofit) + + implementation(projects.utils.commonUtils) + + implementation(libs.bundles.kotlinxSerialization) + implementation(libs.log4j.api.kotlin) + implementation(libs.okhttp.loggingInterceptor) + implementation(libs.retrofit.converter.kotlinxSerialization) +} diff --git a/clients/dos/src/main/kotlin/AuthorizationInterceptor.kt b/clients/dos/src/main/kotlin/AuthorizationInterceptor.kt new file mode 100644 index 0000000000000..c54cd0b509a95 --- /dev/null +++ b/clients/dos/src/main/kotlin/AuthorizationInterceptor.kt @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.clients.dos + +import okhttp3.Interceptor + +import retrofit2.Invocation + +/** + * A custom annotation for skipping the authorization interceptor for certain API calls. + */ +@Target(AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.RUNTIME) +internal annotation class SkipAuthorization + +/** + * An HTTP interceptor to conditionally skip authorization. + */ +internal class AuthorizationInterceptor(private val token: String) : Interceptor { + override fun intercept(chain: Interceptor.Chain): okhttp3.Response { + val original = chain.request() + + val skipAuthorization = original.tag(Invocation::class.java) + ?.method() + ?.isAnnotationPresent(SkipAuthorization::class.java) == true + + if (skipAuthorization) return chain.proceed(original) + + val requestBuilder = original.newBuilder() + .header("Authorization", "Bearer $token") + .method(original.method, original.body) + + return chain.proceed(requestBuilder.build()) + } +} diff --git a/clients/dos/src/main/kotlin/DosClient.kt b/clients/dos/src/main/kotlin/DosClient.kt new file mode 100644 index 0000000000000..3a60ca9f424c9 --- /dev/null +++ b/clients/dos/src/main/kotlin/DosClient.kt @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.clients.dos + +import java.io.File + +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.asRequestBody + +import org.apache.logging.log4j.kotlin.logger + +import org.ossreviewtoolkit.utils.common.formatSizeInMib + +/** + * A client implementation on top of the DOS service. + */ +class DosClient(private val service: DosService) { + /** + * Get the package configuration for the given [purl]. Return a [PackageConfigurationResponseBody] or null on error. + */ + suspend fun getPackageConfiguration(purl: String): PackageConfigurationResponseBody? { + if (purl.isEmpty()) { + logger.error { "The Package URL is required to get the package configuration." } + return null + } + + val requestBody = PackageConfigurationRequestBody(purl) + val response = service.getPackageConfiguration(requestBody) + val responseBody = response.body() + + return if (response.isSuccessful && responseBody != null) { + responseBody + } else { + logger.error { "Error getting the package configuration for PURL $purl: ${response.errorBody()}" } + + null + } + } + + /** + * Return a pre-signed URL as provided by the DOS API to upload a package to S3 object storage for scanning, or null + * if the request did not succeed. + */ + suspend fun getUploadUrl(key: String): String? { + if (key.isEmpty()) { + logger.error { "Need the name of the zipped packet to upload" } + return null + } + + val requestBody = UploadUrlRequestBody(key) + val response = service.getUploadUrl(requestBody) + val responseBody = response.body() + + return if (response.isSuccessful && responseBody != null && responseBody.success) { + response.body()?.presignedUrl + } else { + logger.error { "Unable to get a pre-signed URL for $key: ${response.errorBody()?.string()}" } + null + } + } + + /** + * Upload the file at [file] to S3 using the [pre-signed URL][presignedUrl] and return true on success or false + * other on failure. + */ + suspend fun uploadFile(file: File, presignedUrl: String): Boolean { + logger.info { "Uploading file $file of size ${file.formatSizeInMib} to S3..." } + + val contentType = "application/zip".toMediaType() + val response = service.uploadFile(presignedUrl, file.asRequestBody(contentType)) + + return if (response.isSuccessful) { + logger.info { "Successfully uploaded $file to S3." } + true + } else { + logger.error { "Failed to upload $file to S3: ${response.errorBody()}" } + false + } + } + + /** + * Add a new scan job [zipFileKey] and [purls]. Return a [JobResponseBody] or null on error. + */ + suspend fun addScanJob(zipFileKey: String, purls: List): JobResponseBody? { + if (zipFileKey.isEmpty() || purls.isEmpty()) { + logger.error { "The ZIP file key and Package URLs are required to add a scan job." } + return null + } + + val requestBody = JobRequestBody(zipFileKey, purls) + val response = service.addScanJob(requestBody) + val responseBody = response.body() + + return if (response.isSuccessful && responseBody != null) { + responseBody + } else { + logger.error { "Error adding a new scan job for $zipFileKey and $purls: ${response.errorBody()}" } + + null + } + } + + /** + * Get scan results for a list of [purls]. In case multiple PURLs are provided, it is assumed that they all refer to + * the same provenance (like a monorepo). If [fetchConcluded] is true, return concluded licenses instead of detected + * licenses. Return either existing results, a "pending" message if the package is currently being scanned, a + * "no-results" message if a scan yielded no results, or null on error. + */ + suspend fun getScanResults(purls: List, fetchConcluded: Boolean): ScanResultsResponseBody? { + if (purls.isEmpty()) { + logger.error { "The list of PURLs to get scan results for must not be empty." } + return null + } + + val options = ScanResultsRequestBody.ReqOptions(fetchConcluded) + val requestBody = ScanResultsRequestBody(purls, options) + val response = service.getScanResults(requestBody) + val responseBody = response.body() + + return if (response.isSuccessful && responseBody != null) { + when (responseBody.state.status) { + "no-results" -> logger.info { "No scan results found for $purls." } + "pending" -> logger.info { "Scan pending for $purls." } + "ready" -> { + logger.info { "Scan results ready for $purls." } + + if (fetchConcluded) { + logger.info { "Returning concluded licenses instead of detected licenses." } + } + } + } + + responseBody + } else { + logger.error { "Error getting scan results: ${response.errorBody()}" } + + null + } + } + + /** + * Get the state of the scan job with the given [id]. Return a [JobStateResponseBody] with one of waiting / active / + * completed / failed state, or null on error. + */ + suspend fun getScanJobState(id: String): JobStateResponseBody? { + if (id.isEmpty()) { + logger.error { "Need the job ID to check for job state" } + return null + } + + val response = service.getScanJobState(id) + val responseBody = response.body() + + return if (response.isSuccessful && responseBody != null) { + responseBody + } else { + logger.error { "Error getting the scan state for job $id: ${response.errorBody()}" } + + null + } + } +} diff --git a/clients/dos/src/main/kotlin/DosModel.kt b/clients/dos/src/main/kotlin/DosModel.kt new file mode 100644 index 0000000000000..b1c8f3885f321 --- /dev/null +++ b/clients/dos/src/main/kotlin/DosModel.kt @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.clients.dos + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject + +/** + * The JSON (de-)serialization object used by this service. + */ +val JSON = Json { ignoreUnknownKeys = true } + +@Serializable +data class UploadUrlRequestBody( + val key: String? = null +) + +@Serializable +data class UploadUrlResponseBody( + val success: Boolean, + val presignedUrl: String? = null, + val message: String? = null +) + +@Serializable +data class ScanResultsRequestBody( + val purls: List? = null, + val options: ReqOptions? = null +) { + @Serializable + data class ReqOptions( + val fetchConcluded: Boolean? = false + ) +} + +@Serializable +data class ScanResultsResponseBody( + val state: State, + val purls: List? = null, + val results: JsonObject? = null +) { + @Serializable + data class State( + val status: String, + val jobId: String? = null + ) +} + +@Serializable +data class JobRequestBody( + val zipFileKey: String? = null, + val purls: List? = null +) + +@Serializable +data class JobResponseBody( + val scannerJobId: String? = null, + val message: String? = null +) + +@Serializable +data class JobStateResponseBody( + val state: State, + val results: JsonObject? = null +) { + @Serializable + data class State( + val status: String? = null, + val message: String? = null + ) +} + +@Serializable +data class PackageConfigurationRequestBody( + val purl: String? = null +) + +@Serializable +data class PackageConfigurationResponseBody( + val licenseConclusions: List, + val pathExclusions: List +) { + @Serializable + data class LicenseConclusion( + val path: String, + val detectedLicenseExpressionSPDX: String? = null, + val concludedLicenseExpressionSPDX: String, + val comment: String? = null + ) + + @Serializable + data class PathExclusion( + val pattern: String, + val reason: String, + val comment: String? = null + ) +} diff --git a/clients/dos/src/main/kotlin/DosService.kt b/clients/dos/src/main/kotlin/DosService.kt new file mode 100644 index 0000000000000..f7dbd1c87b98d --- /dev/null +++ b/clients/dos/src/main/kotlin/DosService.kt @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.clients.dos + +import java.time.Duration + +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.RequestBody +import okhttp3.logging.HttpLoggingInterceptor + +import retrofit2.Response +import retrofit2.Retrofit +import retrofit2.converter.kotlinx.serialization.asConverterFactory +import retrofit2.http.Body +import retrofit2.http.GET +import retrofit2.http.POST +import retrofit2.http.PUT +import retrofit2.http.Path +import retrofit2.http.Url + +/** + * The service for the REST API client of the Double Open Server (DOS), see https://github.com/doubleopen-project/dos/. + */ +interface DosService { + companion object { + /** + * Create a new service instance that connects to the specified DOS [url] with an authorization [token] and + * uses the optionally provided [timeout] value and HTTP [client]. + */ + fun create(url: String, token: String, timeout: Duration? = null, client: OkHttpClient? = null): DosService { + val contentType = "application/json; charset=utf-8".toMediaType() + + val loggingInterceptor = HttpLoggingInterceptor().apply { + // For logging basic call-response statuses, use "BASIC". + // For logging the request and response bodies of a call, use "BODY". + level = HttpLoggingInterceptor.Level.NONE + } + + val authorizationInterceptor = AuthorizationInterceptor(token) + + val okHttpClient = client ?: OkHttpClient.Builder() + .apply { + if (timeout != null) { + callTimeout(timeout) + connectTimeout(timeout) + readTimeout(timeout) + writeTimeout(timeout) + } + } + .addInterceptor(loggingInterceptor) + .addInterceptor(authorizationInterceptor) + .build() + + val retrofit = Retrofit.Builder() + .client(okHttpClient) + .baseUrl(url) + .addConverterFactory(JSON.asConverterFactory(contentType)) + .build() + + return retrofit.create(DosService::class.java) + } + } + + /** + * Get the package configuration for specified [purl][PackageConfigurationRequestBody.purl]. + */ + @POST("package-configuration") + suspend fun getPackageConfiguration( + @Body body: PackageConfigurationRequestBody + ): Response + + /** + * Get the pre-signed upload URL for S3 object storage with the specified object [key][UploadUrlRequestBody.key]. + */ + @POST("upload-url") + suspend fun getUploadUrl(@Body body: UploadUrlRequestBody): Response + + /** + * Put the [file] into the S3 object storage at [url]. + */ + @SkipAuthorization + @PUT + suspend fun uploadFile(@Url url: String, @Body file: RequestBody): Response + + /** + * Add new scanner job for specified [purls][JobRequestBody.purls]. In case multiple PURLs are provided, it is + * assumed that they all refer to the same provenance (like a monorepo). + */ + @POST("job") + suspend fun addScanJob(@Body body: JobRequestBody): Response + + /** + * Get scan results for specified [purls][ScanResultsRequestBody.purls]. In case multiple PURLs are provided, it is + * assumed that they all refer to the same provenance (like a monorepo). + */ + @POST("scan-results") + suspend fun getScanResults(@Body body: ScanResultsRequestBody): Response + + /** + * Get the state for scan job with given [id]. + */ + @GET("job-state/{id}") + suspend fun getScanJobState(@Path("id") id: String): Response +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 83a922329c56f..0d1179c766b7f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -150,6 +150,7 @@ maven-resolver-transport-wagon = { module = "org.apache.maven.resolver:maven-res mockk = { module = "io.mockk:mockk", version.ref = "mockk" } mordant = { module = "com.github.ajalt.mordant:mordant", version.ref = "mordant" } okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" } +okhttp-loggingInterceptor = { module = "com.squareup.okhttp3:logging-interceptor", version.ref = "okhttp" } postgres = { module = "org.postgresql:postgresql", version.ref = "postgres" } postgresEmbedded = { module = "com.opentable.components:otj-pg-embedded", version.ref = "postgresEmbedded" } reflections = { module = "org.reflections:reflections", version.ref = "reflections" } diff --git a/settings.gradle.kts b/settings.gradle.kts index cef54ad5d32fe..390f154f17d57 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -28,6 +28,7 @@ include(":analyzer") include(":cli") include(":clients:bazel-module-registry") include(":clients:clearly-defined") +include(":clients:dos") include(":clients:fossid-webapp") include(":clients:github-graphql") include(":clients:nexus-iq") @@ -51,6 +52,7 @@ include(":utils:test") project(":clients:bazel-module-registry").name = "bazel-module-registry-client" project(":clients:clearly-defined").name = "clearly-defined-client" +project(":clients:dos").name = "dos-client" project(":clients:fossid-webapp").name = "fossid-webapp-client" project(":clients:github-graphql").name = "github-graphql-client" project(":clients:nexus-iq").name = "nexus-iq-client" diff --git a/utils/ort/src/test/kotlin/UtilsTest.kt b/utils/ort/src/test/kotlin/UtilsTest.kt index e76a97ea0c1eb..8fd094c1d9664 100644 --- a/utils/ort/src/test/kotlin/UtilsTest.kt +++ b/utils/ort/src/test/kotlin/UtilsTest.kt @@ -74,6 +74,7 @@ class UtilsTest : WordSpec({ "DEV0_9_4", "DEV_0_9_7", "Exoffice", + "PROD_1_1_0", "PROD_1_0_1", "PROD_1_0_2", "PROD_1_0_3", @@ -81,7 +82,7 @@ class UtilsTest : WordSpec({ "before_debug_changes" ) - filterVersionNames("1.0.3", names) shouldHaveSingleElement "PROD_1_0_3" + filterVersionNames("1.1.0", names) shouldHaveSingleElement "PROD_1_1_0" filterVersionNames("1", names) should beEmpty() filterVersionNames("0.3", names) should beEmpty() }