diff --git a/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2-expected-output.yml b/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2-expected-output.yml new file mode 100644 index 0000000000000..b15d1c98fabde --- /dev/null +++ b/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2-expected-output.yml @@ -0,0 +1,261 @@ +--- +project: + id: "Bazel::plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2/MODULE.bazel:" + definition_file_path: "" + declared_licenses: [] + declared_licenses_processed: {} + vcs: + type: "Git" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "" + revision: "" + path: "" + homepage_url: "" + scopes: + - name: "dev" + dependencies: + - id: "Bazel::googletest:1.14.0" + linkage: "STATIC" + dependencies: + - id: "Bazel::abseil-cpp:20230125.1" + linkage: "STATIC" + dependencies: + - id: "Bazel::bazel_skylib:1.6.1" + linkage: "STATIC" + dependencies: + - id: "Bazel::platforms:0.0.9" + linkage: "STATIC" + - id: "Bazel::platforms:0.0.9" + linkage: "STATIC" + - id: "Bazel::rules_cc:0.0.9" + linkage: "STATIC" + - id: "Bazel::platforms:0.0.9" + linkage: "STATIC" + dependencies: + - id: "Bazel::rules_license:0.0.7" + linkage: "STATIC" + - id: "Bazel::rules_cc:0.0.9" + linkage: "STATIC" + dependencies: + - id: "Bazel::platforms:0.0.9" + linkage: "STATIC" + - name: "main" + dependencies: + - id: "Bazel::glog:0.5.0" + linkage: "STATIC" + dependencies: + - id: "Bazel::gflags:2.2.2" + linkage: "STATIC" +packages: +- id: "Bazel::abseil-cpp:20230125.1" + purl: "pkg:generic/abseil-cpp@20230125.1" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://github.com/abseil/abseil-cpp" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.1.tar.gz" + hash: + value: "81311c17599b3712069ded20cca09a62ab0bf2a89dfa16993786c8782b7ed145" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "https://github.com/abseil/abseil-cpp" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/abseil/abseil-cpp.git" + revision: "" + path: "" +- id: "Bazel::bazel_skylib:1.6.1" + purl: "pkg:generic/bazel_skylib@1.6.1" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://github.com/bazelbuild/bazel-skylib" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/bazelbuild/bazel-skylib/releases/download/1.6.1/bazel-skylib-1.6.1.tar.gz" + hash: + value: "9f38886a40548c6e96c106b752f242130ee11aaa068a56ba7e56f4511f33e4f2" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "https://github.com/bazelbuild/bazel-skylib" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/bazelbuild/bazel-skylib.git" + revision: "" + path: "" +- id: "Bazel::gflags:2.2.2" + purl: "pkg:generic/gflags@2.2.2" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://gflags.github.io/gflags/" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/gflags/gflags/archive/refs/tags/v2.2.2.tar.gz" + hash: + value: "34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "https://github.com/gflags/gflags" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/gflags/gflags.git" + revision: "" + path: "" +- id: "Bazel::glog:0.5.0" + purl: "pkg:generic/glog@0.5.0" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://github.com/google/glog" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/google/glog/archive/refs/tags/v0.5.0.tar.gz" + hash: + value: "eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "https://github.com/google/glog" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/google/glog.git" + revision: "" + path: "" +- id: "Bazel::googletest:1.14.0" + purl: "pkg:generic/googletest@1.14.0" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://google.github.io/googletest/" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/google/googletest/archive/refs/tags/v1.14.0.tar.gz" + hash: + value: "8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "https://github.com/google/googletest" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/google/googletest.git" + revision: "" + path: "" +- id: "Bazel::platforms:0.0.9" + purl: "pkg:generic/platforms@0.0.9" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://github.com/bazelbuild/platforms" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/bazelbuild/platforms/releases/download/0.0.9/platforms-0.0.9.tar.gz" + hash: + value: "5eda539c841265031c2f82d8ae7a3a6490bd62176e0c038fc469eabf91f6149b" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "https://github.com/bazelbuild/platforms" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/bazelbuild/platforms.git" + revision: "" + path: "" +- id: "Bazel::rules_cc:0.0.9" + purl: "pkg:generic/rules_cc@0.0.9" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://github.com/bazelbuild/rules_cc" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz" + hash: + value: "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "" + revision: "" + path: "" +- id: "Bazel::rules_license:0.0.7" + purl: "pkg:generic/rules_license@0.0.7" + declared_licenses: [] + declared_licenses_processed: {} + description: "" + homepage_url: "https://github.com/bazelbuild/rules_license" + binary_artifact: + url: "" + hash: + value: "" + algorithm: "" + source_artifact: + url: "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" + hash: + value: "4531deccb913639c30e5c7512a054d5d875698daeb75d8cf90f284375fe7c360" + algorithm: "SHA-256" + vcs: + type: "Git" + url: "https://github.com/bazelbuild/rules_license" + revision: "" + path: "" + vcs_processed: + type: "Git" + url: "https://github.com/bazelbuild/rules_license.git" + revision: "" + path: "" diff --git a/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2/.bazelversion b/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2/.bazelversion new file mode 100644 index 0000000000000..0ee843cc60466 --- /dev/null +++ b/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2/.bazelversion @@ -0,0 +1 @@ +7.2.0 diff --git a/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2/MODULE.bazel b/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2/MODULE.bazel new file mode 100644 index 0000000000000..7bf05d50c9889 --- /dev/null +++ b/plugins/package-managers/bazel/src/funTest/assets/projects/synthetic/bazel-7.2/MODULE.bazel @@ -0,0 +1,2 @@ +bazel_dep(name = "glog", version = "0.5.0", repo_name = "com_github_google_glog") +bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest", dev_dependency = True) diff --git a/plugins/package-managers/bazel/src/funTest/kotlin/BazelFunTest.kt b/plugins/package-managers/bazel/src/funTest/kotlin/BazelFunTest.kt index afdbf505cbf41..91e620962b0f2 100644 --- a/plugins/package-managers/bazel/src/funTest/kotlin/BazelFunTest.kt +++ b/plugins/package-managers/bazel/src/funTest/kotlin/BazelFunTest.kt @@ -46,4 +46,13 @@ class BazelFunTest : StringSpec({ result.toYaml() should matchExpectedResult(expectedResultFile, definitionFile) } + + "Dependencies are detected correctly with Bazel 7.2.0" { + val definitionFile = getAssetFile("projects/synthetic/bazel-7.2/MODULE.bazel") + val expectedResultFile = getAssetFile("projects/synthetic/bazel-7.2-expected-output.yml") + + val result = create("Bazel").resolveSingleProject(definitionFile) + + result.toYaml() should matchExpectedResult(expectedResultFile, definitionFile) + } }) diff --git a/plugins/package-managers/bazel/src/main/kotlin/Bazel.kt b/plugins/package-managers/bazel/src/main/kotlin/Bazel.kt index 228375940e69f..901c82dee3552 100644 --- a/plugins/package-managers/bazel/src/main/kotlin/Bazel.kt +++ b/plugins/package-managers/bazel/src/main/kotlin/Bazel.kt @@ -109,16 +109,7 @@ class Bazel( // If no lockfile is present, getDependencyGraph() runs "bazel mod graph", which creates a MODULE.bazel.lock // file as a side effect. That file contains the URL of the Bazel module registry that was used for dependency // resolution. - val registryUrl = parseLockfile(lockfile).registryUrl() - - val registry = registryUrl.withoutPrefix("file://")?.let { - val localRegistryURL = it.replace("%workspace%", projectDir.absolutePath) - logger.info { - "Using local Bazel module registry at '$localRegistryURL'." - } - - LocalBazelModuleRegistryService(File(localRegistryURL)) - } ?: RemoteBazelModuleRegistryService.create(registryUrl) + val registry = determineRegistry(parseLockfile(lockfile), projectDir) val packages = getPackages(scopes, registry) @@ -143,6 +134,45 @@ class Bazel( ) } + /** + * This function determines the Bazel module registry to use based on the given [lockfile]: If this is a lockfile + * generated by Bazel version >= 7.2.0, a [CompositeBazelModuleRegistryService] based on the "registryFileHashes" + * will be returned. Else, either a [LocalBazelModuleRegistryService] or a [RemoteBazelModuleRegistryService] based + * on the "cmdRegistries" will be returned. + */ + private fun determineRegistry(lockfile: Lockfile, projectDir: File): BazelModuleRegistryService { + // Bazel version < 7.2.0. + if (lockfile.flags != null) { + val registryUrl = lockfile.registryUrl() + + return registryUrl.withoutPrefix("file://")?.let { + val localRegistryURL = it.replace("%workspace%", projectDir.absolutePath) + logger.info { + "Using local Bazel module registry at '$localRegistryURL'." + } + + LocalBazelModuleRegistryService(File(localRegistryURL)) + } ?: RemoteBazelModuleRegistryService.create(registryUrl) + } + + // Bazel version >= 7.2.0. + if (lockfile.registryFileHashes != null) { + val registryFileHashes = lockfile.registryFileHashes.map { (url, _) -> + if (url.startsWith("file://")) { + url.replace("%workspace%", projectDir.absolutePath) + } else { + url + } + }.toSet() + + return CompositeBazelModuleRegistryService.create(registryFileHashes) + } + + val msg = "Bazel registry URL cannot be determined from the lockfile." + logger.error(msg) + error(msg) + } + private fun getPackages(scopes: Set, registry: BazelModuleRegistryService): Set { val ids = scopes.collectDependencies() val moduleMetadataForId = ids.associateWith { getModuleMetadata(it, registry) } diff --git a/plugins/package-managers/bazel/src/main/kotlin/BazelModel.kt b/plugins/package-managers/bazel/src/main/kotlin/BazelModel.kt index c83f61b02ca00..0b3ea2ef8ebca 100644 --- a/plugins/package-managers/bazel/src/main/kotlin/BazelModel.kt +++ b/plugins/package-managers/bazel/src/main/kotlin/BazelModel.kt @@ -31,8 +31,18 @@ private val json = Json { ignoreUnknownKeys = true } */ @Serializable internal data class Lockfile( - val flags: Flags? = null + // The flags containing the registry URLs (Bazel < 7.2.0). + val flags: Flags? = null, + + // The registry URLs of the project's dependencies packages (Bazel >= 7.2.0). + val registryFileHashes: Map? = null ) { + init { + require((flags != null && registryFileHashes == null) || (flags == null && registryFileHashes != null)) { + "Exactly one of 'flags' and 'registryFileHashes' must be set." + } + } + // TODO Support multiple registries. fun registryUrl(): String? = flags?.cmdRegistries?.getOrElse(0) { null } } diff --git a/plugins/package-managers/bazel/src/main/kotlin/CompositeBazelModuleRegistryService.kt b/plugins/package-managers/bazel/src/main/kotlin/CompositeBazelModuleRegistryService.kt new file mode 100644 index 0000000000000..bafeae3991ece --- /dev/null +++ b/plugins/package-managers/bazel/src/main/kotlin/CompositeBazelModuleRegistryService.kt @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.bazel + +import java.io.File + +import org.apache.logging.log4j.kotlin.logger + +import org.ossreviewtoolkit.clients.bazelmoduleregistry.BazelModuleRegistryService +import org.ossreviewtoolkit.clients.bazelmoduleregistry.LocalBazelModuleRegistryService +import org.ossreviewtoolkit.clients.bazelmoduleregistry.ModuleMetadata +import org.ossreviewtoolkit.clients.bazelmoduleregistry.ModuleSourceInfo +import org.ossreviewtoolkit.clients.bazelmoduleregistry.RemoteBazelModuleRegistryService + +/** + * A composite Bazel module registry service that aggregates multiple [BazelModuleRegistryService] instances and + * delegates the package lookup to the corresponding registry. The first registry that contains the package is used. + * [packagesPerRegistry] is a map of [BazelModuleRegistryService] instances to the list of packages they are + * responsible for. + */ +internal class CompositeBazelModuleRegistryService( + private val packagesPerRegistry: Map> +) : BazelModuleRegistryService { + companion object { + // A regular expression to extract the server and package parts of a registry URL. + internal val URL_REGEX = "^(?.*/)modules/(?[^/]+)/[^/]+/source\\.json$".toRegex() + + /** + * Create a Composite Bazel Module Registry client instance. + */ + fun create(urls: Set): CompositeBazelModuleRegistryService { + val packageNamesForServer = urls.filter { it.endsWith("source.json") }.mapNotNull { url -> + val groups = URL_REGEX.matchEntire(url)?.groups + + val serverName = groups?.get("server")?.value ?: let { + logger.warn { + "$it cannot be mapped to a server root." + } + + return@mapNotNull null + } + + val packageName = groups["package"]?.value ?: url + + serverName to packageName + }.groupByTo(mutableMapOf(), { it.first }) { it.second }.mapValues { it.value.toSet() } + + val packageNamesForRegistry = packageNamesForServer.mapKeys { (url, _) -> + if (url.startsWith("file://")) { + logger.info { + "Using local Bazel module registry at '$url'." + } + + LocalBazelModuleRegistryService(File(url)) + } else { + logger.info { + "Using remote Bazel module registry at '$url'." + } + + RemoteBazelModuleRegistryService.create(url) + } + } + + return CompositeBazelModuleRegistryService(packageNamesForRegistry) + } + } + + override suspend fun getModuleMetadata(name: String): ModuleMetadata { + val registry = packagesPerRegistry.entries.find { name in it.value } + ?: throw IllegalArgumentException("No registry found for package '$name'.") + // TODO check if other registry entries contain the package + + return registry.key.getModuleMetadata(name) + } + + override suspend fun getModuleSourceInfo(name: String, version: String): ModuleSourceInfo { + val registry = packagesPerRegistry.entries.find { name in it.value } + ?: throw IllegalArgumentException("No registry found for package '$name'.") + // TODO check if other registry entries contain the package + + return registry.key.getModuleSourceInfo(name, version) + } +} diff --git a/plugins/package-managers/bazel/src/test/kotlin/CompositeBazelModuleRegistryServiceTest.kt b/plugins/package-managers/bazel/src/test/kotlin/CompositeBazelModuleRegistryServiceTest.kt new file mode 100644 index 0000000000000..77a44b3711710 --- /dev/null +++ b/plugins/package-managers/bazel/src/test/kotlin/CompositeBazelModuleRegistryServiceTest.kt @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.plugins.packagemanagers.bazel + +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.shouldBe + +class CompositeBazelModuleRegistryServiceTest : WordSpec({ + "URL_REGEX" should { + "match the server's base url and the package name" { + val expr1 = "https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/modules/abseil-cpp/" + + "20230125.1/source.json" + val group1 = CompositeBazelModuleRegistryService.Companion.URL_REGEX.matchEntire(expr1)?.groups + + group1?.get("server") + ?.value shouldBe "https://raw.githubusercontent.com/bazelbuild/bazel-central-registry/main/" + group1?.get("package")?.value shouldBe "abseil-cpp" + + val expr2 = "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json" + val group2 = CompositeBazelModuleRegistryService.Companion.URL_REGEX.matchEntire(expr2)?.groups + + group2?.get("server")?.value shouldBe "https://bcr.bazel.build/" + group2?.get("package")?.value shouldBe "rules_proto" + + val expr3 = "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json" + val group3 = CompositeBazelModuleRegistryService.Companion.URL_REGEX.matchEntire(expr3)?.groups + + group3?.get("server")?.value shouldBe "https://bcr.bazel.build/" + group3?.get("package")?.value shouldBe "upb" + } + } +})