[build] Support bzlmod for building and installing

See https://bazel.build/external/migration for background. Note that this commit does not pass all tests when bzlmod is enabled; doing so requires enhancements to our runfiles handling, so for now bzlmod is still disabled by default (except for CMake installs). Details: - Add MODULE.bazel and WORKSPACE.bzlmod for bzlmod use. - Note that WORKSPACE.bzlmod is a copy of WORKSPACE with a tweaked overview comment and a single functional change to our bzlmod flag. - Fix latent bug in repository_excludes for rules_python. - Adjust install rule so installed paths are "drake" not "_main". - Disable drake_py add_test_rule pyc files (for newer rules_python). - Tweak local_config_cc spelling again for bzlmod. - Add linter for module<->workspace syncing. - Turn on bzlmod for CMake to achieve test coverage in CI.
RobotLocomotion · Dec 5, 2024 · ad6888d · ad6888d
1 parent cea8a38
commit ad6888d
Show file tree

Hide file tree

Showing 21 changed files with 324 additions and 19 deletions.
diff --git a/.gitignore b/.gitignore
@@ -20,8 +20,9 @@
 /build/
 /cmake-build-*/
 
-# Bazel build artifacts (symlinks)
+# Bazel build artifacts
 /bazel-*
+/MODULE.bazel.lock
 
 # Platform artifacts generated by `install_prereqs`
 /gen/

diff --git a/BUILD.bazel b/BUILD.bazel
@@ -17,6 +17,15 @@ exports_files([
     "package.xml",
 ])
 
+exports_files(
+    [
+        "MODULE.bazel",
+        "WORKSPACE",
+        "WORKSPACE.bzlmod",
+    ],
+    visibility = ["//tools/workspace:__pkg__"],
+)
+
 # A legacy hack module to disambiguate the 'drake' module when Drake is being
 # used as a non-bzlmod external. We should remove this when we drop support for
 # WORKSPACE (i.e., Bazel >= 9).
@@ -85,11 +94,19 @@ filegroup(
 _INSTALL_TEST_COMMANDS = "install_test_commands"
 
 install(
-    name = "install",
-    install_tests_script = _INSTALL_TEST_COMMANDS,
+    name = "install_files",
     data = ["package.xml"],
+    data_dest = "share/drake",
     docs = ["LICENSE.TXT"],
+    doc_dest = "share/doc/drake",
+    visibility = ["//visibility:private"],
+)
+
+install(
+    name = "install",
+    install_tests_script = _INSTALL_TEST_COMMANDS,
     deps = [
+        ":install_files",
         "//bindings/pydrake:install",
         "//common:install",
         "//examples:install",

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -542,8 +542,9 @@ endforeach()
 # however, that the macOS wheel builds also need to know this path, so if it
 # ever changes, tools/wheel/macos/build-wheel.sh will also need to be updated.
 configure_file(cmake/bazel.rc.in drake_build_cwd/.bazelrc @ONLY)
-configure_file(cmake/WORKSPACE.in drake_build_cwd/WORKSPACE.bazel @ONLY)
+configure_file(cmake/WORKSPACE.bzlmod.in drake_build_cwd/WORKSPACE.bzlmod @ONLY)
 file(CREATE_LINK "${PROJECT_SOURCE_DIR}/.bazeliskrc" drake_build_cwd/.bazeliskrc SYMBOLIC)
+file(CREATE_LINK "${PROJECT_SOURCE_DIR}/MODULE.bazel" drake_build_cwd/MODULE.bazel SYMBOLIC)
 
 find_package(Git)
 

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -0,0 +1,21 @@
+# This file marks a workspace root for the Bazel build system.
+# See `https://bazel.build/`.
+
+# This file lists Drake's external dependencies as known to bzlmod.
+#
+# When bzlmod is disabled, this file is NOT used. Instead, only WORKSPACE is
+# used.
+#
+# When bzlmod is enabled, this file + WORKSPACE.bzlmod are both used, and
+# WORKSPACE is ignored.
+
+module(name = "drake")
+
+bazel_dep(name = "bazel_skylib", version = "1.7.1")
+bazel_dep(name = "rules_cc", version = "0.0.17")
+bazel_dep(name = "rules_java", version = "8.6.1")
+bazel_dep(name = "rules_license", version = "1.0.0")
+bazel_dep(name = "rules_python", version = "0.40.0")
+
+cc_configure = use_extension("@rules_cc//cc:extensions.bzl", "cc_configure_extension")
+use_repo(cc_configure, "local_config_cc")
diff --git a/WORKSPACE b/WORKSPACE
@@ -1,11 +1,16 @@
 # This file marks a workspace root for the Bazel build system.
 # See `https://bazel.build/`.
+#
+# When bzlmod is disabled, only this file is used. The related files
+# MODULE.bazel and WORKSPACE.bzlmod are NOT used.
+#
+# When bzlmod is enabled, this file is ignored.
 
 workspace(name = "drake")
 
 load("//tools/workspace:default.bzl", "add_default_workspace")
 
-add_default_workspace()
+add_default_workspace(bzlmod = False)
 
 load("@build_bazel_apple_support//crosstool:setup.bzl", "apple_cc_configure")
 

diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -0,0 +1,38 @@
+# -*- bazel -*-
+#
+# This file lists Drake's external dependencies as known to bzlmod.
+#
+# When bzlmod is disabled, this file is NOT used. Instead, only WORKSPACE is
+# used.
+#
+# When bzlmod is enabled, this file + MODULE.bazel are both used, and WORKSPACE
+# is ignored.
+
+workspace(name = "drake")
+
+load("//tools/workspace:default.bzl", "add_default_workspace")
+
+add_default_workspace(bzlmod = True)
+
+load("@build_bazel_apple_support//crosstool:setup.bzl", "apple_cc_configure")
+
+apple_cc_configure()
+
+# Add some special heuristic logic for using CLion with Drake.
+load("//tools/clion:repository.bzl", "drake_clion_environment")
+
+drake_clion_environment()
+
+load("@bazel_skylib//lib:versions.bzl", "versions")
+
+# This needs to be in WORKSPACE or a repository rule for native.bazel_version
+# to actually be defined. The minimum_bazel_version value should match the
+# version passed to the find_package(Bazel) call in the root CMakeLists.txt.
+versions.check(minimum_bazel_version = "7.1")
+
+# The cargo_universe programs are only used by Drake's new_release tooling, not
+# by any compilation rules. As such, we can put it directly into the WORKSPACE
+# instead of into our `//tools/workspace:default.bzl` repositories.
+load("@rules_rust//crate_universe:repositories.bzl", "crate_universe_dependencies")  # noqa
+
+crate_universe_dependencies(bootstrap = True)
diff --git a/cmake/WORKSPACE.in → cmake/WORKSPACE.bzlmod.in b/cmake/WORKSPACE.in → cmake/WORKSPACE.bzlmod.in
@@ -22,6 +22,7 @@ _BAZEL_WORKSPACE_EXCLUDES = split_cmake_list("@BAZEL_WORKSPACE_EXCLUDES@")
 # For anything not already overridden, use Drake's default externals.
 add_default_workspace(
     repository_excludes = ["python"] + _BAZEL_WORKSPACE_EXCLUDES,
+    bzlmod = True,
 )
 
 load("@build_bazel_apple_support//crosstool:setup.bzl", "apple_cc_configure")

diff --git a/cmake/bazel.rc.in b/cmake/bazel.rc.in
@@ -7,6 +7,11 @@ startup --output_base="@BAZEL_OUTPUT_BASE@"
 # Inherit Drake's default options.
 @BAZELRC_IMPORT@
 
+# By default Drake (currently) opts-out of bzlmod, but for CMake builds we want
+# to enable it as a mechanism for covering our bzlmod changes in CI, and also
+# to be forward-looking since bzlmod is all Bazel >= 9 will support.
+common --enable_bzlmod=true
+
 # Environment variables to be used in repository rules (if any).
 common @BAZEL_REPO_ENV@
 

diff --git a/tools/bazel.rc b/tools/bazel.rc
@@ -1,4 +1,6 @@
 # Don't use bzlmod yet.
+# TODO(jwnimmer-tri) When we enable bzlmod by default here, we should nix the
+# redundant setting drake/cmake/bazel.rc.in at the same time.
 common --enable_workspace=true
 common --enable_bzlmod=false
 

diff --git a/tools/skylark/drake_py.bzl b/tools/skylark/drake_py.bzl
@@ -150,6 +150,7 @@ def drake_py_binary(
             srcs = srcs,
             main = main,
             deps = deps,
+            precompile = "disabled",  # To avoid *.pyc conflicts.
             isolate = isolate,
             args = test_rule_args,
             data = data + test_rule_data,

diff --git a/tools/workspace/BUILD.bazel b/tools/workspace/BUILD.bazel
@@ -72,6 +72,26 @@ drake_py_test(
     deps = [":module_py"],
 )
 
+drake_py_test(
+    name = "workspace_bzlmod_sync_test",
+    srcs = ["workspace_bzlmod_sync_test.py"],
+    allow_import_unittest = True,
+    data = [
+        ":default.bzl",
+        "//:MODULE.bazel",
+        "//:WORKSPACE",
+        "//:WORKSPACE.bzlmod",
+        "//tools/workspace/bazel_skylib:repository.bzl",
+        "//tools/workspace/rules_cc:repository.bzl",
+        "//tools/workspace/rules_license:repository.bzl",
+    ],
+    tags = ["lint"],
+    deps = [
+        ":module_py",
+        "@rules_python//python/runfiles",
+    ],
+)
+
 drake_py_binary(
     name = "cmake_configure_file",
     srcs = ["cmake_configure_file.py"],

diff --git a/tools/workspace/bazel_skylib/BUILD.bazel b/tools/workspace/bazel_skylib/BUILD.bazel
@@ -1,6 +1,9 @@
-# This file exists to make our directory into a Bazel package, so that our
-# neighboring *.bzl file can be loaded elsewhere.
-
 load("//tools/lint:lint.bzl", "add_lint_tests")
 
+# Required for workspace_bzlmod_sync_test.py.
+exports_files(
+    ["repository.bzl"],
+    visibility = ["//tools/workspace:__pkg__"],
+)
+
 add_lint_tests()
diff --git a/tools/workspace/bazel_skylib/repository.bzl b/tools/workspace/bazel_skylib/repository.bzl
@@ -4,6 +4,10 @@ def bazel_skylib_repository(name, mirrors = None):
     github_archive(
         name = name,
         repository = "bazelbuild/bazel-skylib",
+        upgrade_advice = """
+        When updating, you must also manually propagate to the new version
+        number into the MODULE.bazel file (at the top level of Drake).
+        """,
         commit = "1.7.1",
         sha256 = "e3fea03ff75a9821e84199466799ba560dbaebb299c655b5307f4df1e5970696",  # noqa
         mirrors = mirrors,

diff --git a/tools/workspace/cc/repository.bzl b/tools/workspace/cc/repository.bzl
@@ -30,10 +30,6 @@ Argument:
 
 load("//tools/workspace:execute.bzl", "execute_or_fail")
 
-# We can probe whether bzlmod is enabled by checking if labels use one or two
-# leading '@' charaters. (The label doesn't need to be valid.)
-BZLMOD_ENABLED = "@@" in str(Label("//:foo"))
-
 def _check_compiler_version(compiler_id, actual_version, supported_version):
     """
     Check if the compiler is of a supported version and report an error if not.
@@ -91,8 +87,13 @@ def _impl(repository_ctx):
     else:
         cc_environment = {}
 
+    # For Bazel 7.x sometimes we need a weird spelling of @local_config_cc.
+    # We can probably remove this once our minimum supported Bazel is >= 8.
     local_config_cc = "@local_config_cc"
-    if BZLMOD_ENABLED:
+    if all([
+        native.bazel_version.startswith("7."),
+        "@@" in str(Label("//:foo")),
+    ]):
         local_config_cc = "@bazel_tools~cc_configure_extension~local_config_cc"
     executable = repository_ctx.path("identify_compiler")
     execute_or_fail(repository_ctx, [

diff --git a/tools/workspace/default.bzl b/tools/workspace/default.bzl
@@ -106,7 +106,18 @@ load("//tools/workspace/xmlrunner_py:repository.bzl", "xmlrunner_py_repository")
 load("//tools/workspace/yaml_cpp_internal:repository.bzl", "yaml_cpp_internal_repository")  # noqa
 load("//tools/workspace/zlib:repository.bzl", "zlib_repository")
 
-def add_default_repositories(excludes = [], mirrors = DEFAULT_MIRRORS):
+# This is the list of modules that our MODULE.bazel already incorporates.
+REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES = [
+    "bazel_skylib",
+    "rules_cc",
+    "rules_license",
+    "rules_python",
+]
+
+def add_default_repositories(
+        excludes = [],
+        mirrors = DEFAULT_MIRRORS,
+        bzlmod = False):
     """Declares workspace repositories for all externals needed by drake (other
     than those built into Bazel, of course).  This is intended to be loaded and
     called from a WORKSPACE file.
@@ -115,7 +126,11 @@ def add_default_repositories(excludes = [], mirrors = DEFAULT_MIRRORS):
         excludes: list of string names of repositories to exclude; this can
           be useful if a WORKSPACE file has already supplied its own external
           of a given name.
+        bzlmod: when True, skips repositories declared in our MODULE.bazel;
+          set this to True if you are using bzlmod.
     """
+    if bzlmod:
+        excludes = excludes + REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES
     if "abseil_cpp_internal" not in excludes:
         abseil_cpp_internal_repository(name = "abseil_cpp_internal", mirrors = mirrors)  # noqa
     if "bazelisk" not in excludes:
@@ -272,6 +287,8 @@ def add_default_repositories(excludes = [], mirrors = DEFAULT_MIRRORS):
         rules_license_repository(name = "rules_license", mirrors = mirrors)
     if "rules_python" not in excludes:
         rules_python_repository(name = "rules_python", mirrors = mirrors)
+    else:
+        rules_python_repository(name = "rules_python", _constants_only = True)
     if "rules_rust" not in excludes:
         rules_rust_repository(name = "rules_rust", mirrors = mirrors)
     if "rules_rust_tinyjson" not in excludes:
@@ -348,7 +365,8 @@ def add_default_toolchains(excludes = []):
 def add_default_workspace(
         repository_excludes = [],
         toolchain_excludes = [],
-        mirrors = DEFAULT_MIRRORS):
+        mirrors = DEFAULT_MIRRORS,
+        bzlmod = False):
     """Declare repositories in this WORKSPACE for each dependency of @drake
     (e.g., "eigen") that is not explicitly excluded, and register toolchains
     for each language (e.g., "py") not explicitly excluded and/or not using an
@@ -364,5 +382,9 @@ def add_default_workspace(
             default values.
     """
 
-    add_default_repositories(excludes = repository_excludes, mirrors = mirrors)
+    add_default_repositories(
+        excludes = repository_excludes,
+        mirrors = mirrors,
+        bzlmod = bzlmod,
+    )
     add_default_toolchains(excludes = toolchain_excludes)
diff --git a/tools/workspace/rules_cc/BUILD.bazel b/tools/workspace/rules_cc/BUILD.bazel
@@ -1,3 +1,9 @@
 load("//tools/lint:lint.bzl", "add_lint_tests")
 
+# Required for workspace_bzlmod_sync_test.py.
+exports_files(
+    ["repository.bzl"],
+    visibility = ["//tools/workspace:__pkg__"],
+)
+
 add_lint_tests()
diff --git a/tools/workspace/rules_cc/repository.bzl b/tools/workspace/rules_cc/repository.bzl
@@ -10,6 +10,10 @@ def rules_cc_repository(
     github_archive(
         name = name,
         repository = "bazelbuild/rules_cc",  # License: Apache-2.0,
+        upgrade_advice = """
+        When updating, you must also manually propagate to the new version
+        number into the MODULE.bazel file (at the top level of Drake).
+        """,
         commit = "0.0.17",
         sha256 = "abc605dd850f813bb37004b77db20106a19311a96b2da1c92b789da529d28fe1",  # noqa
         patches = [

diff --git a/tools/workspace/rules_license/BUILD.bazel b/tools/workspace/rules_license/BUILD.bazel
@@ -1,3 +1,9 @@
 load("//tools/lint:lint.bzl", "add_lint_tests")
 
+# Required for workspace_bzlmod_sync_test.py.
+exports_files(
+    ["repository.bzl"],
+    visibility = ["//tools/workspace:__pkg__"],
+)
+
 add_lint_tests()
diff --git a/tools/workspace/rules_license/repository.bzl b/tools/workspace/rules_license/repository.bzl
@@ -10,6 +10,10 @@ def rules_license_repository(
     github_archive(
         name = name,
         repository = "bazelbuild/rules_license",  # License: Apache-2.0,
+        upgrade_advice = """
+        When updating, you must also manually propagate to the new version
+        number into the MODULE.bazel file (at the top level of Drake).
+        """,
         commit = "1.0.0",
         sha256 = "75759939aef3aeb726e801417a883deefadadb7fea49946a1f5bb74a5162e81e",  # noqa
         mirrors = mirrors,

diff --git a/tools/workspace/rules_python/repository.bzl b/tools/workspace/rules_python/repository.bzl
@@ -25,17 +25,22 @@ _rules_python_drake_constants_repository = repository_rule(
 
 def rules_python_repository(
         name,
-        mirrors = None):
-    # For Bazel versions < 8, we pin our own particular copy of rules_python.
+        mirrors = None,
+        _constants_only = False):
+    # For Bazel versions < 8, we pin our own particular copy of rules_python,
+    # though when we are skipping rules_python by definition that is unpinned.
     # For Bazel versions >= 8, we'll use Bazel's vendored copy of rules_python.
     # Our minimum version (per WORKSPACE) is 7.1 so we can use a string match.
-    use_drake_rules_python_pin = native.bazel_version[0:2] == "7."
+    use_drake_rules_python_pin = (native.bazel_version[0:2] == "7." and
+                                  not _constants_only)
     _rules_python_drake_constants_repository(
         name = name + "_drake_constants",
         constants_json = json.encode({
             "USE_DRAKE_PIN": 1 if use_drake_rules_python_pin else 0,
         }),
     )
+    if _constants_only:
+        return
     if not use_drake_rules_python_pin:
         return
     github_archive(