From f374260a3d1c764af5a089a20268cfa291ae2604 Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Wed, 18 Dec 2024 16:21:00 +0100 Subject: [PATCH 1/2] [bazel] Introduce an extension for python in repository rules Currently, it is quite tricky to use the python interpreter in the repository rules (like in bitstream_workspace.py) because the repositories created by rules_python are not really meant to be used in the loading phase. With blzmod, this has become quite cumbersome and so this commit introduces a clean mechanism to do so. The first is to create an extension. This extension has a single method to register a python interpret and some packages which are import by pip. The extension will then create a repository containing a single executable which acts as python interpreter taking care of all the detais. Strictly speaking, this could have been achieved without an extension but it would then require to manipulate the pythonpath at every call site and create the required labels every time. By putting everything into an extension, we get to do it once and make it very easy to use everywhere. Signed-off-by: Amaury Pouly --- third_party/python/extension.bzl | 152 +++++++++++++++++++++++++ third_party/python/python.MODULE.bazel | 29 +++++ 2 files changed, 181 insertions(+) create mode 100644 third_party/python/extension.bzl diff --git a/third_party/python/extension.bzl b/third_party/python/extension.bzl new file mode 100644 index 0000000000000..8e77c3231db5c --- /dev/null +++ b/third_party/python/extension.bzl @@ -0,0 +1,152 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +""" +The only extension in this file is the `ot_py_for_repo` extension. +This extension creates repositories that allow to execute python +code in repository rules, where the normal python rules are +unavailable. + +In order to use this extension, first import it as usual: +``` +ot_py_for_repo = use_extension("//third_party/python:extension.bzl", "ot_py_for_repo") +``` + +Now assume that we have already setup a python interpreter as follows: +``` +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(...) +use_repo(python, "pythons_hub") +register_toolchains("@pythons_hub//:all") +# Here it is very important to use the **host** python repository and not +# the usual repository. This repository contaisn symlinks to the actual interpreter. +use_repo(python3_host = "python_3_9_host") +``` + +Further assume that we have setup pip as follows: +``` +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "ot_python_deps", + python_version = "3.9", + requirements_lock = ..., +) +use_repo(pip, "ot_python_deps") +``` + +Now we would like to be able to use the python interpret with some pip dependencies, +say "hjson". We need to register this intention with the extension and it will +create a repository for us. +``` +ot_py_for_repo.register( + # You can choose the following name arbitrarily. + repo_name = "my_favorite_name", + # Here we need to point to the interpret in the repo used above. + python = "@python3_host//:python", + packages = [ + # For each package, we need to pass a label to any file at the root + # of the repository created by rules_python. The general scheme is: + # __ + "@ot_python_deps_39_hjson//:BUILD.bazel" + ], +) +use_repo(ot_py_for_repo, "my_favorite_name") +``` + +Now that we have registered our usage. We can use this interpreter by +simply using the label "@my_favorite_name//:python". This executable +can be called as if it a normal python interpreter. It will automatically +set the PYTHONPATH to the correct value before running so that all the packages +listed at th registrering step at available. + +A quick way to test it is as follows: +``` +bazel run @my_favorite_name//:python +``` +""" + +PYTHON_INTERPRETER_TEMPLATE = """ +# Call the real interpreter with all the arguments +PYTHONPATH={pythonpath} {python_binary} "$@" +""" + +BUILD_CONTENT = """ +exports_files([ + "python" +]) +""" + +def _ot_py_for_repo_repo_impl(ctx): + import_dirs = [ + # We have a path to any file inside the repository, we get + # the parent and then the site packages. + str(ctx.path(pkg).dirname.get_child("site-packages")) + for pkg in ctx.attr.packages + ] + content = PYTHON_INTERPRETER_TEMPLATE.format( + python_binary = ctx.path(ctx.attr.python), + pythonpath = ":".join(import_dirs), + ) + ctx.file("python", content, executable = True) + ctx.file("BUILD", BUILD_CONTENT) + +ot_py_for_repo_repo = repository_rule( + implementation = _ot_py_for_repo_repo_impl, + doc = """ + Create a repository containing a single bash binary: python. + This binary act as a python interpreter and will setup + everything so that the required packages can be used. + """, + attrs = { + "python": attr.label( + allow_single_file = True, + mandatory = True, + doc = "Path to the python interpreter", + ), + "packages": attr.label_list( + default = [], + doc = """ + List of python pip dependencies: point to any file at the root of the repositories + For example, @@ot_python_deps_39_hjson//:BUILD.bazel + """, + ), + }, +) + +_register = tag_class( + attrs = { + "repo_name": attr.string( + mandatory = True, + doc = "Name of the repository to create", + ), + "python": attr.label( + allow_single_file = True, + mandatory = True, + doc = "Path to the python interpreter", + ), + "packages": attr.label_list( + default = [], + doc = "List of python pip dependencies: point to the :pkg inside the bazel repositories", + ), + }, +) + +def _ot_py_for_repo_impl(ctx): + for mod in ctx.modules: + for reg in mod.tags.register: + ot_py_for_repo_repo( + name = reg.repo_name, + python = reg.python, + packages = reg.packages, + ) + + return ctx.extension_metadata(reproducible = True) + +ot_py_for_repo = module_extension( + implementation = _ot_py_for_repo_impl, + tag_classes = { + "register": _register, + }, + doc = "Extensions to allow the use of the Python interpreter and pip dependencies in the repository rules", +) diff --git a/third_party/python/python.MODULE.bazel b/third_party/python/python.MODULE.bazel index 8401df964d7a7..a8735b0551f38 100644 --- a/third_party/python/python.MODULE.bazel +++ b/third_party/python/python.MODULE.bazel @@ -29,3 +29,32 @@ pip.parse( requirements_lock = "//:python-requirements.txt", ) use_repo(pip, "ot_python_deps") + +# List of dependencies that need to used during the loading phase +# by repository rules. +REPO_RULES_PIP_DEPS = [ +] + +# Ask bazel to use all repositories created by rules_python so we +# can access them directly. +[ + use_repo(pip, "ot_python_deps_39_{}".format(pkg)) + for pkg in REPO_RULES_PIP_DEPS +] + +# Call an extension that will create a repository handling all of +# the nasty details so we can run the python interpreter during +# the loading phase with the dependencies from pip. +ot_py_for_repo = use_extension("//third_party/python:extension.bzl", "ot_py_for_repo") +ot_py_for_repo.register( + repo_name = "ot_py_for_repo", + python = "@python3_host//:python", + packages = [ + "@ot_python_deps_39_{}//:BUILD.bazel".format(pkg) + for pkg in REPO_RULES_PIP_DEPS + ], +) + +# The following repository has a python interpreter at //:python +# that will handle dependencies correctly. +use_repo(ot_py_for_repo, "ot_py_for_repo") From 8d4b3dd60478611bda1eb8c8c2c7c7a50753557d Mon Sep 17 00:00:00 2001 From: Amaury Pouly Date: Wed, 18 Dec 2024 16:26:00 +0100 Subject: [PATCH 2/2] [bazel] Convert bitstream workspace rule to the new python ext This python script ideally requires the jsonschema package. Signed-off-by: Amaury Pouly --- rules/bitstreams.bzl | 2 +- third_party/python/python.MODULE.bazel | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/rules/bitstreams.bzl b/rules/bitstreams.bzl index 3f94dafacf6aa..2aea0b80ccec1 100644 --- a/rules/bitstreams.bzl +++ b/rules/bitstreams.bzl @@ -110,7 +110,7 @@ bitstreams_repo = repository_rule( default = 18 * 3600, # Refresh every 18h ), "python_interpreter": attr.label( - default = "@python3_host//:python", + default = "@ot_py_for_repo//:python", allow_single_file = True, doc = "Python interpreter to use.", ), diff --git a/third_party/python/python.MODULE.bazel b/third_party/python/python.MODULE.bazel index a8735b0551f38..b0a314c9dac4d 100644 --- a/third_party/python/python.MODULE.bazel +++ b/third_party/python/python.MODULE.bazel @@ -33,6 +33,11 @@ use_repo(pip, "ot_python_deps") # List of dependencies that need to used during the loading phase # by repository rules. REPO_RULES_PIP_DEPS = [ + # Direct dependencies: + "jsonschema", # In rules/scripts/bitstreams_workspace.py + # Transitive dependencies: + "attrs", # For jsonschema + "pyrsistent", # For jsonschema ] # Ask bazel to use all repositories created by rules_python so we