From 0bb086a5c44c7594efeb515a98b0df189ec20eb6 Mon Sep 17 00:00:00 2001 From: Nikolaus Wittenstein Date: Wed, 7 Feb 2024 23:03:23 -0800 Subject: [PATCH] Make integrity field optional --- CHANGELOG.md | 4 + .../.bazelversion | 2 +- .../bzlmod_build_file_generation/BUILD.bazel | 4 - .../gazelle_python.yaml | 2 +- .../gazelle_python_windows.yaml | 594 ++++++++++++++++++ gazelle/MODULE.bazel | 1 + gazelle/README.md | 4 + gazelle/manifest/BUILD.bazel | 5 + gazelle/manifest/copy_to_source.py | 36 ++ gazelle/manifest/defs.bzl | 136 ++-- gazelle/manifest/generate/generate.go | 55 +- gazelle/manifest/manifest.go | 13 +- gazelle/manifest/manifest_test.go | 4 +- 13 files changed, 767 insertions(+), 93 deletions(-) create mode 100644 examples/bzlmod_build_file_generation/gazelle_python_windows.yaml create mode 100644 gazelle/manifest/copy_to_source.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bb076dc1c..5923c5e61a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -164,6 +164,10 @@ A brief description of the categories of changes: target for each file with `if __name__ == "__main__"` instead of just one `py_binary` for the whole module. +* (gazelle) the Gazelle manifest integrity field is now optional. If the + `requirements` argument to `gazelle_python_manifest` is unset, no integrity + field will be generated. + ### Fixed * (gazelle) The gazelle plugin helper was not working with Python toolchains 3.11 diff --git a/examples/bzlmod_build_file_generation/.bazelversion b/examples/bzlmod_build_file_generation/.bazelversion index 09b254e90c..19b860c187 100644 --- a/examples/bzlmod_build_file_generation/.bazelversion +++ b/examples/bzlmod_build_file_generation/.bazelversion @@ -1 +1 @@ -6.0.0 +6.4.0 diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index bca3b3681b..f47758da29 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -47,10 +47,6 @@ gazelle_python_manifest( name = "gazelle_python_manifest", modules_mapping = ":modules_map", pip_repository_name = "pip", - requirements = [ - "//:requirements_lock.txt", - "//:requirements_windows.txt", - ], tags = ["exclusive"], ) diff --git a/examples/bzlmod_build_file_generation/gazelle_python.yaml b/examples/bzlmod_build_file_generation/gazelle_python.yaml index 46a1c8b337..c402f694fb 100644 --- a/examples/bzlmod_build_file_generation/gazelle_python.yaml +++ b/examples/bzlmod_build_file_generation/gazelle_python.yaml @@ -232,6 +232,7 @@ manifest: isort.wrap: isort isort.wrap_modes: isort lazy_object_proxy: lazy_object_proxy + lazy_object_proxy.cext: lazy_object_proxy lazy_object_proxy.compat: lazy_object_proxy lazy_object_proxy.simple: lazy_object_proxy lazy_object_proxy.slots: lazy_object_proxy @@ -586,4 +587,3 @@ manifest: yamllint.rules.truthy: yamllint pip_repository: name: pip -integrity: cd25503dc6b3d9e1c5f46715ba2d0499ecc8b3d654ebcbf9f4e52f2074290e0a diff --git a/examples/bzlmod_build_file_generation/gazelle_python_windows.yaml b/examples/bzlmod_build_file_generation/gazelle_python_windows.yaml new file mode 100644 index 0000000000..b0b238f68d --- /dev/null +++ b/examples/bzlmod_build_file_generation/gazelle_python_windows.yaml @@ -0,0 +1,594 @@ +# GENERATED FILE - DO NOT EDIT! +# +# To update this file, run: +# bazel run //:gazelle_python_manifest.update + +manifest: + modules_mapping: + S3: s3cmd + S3.ACL: s3cmd + S3.AccessLog: s3cmd + S3.BidirMap: s3cmd + S3.CloudFront: s3cmd + S3.Config: s3cmd + S3.ConnMan: s3cmd + S3.Crypto: s3cmd + S3.Custom_httplib27: s3cmd + S3.Custom_httplib3x: s3cmd + S3.Exceptions: s3cmd + S3.ExitCodes: s3cmd + S3.FileDict: s3cmd + S3.FileLists: s3cmd + S3.HashCache: s3cmd + S3.MultiPart: s3cmd + S3.PkgInfo: s3cmd + S3.Progress: s3cmd + S3.S3: s3cmd + S3.S3Uri: s3cmd + S3.SortedDict: s3cmd + S3.Utils: s3cmd + astroid: astroid + astroid.arguments: astroid + astroid.astroid_manager: astroid + astroid.bases: astroid + astroid.brain: astroid + astroid.brain.brain_argparse: astroid + astroid.brain.brain_attrs: astroid + astroid.brain.brain_boto3: astroid + astroid.brain.brain_builtin_inference: astroid + astroid.brain.brain_collections: astroid + astroid.brain.brain_crypt: astroid + astroid.brain.brain_ctypes: astroid + astroid.brain.brain_curses: astroid + astroid.brain.brain_dataclasses: astroid + astroid.brain.brain_dateutil: astroid + astroid.brain.brain_fstrings: astroid + astroid.brain.brain_functools: astroid + astroid.brain.brain_gi: astroid + astroid.brain.brain_hashlib: astroid + astroid.brain.brain_http: astroid + astroid.brain.brain_hypothesis: astroid + astroid.brain.brain_io: astroid + astroid.brain.brain_mechanize: astroid + astroid.brain.brain_multiprocessing: astroid + astroid.brain.brain_namedtuple_enum: astroid + astroid.brain.brain_nose: astroid + astroid.brain.brain_numpy_core_einsumfunc: astroid + astroid.brain.brain_numpy_core_fromnumeric: astroid + astroid.brain.brain_numpy_core_function_base: astroid + astroid.brain.brain_numpy_core_multiarray: astroid + astroid.brain.brain_numpy_core_numeric: astroid + astroid.brain.brain_numpy_core_numerictypes: astroid + astroid.brain.brain_numpy_core_umath: astroid + astroid.brain.brain_numpy_ma: astroid + astroid.brain.brain_numpy_ndarray: astroid + astroid.brain.brain_numpy_random_mtrand: astroid + astroid.brain.brain_numpy_utils: astroid + astroid.brain.brain_pathlib: astroid + astroid.brain.brain_pkg_resources: astroid + astroid.brain.brain_pytest: astroid + astroid.brain.brain_qt: astroid + astroid.brain.brain_random: astroid + astroid.brain.brain_re: astroid + astroid.brain.brain_responses: astroid + astroid.brain.brain_scipy_signal: astroid + astroid.brain.brain_signal: astroid + astroid.brain.brain_six: astroid + astroid.brain.brain_sqlalchemy: astroid + astroid.brain.brain_ssl: astroid + astroid.brain.brain_subprocess: astroid + astroid.brain.brain_threading: astroid + astroid.brain.brain_type: astroid + astroid.brain.brain_typing: astroid + astroid.brain.brain_unittest: astroid + astroid.brain.brain_uuid: astroid + astroid.brain.helpers: astroid + astroid.builder: astroid + astroid.const: astroid + astroid.context: astroid + astroid.decorators: astroid + astroid.exceptions: astroid + astroid.filter_statements: astroid + astroid.helpers: astroid + astroid.inference: astroid + astroid.inference_tip: astroid + astroid.interpreter: astroid + astroid.interpreter.dunder_lookup: astroid + astroid.interpreter.objectmodel: astroid + astroid.manager: astroid + astroid.mixins: astroid + astroid.modutils: astroid + astroid.node_classes: astroid + astroid.nodes: astroid + astroid.nodes.as_string: astroid + astroid.nodes.const: astroid + astroid.nodes.node_classes: astroid + astroid.nodes.node_ng: astroid + astroid.nodes.scoped_nodes: astroid + astroid.nodes.scoped_nodes.mixin: astroid + astroid.nodes.scoped_nodes.scoped_nodes: astroid + astroid.nodes.scoped_nodes.utils: astroid + astroid.nodes.utils: astroid + astroid.objects: astroid + astroid.protocols: astroid + astroid.raw_building: astroid + astroid.rebuilder: astroid + astroid.scoped_nodes: astroid + astroid.test_utils: astroid + astroid.transforms: astroid + astroid.typing: astroid + astroid.util: astroid + certifi: certifi + certifi.core: certifi + chardet: chardet + chardet.big5freq: chardet + chardet.big5prober: chardet + chardet.chardistribution: chardet + chardet.charsetgroupprober: chardet + chardet.charsetprober: chardet + chardet.cli: chardet + chardet.cli.chardetect: chardet + chardet.codingstatemachine: chardet + chardet.compat: chardet + chardet.cp949prober: chardet + chardet.enums: chardet + chardet.escprober: chardet + chardet.escsm: chardet + chardet.eucjpprober: chardet + chardet.euckrfreq: chardet + chardet.euckrprober: chardet + chardet.euctwfreq: chardet + chardet.euctwprober: chardet + chardet.gb2312freq: chardet + chardet.gb2312prober: chardet + chardet.hebrewprober: chardet + chardet.jisfreq: chardet + chardet.jpcntx: chardet + chardet.langbulgarianmodel: chardet + chardet.langgreekmodel: chardet + chardet.langhebrewmodel: chardet + chardet.langhungarianmodel: chardet + chardet.langrussianmodel: chardet + chardet.langthaimodel: chardet + chardet.langturkishmodel: chardet + chardet.latin1prober: chardet + chardet.mbcharsetprober: chardet + chardet.mbcsgroupprober: chardet + chardet.mbcssm: chardet + chardet.metadata: chardet + chardet.metadata.languages: chardet + chardet.sbcharsetprober: chardet + chardet.sbcsgroupprober: chardet + chardet.sjisprober: chardet + chardet.universaldetector: chardet + chardet.utf8prober: chardet + chardet.version: chardet + colorama: colorama + colorama.ansi: colorama + colorama.ansitowin32: colorama + colorama.initialise: colorama + colorama.win32: colorama + colorama.winterm: colorama + dateutil: python_dateutil + dateutil.easter: python_dateutil + dateutil.parser: python_dateutil + dateutil.parser.isoparser: python_dateutil + dateutil.relativedelta: python_dateutil + dateutil.rrule: python_dateutil + dateutil.tz: python_dateutil + dateutil.tz.tz: python_dateutil + dateutil.tz.win: python_dateutil + dateutil.tzwin: python_dateutil + dateutil.utils: python_dateutil + dateutil.zoneinfo: python_dateutil + dateutil.zoneinfo.rebuild: python_dateutil + dill: dill + dill.detect: dill + dill.logger: dill + dill.objtypes: dill + dill.pointers: dill + dill.session: dill + dill.settings: dill + dill.source: dill + dill.temp: dill + idna: idna + idna.codec: idna + idna.compat: idna + idna.core: idna + idna.idnadata: idna + idna.intranges: idna + idna.package_data: idna + idna.uts46data: idna + isort: isort + isort.api: isort + isort.comments: isort + isort.core: isort + isort.deprecated: isort + isort.deprecated.finders: isort + isort.exceptions: isort + isort.files: isort + isort.format: isort + isort.hooks: isort + isort.identify: isort + isort.io: isort + isort.literal: isort + isort.logo: isort + isort.main: isort + isort.output: isort + isort.parse: isort + isort.place: isort + isort.profiles: isort + isort.pylama_isort: isort + isort.sections: isort + isort.settings: isort + isort.setuptools_commands: isort + isort.sorting: isort + isort.stdlibs: isort + isort.stdlibs.all: isort + isort.stdlibs.py2: isort + isort.stdlibs.py27: isort + isort.stdlibs.py3: isort + isort.stdlibs.py310: isort + isort.stdlibs.py311: isort + isort.stdlibs.py36: isort + isort.stdlibs.py37: isort + isort.stdlibs.py38: isort + isort.stdlibs.py39: isort + isort.utils: isort + isort.wrap: isort + isort.wrap_modes: isort + lazy_object_proxy: lazy_object_proxy + lazy_object_proxy.compat: lazy_object_proxy + lazy_object_proxy.simple: lazy_object_proxy + lazy_object_proxy.slots: lazy_object_proxy + lazy_object_proxy.utils: lazy_object_proxy + magic: python_magic + magic.compat: python_magic + magic.loader: python_magic + mccabe: mccabe + pathspec: pathspec + pathspec.gitignore: pathspec + pathspec.pathspec: pathspec + pathspec.pattern: pathspec + pathspec.patterns: pathspec + pathspec.patterns.gitwildmatch: pathspec + pathspec.util: pathspec + pkg_resources: setuptools + pkg_resources.extern: setuptools + platformdirs: platformdirs + platformdirs.android: platformdirs + platformdirs.api: platformdirs + platformdirs.macos: platformdirs + platformdirs.unix: platformdirs + platformdirs.version: platformdirs + platformdirs.windows: platformdirs + pylint: pylint + pylint.checkers: pylint + pylint.checkers.async: pylint + pylint.checkers.base: pylint + pylint.checkers.base.basic_checker: pylint + pylint.checkers.base.basic_error_checker: pylint + pylint.checkers.base.comparison_checker: pylint + pylint.checkers.base.docstring_checker: pylint + pylint.checkers.base.name_checker: pylint + pylint.checkers.base.name_checker.checker: pylint + pylint.checkers.base.name_checker.naming_style: pylint + pylint.checkers.base.pass_checker: pylint + pylint.checkers.base_checker: pylint + pylint.checkers.classes: pylint + pylint.checkers.classes.class_checker: pylint + pylint.checkers.classes.special_methods_checker: pylint + pylint.checkers.deprecated: pylint + pylint.checkers.design_analysis: pylint + pylint.checkers.dunder_methods: pylint + pylint.checkers.ellipsis_checker: pylint + pylint.checkers.exceptions: pylint + pylint.checkers.format: pylint + pylint.checkers.imports: pylint + pylint.checkers.lambda_expressions: pylint + pylint.checkers.logging: pylint + pylint.checkers.mapreduce_checker: pylint + pylint.checkers.method_args: pylint + pylint.checkers.misc: pylint + pylint.checkers.modified_iterating_checker: pylint + pylint.checkers.newstyle: pylint + pylint.checkers.non_ascii_names: pylint + pylint.checkers.raw_metrics: pylint + pylint.checkers.refactoring: pylint + pylint.checkers.refactoring.implicit_booleaness_checker: pylint + pylint.checkers.refactoring.not_checker: pylint + pylint.checkers.refactoring.recommendation_checker: pylint + pylint.checkers.refactoring.refactoring_checker: pylint + pylint.checkers.similar: pylint + pylint.checkers.spelling: pylint + pylint.checkers.stdlib: pylint + pylint.checkers.strings: pylint + pylint.checkers.threading_checker: pylint + pylint.checkers.typecheck: pylint + pylint.checkers.unicode: pylint + pylint.checkers.unsupported_version: pylint + pylint.checkers.utils: pylint + pylint.checkers.variables: pylint + pylint.config: pylint + pylint.config.argument: pylint + pylint.config.arguments_manager: pylint + pylint.config.arguments_provider: pylint + pylint.config.callback_actions: pylint + pylint.config.config_file_parser: pylint + pylint.config.config_initialization: pylint + pylint.config.configuration_mixin: pylint + pylint.config.deprecation_actions: pylint + pylint.config.environment_variable: pylint + pylint.config.exceptions: pylint + pylint.config.find_default_config_files: pylint + pylint.config.help_formatter: pylint + pylint.config.option: pylint + pylint.config.option_manager_mixin: pylint + pylint.config.option_parser: pylint + pylint.config.options_provider_mixin: pylint + pylint.config.utils: pylint + pylint.constants: pylint + pylint.epylint: pylint + pylint.exceptions: pylint + pylint.extensions: pylint + pylint.extensions.bad_builtin: pylint + pylint.extensions.broad_try_clause: pylint + pylint.extensions.check_elif: pylint + pylint.extensions.code_style: pylint + pylint.extensions.comparetozero: pylint + pylint.extensions.comparison_placement: pylint + pylint.extensions.confusing_elif: pylint + pylint.extensions.consider_ternary_expression: pylint + pylint.extensions.docparams: pylint + pylint.extensions.docstyle: pylint + pylint.extensions.empty_comment: pylint + pylint.extensions.emptystring: pylint + pylint.extensions.eq_without_hash: pylint + pylint.extensions.for_any_all: pylint + pylint.extensions.mccabe: pylint + pylint.extensions.no_self_use: pylint + pylint.extensions.overlapping_exceptions: pylint + pylint.extensions.private_import: pylint + pylint.extensions.redefined_loop_name: pylint + pylint.extensions.redefined_variable_type: pylint + pylint.extensions.set_membership: pylint + pylint.extensions.typing: pylint + pylint.extensions.while_used: pylint + pylint.graph: pylint + pylint.interfaces: pylint + pylint.lint: pylint + pylint.lint.base_options: pylint + pylint.lint.caching: pylint + pylint.lint.expand_modules: pylint + pylint.lint.message_state_handler: pylint + pylint.lint.parallel: pylint + pylint.lint.pylinter: pylint + pylint.lint.report_functions: pylint + pylint.lint.run: pylint + pylint.lint.utils: pylint + pylint.message: pylint + pylint.message.message: pylint + pylint.message.message_definition: pylint + pylint.message.message_definition_store: pylint + pylint.message.message_id_store: pylint + pylint.pyreverse: pylint + pylint.pyreverse.diadefslib: pylint + pylint.pyreverse.diagrams: pylint + pylint.pyreverse.dot_printer: pylint + pylint.pyreverse.inspector: pylint + pylint.pyreverse.main: pylint + pylint.pyreverse.mermaidjs_printer: pylint + pylint.pyreverse.plantuml_printer: pylint + pylint.pyreverse.printer: pylint + pylint.pyreverse.printer_factory: pylint + pylint.pyreverse.utils: pylint + pylint.pyreverse.vcg_printer: pylint + pylint.pyreverse.writer: pylint + pylint.reporters: pylint + pylint.reporters.base_reporter: pylint + pylint.reporters.collecting_reporter: pylint + pylint.reporters.json_reporter: pylint + pylint.reporters.multi_reporter: pylint + pylint.reporters.reports_handler_mix_in: pylint + pylint.reporters.text: pylint + pylint.reporters.ureports: pylint + pylint.reporters.ureports.base_writer: pylint + pylint.reporters.ureports.nodes: pylint + pylint.reporters.ureports.text_writer: pylint + pylint.testutils: pylint + pylint.testutils.checker_test_case: pylint + pylint.testutils.configuration_test: pylint + pylint.testutils.constants: pylint + pylint.testutils.decorator: pylint + pylint.testutils.functional: pylint + pylint.testutils.functional.find_functional_tests: pylint + pylint.testutils.functional.lint_module_output_update: pylint + pylint.testutils.functional.test_file: pylint + pylint.testutils.functional_test_file: pylint + pylint.testutils.get_test_info: pylint + pylint.testutils.global_test_linter: pylint + pylint.testutils.lint_module_test: pylint + pylint.testutils.output_line: pylint + pylint.testutils.pyreverse: pylint + pylint.testutils.reporter_for_tests: pylint + pylint.testutils.tokenize_str: pylint + pylint.testutils.unittest_linter: pylint + pylint.testutils.utils: pylint + pylint.typing: pylint + pylint.utils: pylint + pylint.utils.ast_walker: pylint + pylint.utils.docs: pylint + pylint.utils.file_state: pylint + pylint.utils.linterstats: pylint + pylint.utils.pragma_parser: pylint + pylint.utils.utils: pylint + requests: requests + requests.adapters: requests + requests.api: requests + requests.auth: requests + requests.certs: requests + requests.compat: requests + requests.cookies: requests + requests.exceptions: requests + requests.help: requests + requests.hooks: requests + requests.models: requests + requests.packages: requests + requests.sessions: requests + requests.status_codes: requests + requests.structures: requests + requests.utils: requests + setuptools: setuptools + setuptools.archive_util: setuptools + setuptools.build_meta: setuptools + setuptools.command: setuptools + setuptools.command.alias: setuptools + setuptools.command.bdist_egg: setuptools + setuptools.command.bdist_rpm: setuptools + setuptools.command.build: setuptools + setuptools.command.build_clib: setuptools + setuptools.command.build_ext: setuptools + setuptools.command.build_py: setuptools + setuptools.command.develop: setuptools + setuptools.command.dist_info: setuptools + setuptools.command.easy_install: setuptools + setuptools.command.editable_wheel: setuptools + setuptools.command.egg_info: setuptools + setuptools.command.install: setuptools + setuptools.command.install_egg_info: setuptools + setuptools.command.install_lib: setuptools + setuptools.command.install_scripts: setuptools + setuptools.command.py36compat: setuptools + setuptools.command.register: setuptools + setuptools.command.rotate: setuptools + setuptools.command.saveopts: setuptools + setuptools.command.sdist: setuptools + setuptools.command.setopt: setuptools + setuptools.command.test: setuptools + setuptools.command.upload: setuptools + setuptools.command.upload_docs: setuptools + setuptools.config: setuptools + setuptools.config.expand: setuptools + setuptools.config.pyprojecttoml: setuptools + setuptools.config.setupcfg: setuptools + setuptools.dep_util: setuptools + setuptools.depends: setuptools + setuptools.discovery: setuptools + setuptools.dist: setuptools + setuptools.errors: setuptools + setuptools.extension: setuptools + setuptools.extern: setuptools + setuptools.glob: setuptools + setuptools.installer: setuptools + setuptools.launch: setuptools + setuptools.logging: setuptools + setuptools.monkey: setuptools + setuptools.msvc: setuptools + setuptools.namespaces: setuptools + setuptools.package_index: setuptools + setuptools.py34compat: setuptools + setuptools.sandbox: setuptools + setuptools.unicode_utils: setuptools + setuptools.version: setuptools + setuptools.wheel: setuptools + setuptools.windows_support: setuptools + six: six + tabulate: tabulate + tabulate.version: tabulate + tomli: tomli + tomlkit: tomlkit + tomlkit.api: tomlkit + tomlkit.container: tomlkit + tomlkit.exceptions: tomlkit + tomlkit.items: tomlkit + tomlkit.parser: tomlkit + tomlkit.source: tomlkit + tomlkit.toml_char: tomlkit + tomlkit.toml_document: tomlkit + tomlkit.toml_file: tomlkit + typing_extensions: typing_extensions + urllib3: urllib3 + urllib3.connection: urllib3 + urllib3.connectionpool: urllib3 + urllib3.contrib: urllib3 + urllib3.contrib.appengine: urllib3 + urllib3.contrib.ntlmpool: urllib3 + urllib3.contrib.pyopenssl: urllib3 + urllib3.contrib.securetransport: urllib3 + urllib3.contrib.socks: urllib3 + urllib3.exceptions: urllib3 + urllib3.fields: urllib3 + urllib3.filepost: urllib3 + urllib3.packages: urllib3 + urllib3.packages.backports: urllib3 + urllib3.packages.backports.makefile: urllib3 + urllib3.packages.six: urllib3 + urllib3.poolmanager: urllib3 + urllib3.request: urllib3 + urllib3.response: urllib3 + urllib3.util: urllib3 + urllib3.util.connection: urllib3 + urllib3.util.proxy: urllib3 + urllib3.util.queue: urllib3 + urllib3.util.request: urllib3 + urllib3.util.response: urllib3 + urllib3.util.retry: urllib3 + urllib3.util.ssl_: urllib3 + urllib3.util.ssl_match_hostname: urllib3 + urllib3.util.ssltransport: urllib3 + urllib3.util.timeout: urllib3 + urllib3.util.url: urllib3 + urllib3.util.wait: urllib3 + wrapt: wrapt + wrapt.arguments: wrapt + wrapt.decorators: wrapt + wrapt.importer: wrapt + wrapt.wrappers: wrapt + yaml: PyYAML + yaml.composer: PyYAML + yaml.constructor: PyYAML + yaml.cyaml: PyYAML + yaml.dumper: PyYAML + yaml.emitter: PyYAML + yaml.error: PyYAML + yaml.events: PyYAML + yaml.loader: PyYAML + yaml.nodes: PyYAML + yaml.parser: PyYAML + yaml.reader: PyYAML + yaml.representer: PyYAML + yaml.resolver: PyYAML + yaml.scanner: PyYAML + yaml.serializer: PyYAML + yaml.tokens: PyYAML + yamllint: yamllint + yamllint.cli: yamllint + yamllint.config: yamllint + yamllint.linter: yamllint + yamllint.parser: yamllint + yamllint.rules: yamllint + yamllint.rules.braces: yamllint + yamllint.rules.brackets: yamllint + yamllint.rules.colons: yamllint + yamllint.rules.commas: yamllint + yamllint.rules.comments: yamllint + yamllint.rules.comments_indentation: yamllint + yamllint.rules.common: yamllint + yamllint.rules.document_end: yamllint + yamllint.rules.document_start: yamllint + yamllint.rules.empty_lines: yamllint + yamllint.rules.empty_values: yamllint + yamllint.rules.float_values: yamllint + yamllint.rules.hyphens: yamllint + yamllint.rules.indentation: yamllint + yamllint.rules.key_duplicates: yamllint + yamllint.rules.key_ordering: yamllint + yamllint.rules.line_length: yamllint + yamllint.rules.new_line_at_end_of_file: yamllint + yamllint.rules.new_lines: yamllint + yamllint.rules.octal_values: yamllint + yamllint.rules.quoted_strings: yamllint + yamllint.rules.trailing_spaces: yamllint + yamllint.rules.truthy: yamllint + pip_repository: + name: pip diff --git a/gazelle/MODULE.bazel b/gazelle/MODULE.bazel index 8c6ad19c7b..940a263953 100644 --- a/gazelle/MODULE.bazel +++ b/gazelle/MODULE.bazel @@ -4,6 +4,7 @@ module( compatibility_level = 1, ) +bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "rules_python", version = "0.18.0") bazel_dep(name = "rules_go", version = "0.41.0", repo_name = "io_bazel_rules_go") bazel_dep(name = "gazelle", version = "0.33.0", repo_name = "bazel_gazelle") diff --git a/gazelle/README.md b/gazelle/README.md index 0a5b1046b7..c1221a6015 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -114,6 +114,10 @@ gazelle_python_manifest( pip_repository_name = "pip", # This should point to wherever we declare our python dependencies # (the same as what we passed to the modules_mapping rule in WORKSPACE) + # This argument is optional. If provided, the `.test` target is very + # fast because it just has to check an integrity field. If not provided, + # the integrity field is not added to the manifest which can help avoid + # merge conflicts in large repos. requirements = "//:requirements_lock.txt", ) ``` diff --git a/gazelle/manifest/BUILD.bazel b/gazelle/manifest/BUILD.bazel index fc7fa09632..33b5a46947 100644 --- a/gazelle/manifest/BUILD.bazel +++ b/gazelle/manifest/BUILD.bazel @@ -1,5 +1,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +exports_files([ + # This gets wrapped up into a py_binary with args inside of the gazelle_python_manifest macro. + "copy_to_source.py", +]) + go_library( name = "manifest", srcs = ["manifest.go"], diff --git a/gazelle/manifest/copy_to_source.py b/gazelle/manifest/copy_to_source.py new file mode 100644 index 0000000000..6854192332 --- /dev/null +++ b/gazelle/manifest/copy_to_source.py @@ -0,0 +1,36 @@ +"""Copy a generated file to the source tree. + +Run like: + copy_to_source path/to/generated_file path/to/source_file_to_overwrite +""" + +import os +import shutil +import stat +import sys +from pathlib import Path + + +def copy_to_source(generated_relative_path: Path, target_relative_path: Path) -> None: + """Copy the generated file to the target file path. + + Expands the relative paths by looking at Bazel env vars to figure out which absolute paths to use. + """ + # This script normally gets executed from the runfiles dir, so find the absolute path to the generated file based on that. + generated_absolute_path = Path.cwd() / generated_relative_path + + # Similarly, the target is relative to the source directory. + target_absolute_path = os.getenv("BUILD_WORKSPACE_DIRECTORY") / target_relative_path + + print(f"Copying {generated_absolute_path} to {target_absolute_path}") + target_absolute_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(generated_absolute_path, target_absolute_path) + + target_absolute_path.chmod(0O664) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit("Usage: copy_to_source ") + + copy_to_source(Path(sys.argv[1]), Path(sys.argv[2])) diff --git a/gazelle/manifest/defs.bzl b/gazelle/manifest/defs.bzl index ef0f275464..af900b5ece 100644 --- a/gazelle/manifest/defs.bzl +++ b/gazelle/manifest/defs.bzl @@ -16,29 +16,35 @@ for updating and testing the Gazelle manifest file. """ -load("@io_bazel_rules_go//go:def.bzl", "GoSource", "go_binary", "go_test") +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") +load("@io_bazel_rules_go//go:def.bzl", "GoSource", "go_test") +load("@rules_python//python:defs.bzl", "py_binary") def gazelle_python_manifest( name, - requirements, modules_mapping, + requirements = [], pip_repository_name = "", pip_deps_repository_name = "", manifest = ":gazelle_python.yaml", + manifest_windows = ":gazelle_python_windows.yaml", **kwargs): """A macro for defining the updating and testing targets for the Gazelle manifest file. Args: name: the name used as a base for the targets. + modules_mapping: the target for the generated modules_mapping.json file. requirements: the target for the requirements.txt file or a list of requirements files that will be concatenated before passing on to - the manifest generator. + the manifest generator. If unset, no integrity field is added to the + manifest, meaning testing it is just as expensive as generating it, + but modifying it is much less likely to result in a merge conflict. pip_repository_name: the name of the pip_install or pip_repository target. pip_deps_repository_name: deprecated - the old pip_install target name. - modules_mapping: the target for the generated modules_mapping.json file. - manifest: the target for the Gazelle manifest file. - **kwargs: other bazel attributes passed to the target target generated by - this macro. + manifest: the Gazelle manifest file. + manifest_windows: the Gazelle manifest file for Windows. + **kwargs: other bazel attributes passed to the generate and test targets + generated by this macro. """ if pip_deps_repository_name != "": # buildifier: disable=print @@ -52,12 +58,22 @@ def gazelle_python_manifest( # This is a temporary check while pip_deps_repository_name exists as deprecated. fail("pip_repository_name must be set in //{}:{}".format(native.package_name(), name)) + test_target = "{}.test".format(name) update_target = "{}.update".format(name) update_target_label = "//{}:{}".format(native.package_name(), update_target) + manifest_genrule = name + ".genrule" + generated_manifest = name + ".generated_manifest" + manifest_generator = Label("//manifest/generate:generate") manifest_generator_hash = Label("//manifest/generate:generate_lib_sources_hash") - if type(requirements) == "list": + selected_manifest_list = select({ + "@platforms//os:windows": [manifest_windows], + "//conditions:default": [manifest], + }) + + if requirements and type(requirements) == "list": + # This runs if requirements is a list or is unset (default value is empty list) native.genrule( name = name + "_requirements_gen", srcs = sorted(requirements), @@ -68,59 +84,77 @@ def gazelle_python_manifest( requirements = name + "_requirements_gen" update_args = [ - "--manifest-generator-hash", - "$(rootpath {})".format(manifest_generator_hash), - "--requirements", - "$(rootpath {})".format(requirements), - "--pip-repository-name", - pip_repository_name, - "--modules-mapping", - "$(rootpath {})".format(modules_mapping), - "--output", - "$(rootpath {})".format(manifest), - "--update-target", - update_target_label, + "--manifest-generator-hash=$(execpath {})".format(manifest_generator_hash), + "--requirements=$(rootpath {})".format(requirements) if requirements else "--requirements=", + "--pip-repository-name={}".format(pip_repository_name), + "--modules-mapping=$(execpath {})".format(modules_mapping), + "--output=$(execpath {})".format(generated_manifest), + "--update-target={}".format(update_target_label), ] - go_binary( - name = update_target, - embed = [Label("//manifest/generate:generate_lib")], - data = [ - manifest, + native.genrule( + name = manifest_genrule, + outs = [generated_manifest], + cmd = "$(execpath {}) {}".format(manifest_generator, " ".join(update_args)), + tools = [manifest_generator], + srcs = [ modules_mapping, - requirements, manifest_generator_hash, - ], - args = update_args, - visibility = ["//visibility:private"], - tags = ["manual"], + ] + ([requirements] if requirements else []), ) - attrs = { - "env": { - "_TEST_MANIFEST": "$(rootpath {})".format(manifest), - "_TEST_MANIFEST_GENERATOR_HASH": "$(rootpath {})".format(manifest_generator_hash), - "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements), - }, - "size": "small", - } - go_test( - name = "{}.test".format(name), - srcs = [Label("//manifest/test:test.go")], - data = [ - manifest, - requirements, - manifest_generator_hash, - ], - rundir = ".", - deps = [Label("//manifest")], - # kwargs could contain test-specific attributes like size or timeout - **dict(attrs, **kwargs) + py_binary( + name = update_target, + srcs = [Label("//manifest:copy_to_source.py")], + main = Label("//manifest:copy_to_source.py"), + args = [ + "$(rootpath {})".format(generated_manifest), + ] + select({ + "@platforms//os:windows": ["$(rootpath {})".format(manifest_windows)], + "//conditions:default": ["$(rootpath {})".format(manifest)], + }), + data = [generated_manifest] + selected_manifest_list, + **kwargs ) + if requirements: + attrs = { + "env": { + "_TEST_MANIFEST_GENERATOR_HASH": "$(rootpath {})".format(manifest_generator_hash), + "_TEST_REQUIREMENTS": "$(rootpath {})".format(requirements), + } | select({ + "@platforms//os:windows": {"_TEST_MANIFEST": "$(rootpath {})".format(manifest_windows)}, + "//conditions:default": {"_TEST_MANIFEST": "$(rootpath {})".format(manifest)}, + }), + "size": "small", + } + go_test( + name = test_target, + srcs = [Label("//manifest/test:test.go")], + data = [ + requirements, + manifest_generator_hash, + ] + selected_manifest_list, + rundir = ".", + deps = [Label("//manifest")], + # kwargs could contain test-specific attributes like size or timeout + **dict(attrs, **kwargs) + ) + else: + diff_test( + name = test_target, + file1 = generated_manifest, + file2 = select({ + "@platforms//os:windows": manifest_windows, + "//conditions:default": manifest, + }), + failure_message = "Gazelle manifest is out of date. Run 'bazel run {}' to update it.".format(native.package_relative_label(update_target)), + **kwargs + ) + native.filegroup( name = name, - srcs = [manifest], + srcs = selected_manifest_list, tags = ["manual"], visibility = ["//visibility:public"], ) diff --git a/gazelle/manifest/generate/generate.go b/gazelle/manifest/generate/generate.go index bdd0206ccb..19ca08a2d6 100644 --- a/gazelle/manifest/generate/generate.go +++ b/gazelle/manifest/generate/generate.go @@ -31,12 +31,6 @@ import ( "github.com/bazelbuild/rules_python/gazelle/manifest" ) -func init() { - if os.Getenv("BUILD_WORKSPACE_DIRECTORY") == "" { - log.Fatalln("ERROR: this program must run under Bazel") - } -} - func main() { var ( manifestGeneratorHashPath string @@ -79,10 +73,6 @@ func main() { "The Bazel target to update the YAML manifest file.") flag.Parse() - if requirementsPath == "" { - log.Fatalln("ERROR: --requirements must be set") - } - if modulesMappingPath == "" { log.Fatalln("ERROR: --modules-mapping must be set") } @@ -102,12 +92,12 @@ func main() { header := generateHeader(updateTarget) repository := manifest.PipRepository{ - Name: pipRepositoryName, + Name: pipRepositoryName, } manifestFile := manifest.NewFile(&manifest.Manifest{ ModulesMapping: modulesMapping, - PipRepository: &repository, + PipRepository: &repository, }) if err := writeOutput( outputPath, @@ -155,12 +145,7 @@ func writeOutput( manifestGeneratorHashPath string, requirementsPath string, ) error { - stat, err := os.Stat(outputPath) - if err != nil { - return fmt.Errorf("failed to write output: %w", err) - } - - outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_TRUNC, stat.Mode()) + outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) if err != nil { return fmt.Errorf("failed to write output: %w", err) } @@ -170,20 +155,26 @@ func writeOutput( return fmt.Errorf("failed to write output: %w", err) } - manifestGeneratorHash, err := os.Open(manifestGeneratorHashPath) - if err != nil { - return fmt.Errorf("failed to write output: %w", err) - } - defer manifestGeneratorHash.Close() - - requirements, err := os.Open(requirementsPath) - if err != nil { - return fmt.Errorf("failed to write output: %w", err) - } - defer requirements.Close() - - if err := manifestFile.Encode(outputFile, manifestGeneratorHash, requirements); err != nil { - return fmt.Errorf("failed to write output: %w", err) + if requirementsPath != "" { + manifestGeneratorHash, err := os.Open(manifestGeneratorHashPath) + if err != nil { + return fmt.Errorf("failed to write output: %w", err) + } + defer manifestGeneratorHash.Close() + + requirements, err := os.Open(requirementsPath) + if err != nil { + return fmt.Errorf("failed to write output: %w", err) + } + defer requirements.Close() + + if err := manifestFile.EncodeWithIntegrity(outputFile, manifestGeneratorHash, requirements); err != nil { + return fmt.Errorf("failed to write output: %w", err) + } + } else { + if err := manifestFile.EncodeWithoutIntegrity(outputFile); err != nil { + return fmt.Errorf("failed to write output: %w", err) + } } return nil diff --git a/gazelle/manifest/manifest.go b/gazelle/manifest/manifest.go index fb146f9439..26b0dfb394 100644 --- a/gazelle/manifest/manifest.go +++ b/gazelle/manifest/manifest.go @@ -31,7 +31,7 @@ type File struct { // Integrity is the hash of the requirements.txt file and the Manifest for // ensuring the integrity of the entire gazelle_python.yaml file. This // controls the testing to keep the gazelle_python.yaml file up-to-date. - Integrity string `yaml:"integrity"` + Integrity string `yaml:"integrity,omitempty"` } // NewFile creates a new File with a given Manifest. @@ -40,12 +40,21 @@ func NewFile(manifest *Manifest) *File { } // Encode encodes the manifest file to the given writer. -func (f *File) Encode(w io.Writer, manifestGeneratorHashFile, requirements io.Reader) error { +func (f *File) EncodeWithIntegrity(w io.Writer, manifestGeneratorHashFile, requirements io.Reader) error { integrityBytes, err := f.calculateIntegrity(manifestGeneratorHashFile, requirements) if err != nil { return fmt.Errorf("failed to encode manifest file: %w", err) } f.Integrity = fmt.Sprintf("%x", integrityBytes) + + return f.encode(w) +} + +func (f *File) EncodeWithoutIntegrity(w io.Writer) error { + return f.encode(w) +} + +func (f *File) encode(w io.Writer) error { encoder := yaml.NewEncoder(w) defer encoder.Close() if err := encoder.Encode(f); err != nil { diff --git a/gazelle/manifest/manifest_test.go b/gazelle/manifest/manifest_test.go index 43c4099aa1..2749733704 100644 --- a/gazelle/manifest/manifest_test.go +++ b/gazelle/manifest/manifest_test.go @@ -40,7 +40,7 @@ var modulesMapping = manifest.ModulesMapping{ const pipDepsRepositoryName = "test_repository_name" func TestFile(t *testing.T) { - t.Run("Encode", func(t *testing.T) { + t.Run("EncodeWithIntegrity", func(t *testing.T) { f := manifest.NewFile(&manifest.Manifest{ ModulesMapping: modulesMapping, PipDepsRepositoryName: pipDepsRepositoryName, @@ -53,7 +53,7 @@ func TestFile(t *testing.T) { t.FailNow() } defer requirements.Close() - if err := f.Encode(&b, manifestGeneratorHashFile, requirements); err != nil { + if err := f.EncodeWithIntegrity(&b, manifestGeneratorHashFile, requirements); err != nil { log.Println(err) t.FailNow() }