From 2abca35cf67f60d41485eaab1f53ff7a3c6e9b50 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:19:26 +0900 Subject: [PATCH] refactor(pypi): move config setting processing to the macro (#2424) Before hand we would pass around `whl_alias` struct instances and it would go through rounds of starlark functions which would end up rendered as BUILD.bazel files in the hub repository. This means that every time the output of the said functions would change, the `MODULE.bazel.lock` would also possibly change. Since we now have much better unit testing framework, we start relying on these functions being evaluated at loading phase instead of repo rule phase, which means that we can make the `BUILD.bazel` files not change a lot when the underlying code changes, this should make the code more maintainable in the long run and unblocks the code to accept extra things, like env marker values or similar. Summary: - Remove the `repo` field from `whl_alias`. - Rename `whl_alias` to `whl_config_setting` - Move the tests around - Simplify the `pkg_aliases` tests to use `contains` instead of exact equality to make things less brittle. - Simplify the `pkg_aliases` tests to use different mocks to make expectations easier to understand. - Make `whl_config_setting` hashable by using tuples instead of lists. - Adjust how we store the `whl_config_setting` in the PyPI extension and optimize the passing to the `hub_repository`. This is needed for #2319 and #2423. To be in future PRs: * Remove the need to pass `osx_versions`, etc to the `pkg_aliases` macro. --- python/private/pypi/BUILD.bazel | 10 +- python/private/pypi/extension.bzl | 53 +-- python/private/pypi/hub_repository.bzl | 51 +- python/private/pypi/pip_repository.bzl | 4 +- python/private/pypi/pkg_aliases.bzl | 272 ++++++++++- python/private/pypi/render_pkg_aliases.bzl | 383 +++------------ python/private/pypi/whl_config_setting.bzl | 50 ++ python/private/text_util.bzl | 17 + tests/pypi/extension/extension_tests.bzl | 123 ++--- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 441 ++++++++++++++---- .../render_pkg_aliases_test.bzl | 328 +++---------- 11 files changed, 945 insertions(+), 787 deletions(-) create mode 100644 python/private/pypi/whl_config_setting.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index dafbcfb757..2cc073a832 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -64,6 +64,7 @@ bzl_library( ":parse_whl_name_bzl", ":pip_repository_attrs_bzl", ":simpleapi_download_bzl", + ":whl_config_setting_bzl", ":whl_library_bzl", ":whl_repo_name_bzl", "//python/private:full_version_bzl", @@ -156,7 +157,7 @@ bzl_library( name = "multi_pip_parse_bzl", srcs = ["multi_pip_parse.bzl"], deps = [ - "pip_repository_bzl", + ":pip_repository_bzl", "//python/private:text_util_bzl", ], ) @@ -230,6 +231,7 @@ bzl_library( ":parse_requirements_bzl", ":pip_repository_attrs_bzl", ":render_pkg_aliases_bzl", + ":whl_config_setting_bzl", "//python/private:normalize_name_bzl", "//python/private:repo_utils_bzl", "//python/private:text_util_bzl", @@ -257,6 +259,7 @@ bzl_library( deps = [ ":generate_group_library_build_bazel_bzl", ":parse_whl_name_bzl", + ":whl_config_setting_bzl", ":whl_target_platforms_bzl", "//python/private:normalize_name_bzl", "//python/private:text_util_bzl", @@ -283,6 +286,11 @@ bzl_library( ], ) +bzl_library( + name = "whl_config_setting_bzl", + srcs = ["whl_config_setting.bzl"], +) + bzl_library( name = "whl_library_alias_bzl", srcs = ["whl_library_alias.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 2ed946fbca..fd224d1592 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -23,13 +23,13 @@ load("//python/private:semver.bzl", "semver") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS") -load(":hub_repository.bzl", "hub_repository") +load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") load(":pip_repository_attrs.bzl", "ATTRS") -load(":render_pkg_aliases.bzl", "whl_alias") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") +load(":whl_config_setting.bzl", "whl_config_setting") load(":whl_library.bzl", "whl_library") load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name") @@ -87,7 +87,7 @@ def _create_whl_repos( Returns a {type}`struct` with the following attributes: whl_map: {type}`dict[str, list[struct]]` the output is keyed by the normalized package name and the values are the instances of the - {bzl:obj}`whl_alias` return values. + {bzl:obj}`whl_config_setting` return values. exposed_packages: {type}`dict[str, Any]` this is just a way to represent a set of string values. whl_libraries: {type}`dict[str, dict[str, Any]]` the keys are the @@ -304,14 +304,11 @@ def _create_whl_repos( whl_libraries[repo_name] = args - whl_map.setdefault(whl_name, []).append( - whl_alias( - repo = repo_name, - version = major_minor, - filename = distribution.filename, - target_platforms = target_platforms, - ), - ) + whl_map.setdefault(whl_name, {})[whl_config_setting( + version = major_minor, + filename = distribution.filename, + target_platforms = target_platforms, + )] = repo_name if found_something: if is_exposed: @@ -339,13 +336,10 @@ def _create_whl_repos( *target_platforms ) whl_libraries[repo_name] = args - whl_map.setdefault(whl_name, []).append( - whl_alias( - repo = repo_name, - version = major_minor, - target_platforms = target_platforms or None, - ), - ) + whl_map.setdefault(whl_name, {})[whl_config_setting( + version = major_minor, + target_platforms = target_platforms or None, + )] = repo_name if is_exposed: exposed_packages[whl_name] = None @@ -488,7 +482,8 @@ You cannot use both the additive_build_content and additive_build_content_file a ) hub_whl_map.setdefault(hub_name, {}) for key, settings in out.whl_map.items(): - hub_whl_map[hub_name].setdefault(key, []).extend(settings) + for setting, repo in settings.items(): + hub_whl_map[hub_name].setdefault(key, {}).setdefault(repo, []).append(setting) extra_aliases.setdefault(hub_name, {}) for whl_name, aliases in out.extra_aliases.items(): extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) @@ -508,7 +503,7 @@ You cannot use both the additive_build_content and additive_build_content_file a whl_mods = dict(sorted(whl_mods.items())), hub_whl_map = { hub_name: { - whl_name: sorted(settings, key = lambda x: (x.version, x.filename)) + whl_name: dict(settings) for whl_name, settings in sorted(whl_map.items()) } for hub_name, whl_map in sorted(hub_whl_map.items()) @@ -538,20 +533,6 @@ You cannot use both the additive_build_content and additive_build_content_file a is_reproducible = is_reproducible, ) -def _alias_dict(a): - ret = { - "repo": a.repo, - } - if a.config_setting: - ret["config_setting"] = a.config_setting - if a.filename: - ret["filename"] = a.filename - if a.target_platforms: - ret["target_platforms"] = a.target_platforms - if a.version: - ret["version"] = a.version - return ret - def _pip_impl(module_ctx): """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories. @@ -632,8 +613,8 @@ def _pip_impl(module_ctx): repo_name = hub_name, extra_hub_aliases = mods.extra_aliases.get(hub_name, {}), whl_map = { - key: json.encode([_alias_dict(a) for a in aliases]) - for key, aliases in whl_map.items() + key: whl_config_settings_to_json(values) + for key, values in whl_map.items() }, packages = mods.exposed_packages.get(hub_name, []), groups = mods.hub_group_map.get(hub_name), diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 69d937142a..48245b4106 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -15,11 +15,8 @@ "" load("//python/private:text_util.bzl", "render") -load( - ":render_pkg_aliases.bzl", - "render_multiplatform_pkg_aliases", - "whl_alias", -) +load(":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases") +load(":whl_config_setting.bzl", "whl_config_setting") _BUILD_FILE_CONTENTS = """\ package(default_visibility = ["//visibility:public"]) @@ -32,7 +29,7 @@ def _impl(rctx): bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { - key: [whl_alias(**v) for v in json.decode(values)] + key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, extra_hub_aliases = rctx.attr.extra_hub_aliases, @@ -97,3 +94,45 @@ in the pip.parse tag class. doc = """A rule for bzlmod mulitple pip repository creation. PRIVATE USE ONLY.""", implementation = _impl, ) + +def _whl_config_settings_from_json(repo_mapping_json): + """Deserialize the serialized values with whl_config_settings_to_json. + + Args: + repo_mapping_json: {type}`str` + + Returns: + What `whl_config_settings_to_json` accepts. + """ + return { + whl_config_setting(**v): repo + for repo, values in json.decode(repo_mapping_json).items() + for v in values + } + +def whl_config_settings_to_json(repo_mapping): + """A function to serialize the aliases so that `hub_repository` can accept them. + + Args: + repo_mapping: {type}`dict[str, list[struct]]` repo to + {obj}`whl_config_setting` mapping. + + Returns: + A deserializable JSON string + """ + return json.encode({ + repo: [_whl_config_setting_dict(s) for s in settings] + for repo, settings in repo_mapping.items() + }) + +def _whl_config_setting_dict(a): + ret = {} + if a.config_setting: + ret["config_setting"] = a.config_setting + if a.filename: + ret["filename"] = a.filename + if a.target_platforms: + ret["target_platforms"] = a.target_platforms + if a.version: + ret["version"] = a.version + return ret diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 597b37f52c..47fa31f1bc 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -21,7 +21,7 @@ load("//python/private:text_util.bzl", "render") load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pip_repository_attrs.bzl", "ATTRS") -load(":render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") +load(":render_pkg_aliases.bzl", "render_pkg_aliases") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") def _get_python_interpreter_attr(rctx): @@ -174,7 +174,7 @@ def _pip_repository_impl(rctx): aliases = render_pkg_aliases( aliases = { - pkg: [whl_alias(repo = rctx.attr.name + "_" + pkg)] + pkg: rctx.attr.name + "_" + pkg for pkg in bzl_packages or [] }, extra_hub_aliases = rctx.attr.extra_hub_aliases, diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index de9545e986..284f8e9ed0 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -16,6 +16,7 @@ This is used in bzlmod and non-bzlmod setups.""" +load("@bazel_skylib//lib:selects.bzl", "selects") load("//python/private:text_util.bzl", "render") load( ":labels.bzl", @@ -26,6 +27,14 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) +load(":parse_whl_name.bzl", "parse_whl_name") +load(":whl_target_platforms.bzl", "whl_target_platforms") + +# This value is used as sentinel value in the alias/config setting machinery +# for libc and osx versions. If we encounter this version in this part of the +# code, then it means that we have a bug in rules_python and that we should fix +# it. It is more of an internal consistency check. +_VERSION_NONE = (0, 0) _NO_MATCH_ERROR_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -53,7 +62,11 @@ def _no_match_error(actual): return _NO_MATCH_ERROR_TEMPLATE.format( config_settings = render.indent( - "\n".join(sorted(actual.keys())), + "\n".join(sorted([ + value + for key in actual + for value in (key if type(key) == "tuple" else [key]) + ])), ).lstrip(), rules_python = "rules_python", ) @@ -65,17 +78,21 @@ def pkg_aliases( group_name = None, extra_aliases = None, native = native, - select = select): + select = selects.with_or, + **kwargs): """Create aliases for an actual package. Args: name: {type}`str` The name of the package. - actual: {type}`dict[Label, str] | str` The name of the repo the aliases point to, or a dict of select conditions to repo names for the aliases to point to - mapping to repositories. + actual: {type}`dict[Label | tuple, str] | str` The name of the repo the + aliases point to, or a dict of select conditions to repo names for + the aliases to point to mapping to repositories. The keys are passed + to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. extra_aliases: {type}`list[str]` The extra aliases to be created. native: {type}`struct` used in unit tests. select: {type}`select` used in unit tests. + **kwargs: extra kwargs to pass to {bzl:obj}`get_filename_config_settings`. """ native.alias( name = name, @@ -91,6 +108,8 @@ def pkg_aliases( x: x for x in extra_aliases or [] } + + actual = multiplatform_whl_aliases(aliases = actual, **kwargs) no_match_error = _no_match_error(actual) for name, target_name in target_names.items(): @@ -102,11 +121,11 @@ def pkg_aliases( elif type(actual) == type({}): _actual = select( { - config_setting: "@{repo}//:{target_name}".format( + v: "@{repo}//:{target_name}".format( repo = repo, target_name = name, ) - for config_setting, repo in actual.items() + for v, repo in actual.items() }, no_match_error = no_match_error, ) @@ -132,3 +151,244 @@ def pkg_aliases( name = WHEEL_FILE_PUBLIC_LABEL, actual = "//_groups:{}_whl".format(group_name), ) + +def _normalize_versions(name, versions): + if not versions: + return [] + + if _VERSION_NONE in versions: + fail("a sentinel version found in '{}', check render_pkg_aliases for bugs".format(name)) + + return sorted(versions) + +def multiplatform_whl_aliases( + *, + aliases = [], + glibc_versions = [], + muslc_versions = [], + osx_versions = []): + """convert a list of aliases from filename to config_setting ones. + + Args: + aliases: {type}`str | dict[whl_config_setting | str, str]`: The aliases + to process. Any aliases that have the filename set will be + converted to a dict of config settings to repo names. + glibc_versions: {type}`list[tuple[int, int]]` list of versions that can be + used in this hub repo. + muslc_versions: {type}`list[tuple[int, int]]` list of versions that can be + used in this hub repo. + osx_versions: {type}`list[tuple[int, int]]` list of versions that can be + used in this hub repo. + + Returns: + A dict with of config setting labels to repo names or the repo name itself. + """ + + if type(aliases) == type(""): + # We don't have any aliases, this is a repo name + return aliases + + # TODO @aignas 2024-11-17: we might be able to use FeatureFlagInfo and some + # code gen to create a version_lt_x target, which would allow us to check + # if the libc version is in a particular range. + glibc_versions = _normalize_versions("glibc_versions", glibc_versions) + muslc_versions = _normalize_versions("muslc_versions", muslc_versions) + osx_versions = _normalize_versions("osx_versions", osx_versions) + + ret = {} + versioned_additions = {} + for alias, repo in aliases.items(): + if type(alias) != "struct": + ret[alias] = repo + continue + elif not (alias.filename or alias.target_platforms): + # This is an internal consistency check + fail("Expected to have either 'filename' or 'target_platforms' set, got: {}".format(alias)) + + config_settings, all_versioned_settings = get_filename_config_settings( + filename = alias.filename or "", + target_platforms = alias.target_platforms, + python_version = alias.version, + # If we have multiple platforms but no wheel filename, lets use different + # config settings. + non_whl_prefix = "sdist" if alias.filename else "", + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + osx_versions = osx_versions, + ) + + for setting in config_settings: + ret["//_config" + setting] = repo + + # Now for the versioned platform config settings, we need to select one + # that best fits the bill and if there are multiple wheels, e.g. + # manylinux_2_17_x86_64 and manylinux_2_28_x86_64, then we need to select + # the former when the glibc is in the range of [2.17, 2.28) and then chose + # the later if it is [2.28, ...). If the 2.28 wheel was not present in + # the hub, then we would need to use 2.17 for all the glibc version + # configurations. + # + # Here we add the version settings to a dict where we key the range of + # versions that the whl spans. If the wheel supports musl and glibc at + # the same time, we do this for each supported platform, hence the + # double dict. + for default_setting, versioned in all_versioned_settings.items(): + versions = sorted(versioned) + min_version = versions[0] + max_version = versions[-1] + + versioned_additions.setdefault(default_setting, {})[(min_version, max_version)] = struct( + repo = repo, + settings = versioned, + ) + + versioned = {} + for default_setting, candidates in versioned_additions.items(): + # Sort the candidates by the range of versions the span, so that we + # start with the lowest version. + for _, candidate in sorted(candidates.items()): + # Set the default with the first candidate, which gives us the highest + # compatibility. If the users want to use a higher-version than the default + # they can configure the glibc_version flag. + versioned.setdefault("//_config" + default_setting, candidate.repo) + + # We will be overwriting previously added entries, but that is intended. + for _, setting in candidate.settings.items(): + versioned["//_config" + setting] = candidate.repo + + ret.update(versioned) + return ret + +def get_filename_config_settings( + *, + filename, + target_platforms, + python_version, + glibc_versions = None, + muslc_versions = None, + osx_versions = None, + non_whl_prefix = "sdist"): + """Get the filename config settings. + + Args: + filename: the distribution filename (can be a whl or an sdist). + target_platforms: list[str], target platforms in "{abi}_{os}_{cpu}" format. + glibc_versions: list[tuple[int, int]], list of versions. + muslc_versions: list[tuple[int, int]], list of versions. + osx_versions: list[tuple[int, int]], list of versions. + python_version: the python version to generate the config_settings for. + non_whl_prefix: the prefix of the config setting when the whl we don't have + a filename ending with ".whl". + + Returns: + A tuple: + * A list of config settings that are generated by ./pip_config_settings.bzl + * The list of default version settings. + """ + prefixes = [] + suffixes = [] + setting_supported_versions = {} + + if filename.endswith(".whl"): + parsed = parse_whl_name(filename) + if parsed.python_tag == "py2.py3": + py = "py" + elif parsed.python_tag.startswith("cp"): + py = "cp3x" + else: + py = "py3" + + if parsed.abi_tag.startswith("cp"): + abi = "cp" + else: + abi = parsed.abi_tag + + if parsed.platform_tag == "any": + prefixes = ["_{}_{}_any".format(py, abi)] + else: + prefixes = ["_{}_{}".format(py, abi)] + suffixes = _whl_config_setting_suffixes( + platform_tag = parsed.platform_tag, + glibc_versions = glibc_versions, + muslc_versions = muslc_versions, + osx_versions = osx_versions, + setting_supported_versions = setting_supported_versions, + ) + else: + prefixes = [""] if not non_whl_prefix else ["_" + non_whl_prefix] + + versioned = { + ":is_cp{}{}_{}".format(python_version, p, suffix): { + version: ":is_cp{}{}_{}".format(python_version, p, setting) + for version, setting in versions.items() + } + for p in prefixes + for suffix, versions in setting_supported_versions.items() + } + + if suffixes or target_platforms or versioned: + target_platforms = target_platforms or [] + suffixes = suffixes or [_non_versioned_platform(p) for p in target_platforms] + return [ + ":is_cp{}{}_{}".format(python_version, p, s) + for p in prefixes + for s in suffixes + ], versioned + else: + return [":is_cp{}{}".format(python_version, p) for p in prefixes], setting_supported_versions + +def _whl_config_setting_suffixes( + platform_tag, + glibc_versions, + muslc_versions, + osx_versions, + setting_supported_versions): + suffixes = [] + for platform_tag in platform_tag.split("."): + for p in whl_target_platforms(platform_tag): + prefix = p.os + suffix = p.cpu + if "manylinux" in platform_tag: + prefix = "manylinux" + versions = glibc_versions + elif "musllinux" in platform_tag: + prefix = "musllinux" + versions = muslc_versions + elif p.os in ["linux", "windows"]: + versions = [(0, 0)] + elif p.os == "osx": + versions = osx_versions + if "universal2" in platform_tag: + suffix += "_universal2" + else: + fail("Unsupported whl os: {}".format(p.os)) + + default_version_setting = "{}_{}".format(prefix, suffix) + supported_versions = {} + for v in versions: + if v == (0, 0): + suffixes.append(default_version_setting) + elif v >= p.version: + supported_versions[v] = "{}_{}_{}_{}".format( + prefix, + v[0], + v[1], + suffix, + ) + if supported_versions: + setting_supported_versions[default_version_setting] = supported_versions + + return suffixes + +def _non_versioned_platform(p, *, strict = False): + """A small utility function that converts 'cp311_linux_x86_64' to 'linux_x86_64'. + + This is so that we can tighten the code structure later by using strict = True. + """ + has_abi = p.startswith("cp") + if has_abi: + return p.partition("_")[-1] + elif not strict: + return p + else: + fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 98c9d0906f..66968c11e2 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -61,34 +61,53 @@ If the value is missing, then the "default" Python version is being used, which has a "null" version value and will not match version constraints. """ +def _repr_dict(*, value_repr = repr, **kwargs): + return {k: value_repr(v) for k, v in kwargs.items() if v} + +def _repr_config_setting(alias): + if alias.filename: + return render.call( + "whl_config_setting", + **_repr_dict( + filename = alias.filename, + target_platforms = alias.target_platforms, + version = alias.version, + ) + ) + else: + return repr( + alias.config_setting or ("//_config:is_python_" + alias.version), + ) + def _repr_actual(aliases): - if len(aliases) == 1 and not aliases[0].version and not aliases[0].config_setting: - return repr(aliases[0].repo) + if type(aliases) == type(""): + return repr(aliases) + else: + return render.dict(aliases, key_repr = _repr_config_setting) - actual = {} - for alias in aliases: - actual[alias.config_setting or ("//_config:is_python_" + alias.version)] = alias.repo - return render.indent(render.dict(actual)).lstrip() +def _render_common_aliases(*, name, aliases, **kwargs): + pkg_aliases = render.call( + "pkg_aliases", + name = repr(name), + actual = _repr_actual(aliases), + **_repr_dict(**kwargs) + ) + extra_loads = "" + if "whl_config_setting" in pkg_aliases: + extra_loads = """load("@rules_python//python/private/pypi:whl_config_setting.bzl", "whl_config_setting")""" + extra_loads += "\n" -def _render_common_aliases(*, name, aliases, extra_aliases = [], group_name = None): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") - +{extra_loads} package(default_visibility = ["//visibility:public"]) -pkg_aliases( - name = "{name}", - actual = {actual}, - group_name = {group_name}, - extra_aliases = {extra_aliases}, -)""".format( - name = name, - actual = _repr_actual(aliases), - group_name = repr(group_name), - extra_aliases = repr(extra_aliases), +{aliases}""".format( + aliases = pkg_aliases, + extra_loads = extra_loads, ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}): +def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -97,10 +116,11 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases Args: aliases: dict, the keys are normalized distribution names and values are the - whl_alias instances. + whl_config_setting instances. requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + **kwargs: Extra kwargs to pass to the rules. Returns: A dict of file paths and their contents. @@ -130,6 +150,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + **kwargs ).strip() for name, pkg_aliases in aliases.items() } @@ -138,48 +159,11 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files -def whl_alias(*, repo, version = None, config_setting = None, filename = None, target_platforms = None): - """The bzl_packages value used by by the render_pkg_aliases function. - - This contains the minimum amount of information required to generate correct - aliases in a hub repository. - - Args: - repo: str, the repo of where to find the things to be aliased. - version: optional(str), the version of the python toolchain that this - whl alias is for. If not set, then non-version aware aliases will be - constructed. This is mainly used for better error messages when there - is no match found during a select. - config_setting: optional(Label or str), the config setting that we should use. Defaults - to "//_config:is_python_{version}". - filename: optional(str), the distribution filename to derive the config_setting. - target_platforms: optional(list[str]), the list of target_platforms for this - distribution. - - Returns: - a struct with the validated and parsed values. - """ - if not repo: - fail("'repo' must be specified") - - if target_platforms: - for p in target_platforms: - if not p.startswith("cp"): - fail("target_platform should start with 'cp' denoting the python version, got: " + p) - - return struct( - config_setting = config_setting, - filename = filename, - repo = repo, - target_platforms = target_platforms, - version = version, - ) - def render_multiplatform_pkg_aliases(*, aliases, **kwargs): """Render the multi-platform pkg aliases. Args: - aliases: dict[str, list(whl_alias)] A list of aliases that will be + aliases: dict[str, list(whl_config_setting)] A list of aliases that will be transformed from ones having `filename` to ones having `config_setting`. **kwargs: extra arguments passed to render_pkg_aliases. @@ -188,142 +172,45 @@ def render_multiplatform_pkg_aliases(*, aliases, **kwargs): """ flag_versions = get_whl_flag_versions( - aliases = [ + settings = [ a for bunch in aliases.values() for a in bunch ], ) - config_setting_aliases = { - pkg: multiplatform_whl_aliases( - aliases = pkg_aliases, - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), - ) - for pkg, pkg_aliases in aliases.items() - } - contents = render_pkg_aliases( - aliases = config_setting_aliases, + aliases = aliases, + glibc_versions = flag_versions.get("glibc_versions", []), + muslc_versions = flag_versions.get("muslc_versions", []), + osx_versions = flag_versions.get("osx_versions", []), **kwargs ) - contents["_config/BUILD.bazel"] = _render_config_settings(**flag_versions) + contents["_config/BUILD.bazel"] = _render_config_settings( + glibc_versions = flag_versions.get("glibc_versions", []), + muslc_versions = flag_versions.get("muslc_versions", []), + osx_versions = flag_versions.get("osx_versions", []), + python_versions = flag_versions.get("python_versions", []), + target_platforms = flag_versions.get("target_platforms", []), + visibility = ["//:__subpackages__"], + ) return contents -def multiplatform_whl_aliases(*, aliases, **kwargs): - """convert a list of aliases from filename to config_setting ones. - - Args: - aliases: list(whl_alias): The aliases to process. Any aliases that have - the filename set will be converted to a list of aliases, each with - an appropriate config_setting value. - **kwargs: Extra parameters passed to get_filename_config_settings. - - Returns: - A dict with aliases to be used in the hub repo. - """ - - ret = [] - versioned_additions = {} - for alias in aliases: - if not alias.filename and not alias.target_platforms: - ret.append(alias) - continue - - config_settings, all_versioned_settings = get_filename_config_settings( - # TODO @aignas 2024-05-27: pass the parsed whl to reduce the - # number of duplicate operations. - filename = alias.filename or "", - target_platforms = alias.target_platforms, - python_version = alias.version, - # If we have multiple platforms but no wheel filename, lets use different - # config settings. - non_whl_prefix = "sdist" if alias.filename else "", - **kwargs - ) - - for setting in config_settings: - ret.append(whl_alias( - repo = alias.repo, - version = alias.version, - config_setting = "//_config" + setting, - )) - - # Now for the versioned platform config settings, we need to select one - # that best fits the bill and if there are multiple wheels, e.g. - # manylinux_2_17_x86_64 and manylinux_2_28_x86_64, then we need to select - # the former when the glibc is in the range of [2.17, 2.28) and then chose - # the later if it is [2.28, ...). If the 2.28 wheel was not present in - # the hub, then we would need to use 2.17 for all the glibc version - # configurations. - # - # Here we add the version settings to a dict where we key the range of - # versions that the whl spans. If the wheel supports musl and glibc at - # the same time, we do this for each supported platform, hence the - # double dict. - for default_setting, versioned in all_versioned_settings.items(): - versions = sorted(versioned) - min_version = versions[0] - max_version = versions[-1] - - versioned_additions.setdefault(default_setting, {})[(min_version, max_version)] = struct( - repo = alias.repo, - python_version = alias.version, - settings = versioned, - ) - - versioned = {} - for default_setting, candidates in versioned_additions.items(): - # Sort the candidates by the range of versions the span, so that we - # start with the lowest version. - for _, candidate in sorted(candidates.items()): - # Set the default with the first candidate, which gives us the highest - # compatibility. If the users want to use a higher-version than the default - # they can configure the glibc_version flag. - versioned.setdefault(default_setting, whl_alias( - version = candidate.python_version, - config_setting = "//_config" + default_setting, - repo = candidate.repo, - )) - - # We will be overwriting previously added entries, but that is intended. - for _, setting in sorted(candidate.settings.items()): - versioned[setting] = whl_alias( - version = candidate.python_version, - config_setting = "//_config" + setting, - repo = candidate.repo, - ) - - ret.extend(versioned.values()) - return ret - -def _render_config_settings(python_versions = [], target_platforms = [], osx_versions = [], glibc_versions = [], muslc_versions = []): +def _render_config_settings(**kwargs): return """\ load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings") -config_settings( - name = "config_settings", - glibc_versions = {glibc_versions}, - muslc_versions = {muslc_versions}, - osx_versions = {osx_versions}, - python_versions = {python_versions}, - target_platforms = {target_platforms}, - visibility = ["//:__subpackages__"], -)""".format( - glibc_versions = render.indent(render.list(glibc_versions)).lstrip(), - muslc_versions = render.indent(render.list(muslc_versions)).lstrip(), - osx_versions = render.indent(render.list(osx_versions)).lstrip(), - python_versions = render.indent(render.list(python_versions)).lstrip(), - target_platforms = render.indent(render.list(target_platforms)).lstrip(), - ) +{}""".format(render.call( + "config_settings", + name = repr("config_settings"), + **_repr_dict(value_repr = render.list, **kwargs) + )) -def get_whl_flag_versions(aliases): - """Return all of the flag versions that is used by the aliases +def get_whl_flag_versions(settings): + """Return all of the flag versions that is used by the settings Args: - aliases: list[whl_alias] + settings: list[whl_config_setting] Returns: dict, which may have keys: @@ -335,17 +222,17 @@ def get_whl_flag_versions(aliases): muslc_versions = {} osx_versions = {} - for a in aliases: - if not a.version and not a.filename: + for setting in settings: + if not setting.version and not setting.filename: continue - if a.version: - python_versions[a.version] = None + if setting.version: + python_versions[setting.version] = None - if a.filename and a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"): - parsed = parse_whl_name(a.filename) + if setting.filename and setting.filename.endswith(".whl") and not setting.filename.endswith("-any.whl"): + parsed = parse_whl_name(setting.filename) else: - for plat in a.target_platforms or []: + for plat in setting.target_platforms or []: target_platforms[_non_versioned_platform(plat)] = None continue @@ -396,131 +283,3 @@ def _non_versioned_platform(p, *, strict = False): return p else: fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) - -def get_filename_config_settings( - *, - filename, - target_platforms, - python_version, - glibc_versions = None, - muslc_versions = None, - osx_versions = None, - non_whl_prefix = "sdist"): - """Get the filename config settings. - - Args: - filename: the distribution filename (can be a whl or an sdist). - target_platforms: list[str], target platforms in "{abi}_{os}_{cpu}" format. - glibc_versions: list[tuple[int, int]], list of versions. - muslc_versions: list[tuple[int, int]], list of versions. - osx_versions: list[tuple[int, int]], list of versions. - python_version: the python version to generate the config_settings for. - non_whl_prefix: the prefix of the config setting when the whl we don't have - a filename ending with ".whl". - - Returns: - A tuple: - * A list of config settings that are generated by ./pip_config_settings.bzl - * The list of default version settings. - """ - prefixes = [] - suffixes = [] - setting_supported_versions = {} - - if filename.endswith(".whl"): - if (0, 0) in glibc_versions: - fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value") - if (0, 0) in muslc_versions: - fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value") - if (0, 0) in osx_versions: - fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value") - - glibc_versions = sorted(glibc_versions) - muslc_versions = sorted(muslc_versions) - osx_versions = sorted(osx_versions) - - parsed = parse_whl_name(filename) - if parsed.python_tag == "py2.py3": - py = "py" - elif parsed.python_tag.startswith("cp"): - py = "cp3x" - else: - py = "py3" - - if parsed.abi_tag.startswith("cp"): - abi = "cp" - else: - abi = parsed.abi_tag - - if parsed.platform_tag == "any": - prefixes = ["_{}_{}_any".format(py, abi)] - suffixes = [_non_versioned_platform(p) for p in target_platforms or []] - else: - prefixes = ["_{}_{}".format(py, abi)] - suffixes = _whl_config_setting_suffixes( - platform_tag = parsed.platform_tag, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - setting_supported_versions = setting_supported_versions, - ) - else: - prefixes = [""] if not non_whl_prefix else ["_" + non_whl_prefix] - suffixes = [_non_versioned_platform(p) for p in target_platforms or []] - - versioned = { - ":is_cp{}{}_{}".format(python_version, p, suffix): { - version: ":is_cp{}{}_{}".format(python_version, p, setting) - for version, setting in versions.items() - } - for p in prefixes - for suffix, versions in setting_supported_versions.items() - } - - if suffixes or versioned: - return [":is_cp{}{}_{}".format(python_version, p, s) for p in prefixes for s in suffixes], versioned - else: - return [":is_cp{}{}".format(python_version, p) for p in prefixes], setting_supported_versions - -def _whl_config_setting_suffixes( - platform_tag, - glibc_versions, - muslc_versions, - osx_versions, - setting_supported_versions): - suffixes = [] - for platform_tag in platform_tag.split("."): - for p in whl_target_platforms(platform_tag): - prefix = p.os - suffix = p.cpu - if "manylinux" in platform_tag: - prefix = "manylinux" - versions = glibc_versions - elif "musllinux" in platform_tag: - prefix = "musllinux" - versions = muslc_versions - elif p.os in ["linux", "windows"]: - versions = [(0, 0)] - elif p.os == "osx": - versions = osx_versions - if "universal2" in platform_tag: - suffix += "_universal2" - else: - fail("Unsupported whl os: {}".format(p.os)) - - default_version_setting = "{}_{}".format(prefix, suffix) - supported_versions = {} - for v in versions: - if v == (0, 0): - suffixes.append(default_version_setting) - elif v >= p.version: - supported_versions[v] = "{}_{}_{}_{}".format( - prefix, - v[0], - v[1], - suffix, - ) - if supported_versions: - setting_supported_versions[default_version_setting] = supported_versions - - return suffixes diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl new file mode 100644 index 0000000000..e46c7d37d7 --- /dev/null +++ b/python/private/pypi/whl_config_setting.bzl @@ -0,0 +1,50 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"A small function to create an alias for a whl distribution" + +def whl_config_setting(*, repo = None, version = None, config_setting = None, filename = None, target_platforms = None): + """The bzl_packages value used by by the render_pkg_aliases function. + + This contains the minimum amount of information required to generate correct + aliases in a hub repository. + + Args: + repo: str, the repo of where to find the things to be aliased. + version: optional(str), the version of the python toolchain that this + whl alias is for. If not set, then non-version aware aliases will be + constructed. This is mainly used for better error messages when there + is no match found during a select. + config_setting: optional(Label or str), the config setting that we should use. Defaults + to "//_config:is_python_{version}". + filename: optional(str), the distribution filename to derive the config_setting. + target_platforms: optional(list[str]), the list of target_platforms for this + distribution. + + Returns: + a struct with the validated and parsed values. + """ + if target_platforms: + for p in target_platforms: + if not p.startswith("cp"): + fail("target_platform should start with 'cp' denoting the python version, got: " + p) + + return struct( + config_setting = config_setting, + filename = filename, + repo = repo, + # Make the struct hashable + target_platforms = tuple(target_platforms) if target_platforms else None, + version = version, + ) diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl index 38f2b0e404..a64b5d6243 100644 --- a/python/private/text_util.bzl +++ b/python/private/text_util.bzl @@ -124,6 +124,21 @@ def _render_tuple(items, *, value_repr = repr): ")", ]) +def _render_kwargs(items, *, value_repr = repr): + if not items: + return "" + + return "\n".join([ + "{} = {},".format(k, value_repr(v)).lstrip() + for k, v in items.items() + ]) + +def _render_call(fn_name, **kwargs): + if not kwargs: + return fn_name + "()" + + return "{}(\n{}\n)".format(fn_name, _indent(_render_kwargs(kwargs, value_repr = lambda x: x))) + def _toolchain_prefix(index, name, pad_length): """Prefixes the given name with the index, padded with zeros to ensure lexicographic sorting. @@ -141,8 +156,10 @@ def _left_pad_zero(index, length): render = struct( alias = _render_alias, dict = _render_dict, + call = _render_call, hanging_indent = _hanging_indent, indent = _indent, + kwargs = _render_kwargs, left_pad_zero = _left_pad_zero, list = _render_list, select = _render_select, diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 39670cd71f..7dfd8762a7 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -17,7 +17,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disable=bzl-visibility -load("//python/private/pypi:render_pkg_aliases.bzl", "whl_alias") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility _tests = [] @@ -158,12 +158,13 @@ def _test_simple(env): pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { - "simple": [ - whl_alias( - repo = "pypi_315_simple", - version = "3.15", - ), - ], + "simple": { + "pypi_315_simple": [ + whl_config_setting( + version = "3.15", + ), + ], + }, }}) pypi.whl_libraries().contains_exactly({ "pypi_315_simple": { @@ -206,23 +207,25 @@ def _test_simple_multiple_requirements(env): pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { - "simple": [ - whl_alias( - repo = "pypi_315_simple_windows_x86_64", - target_platforms = [ - "cp315_windows_x86_64", - ], - version = "3.15", - ), - whl_alias( - repo = "pypi_315_simple_osx_aarch64_osx_x86_64", - target_platforms = [ - "cp315_osx_aarch64", - "cp315_osx_x86_64", - ], - version = "3.15", - ), - ], + "simple": { + "pypi_315_simple_osx_aarch64_osx_x86_64": [ + whl_config_setting( + target_platforms = [ + "cp315_osx_aarch64", + "cp315_osx_x86_64", + ], + version = "3.15", + ), + ], + "pypi_315_simple_windows_x86_64": [ + whl_config_setting( + target_platforms = [ + "cp315_windows_x86_64", + ], + version = "3.15", + ), + ], + }, }}) pypi.whl_libraries().contains_exactly({ "pypi_315_simple_osx_aarch64_osx_x86_64": { @@ -289,24 +292,25 @@ simple==0.0.3 --hash=sha256:deadbaaf pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { - "extra": [ - whl_alias( - repo = "pypi_315_extra", - version = "3.15", - ), - ], - "simple": [ - whl_alias( - repo = "pypi_315_simple_linux_x86_64", - target_platforms = ["cp315_linux_x86_64"], - version = "3.15", - ), - whl_alias( - repo = "pypi_315_simple_osx_aarch64", - target_platforms = ["cp315_osx_aarch64"], - version = "3.15", - ), - ], + "extra": { + "pypi_315_extra": [ + whl_config_setting(version = "3.15"), + ], + }, + "simple": { + "pypi_315_simple_linux_x86_64": [ + whl_config_setting( + target_platforms = ["cp315_linux_x86_64"], + version = "3.15", + ), + ], + "pypi_315_simple_osx_aarch64": [ + whl_config_setting( + target_platforms = ["cp315_osx_aarch64"], + version = "3.15", + ), + ], + }, }}) pypi.whl_libraries().contains_exactly({ "pypi_315_extra": { @@ -404,24 +408,23 @@ some_pkg==0.0.1 pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({ "pypi": { - "simple": [ - whl_alias( - filename = "simple-0.0.1-py3-none-any.whl", - repo = "pypi_315_simple_py3_none_any_deadb00f", - version = "3.15", - ), - whl_alias( - filename = "simple-0.0.1.tar.gz", - repo = "pypi_315_simple_sdist_deadbeef", - version = "3.15", - ), - ], - "some_pkg": [ - whl_alias( - repo = "pypi_315_some_pkg", - version = "3.15", - ), - ], + "simple": { + "pypi_315_simple_py3_none_any_deadb00f": [ + whl_config_setting( + filename = "simple-0.0.1-py3-none-any.whl", + version = "3.15", + ), + ], + "pypi_315_simple_sdist_deadbeef": [ + whl_config_setting( + filename = "simple-0.0.1.tar.gz", + version = "3.15", + ), + ], + }, + "some_pkg": { + "pypi_315_some_pkg": [whl_config_setting(version = "3.15")], + }, }, }) pypi.whl_libraries().contains_exactly({ diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index cf801ae643..0fa66d05eb 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -15,80 +15,54 @@ """pkg_aliases tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility load( "//python/private/pypi:pkg_aliases.bzl", + "multiplatform_whl_aliases", "pkg_aliases", ) # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility _tests = [] def _test_legacy_aliases(env): - actual = [] + got = {} pkg_aliases( name = "foo", actual = "repo", native = struct( - alias = lambda **kwargs: actual.append(kwargs), + alias = lambda name, actual: got.update({name: actual}), ), extra_aliases = ["my_special"], ) # buildifier: disable=unsorted-dict-items - want = [ - { - "name": "foo", - "actual": ":pkg", - }, - { - "name": "pkg", - "actual": "@repo//:pkg", - }, - { - "name": "whl", - "actual": "@repo//:whl", - }, - { - "name": "data", - "actual": "@repo//:data", - }, - { - "name": "dist_info", - "actual": "@repo//:dist_info", - }, - { - "name": "my_special", - "actual": "@repo//:my_special", - }, - ] + want = { + "foo": ":pkg", + "pkg": "@repo//:pkg", + "whl": "@repo//:whl", + "data": "@repo//:data", + "dist_info": "@repo//:dist_info", + "my_special": "@repo//:my_special", + } - env.expect.that_collection(actual).contains_exactly(want) + env.expect.that_dict(got).contains_exactly(want) _tests.append(_test_legacy_aliases) def _test_config_setting_aliases(env): # Use this function as it is used in pip_repository - actual = [] + got = {} actual_no_match_error = [] def mock_select(value, no_match_error = None): actual_no_match_error.append(no_match_error) - env.expect.that_str(no_match_error).equals("""\ -No matching wheel for current configuration's Python version. - -The current build configuration's Python version doesn't match any of the Python -wheels available for this wheel. This wheel supports the following Python + env.expect.that_str(no_match_error).contains("""\ configuration settings: //:my_config_setting -To determine the current configuration's Python version, run: - `bazel config ` (shown further below) -and look for - rules_python//python/config_settings:python_version - -If the value is missing, then the "default" Python version is being used, -which has a "null" version value and will not match version constraints. """) - return struct(value = value, no_match_error = no_match_error != None) + return value pkg_aliases( name = "bar_baz", @@ -97,66 +71,123 @@ which has a "null" version value and will not match version constraints. }, extra_aliases = ["my_special"], native = struct( - alias = lambda **kwargs: actual.append(kwargs), + alias = lambda name, actual: got.update({name: actual}), ), select = mock_select, ) # buildifier: disable=unsorted-dict-items - want = [ - { - "name": "bar_baz", - "actual": ":pkg", - }, - { - "name": "pkg", - "actual": struct( - value = { - "//:my_config_setting": "@bar_baz_repo//:pkg", - }, - no_match_error = True, - ), + want = { + "pkg": { + "//:my_config_setting": "@bar_baz_repo//:pkg", }, - { - "name": "whl", - "actual": struct( - value = { - "//:my_config_setting": "@bar_baz_repo//:whl", - }, - no_match_error = True, - ), + } + env.expect.that_dict(got).contains_at_least(want) + +_tests.append(_test_config_setting_aliases) + +def _test_config_setting_aliases_many(env): + # Use this function as it is used in pip_repository + got = {} + actual_no_match_error = [] + + def mock_select(value, no_match_error = None): + actual_no_match_error.append(no_match_error) + env.expect.that_str(no_match_error).contains("""\ +configuration settings: + //:another_config_setting + //:my_config_setting + //:third_config_setting +""") + return value + + pkg_aliases( + name = "bar_baz", + actual = { + ( + "//:my_config_setting", + "//:another_config_setting", + ): "bar_baz_repo", + "//:third_config_setting": "foo_repo", }, - { - "name": "data", - "actual": struct( - value = { - "//:my_config_setting": "@bar_baz_repo//:data", - }, - no_match_error = True, - ), + extra_aliases = ["my_special"], + native = struct( + alias = lambda name, actual: got.update({name: actual}), + ), + select = mock_select, + ) + + # buildifier: disable=unsorted-dict-items + want = { + "my_special": { + ( + "//:my_config_setting", + "//:another_config_setting", + ): "@bar_baz_repo//:my_special", + "//:third_config_setting": "@foo_repo//:my_special", }, - { - "name": "dist_info", - "actual": struct( - value = { - "//:my_config_setting": "@bar_baz_repo//:dist_info", - }, - no_match_error = True, - ), + } + env.expect.that_dict(got).contains_at_least(want) + +_tests.append(_test_config_setting_aliases_many) + +def _test_multiplatform_whl_aliases(env): + # Use this function as it is used in pip_repository + got = {} + actual_no_match_error = [] + + def mock_select(value, no_match_error = None): + actual_no_match_error.append(no_match_error) + env.expect.that_str(no_match_error).contains("""\ +configuration settings: + //:my_config_setting + //_config:is_cp3.9_linux_x86_64 + //_config:is_cp3.9_py3_none_any + //_config:is_cp3.9_py3_none_any_linux_x86_64 + +""") + return value + + pkg_aliases( + name = "bar_baz", + actual = { + whl_config_setting( + filename = "foo-0.0.0-py3-none-any.whl", + version = "3.9", + ): "filename_repo", + whl_config_setting( + filename = "foo-0.0.0-py3-none-any.whl", + version = "3.9", + target_platforms = ["cp39_linux_x86_64"], + ): "filename_repo_for_platform", + whl_config_setting( + version = "3.9", + target_platforms = ["cp39_linux_x86_64"], + ): "bzlmod_repo_for_a_particular_platform", + "//:my_config_setting": "bzlmod_repo", }, - { - "name": "my_special", - "actual": struct( - value = { - "//:my_config_setting": "@bar_baz_repo//:my_special", - }, - no_match_error = True, - ), + extra_aliases = [], + native = struct( + alias = lambda name, actual: got.update({name: actual}), + ), + select = mock_select, + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + ) + + # buildifier: disable=unsorted-dict-items + want = { + "pkg": { + "//:my_config_setting": "@bzlmod_repo//:pkg", + "//_config:is_cp3.9_linux_x86_64": "@bzlmod_repo_for_a_particular_platform//:pkg", + "//_config:is_cp3.9_py3_none_any": "@filename_repo//:pkg", + "//_config:is_cp3.9_py3_none_any_linux_x86_64": "@filename_repo_for_platform//:pkg", }, - ] - env.expect.that_collection(actual).contains_exactly(want) + } + env.expect.that_dict(got).contains_at_least(want) -_tests.append(_test_config_setting_aliases) +_tests.append(_test_multiplatform_whl_aliases) def _test_group_aliases(env): # Use this function as it is used in pip_repository @@ -208,6 +239,232 @@ def _test_group_aliases(env): _tests.append(_test_group_aliases) +def _test_multiplatform_whl_aliases_empty(env): + # Check that we still work with an empty requirements.txt + got = multiplatform_whl_aliases(aliases = {}) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_multiplatform_whl_aliases_empty) + +def _test_multiplatform_whl_aliases_nofilename(env): + aliases = { + "//:label": "foo", + } + got = multiplatform_whl_aliases(aliases = aliases) + env.expect.that_dict(got).contains_exactly(aliases) + +_tests.append(_test_multiplatform_whl_aliases_nofilename) + +def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): + aliases = { + whl_config_setting( + config_setting = "//:ignored", + version = "3.1", + target_platforms = [ + "cp31_linux_x86_64", + "cp31_linux_aarch64", + ], + ): "foo", + } + + got = multiplatform_whl_aliases(aliases = aliases) + + want = { + "//_config:is_cp3.1_linux_aarch64": "foo", + "//_config:is_cp3.1_linux_x86_64": "foo", + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms) + +def _test_multiplatform_whl_aliases_filename(env): + aliases = { + whl_config_setting( + filename = "foo-0.0.3-py3-none-any.whl", + version = "3.2", + ): "foo-py3-0.0.3", + whl_config_setting( + filename = "foo-0.0.1-py3-none-any.whl", + version = "3.1", + ): "foo-py3-0.0.1", + whl_config_setting( + filename = "foo-0.0.2-py3-none-any.whl", + version = "3.1", + target_platforms = [ + "cp31_linux_x86_64", + "cp31_linux_aarch64", + ], + ): "foo-0.0.2", + } + got = multiplatform_whl_aliases( + aliases = aliases, + glibc_versions = [], + muslc_versions = [], + osx_versions = [], + ) + want = { + "//_config:is_cp3.1_py3_none_any": "foo-py3-0.0.1", + "//_config:is_cp3.1_py3_none_any_linux_aarch64": "foo-0.0.2", + "//_config:is_cp3.1_py3_none_any_linux_x86_64": "foo-0.0.2", + "//_config:is_cp3.2_py3_none_any": "foo-py3-0.0.3", + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_filename) + +def _test_multiplatform_whl_aliases_filename_versioned(env): + aliases = { + whl_config_setting( + filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", + version = "3.1", + ): "glibc-2.17", + whl_config_setting( + filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", + version = "3.1", + ): "glibc-2.18", + whl_config_setting( + filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", + version = "3.1", + ): "musl-1.1", + } + got = multiplatform_whl_aliases( + aliases = aliases, + glibc_versions = [(2, 17), (2, 18)], + muslc_versions = [(1, 1), (1, 2)], + osx_versions = [], + ) + want = { + # This could just work with: + # select({ + # "//_config:is_gt_eq_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", + # "//conditions:default": "//_config:is_gt_eq_2.18", + # }): "glibc-2.18", + # select({ + # "//_config:is_range_2.17_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", + # "//_config:is_glibc_default": "//_config:is_cp3.1_py3_none_manylinux_x86_64", + # "//conditions:default": "//_config:is_glibc_default", + # }): "glibc-2.17", + # ( + # "//_config:is_gt_musl_1.1": "musl-1.1", + # "//_config:is_musl_default": "musl-1.1", + # ): "musl-1.1", + # + # For this to fully work we need to have the pypi:config_settings.bzl to generate the + # extra targets that use the FeatureFlagInfo and this to generate extra aliases for the + # config settings. + "//_config:is_cp3.1_py3_none_manylinux_2_17_x86_64": "glibc-2.17", + "//_config:is_cp3.1_py3_none_manylinux_2_18_x86_64": "glibc-2.18", + "//_config:is_cp3.1_py3_none_manylinux_x86_64": "glibc-2.17", + "//_config:is_cp3.1_py3_none_musllinux_1_1_x86_64": "musl-1.1", + "//_config:is_cp3.1_py3_none_musllinux_1_2_x86_64": "musl-1.1", + "//_config:is_cp3.1_py3_none_musllinux_x86_64": "musl-1.1", + } + env.expect.that_dict(got).contains_exactly(want) + +_tests.append(_test_multiplatform_whl_aliases_filename_versioned) + +def _mock_alias(container): + return lambda name, **kwargs: container.append(name) + +def _mock_config_setting(container): + def _inner(name, flag_values = None, constraint_values = None, **_): + if flag_values or constraint_values: + container.append(name) + return + + fail("At least one of 'flag_values' or 'constraint_values' needs to be set") + + return _inner + +def _test_config_settings_exist_legacy(env): + aliases = { + whl_config_setting( + version = "3.11", + target_platforms = [ + "cp311_linux_aarch64", + "cp311_linux_x86_64", + ], + ): "repo", + } + available_config_settings = [] + config_settings( + python_versions = ["3.11"], + native = struct( + alias = _mock_alias(available_config_settings), + config_setting = _mock_config_setting(available_config_settings), + ), + target_platforms = [ + "linux_aarch64", + "linux_x86_64", + ], + ) + + got_aliases = multiplatform_whl_aliases( + aliases = aliases, + ) + got = [a.partition(":")[-1] for a in got_aliases] + + env.expect.that_collection(available_config_settings).contains_at_least(got) + +_tests.append(_test_config_settings_exist_legacy) + +def _test_config_settings_exist(env): + for py_tag in ["py2.py3", "py3", "py311", "cp311"]: + if py_tag == "py2.py3": + abis = ["none"] + elif py_tag.startswith("py"): + abis = ["none", "abi3"] + else: + abis = ["none", "abi3", "cp311"] + + for abi_tag in abis: + for platform_tag, kwargs in { + "any": {}, + "macosx_11_0_arm64": { + "osx_versions": [(11, 0)], + "target_platforms": ["osx_aarch64"], + }, + "manylinux_2_17_x86_64": { + "glibc_versions": [(2, 17), (2, 18)], + "target_platforms": ["linux_x86_64"], + }, + "manylinux_2_18_x86_64": { + "glibc_versions": [(2, 17), (2, 18)], + "target_platforms": ["linux_x86_64"], + }, + "musllinux_1_1_aarch64": { + "muslc_versions": [(1, 2), (1, 1), (1, 0)], + "target_platforms": ["linux_aarch64"], + }, + }.items(): + aliases = { + whl_config_setting( + filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), + version = "3.11", + ): "repo", + } + available_config_settings = [] + config_settings( + python_versions = ["3.11"], + native = struct( + alias = _mock_alias(available_config_settings), + config_setting = _mock_config_setting(available_config_settings), + ), + **kwargs + ) + + got_aliases = multiplatform_whl_aliases( + aliases = aliases, + glibc_versions = kwargs.get("glibc_versions", []), + muslc_versions = kwargs.get("muslc_versions", []), + osx_versions = kwargs.get("osx_versions", []), + ) + got = [a.partition(":")[-1] for a in got_aliases] + + env.expect.that_collection(available_config_settings).contains_at_least(got) + +_tests.append(_test_config_settings_exist) + def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 4741df04b4..0ba642eca9 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -15,16 +15,17 @@ """render_pkg_aliases tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:config_settings.bzl", "config_settings") # buildifier: disable=bzl-visibility load( - "//python/private/pypi:render_pkg_aliases.bzl", + "//python/private/pypi:pkg_aliases.bzl", "get_filename_config_settings", +) # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:render_pkg_aliases.bzl", "get_whl_flag_versions", - "multiplatform_whl_aliases", "render_multiplatform_pkg_aliases", "render_pkg_aliases", - "whl_alias", ) # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility _tests = [] @@ -42,9 +43,7 @@ _tests.append(_test_empty) def _test_legacy_aliases(env): actual = render_pkg_aliases( aliases = { - "foo": [ - whl_alias(repo = "pypi_foo"), - ], + "foo": "pypi_foo", }, ) @@ -57,8 +56,6 @@ package(default_visibility = ["//visibility:public"]) pkg_aliases( name = "foo", actual = "pypi_foo", - group_name = None, - extra_aliases = [], )""" env.expect.that_dict(actual).contains_exactly({want_key: want_content}) @@ -69,15 +66,24 @@ def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( aliases = { - "bar-baz": [ - whl_alias(version = "3.2", repo = "pypi_32_bar_baz", config_setting = "//:my_config_setting"), - ], + "bar-baz": { + whl_config_setting( + version = "3.2", + config_setting = "//:my_config_setting", + ): "pypi_32_bar_baz", + whl_config_setting( + version = "3.2", + filename = "foo-0.0.0-py3-none-any.whl", + ): "filename_repo", + }, }, + extra_hub_aliases = {"bar_baz": ["foo"]}, ) want_key = "bar_baz/BUILD.bazel" want_content = """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") +load("@rules_python//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") package(default_visibility = ["//visibility:public"]) @@ -85,9 +91,12 @@ pkg_aliases( name = "bar_baz", actual = { "//:my_config_setting": "pypi_32_bar_baz", + whl_config_setting( + filename = "foo-0.0.0-py3-none-any.whl", + version = "3.2", + ): "filename_repo", }, - group_name = None, - extra_aliases = [], + extra_aliases = ["foo"], )""" env.expect.that_str(actual.pop("_config/BUILD.bazel")).equals( @@ -96,11 +105,7 @@ load("@rules_python//python/private/pypi:config_settings.bzl", "config_settings" config_settings( name = "config_settings", - glibc_versions = [], - muslc_versions = [], - osx_versions = [], python_versions = ["3.2"], - target_platforms = [], visibility = ["//:__subpackages__"], )""", ) @@ -112,14 +117,14 @@ _tests.append(_test_bzlmod_aliases) def _test_aliases_are_created_for_all_wheels(env): actual = render_pkg_aliases( aliases = { - "bar": [ - whl_alias(version = "3.1", repo = "pypi_31_bar"), - whl_alias(version = "3.2", repo = "pypi_32_bar"), - ], - "foo": [ - whl_alias(version = "3.1", repo = "pypi_32_foo"), - whl_alias(version = "3.2", repo = "pypi_31_foo"), - ], + "bar": { + whl_config_setting(version = "3.1"): "pypi_31_bar", + whl_config_setting(version = "3.2"): "pypi_32_bar", + }, + "foo": { + whl_config_setting(version = "3.1"): "pypi_32_foo", + whl_config_setting(version = "3.2"): "pypi_31_foo", + }, }, ) @@ -135,18 +140,18 @@ _tests.append(_test_aliases_are_created_for_all_wheels) def _test_aliases_with_groups(env): actual = render_pkg_aliases( aliases = { - "bar": [ - whl_alias(version = "3.1", repo = "pypi_31_bar"), - whl_alias(version = "3.2", repo = "pypi_32_bar"), - ], - "baz": [ - whl_alias(version = "3.1", repo = "pypi_31_baz"), - whl_alias(version = "3.2", repo = "pypi_32_baz"), - ], - "foo": [ - whl_alias(version = "3.1", repo = "pypi_32_foo"), - whl_alias(version = "3.2", repo = "pypi_31_foo"), - ], + "bar": { + whl_config_setting(version = "3.1"): "pypi_31_bar", + whl_config_setting(version = "3.2"): "pypi_32_bar", + }, + "baz": { + whl_config_setting(version = "3.1"): "pypi_31_baz", + whl_config_setting(version = "3.2"): "pypi_32_baz", + }, + "foo": { + whl_config_setting(version = "3.1"): "pypi_32_foo", + whl_config_setting(version = "3.2"): "pypi_31_foo", + }, }, requirement_cycles = { "group": ["bar", "baz"], @@ -175,7 +180,7 @@ _tests.append(_test_aliases_with_groups) def _test_empty_flag_versions(env): got = get_whl_flag_versions( - aliases = [], + settings = [], ) want = {} env.expect.that_dict(got).contains_exactly(want) @@ -184,10 +189,10 @@ _tests.append(_test_empty_flag_versions) def _test_get_python_versions(env): got = get_whl_flag_versions( - aliases = [ - whl_alias(repo = "foo", version = "3.3"), - whl_alias(repo = "foo", version = "3.2"), - ], + settings = { + whl_config_setting(version = "3.3"): "foo", + whl_config_setting(version = "3.2"): "foo", + }, ) want = { "python_versions": ["3.2", "3.3"], @@ -198,9 +203,9 @@ _tests.append(_test_get_python_versions) def _test_get_python_versions_with_target_platforms(env): got = get_whl_flag_versions( - aliases = [ - whl_alias(repo = "foo", version = "3.3", target_platforms = ["cp33_linux_x86_64"]), - whl_alias(repo = "foo", version = "3.2", target_platforms = ["cp32_linux_x86_64", "cp32_osx_aarch64"]), + settings = [ + whl_config_setting(repo = "foo", version = "3.3", target_platforms = ["cp33_linux_x86_64"]), + whl_config_setting(repo = "foo", version = "3.2", target_platforms = ["cp32_linux_x86_64", "cp32_osx_aarch64"]), ], ) want = { @@ -216,8 +221,8 @@ _tests.append(_test_get_python_versions_with_target_platforms) def _test_get_python_versions_from_filenames(env): got = get_whl_flag_versions( - aliases = [ - whl_alias( + settings = [ + whl_config_setting( repo = "foo", version = "3.3", filename = "foo-0.0.0-py3-none-" + plat + ".whl", @@ -254,8 +259,8 @@ _tests.append(_test_get_python_versions_from_filenames) def _test_get_flag_versions_from_alias_target_platforms(env): got = get_whl_flag_versions( - aliases = [ - whl_alias( + settings = [ + whl_config_setting( repo = "foo", version = "3.3", filename = "foo-0.0.0-py3-none-" + plat + ".whl", @@ -264,7 +269,7 @@ def _test_get_flag_versions_from_alias_target_platforms(env): "windows_x86_64", ] ] + [ - whl_alias( + whl_config_setting( repo = "foo", version = "3.3", filename = "foo-0.0.0-py3-none-any.whl", @@ -467,227 +472,6 @@ def _test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64(env): _tests.append(_test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64) -def _test_multiplatform_whl_aliases_empty(env): - # Check that we still work with an empty requirements.txt - got = multiplatform_whl_aliases(aliases = []) - env.expect.that_collection(got).contains_exactly([]) - -_tests.append(_test_multiplatform_whl_aliases_empty) - -def _test_multiplatform_whl_aliases_nofilename(env): - aliases = [ - whl_alias( - repo = "foo", - config_setting = "//:label", - version = "3.1", - ), - ] - got = multiplatform_whl_aliases(aliases = aliases) - env.expect.that_collection(got).contains_exactly(aliases) - -_tests.append(_test_multiplatform_whl_aliases_nofilename) - -def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): - aliases = [ - whl_alias( - repo = "foo", - config_setting = "//:ignored", - version = "3.1", - target_platforms = [ - "cp31_linux_x86_64", - "cp31_linux_aarch64", - ], - ), - ] - - got = multiplatform_whl_aliases(aliases = aliases) - - want = [ - whl_alias(config_setting = "//_config:is_cp3.1_linux_x86_64", repo = "foo", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_linux_aarch64", repo = "foo", version = "3.1"), - ] - env.expect.that_collection(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms) - -def _test_multiplatform_whl_aliases_filename(env): - aliases = [ - whl_alias( - repo = "foo-py3-0.0.3", - filename = "foo-0.0.3-py3-none-any.whl", - version = "3.2", - ), - whl_alias( - repo = "foo-py3-0.0.1", - filename = "foo-0.0.1-py3-none-any.whl", - version = "3.1", - ), - whl_alias( - repo = "foo-0.0.2", - filename = "foo-0.0.2-py3-none-any.whl", - version = "3.1", - target_platforms = [ - "cp31_linux_x86_64", - "cp31_linux_aarch64", - ], - ), - ] - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], - ) - want = [ - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any", repo = "foo-py3-0.0.1", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any_linux_aarch64", repo = "foo-0.0.2", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_any_linux_x86_64", repo = "foo-0.0.2", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.2_py3_none_any", repo = "foo-py3-0.0.3", version = "3.2"), - ] - env.expect.that_collection(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename) - -def _test_multiplatform_whl_aliases_filename_versioned(env): - aliases = [ - whl_alias( - repo = "glibc-2.17", - filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", - version = "3.1", - ), - whl_alias( - repo = "glibc-2.18", - filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", - version = "3.1", - ), - whl_alias( - repo = "musl", - filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", - version = "3.1", - ), - ] - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [(2, 17), (2, 18)], - muslc_versions = [(1, 1), (1, 2)], - osx_versions = [], - ) - want = [ - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_2_17_x86_64", repo = "glibc-2.17", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_2_18_x86_64", repo = "glibc-2.18", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_manylinux_x86_64", repo = "glibc-2.17", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_1_1_x86_64", repo = "musl", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_1_2_x86_64", repo = "musl", version = "3.1"), - whl_alias(config_setting = "//_config:is_cp3.1_py3_none_musllinux_x86_64", repo = "musl", version = "3.1"), - ] - env.expect.that_collection(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename_versioned) - -def _mock_alias(container): - return lambda name, **kwargs: container.append(name) - -def _mock_config_setting(container): - def _inner(name, flag_values = None, constraint_values = None, **_): - if flag_values or constraint_values: - container.append(name) - return - - fail("At least one of 'flag_values' or 'constraint_values' needs to be set") - - return _inner - -def _test_config_settings_exist_legacy(env): - aliases = [ - whl_alias( - repo = "repo", - version = "3.11", - target_platforms = [ - "cp311_linux_aarch64", - "cp311_linux_x86_64", - ], - ), - ] - available_config_settings = [] - config_settings( - python_versions = ["3.11"], - native = struct( - alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting(available_config_settings), - ), - target_platforms = [ - "linux_aarch64", - "linux_x86_64", - ], - ) - - got_aliases = multiplatform_whl_aliases( - aliases = aliases, - ) - got = [a.config_setting.partition(":")[-1] for a in got_aliases] - - env.expect.that_collection(available_config_settings).contains_at_least(got) - -_tests.append(_test_config_settings_exist_legacy) - -def _test_config_settings_exist(env): - for py_tag in ["py2.py3", "py3", "py311", "cp311"]: - if py_tag == "py2.py3": - abis = ["none"] - elif py_tag.startswith("py"): - abis = ["none", "abi3"] - else: - abis = ["none", "abi3", "cp311"] - - for abi_tag in abis: - for platform_tag, kwargs in { - "any": {}, - "macosx_11_0_arm64": { - "osx_versions": [(11, 0)], - "target_platforms": ["osx_aarch64"], - }, - "manylinux_2_17_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "target_platforms": ["linux_x86_64"], - }, - "manylinux_2_18_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "target_platforms": ["linux_x86_64"], - }, - "musllinux_1_1_aarch64": { - "muslc_versions": [(1, 2), (1, 1), (1, 0)], - "target_platforms": ["linux_aarch64"], - }, - }.items(): - aliases = [ - whl_alias( - repo = "repo", - filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), - version = "3.11", - ), - ] - available_config_settings = [] - config_settings( - python_versions = ["3.11"], - native = struct( - alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting(available_config_settings), - ), - **kwargs - ) - - got_aliases = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = kwargs.get("glibc_versions", []), - muslc_versions = kwargs.get("muslc_versions", []), - osx_versions = kwargs.get("osx_versions", []), - ) - got = [a.config_setting.partition(":")[-1] for a in got_aliases] - - env.expect.that_collection(available_config_settings).contains_at_least(got) - -_tests.append(_test_config_settings_exist) - def render_pkg_aliases_test_suite(name): """Create the test suite.