From 18eed9191dbcb84cd6099a97d5ad778ac561cd4d Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Sun, 10 Sep 2023 17:15:37 +0530 Subject: [PATCH] Introduce short package names for publish docs (#34253) * Introduce short package names for publish docs * inlcuding docs * Refactoring to a general utils file --- BREEZE.rst | 14 ++ .../commands/release_management_commands.py | 15 +- .../airflow_breeze/params/doc_build_params.py | 16 +- .../src/airflow_breeze/utils/general_utils.py | 32 +++ .../utils/publish_docs_helpers.py | 11 +- images/breeze/output-commands-hash.txt | 4 +- images/breeze/output_release-management.svg | 58 +++--- ...output_release-management_publish-docs.svg | 184 +++++------------- 8 files changed, 142 insertions(+), 192 deletions(-) create mode 100644 dev/breeze/src/airflow_breeze/utils/general_utils.py diff --git a/BREEZE.rst b/BREEZE.rst index f791ee73a5444..4ee2b5b5e826d 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2080,6 +2080,9 @@ while publishing the documentation. breeze release-management publish-docs --airflow-site-directory +You can also use shorthand names as arguments instead of using the full names +for airflow providers. To find the short hand names, follow the instructions in :ref:`generating_short_form_names`. + The flag ``--airflow-site-directory`` takes the path of the cloned ``airflow-site``. The command will not proceed if this is an invalid path. @@ -2093,6 +2096,17 @@ These are all available flags of ``release-management publish-docs`` command: :width: 100% :alt: Breeze Publish documentation +.. _generating_short_form_names: + +Generating short form names for Providers +----------------------------------------- + +Skip the ``apache-airflow-providers-`` from the usual provider full names. +Now with the remaining part, replace every ``dash("-")`` with a ``dot(".")``. + +Example: +If the provider name is ``apache-airflow-providers-cncf-kubernetes``, it will be ``cncf.kubernetes``. + Adding back referencing HTML for the documentation """""""""""""""""""""""""""""""""""""""""""""""""" diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 5558e78b01c2e..daa04f04483a1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -53,6 +53,7 @@ from airflow_breeze.utils.common_options import ( argument_packages, argument_packages_plus_all_providers, + argument_packages_plus_all_providers_for_shorthand, option_airflow_constraints_mode_ci, option_airflow_constraints_mode_update, option_airflow_constraints_reference, @@ -82,7 +83,7 @@ ) from airflow_breeze.utils.confirm import Answer, user_confirm from airflow_breeze.utils.console import Output, get_console -from airflow_breeze.utils.custom_param_types import BetterChoice, NotVerifiedBetterChoice +from airflow_breeze.utils.custom_param_types import BetterChoice from airflow_breeze.utils.docker_command_utils import ( check_remote_ghcr_io_commands, get_env_variables_for_docker_commands, @@ -837,10 +838,13 @@ def run_publish_docs_in_parallel( ) @click.option("-s", "--override-versioned", help="Overrides versioned directories.", is_flag=True) @option_airflow_site_directory +@argument_packages_plus_all_providers_for_shorthand @click.option( "--package-filter", - help="List of packages to consider.", - type=NotVerifiedBetterChoice(get_available_documentation_packages()), + help="List of packages to consider. You can use the full names like apache-airflow-providers-, " + "the short hand names or the glob pattern matching the full package name. " + "The list of short hand names can be found in --help output", + type=str, multiple=True, ) @option_run_in_parallel @@ -853,6 +857,7 @@ def run_publish_docs_in_parallel( def publish_docs( override_versioned: bool, airflow_site_directory: str, + packages_plus_all_providers: tuple[str], package_filter: tuple[str], run_in_parallel: bool, parallelism: int, @@ -869,8 +874,10 @@ def publish_docs( available_packages = get_available_packages() package_filters = package_filter + current_packages = process_package_filters( + available_packages, package_filters, packages_plus_all_providers + ) - current_packages = process_package_filters(available_packages, package_filters) print(f"Publishing docs for {len(current_packages)} package(s)") for pkg in current_packages: print(f" - {pkg}") diff --git a/dev/breeze/src/airflow_breeze/params/doc_build_params.py b/dev/breeze/src/airflow_breeze/params/doc_build_params.py index bd02ae7903541..32da20920428e 100644 --- a/dev/breeze/src/airflow_breeze/params/doc_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/doc_build_params.py @@ -20,21 +20,7 @@ from dataclasses import dataclass from airflow_breeze.branch_defaults import AIRFLOW_BRANCH - -providers_prefix = "apache-airflow-providers-" - - -def get_provider_name_from_short_hand(short_form_providers: tuple[str]): - providers = [] - for short_form_provider in short_form_providers: - if short_form_provider == "providers-index": - providers.append("apache-airflow-providers") - continue - - short_form_provider.split(".") - parts = "-".join(short_form_provider.split(".")) - providers.append(providers_prefix + parts) - return tuple(providers) +from airflow_breeze.utils.general_utils import get_provider_name_from_short_hand @dataclass diff --git a/dev/breeze/src/airflow_breeze/utils/general_utils.py b/dev/breeze/src/airflow_breeze/utils/general_utils.py new file mode 100644 index 0000000000000..d37d6d42c7d9c --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/general_utils.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +providers_prefix = "apache-airflow-providers-" + + +def get_provider_name_from_short_hand(short_form_providers: tuple[str]): + providers = [] + for short_form_provider in short_form_providers: + if short_form_provider == "providers-index": + providers.append("apache-airflow-providers") + continue + + short_form_provider.split(".") + parts = "-".join(short_form_provider.split(".")) + providers.append(providers_prefix + parts) + return tuple(providers) diff --git a/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py index 7d41c7c7b2cfb..7cd436e0e99b7 100644 --- a/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py +++ b/dev/breeze/src/airflow_breeze/utils/publish_docs_helpers.py @@ -26,6 +26,8 @@ import yaml +from airflow_breeze.utils.general_utils import get_provider_name_from_short_hand + CONSOLE_WIDTH = 180 ROOT_DIR = Path(__file__).parents[5].resolve() @@ -99,14 +101,19 @@ def get_available_packages(): ] -def process_package_filters(available_packages: list[str], package_filters: list[str] | None): +def process_package_filters( + available_packages: list[str], package_filters: list[str] | None, packages_short_form: tuple[str] +): """Filters the package list against a set of filters. A packet is returned if it matches at least one filter. The function keeps the order of the packages. """ - if not package_filters: + if not package_filters and not packages_short_form: return available_packages + expanded_short_form_packages = get_provider_name_from_short_hand(packages_short_form) + package_filters = list(package_filters + expanded_short_form_packages) + invalid_filters = [ f for f in package_filters if not any(fnmatch.fnmatch(p, f) for p in available_packages) ] diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index b21c8031fc04e..021b5060cf698 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -45,13 +45,13 @@ release-management:install-provider-packages:34c38aca17d23dbb454fe7a6bfd8e630 release-management:prepare-airflow-package:85d01c57e5b5ee0fb9e5f9d9706ed3b5 release-management:prepare-provider-documentation:a53cd338bd719c77108b53bc8d45b634 release-management:prepare-provider-packages:fc69d2ab8abdcbafcaaa63da380f4b76 -release-management:publish-docs:1c7e6b9af76d663d3af5bec29a21a2ee +release-management:publish-docs:45a6ea090bfcf564ea0dd8fc61655d8a release-management:release-prod-images:cfbfe8b19fee91fd90718f98ef2fd078 release-management:start-rc-process:b27bd524dd3c89f50a747b60a7e892c1 release-management:start-release:419f48f6a4ff4457cb9de7ff496aebbe release-management:update-constraints:02ec4b119150e3fdbac52026e94820ef release-management:verify-provider-packages:96dce5644aad6b37080acf77b3d8de3a -release-management:1098994e77919b0de653e5029bccbd14 +release-management:885a5fe8a39a3773011cf1f9bd2983ad sbom:generate-provider-requirements:2ea27f55be4e6ab2b0b5a410330f2747 sbom:update-sbom-information:653be48be70b4b7ff5172d491aadc694 sbom:ff73e9f941b6d6053e680ca53e9d1b33 diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg index 1b85a1bd798c0..fad73508b9b50 100644 --- a/images/breeze/output_release-management.svg +++ b/images/breeze/output_release-management.svg @@ -35,8 +35,8 @@ .breeze-release-management-r1 { fill: #c5c8c6;font-weight: bold } .breeze-release-management-r2 { fill: #c5c8c6 } .breeze-release-management-r3 { fill: #d0b344;font-weight: bold } -.breeze-release-management-r4 { fill: #868887 } -.breeze-release-management-r5 { fill: #68a0b3;font-weight: bold } +.breeze-release-management-r4 { fill: #68a0b3;font-weight: bold } +.breeze-release-management-r5 { fill: #868887 } .breeze-release-management-r6 { fill: #98a84b;font-weight: bold } @@ -147,36 +147,36 @@ -Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... +Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... Tools that release managers can use to prepare and manage Airflow releases -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Airflow release commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│prepare-airflow-package      Prepare sdist/whl package of Airflow.                                                │ -│create-minor-branch          Create a new version branch and update the default branches in main                  │ -│start-rc-process             Start RC process                                                                     │ -│start-release                Start Airflow release process                                                        │ -│release-prod-images          Release production images to DockerHub (needs DockerHub permissions).                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Providers release commands ─────────────────────────────────────────────────────────────────────────────────────────╮ -│prepare-provider-documentation      Prepare CHANGELOG, README and COMMITS information for providers.              │ -│prepare-provider-packages           Prepare sdist/whl packages of Airflow Providers.                              │ -│install-provider-packages           Installs provider packages that can be found in dist.                         │ -│verify-provider-packages            Verifies if all provider code is following expectations for providers.        │ -│generate-providers-metadata         Generates metadata for providers.                                             │ -│generate-issue-content-providers    Generates content for issue to test the release.                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│publish-docs           Command to publish generated documentation to airflow-site                                 │ -│generate-constraints   Generates pinned constraint files with all extras from setup.py in parallel.               │ -│add-back-references    Command to add back references for documentation to make it backward compatible.           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│update-constraints              Update released constraints with manual changes.                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Airflow release commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│prepare-airflow-package      Prepare sdist/whl package of Airflow.                                                │ +│create-minor-branch          Create a new version branch and update the default branches in main                  │ +│start-rc-process             Start RC process                                                                     │ +│start-release                Start Airflow release process                                                        │ +│release-prod-images          Release production images to DockerHub (needs DockerHub permissions).                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Providers release commands ─────────────────────────────────────────────────────────────────────────────────────────╮ +│prepare-provider-documentation      Prepare CHANGELOG, README and COMMITS information for providers.              │ +│prepare-provider-packages           Prepare sdist/whl packages of Airflow Providers.                              │ +│install-provider-packages           Installs provider packages that can be found in dist.                         │ +│verify-provider-packages            Verifies if all provider code is following expectations for providers.        │ +│generate-providers-metadata         Generates metadata for providers.                                             │ +│generate-issue-content-providers    Generates content for issue to test the release.                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│publish-docs           Command to publish generated documentation to airflow-site                                 │ +│generate-constraints   Generates pinned constraint files with all extras from setup.py in parallel.               │ +│add-back-references    Command to add back references for documentation to make it backward compatible.           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│update-constraints              Update released constraints with manual changes.                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_publish-docs.svg b/images/breeze/output_release-management_publish-docs.svg index 953fd4553d0ee..cab90ff6e2382 100644 --- a/images/breeze/output_release-management_publish-docs.svg +++ b/images/breeze/output_release-management_publish-docs.svg @@ -1,4 +1,4 @@ - +