Fix latest botocore tests conflicting dependencies (#41267)

The "latest botocore" tests have conflicting dependencies for
latest yandexcloud provider - their "requests" support conflicts
with apache.beam and latest botocore.

For now we remove yandexcloud for those tests hoping that soon
apache.beam will support higher requests versions.

And issue has been created in apache.beam to remove the limitation
as the root cause seems to be already fixed.

See apache/beam#32080

Dockerfile.ci

@@ -1022,11 +1022,16 @@ function check_boto_upgrade() {
     echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
     echo
     # shellcheck disable=SC2086
-    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true
+    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true
     # We need to include few dependencies to pass pip check with other dependencies:
     #   * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider)
     #   * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider)
     #   * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider)
+    #   * yandexcloud requirements for requests does not match those of apache.beam and latest botocore
+    #   Both requests and yandexcloud exclusion above might be removed after
+    #   https://github.com/apache/beam/issues/32080 is addressed
+    #   When you remove yandexcloud from the above list, also remove it from "test_example_dags.py"
+    #   in "tests/always".
     set -x
     # shellcheck disable=SC2086
     ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \

pyproject.toml

@@ -409,6 +409,7 @@ combine-as-imports = true
 "tests/providers/qdrant/hooks/test_qdrant.py" = ["E402"]
 "tests/providers/qdrant/operators/test_qdrant.py" = ["E402"]
 "tests/providers/snowflake/operators/test_snowflake_sql.py" = ["E402"]
+"tests/providers/yandex/*/*.py" = ["E402"]
 
 # All the modules which do not follow B028 yet: https://docs.astral.sh/ruff/rules/no-explicit-stacklevel/
 "helm_tests/airflow_aux/test_basic_helm_chart.py" = ["B028"]

scripts/docker/entrypoint_ci.sh

@@ -242,11 +242,16 @@ function check_boto_upgrade() {
     echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}"
     echo
     # shellcheck disable=SC2086
-    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs || true
+    ${PACKAGING_TOOL_CMD} uninstall ${EXTRA_UNINSTALL_FLAGS} aiobotocore s3fs yandexcloud || true
     # We need to include few dependencies to pass pip check with other dependencies:
     #   * oss2 as dependency as otherwise jmespath will be bumped (sync with alibaba provider)
     #   * gcloud-aio-auth limit is needed to be included as it bumps cryptography (sync with google provider)
     #   * requests needs to be limited to be compatible with apache beam (sync with apache-beam provider)
+    #   * yandexcloud requirements for requests does not match those of apache.beam and latest botocore
+    #   Both requests and yandexcloud exclusion above might be removed after
+    #   https://github.com/apache/beam/issues/32080 is addressed
+    #   When you remove yandexcloud from the above list, also remove it from "test_example_dags.py"
+    #   in "tests/always".
     set -x
     # shellcheck disable=SC2086
     ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \

tests/always/test_example_dags.py

@@ -124,6 +124,13 @@ def example_not_excluded_dags(xfail_db_exception: bool = False):
         for prefix in PROVIDERS_PREFIXES
         for provider in suspended_providers_folders
     ]
+    temporary_excluded_upgrade_boto_providers_folders = [
+        AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix()
+        for prefix in PROVIDERS_PREFIXES
+        # TODO - remove me when https://github.com/apache/beam/issues/32080 is addressed
+        #        and we bring back yandex to be run in case of upgrade boto
+        for provider in ["yandex"]
+    ]
     current_python_excluded_providers_folders = [
         AIRFLOW_SOURCES_ROOT.joinpath(prefix, provider).as_posix()
         for prefix in PROVIDERS_PREFIXES
@@ -139,6 +146,11 @@ def example_not_excluded_dags(xfail_db_exception: bool = False):
             if candidate.startswith(tuple(suspended_providers_folders)):
                 param_marks.append(pytest.mark.skip(reason="Suspended provider"))
 
+            if os.environ.get("UPGRADE_BOTO", "false") == "true" and candidate.startswith(
+                tuple(temporary_excluded_upgrade_boto_providers_folders)
+            ):
+                param_marks.append(pytest.mark.skip(reason="Temporary excluded upgrade boto provider"))
+
             if candidate.startswith(tuple(current_python_excluded_providers_folders)):
                 param_marks.append(
                     pytest.mark.skip(reason=f"Not supported for Python {CURRENT_PYTHON_VERSION}")

tests/providers/yandex/hooks/test_dataproc.py

@@ -19,6 +19,10 @@
 import json
 from unittest import mock
 
+import pytest
+
+yandexlcloud = pytest.importorskip("yandexcloud")
+
 from airflow.models import Connection
 from airflow.providers.yandex.hooks.dataproc import DataprocHook
 

tests/providers/yandex/hooks/test_yandex.py

@@ -23,6 +23,8 @@
 
 import pytest
 
+yandexcloud = pytest.importorskip("yandexcloud")
+
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook
 from tests.test_utils.config import conf_vars

tests/providers/yandex/hooks/test_yq.py

@@ -20,6 +20,10 @@
 from datetime import timedelta
 from unittest import mock
 
+import pytest
+
+yandexcloud = pytest.importorskip("yandexcloud")
+
 import responses
 from responses import matchers
 

tests/providers/yandex/links/test_yq.py

@@ -18,11 +18,15 @@
 
 from unittest import mock
 
+import pytest
+
 from airflow.models.taskinstance import TaskInstance
 from airflow.models.xcom import XCom
 from airflow.providers.yandex.links.yq import YQLink
 from tests.test_utils.mock_operators import MockOperator
 
+yandexcloud = pytest.importorskip("yandexcloud")
+
 
 def test_persist():
     mock_context = mock.MagicMock()

tests/providers/yandex/operators/test_dataproc.py

@@ -19,6 +19,10 @@
 import datetime
 from unittest.mock import MagicMock, call, patch
 
+import pytest
+
+yandexcloud = pytest.importorskip("yandexcloud")
+
 from airflow.models.dag import DAG
 from airflow.providers.yandex.operators.dataproc import (
     DataprocCreateClusterOperator,

tests/providers/yandex/operators/test_yq.py

@@ -21,6 +21,9 @@
 from unittest.mock import MagicMock, call, patch
 
 import pytest
+
+yandexcloud = pytest.importorskip("yandexcloud")
+
 import responses
 from responses import matchers
 

tests/providers/yandex/secrets/test_lockbox.py

@@ -20,6 +20,9 @@
 from unittest.mock import MagicMock, Mock, patch
 
 import pytest
+
+yandexcloud = pytest.importorskip("yandexcloud")
+
 import yandex.cloud.lockbox.v1.payload_pb2 as payload_pb
 import yandex.cloud.lockbox.v1.secret_pb2 as secret_pb
 import yandex.cloud.lockbox.v1.secret_service_pb2 as secret_service_pb

tests/providers/yandex/utils/test_user_agent.py

@@ -18,6 +18,10 @@
 
 from unittest import mock
 
+import pytest
+
+yandexcloud = pytest.importorskip("yandexcloud")
+
 from airflow.providers.yandex.utils.user_agent import provider_user_agent