diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py index b5b86bab4d5f..27088881908e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py @@ -60,6 +60,7 @@ async def build(ctx: click.Context, use_host_gradle_dist_tar: bool, build_archit ci_context=ctx.obj.get("ci_context"), ci_gcp_credentials=ctx.obj["ci_gcp_credentials"], use_local_cdk=ctx.obj.get("use_local_cdk"), + use_cdk_ref=ctx.obj.get("use_cdk_ref"), enable_report_auto_open=ctx.obj.get("enable_report_auto_open"), use_host_gradle_dist_tar=use_host_gradle_dist_tar, s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py index ad78e032d970..ca69acb7a4a0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -207,6 +207,15 @@ def validate_environment(is_local: bool) -> None: default=False, type=bool, ) +@click.option( + "--use-cdk-ref", + help=( + "Build with the airbyte-cdk from the specified git ref. " + "This is useful for testing against dev versions or previous versions of the CDK. " + "Ignored for java connectors and if `--use-local-cdk` is set." + ), + type=str, +) @click.option( "--enable-report-auto-open/--disable-report-auto-open", is_flag=True, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py index 3c2300e72b25..52deb09d7c01 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -60,6 +60,7 @@ def __init__( should_save_report: bool = True, code_tests_only: bool = False, use_local_cdk: bool = False, + use_cdk_ref: Optional[str] = None, use_host_gradle_dist_tar: bool = False, enable_report_auto_open: bool = True, docker_hub_username: Optional[Secret] = None, @@ -110,6 +111,7 @@ def __init__( self.should_save_report = should_save_report self.code_tests_only = code_tests_only self.use_local_cdk = use_local_cdk + self.use_cdk_ref = use_cdk_ref self.use_host_gradle_dist_tar = use_host_gradle_dist_tar self.enable_report_auto_open = enable_report_auto_open self.docker_hub_username = docker_hub_username diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py index 6a8ec464d40e..5a93fa05b2f9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -187,6 +187,7 @@ async def publish( s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), use_local_cdk=ctx.obj.get("use_local_cdk"), + use_cdk_ref=ctx.obj.get("use_cdk_ref"), python_registry_token=python_registry_token, python_registry_url=python_registry_url, python_registry_check_url=python_registry_check_url, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index 6acc02f07e49..f4e3e132eccc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -54,6 +54,7 @@ def __init__( s3_build_cache_access_key_id: Optional[Secret] = None, s3_build_cache_secret_key: Optional[Secret] = None, use_local_cdk: bool = False, + use_cdk_ref: Optional[str] = None, python_registry_token: Optional[Secret] = None, ci_github_access_token: Optional[Secret] = None, ) -> None: @@ -70,8 +71,8 @@ def __init__( pipeline_name = f"{rollout_mode.value} {connector.technical_name}" pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name - if use_local_cdk and not self.pre_release: - raise click.UsageError("Publishing with the local CDK is only supported for pre-release publishing.") + if (use_local_cdk or use_cdk_ref) and not self.pre_release: + raise click.UsageError("Publishing with CDK overrides is only supported for pre-release publishing.") super().__init__( pipeline_name=pipeline_name, @@ -91,6 +92,7 @@ def __init__( ci_gcp_credentials=ci_gcp_credentials, should_save_report=True, use_local_cdk=use_local_cdk, + use_cdk_ref=use_cdk_ref, docker_hub_username=docker_hub_username, docker_hub_password=docker_hub_password, s3_build_cache_access_key_id=s3_build_cache_access_key_id, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py index 685096215b72..442a5a824245 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -156,6 +156,7 @@ async def test( ci_gcp_credentials=ctx.obj["ci_gcp_credentials"], code_tests_only=code_tests_only, use_local_cdk=ctx.obj.get("use_local_cdk"), + use_cdk_ref=ctx.obj.get("use_cdk_ref"), s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), docker_hub_username=ctx.obj.get("docker_hub_username"), diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py index 84727d9908db..e0d3990d558b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -240,19 +240,47 @@ def with_python_connector_source(context: ConnectorContext) -> Container: async def apply_python_development_overrides(context: ConnectorContext, connector_container: Container) -> Container: # Run the connector using the local cdk if flag is set if context.use_local_cdk: - context.logger.info("Using local CDK") - # mount the local cdk - path_to_cdk = "airbyte-cdk/python/" - directory_to_mount = context.get_repo_dir(path_to_cdk) + # Assume CDK is cloned in a sibling dir to `airbyte`: + path_to_cdk = str(Path("../airbyte-python-cdk").resolve()) + if not Path(path_to_cdk).exists(): + raise FileExistsError(f"Local CDK not found at '{path_to_cdk}'") + context.logger.info(f"Using local CDK found at: '{path_to_cdk}'") - context.logger.info(f"Mounting CDK from {directory_to_mount}") + directory_to_mount = context.dagger_client.host().directory(path_to_cdk) + cdk_mount_dir = "/airbyte-cdk/python" + + context.logger.info(f"Mounting CDK from '{path_to_cdk}' to '{cdk_mount_dir}'") # Install the airbyte-cdk package from the local directory - # We use `--force-reinstall` to use local CDK with the latest updates and dependencies - connector_container = connector_container.with_mounted_directory(f"/{path_to_cdk}", directory_to_mount).with_exec( - ["pip", "install", "--force-reinstall", f"/{path_to_cdk}"] + connector_container = ( + connector_container.with_env_variable( + "POETRY_DYNAMIC_VERSIONING_BYPASS", + "0.0.0-dev.0", # Replace dynamic versioning with dev version + ) + .with_directory( + cdk_mount_dir, + directory_to_mount, + ) + .with_exec( + ["pip", "install", "--force-reinstall", f"{cdk_mount_dir}"], + # TODO: Consider moving to Poetry-native installation: + # ["poetry", "add", cdk_mount_dir] + ) + ) + elif context.use_cdk_ref: + cdk_ref = context.use_cdk_ref + if " " in cdk_ref: + raise ValueError("CDK ref should not contain spaces") + + context.logger.info("Using CDK ref: '{cdk_ref}'") + # Install the airbyte-cdk package from provided ref + connector_container = connector_container.with_exec( + [ + "pip", + "install", + f"git+https://github.com/airbytehq/airbyte-python-cdk.git#{cdk_ref}", + ], ) - return connector_container