add support for '--use-cdk-ref'

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py

@@ -60,6 +60,7 @@ async def build(ctx: click.Context, use_host_gradle_dist_tar: bool, build_archit
             ci_context=ctx.obj.get("ci_context"),
             ci_gcp_credentials=ctx.obj["ci_gcp_credentials"],
             use_local_cdk=ctx.obj.get("use_local_cdk"),
+            use_cdk_ref=ctx.obj.get("use_cdk_ref"),
             enable_report_auto_open=ctx.obj.get("enable_report_auto_open"),
             use_host_gradle_dist_tar=use_host_gradle_dist_tar,
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py

@@ -207,6 +207,15 @@ def validate_environment(is_local: bool) -> None:
     default=False,
     type=bool,
 )
+@click.option(
+    "--use-cdk-ref",
+    help=(
+        "Build with the airbyte-cdk from the specified git ref. "
+        "This is useful for testing against dev versions or previous versions of the CDK. "
+        "Ignored for java connectors and if `--use-local-cdk` is set."
+    ),
+    type=str,
+)
 @click.option(
     "--enable-report-auto-open/--disable-report-auto-open",
     is_flag=True,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py

@@ -60,6 +60,7 @@ def __init__(
         should_save_report: bool = True,
         code_tests_only: bool = False,
         use_local_cdk: bool = False,
+        use_cdk_ref: Optional[str] = None,
         use_host_gradle_dist_tar: bool = False,
         enable_report_auto_open: bool = True,
         docker_hub_username: Optional[Secret] = None,
@@ -110,6 +111,7 @@ def __init__(
         self.should_save_report = should_save_report
         self.code_tests_only = code_tests_only
         self.use_local_cdk = use_local_cdk
+        self.use_cdk_ref = use_cdk_ref
         self.use_host_gradle_dist_tar = use_host_gradle_dist_tar
         self.enable_report_auto_open = enable_report_auto_open
         self.docker_hub_username = docker_hub_username

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py

@@ -187,6 +187,7 @@ async def publish(
                 s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
                 s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
                 use_local_cdk=ctx.obj.get("use_local_cdk"),
+                use_cdk_ref=ctx.obj.get("use_cdk_ref"),
                 python_registry_token=python_registry_token,
                 python_registry_url=python_registry_url,
                 python_registry_check_url=python_registry_check_url,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py

@@ -54,6 +54,7 @@ def __init__(
         s3_build_cache_access_key_id: Optional[Secret] = None,
         s3_build_cache_secret_key: Optional[Secret] = None,
         use_local_cdk: bool = False,
+        use_cdk_ref: Optional[str] = None,
         python_registry_token: Optional[Secret] = None,
         ci_github_access_token: Optional[Secret] = None,
     ) -> None:
@@ -70,8 +71,8 @@ def __init__(
         pipeline_name = f"{rollout_mode.value} {connector.technical_name}"
         pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name
 
-        if use_local_cdk and not self.pre_release:
-            raise click.UsageError("Publishing with the local CDK is only supported for pre-release publishing.")
+        if (use_local_cdk or use_cdk_ref) and not self.pre_release:
+            raise click.UsageError("Publishing with CDK overrides is only supported for pre-release publishing.")
 
         super().__init__(
             pipeline_name=pipeline_name,
@@ -91,6 +92,7 @@ def __init__(
             ci_gcp_credentials=ci_gcp_credentials,
             should_save_report=True,
             use_local_cdk=use_local_cdk,
+            use_cdk_ref=use_cdk_ref,
             docker_hub_username=docker_hub_username,
             docker_hub_password=docker_hub_password,
             s3_build_cache_access_key_id=s3_build_cache_access_key_id,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py

@@ -156,6 +156,7 @@ async def test(
             ci_gcp_credentials=ctx.obj["ci_gcp_credentials"],
             code_tests_only=code_tests_only,
             use_local_cdk=ctx.obj.get("use_local_cdk"),
+            use_cdk_ref=ctx.obj.get("use_cdk_ref"),
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
             s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
             docker_hub_username=ctx.obj.get("docker_hub_username"),

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py

@@ -240,12 +240,11 @@ def with_python_connector_source(context: ConnectorContext) -> Container:
 async def apply_python_development_overrides(context: ConnectorContext, connector_container: Container) -> Container:
     # Run the connector using the local cdk if flag is set
     if context.use_local_cdk:
-        context.logger.info("Using local CDK")
-        # mount the local cdk
         # Assume CDK is cloned in a sibling dir to `airbyte`:
         path_to_cdk = str(Path("../airbyte-python-cdk").resolve())
         if not Path(path_to_cdk).exists():
             raise FileExistsError(f"Local CDK not found at '{path_to_cdk}'")
+        context.logger.info(f"Using local CDK found at: '{path_to_cdk}'")
 
         directory_to_mount = context.dagger_client.host().directory(path_to_cdk)
         cdk_mount_dir = "/airbyte-cdk/python"
@@ -268,7 +267,20 @@ async def apply_python_development_overrides(context: ConnectorContext, connecto
                 # ["poetry", "add", cdk_mount_dir]
             )
         )
-
+    elif context.use_cdk_ref:
+        cdk_ref = context.use_cdk_ref
+        if " " in cdk_ref:
+            raise ValueError("CDK ref should not contain spaces")
+
+        context.logger.info("Using CDK ref: '{cdk_ref}'")
+        # Install the airbyte-cdk package from provided ref
+        connector_container = connector_container.with_exec(
+            [
+                "pip",
+                "install",
+                f"git+https://github.com/airbytehq/airbyte-python-cdk.git#{cdk_ref}",
+            ],
+        )
     return connector_container