Draft: update airbyte-ci local cdk code (#48457)

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py

@@ -60,6 +60,7 @@ async def build(ctx: click.Context, use_host_gradle_dist_tar: bool, build_archit
             ci_context=ctx.obj.get("ci_context"),
             ci_gcp_credentials=ctx.obj["ci_gcp_credentials"],
             use_local_cdk=ctx.obj.get("use_local_cdk"),
+            use_cdk_ref=ctx.obj.get("use_cdk_ref"),
             enable_report_auto_open=ctx.obj.get("enable_report_auto_open"),
             use_host_gradle_dist_tar=use_host_gradle_dist_tar,
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py

@@ -207,6 +207,15 @@ def validate_environment(is_local: bool) -> None:
     default=False,
     type=bool,
 )
+@click.option(
+    "--use-cdk-ref",
+    help=(
+        "Build with the airbyte-cdk from the specified git ref. "
+        "This is useful for testing against dev versions or previous versions of the CDK. "
+        "Ignored for java connectors and if `--use-local-cdk` is set."
+    ),
+    type=str,
+)
 @click.option(
     "--enable-report-auto-open/--disable-report-auto-open",
     is_flag=True,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py

@@ -60,6 +60,7 @@ def __init__(
         should_save_report: bool = True,
         code_tests_only: bool = False,
         use_local_cdk: bool = False,
+        use_cdk_ref: Optional[str] = None,
         use_host_gradle_dist_tar: bool = False,
         enable_report_auto_open: bool = True,
         docker_hub_username: Optional[Secret] = None,
@@ -110,6 +111,7 @@ def __init__(
         self.should_save_report = should_save_report
         self.code_tests_only = code_tests_only
         self.use_local_cdk = use_local_cdk
+        self.use_cdk_ref = use_cdk_ref
         self.use_host_gradle_dist_tar = use_host_gradle_dist_tar
         self.enable_report_auto_open = enable_report_auto_open
         self.docker_hub_username = docker_hub_username

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py

@@ -187,6 +187,7 @@ async def publish(
                 s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
                 s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
                 use_local_cdk=ctx.obj.get("use_local_cdk"),
+                use_cdk_ref=ctx.obj.get("use_cdk_ref"),
                 python_registry_token=python_registry_token,
                 python_registry_url=python_registry_url,
                 python_registry_check_url=python_registry_check_url,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py

@@ -54,6 +54,7 @@ def __init__(
         s3_build_cache_access_key_id: Optional[Secret] = None,
         s3_build_cache_secret_key: Optional[Secret] = None,
         use_local_cdk: bool = False,
+        use_cdk_ref: Optional[str] = None,
         python_registry_token: Optional[Secret] = None,
         ci_github_access_token: Optional[Secret] = None,
     ) -> None:
@@ -70,8 +71,8 @@ def __init__(
         pipeline_name = f"{rollout_mode.value} {connector.technical_name}"
         pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name
 
-        if use_local_cdk and not self.pre_release:
-            raise click.UsageError("Publishing with the local CDK is only supported for pre-release publishing.")
+        if (use_local_cdk or use_cdk_ref) and not self.pre_release:
+            raise click.UsageError("Publishing with CDK overrides is only supported for pre-release publishing.")
 
         super().__init__(
             pipeline_name=pipeline_name,
@@ -91,6 +92,7 @@ def __init__(
             ci_gcp_credentials=ci_gcp_credentials,
             should_save_report=True,
             use_local_cdk=use_local_cdk,
+            use_cdk_ref=use_cdk_ref,
             docker_hub_username=docker_hub_username,
             docker_hub_password=docker_hub_password,
             s3_build_cache_access_key_id=s3_build_cache_access_key_id,

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py

@@ -156,6 +156,7 @@ async def test(
             ci_gcp_credentials=ctx.obj["ci_gcp_credentials"],
             code_tests_only=code_tests_only,
             use_local_cdk=ctx.obj.get("use_local_cdk"),
+            use_cdk_ref=ctx.obj.get("use_cdk_ref"),
             s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
             s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
             docker_hub_username=ctx.obj.get("docker_hub_username"),

airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py

@@ -240,19 +240,47 @@ def with_python_connector_source(context: ConnectorContext) -> Container:
 async def apply_python_development_overrides(context: ConnectorContext, connector_container: Container) -> Container:
     # Run the connector using the local cdk if flag is set
     if context.use_local_cdk:
-        context.logger.info("Using local CDK")
-        # mount the local cdk
-        path_to_cdk = "airbyte-cdk/python/"
-        directory_to_mount = context.get_repo_dir(path_to_cdk)
+        # Assume CDK is cloned in a sibling dir to `airbyte`:
+        path_to_cdk = str(Path("../airbyte-python-cdk").resolve())
+        if not Path(path_to_cdk).exists():
+            raise FileExistsError(f"Local CDK not found at '{path_to_cdk}'")
+        context.logger.info(f"Using local CDK found at: '{path_to_cdk}'")
 
-        context.logger.info(f"Mounting CDK from {directory_to_mount}")
+        directory_to_mount = context.dagger_client.host().directory(path_to_cdk)
+        cdk_mount_dir = "/airbyte-cdk/python"
+
+        context.logger.info(f"Mounting CDK from '{path_to_cdk}' to '{cdk_mount_dir}'")
 
         # Install the airbyte-cdk package from the local directory
-        # We use `--force-reinstall` to use local CDK with the latest updates and dependencies
-        connector_container = connector_container.with_mounted_directory(f"/{path_to_cdk}", directory_to_mount).with_exec(
-            ["pip", "install", "--force-reinstall", f"/{path_to_cdk}"]
+        connector_container = (
+            connector_container.with_env_variable(
+                "POETRY_DYNAMIC_VERSIONING_BYPASS",
+                "0.0.0-dev.0",  # Replace dynamic versioning with dev version
+            )
+            .with_directory(
+                cdk_mount_dir,
+                directory_to_mount,
+            )
+            .with_exec(
+                ["pip", "install", "--force-reinstall", f"{cdk_mount_dir}"],
+                # TODO: Consider moving to Poetry-native installation:
+                # ["poetry", "add", cdk_mount_dir]
+            )
+        )
+    elif context.use_cdk_ref:
+        cdk_ref = context.use_cdk_ref
+        if " " in cdk_ref:
+            raise ValueError("CDK ref should not contain spaces")
+
+        context.logger.info("Using CDK ref: '{cdk_ref}'")
+        # Install the airbyte-cdk package from provided ref
+        connector_container = connector_container.with_exec(
+            [
+                "pip",
+                "install",
+                f"git+https://github.com/airbytehq/airbyte-python-cdk.git#{cdk_ref}",
+            ],
         )
-
     return connector_container