From cbb34c9865430ff8288d56e277fa4b75b36a4ed4 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Wed, 13 Nov 2024 14:44:05 -0800
Subject: [PATCH 01/28] add manifest connectors to test matrix

---
 .github/workflows/connector-tests.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml
index 8a934d14..83574245 100644
--- a/.github/workflows/connector-tests.yml
+++ b/.github/workflows/connector-tests.yml
@@ -81,6 +81,12 @@ jobs:
             cdk_extra: vector-db-based
           - connector: destination-motherduck
             cdk_extra: sql
+          # TODO: These are manifest connectors and won't work as expectd until we
+          #       add `--use-local-cdk` support for manifest connectors
+          - connector: source-the-guardian-api
+            cdk_extra: n/a
+          - connector: source-pokeapi
+            cdk_extra: n/a
 
     name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
     steps:

From 6979c688d6930c32b6e3132acf28f4b4e59ec816 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Wed, 13 Nov 2024 14:44:23 -0800
Subject: [PATCH 02/28] ci: add docker build job

---
 .github/workflows/docker-build.yml | 22 ++++++++++++++++++++++
 Dockerfile                         | 12 ++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 .github/workflows/docker-build.yml
 create mode 100644 Dockerfile

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
new file mode 100644
index 00000000..8caa69c6
--- /dev/null
+++ b/.github/workflows/docker-build.yml
@@ -0,0 +1,22 @@
+name: SCD Docker Build (Dev Images)
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  docker_publish:
+    name: Build and Publish `source-declarative-manifest` Docker Image
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
+      contents: write # Needed to upload artifacts to the release
+    env: {}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - run: |
+          docker login
+          docker build -t airbyte/source-declarative-manifest:${{ github.sha }} .
+          docker push airbyte/source-declarative-manifest:${{ github.sha }}
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..3d4572fb
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,12 @@
+# Builds as `airbyte/source-declarative-manifest`
+# Usage:
+#  docker build -t airbyte/source-declarative-manifest .
+#  docker run airbyte/source-declarative-manifest --help
+#  docker run airbyte/source-declarative-manifest spec
+FROM docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
+
+# Copy source code into image
+COPY .
+
+RUN poetry install --no-dev
+ENTRYPOINT [ "poetry run source-declarative-manifest" ]

From f0566c3afc14f786c4c1fb54a0210b302f41b92e Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Wed, 13 Nov 2024 14:44:43 -0800
Subject: [PATCH 03/28] feat: add cli script for sdm

---
 pyproject.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pyproject.toml b/pyproject.toml
index 13f2e74c..eb3645eb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -104,6 +104,10 @@ sphinx-docs = ["Sphinx", "sphinx-rtd-theme"]
 vector-db-based = ["langchain", "openai", "cohere", "tiktoken"]
 sql = ["sqlalchemy"]
 
+[tool.poetry.scripts]
+
+source-declarative-manifest = "airbyte_cdk.cli.source_declarative_manifest:run"
+
 [tool.isort]
 skip = ["__init__.py"]  # TODO: Remove after this is fixed: https://github.com/airbytehq/airbyte-python-cdk/issues/12
 

From 423d6b31b3c14e12782fbc9bd4d48891b8ae251a Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Wed, 13 Nov 2024 14:45:11 -0800
Subject: [PATCH 04/28] feat: add sdm cli

---
 airbyte_cdk/cli/__init__.py                   |   1 +
 .../source_declarative_manifest/__init__.py   |   6 +
 .../cli/source_declarative_manifest/_run.py   | 191 ++++++++++++++++++
 .../cli/source_declarative_manifest/spec.json |  17 ++
 4 files changed, 215 insertions(+)
 create mode 100644 airbyte_cdk/cli/__init__.py
 create mode 100644 airbyte_cdk/cli/source_declarative_manifest/__init__.py
 create mode 100644 airbyte_cdk/cli/source_declarative_manifest/_run.py
 create mode 100644 airbyte_cdk/cli/source_declarative_manifest/spec.json

diff --git a/airbyte_cdk/cli/__init__.py b/airbyte_cdk/cli/__init__.py
new file mode 100644
index 00000000..7f66676b
--- /dev/null
+++ b/airbyte_cdk/cli/__init__.py
@@ -0,0 +1 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
diff --git a/airbyte_cdk/cli/source_declarative_manifest/__init__.py b/airbyte_cdk/cli/source_declarative_manifest/__init__.py
new file mode 100644
index 00000000..8a8d225e
--- /dev/null
+++ b/airbyte_cdk/cli/source_declarative_manifest/__init__.py
@@ -0,0 +1,6 @@
+from airbyte_cdk.cli.source_declarative_manifest._run import run
+
+
+__all__ = [
+    "run",
+]
diff --git a/airbyte_cdk/cli/source_declarative_manifest/_run.py b/airbyte_cdk/cli/source_declarative_manifest/_run.py
new file mode 100644
index 00000000..c3d399ae
--- /dev/null
+++ b/airbyte_cdk/cli/source_declarative_manifest/_run.py
@@ -0,0 +1,191 @@
+# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+
+
+from __future__ import annotations
+
+import json
+import pkgutil
+import sys
+import traceback
+from datetime import datetime
+from pathlib import Path
+from typing import Any, List, Mapping, Optional
+
+from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
+from airbyte_cdk.models import (
+    AirbyteErrorTraceMessage,
+    AirbyteMessage,
+    AirbyteMessageSerializer,
+    AirbyteStateMessage,
+    AirbyteTraceMessage,
+    ConfiguredAirbyteCatalog,
+    ConnectorSpecificationSerializer,
+    TraceType,
+    Type,
+)
+from airbyte_cdk.sources.declarative.concurrent_declarative_source import (
+    ConcurrentDeclarativeSource,
+)
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+from airbyte_cdk.sources.source import TState
+from orjson import orjson
+
+
+class SourceLocalYaml(YamlDeclarativeSource):
+    """
+    Declarative source defined by a yaml file in the local filesystem
+    """
+
+    def __init__(
+        self,
+        catalog: Optional[ConfiguredAirbyteCatalog],
+        config: Optional[Mapping[str, Any]],
+        state: TState,
+        **kwargs,
+    ):
+        """
+        HACK!
+            Problem: YamlDeclarativeSource relies on the calling module name/path to find the yaml file.
+            Implication: If you call YamlDeclarativeSource directly it will look for the yaml file in the wrong place. (e.g. the airbyte-cdk package)
+            Solution: Subclass YamlDeclarativeSource from the same location as the manifest to load.
+
+            When can we remove this?
+                When the airbyte-cdk is updated to not rely on the calling module name/path to find the yaml file.
+                When all manifest connectors are updated to use the new airbyte-cdk.
+                When all manifest connectors are updated to use the source-declarative-manifest as the base image.
+        """
+        super().__init__(
+            catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"}
+        )
+
+
+def _is_local_manifest_command(args: List[str]) -> bool:
+    # Check for a local manifest.yaml file
+    return Path("/airbyte/integration_code/source_declarative_manifest/manifest.yaml").exists()
+
+
+def handle_command(args: List[str]) -> None:
+    if _is_local_manifest_command(args):
+        handle_local_manifest_command(args)
+    else:
+        handle_remote_manifest_command(args)
+
+
+def _get_local_yaml_source(args: List[str]) -> SourceLocalYaml:
+    try:
+        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        return SourceLocalYaml(config=config, catalog=catalog, state=state)
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        raise error
+
+
+def handle_local_manifest_command(args: List[str]) -> None:
+    source = _get_local_yaml_source(args)
+    launch(source, args)
+
+
+def handle_remote_manifest_command(args: List[str]) -> None:
+    """Overrides the spec command to return the generalized spec for the declarative manifest source.
+
+    This is different from a typical low-code, but built and published separately source built as a ManifestDeclarativeSource,
+    because that will have a spec method that returns the spec for that specific source. Other than spec,
+    the generalized connector behaves the same as any other, since the manifest is provided in the config.
+    """
+    if args[0] == "spec":
+        json_spec = pkgutil.get_data(
+            "airbyte_cdk.cli.source_declarative_manifest",
+            "spec.json",
+        )
+        spec_obj = json.loads(json_spec)
+        spec = ConnectorSpecificationSerializer.load(spec_obj)
+
+        message = AirbyteMessage(type=Type.SPEC, spec=spec)
+        print(AirbyteEntrypoint.airbyte_message_to_string(message))
+    else:
+        source = create_declarative_source(args)
+        launch(source, args)
+
+
+def create_declarative_source(args: List[str]) -> ConcurrentDeclarativeSource:
+    """Creates the source with the injected config.
+
+    This essentially does what other low-code sources do at build time, but at runtime,
+    with a user-provided manifest in the config. This better reflects what happens in the
+    connector builder.
+    """
+    try:
+        config, catalog, state = _parse_inputs_into_config_catalog_state(args)
+        if "__injected_declarative_manifest" not in config:
+            raise ValueError(
+                f"Invalid config: `__injected_declarative_manifest` should be provided at the root of the config but config only has keys {list(config.keys())}"
+            )
+        return ConcurrentDeclarativeSource(
+            config=config,
+            catalog=catalog,
+            state=state,
+            source_config=config.get("__injected_declarative_manifest"),
+        )
+    except Exception as error:
+        print(
+            orjson.dumps(
+                AirbyteMessageSerializer.dump(
+                    AirbyteMessage(
+                        type=Type.TRACE,
+                        trace=AirbyteTraceMessage(
+                            type=TraceType.ERROR,
+                            emitted_at=int(datetime.now().timestamp() * 1000),
+                            error=AirbyteErrorTraceMessage(
+                                message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}",
+                                stack_trace=traceback.format_exc(),
+                            ),
+                        ),
+                    )
+                )
+            ).decode()
+        )
+        raise error
+
+
+def _parse_inputs_into_config_catalog_state(
+    args: List[str],
+) -> (Optional[Mapping[str, Any]], Optional[ConfiguredAirbyteCatalog], List[AirbyteStateMessage]):
+    parsed_args = AirbyteEntrypoint.parse_args(args)
+    config = (
+        ConcurrentDeclarativeSource.read_config(parsed_args.config)
+        if hasattr(parsed_args, "config")
+        else None
+    )
+    catalog = (
+        ConcurrentDeclarativeSource.read_catalog(parsed_args.catalog)
+        if hasattr(parsed_args, "catalog")
+        else None
+    )
+    state = (
+        ConcurrentDeclarativeSource.read_state(parsed_args.state)
+        if hasattr(parsed_args, "state")
+        else []
+    )
+
+    return config, catalog, state
+
+
+def run() -> None:
+    args = sys.argv[1:]
+    handle_command(args)
diff --git a/airbyte_cdk/cli/source_declarative_manifest/spec.json b/airbyte_cdk/cli/source_declarative_manifest/spec.json
new file mode 100644
index 00000000..73d6a81a
--- /dev/null
+++ b/airbyte_cdk/cli/source_declarative_manifest/spec.json
@@ -0,0 +1,17 @@
+{
+  "documentationUrl": "https://docs.airbyte.com/integrations/sources/low-code",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Low-code source spec",
+    "type": "object",
+    "required": ["__injected_declarative_manifest"],
+    "additionalProperties": true,
+    "properties": {
+      "__injected_declarative_manifest": {
+        "title": "Low-code manifest",
+        "type": "object",
+        "description": "The low-code manifest that defines the components of the source."
+      }
+    }
+  }
+}

From 17c5ee6a7b196702fa19486079ebeffd1855c489 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Wed, 13 Nov 2024 14:46:06 -0800
Subject: [PATCH 05/28] feat: publish official sdm docker image after pypi
 publish

---
 .github/workflows/pypi_publish.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml
index c1af4e6c..777d4971 100644
--- a/.github/workflows/pypi_publish.yml
+++ b/.github/workflows/pypi_publish.yml
@@ -45,3 +45,25 @@ jobs:
         with:
           # Can be toggled at the repository level between `https://upload.pypi.org/legacy/` and `https://test.pypi.org/legacy/`
           repository-url: ${{vars.PYPI_PUBLISH_URL}}
+
+  # TODO: Need to test this on the other workflow first, then update here when we have
+  #       something that works.
+  docker_publish:
+    name: Build and Publish `source-declarative-manifest` Docker Image
+    runs-on: ubuntu-latest
+    needs: [build]
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
+      contents: write # Needed to upload artifacts to the release
+    environment:
+      name: Docker Hub
+      url: "https://hub.docker.com/r/airbytehq/airbyte-cdk"
+    env:
+      # TODO: Clean this up with a real version tag, remove the 'refs/tags/v' prefix
+      VERSION: ${{ github.ref }}
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - run: |
+          docker login
+          docker build -t airbyte/source-declarative-manifest:${VERSION} .
+          docker push airbyte/source-declarative-manifest:${VERSION}

From 9094390e9a1d2b8231183342ddcf92e7dd332081 Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Thu, 14 Nov 2024 12:12:54 -0800
Subject: [PATCH 06/28] feat: working Dockerfile for declarative-manifest

---
 Dockerfile     | 24 +++++++++++++++---------
 poetry.lock    | 12 ++++++------
 pyproject.toml |  1 +
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 3d4572fb..dd6a841e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,18 @@
-# Builds as `airbyte/source-declarative-manifest`
-# Usage:
-#  docker build -t airbyte/source-declarative-manifest .
-#  docker run airbyte/source-declarative-manifest --help
-#  docker run airbyte/source-declarative-manifest spec
 FROM docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
 
-# Copy source code into image
-COPY .
+WORKDIR /airbyte/integration_code
 
-RUN poetry install --no-dev
-ENTRYPOINT [ "poetry run source-declarative-manifest" ]
+# Copy project files needed for build
+COPY pyproject.toml poetry.lock README.md ./
+
+# Install dependencies - ignore keyring warnings
+RUN poetry config virtualenvs.create false \
+    && poetry install --only main --no-interaction --no-ansi || true
+
+# Copy source code
+COPY airbyte_cdk ./airbyte_cdk
+
+# Build and install the package
+RUN poetry build && pip install dist/*.whl
+
+ENTRYPOINT ["poetry", "run", "source-declarative-manifest"]
diff --git a/poetry.lock b/poetry.lock
index a9efac1e..3ef60420 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
 
 [[package]]
 name = "aiohappyeyeballs"
@@ -1855,8 +1855,8 @@ files = [
 httpx = ">=0.23.0,<1"
 orjson = ">=3.9.14,<4.0.0"
 pydantic = [
-    {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""},
     {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""},
+    {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""},
 ]
 requests = ">=2,<3"
 requests-toolbelt = ">=1.0.0,<2.0.0"
@@ -2734,9 +2734,9 @@ files = [
 
 [package.dependencies]
 numpy = [
+    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
     {version = ">=1.22.4", markers = "python_version < \"3.11\""},
     {version = ">=1.23.2", markers = "python_version == \"3.11\""},
-    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -3255,8 +3255,8 @@ files = [
 annotated-types = ">=0.6.0"
 pydantic-core = "2.23.4"
 typing-extensions = [
-    {version = ">=4.6.1", markers = "python_version < \"3.13\""},
     {version = ">=4.12.2", markers = "python_version >= \"3.13\""},
+    {version = ">=4.6.1", markers = "python_version < \"3.13\""},
 ]
 
 [package.extras]
@@ -3891,7 +3891,7 @@ files = [
 name = "rapidfuzz"
 version = "3.10.1"
 description = "rapid fuzzy string matching"
-optional = true
+optional = false
 python-versions = ">=3.9"
 files = [
     {file = "rapidfuzz-3.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f17d9f21bf2f2f785d74f7b0d407805468b4c173fa3e52c86ec94436b338e74a"},
@@ -5253,4 +5253,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "fb563a9483e45cd18a6f508b5d8f7b9a6362a9157a454c686a0fa212b0442021"
+content-hash = "812f9d2f5348c4cf0c052928219db91e98410c1827e45a50dbfc315c85caba89"
diff --git a/pyproject.toml b/pyproject.toml
index eb3645eb..dd953a15 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,6 +49,7 @@ pyrate-limiter = "~3.1.0"
 python-dateutil = "*"
 python-ulid = "^3.0.0"
 PyYAML = "^6.0.1"
+rapidfuzz = "^3.10.1"
 requests = "*"
 requests_cache = "*"
 wcmatch = "10.0"

From 91a979963361551317fbe5d7386de61b3b8d3916 Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Thu, 14 Nov 2024 13:24:03 -0800
Subject: [PATCH 07/28] update docker-build action

---
 .github/workflows/docker-build.yml | 35 +++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 8caa69c6..51937820 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -1,22 +1,37 @@
-name: SCD Docker Build (Dev Images)
+name: SDM Docker Build (Test)
 
 on:
   push:
+    branches:
+      - main
   workflow_dispatch:
 
 jobs:
-  docker_publish:
-    name: Build and Publish `source-declarative-manifest` Docker Image
+  docker_build:
+    name: Build and Publish source-declarative-manifest Docker Image
     runs-on: ubuntu-latest
     permissions:
-      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
-      contents: write # Needed to upload artifacts to the release
-    env: {}
+      id-token: write
+      contents: write
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - run: |
-          docker login
-          docker build -t airbyte/source-declarative-manifest:${{ github.sha }} .
-          docker push airbyte/source-declarative-manifest:${{ github.sha }}
+      
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}  # Only push on main
+          tags: airbyte/source-declarative-manifest:test-${{ github.sha }}
+
+      - name: Test image
+        run: |
+          docker pull airbyte/source-declarative-manifest:test-${{ github.sha }}
+          docker run airbyte/source-declarative-manifest:test-${{ github.sha }} spec

From b72056be74a517b952b0b1198d32aaa508667688 Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Thu, 14 Nov 2024 13:52:38 -0800
Subject: [PATCH 08/28] modify docker-build action

---
 .github/workflows/docker-build.yml | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 51937820..d852613a 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -1,10 +1,19 @@
-name: SDM Docker Build (Test)
+name: SDM Docker Build
 
 on:
   push:
     branches:
       - main
+    paths:
+      - 'airbyte_cdk/**'  # Any CDK changes could affect SDM
+      - '.github/workflows/docker-build.yml'
+      - 'Dockerfile'
   workflow_dispatch:
+    inputs:
+      version-tag:
+        description: "Version tag for the image (e.g. 0.1.0)"
+        required: false
+        type: string
 
 jobs:
   docker_build:
@@ -28,10 +37,13 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}  # Only push on main
-          tags: airbyte/source-declarative-manifest:test-${{ github.sha }}
+          push: true
+          tags: |
+            airbyte/source-declarative-manifest:latest
+            airbyte/source-declarative-manifest:${{ github.sha }}
+            ${{ github.event.inputs.version-tag != '' && format('airbyte/source-declarative-manifest:{0}', github.event.inputs.version-tag) || '' }}
 
       - name: Test image
         run: |
-          docker pull airbyte/source-declarative-manifest:test-${{ github.sha }}
-          docker run airbyte/source-declarative-manifest:test-${{ github.sha }} spec
+          docker pull airbyte/source-declarative-manifest:${{ github.sha }}
+          docker run airbyte/source-declarative-manifest:${{ github.sha }} spec

From 87c06eaf5192d3813498c8e9fcc997e6a2b7b92b Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Thu, 14 Nov 2024 14:31:10 -0800
Subject: [PATCH 09/28] refactor docker-build action

---
 .github/workflows/docker-build.yml | 52 +++++++++++++++++++-----------
 .github/workflows/pypi_publish.yml | 22 -------------
 2 files changed, 33 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index d852613a..286c7e2a 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -5,45 +5,59 @@ on:
     branches:
       - main
     paths:
-      - 'airbyte_cdk/**'  # Any CDK changes could affect SDM
+      - 'airbyte_cdk/**'
       - '.github/workflows/docker-build.yml'
       - 'Dockerfile'
   workflow_dispatch:
     inputs:
       version-tag:
-        description: "Version tag for the image (e.g. 0.1.0)"
+        description: "Version tag for the image (optional)"
         required: false
         type: string
 
 jobs:
   docker_build:
-    name: Build and Publish source-declarative-manifest Docker Image
+    name: Build and Publish SDM Docker Image
     runs-on: ubuntu-latest
     permissions:
-      id-token: write
-      contents: write
+      id-token: write  # Required for trusted publishing
+      contents: write  # Required for artifact uploads
+
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      
+
+      - name: Build Docker image
+        run: |
+          docker build -t airbyte/source-declarative-manifest:build-test .
+
+      - name: Test image
+        run: |
+          docker run airbyte/source-declarative-manifest:build-test spec
+
       - name: Login to Docker Hub
+        if: ${{ success() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: |
-            airbyte/source-declarative-manifest:latest
-            airbyte/source-declarative-manifest:${{ github.sha }}
-            ${{ github.event.inputs.version-tag != '' && format('airbyte/source-declarative-manifest:{0}', github.event.inputs.version-tag) || '' }}
-
-      - name: Test image
+      - name: Push to Docker Hub
+        if: ${{ success() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
         run: |
-          docker pull airbyte/source-declarative-manifest:${{ github.sha }}
-          docker run airbyte/source-declarative-manifest:${{ github.sha }} spec
+          # Always tag with commit SHA
+          docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:${{ github.sha }}
+          docker push airbyte/source-declarative-manifest:${{ github.sha }}
+          
+          # Tag as latest if on main branch
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:latest
+            docker push airbyte/source-declarative-manifest:latest
+          fi
+          
+          # Add version tag if provided
+          if [[ -n "${{ github.event.inputs.version-tag }}" ]]; then
+            docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:${{ github.event.inputs.version-tag }}
+            docker push airbyte/source-declarative-manifest:${{ github.event.inputs.version-tag }}
+          fi
diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml
index 777d4971..c1af4e6c 100644
--- a/.github/workflows/pypi_publish.yml
+++ b/.github/workflows/pypi_publish.yml
@@ -45,25 +45,3 @@ jobs:
         with:
           # Can be toggled at the repository level between `https://upload.pypi.org/legacy/` and `https://test.pypi.org/legacy/`
           repository-url: ${{vars.PYPI_PUBLISH_URL}}
-
-  # TODO: Need to test this on the other workflow first, then update here when we have
-  #       something that works.
-  docker_publish:
-    name: Build and Publish `source-declarative-manifest` Docker Image
-    runs-on: ubuntu-latest
-    needs: [build]
-    permissions:
-      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
-      contents: write # Needed to upload artifacts to the release
-    environment:
-      name: Docker Hub
-      url: "https://hub.docker.com/r/airbytehq/airbyte-cdk"
-    env:
-      # TODO: Clean this up with a real version tag, remove the 'refs/tags/v' prefix
-      VERSION: ${{ github.ref }}
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - run: |
-          docker login
-          docker build -t airbyte/source-declarative-manifest:${VERSION} .
-          docker push airbyte/source-declarative-manifest:${VERSION}

From b5be82ca6ea127797d6e39ba405c668921a12cab Mon Sep 17 00:00:00 2001
From: "Aaron (\"AJ\") Steers" <aj@airbyte.io>
Date: Thu, 14 Nov 2024 17:31:08 -0800
Subject: [PATCH 10/28] Apply suggestions from code review

---
 .github/workflows/connector-tests.yml              |  4 ++--
 .../cli/source_declarative_manifest/_run.py        | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml
index 83574245..14cfb6cd 100644
--- a/.github/workflows/connector-tests.yml
+++ b/.github/workflows/connector-tests.yml
@@ -81,8 +81,8 @@ jobs:
             cdk_extra: vector-db-based
           - connector: destination-motherduck
             cdk_extra: sql
-          # TODO: These are manifest connectors and won't work as expectd until we
-          #       add `--use-local-cdk` support for manifest connectors
+          # TODO: These are manifest connectors and won't work as expected until we
+          #       add `--use-local-cdk` support for manifest connectors.
           - connector: source-the-guardian-api
             cdk_extra: n/a
           - connector: source-pokeapi
diff --git a/airbyte_cdk/cli/source_declarative_manifest/_run.py b/airbyte_cdk/cli/source_declarative_manifest/_run.py
index c3d399ae..b5780631 100644
--- a/airbyte_cdk/cli/source_declarative_manifest/_run.py
+++ b/airbyte_cdk/cli/source_declarative_manifest/_run.py
@@ -1,4 +1,18 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
+"""Defines the `source-declarative-manifest` connector, which installs alongside CDK.
+
+This file was originally imported from the dedicated connector directory, under the
+`airbyte` monorepo.
+
+Usage:
+
+```
+pipx install airbyte-cdk
+source-declarative-manifest --help
+source-declarative-manifest spec
+...
+```
+"""
 
 
 from __future__ import annotations

From 089cf02dc106b18a3855e91e6e6e2dcd97879d15 Mon Sep 17 00:00:00 2001
From: octavia-squidington-iii <contact@airbyte.com>
Date: Fri, 15 Nov 2024 05:46:19 +0000
Subject: [PATCH 11/28] Auto-fix lint and format issues

---
 airbyte_cdk/cli/source_declarative_manifest/_run.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/airbyte_cdk/cli/source_declarative_manifest/_run.py b/airbyte_cdk/cli/source_declarative_manifest/_run.py
index b5780631..9e19c629 100644
--- a/airbyte_cdk/cli/source_declarative_manifest/_run.py
+++ b/airbyte_cdk/cli/source_declarative_manifest/_run.py
@@ -14,7 +14,6 @@
 ```
 """
 
-
 from __future__ import annotations
 
 import json

From 176c901e186efd3df69d0929beb974e50bdca8af Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Thu, 14 Nov 2024 21:59:25 -0800
Subject: [PATCH 12/28] add this branch for testing docker builds

---
 .github/workflows/docker-build.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 286c7e2a..0e014544 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - christo/sdm-test (dev/test branch)
     paths:
       - 'airbyte_cdk/**'
       - '.github/workflows/docker-build.yml'
@@ -49,13 +50,13 @@ jobs:
           # Always tag with commit SHA
           docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:${{ github.sha }}
           docker push airbyte/source-declarative-manifest:${{ github.sha }}
-          
+
           # Tag as latest if on main branch
           if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
             docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:latest
             docker push airbyte/source-declarative-manifest:latest
           fi
-          
+
           # Add version tag if provided
           if [[ -n "${{ github.event.inputs.version-tag }}" ]]; then
             docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:${{ github.event.inputs.version-tag }}

From 84ba231f28fa6e7689446021e35c0d3a6f93bb05 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Thu, 14 Nov 2024 22:03:05 -0800
Subject: [PATCH 13/28] fix  comment syntax

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 0e014544..be40eaaf 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - main
-      - christo/sdm-test (dev/test branch)
+      - christo/sdm-test # (dev/test branch)
     paths:
       - 'airbyte_cdk/**'
       - '.github/workflows/docker-build.yml'

From c8ff5363fe93e77ee73e526855c0542565a94881 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Thu, 14 Nov 2024 22:13:16 -0800
Subject: [PATCH 14/28] full test on dev branch

---
 .github/workflows/docker-build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index be40eaaf..e3b90cb5 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -38,14 +38,14 @@ jobs:
           docker run airbyte/source-declarative-manifest:build-test spec
 
       - name: Login to Docker Hub
-        if: ${{ success() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+        if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/christo/sdm-test' || github.event_name == 'workflow_dispatch') }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Push to Docker Hub
-        if: ${{ success() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+        if: ${{ success() && (github.ref == 'refs/heads/main'  || github.ref == 'refs/heads/christo/sdm-test'|| github.event_name == 'workflow_dispatch') }}
         run: |
           # Always tag with commit SHA
           docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:${{ github.sha }}

From 374e4198c0a11095b0adad32471214b564bbeab7 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Thu, 14 Nov 2024 22:22:31 -0800
Subject: [PATCH 15/28] fix secrets names

---
 .github/workflows/docker-build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index e3b90cb5..176d755f 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -41,8 +41,8 @@ jobs:
         if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/christo/sdm-test' || github.event_name == 'workflow_dispatch') }}
         uses: docker/login-action@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
       - name: Push to Docker Hub
         if: ${{ success() && (github.ref == 'refs/heads/main'  || github.ref == 'refs/heads/christo/sdm-test'|| github.event_name == 'workflow_dispatch') }}

From 0137ba28445a598683887a1f844c4c5df6aaa763 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Thu, 14 Nov 2024 22:44:58 -0800
Subject: [PATCH 16/28] fix linting/typing issues

---
 .../cli/source_declarative_manifest/_run.py   | 48 ++++++++++++-------
 .../declarative/yaml_declarative_source.py    |  3 +-
 2 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/airbyte_cdk/cli/source_declarative_manifest/_run.py b/airbyte_cdk/cli/source_declarative_manifest/_run.py
index 9e19c629..e871ad71 100644
--- a/airbyte_cdk/cli/source_declarative_manifest/_run.py
+++ b/airbyte_cdk/cli/source_declarative_manifest/_run.py
@@ -20,9 +20,10 @@
 import pkgutil
 import sys
 import traceback
+from collections.abc import Mapping
 from datetime import datetime
 from pathlib import Path
-from typing import Any, List, Mapping, Optional
+from typing import Any
 
 from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.models import (
@@ -51,11 +52,11 @@ class SourceLocalYaml(YamlDeclarativeSource):
 
     def __init__(
         self,
-        catalog: Optional[ConfiguredAirbyteCatalog],
-        config: Optional[Mapping[str, Any]],
+        catalog: ConfiguredAirbyteCatalog | None,
+        config: Mapping[str, Any] | None,
         state: TState,
-        **kwargs,
-    ):
+        **kwargs: Any,
+    ) -> None:
         """
         HACK!
             Problem: YamlDeclarativeSource relies on the calling module name/path to find the yaml file.
@@ -68,23 +69,26 @@ def __init__(
                 When all manifest connectors are updated to use the source-declarative-manifest as the base image.
         """
         super().__init__(
-            catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"}
+            catalog=catalog,
+            config=config,
+            state=state,
+            path_to_yaml="manifest.yaml",
         )
 
 
-def _is_local_manifest_command(args: List[str]) -> bool:
+def _is_local_manifest_command(args: list[str]) -> bool:
     # Check for a local manifest.yaml file
     return Path("/airbyte/integration_code/source_declarative_manifest/manifest.yaml").exists()
 
 
-def handle_command(args: List[str]) -> None:
+def handle_command(args: list[str]) -> None:
     if _is_local_manifest_command(args):
         handle_local_manifest_command(args)
     else:
         handle_remote_manifest_command(args)
 
 
-def _get_local_yaml_source(args: List[str]) -> SourceLocalYaml:
+def _get_local_yaml_source(args: list[str]) -> SourceLocalYaml:
     try:
         config, catalog, state = _parse_inputs_into_config_catalog_state(args)
         return SourceLocalYaml(config=config, catalog=catalog, state=state)
@@ -109,12 +113,15 @@ def _get_local_yaml_source(args: List[str]) -> SourceLocalYaml:
         raise error
 
 
-def handle_local_manifest_command(args: List[str]) -> None:
+def handle_local_manifest_command(args: list[str]) -> None:
     source = _get_local_yaml_source(args)
-    launch(source, args)
+    launch(
+        source=source,
+        args=args,
+    )
 
 
-def handle_remote_manifest_command(args: List[str]) -> None:
+def handle_remote_manifest_command(args: list[str]) -> None:
     """Overrides the spec command to return the generalized spec for the declarative manifest source.
 
     This is different from a typical low-code, but built and published separately source built as a ManifestDeclarativeSource,
@@ -133,10 +140,13 @@ def handle_remote_manifest_command(args: List[str]) -> None:
         print(AirbyteEntrypoint.airbyte_message_to_string(message))
     else:
         source = create_declarative_source(args)
-        launch(source, args)
+        launch(
+            source=source,
+            args=args,
+        )
 
 
-def create_declarative_source(args: List[str]) -> ConcurrentDeclarativeSource:
+def create_declarative_source(args: list[str]) -> ConcurrentDeclarativeSource:
     """Creates the source with the injected config.
 
     This essentially does what other low-code sources do at build time, but at runtime,
@@ -177,8 +187,12 @@ def create_declarative_source(args: List[str]) -> ConcurrentDeclarativeSource:
 
 
 def _parse_inputs_into_config_catalog_state(
-    args: List[str],
-) -> (Optional[Mapping[str, Any]], Optional[ConfiguredAirbyteCatalog], List[AirbyteStateMessage]):
+    args: list[str],
+) -> tuple[
+    Mapping[str, Any] | None,
+    ConfiguredAirbyteCatalog | None,
+    list[AirbyteStateMessage],
+]:
     parsed_args = AirbyteEntrypoint.parse_args(args)
     config = (
         ConcurrentDeclarativeSource.read_config(parsed_args.config)
@@ -200,5 +214,5 @@ def _parse_inputs_into_config_catalog_state(
 
 
 def run() -> None:
-    args = sys.argv[1:]
+    args: list[str] = sys.argv[1:]
     handle_command(args)
diff --git a/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte_cdk/sources/declarative/yaml_declarative_source.py
index cecb91a6..c23f7d47 100644
--- a/airbyte_cdk/sources/declarative/yaml_declarative_source.py
+++ b/airbyte_cdk/sources/declarative/yaml_declarative_source.py
@@ -19,10 +19,11 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
     def __init__(
         self,
         path_to_yaml: str,
-        debug: bool = False,
         catalog: Optional[ConfiguredAirbyteCatalog] = None,
         config: Optional[Mapping[str, Any]] = None,
         state: Optional[List[AirbyteStateMessage]] = None,
+        *,
+        debug: bool = False,
     ) -> None:
         """
         :param path_to_yaml: Path to the yaml file describing the source

From 78f770d5b1eb091c03dc212817fb1ef1256f0587 Mon Sep 17 00:00:00 2001
From: Aaron Steers <aj@airbyte.io>
Date: Thu, 14 Nov 2024 22:49:48 -0800
Subject: [PATCH 17/28] fix more mypy issues

---
 airbyte_cdk/cli/source_declarative_manifest/_run.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/airbyte_cdk/cli/source_declarative_manifest/_run.py b/airbyte_cdk/cli/source_declarative_manifest/_run.py
index e871ad71..ece30309 100644
--- a/airbyte_cdk/cli/source_declarative_manifest/_run.py
+++ b/airbyte_cdk/cli/source_declarative_manifest/_run.py
@@ -23,7 +23,7 @@
 from collections.abc import Mapping
 from datetime import datetime
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch
 from airbyte_cdk.models import (
@@ -133,6 +133,11 @@ def handle_remote_manifest_command(args: list[str]) -> None:
             "airbyte_cdk.cli.source_declarative_manifest",
             "spec.json",
         )
+        if json_spec is None:
+            raise FileNotFoundError(
+                "Could not find `spec.json` file for source-declarative-manifest"
+            )
+
         spec_obj = json.loads(json_spec)
         spec = ConnectorSpecificationSerializer.load(spec_obj)
 
@@ -163,7 +168,7 @@ def create_declarative_source(args: list[str]) -> ConcurrentDeclarativeSource:
             config=config,
             catalog=catalog,
             state=state,
-            source_config=config.get("__injected_declarative_manifest"),
+            source_config=cast(dict[str, Any], config["__injected_declarative_manifest"]),
         )
     except Exception as error:
         print(

From dd01890f5769b6b93add941d6744aa6b6974cf76 Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Mon, 18 Nov 2024 11:44:59 -0800
Subject: [PATCH 18/28] add multi-arch build and vulnerability scanning

---
 .github/workflows/docker-build.yml | 56 ++++++++++++++++++------------
 1 file changed, 34 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 176d755f..2a8a2733 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -1,4 +1,4 @@
-name: SDM Docker Build
+name: Source Declarative Manifest - Docker Build
 
 on:
   push:
@@ -29,13 +29,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Build Docker image
-        run: |
-          docker build -t airbyte/source-declarative-manifest:build-test .
+      - name: Set up QEMU for mulit-platform builds
+        uses: docker/setup-qemu-action@v3
 
-      - name: Test image
-        run: |
-          docker run airbyte/source-declarative-manifest:build-test spec
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to Docker Hub
         if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/christo/sdm-test' || github.event_name == 'workflow_dispatch') }}
@@ -44,21 +42,35 @@ jobs:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
-      - name: Push to Docker Hub
-        if: ${{ success() && (github.ref == 'refs/heads/main'  || github.ref == 'refs/heads/christo/sdm-test'|| github.event_name == 'workflow_dispatch') }}
+      - name: Build test image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          load: true
+          tags: airbyte/source-declarative-manifest:build-test
+
+      - name: Test image
         run: |
-          # Always tag with commit SHA
-          docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:${{ github.sha }}
-          docker push airbyte/source-declarative-manifest:${{ github.sha }}
+          docker run airbyte/source-declarative-manifest:build-test spec 
 
-          # Tag as latest if on main branch
-          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
-            docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:latest
-            docker push airbyte/source-declarative-manifest:latest
-          fi
+      - name: Scan for vulnerabilities
+        uses: aquasecurity/trivy-action@master
+        continue-on-error: true # Prevent security scan from failing the build
+        with:
+          image-ref: airbyte/source-declarative-manifest:build-test
+          format: table
+          exit-code: 1
+          severity: CRITICAL
 
-          # Add version tag if provided
-          if [[ -n "${{ github.event.inputs.version-tag }}" ]]; then
-            docker tag airbyte/source-declarative-manifest:build-test airbyte/source-declarative-manifest:${{ github.event.inputs.version-tag }}
-            docker push airbyte/source-declarative-manifest:${{ github.event.inputs.version-tag }}
-          fi
+      - name: Build and push
+        if: ${{ success() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/source-declarative-manifest:${{ github.sha }}
+            ${{ github.ref == 'refs/heads/main' && 'airbyte/source-declarative-manifest:latest' || '' }}
+            ${{ github.event.inputs.version-tag != '' && format('airbyte/source-declarative-manifest:{0}', github.event.inputs.version-tag) || '' }}

From c66b05e781e8aca6d84ddca5819e75c093b30b9d Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Mon, 18 Nov 2024 12:01:19 -0800
Subject: [PATCH 19/28] chore: add test branch to publish step

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 2a8a2733..328fb560 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -64,7 +64,7 @@ jobs:
           severity: CRITICAL
 
       - name: Build and push
-        if: ${{ success() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+        if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == "refs/head/christo/sdm-test" || github.event_name == 'workflow_dispatch') }}
         uses: docker/build-push-action@v5
         with:
           context: .

From a4a52e00d90acf4d79ae0ba191f08c9a0459d3bb Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Mon, 18 Nov 2024 12:06:49 -0800
Subject: [PATCH 20/28] chore: fix double-quotes in build step

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 328fb560..c4cebdd3 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -64,7 +64,7 @@ jobs:
           severity: CRITICAL
 
       - name: Build and push
-        if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == "refs/head/christo/sdm-test" || github.event_name == 'workflow_dispatch') }}
+        if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/head/christo/sdm-test' || github.event_name == 'workflow_dispatch') }}
         uses: docker/build-push-action@v5
         with:
           context: .

From 8e6f1a7c92ffe68c1e6bf288499c222bba649652 Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Mon, 18 Nov 2024 12:12:21 -0800
Subject: [PATCH 21/28] chore: define single architecture for test build

---
 .github/workflows/docker-build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index c4cebdd3..7861c8db 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -46,7 +46,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64 # Just build for the runner's architecture during test
           load: true
           tags: airbyte/source-declarative-manifest:build-test
 

From 12984d26fe333d2aba0d15d40b385e2425fc95d1 Mon Sep 17 00:00:00 2001
From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com>
Date: Mon, 18 Nov 2024 13:38:39 -0800
Subject: [PATCH 22/28] Apply suggestions from code review

Co-authored-by: Augustin <augustin@airbyte.io>
---
 .github/workflows/docker-build.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 7861c8db..b7787f6d 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -4,10 +4,8 @@ on:
   push:
     branches:
       - main
-      - christo/sdm-test # (dev/test branch)
     paths:
-      - 'airbyte_cdk/**'
-      - '.github/workflows/docker-build.yml'
+      - 'airbyte_cdk/pyproject.toml' # To only publish on CDK version change
       - 'Dockerfile'
   workflow_dispatch:
     inputs:

From 9709c3c57d87aa0fd126039d3f84c2eccfcd69fd Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Mon, 18 Nov 2024 14:01:43 -0800
Subject: [PATCH 23/28] address review comments

---
 .../{docker-build.yml => cdk-publish.yml}     | 74 ++++++++++++-------
 1 file changed, 46 insertions(+), 28 deletions(-)
 rename .github/workflows/{docker-build.yml => cdk-publish.yml} (54%)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/cdk-publish.yml
similarity index 54%
rename from .github/workflows/docker-build.yml
rename to .github/workflows/cdk-publish.yml
index b7787f6d..d4c7d367 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/cdk-publish.yml
@@ -1,45 +1,26 @@
-name: Source Declarative Manifest - Docker Build
-
+name: Publish CDK and Source Declarative Manifest
 on:
   push:
-    branches:
-      - main
     paths:
       - 'airbyte_cdk/pyproject.toml' # To only publish on CDK version change
       - 'Dockerfile'
   workflow_dispatch:
-    inputs:
-      version-tag:
-        description: "Version tag for the image (optional)"
-        required: false
-        type: string
 
 jobs:
-  docker_build:
-    name: Build and Publish SDM Docker Image
+  test:
+    name: Test Source Declarative Manifest Docker Build
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write  # Required for trusted publishing
-      contents: write  # Required for artifact uploads
-
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Set up QEMU for mulit-platform builds
+      - name: Set up QEMU for multi-platform builds
         uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to Docker Hub
-        if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/christo/sdm-test' || github.event_name == 'workflow_dispatch') }}
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
       - name: Build test image
         uses: docker/build-push-action@v5
         with:
@@ -50,7 +31,7 @@ jobs:
 
       - name: Test image
         run: |
-          docker run airbyte/source-declarative-manifest:build-test spec 
+          docker run airbyte/source-declarative-manifest:build-test spec
 
       - name: Scan for vulnerabilities
         uses: aquasecurity/trivy-action@master
@@ -61,14 +42,51 @@ jobs:
           exit-code: 1
           severity: CRITICAL
 
+  publish:
+    name: Publish SDM Docker Image
+    runs-on: ubuntu-latest
+    needs: test
+    if: ${{ success() && (github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch') }}
+    permissions:
+      id-token: write  # Required for trusted publishing
+      contents: write  # Required for artifact uploads
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU for multi-platform builds
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Get CDK version
+        run: |
+          cdk_version=$(poetry version --short)
+          echo "CDK_VERSION=$cdk_version" >> $GITHUB_ENV
+
+      - name: Check if tag already exists
+        run: |
+          tag="airbyte/source-declarative-manifest:${{ env.CDK_VERSION}}-${{ github.run_number }}"
+          if docker manifest inspect $tag > /dev/null 2>&1; then
+            echo "The tag $tag already exists on Dockerhub. Skipping publish to prevent overwrite."
+            exit 1
+          fi
+
       - name: Build and push
-        if: ${{ success() && (github.ref == 'refs/heads/main' || github.ref == 'refs/head/christo/sdm-test' || github.event_name == 'workflow_dispatch') }}
         uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            airbyte/source-declarative-manifest:${{ github.sha }}
-            ${{ github.ref == 'refs/heads/main' && 'airbyte/source-declarative-manifest:latest' || '' }}
-            ${{ github.event.inputs.version-tag != '' && format('airbyte/source-declarative-manifest:{0}', github.event.inputs.version-tag) || '' }}
+            airbyte/source-declarative-manifest:latest
+            airbyte/source-declarative-manifest:${{ env.CDK_VERSION }}
+            airbyte/source-declarative-manifest:${{ env.CDK_VERSION }}-${{ github.run_number }}

From 8d0ffc9b1e81cdd796aa9e12f7ae1c5476210129 Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Mon, 18 Nov 2024 14:43:30 -0800
Subject: [PATCH 24/28] chore: update lockfile

---
 poetry.lock | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 00ab177d..fc666274 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1841,8 +1841,8 @@ files = [
 httpx = ">=0.23.0,<1"
 orjson = ">=3.9.14,<4.0.0"
 pydantic = [
-    {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""},
     {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""},
+    {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""},
 ]
 requests = ">=2,<3"
 requests-toolbelt = ">=1.0.0,<2.0.0"
@@ -2720,9 +2720,9 @@ files = [
 
 [package.dependencies]
 numpy = [
-    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
     {version = ">=1.22.4", markers = "python_version < \"3.11\""},
     {version = ">=1.23.2", markers = "python_version == \"3.11\""},
+    {version = ">=1.26.0", markers = "python_version >= \"3.12\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -3240,10 +3240,7 @@ files = [
 [package.dependencies]
 annotated-types = ">=0.6.0"
 pydantic-core = "2.23.4"
-typing-extensions = [
-    {version = ">=4.12.2", markers = "python_version >= \"3.13\""},
-    {version = ">=4.6.1", markers = "python_version < \"3.13\""},
-]
+typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""}
 
 [package.extras]
 email = ["email-validator (>=2.0.0)"]
@@ -4226,6 +4223,11 @@ files = [
     {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f60021ec1574e56632be2a36b946f8143bf4e5e6af4a06d85281adc22938e0dd"},
     {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:394397841449853c2290a32050382edaec3da89e35b3e03d6cc966aebc6a8ae6"},
     {file = "scikit_learn-1.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:57cc1786cfd6bd118220a92ede80270132aa353647684efa385a74244a41e3b1"},
+    {file = "scikit_learn-1.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9a702e2de732bbb20d3bad29ebd77fc05a6b427dc49964300340e4c9328b3f5"},
+    {file = "scikit_learn-1.5.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:b0768ad641981f5d3a198430a1d31c3e044ed2e8a6f22166b4d546a5116d7908"},
+    {file = "scikit_learn-1.5.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:178ddd0a5cb0044464fc1bfc4cca5b1833bfc7bb022d70b05db8530da4bb3dd3"},
+    {file = "scikit_learn-1.5.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7284ade780084d94505632241bf78c44ab3b6f1e8ccab3d2af58e0e950f9c12"},
+    {file = "scikit_learn-1.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:b7b0f9a0b1040830d38c39b91b3a44e1b643f4b36e36567b80b7c6bd2202a27f"},
     {file = "scikit_learn-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:757c7d514ddb00ae249832fe87100d9c73c6ea91423802872d9e74970a0e40b9"},
     {file = "scikit_learn-1.5.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:52788f48b5d8bca5c0736c175fa6bdaab2ef00a8f536cda698db61bd89c551c1"},
     {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:643964678f4b5fbdc95cbf8aec638acc7aa70f5f79ee2cdad1eec3df4ba6ead8"},
@@ -5234,4 +5236,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10,<3.13"
-content-hash = "e7ecc6f9875e1403a581a81c13595e4fed001e649face2bc3e466aa676a71fae"
+content-hash = "e7ba21b5836e45357136ba5eff70d137327fdb6d20a64bcfee9423ce447774e5"

From bc5de0d796381de6f5de674b9db6c118edbb8bcf Mon Sep 17 00:00:00 2001
From: ChristoGrab <christo.grab@gmail.com>
Date: Mon, 18 Nov 2024 15:27:53 -0800
Subject: [PATCH 25/28] revert change to yaml_declarative_source

---
 airbyte_cdk/sources/declarative/yaml_declarative_source.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/airbyte_cdk/sources/declarative/yaml_declarative_source.py b/airbyte_cdk/sources/declarative/yaml_declarative_source.py
index c23f7d47..cecb91a6 100644
--- a/airbyte_cdk/sources/declarative/yaml_declarative_source.py
+++ b/airbyte_cdk/sources/declarative/yaml_declarative_source.py
@@ -19,11 +19,10 @@ class YamlDeclarativeSource(ConcurrentDeclarativeSource[List[AirbyteStateMessage
     def __init__(
         self,
         path_to_yaml: str,
+        debug: bool = False,
         catalog: Optional[ConfiguredAirbyteCatalog] = None,
         config: Optional[Mapping[str, Any]] = None,
         state: Optional[List[AirbyteStateMessage]] = None,
-        *,
-        debug: bool = False,
     ) -> None:
         """
         :param path_to_yaml: Path to the yaml file describing the source

From 3b866029d7c29725e72adea7cb70695e7d043713 Mon Sep 17 00:00:00 2001
From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com>
Date: Mon, 18 Nov 2024 18:33:27 -0500
Subject: [PATCH 26/28] Update .github/workflows/cdk-publish.yml

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
---
 .github/workflows/cdk-publish.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-publish.yml b/.github/workflows/cdk-publish.yml
index d4c7d367..faa6cf77 100644
--- a/.github/workflows/cdk-publish.yml
+++ b/.github/workflows/cdk-publish.yml
@@ -69,7 +69,7 @@ jobs:
 
       - name: Get CDK version
         run: |
-          cdk_version=$(poetry version --short)
+          cdk_version="$(poetry version --short)"
           echo "CDK_VERSION=$cdk_version" >> $GITHUB_ENV
 
       - name: Check if tag already exists

From 87acda06e577edd3c0209bc434158df766c72889 Mon Sep 17 00:00:00 2001
From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com>
Date: Mon, 18 Nov 2024 18:39:34 -0500
Subject: [PATCH 27/28] Update .github/workflows/cdk-publish.yml

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
---
 .github/workflows/cdk-publish.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cdk-publish.yml b/.github/workflows/cdk-publish.yml
index faa6cf77..28356afc 100644
--- a/.github/workflows/cdk-publish.yml
+++ b/.github/workflows/cdk-publish.yml
@@ -38,9 +38,11 @@ jobs:
         continue-on-error: true # Prevent security scan from failing the build
         with:
           image-ref: airbyte/source-declarative-manifest:build-test
-          format: table
+          format: 'table,sarif'
+          output: 'trivy-results.sarif'
           exit-code: 1
-          severity: CRITICAL
+          severity: 'CRITICAL,HIGH'
+          timeout: '5m'
 
   publish:
     name: Publish SDM Docker Image

From 7df39551da726e532e8350f51de4818c1da5a821 Mon Sep 17 00:00:00 2001
From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com>
Date: Mon, 18 Nov 2024 18:47:08 -0500
Subject: [PATCH 28/28] Update .github/workflows/cdk-publish.yml

Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com>
---
 .github/workflows/cdk-publish.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/cdk-publish.yml b/.github/workflows/cdk-publish.yml
index 28356afc..2e5ab8e2 100644
--- a/.github/workflows/cdk-publish.yml
+++ b/.github/workflows/cdk-publish.yml
@@ -71,17 +71,16 @@ jobs:
 
       - name: Get CDK version
         run: |
-          cdk_version="$(poetry version --short)"
+          cdk_version="$(poetry version --short | tr -d '[:space:]')"
           echo "CDK_VERSION=$cdk_version" >> $GITHUB_ENV
 
       - name: Check if tag already exists
         run: |
           tag="airbyte/source-declarative-manifest:${{ env.CDK_VERSION}}-${{ github.run_number }}"
-          if docker manifest inspect $tag > /dev/null 2>&1; then
+          if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
             echo "The tag $tag already exists on Dockerhub. Skipping publish to prevent overwrite."
             exit 1
           fi
-
       - name: Build and push
         uses: docker/build-push-action@v5
         with: