Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Publish the response to process as stream_status.response in the transformation's context #190

Closed
wants to merge 3 commits into from
Closed

Feat: Publish the response to process as stream_status.response in the transformation's context #190

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
02cc6c4
Merge branch 'main' of https://github.com/rpopov/airbyte-python-cdk
rpopov Dec 24, 2024
8bfa7ab
#49971 Publish the response object in the transformation's context
rpopov Dec 26, 2024
e239779
50395 Formatted the source code
rpopov Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions airbyte_cdk/sources/declarative/extractors/record_selector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import requests

from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
Expand All @@ -21,6 +22,8 @@
SchemaNormalization.Default: TransformConfig.DefaultSchemaNormalization,
}

STREAM_SLICE_RESPONSE_KEY = "response"


@dataclass
class RecordSelector(HttpSelector):
Expand Down Expand Up @@ -51,6 +54,7 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
if isinstance(self._name, str)
else self._name
)
self.response_root_extractor = DpathExtractor(field_path=[], config={}, parameters={})

@property # type: ignore
def name(self) -> str:
Expand Down Expand Up @@ -86,9 +90,15 @@ def select_records(
:return: List of Records selected from the response
"""
all_data: Iterable[Mapping[str, Any]] = self.extractor.extract_records(response)
yield from self.filter_and_transform(
all_data, stream_state, records_schema, stream_slice, next_page_token
)

response_root_data = self.response_root_extractor.extract_records(response)
stream_state.update({STREAM_SLICE_RESPONSE_KEY: response_root_data})
try:
yield from self.filter_and_transform(
all_data, stream_state, records_schema, stream_slice, next_page_token
)
finally:
stream_state.pop(STREAM_SLICE_RESPONSE_KEY)

def filter_and_transform(
self,
Expand Down
Loading