Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Publish the response to process as stream_status.response in the transformation's context #191

Closed
wants to merge 4 commits into from
+26 −7
Closed

Feat: Publish the response to process as stream_status.response in the transformation's context #191

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
02cc6c4
Merge branch 'main' of https://github.com/rpopov/airbyte-python-cdk
rpopov Dec 24, 2024
a7b010b
Feat: Publish the response to process as stream_status.response in th…
rpopov Dec 26, 2024
fabdf7c
Resolved mypy's complain on next() use on iterable by building an ite…
rpopov Dec 26, 2024
c7f16eb
Used a dictionary instead of StreamState and mypy agreed.
rpopov Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions airbyte_cdk/sources/declarative/extractors/record_selector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import requests

from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector
from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter
Expand All @@ -21,6 +22,8 @@
SchemaNormalization.Default: TransformConfig.DefaultSchemaNormalization,
}

STREAM_SLICE_RESPONSE_ROOT_KEY = "response"


@dataclass
class RecordSelector(HttpSelector):
Expand Down Expand Up @@ -51,6 +54,7 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
if isinstance(self._name, str)
else self._name
)
self.response_root_extractor = DpathExtractor(field_path=[], config={}, parameters={})

@property # type: ignore
def name(self) -> str:
Expand Down Expand Up @@ -86,9 +90,15 @@ def select_records(
:return: List of Records selected from the response
"""
all_data: Iterable[Mapping[str, Any]] = self.extractor.extract_records(response)
yield from self.filter_and_transform(
all_data, stream_state, records_schema, stream_slice, next_page_token
)

response_root_iterator = self.response_root_extractor.extract_records(response)
stream_state.update({STREAM_SLICE_RESPONSE_ROOT_KEY: next(response_root_iterator, None)})
try:
yield from self.filter_and_transform(
all_data, stream_state, records_schema, stream_slice, next_page_token
)
finally:
stream_state.pop(STREAM_SLICE_RESPONSE_ROOT_KEY)

def filter_and_transform(
self,
Expand Down
13 changes: 13 additions & 0 deletions unit_tests/sources/declarative/extractors/test_record_selector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,19 @@
[],
[],
),
(
"test_the original response is available in filters and transformations",
["data"],
"{{ record['created_at'] == stream_state.response.data[1].created_at }}",
{
"data": [
{"id": 1, "created_at": "06-06-21"},
{"id": 2, "created_at": "06-07-21"},
{"id": 3, "created_at": "06-08-21"},
]
},
[{"id": 2, "created_at": "06-07-21"}],
),
],
)
def test_record_filter(test_name, field_path, filter_template, body, expected_data):
Expand Down
Loading