diff --git a/.github/workflows/autofix-command.yml b/.github/workflows/autofix-command.yml index 03ec62c0..f438a53c 100644 --- a/.github/workflows/autofix-command.yml +++ b/.github/workflows/autofix-command.yml @@ -75,7 +75,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 0f7033f1..666b722e 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -94,6 +94,8 @@ jobs: # cdk_extra: n/a name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" + permissions: + checks: write steps: - name: Abort if extra not changed (${{matrix.cdk_extra}}) id: no_changes @@ -131,7 +133,9 @@ jobs: run: | cd airbyte make tools.airbyte-ci-dev.install - airbyte-ci-dev connectors \ + airbyte-ci-dev \ + --ci-report-bucket-name=airbyte-ci-reports-multi \ + connectors \ --name ${{matrix.connector}} \ --use-local-cdk \ test \ @@ -139,7 +143,8 @@ jobs: --skip-step qa_checks \ --skip-step connector_live_tests - - name: Evaluate Test Output + - name: Evaluate Test + id: evaluate_output if: always() && steps.no_changes.outputs.status != 'cancelled' run: | # save job output json file as ci step output @@ -148,7 +153,9 @@ jobs: success=$(echo ${job_output} | jq -r '.success') failed_step=$(echo ${job_output} | jq -r '.failed_steps | select(length > 0) | .[0] // "None"') run_duration=$(echo ${job_output} | jq -r '.run_duration') + html_report_url=$(echo ${job_output} | jq -r '.html_report_url') echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY + echo "- [HTML Report](${html_report_url})" >> $GITHUB_STEP_SUMMARY echo "- Success: ${success}" >> $GITHUB_STEP_SUMMARY echo "- Test Duration: $(printf "%.0f" ${run_duration})s" >> $GITHUB_STEP_SUMMARY if [ "${success}" != "true" ]; then @@ -159,6 +166,8 @@ jobs: echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. Check the logs for more details." exit 1 fi + echo "success=${success}" >> $GITHUB_OUTPUT + echo "html_report_url=${html_report_url}" >> $GITHUB_OUTPUT # Upload the job output to the artifacts - name: Upload Job Output diff --git a/.github/workflows/pdoc_preview.yml b/.github/workflows/pdoc_preview.yml index c260e458..3a7f6b25 100644 --- a/.github/workflows/pdoc_preview.yml +++ b/.github/workflows/pdoc_preview.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pdoc_publish.yml b/.github/workflows/pdoc_publish.yml index fc4107b3..0af8fe4e 100644 --- a/.github/workflows/pdoc_publish.yml +++ b/.github/workflows/pdoc_publish.yml @@ -33,7 +33,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/poetry-lock-command.yml b/.github/workflows/poetry-lock-command.yml index 6a968874..a905a837 100644 --- a/.github/workflows/poetry-lock-command.yml +++ b/.github/workflows/poetry-lock-command.yml @@ -72,7 +72,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml index d3cb4a8d..2beebb95 100644 --- a/.github/workflows/pytest_fast.yml +++ b/.github/workflows/pytest_fast.yml @@ -14,6 +14,14 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.8.4" + + - name: Check Poetry lock file is current + run: poetry check + - uses: hynek/build-and-inspect-python-package@v2 env: # Pass in dummy version '0.0.0dev0' version to appease dynamic versioning @@ -36,7 +44,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/pytest_matrix.yml b/.github/workflows/pytest_matrix.yml index fce75a27..bc767e19 100644 --- a/.github/workflows/pytest_matrix.yml +++ b/.github/workflows/pytest_matrix.yml @@ -23,8 +23,8 @@ jobs: name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }}) # Don't run on forks. Run on pushes to main, and on PRs that are not from forks. if: > - (github.event_name == 'push' && github.ref == 'refs/heads/main') || - (github.event.pull_request.head.repo.fork == false) + github.event_name == 'pull_request' || + (github.event_name == 'push' && github.ref == 'refs/heads/main') strategy: matrix: python-version: [ @@ -61,7 +61,7 @@ jobs: uses: Gr1N/setup-poetry@v9 if: steps.changes.outputs.src == 'true' with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 if: steps.changes.outputs.src == 'true' diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index 6dbc6b1c..f92ae7f0 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: @@ -40,7 +40,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: @@ -63,7 +63,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index ed469441..47023436 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -85,7 +85,7 @@ jobs: - name: Set up Poetry uses: Gr1N/setup-poetry@v9 with: - poetry-version: "1.7.1" + poetry-version: "1.8.4" - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/airbyte_cdk/sources/declarative/concurrent_declarative_source.py b/airbyte_cdk/sources/declarative/concurrent_declarative_source.py index 371e34a8..0d3a176c 100644 --- a/airbyte_cdk/sources/declarative/concurrent_declarative_source.py +++ b/airbyte_cdk/sources/declarative/concurrent_declarative_source.py @@ -206,7 +206,7 @@ def _group_streams( # so we need to treat them as synchronous if ( isinstance(declarative_stream, DeclarativeStream) - and name_to_stream_mapping[declarative_stream.name].get("retriever")["type"] + and name_to_stream_mapping[declarative_stream.name]["retriever"]["type"] == "SimpleRetriever" ): incremental_sync_component_definition = name_to_stream_mapping[ @@ -215,7 +215,7 @@ def _group_streams( partition_router_component_definition = ( name_to_stream_mapping[declarative_stream.name] - .get("retriever") + .get("retriever", {}) .get("partition_router") ) is_without_partition_router_or_cursor = not bool( @@ -237,7 +237,7 @@ def _group_streams( cursor = self._constructor.create_concurrent_cursor_from_datetime_based_cursor( state_manager=state_manager, model_type=DatetimeBasedCursorModel, - component_definition=incremental_sync_component_definition, + component_definition=incremental_sync_component_definition, # type: ignore # Not None because of the if condition above stream_name=declarative_stream.name, stream_namespace=declarative_stream.namespace, config=config or {}, @@ -320,10 +320,11 @@ def _group_streams( def _is_datetime_incremental_without_partition_routing( self, declarative_stream: DeclarativeStream, - incremental_sync_component_definition: Mapping[str, Any], + incremental_sync_component_definition: Mapping[str, Any] | None, ) -> bool: return ( - bool(incremental_sync_component_definition) + incremental_sync_component_definition is not None + and bool(incremental_sync_component_definition) and incremental_sync_component_definition.get("type", "") == DatetimeBasedCursorModel.__name__ and self._stream_supports_concurrent_partition_processing( diff --git a/poetry.lock b/poetry.lock index 07ddb189..44b500df 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5047,4 +5047,3 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.0" python-versions = "^3.10,<3.13" -content-hash = "2b774b097cd8c896e59f37d0ca640503b324ab8aebe8f3ba9b5fc2dfd4b63049"