Merge branch 'main' into 123-purge-sphinx

airbytehq · Dec 5, 2024 · 33ce2ef · 33ce2ef
2 parents 159e77b + f5e2a65
commit 33ce2ef
Show file tree

Hide file tree

Showing 11 changed files with 37 additions and 20 deletions.
diff --git a/.github/workflows/autofix-command.yml b/.github/workflows/autofix-command.yml
@@ -75,7 +75,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml
@@ -94,6 +94,8 @@ jobs:
           #   cdk_extra: n/a
 
     name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
+    permissions:
+      checks: write
     steps:
       - name: Abort if extra not changed (${{matrix.cdk_extra}})
         id: no_changes
@@ -131,15 +133,18 @@ jobs:
         run: |
           cd airbyte
           make tools.airbyte-ci-dev.install
-          airbyte-ci-dev connectors \
+          airbyte-ci-dev \
+          --ci-report-bucket-name=airbyte-ci-reports-multi \
+          connectors \
           --name ${{matrix.connector}} \
           --use-local-cdk \
           test \
           --fail-fast \
           --skip-step qa_checks \
           --skip-step connector_live_tests
 
-      - name: Evaluate Test Output
+      - name: Evaluate Test
+        id: evaluate_output
         if: always() && steps.no_changes.outputs.status != 'cancelled'
         run: |
           # save job output json file as ci step output
@@ -148,7 +153,9 @@ jobs:
           success=$(echo ${job_output} | jq -r '.success')
           failed_step=$(echo ${job_output} | jq -r '.failed_steps | select(length > 0) | .[0] // "None"')
           run_duration=$(echo ${job_output} | jq -r '.run_duration')
+          html_report_url=$(echo ${job_output} | jq -r '.html_report_url')
           echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY
+          echo "- [HTML Report](${html_report_url})" >> $GITHUB_STEP_SUMMARY
           echo "- Success: ${success}" >> $GITHUB_STEP_SUMMARY
           echo "- Test Duration: $(printf "%.0f" ${run_duration})s" >> $GITHUB_STEP_SUMMARY
           if [ "${success}" != "true" ]; then
@@ -159,6 +166,8 @@ jobs:
             echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. Check the logs for more details."
             exit 1
           fi
+          echo "success=${success}" >> $GITHUB_OUTPUT
+          echo "html_report_url=${html_report_url}" >> $GITHUB_OUTPUT
 
       # Upload the job output to the artifacts
       - name: Upload Job Output

diff --git a/.github/workflows/pdoc_preview.yml b/.github/workflows/pdoc_preview.yml
@@ -16,7 +16,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

diff --git a/.github/workflows/pdoc_publish.yml b/.github/workflows/pdoc_publish.yml
@@ -33,7 +33,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

diff --git a/.github/workflows/poetry-lock-command.yml b/.github/workflows/poetry-lock-command.yml
@@ -72,7 +72,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml
@@ -14,6 +14,14 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Poetry
+        uses: Gr1N/setup-poetry@v9
+        with:
+          poetry-version: "1.8.4"
+
+      - name: Check Poetry lock file is current
+        run: poetry check
+
       - uses: hynek/build-and-inspect-python-package@v2
         env:
           # Pass in dummy version '0.0.0dev0' version to appease dynamic versioning
@@ -36,7 +44,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

diff --git a/.github/workflows/pytest_matrix.yml b/.github/workflows/pytest_matrix.yml
@@ -23,8 +23,8 @@ jobs:
     name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }})
     # Don't run on forks. Run on pushes to main, and on PRs that are not from forks.
     if: >
-      (github.event_name == 'push' && github.ref == 'refs/heads/main') ||
-      (github.event.pull_request.head.repo.fork == false)
+      github.event_name == 'pull_request' ||
+      (github.event_name == 'push' && github.ref == 'refs/heads/main')
     strategy:
       matrix:
         python-version: [
@@ -61,7 +61,7 @@ jobs:
         uses: Gr1N/setup-poetry@v9
         if: steps.changes.outputs.src == 'true'
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         if: steps.changes.outputs.src == 'true'

diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml
@@ -17,7 +17,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -40,7 +40,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -63,7 +63,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -85,7 +85,7 @@ jobs:
       - name: Set up Poetry
         uses: Gr1N/setup-poetry@v9
         with:
-          poetry-version: "1.7.1"
+          poetry-version: "1.8.4"
       - name: Set up Python
         uses: actions/setup-python@v5
         with:

diff --git a/airbyte_cdk/sources/declarative/concurrent_declarative_source.py b/airbyte_cdk/sources/declarative/concurrent_declarative_source.py
@@ -206,7 +206,7 @@ def _group_streams(
             # so we need to treat them as synchronous
             if (
                 isinstance(declarative_stream, DeclarativeStream)
-                and name_to_stream_mapping[declarative_stream.name].get("retriever")["type"]
+                and name_to_stream_mapping[declarative_stream.name]["retriever"]["type"]
                 == "SimpleRetriever"
             ):
                 incremental_sync_component_definition = name_to_stream_mapping[
@@ -215,7 +215,7 @@ def _group_streams(
 
                 partition_router_component_definition = (
                     name_to_stream_mapping[declarative_stream.name]
-                    .get("retriever")
+                    .get("retriever", {})
                     .get("partition_router")
                 )
                 is_without_partition_router_or_cursor = not bool(
@@ -237,7 +237,7 @@ def _group_streams(
                     cursor = self._constructor.create_concurrent_cursor_from_datetime_based_cursor(
                         state_manager=state_manager,
                         model_type=DatetimeBasedCursorModel,
-                        component_definition=incremental_sync_component_definition,
+                        component_definition=incremental_sync_component_definition,  # type: ignore  # Not None because of the if condition above
                         stream_name=declarative_stream.name,
                         stream_namespace=declarative_stream.namespace,
                         config=config or {},
@@ -320,10 +320,11 @@ def _group_streams(
     def _is_datetime_incremental_without_partition_routing(
         self,
         declarative_stream: DeclarativeStream,
-        incremental_sync_component_definition: Mapping[str, Any],
+        incremental_sync_component_definition: Mapping[str, Any] | None,
     ) -> bool:
         return (
-            bool(incremental_sync_component_definition)
+            incremental_sync_component_definition is not None
+            and bool(incremental_sync_component_definition)
             and incremental_sync_component_definition.get("type", "")
             == DatetimeBasedCursorModel.__name__
             and self._stream_supports_concurrent_partition_processing(

diff --git a/poetry.lock b/poetry.lock