Merge branch 'main' into davinchia/is_resumable

airbytehq · Apr 17, 2024 · 756457b · 756457b
2 parents eadfbf6 + b93c4fd
commit 756457b
Show file tree

Hide file tree

Showing 8 changed files with 192 additions and 4 deletions.
diff --git a/.env b/.env
@@ -1,3 +1,3 @@
 # x-release-please-start-version
-VERSION=0.5.1
+VERSION=0.9.0
 # x-release-please-end
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,47 @@
 # Changelog
 
+## [0.9.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.8.0...v0.9.0) (2024-04-02)
+
+
+### Features
+
+* Proposal for refreshes needed metadata ([#67](https://github.com/airbytehq/airbyte-protocol/issues/67)) ([7e8a487](https://github.com/airbytehq/airbyte-protocol/commit/7e8a487603af231e29e3031ca631a9f54a7c366a))
+
+## [0.8.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.7.0...v0.8.0) (2024-03-19)
+
+
+### Features
+
+* add transient_error as new failure type ([#69](https://github.com/airbytehq/airbyte-protocol/issues/69)) ([8fada32](https://github.com/airbytehq/airbyte-protocol/commit/8fada32aedb559731c62f0e35443e4e6dfd3d600))
+
+## [0.7.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.6.0...v0.7.0) (2024-03-06)
+
+
+### Features
+
+* Add DESTINATION_TYPECAST_ERROR to reason enum value ([#65](https://github.com/airbytehq/airbyte-protocol/issues/65)) ([86f03e9](https://github.com/airbytehq/airbyte-protocol/commit/86f03e9259c5e03b767a28400080ad63e91e1b29))
+
+## [0.6.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.3...v0.6.0) (2024-01-24)
+
+
+### Features
+
+* `AirbyteRecordMessageMeta` for per-record lineage and changes ([#56](https://github.com/airbytehq/airbyte-protocol/issues/56)) ([45461e1](https://github.com/airbytehq/airbyte-protocol/commit/45461e133e001a533c6fb01a7aa26e94dc0f13d2))
+
+## [0.5.3](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.2...v0.5.3) (2024-01-04)
+
+
+### Bug Fixes
+
+* Add py.typed to python distribution ([#57](https://github.com/airbytehq/airbyte-protocol/issues/57)) ([212095d](https://github.com/airbytehq/airbyte-protocol/commit/212095d554c8fef0b2389129173de11cf032e670))
+
+## [0.5.2](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.1...v0.5.2) (2023-12-26)
+
+
+### Bug Fixes
+
+* Removed unused V1. ([#58](https://github.com/airbytehq/airbyte-protocol/issues/58)) ([24be395](https://github.com/airbytehq/airbyte-protocol/commit/24be3958a2685def17c07992fbe87f6ac9867af7))
+
 ## [0.5.1](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.0...v0.5.1) (2023-12-08)
 
 

diff --git a/protocol-models/python/airbyte_protocol/py.typed b/protocol-models/python/airbyte_protocol/py.typed
diff --git a/protocol-models/python/setup.py b/protocol-models/python/setup.py
@@ -56,6 +56,7 @@
         "Tracker": "https://github.com/airbytehq/airbyte-protocol/issues",
     },
     packages=['airbyte_protocol.models'],
+    package_data={"airbyte_protocol": ["py.typed"]},
     setup_requires=['python-dotenv'],
     install_requires=[
         "pydantic>=1.9.2,<2.0.0",

diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml
@@ -73,6 +73,55 @@ definitions:
       emitted_at:
         description: "when the data was emitted from the source. epoch in millisecond."
         type: integer
+      meta:
+        description: Information about this record added mid-sync
+        "$ref": "#/definitions/AirbyteRecordMessageMeta"
+  AirbyteRecordMessageMeta:
+    type: object
+    additionalProperties: true
+    properties:
+      changes:
+        description: Lists of changes to the content of this record which occurred during syncing
+        type: array
+        items:
+          "$ref": "#/definitions/AirbyteRecordMessageMetaChange"
+  AirbyteRecordMessageMetaChange:
+    type: object
+    additionalProperties: true
+    required:
+      - field
+      - reason
+      - change
+    properties:
+      field:
+        type: string
+        description: The field that had the change occur (required)
+      change:
+        type: string
+        description: The type of change that occurred
+        enum:
+          - NULLED
+          - TRUNCATED
+      reason:
+        type: string
+        description: The reason that the change occurred
+        enum:
+          # The record, in aggregate, was too large to be processed
+          - SOURCE_RECORD_SIZE_LIMITATION
+          - DESTINATION_RECORD_SIZE_LIMITATION
+          - PLATFORM_RECORD_SIZE_LIMITATION
+          # A single field, was too large to be processed
+          - SOURCE_FIELD_SIZE_LIMITATION
+          - DESTINATION_FIELD_SIZE_LIMITATION
+          - PLATFORM_FIELD_SIZE_LIMITATION
+          # The field could not be read or written
+          - SOURCE_SERIALIZATION_ERROR
+          - DESTINATION_SERIALIZATION_ERROR
+          - PLATFORM_SERIALIZATION_ERROR
+          # Errors producing the field
+          - SOURCE_RETRIEVAL_ERROR
+          # Errors casting to appropriate type
+          - DESTINATION_TYPECAST_ERROR
   AirbyteStateMessage:
     type: object
     additionalProperties: true
@@ -147,7 +196,7 @@ definitions:
     additionalProperties: true
     properties:
       recordCount:
-        description: "the number of records which were emitted for this state message, for this stream or global"
+        description: "the number of records which were emitted for this state message, for this stream or global. While the value should always be a round number, it is defined as a double to account for integer overflows, and the value should always have a decimal point for proper serialization."
         type: number
   AirbyteLogMessage:
     type: object
@@ -223,6 +272,7 @@ definitions:
         enum:
           - system_error
           - config_error
+          - transient_error
       stream_descriptor:
         description: "The stream associated with the error, if known (optional)"
         "$ref": "#/definitions/StreamDescriptor"
@@ -437,6 +487,28 @@ definitions:
           type: array
           items:
             type: string
+      generation_id:
+        description: 
+          "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs.
+
+          If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup.
+          "
+        type: integer
+      minimum_generation_id:
+        description: 
+          "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property.
+          If the minimum generation is equals to 0, no data deletion is expected from the destiantion
+
+          If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup.
+          "
+        type: integer
+      sync_id:
+        description: 
+          "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync.
+
+          If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup.
+          "
+        type: integer
   SyncMode:
     type: string
     enum:

diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml
@@ -73,6 +73,56 @@ definitions:
       emitted_at:
         description: "when the data was emitted from the source. epoch in millisecond."
         type: integer
+      meta:
+        description: Information about this record added mid-sync
+        "$ref": "#/definitions/AirbyteRecordMessageMeta"
+
+  AirbyteRecordMessageMeta:
+    type: object
+    additionalProperties: true
+    properties:
+      changes:
+        description: Lists of changes to the content of this record which occurred during syncing
+        type: array
+        items:
+          "$ref": "#/definitions/AirbyteRecordMessageMetaChange"
+
+  AirbyteRecordMessageMetaChange:
+    type: object
+    additionalProperties: true
+    required:
+      - field
+      - reason
+      - change
+    properties:
+      field:
+        type: string
+        description: The field that had the change occur (required)
+      change:
+        type: string
+        description: The type of change that occurred
+        enum:
+          - NULLED
+          - TRUNCATED
+      reason:
+        type: string
+        description: The reason that the change occurred
+        enum:
+          # The record, in aggregate, was too large to be processed
+          - SOURCE_RECORD_SIZE_LIMITATION
+          - DESTINATION_RECORD_SIZE_LIMITATION
+          - PLATFORM_RECORD_SIZE_LIMITATION
+          # A single field, was too large to be processed
+          - SOURCE_FIELD_SIZE_LIMITATION
+          - DESTINATION_FIELD_SIZE_LIMITATION
+          - PLATFORM_FIELD_SIZE_LIMITATION
+          # The field could not be read or written
+          - SOURCE_SERIALIZATION_ERROR
+          - DESTINATION_SERIALIZATION_ERROR
+          - PLATFORM_SERIALIZATION_ERROR
+          # Errors producing the field
+          - SOURCE_RETRIEVAL_ERROR
+
   AirbyteStateMessage:
     type: object
     additionalProperties: true
@@ -223,6 +273,7 @@ definitions:
         enum:
           - system_error
           - config_error
+          - transient_error
       stream_descriptor:
         description: "The stream associated with the error, if known (optional)"
         "$ref": "#/definitions/StreamDescriptor"
@@ -434,6 +485,28 @@ definitions:
           type: array
           items:
             type: string
+      generation_id:
+        description: 
+          "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs.
+
+          If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup.
+          "
+        type: integer
+      minimum_generation_id:
+        description: 
+          "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property.
+          If the minimum generation is equals to 0, no data deletion is expected from the destiantion
+        
+          If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup.
+          "
+        type: integer
+      sync_id:
+        description: 
+          "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync.
+        
+          If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup.
+          "
+        type: integer
   SyncMode:
     type: string
     enum:

diff --git a/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml
diff --git a/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java
@@ -278,7 +278,7 @@ void testCatalogDiffWithoutStreamConfig() throws IOException {
 
     final Set<StreamTransform> diff = CatalogHelpers.getCatalogDiff(catalog1, catalog2, configuredAirbyteCatalog);
 
-    //configuredCatalog is for a different stream, so no diff should be found
+    // configuredCatalog is for a different stream, so no diff should be found
     Assertions.assertThat(diff).hasSize(0);
   }
 
@@ -305,4 +305,5 @@ void testCatalogDiffStreamChangeWithNoFieldTransform() throws IOException {
 
     Assertions.assertThat(actualDiff).containsExactlyInAnyOrderElementsOf(expectedDiff);
   }
+
 }