diff --git a/.env b/.env index 80f85a2..3afdf1b 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.5.1 +VERSION=0.9.0 # x-release-please-end diff --git a/CHANGELOG.md b/CHANGELOG.md index ba61a73..49add00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,47 @@ # Changelog +## [0.9.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.8.0...v0.9.0) (2024-04-02) + + +### Features + +* Proposal for refreshes needed metadata ([#67](https://github.com/airbytehq/airbyte-protocol/issues/67)) ([7e8a487](https://github.com/airbytehq/airbyte-protocol/commit/7e8a487603af231e29e3031ca631a9f54a7c366a)) + +## [0.8.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.7.0...v0.8.0) (2024-03-19) + + +### Features + +* add transient_error as new failure type ([#69](https://github.com/airbytehq/airbyte-protocol/issues/69)) ([8fada32](https://github.com/airbytehq/airbyte-protocol/commit/8fada32aedb559731c62f0e35443e4e6dfd3d600)) + +## [0.7.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.6.0...v0.7.0) (2024-03-06) + + +### Features + +* Add DESTINATION_TYPECAST_ERROR to reason enum value ([#65](https://github.com/airbytehq/airbyte-protocol/issues/65)) ([86f03e9](https://github.com/airbytehq/airbyte-protocol/commit/86f03e9259c5e03b767a28400080ad63e91e1b29)) + +## [0.6.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.3...v0.6.0) (2024-01-24) + + +### Features + +* `AirbyteRecordMessageMeta` for per-record lineage and changes ([#56](https://github.com/airbytehq/airbyte-protocol/issues/56)) ([45461e1](https://github.com/airbytehq/airbyte-protocol/commit/45461e133e001a533c6fb01a7aa26e94dc0f13d2)) + +## [0.5.3](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.2...v0.5.3) (2024-01-04) + + +### Bug Fixes + +* Add py.typed to python distribution ([#57](https://github.com/airbytehq/airbyte-protocol/issues/57)) ([212095d](https://github.com/airbytehq/airbyte-protocol/commit/212095d554c8fef0b2389129173de11cf032e670)) + +## [0.5.2](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.1...v0.5.2) (2023-12-26) + + +### Bug Fixes + +* Removed unused V1. ([#58](https://github.com/airbytehq/airbyte-protocol/issues/58)) ([24be395](https://github.com/airbytehq/airbyte-protocol/commit/24be3958a2685def17c07992fbe87f6ac9867af7)) + ## [0.5.1](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.0...v0.5.1) (2023-12-08) diff --git a/protocol-models/python/airbyte_protocol/py.typed b/protocol-models/python/airbyte_protocol/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/protocol-models/python/setup.py b/protocol-models/python/setup.py index 783eb0f..a4586f5 100644 --- a/protocol-models/python/setup.py +++ b/protocol-models/python/setup.py @@ -56,6 +56,7 @@ "Tracker": "https://github.com/airbytehq/airbyte-protocol/issues", }, packages=['airbyte_protocol.models'], + package_data={"airbyte_protocol": ["py.typed"]}, setup_requires=['python-dotenv'], install_requires=[ "pydantic>=1.9.2,<2.0.0", diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index d745b10..b85099c 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -73,6 +73,55 @@ definitions: emitted_at: description: "when the data was emitted from the source. epoch in millisecond." type: integer + meta: + description: Information about this record added mid-sync + "$ref": "#/definitions/AirbyteRecordMessageMeta" + AirbyteRecordMessageMeta: + type: object + additionalProperties: true + properties: + changes: + description: Lists of changes to the content of this record which occurred during syncing + type: array + items: + "$ref": "#/definitions/AirbyteRecordMessageMetaChange" + AirbyteRecordMessageMetaChange: + type: object + additionalProperties: true + required: + - field + - reason + - change + properties: + field: + type: string + description: The field that had the change occur (required) + change: + type: string + description: The type of change that occurred + enum: + - NULLED + - TRUNCATED + reason: + type: string + description: The reason that the change occurred + enum: + # The record, in aggregate, was too large to be processed + - SOURCE_RECORD_SIZE_LIMITATION + - DESTINATION_RECORD_SIZE_LIMITATION + - PLATFORM_RECORD_SIZE_LIMITATION + # A single field, was too large to be processed + - SOURCE_FIELD_SIZE_LIMITATION + - DESTINATION_FIELD_SIZE_LIMITATION + - PLATFORM_FIELD_SIZE_LIMITATION + # The field could not be read or written + - SOURCE_SERIALIZATION_ERROR + - DESTINATION_SERIALIZATION_ERROR + - PLATFORM_SERIALIZATION_ERROR + # Errors producing the field + - SOURCE_RETRIEVAL_ERROR + # Errors casting to appropriate type + - DESTINATION_TYPECAST_ERROR AirbyteStateMessage: type: object additionalProperties: true @@ -147,7 +196,7 @@ definitions: additionalProperties: true properties: recordCount: - description: "the number of records which were emitted for this state message, for this stream or global" + description: "the number of records which were emitted for this state message, for this stream or global. While the value should always be a round number, it is defined as a double to account for integer overflows, and the value should always have a decimal point for proper serialization." type: number AirbyteLogMessage: type: object @@ -223,6 +272,7 @@ definitions: enum: - system_error - config_error + - transient_error stream_descriptor: description: "The stream associated with the error, if known (optional)" "$ref": "#/definitions/StreamDescriptor" @@ -437,6 +487,28 @@ definitions: type: array items: type: string + generation_id: + description: + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + minimum_generation_id: + description: + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + sync_id: + description: + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer SyncMode: type: string enum: diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 85960b4..f01e772 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -73,6 +73,56 @@ definitions: emitted_at: description: "when the data was emitted from the source. epoch in millisecond." type: integer + meta: + description: Information about this record added mid-sync + "$ref": "#/definitions/AirbyteRecordMessageMeta" + + AirbyteRecordMessageMeta: + type: object + additionalProperties: true + properties: + changes: + description: Lists of changes to the content of this record which occurred during syncing + type: array + items: + "$ref": "#/definitions/AirbyteRecordMessageMetaChange" + + AirbyteRecordMessageMetaChange: + type: object + additionalProperties: true + required: + - field + - reason + - change + properties: + field: + type: string + description: The field that had the change occur (required) + change: + type: string + description: The type of change that occurred + enum: + - NULLED + - TRUNCATED + reason: + type: string + description: The reason that the change occurred + enum: + # The record, in aggregate, was too large to be processed + - SOURCE_RECORD_SIZE_LIMITATION + - DESTINATION_RECORD_SIZE_LIMITATION + - PLATFORM_RECORD_SIZE_LIMITATION + # A single field, was too large to be processed + - SOURCE_FIELD_SIZE_LIMITATION + - DESTINATION_FIELD_SIZE_LIMITATION + - PLATFORM_FIELD_SIZE_LIMITATION + # The field could not be read or written + - SOURCE_SERIALIZATION_ERROR + - DESTINATION_SERIALIZATION_ERROR + - PLATFORM_SERIALIZATION_ERROR + # Errors producing the field + - SOURCE_RETRIEVAL_ERROR + AirbyteStateMessage: type: object additionalProperties: true @@ -223,6 +273,7 @@ definitions: enum: - system_error - config_error + - transient_error stream_descriptor: description: "The stream associated with the error, if known (optional)" "$ref": "#/definitions/StreamDescriptor" @@ -434,6 +485,28 @@ definitions: type: array items: type: string + generation_id: + description: + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + minimum_generation_id: + description: + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + sync_id: + description: + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer SyncMode: type: string enum: diff --git a/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml deleted file mode 120000 index c327ab2..0000000 --- a/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml +++ /dev/null @@ -1 +0,0 @@ -../airbyte_protocol.yaml \ No newline at end of file diff --git a/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 63ba490..257c137 100644 --- a/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -278,7 +278,7 @@ void testCatalogDiffWithoutStreamConfig() throws IOException { final Set diff = CatalogHelpers.getCatalogDiff(catalog1, catalog2, configuredAirbyteCatalog); - //configuredCatalog is for a different stream, so no diff should be found + // configuredCatalog is for a different stream, so no diff should be found Assertions.assertThat(diff).hasSize(0); } @@ -305,4 +305,5 @@ void testCatalogDiffStreamChangeWithNoFieldTransform() throws IOException { Assertions.assertThat(actualDiff).containsExactlyInAnyOrderElementsOf(expectedDiff); } + }