From 24be3958a2685def17c07992fbe87f6ac9867af7 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Tue, 26 Dec 2023 09:32:09 -0800 Subject: [PATCH 01/13] fix: Removed unused V1. (#58) We do not use V1. Instead of duplicate, potential drift and confusion, let's remove V1 for now and re-introduce when we are ready to move to V1. --- .../src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml | 1 - 1 file changed, 1 deletion(-) delete mode 120000 protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml diff --git a/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml deleted file mode 120000 index c327ab2..0000000 --- a/protocol-models/src/main/resources/airbyte_protocol/v1/airbyte_protocol.yaml +++ /dev/null @@ -1 +0,0 @@ -../airbyte_protocol.yaml \ No newline at end of file From 8f111ab9ff11100f911355195e918fb9d1222a6b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 11:23:18 -0800 Subject: [PATCH 02/13] chore(main): release 0.5.2 (#59) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 80f85a2..d91b75f 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.5.1 +VERSION=0.5.2 # x-release-please-end diff --git a/CHANGELOG.md b/CHANGELOG.md index ba61a73..5b1308d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.5.2](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.1...v0.5.2) (2023-12-26) + + +### Bug Fixes + +* Removed unused V1. ([#58](https://github.com/airbytehq/airbyte-protocol/issues/58)) ([24be395](https://github.com/airbytehq/airbyte-protocol/commit/24be3958a2685def17c07992fbe87f6ac9867af7)) + ## [0.5.1](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.0...v0.5.1) (2023-12-08) From 212095d554c8fef0b2389129173de11cf032e670 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 4 Jan 2024 15:54:29 +0100 Subject: [PATCH 03/13] fix: Add py.typed to python distribution (#57) --- protocol-models/python/airbyte_protocol/py.typed | 0 protocol-models/python/setup.py | 1 + 2 files changed, 1 insertion(+) create mode 100644 protocol-models/python/airbyte_protocol/py.typed diff --git a/protocol-models/python/airbyte_protocol/py.typed b/protocol-models/python/airbyte_protocol/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/protocol-models/python/setup.py b/protocol-models/python/setup.py index 783eb0f..a4586f5 100644 --- a/protocol-models/python/setup.py +++ b/protocol-models/python/setup.py @@ -56,6 +56,7 @@ "Tracker": "https://github.com/airbytehq/airbyte-protocol/issues", }, packages=['airbyte_protocol.models'], + package_data={"airbyte_protocol": ["py.typed"]}, setup_requires=['python-dotenv'], install_requires=[ "pydantic>=1.9.2,<2.0.0", From c625e8544d6122e8084b948ec958dd9b55cd2c2c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 18:27:04 +0100 Subject: [PATCH 04/13] chore(main): release 0.5.3 (#61) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.env b/.env index d91b75f..b27882d 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.5.2 +VERSION=0.5.3 # x-release-please-end diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b1308d..a0d0542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.5.3](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.2...v0.5.3) (2024-01-04) + + +### Bug Fixes + +* Add py.typed to python distribution ([#57](https://github.com/airbytehq/airbyte-protocol/issues/57)) ([212095d](https://github.com/airbytehq/airbyte-protocol/commit/212095d554c8fef0b2389129173de11cf032e670)) + ## [0.5.2](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.1...v0.5.2) (2023-12-26) From 45461e133e001a533c6fb01a7aa26e94dc0f13d2 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Wed, 24 Jan 2024 12:01:21 -0800 Subject: [PATCH 05/13] feat: `AirbyteRecordMessageMeta` for per-record lineage and changes (#56) --- .../airbyte_protocol/airbyte_protocol.yaml | 47 +++++++++++++++++ .../airbyte_protocol/v0/airbyte_protocol.yaml | 50 +++++++++++++++++++ .../protocol/models/CatalogHelpersTest.java | 3 +- 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 3665055..77aea3d 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -73,6 +73,53 @@ definitions: emitted_at: description: "when the data was emitted from the source. epoch in millisecond." type: integer + meta: + description: Information about this record added mid-sync + "$ref": "#/definitions/AirbyteRecordMessageMeta" + AirbyteRecordMessageMeta: + type: object + additionalProperties: true + properties: + changes: + description: Lists of changes to the content of this record which occurred during syncing + type: array + items: + "$ref": "#/definitions/AirbyteRecordMessageMetaChange" + AirbyteRecordMessageMetaChange: + type: object + additionalProperties: true + required: + - field + - reason + - change + properties: + field: + type: string + description: The field that had the change occur (required) + change: + type: string + description: The type of change that occurred + enum: + - NULLED + - TRUNCATED + reason: + type: string + description: The reason that the change occurred + enum: + # The record, in aggregate, was too large to be processed + - SOURCE_RECORD_SIZE_LIMITATION + - DESTINATION_RECORD_SIZE_LIMITATION + - PLATFORM_RECORD_SIZE_LIMITATION + # A single field, was too large to be processed + - SOURCE_FIELD_SIZE_LIMITATION + - DESTINATION_FIELD_SIZE_LIMITATION + - PLATFORM_FIELD_SIZE_LIMITATION + # The field could not be read or written + - SOURCE_SERIALIZATION_ERROR + - DESTINATION_SERIALIZATION_ERROR + - PLATFORM_SERIALIZATION_ERROR + # Errors producing the field + - SOURCE_RETRIEVAL_ERROR AirbyteStateMessage: type: object additionalProperties: true diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 85960b4..31c8a17 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -73,6 +73,56 @@ definitions: emitted_at: description: "when the data was emitted from the source. epoch in millisecond." type: integer + meta: + description: Information about this record added mid-sync + "$ref": "#/definitions/AirbyteRecordMessageMeta" + + AirbyteRecordMessageMeta: + type: object + additionalProperties: true + properties: + changes: + description: Lists of changes to the content of this record which occurred during syncing + type: array + items: + "$ref": "#/definitions/AirbyteRecordMessageMetaChange" + + AirbyteRecordMessageMetaChange: + type: object + additionalProperties: true + required: + - field + - reason + - change + properties: + field: + type: string + description: The field that had the change occur (required) + change: + type: string + description: The type of change that occurred + enum: + - NULLED + - TRUNCATED + reason: + type: string + description: The reason that the change occurred + enum: + # The record, in aggregate, was too large to be processed + - SOURCE_RECORD_SIZE_LIMITATION + - DESTINATION_RECORD_SIZE_LIMITATION + - PLATFORM_RECORD_SIZE_LIMITATION + # A single field, was too large to be processed + - SOURCE_FIELD_SIZE_LIMITATION + - DESTINATION_FIELD_SIZE_LIMITATION + - PLATFORM_FIELD_SIZE_LIMITATION + # The field could not be read or written + - SOURCE_SERIALIZATION_ERROR + - DESTINATION_SERIALIZATION_ERROR + - PLATFORM_SERIALIZATION_ERROR + # Errors producing the field + - SOURCE_RETRIEVAL_ERROR + AirbyteStateMessage: type: object additionalProperties: true diff --git a/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java b/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java index 63ba490..257c137 100644 --- a/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java +++ b/protocol-models/src/test/java/io/airbyte/protocol/models/CatalogHelpersTest.java @@ -278,7 +278,7 @@ void testCatalogDiffWithoutStreamConfig() throws IOException { final Set diff = CatalogHelpers.getCatalogDiff(catalog1, catalog2, configuredAirbyteCatalog); - //configuredCatalog is for a different stream, so no diff should be found + // configuredCatalog is for a different stream, so no diff should be found Assertions.assertThat(diff).hasSize(0); } @@ -305,4 +305,5 @@ void testCatalogDiffStreamChangeWithNoFieldTransform() throws IOException { Assertions.assertThat(actualDiff).containsExactlyInAnyOrderElementsOf(expectedDiff); } + } From 8bcf5165201fbfe48cc5ca1d58b1d202d3a2f91d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 12:02:48 -0800 Subject: [PATCH 06/13] chore(main): release 0.6.0 (#63) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.env b/.env index b27882d..ede2e26 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.5.3 +VERSION=0.6.0 # x-release-please-end diff --git a/CHANGELOG.md b/CHANGELOG.md index a0d0542..cf2a341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.6.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.3...v0.6.0) (2024-01-24) + + +### Features + +* `AirbyteRecordMessageMeta` for per-record lineage and changes ([#56](https://github.com/airbytehq/airbyte-protocol/issues/56)) ([45461e1](https://github.com/airbytehq/airbyte-protocol/commit/45461e133e001a533c6fb01a7aa26e94dc0f13d2)) + ## [0.5.3](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.2...v0.5.3) (2024-01-04) From 86f03e9259c5e03b767a28400080ad63e91e1b29 Mon Sep 17 00:00:00 2001 From: Joe Bell Date: Wed, 6 Mar 2024 08:26:23 -0800 Subject: [PATCH 07/13] feat: Add DESTINATION_TYPECAST_ERROR to reason enum value (#65) --- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 77aea3d..74392bc 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -120,6 +120,8 @@ definitions: - PLATFORM_SERIALIZATION_ERROR # Errors producing the field - SOURCE_RETRIEVAL_ERROR + # Errors casting to appropriate type + - DESTINATION_TYPECAST_ERROR AirbyteStateMessage: type: object additionalProperties: true From d7be570ff40608d0f49af1b9aa3bb446b074ec6d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 08:47:48 -0800 Subject: [PATCH 08/13] chore(main): release 0.7.0 (#68) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.env b/.env index ede2e26..5f57e15 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.6.0 +VERSION=0.7.0 # x-release-please-end diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2a341..821f1a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.7.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.6.0...v0.7.0) (2024-03-06) + + +### Features + +* Add DESTINATION_TYPECAST_ERROR to reason enum value ([#65](https://github.com/airbytehq/airbyte-protocol/issues/65)) ([86f03e9](https://github.com/airbytehq/airbyte-protocol/commit/86f03e9259c5e03b767a28400080ad63e91e1b29)) + ## [0.6.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.5.3...v0.6.0) (2024-01-24) From 8fada32aedb559731c62f0e35443e4e6dfd3d600 Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Tue, 19 Mar 2024 13:31:32 -0700 Subject: [PATCH 09/13] feat: add transient_error as new failure type (#69) --- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 1 + .../src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 74392bc..563632a 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -272,6 +272,7 @@ definitions: enum: - system_error - config_error + - transient_error stream_descriptor: description: "The stream associated with the error, if known (optional)" "$ref": "#/definitions/StreamDescriptor" diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 31c8a17..7626c43 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -273,6 +273,7 @@ definitions: enum: - system_error - config_error + - transient_error stream_descriptor: description: "The stream associated with the error, if known (optional)" "$ref": "#/definitions/StreamDescriptor" From f92b4afe7b9d2ca3f35ed152f9c3ae212235d472 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 16:43:39 -0700 Subject: [PATCH 10/13] chore(main): release 0.8.0 (#70) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 5f57e15..ecd7784 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.7.0 +VERSION=0.8.0 # x-release-please-end diff --git a/CHANGELOG.md b/CHANGELOG.md index 821f1a6..06ace74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.8.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.7.0...v0.8.0) (2024-03-19) + + +### Features + +* add transient_error as new failure type ([#69](https://github.com/airbytehq/airbyte-protocol/issues/69)) ([8fada32](https://github.com/airbytehq/airbyte-protocol/commit/8fada32aedb559731c62f0e35443e4e6dfd3d600)) + ## [0.7.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.6.0...v0.7.0) (2024-03-06) From 2a403a601e686cbc181e75830d6d2d8854c519ab Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 27 Mar 2024 09:16:29 -0700 Subject: [PATCH 11/13] docs: Explain why recordCount is a number instead of an integer (#71) --- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 563632a..2f33a4f 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -196,7 +196,7 @@ definitions: additionalProperties: true properties: recordCount: - description: "the number of records which were emitted for this state message, for this stream or global" + description: "the number of records which were emitted for this state message, for this stream or global. While the value should always be a round number, it is defined as a double to account for integer overflows, and the value should always have a decimal point for proper serialization." type: number AirbyteLogMessage: type: object From 7e8a487603af231e29e3031ca631a9f54a7c366a Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Mon, 1 Apr 2024 18:45:05 -0700 Subject: [PATCH 12/13] feat: Proposal for refreshes needed metadata (#67) --- .../airbyte_protocol/airbyte_protocol.yaml | 22 +++++++++++++++++++ .../airbyte_protocol/v0/airbyte_protocol.yaml | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 2f33a4f..fad870a 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -484,6 +484,28 @@ definitions: type: array items: type: string + generation_id: + description: + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + minimum_generation_id: + description: + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + sync_id: + description: + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer SyncMode: type: string enum: diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 7626c43..f01e772 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -485,6 +485,28 @@ definitions: type: array items: type: string + generation_id: + description: + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + minimum_generation_id: + description: + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer + sync_id: + description: + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " + type: integer SyncMode: type: string enum: From b93c4fde73c7131c2bc90158b1a209d951eb385b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 15:57:41 -0700 Subject: [PATCH 13/13] chore(main): release 0.9.0 (#72) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .env | 2 +- CHANGELOG.md | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.env b/.env index ecd7784..3afdf1b 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.8.0 +VERSION=0.9.0 # x-release-please-end diff --git a/CHANGELOG.md b/CHANGELOG.md index 06ace74..49add00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.9.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.8.0...v0.9.0) (2024-04-02) + + +### Features + +* Proposal for refreshes needed metadata ([#67](https://github.com/airbytehq/airbyte-protocol/issues/67)) ([7e8a487](https://github.com/airbytehq/airbyte-protocol/commit/7e8a487603af231e29e3031ca631a9f54a7c366a)) + ## [0.8.0](https://github.com/airbytehq/airbyte-protocol/compare/v0.7.0...v0.8.0) (2024-03-19)