From 771d56d91abb1f879e9730579a9ecfe4bf520ce2 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Wed, 6 Mar 2024 08:09:57 -0800 Subject: [PATCH 01/11] Proposal for refreshes needed metadata --- .../airbyte_protocol/v0/airbyte_protocol.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 31c8a17..92b46b2 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -456,6 +456,22 @@ definitions: type: array items: "$ref": "#/definitions/ConfiguredAirbyteStream" + sync_metadata: + "$ref": "#/definitions/SyncMetadata" + SyncMetadata: + type: object + additionalProperties: true + description: Information about the sync which is currently running + required: + - job_id + - attempt_number + properties: + job_id: + description: Optional job Id which represent the job id in which the catalog is use + type: integer + attempt_number: + description: Optional attempt number which represent the attempt number of the current job id + type: integer ConfiguredAirbyteStream: type: object additionalProperties: true From 574208db43a75be972658761e16b6171c9586c8f Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Wed, 6 Mar 2024 08:25:45 -0800 Subject: [PATCH 02/11] Update naming --- .../resources/airbyte_protocol/v0/airbyte_protocol.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 92b46b2..2bd1936 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -463,14 +463,14 @@ definitions: additionalProperties: true description: Information about the sync which is currently running required: - - job_id + - sync_id - attempt_number properties: - job_id: - description: Optional job Id which represent the job id in which the catalog is use + sync_id: + description: Incremental id representing the synchronization number type: integer attempt_number: - description: Optional attempt number which represent the attempt number of the current job id + description: Incremental number representing the attempt number within the same sync_id type: integer ConfiguredAirbyteStream: type: object From e56fc4538b119bbb596295232fa7f7a3f804729b Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 18 Mar 2024 15:52:46 -0700 Subject: [PATCH 03/11] Remove attempt number --- .../airbyte_protocol/v0/airbyte_protocol.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 2bd1936..4300253 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -463,14 +463,10 @@ definitions: additionalProperties: true description: Information about the sync which is currently running required: - - sync_id - - attempt_number + - job_id properties: - sync_id: - description: Incremental id representing the synchronization number - type: integer - attempt_number: - description: Incremental number representing the attempt number within the same sync_id + job_id: + description: Incremental id representing the job Id type: integer ConfiguredAirbyteStream: type: object From 37a3de00458985201c7e09127d5d13640ec82494 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Fri, 22 Mar 2024 14:17:57 -0700 Subject: [PATCH 04/11] Update --- .../airbyte_protocol/v0/airbyte_protocol.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index a279731..9aeb73b 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -497,6 +497,20 @@ definitions: type: array items: type: string + StreamMetadata: + type: object + additionalProperties: true + description: Information about the stream + required: + - generation_id + - minimum_generation + properties: + generation_id: + description: Incremental id representing the current generation of a stream + type: integer + minimum_generation: + description: The minimum generation id wanted in the stream. All data with a lower generation will be removed from the destination + type: integer SyncMode: type: string enum: From 29223c28fab7481e1d67939c81aee8ae06c5639c Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 1 Apr 2024 08:47:04 -0700 Subject: [PATCH 05/11] Update protocol --- .../airbyte_protocol/v0/airbyte_protocol.yaml | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 9aeb73b..6caecea 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -457,18 +457,6 @@ definitions: type: array items: "$ref": "#/definitions/ConfiguredAirbyteStream" - sync_metadata: - "$ref": "#/definitions/SyncMetadata" - SyncMetadata: - type: object - additionalProperties: true - description: Information about the sync which is currently running - required: - - job_id - properties: - job_id: - description: Incremental id representing the job Id - type: integer ConfiguredAirbyteStream: type: object additionalProperties: true @@ -497,19 +485,22 @@ definitions: type: array items: type: string - StreamMetadata: + StreamConfig: type: object additionalProperties: true description: Information about the stream required: - - generation_id - - minimum_generation + - current_generation_id + - sync_id properties: generation_id: - description: Incremental id representing the current generation of a stream + description: Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs + type: integer + minimum_generation_id: + description: The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. If it is absent, the destination won't try to delete data type: integer - minimum_generation: - description: The minimum generation id wanted in the stream. All data with a lower generation will be removed from the destination + sync_id: + description: Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. type: integer SyncMode: type: string From fba5ce85a3b0bf272e30891fcfb52c1b68fe4888 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 1 Apr 2024 08:50:01 -0700 Subject: [PATCH 06/11] More PR updates --- .../airbyte_protocol/v0/airbyte_protocol.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 6caecea..3918002 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -464,6 +464,8 @@ definitions: - stream - sync_mode - destination_sync_mode + - current_generation_id + - sync_id properties: stream: "$ref": "#/definitions/AirbyteStream" @@ -485,14 +487,6 @@ definitions: type: array items: type: string - StreamConfig: - type: object - additionalProperties: true - description: Information about the stream - required: - - current_generation_id - - sync_id - properties: generation_id: description: Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs type: integer From e3ce32ec30b784b6593a5b37f9a7e18859e134a0 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 1 Apr 2024 14:44:29 -0700 Subject: [PATCH 07/11] PR comments and version --- .env | 2 +- .../resources/airbyte_protocol/airbyte_protocol.yaml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.env b/.env index ecd7784..581b2a7 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.8.0 +VERSION=0.8.1 # x-release-please-end diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 2f33a4f..02b0421 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -484,6 +484,15 @@ definitions: type: array items: type: string + generation_id: + description: Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs + type: integer + minimum_generation_id: + description: The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. If it is absent, the destination won't try to delete data + type: integer + sync_id: + description: Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + type: integer SyncMode: type: string enum: From c762ef394e8841a6e72c013237ce3bab8f7e00e5 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 1 Apr 2024 14:48:56 -0700 Subject: [PATCH 08/11] Missing stuff --- .env | 2 +- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 581b2a7..ecd7784 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ # x-release-please-start-version -VERSION=0.8.1 +VERSION=0.8.0 # x-release-please-end diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 02b0421..6e299e9 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -463,6 +463,8 @@ definitions: - stream - sync_mode - destination_sync_mode + - generation_id + - sync_id properties: stream: "$ref": "#/definitions/AirbyteStream" From 912eec5ce67ec21182477b8d05ff46f7d7ce2a40 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 1 Apr 2024 16:42:07 -0700 Subject: [PATCH 09/11] PR comments --- .../airbyte_protocol/airbyte_protocol.yaml | 21 ++++++++++++++----- .../airbyte_protocol/v0/airbyte_protocol.yaml | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 6e299e9..6fc1e7d 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -463,8 +463,6 @@ definitions: - stream - sync_mode - destination_sync_mode - - generation_id - - sync_id properties: stream: "$ref": "#/definitions/AirbyteStream" @@ -487,13 +485,26 @@ definitions: items: type: string generation_id: - description: Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs + description: + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer minimum_generation_id: - description: The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. If it is absent, the destination won't try to delete data + description: + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer sync_id: - description: Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + description: + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer SyncMode: type: string diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 3918002..2416153 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -464,7 +464,7 @@ definitions: - stream - sync_mode - destination_sync_mode - - current_generation_id + - generation_id - sync_id properties: stream: From 2ab696429f2e9704cbbcddb31a6381ebd14a2fe5 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 1 Apr 2024 16:43:49 -0700 Subject: [PATCH 10/11] PR comments --- .../airbyte_protocol/v0/airbyte_protocol.yaml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 2416153..26cd2d9 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -464,8 +464,6 @@ definitions: - stream - sync_mode - destination_sync_mode - - generation_id - - sync_id properties: stream: "$ref": "#/definitions/AirbyteStream" @@ -488,13 +486,26 @@ definitions: items: type: string generation_id: - description: Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs + description: + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer minimum_generation_id: - description: The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. If it is absent, the destination won't try to delete data + description: + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer sync_id: - description: Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + description: + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer SyncMode: type: string From db58c46cfa536eaea045a1201891f60b5b89c4e3 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Mon, 1 Apr 2024 18:39:45 -0700 Subject: [PATCH 11/11] Format --- .../airbyte_protocol/airbyte_protocol.yaml | 24 +++++++++---------- .../airbyte_protocol/v0/airbyte_protocol.yaml | 20 ++++++++-------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 6fc1e7d..fad870a 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -486,25 +486,25 @@ definitions: type: string generation_id: description: - "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. - If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. - " + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer minimum_generation_id: description: - "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. - If the minimum generation is equals to 0, no data deletion is expected from the destiantion - - If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. - " + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer sync_id: description: - "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. - - If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. - " + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer SyncMode: type: string diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 26cd2d9..f01e772 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -487,25 +487,25 @@ definitions: type: string generation_id: description: - "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. + "Monotically increasing numeric id representing the current generation of a stream. This id can be shared across syncs. - If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. - " + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer minimum_generation_id: description: - "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. - If the minimum generation is equals to 0, no data deletion is expected from the destiantion + "The minimum generation id which is needed in a stream. If it is present, the destination will try to delete the data that are part of a generation lower than this property. + If the minimum generation is equals to 0, no data deletion is expected from the destiantion - If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. - " + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer sync_id: description: - "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. + "Monotically increasing numeric id representing the current sync id. This is aimed to be unique per sync. - If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. - " + If this is null, it means that the platform is not supporting the refresh and it is expected that no extra id will be added to the records and no data from previous generation will be cleanup. + " type: integer SyncMode: type: string