From 777b9a41c8d3fe365850dfd74a6f37d1607266fd Mon Sep 17 00:00:00 2001 From: subodhchaturvedi Date: Fri, 3 May 2024 17:01:05 +0530 Subject: [PATCH 1/6] feat: add RATE_LIMITED enum + metadata to AirbyteStreamStatusTraceMessage --- .../main/resources/airbyte_protocol/airbyte_protocol.yaml | 5 +++++ .../main/resources/airbyte_protocol/v0/airbyte_protocol.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index fad870a..c047b68 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -315,6 +315,7 @@ definitions: - RUNNING # Stream has read its first byte/message - COMPLETE # Stream has completed executing without interruption or error - INCOMPLETE # Stream has stopped due to an interruption or error + - RATE_LIMITED # Stream has stopped due to the source API hitting RATE limit AirbyteStreamStatusTraceMessage: type: object additionalProperties: true @@ -328,6 +329,10 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" + metadata: + description: "Any metadata associated with the stream status" + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode AirbyteAnalyticsTraceMessage: type: object additionalProperties: true diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index f01e772..90111b5 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -316,6 +316,7 @@ definitions: - RUNNING # Stream has read its first byte/message - COMPLETE # Stream has completed executing without interruption or error - INCOMPLETE # Stream has stopped due to an interruption or error + - RATE_LIMITED # Stream has stopped due to the source API hitting RATE limit AirbyteStreamStatusTraceMessage: type: object additionalProperties: true @@ -329,6 +330,10 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" + metadata: + description: "Any metadata associated with the stream status" + type: object + existingJavaType: com.fasterxml.jackson.databind.JsonNode AirbyteAnalyticsTraceMessage: type: object additionalProperties: true From ab3e30286db6d69e116e583f1aa6335386d7554d Mon Sep 17 00:00:00 2001 From: subodhchaturvedi Date: Fri, 3 May 2024 17:49:12 +0530 Subject: [PATCH 2/6] use timestamp instead of metadata --- .../main/resources/airbyte_protocol/airbyte_protocol.yaml | 7 +++---- .../resources/airbyte_protocol/v0/airbyte_protocol.yaml | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index c047b68..115548c 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -329,10 +329,9 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" - metadata: - description: "Any metadata associated with the stream status" - type: object - existingJavaType: com.fasterxml.jackson.databind.JsonNode + valid_till: + description: "Optional time in ms that the status is going to be valid till" + type: number AirbyteAnalyticsTraceMessage: type: object additionalProperties: true diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 90111b5..998e237 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -330,10 +330,9 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" - metadata: - description: "Any metadata associated with the stream status" - type: object - existingJavaType: com.fasterxml.jackson.databind.JsonNode + valid_till: + description: "Optional time in ms that the status is going to be valid till" + type: number AirbyteAnalyticsTraceMessage: type: object additionalProperties: true From 684fcabdb01d393b5322624687fa7a9defd605ae Mon Sep 17 00:00:00 2001 From: subodhchaturvedi Date: Sat, 4 May 2024 02:53:32 +0530 Subject: [PATCH 3/6] til instead of till --- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 2 +- .../main/resources/airbyte_protocol/v0/airbyte_protocol.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 115548c..a17f7c0 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -329,7 +329,7 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" - valid_till: + valid_til: description: "Optional time in ms that the status is going to be valid till" type: number AirbyteAnalyticsTraceMessage: diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 998e237..2a98d54 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -330,7 +330,7 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" - valid_till: + valid_til: description: "Optional time in ms that the status is going to be valid till" type: number AirbyteAnalyticsTraceMessage: From e2c7ad466d057e210d5e9033d6876e07ddb17931 Mon Sep 17 00:00:00 2001 From: subodhchaturvedi Date: Thu, 9 May 2024 03:05:03 +0530 Subject: [PATCH 4/6] update protocol changes according to discussion --- .../airbyte_protocol/airbyte_protocol.yaml | 33 ++++++++++++++++--- .../airbyte_protocol/v0/airbyte_protocol.yaml | 33 ++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index a17f7c0..9db85ac 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -315,7 +315,30 @@ definitions: - RUNNING # Stream has read its first byte/message - COMPLETE # Stream has completed executing without interruption or error - INCOMPLETE # Stream has stopped due to an interruption or error - - RATE_LIMITED # Stream has stopped due to the source API hitting RATE limit + AirbyteStreamStatusReasonType: + type: string + description: > + Type of reason + enum: + - RATE_LIMITED + AirbyteStreamStatusRateLimitedReason: + type: object + description: Rate Limited Information + properties: + quota_reset: + description: "Optional time in ms representing when the API quota is going to be reset" + type: number + AirbyteStreamStatusReason: + type: object + required: + - type + description: > + The reason associated with the status of the stream. + properties: + type: + "$ref": "#/definitions/AirbyteStreamStatusReasonType" + rate_limited: + "$ref": "#/definitions/AirbyteStreamStatusRateLimitedReason" AirbyteStreamStatusTraceMessage: type: object additionalProperties: true @@ -329,9 +352,11 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" - valid_til: - description: "Optional time in ms that the status is going to be valid till" - type: number + reasons: + description: "The reasons associated with the status of the stream" + type: array + items: + "$ref": "#/definitions/AirbyteStreamStatusReason" AirbyteAnalyticsTraceMessage: type: object additionalProperties: true diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 2a98d54..be2bb82 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -316,7 +316,30 @@ definitions: - RUNNING # Stream has read its first byte/message - COMPLETE # Stream has completed executing without interruption or error - INCOMPLETE # Stream has stopped due to an interruption or error - - RATE_LIMITED # Stream has stopped due to the source API hitting RATE limit + AirbyteStreamStatusReasonType: + type: string + description: > + Type of reason + enum: + - RATE_LIMITED + AirbyteStreamStatusRateLimitedReason: + type: object + description: Rate Limited Information + properties: + quota_reset: + description: "Optional time in ms representing when the API quota is going to be reset" + type: number + AirbyteStreamStatusReason: + type: object + required: + - type + description: > + The reason associated with the status of the stream. + properties: + type: + "$ref": "#/definitions/AirbyteStreamStatusReasonType" + rate_limited: + "$ref": "#/definitions/AirbyteStreamStatusRateLimitedReason" AirbyteStreamStatusTraceMessage: type: object additionalProperties: true @@ -330,9 +353,11 @@ definitions: status: description: "The current status of the stream" "$ref": "#/definitions/AirbyteStreamStatus" - valid_til: - description: "Optional time in ms that the status is going to be valid till" - type: number + reasons: + description: "The reasons associated with the status of the stream" + type: array + items: + "$ref": "#/definitions/AirbyteStreamStatusReason" AirbyteAnalyticsTraceMessage: type: object additionalProperties: true From c6523f67f2aea4d1a31a265a0112becd3a437967 Mon Sep 17 00:00:00 2001 From: subodhchaturvedi Date: Mon, 13 May 2024 21:35:56 +0530 Subject: [PATCH 5/6] use integer instead of number --- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 3 ++- .../main/resources/airbyte_protocol/v0/airbyte_protocol.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 9db85ac..2e6ee5f 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -327,7 +327,7 @@ definitions: properties: quota_reset: description: "Optional time in ms representing when the API quota is going to be reset" - type: number + type: integer AirbyteStreamStatusReason: type: object required: @@ -396,6 +396,7 @@ definitions: - CONNECTOR_CONFIG emitted_at: description: "the time in ms that the message was emitted" + # TODO : This should have been an integer instead of a number, need to fix this type: number connectorConfig: description: "connector config orchestrator message: the updated config for the platform to store for this connector" diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index be2bb82..98bd414 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -328,7 +328,7 @@ definitions: properties: quota_reset: description: "Optional time in ms representing when the API quota is going to be reset" - type: number + type: integer AirbyteStreamStatusReason: type: object required: @@ -397,6 +397,7 @@ definitions: - CONNECTOR_CONFIG emitted_at: description: "the time in ms that the message was emitted" + # TODO : This should have been an integer instead of a number, need to fix this type: number connectorConfig: description: "connector config orchestrator message: the updated config for the platform to store for this connector" From c76a2f9280a8a620b17de25e4b777fdd21b7cf0d Mon Sep 17 00:00:00 2001 From: subodhchaturvedi Date: Mon, 13 May 2024 21:40:32 +0530 Subject: [PATCH 6/6] add TODOs --- .../src/main/resources/airbyte_protocol/airbyte_protocol.yaml | 1 + .../src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 2e6ee5f..e383230 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -239,6 +239,7 @@ definitions: - ANALYTICS emitted_at: description: "the time in ms that the message was emitted" + # TODO : This should have been an integer instead of a number, need to fix this type: number error: description: "error trace message: the error object" diff --git a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml index 98bd414..0728a4e 100644 --- a/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml +++ b/protocol-models/src/main/resources/airbyte_protocol/v0/airbyte_protocol.yaml @@ -239,6 +239,7 @@ definitions: - STREAM_STATUS - ANALYTICS emitted_at: + # TODO : This should have been an integer instead of a number, need to fix this description: "the time in ms that the message was emitted" type: number error: