From c611436e71287cad85514df3bc76f949a8926ba8 Mon Sep 17 00:00:00 2001 From: Augustin Date: Wed, 18 Dec 2024 15:49:38 +0100 Subject: [PATCH 1/2] base-image-migration --- .../destination-redshift/metadata.yaml | 113 ++++++++++-------- docs/integrations/destinations/redshift.md | 9 +- 2 files changed, 69 insertions(+), 53 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 2b905e9e755f..921c9028ac90 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -2,10 +2,46 @@ data: ab_internal: ql: 200 sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/java-connector-base:1.0.0@sha256:be86e5684e1e6d9280512d3d8071b47153698fe08ad990949c8eeff02803201a connectorSubtype: database + connectorTestSuitesOptions: + - suite: unitTests + - suite: integrationTests + testSecrets: + - fileName: 1s1t_config.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: 1s1t_config_raw_schema_override.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: 1s1t_config_staging.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: 1s1t_config_staging_raw_schema_override.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: config_staging.json + name: SECRET_DESTINATION-REDSHIFT_STAGING__CREDS + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: config.json + name: SECRET_DESTINATION-REDSHIFT__CREDS + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 3.5.0 + dockerImageTag: 3.5.1 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift @@ -17,61 +53,40 @@ data: enabled: true oss: enabled: true + releaseStage: generally_available releases: breakingChanges: 2.0.0: - message: > - This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). - Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. - upgradeDeadline: "2024-03-15" + message: 'This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), + which provides better error handling, incremental delivery of data for large + syncs, and improved final table structures. To review the breaking changes, + and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). + These changes will likely require updates to downstream dbt / SQL models, + which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). + Selecting `Upgrade` will upgrade **all** connections using this destination + at their next sync. You can manually sync existing connections prior to + the next scheduled sync to start the upgrade early. + + ' + upgradeDeadline: '2024-03-15' 3.0.0: - message: > - Version 3.0.0 of destination-redshift removes support for the "standard inserts" mode. S3 staging was always preferred for being faster and less expensive, and as part of Airbyte 1.0, we are officially removing the inferior "standard inserts" mode. Upgrading to this version of the destination will require a configuration with an S3 staging area. - upgradeDeadline: "2024-07-31" + message: 'Version 3.0.0 of destination-redshift removes support for the "standard + inserts" mode. S3 staging was always preferred for being faster and less + expensive, and as part of Airbyte 1.0, we are officially removing the inferior + "standard inserts" mode. Upgrading to this version of the destination will + require a configuration with an S3 staging area. + + ' + upgradeDeadline: '2024-07-31' resourceRequirements: jobSpecific: - - jobType: sync - resourceRequirements: - memory_limit: 1Gi - memory_request: 1Gi - releaseStage: generally_available + - jobType: sync + resourceRequirements: + memory_limit: 1Gi + memory_request: 1Gi supportLevel: certified supportsDbt: true supportsRefreshes: true tags: - - language:java - connectorTestSuitesOptions: - - suite: unitTests - - suite: integrationTests - testSecrets: - - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG - fileName: 1s1t_config.json - secretStore: - type: GSM - alias: airbyte-connector-testing-secret-store - - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING - fileName: 1s1t_config_raw_schema_override.json - secretStore: - type: GSM - alias: airbyte-connector-testing-secret-store - - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING - fileName: 1s1t_config_staging.json - secretStore: - type: GSM - alias: airbyte-connector-testing-secret-store - - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING - fileName: 1s1t_config_staging_raw_schema_override.json - secretStore: - type: GSM - alias: airbyte-connector-testing-secret-store - - name: SECRET_DESTINATION-REDSHIFT_STAGING__CREDS - fileName: config_staging.json - secretStore: - type: GSM - alias: airbyte-connector-testing-secret-store - - name: SECRET_DESTINATION-REDSHIFT__CREDS - fileName: config.json - secretStore: - type: GSM - alias: airbyte-connector-testing-secret-store -metadataSpecVersion: "1.0" + - language:java +metadataSpecVersion: '1.0' diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 1e176451567c..96657e0d720f 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -222,10 +222,11 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 3.5.0 | 2024-09-18 | [45435](https://github.com/airbytehq/airbyte/pull/45435) | upgrade all dependencies | -| 3.4.4 | 2024-08-20 | [44476](https://github.com/airbytehq/airbyte/pull/44476) | Increase message parsing limit to 100mb | -| 3.4.3 | 2024-08-22 | [44526](https://github.com/airbytehq/airbyte/pull/44526) | Revert protocol compliance fix | -| 3.4.2 | 2024-08-15 | [42506](https://github.com/airbytehq/airbyte/pull/42506) | Fix bug in refreshes logic (already mitigated in platform, just fixing protocol compliance) | +| 3.5.1 | 2024-12-18 | [49903](https://github.com/airbytehq/airbyte/pull/49903) | Use a base image: airbyte/java-connector-base:1.0.0 | +| 3.5.0 | 2024-09-18 | [45435](https://github.com/airbytehq/airbyte/pull/45435) | upgrade all dependencies | +| 3.4.4 | 2024-08-20 | [44476](https://github.com/airbytehq/airbyte/pull/44476) | Increase message parsing limit to 100mb | +| 3.4.3 | 2024-08-22 | [44526](https://github.com/airbytehq/airbyte/pull/44526) | Revert protocol compliance fix | +| 3.4.2 | 2024-08-15 | [42506](https://github.com/airbytehq/airbyte/pull/42506) | Fix bug in refreshes logic (already mitigated in platform, just fixing protocol compliance) | | 3.4.1 | 2024-08-13 | [xxx](https://github.com/airbytehq/airbyte/pull/xxx) | Simplify Redshift Options | | 3.4.0 | 2024-07-23 | [42445](https://github.com/airbytehq/airbyte/pull/42445) | Respect the `drop cascade` option on raw tables | | 3.3.1 | 2024-07-15 | [41968](https://github.com/airbytehq/airbyte/pull/41968) | Don't hang forever on empty stream list; shorten error message on INCOMPLETE stream status | From 4e8ae3fe71360a8150764d1c81f6e76456110eea Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 18 Dec 2024 15:05:14 +0000 Subject: [PATCH 2/2] chore: auto-fix lint and format issues --- .../destination-redshift/metadata.yaml | 90 ++++++++++--------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 921c9028ac90..8cbdfc8914ab 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -6,39 +6,39 @@ data: baseImage: docker.io/airbyte/java-connector-base:1.0.0@sha256:be86e5684e1e6d9280512d3d8071b47153698fe08ad990949c8eeff02803201a connectorSubtype: database connectorTestSuitesOptions: - - suite: unitTests - - suite: integrationTests - testSecrets: - - fileName: 1s1t_config.json - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG - secretStore: - alias: airbyte-connector-testing-secret-store - type: GSM - - fileName: 1s1t_config_raw_schema_override.json - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING - secretStore: - alias: airbyte-connector-testing-secret-store - type: GSM - - fileName: 1s1t_config_staging.json - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING - secretStore: - alias: airbyte-connector-testing-secret-store - type: GSM - - fileName: 1s1t_config_staging_raw_schema_override.json - name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING - secretStore: - alias: airbyte-connector-testing-secret-store - type: GSM - - fileName: config_staging.json - name: SECRET_DESTINATION-REDSHIFT_STAGING__CREDS - secretStore: - alias: airbyte-connector-testing-secret-store - type: GSM - - fileName: config.json - name: SECRET_DESTINATION-REDSHIFT__CREDS - secretStore: - alias: airbyte-connector-testing-secret-store - type: GSM + - suite: unitTests + - suite: integrationTests + testSecrets: + - fileName: 1s1t_config.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: 1s1t_config_raw_schema_override.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: 1s1t_config_staging.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: 1s1t_config_staging_raw_schema_override.json + name: SECRET_DESTINATION-REDSHIFT_1S1T_CONFIG_STAGING_RAW_SCHEMA_OVERRIDE_DISABLE_TYPING_DEDUPING + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: config_staging.json + name: SECRET_DESTINATION-REDSHIFT_STAGING__CREDS + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM + - fileName: config.json + name: SECRET_DESTINATION-REDSHIFT__CREDS + secretStore: + alias: airbyte-connector-testing-secret-store + type: GSM connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc dockerImageTag: 3.5.1 @@ -57,7 +57,8 @@ data: releases: breakingChanges: 2.0.0: - message: 'This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), + message: + "This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). @@ -67,26 +68,27 @@ data: at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. - ' - upgradeDeadline: '2024-03-15' + " + upgradeDeadline: "2024-03-15" 3.0.0: - message: 'Version 3.0.0 of destination-redshift removes support for the "standard + message: + 'Version 3.0.0 of destination-redshift removes support for the "standard inserts" mode. S3 staging was always preferred for being faster and less expensive, and as part of Airbyte 1.0, we are officially removing the inferior "standard inserts" mode. Upgrading to this version of the destination will require a configuration with an S3 staging area. ' - upgradeDeadline: '2024-07-31' + upgradeDeadline: "2024-07-31" resourceRequirements: jobSpecific: - - jobType: sync - resourceRequirements: - memory_limit: 1Gi - memory_request: 1Gi + - jobType: sync + resourceRequirements: + memory_limit: 1Gi + memory_request: 1Gi supportLevel: certified supportsDbt: true supportsRefreshes: true tags: - - language:java -metadataSpecVersion: '1.0' + - language:java +metadataSpecVersion: "1.0"