From a19a6daf54998b0394c59d87bcc553708b1151ff Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Thu, 24 Aug 2023 14:15:17 +0200 Subject: [PATCH 001/201] remove super deep nesting that was breaking pmd (#8467) --- .../job/tracker/TrackingMetadata.java | 161 +++++++++--------- .../job/tracker/TrackingMetadataTest.java | 37 ++++ 2 files changed, 120 insertions(+), 78 deletions(-) diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java index 74d9082899b..7702d1c51a7 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/tracker/TrackingMetadata.java @@ -142,89 +142,94 @@ private static Map generateActorDefinitionVersionMetadata(final */ public static Map generateJobAttemptMetadata(final Job job) { final Builder metadata = ImmutableMap.builder(); - if (job != null) { - final List attempts = job.getAttempts(); - if (attempts != null && !attempts.isEmpty()) { - final Attempt lastAttempt = attempts.get(attempts.size() - 1); - if (lastAttempt.getOutput() != null && lastAttempt.getOutput().isPresent()) { - final JobOutput jobOutput = lastAttempt.getOutput().get(); - if (jobOutput.getSync() != null) { - final StandardSyncSummary syncSummary = jobOutput.getSync().getStandardSyncSummary(); - final SyncStats totalStats = syncSummary.getTotalStats(); - final NormalizationSummary normalizationSummary = jobOutput.getSync().getNormalizationSummary(); - - if (syncSummary.getStartTime() != null) { - metadata.put("sync_start_time", syncSummary.getStartTime()); - } - if (syncSummary.getEndTime() != null && syncSummary.getStartTime() != null) { - metadata.put("duration", Math.round((syncSummary.getEndTime() - syncSummary.getStartTime()) / 1000.0)); - } - if (syncSummary.getBytesSynced() != null) { - metadata.put("volume_mb", syncSummary.getBytesSynced()); - } - if (syncSummary.getRecordsSynced() != null) { - metadata.put("volume_rows", syncSummary.getRecordsSynced()); - } - if (totalStats.getSourceStateMessagesEmitted() != null) { - metadata.put("count_state_messages_from_source", syncSummary.getTotalStats().getSourceStateMessagesEmitted()); - } - if (totalStats.getDestinationStateMessagesEmitted() != null) { - metadata.put("count_state_messages_from_destination", syncSummary.getTotalStats().getDestinationStateMessagesEmitted()); - } - if (totalStats.getMaxSecondsBeforeSourceStateMessageEmitted() != null) { - metadata.put("max_seconds_before_source_state_message_emitted", - totalStats.getMaxSecondsBeforeSourceStateMessageEmitted()); - } - if (totalStats.getMeanSecondsBeforeSourceStateMessageEmitted() != null) { - metadata.put("mean_seconds_before_source_state_message_emitted", - totalStats.getMeanSecondsBeforeSourceStateMessageEmitted()); - } - if (totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted() != null) { - metadata.put("max_seconds_between_state_message_emit_and_commit", - totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()); - } - if (totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted() != null) { - metadata.put("mean_seconds_between_state_message_emit_and_commit", - totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()); - } + // Early returns in case we're missing the relevant stats. + if (job == null) { + return metadata.build(); + } + final List attempts = job.getAttempts(); + if (attempts == null || attempts.isEmpty()) { + return metadata.build(); + } + final Attempt lastAttempt = attempts.get(attempts.size() - 1); + if (lastAttempt.getOutput() == null || lastAttempt.getOutput().isEmpty()) { + return metadata.build(); + } + final JobOutput jobOutput = lastAttempt.getOutput().get(); + if (jobOutput.getSync() == null) { + return metadata.build(); + } + final StandardSyncSummary syncSummary = jobOutput.getSync().getStandardSyncSummary(); + final SyncStats totalStats = syncSummary.getTotalStats(); + final NormalizationSummary normalizationSummary = jobOutput.getSync().getNormalizationSummary(); - if (totalStats.getReplicationStartTime() != null) { - metadata.put("replication_start_time", totalStats.getReplicationStartTime()); - } - if (totalStats.getReplicationEndTime() != null) { - metadata.put("replication_end_time", totalStats.getReplicationEndTime()); - } - if (totalStats.getSourceReadStartTime() != null) { - metadata.put("source_read_start_time", totalStats.getSourceReadStartTime()); - } - if (totalStats.getSourceReadEndTime() != null) { - metadata.put("source_read_end_time", totalStats.getSourceReadEndTime()); - } - if (totalStats.getDestinationWriteStartTime() != null) { - metadata.put("destination_write_start_time", totalStats.getDestinationWriteStartTime()); - } - if (totalStats.getDestinationWriteEndTime() != null) { - metadata.put("destination_write_end_time", totalStats.getDestinationWriteEndTime()); - } + if (syncSummary.getStartTime() != null) { + metadata.put("sync_start_time", syncSummary.getStartTime()); + } + if (syncSummary.getEndTime() != null && syncSummary.getStartTime() != null) { + metadata.put("duration", Math.round((syncSummary.getEndTime() - syncSummary.getStartTime()) / 1000.0)); + } + if (syncSummary.getBytesSynced() != null) { + metadata.put("volume_mb", syncSummary.getBytesSynced()); + } + if (syncSummary.getRecordsSynced() != null) { + metadata.put("volume_rows", syncSummary.getRecordsSynced()); + } + if (totalStats.getSourceStateMessagesEmitted() != null) { + metadata.put("count_state_messages_from_source", syncSummary.getTotalStats().getSourceStateMessagesEmitted()); + } + if (totalStats.getDestinationStateMessagesEmitted() != null) { + metadata.put("count_state_messages_from_destination", syncSummary.getTotalStats().getDestinationStateMessagesEmitted()); + } + if (totalStats.getMaxSecondsBeforeSourceStateMessageEmitted() != null) { + metadata.put("max_seconds_before_source_state_message_emitted", + totalStats.getMaxSecondsBeforeSourceStateMessageEmitted()); + } + if (totalStats.getMeanSecondsBeforeSourceStateMessageEmitted() != null) { + metadata.put("mean_seconds_before_source_state_message_emitted", + totalStats.getMeanSecondsBeforeSourceStateMessageEmitted()); + } + if (totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted() != null) { + metadata.put("max_seconds_between_state_message_emit_and_commit", + totalStats.getMaxSecondsBetweenStateMessageEmittedandCommitted()); + } + if (totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted() != null) { + metadata.put("mean_seconds_between_state_message_emit_and_commit", + totalStats.getMeanSecondsBetweenStateMessageEmittedandCommitted()); + } - if (normalizationSummary != null) { - if (normalizationSummary.getStartTime() != null) { - metadata.put("normalization_start_time", normalizationSummary.getStartTime()); + if (totalStats.getReplicationStartTime() != null) { + metadata.put("replication_start_time", totalStats.getReplicationStartTime()); + } + if (totalStats.getReplicationEndTime() != null) { + metadata.put("replication_end_time", totalStats.getReplicationEndTime()); + } + if (totalStats.getSourceReadStartTime() != null) { + metadata.put("source_read_start_time", totalStats.getSourceReadStartTime()); + } + if (totalStats.getSourceReadEndTime() != null) { + metadata.put("source_read_end_time", totalStats.getSourceReadEndTime()); + } + if (totalStats.getDestinationWriteStartTime() != null) { + metadata.put("destination_write_start_time", totalStats.getDestinationWriteStartTime()); + } + if (totalStats.getDestinationWriteEndTime() != null) { + metadata.put("destination_write_end_time", totalStats.getDestinationWriteEndTime()); + } - } - if (normalizationSummary.getEndTime() != null) { - metadata.put("normalization_end_time", normalizationSummary.getEndTime()); - } - } - } - } + if (normalizationSummary != null) { + if (normalizationSummary.getStartTime() != null) { + metadata.put("normalization_start_time", normalizationSummary.getStartTime()); - final List failureReasons = failureReasonsList(attempts); - if (!failureReasons.isEmpty()) { - metadata.put("failure_reasons", failureReasonsListAsJson(failureReasons).toString()); - metadata.put("main_failure_reason", failureReasonAsJson(failureReasons.get(0)).toString()); - } } + if (normalizationSummary.getEndTime() != null) { + metadata.put("normalization_end_time", normalizationSummary.getEndTime()); + } + } + + final List failureReasons = failureReasonsList(attempts); + if (!failureReasons.isEmpty()) { + metadata.put("failure_reasons", failureReasonsListAsJson(failureReasons).toString()); + metadata.put("main_failure_reason", failureReasonAsJson(failureReasons.get(0)).toString()); } return metadata.build(); } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java index b7e44136d4a..08ca67abf92 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java @@ -4,11 +4,13 @@ package io.airbyte.persistence.job.tracker; +import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import io.airbyte.config.AttemptSyncConfig; +import io.airbyte.config.JobConfig; import io.airbyte.config.JobOutput; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardSync; @@ -18,10 +20,12 @@ import io.airbyte.persistence.job.models.Attempt; import io.airbyte.persistence.job.models.AttemptStatus; import io.airbyte.persistence.job.models.Job; +import io.airbyte.persistence.job.models.JobStatus; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.Test; @@ -79,4 +83,37 @@ void testgenerateJobAttemptMetadataWithNulls() { assertEquals(expected, actual); } + @Test + void testGenerateJobAttemptMetadataToleratesNullInputs() { + // Null job. + assertTrue(TrackingMetadata.generateJobAttemptMetadata(null).isEmpty()); + + // There is a job, but it has no attempts. + final Job jobWithNoAttempts = new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithNoAttempts).isEmpty()); + + // There is a job, and it has an attempt, but the attempt has null output. + final Attempt mockAttemptWithNullOutput = mock(Attempt.class); + when(mockAttemptWithNullOutput.getOutput()).thenReturn(null); + final Job jobWithNullOutput = + new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithNullOutput).isEmpty()); + + // There is a job, and it has an attempt, but the attempt has empty output. + final Attempt mockAttemptWithEmptyOutput = mock(Attempt.class); + when(mockAttemptWithEmptyOutput.getOutput()).thenReturn(Optional.empty()); + final Job jobWithEmptyOutput = + new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithEmptyOutput).isEmpty()); + + // There is a job, and it has an attempt, and the attempt has output, but the output has no sync + // info. + final Attempt mockAttemptWithOutput = mock(Attempt.class); + final JobOutput mockJobOutputWithoutSync = mock(JobOutput.class); + when(mockAttemptWithOutput.getOutput()).thenReturn(Optional.of(mockJobOutputWithoutSync)); + when(mockJobOutputWithoutSync.getSync()).thenReturn(null); + final Job jobWithoutSyncInfo = new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(mockAttemptWithOutput), JobStatus.PENDING, 0L, 0, 0); + assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithoutSyncInfo).isEmpty()); + } + } From af0a1a3b440b9121badfe0f6bd44313ba33e21b9 Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Thu, 24 Aug 2023 15:36:27 +0200 Subject: [PATCH 002/201] make sure we re-run integration tests (#8520) --- airbyte-tests/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-tests/build.gradle.kts b/airbyte-tests/build.gradle.kts index 0e54d02d56a..662ee82d51a 100644 --- a/airbyte-tests/build.gradle.kts +++ b/airbyte-tests/build.gradle.kts @@ -64,6 +64,8 @@ fun registerTestSuite(name: String, type: String, dirName: String, deps: JvmComp events = setOf(TestLogEvent.PASSED, TestLogEvent.FAILED) } shouldRunAfter(suites.named("test")) + // Ensure they re-run since these are integration tests. + outputs.upToDateWhen { false } } } } From 363e6568f454afe89d7847b476a3978a1dd883bc Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 24 Aug 2023 10:52:43 -0400 Subject: [PATCH 003/201] =?UTF-8?q?=F0=9F=AA=9F=20Add=20message=20to=20use?= =?UTF-8?q?rs=20about=20auto-propagation=20setting=20(#8482)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NonBreakingChangesPreferenceField.tsx | 59 +++++++++++-------- airbyte-webapp/src/locales/en.json | 1 + 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceField.tsx index 58d678706b9..f6688c24706 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceField.tsx @@ -1,9 +1,10 @@ import { FieldProps } from "formik"; import { useMemo } from "react"; -import { useIntl } from "react-intl"; +import { FormattedMessage, useIntl } from "react-intl"; import { ControlLabels } from "components"; import { DropDown } from "components/ui/DropDown"; +import { Message } from "components/ui/Message"; import { NonBreakingChangesPreference } from "core/request/AirbyteClient"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; @@ -12,12 +13,18 @@ import { useExperiment } from "hooks/services/Experiment"; import { FormFieldLayout } from "./FormFieldLayout"; export const NonBreakingChangesPreferenceField: React.FC> = ({ field, form }) => { + const { connection, mode } = useConnectionFormService(); const autoPropagationEnabled = useExperiment("autopropagation.enabled", true); const autoPropagationPrefix = autoPropagationEnabled ? "autopropagation." : ""; const labelKey = autoPropagationEnabled ? "connectionForm.nonBreakingChangesPreference.autopropagation.label" : "connectionForm.nonBreakingChangesPreference.label"; + const showAutoPropagationMessage = + mode === "edit" && + field.value !== connection.nonBreakingChangesPreference && + (field.value === "propagate_columns" || field.value === "propagate_fully"); + const supportedPreferences = useMemo(() => { if (autoPropagationEnabled) { return [ @@ -40,28 +47,34 @@ export const NonBreakingChangesPreferenceField: React.FC> = ( })); }, [formatMessage, supportedPreferences, autoPropagationPrefix]); - const { mode } = useConnectionFormService(); - return ( - - - form.setFieldValue(field.name, value)} - /> - + <> + + + form.setFieldValue(field.name, value)} + /> + + {showAutoPropagationMessage && ( + } + /> + )} + ); }; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 0ff65b2d952..ef2323caf16 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -213,6 +213,7 @@ "connectionForm.nonBreakingChangesPreference.autopropagation.disable": "Pause connection", "connectionForm.nonBreakingChangesPreference.autopropagation.propagate_columns": "Propagate column changes only", "connectionForm.nonBreakingChangesPreference.autopropagation.propagate_fully": "Propagate all changes", + "connectionForm.nonBreakingChangesPreference.autopropagtion.message": "Auto-propagation will start with your next scheduled sync. To apply immediately, run a manual sync after saving the connection.", "connectionForm.schemaChangesBackdrop.message": "Please review the schema updates before making changes to the connection", "connectionForm.jobHistory": "Job History", "connectionForm.update.success": "Connection update succeeded", From 5cfd590f669ef9d929d1c99403ff98bccac6a509 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Thu, 24 Aug 2023 18:37:09 +0200 Subject: [PATCH 004/201] Call supportstateupdater when getting new definitions (#8508) Co-authored-by: Pedro S. Lopez --- .../io/airbyte/bootloader/BootloaderTest.java | 9 +++- .../DestinationDefinitionsHandler.java | 10 ++++ .../handlers/SourceDefinitionsHandler.java | 9 ++++ .../DestinationDefinitionsHandlerTest.java | 42 ++++++++++------ .../SourceDefinitionsHandlerTest.java | 45 ++++++++++------- .../persistence/SupportStateUpdater.java | 9 ++++ .../config/init/ApplyDefinitionsHelper.java | 10 +++- .../init/ApplyDefinitionsHelperTest.java | 50 ++++++++++++++++--- .../src/main/kotlin/FlagDefinitions.kt | 2 + 9 files changed, 144 insertions(+), 42 deletions(-) diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index ff56104a407..3ef79255115 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -20,6 +20,7 @@ import io.airbyte.config.init.DeclarativeSourceUpdater; import io.airbyte.config.init.PostLoadExecutor; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.config.specs.LocalDefinitionsProvider; import io.airbyte.db.factory.DSLContextFactory; @@ -132,7 +133,9 @@ void testBootloaderAppBlankDb() throws Exception { val jobsDatabaseMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); val jobsPersistence = new DefaultJobPersistence(jobDatabase); val protocolVersionChecker = new ProtocolVersionChecker(jobsPersistence, airbyteProtocolRange, configRepository, definitionsProvider); - val applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient); + val supportStateUpdater = new SupportStateUpdater(configRepository); + val applyDefinitionsHelper = + new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient, supportStateUpdater); final CdkVersionProvider cdkVersionProvider = mock(CdkVersionProvider.class); when(cdkVersionProvider.getCdkVersion()).thenReturn(CDK_VERSION); val declarativeSourceUpdater = new DeclarativeSourceUpdater(configRepository, cdkVersionProvider); @@ -185,8 +188,10 @@ void testRequiredVersionUpgradePredicate() throws Exception { jobsDatabaseInitializationTimeoutMs, MoreResources.readResource(DatabaseConstants.JOBS_INITIAL_SCHEMA_PATH)); val jobsDatabaseMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); val jobsPersistence = new DefaultJobPersistence(jobDatabase); + val supportStateUpdater = new SupportStateUpdater(configRepository); val protocolVersionChecker = new ProtocolVersionChecker(jobsPersistence, airbyteProtocolRange, configRepository, definitionsProvider); - val applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient); + val applyDefinitionsHelper = + new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient, supportStateUpdater); final CdkVersionProvider cdkVersionProvider = mock(CdkVersionProvider.class); when(cdkVersionProvider.getCdkVersion()).thenReturn(CDK_VERSION); val declarativeSourceUpdater = new DeclarativeSourceUpdater(configRepository, cdkVersionProvider); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java index b75ca9b54d8..dd609bc6bca 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java @@ -35,12 +35,14 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.Workspace; import io.airbyte.validation.json.JsonValidationException; import jakarta.inject.Singleton; @@ -70,6 +72,7 @@ public class DestinationDefinitionsHandler { private final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private final RemoteDefinitionsProvider remoteDefinitionsProvider; private final DestinationHandler destinationHandler; + private final SupportStateUpdater supportStateUpdater; private final FeatureFlagClient featureFlagClient; @VisibleForTesting @@ -78,12 +81,14 @@ public DestinationDefinitionsHandler(final ConfigRepository configRepository, final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper, final RemoteDefinitionsProvider remoteDefinitionsProvider, final DestinationHandler destinationHandler, + final SupportStateUpdater supportStateUpdater, final FeatureFlagClient featureFlagClient) { this.configRepository = configRepository; this.uuidSupplier = uuidSupplier; this.actorDefinitionHandlerHelper = actorDefinitionHandlerHelper; this.remoteDefinitionsProvider = remoteDefinitionsProvider; this.destinationHandler = destinationHandler; + this.supportStateUpdater = supportStateUpdater; this.featureFlagClient = featureFlagClient; } @@ -284,6 +289,11 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); } + if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { + final StandardDestinationDefinition updatedDestinationDefinition = configRepository + .getStandardDestinationDefinition(destinationDefinitionUpdate.getDestinationDefinitionId()); + supportStateUpdater.updateSupportStatesForDestinationDefinition(updatedDestinationDefinition); + } return buildDestinationDefinitionRead(newDestination, newVersion); } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java index 753bc26e169..f37c177e521 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java @@ -36,11 +36,13 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.SourceDefinition; import io.airbyte.featureflag.Workspace; import io.airbyte.validation.json.JsonValidationException; @@ -70,6 +72,7 @@ public class SourceDefinitionsHandler { private final RemoteDefinitionsProvider remoteDefinitionsProvider; private final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private final SourceHandler sourceHandler; + private final SupportStateUpdater supportStateUpdater; private final FeatureFlagClient featureFlagClient; @Inject @@ -78,12 +81,14 @@ public SourceDefinitionsHandler(final ConfigRepository configRepository, final ActorDefinitionHandlerHelper actorDefinitionHandlerHelper, final RemoteDefinitionsProvider remoteDefinitionsProvider, final SourceHandler sourceHandler, + final SupportStateUpdater supportStateUpdater, final FeatureFlagClient featureFlagClient) { this.configRepository = configRepository; this.uuidSupplier = uuidSupplier; this.actorDefinitionHandlerHelper = actorDefinitionHandlerHelper; this.remoteDefinitionsProvider = remoteDefinitionsProvider; this.sourceHandler = sourceHandler; + this.supportStateUpdater = supportStateUpdater; this.featureFlagClient = featureFlagClient; } @@ -284,6 +289,10 @@ public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); } + if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { + final StandardSourceDefinition updatedSourceDefinition = configRepository.getStandardSourceDefinition(newSource.getSourceDefinitionId()); + supportStateUpdater.updateSupportStatesForSourceDefinition(updatedSourceDefinition); + } return buildSourceDefinitionRead(newSource, newVersion); } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java index 445fa7e5690..dc7149efa39 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java @@ -15,6 +15,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -52,12 +53,14 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.TestClient; import io.airbyte.featureflag.Workspace; import io.airbyte.protocol.models.ConnectorSpecification; @@ -94,6 +97,7 @@ class DestinationDefinitionsHandlerTest { private ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private RemoteDefinitionsProvider remoteDefinitionsProvider; private DestinationHandler destinationHandler; + private SupportStateUpdater supportStateUpdater; private UUID workspaceId; private UUID organizationId; private FeatureFlagClient featureFlagClient; @@ -110,6 +114,7 @@ void setUp() { actorDefinitionHandlerHelper = mock(ActorDefinitionHandlerHelper.class); remoteDefinitionsProvider = mock(RemoteDefinitionsProvider.class); destinationHandler = mock(DestinationHandler.class); + supportStateUpdater = mock(SupportStateUpdater.class); workspaceId = UUID.randomUUID(); organizationId = UUID.randomUUID(); featureFlagClient = mock(TestClient.class); @@ -119,6 +124,7 @@ void setUp() { actorDefinitionHandlerHelper, remoteDefinitionsProvider, destinationHandler, + supportStateUpdater, featureFlagClient); when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); @@ -716,24 +722,26 @@ void testCreateCustomDestinationDefinitionShouldCheckProtocolVersion() throws UR @ParameterizedTest @ValueSource(booleans = {true, false}) @DisplayName("updateDestinationDefinition should correctly update a destinationDefinition") - void testUpdateDestination(final boolean ingestBreakingChangesFF) throws ConfigNotFoundException, IOException, JsonValidationException { - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(ingestBreakingChangesFF); + void testUpdateDestination(final boolean runSupportStateUpdaterFlagValue) throws ConfigNotFoundException, IOException, JsonValidationException { + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(runSupportStateUpdaterFlagValue); - when(configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId())).thenReturn(destinationDefinition); - when(configRepository.getActorDefinitionVersion(destinationDefinition.getDefaultVersionId())) - .thenReturn(destinationDefinitionVersion); - final DestinationDefinitionRead currentDestination = destinationDefinitionsHandler - .getDestinationDefinition( - new DestinationDefinitionIdRequestBody().destinationDefinitionId(destinationDefinition.getDestinationDefinitionId())); - final String currentTag = currentDestination.getDockerImageTag(); final String newDockerImageTag = "averydifferenttag"; - assertNotEquals(newDockerImageTag, currentTag); - final StandardDestinationDefinition updatedDestination = Jsons.clone(destinationDefinition).withDefaultVersionId(null); final ActorDefinitionVersion updatedDestinationDefVersion = generateVersionFromDestinationDefinition(updatedDestination) - .withDockerImageTag(newDockerImageTag); + .withDockerImageTag(newDockerImageTag) + .withVersionId(UUID.randomUUID()); + + final StandardDestinationDefinition persistedUpdatedDestination = + Jsons.clone(updatedDestination).withDefaultVersionId(updatedDestinationDefVersion.getVersionId()); + + when(configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId())) + .thenReturn(destinationDefinition) // Call at the beginning of the method + .thenReturn(persistedUpdatedDestination); // Call after we've persisted + + when(configRepository.getActorDefinitionVersion(destinationDefinition.getDefaultVersionId())) + .thenReturn(destinationDefinitionVersion); when(actorDefinitionHandlerHelper.defaultDefinitionVersionFromUpdate(destinationDefinitionVersion, ActorType.DESTINATION, newDockerImageTag, destinationDefinition.getCustom())).thenReturn(updatedDestinationDefVersion); @@ -751,11 +759,13 @@ void testUpdateDestination(final boolean ingestBreakingChangesFF) throws ConfigN destinationDefinition.getCustom()); verify(actorDefinitionHandlerHelper).getBreakingChanges(updatedDestinationDefVersion, ActorType.DESTINATION); verify(configRepository).writeDestinationDefinitionAndDefaultVersion(updatedDestination, updatedDestinationDefVersion, breakingChanges); - if (ingestBreakingChangesFF) { - verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + if (runSupportStateUpdaterFlagValue) { + verify(supportStateUpdater).updateSupportStatesForDestinationDefinition(persistedUpdatedDestination); + } else { + verifyNoInteractions(supportStateUpdater); } - - verifyNoMoreInteractions(actorDefinitionHandlerHelper); + verifyNoMoreInteractions(actorDefinitionHandlerHelper, supportStateUpdater); } @Test diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java index 71495ed40d8..2ddd94abd13 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java @@ -16,6 +16,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -53,11 +54,13 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.SourceDefinition; import io.airbyte.featureflag.TestClient; import io.airbyte.featureflag.Workspace; @@ -92,6 +95,7 @@ class SourceDefinitionsHandlerTest { private ActorDefinitionHandlerHelper actorDefinitionHandlerHelper; private RemoteDefinitionsProvider remoteDefinitionsProvider; private SourceHandler sourceHandler; + private SupportStateUpdater supportStateUpdater; private UUID workspaceId; private UUID organizationId; private FeatureFlagClient featureFlagClient; @@ -104,6 +108,7 @@ void setUp() { actorDefinitionHandlerHelper = mock(ActorDefinitionHandlerHelper.class); remoteDefinitionsProvider = mock(RemoteDefinitionsProvider.class); sourceHandler = mock(SourceHandler.class); + supportStateUpdater = mock(SupportStateUpdater.class); workspaceId = UUID.randomUUID(); organizationId = UUID.randomUUID(); sourceDefinition = generateSourceDefinition(); @@ -111,7 +116,9 @@ void setUp() { featureFlagClient = mock(TestClient.class); sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, uuidSupplier, actorDefinitionHandlerHelper, remoteDefinitionsProvider, sourceHandler, - featureFlagClient); + supportStateUpdater, featureFlagClient); + + when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); } private StandardSourceDefinition generateSourceDefinition() { @@ -649,24 +656,26 @@ void testCreateCustomSourceDefinitionShouldCheckProtocolVersion() throws URISynt @ParameterizedTest @ValueSource(booleans = {true, false}) @DisplayName("updateSourceDefinition should correctly update a sourceDefinition") - void testUpdateSource(final boolean ingestBreakingChangesFF) throws ConfigNotFoundException, IOException, JsonValidationException { - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(ingestBreakingChangesFF); + void testUpdateSource(final boolean runSupportStateUpdaterFlagValue) throws ConfigNotFoundException, IOException, JsonValidationException { + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(runSupportStateUpdaterFlagValue); - when(configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())).thenReturn(sourceDefinition); - when(configRepository.getActorDefinitionVersion(sourceDefinition.getDefaultVersionId())) - .thenReturn(sourceDefinitionVersion); - final SourceDefinitionRead currentSource = sourceDefinitionsHandler - .getSourceDefinition( - new SourceDefinitionIdRequestBody().sourceDefinitionId(sourceDefinition.getSourceDefinitionId())); - final String currentTag = currentSource.getDockerImageTag(); final String newDockerImageTag = "averydifferenttag"; - assertNotEquals(newDockerImageTag, currentTag); - final StandardSourceDefinition updatedSource = Jsons.clone(sourceDefinition).withDefaultVersionId(null); final ActorDefinitionVersion updatedSourceDefVersion = generateVersionFromSourceDefinition(updatedSource) - .withDockerImageTag(newDockerImageTag); + .withDockerImageTag(newDockerImageTag) + .withVersionId(UUID.randomUUID()); + + final StandardSourceDefinition persistedUpdatedSource = + Jsons.clone(updatedSource).withDefaultVersionId(updatedSourceDefVersion.getVersionId()); + + when(configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())) + .thenReturn(sourceDefinition) // Call at the beginning of the method + .thenReturn(persistedUpdatedSource); // Call after we've persisted + + when(configRepository.getActorDefinitionVersion(sourceDefinition.getDefaultVersionId())) + .thenReturn(sourceDefinitionVersion); when(actorDefinitionHandlerHelper.defaultDefinitionVersionFromUpdate(sourceDefinitionVersion, ActorType.SOURCE, newDockerImageTag, sourceDefinition.getCustom())).thenReturn(updatedSourceDefVersion); @@ -684,11 +693,13 @@ void testUpdateSource(final boolean ingestBreakingChangesFF) throws ConfigNotFou sourceDefinition.getCustom()); verify(actorDefinitionHandlerHelper).getBreakingChanges(updatedSourceDefVersion, ActorType.SOURCE); verify(configRepository).writeSourceDefinitionAndDefaultVersion(updatedSource, updatedSourceDefVersion, breakingChanges); - if (ingestBreakingChangesFF) { - verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + if (runSupportStateUpdaterFlagValue) { + verify(supportStateUpdater).updateSupportStatesForSourceDefinition(persistedUpdatedSource); + } else { + verifyNoInteractions(supportStateUpdater); } - - verifyNoMoreInteractions(actorDefinitionHandlerHelper); + verifyNoMoreInteractions(actorDefinitionHandlerHelper, supportStateUpdater); } @Test diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java index 317267335ca..a4aff2ae307 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java @@ -22,6 +22,8 @@ import java.util.UUID; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Updates the support state of actor definition versions according to breaking changes. @@ -49,6 +51,7 @@ public static SupportStateUpdate merge(final SupportStateUpdate a, final Support } private final ConfigRepository configRepository; + private static final Logger LOGGER = LoggerFactory.getLogger(SupportStateUpdater.class); public SupportStateUpdater(final ConfigRepository configRepository) { this.configRepository = configRepository; @@ -135,9 +138,11 @@ private SupportState calcVersionSupportState(final Version version, */ public void updateSupportStatesForSourceDefinition(final StandardSourceDefinition sourceDefinition) throws ConfigNotFoundException, IOException { if (!sourceDefinition.getCustom()) { + LOGGER.info("Updating support states for source definition: {}", sourceDefinition.getName()); final ActorDefinitionVersion defaultActorDefinitionVersion = configRepository.getActorDefinitionVersion(sourceDefinition.getDefaultVersionId()); final Version currentDefaultVersion = new Version(defaultActorDefinitionVersion.getDockerImageTag()); updateSupportStatesForActorDefinition(sourceDefinition.getSourceDefinitionId(), currentDefaultVersion); + LOGGER.info("Finished updating support states for source definition: {}", sourceDefinition.getName()); } } @@ -147,10 +152,12 @@ public void updateSupportStatesForSourceDefinition(final StandardSourceDefinitio public void updateSupportStatesForDestinationDefinition(final StandardDestinationDefinition destinationDefinition) throws ConfigNotFoundException, IOException { if (!destinationDefinition.getCustom()) { + LOGGER.info("Updating support states for destination definition: {}", destinationDefinition.getName()); final ActorDefinitionVersion defaultActorDefinitionVersion = configRepository.getActorDefinitionVersion(destinationDefinition.getDefaultVersionId()); final Version currentDefaultVersion = new Version(defaultActorDefinitionVersion.getDockerImageTag()); updateSupportStatesForActorDefinition(destinationDefinition.getDestinationDefinitionId(), currentDefaultVersion); + LOGGER.info("Finished updating support states for destination definition: {}", destinationDefinition.getName()); } } @@ -174,6 +181,7 @@ private Version getVersionTag(final List actorDefinition * Updates the version support states for all source and destination definitions. */ public void updateSupportStates() throws IOException { + LOGGER.info("Updating support states for all definitions"); final List sourceDefinitions = configRepository.listPublicSourceDefinitions(false); final List destinationDefinitions = configRepository.listPublicDestinationDefinitions(false); final List breakingChanges = configRepository.listBreakingChanges(); @@ -203,6 +211,7 @@ public void updateSupportStates() throws IOException { } executeSupportStateUpdate(comboSupportStateUpdate); + LOGGER.info("Finished updating support states for all definitions"); } /** diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java index f0f870960dd..9b0f38d91fc 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java @@ -17,9 +17,11 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.IngestBreakingChanges; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.Workspace; import io.airbyte.persistence.job.JobPersistence; import io.airbyte.validation.json.JsonValidationException; @@ -51,6 +53,7 @@ public class ApplyDefinitionsHelper { private final JobPersistence jobPersistence; private final ConfigRepository configRepository; private final FeatureFlagClient featureFlagClient; + private final SupportStateUpdater supportStateUpdater; private int newConnectorCount; private int changedConnectorCount; private List allBreakingChanges; @@ -59,10 +62,12 @@ public class ApplyDefinitionsHelper { public ApplyDefinitionsHelper(@Named("seedDefinitionsProvider") final DefinitionsProvider definitionsProvider, final JobPersistence jobPersistence, final ConfigRepository configRepository, - final FeatureFlagClient featureFlagClient) { + final FeatureFlagClient featureFlagClient, + final SupportStateUpdater supportStateUpdater) { this.definitionsProvider = definitionsProvider; this.jobPersistence = jobPersistence; this.configRepository = configRepository; + this.supportStateUpdater = supportStateUpdater; this.featureFlagClient = featureFlagClient; } @@ -103,6 +108,9 @@ public void apply(final boolean updateAll) throws JsonValidationException, IOExc if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { configRepository.writeActorDefinitionBreakingChanges(allBreakingChanges); } + if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { + supportStateUpdater.updateSupportStates(); + } LOGGER.info("New connectors added: {}", newConnectorCount); LOGGER.info("Version changes applied: {}", changedConnectorCount); diff --git a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java index d13b8cd68dd..3a607431c01 100644 --- a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java +++ b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java @@ -24,9 +24,11 @@ import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.IngestBreakingChanges; +import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.TestClient; import io.airbyte.featureflag.Workspace; import io.airbyte.persistence.job.JobPersistence; @@ -54,6 +56,7 @@ class ApplyDefinitionsHelperTest { private ConfigRepository configRepository; private DefinitionsProvider definitionsProvider; private JobPersistence jobPersistence; + private SupportStateUpdater supportStateUpdater; private FeatureFlagClient featureFlagClient; private ApplyDefinitionsHelper applyDefinitionsHelper; @@ -101,11 +104,14 @@ void setup() { configRepository = mock(ConfigRepository.class); definitionsProvider = mock(DefinitionsProvider.class); jobPersistence = mock(JobPersistence.class); + supportStateUpdater = mock(SupportStateUpdater.class); featureFlagClient = mock(TestClient.class); - applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobPersistence, configRepository, featureFlagClient); + applyDefinitionsHelper = + new ApplyDefinitionsHelper(definitionsProvider, jobPersistence, configRepository, featureFlagClient, supportStateUpdater); when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); // Default calls to empty. when(definitionsProvider.getDestinationDefinitions()).thenReturn(Collections.emptyList()); @@ -143,8 +149,9 @@ void testNewConnectorIsWritten(final boolean updateAll) ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3)); verify(configRepository).writeActorDefinitionBreakingChanges(List.of()); + verify(supportStateUpdater).updateSupportStates(); - verifyNoMoreInteractions(configRepository); + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @ParameterizedTest @@ -170,8 +177,9 @@ void testConnectorIsUpdatedIfItIsNotInUse(final boolean updateAll) ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); + verify(supportStateUpdater).updateSupportStates(); - verifyNoMoreInteractions(configRepository); + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @ParameterizedTest @@ -202,7 +210,9 @@ void testUpdateBehaviorIfConnectorIsInUse(final boolean updateAll) verify(configRepository).updateStandardDestinationDefinition(ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2)); verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); } - verifyNoMoreInteractions(configRepository); + verify(supportStateUpdater).updateSupportStates(); + + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @ParameterizedTest @@ -240,7 +250,33 @@ void testDefinitionsFiltering(final boolean updateAll) ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); - verifyNoMoreInteractions(configRepository); + verify(supportStateUpdater).updateSupportStates(); + + verifyNoMoreInteractions(configRepository, supportStateUpdater); + } + + @Test + void testTurnOffRunSupportStateUpdaterFeatureFlag() throws JsonValidationException, ConfigNotFoundException, IOException { + when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(false); + + when(definitionsProvider.getSourceDefinitions()).thenReturn(List.of(SOURCE_POSTGRES_2)); + when(definitionsProvider.getDestinationDefinitions()).thenReturn(List.of(DESTINATION_S3_2)); + + applyDefinitionsHelper.apply(true); + verifyConfigRepositoryGetInteractions(); + + verify(configRepository).writeSourceDefinitionAndDefaultVersion( + ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), + ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), + ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); + verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), + ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), + ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); + verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); + + verify(supportStateUpdater, never()).updateSupportStates(); + verifyNoMoreInteractions(configRepository, supportStateUpdater); } @Test @@ -263,7 +299,9 @@ void testTurnOffIngestBreakingChangesFeatureFlag() throws JsonValidationExceptio ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); verify(configRepository, never()).writeActorDefinitionBreakingChanges(any()); - verifyNoMoreInteractions(configRepository); + verify(supportStateUpdater).updateSupportStates(); + + verifyNoMoreInteractions(configRepository, supportStateUpdater); } private static List getExpectedBreakingChanges() { diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index fb46158d4aa..5a3ffb223d0 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -65,6 +65,8 @@ object UseActorScopedDefaultVersions : Temporary(key = "connectors.useA object IngestBreakingChanges : Temporary(key = "connectors.ingestBreakingChanges", default = true) +object RunSupportStateUpdater : Temporary(key = "connectors.runSupportStateUpdater", default = true) + object RefreshSchemaPeriod : Temporary(key = "refreshSchema.period.hours", default = 24) object ConcurrentSourceStreamRead : Temporary(key = "concurrent.source.stream.read", default = false) From 67cb8c4ca6b80a43a245941952bf7895cc04c7f5 Mon Sep 17 00:00:00 2001 From: terencecho <3916587+terencecho@users.noreply.github.com> Date: Thu, 24 Aug 2023 10:44:35 -0700 Subject: [PATCH 005/201] airbyte-api-server: Fix jobs list endpoint (#8517) --- .../kotlin/io/airbyte/api/server/services/JobService.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt index c6cfcc4a9ba..30cec987772 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt @@ -168,9 +168,9 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi .createdAtEnd(jobsFilter.createdAtEnd) .updatedAtStart(jobsFilter.updatedAtStart) .updatedAtEnd(jobsFilter.updatedAtEnd) - .orderByField(JobListRequestBody.OrderByFieldEnum.fromValue(orderByField.name)) + .orderByField(JobListRequestBody.OrderByFieldEnum.valueOf(orderByField.name)) .orderByMethod( - JobListRequestBody.OrderByMethodEnum.fromValue(orderByMethod.name), + JobListRequestBody.OrderByMethodEnum.valueOf(orderByMethod.name), ) val response = try { @@ -215,8 +215,8 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi .createdAtEnd(jobsFilter.createdAtEnd) .updatedAtStart(jobsFilter.updatedAtStart) .updatedAtEnd(jobsFilter.updatedAtEnd) - .orderByField(OrderByFieldEnum.fromValue(orderByField.name)) - .orderByMethod(OrderByMethodEnum.fromValue(orderByMethod.name)) + .orderByField(OrderByFieldEnum.valueOf(orderByField.name)) + .orderByMethod(OrderByMethodEnum.valueOf(orderByMethod.name)) val response = try { configApiClient.getJobListForWorkspaces(requestBody, userInfo) From 6a113351adb73e60681fb89ab5e5d8d159927c95 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Thu, 24 Aug 2023 13:07:48 -0700 Subject: [PATCH 006/201] Clear JobMdc when done (#8526) --- .../config/helpers/LogClientSingleton.java | 16 ++++++++++---- .../temporal/TemporalAttemptExecution.java | 6 +++++ .../AppendToAttemptLogActivityImpl.java | 22 +++++++++++++------ ...obCreationAndStatusUpdateActivityImpl.java | 2 ++ ...obCreationAndStatusUpdateActivityTest.java | 12 ++++++---- 5 files changed, 43 insertions(+), 15 deletions(-) diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java index c17a51eabe2..0bc9e03694a 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java @@ -183,17 +183,25 @@ public void deleteLogs(final WorkerEnvironment workerEnvironment, final LogConfi * * @param workerEnvironment environment of worker. * @param logConfigs configuration for logs - * @param path log path + * @param path log path, if path is null, it will clear the JobMdc instead */ public void setJobMdc(final WorkerEnvironment workerEnvironment, final LogConfigs logConfigs, final Path path) { if (shouldUseLocalLogs(workerEnvironment)) { LOGGER.debug("Setting docker job mdc"); - final String resolvedPath = path.resolve(LogClientSingleton.LOG_FILENAME).toString(); - MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, resolvedPath); + if (path != null) { + final String resolvedPath = path.resolve(LogClientSingleton.LOG_FILENAME).toString(); + MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, resolvedPath); + } else { + MDC.remove(LogClientSingleton.JOB_LOG_PATH_MDC_KEY); + } } else { LOGGER.debug("Setting kube job mdc"); createCloudClientIfNull(logConfigs); - MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + if (path != null) { + MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); + } else { + MDC.remove(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY); + } } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index 73a9fda1113..6c9158442c5 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -165,6 +165,8 @@ public OUTPUT get() { } } catch (final Exception e) { throw Activity.wrap(e); + } finally { + mdcSetter.accept(null); } } @@ -197,6 +199,8 @@ private Thread getWorkerThread(final Worker worker, final Complet } catch (final Throwable e) { LOGGER.info("Completing future exceptionally...", e); outputFuture.completeExceptionally(e); + } finally { + mdcSetter.accept(null); } }); } @@ -244,6 +248,8 @@ private Runnable getCancellationChecker(final Worker worker, cancellationHandler.checkAndHandleCancellation(onCancellationCallback); } catch (final Exception e) { LOGGER.debug("Cancellation checker exception", e); + } finally { + mdcSetter.accept(null); } }; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AppendToAttemptLogActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AppendToAttemptLogActivityImpl.java index 7002fdb0c04..ce29555d78b 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AppendToAttemptLogActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/AppendToAttemptLogActivityImpl.java @@ -47,15 +47,19 @@ public LogOutput log(final LogInput input) { setMdc(input); - final var msg = input.getMessage(); + try { + final var msg = input.getMessage(); - switch (input.getLevel()) { - case ERROR -> logger.error(msg); - case WARN -> logger.warn(msg); - default -> logger.info(msg); - } + switch (input.getLevel()) { + case ERROR -> logger.error(msg); + case WARN -> logger.warn(msg); + default -> logger.info(msg); + } - return new LogOutput(true); + return new LogOutput(true); + } finally { + unsetMdc(); + } } private void setMdc(final LogInput input) { @@ -67,4 +71,8 @@ private void setMdc(final LogInput input) { logClientSingleton.setJobMdc(workerEnvironment, logConfigs, jobRoot); } + private void unsetMdc() { + logClientSingleton.setJobMdc(workerEnvironment, logConfigs, null); + } + } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java index abcdd3b6528..64923505fe4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityImpl.java @@ -126,6 +126,8 @@ public AttemptNumberCreationOutput createNewAttemptNumber(final AttemptCreationI } catch (final IOException e) { log.error("createNewAttemptNumber for job {} failed with exception: {}", input.getJobId(), e.getMessage(), e); throw new RetryableException(e); + } finally { + LogClientSingleton.getInstance().setJobMdc(workerEnvironment, logConfigs, null); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java index 0e568b48e4c..06abd60b7ce 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/JobCreationAndStatusUpdateActivityTest.java @@ -64,12 +64,12 @@ import java.util.Set; import java.util.UUID; import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InOrder; -import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -89,8 +89,7 @@ class JobCreationAndStatusUpdateActivityTest { @Mock private Path mPath; - @Mock - private WorkerEnvironment mWorkerEnvironment; + private final WorkerEnvironment mWorkerEnvironment = WorkerEnvironment.DOCKER; @Mock private LogConfigs mLogConfigs; @@ -110,7 +109,6 @@ class JobCreationAndStatusUpdateActivityTest { @Mock private JobsApi jobsApi; - @InjectMocks private JobCreationAndStatusUpdateActivityImpl jobCreationAndStatusUpdateActivity; private static final UUID CONNECTION_ID = UUID.randomUUID(); @@ -135,6 +133,12 @@ class JobCreationAndStatusUpdateActivityTest { new FailureReason() .withFailureOrigin(FailureOrigin.SOURCE))); + @BeforeEach + void beforeEach() { + jobCreationAndStatusUpdateActivity = new JobCreationAndStatusUpdateActivityImpl( + mJobPersistence, mPath, mWorkerEnvironment, mLogConfigs, mJobNotifier, mJobtracker, mConfigRepository, mJobErrorReporter, jobsApi); + } + @Nested class Creation { From dacfafff4103dde7433242cabc7adc11c97fa4ee Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Thu, 24 Aug 2023 16:15:37 -0700 Subject: [PATCH 007/201] InstanceConfiguration API: Add `/setup` endpoint and GET default User/Org info (#8400) Co-authored-by: Joey Marshment-Howell --- .../src/main/openapi/cloud-config.yaml | 2 + airbyte-api/src/main/openapi/config.yaml | 62 ++++++ .../io/airbyte/bootloader/Bootloader.java | 5 +- .../config/DatabaseBeanFactory.java | 18 ++ airbyte-commons-license/build.gradle | 2 + .../annotation/RequiresAirbyteProEnabled.java | 2 +- .../condition/AirbyteProEnabledCondition.java | 7 +- .../AirbyteConfigurationBeanFactory.java | 9 + .../InstanceConfigurationHandler.java | 170 +++++++++++++++ .../DefaultInstanceConfigurationHandler.java | 38 ---- .../InstanceConfigurationHandler.java | 16 -- .../ProInstanceConfigurationHandler.java | 65 ------ .../InstanceConfigurationHandlerTest.java | 202 ++++++++++++++++++ ...faultInstanceConfigurationHandlerTest.java | 38 ---- .../ProInstanceConfigurationHandlerTest.java | 53 ----- .../main/java/io/airbyte/config/Configs.java | 10 + .../persistence/OrganizationPersistence.java | 18 +- .../config/persistence/UserPersistence.java | 15 ++ .../persistence/WorkspacePersistence.java | 26 +++ .../OrganizationPersistenceTest.java | 13 ++ .../InstanceConfigurationApiController.java | 10 +- .../src/main/resources/application.yml | 2 +- ...nstanceConfigurationApiControllerTest.java | 15 +- airbyte-webapp/cypress/commands/common.ts | 4 - airbyte-webapp/cypress/e2e/onboarding.cy.ts | 17 +- airbyte-webapp/src/App.tsx | 6 +- .../settings/SetupForm/SetupForm.tsx | 17 +- .../core/api/hooks/cloud/cloudWorkspaces.ts | 2 +- .../cloud/usePrefetchCloudWorkspaceData.ts | 2 +- airbyte-webapp/src/core/api/hooks/index.ts | 1 + .../core/api/hooks/instanceConfiguration.ts | 23 +- airbyte-webapp/src/core/api/hooks/users.ts | 19 ++ .../src/core/services/auth/AuthContext.ts | 76 +++++++ .../core/services/auth/DefaultAuthService.tsx | 24 +++ .../services/auth}/KeycloakAuthService.tsx | 46 ++-- .../src/core/services/auth/OSSAuthService.tsx | 17 ++ .../services}/auth/RequestAuthMiddleware.ts | 0 .../src/core/services/auth/index.ts | 3 + .../auth => core/utils}/freeEmailProviders.ts | 3 + airbyte-webapp/src/locales/en.json | 2 + .../src/packages/cloud/cloudRoutes.tsx | 19 +- .../useShowEnrollmentModal.tsx | 4 +- .../packages/cloud/lib/auth/AuthProviders.ts | 5 - .../cloud/lib/auth/GoogleAuthService.ts | 3 +- .../cloud/services/AppServicesProvider.tsx | 2 +- .../cloud/services/auth/AuthService.tsx | 70 +----- .../launchdarkly/LDExperimentService.tsx | 2 +- .../thirdParty/launchdarkly/contexts.ts | 4 +- .../thirdParty/zendesk/ZendeskProvider.tsx | 3 +- .../cloud/views/AcceptEmailInvite.tsx | 10 +- .../cloud/views/FirebaseActionRoute.tsx | 28 ++- .../cloud/views/UpcomingFeaturesPage.tsx | 2 +- .../ConfirmPasswordResetPage.tsx | 9 +- .../cloud/views/auth/LoginPage/LoginPage.tsx | 18 +- .../views/auth/OAuthLogin/OAuthLogin.test.tsx | 23 +- .../views/auth/OAuthLogin/OAuthLogin.tsx | 10 +- .../ResetPasswordPage/ResetPasswordPage.tsx | 9 +- .../views/auth/SignupPage/SignupPage.tsx | 6 +- .../auth/SignupPage/components/SignupForm.tsx | 9 +- .../components/EmailVerificationHint.tsx | 6 +- .../components/RemainingCredits.tsx | 7 +- .../layout/CloudMainView/CloudMainView.tsx | 7 +- .../AccountSettingsView.tsx | 22 +- .../components/EmailSection.tsx | 2 +- .../components/LogoutSection.tsx | 22 ++ .../components/NameSection.tsx | 9 +- .../components/PasswordSection.tsx | 14 +- .../UsersSettingsView/UsersSettingsView.tsx | 2 +- airbyte-webapp/src/pages/routes.tsx | 5 +- .../mock-data/mockInstanceConfig.ts | 4 + 70 files changed, 948 insertions(+), 448 deletions(-) create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java delete mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandler.java delete mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/InstanceConfigurationHandler.java delete mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandler.java create mode 100644 airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java delete mode 100644 airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandlerTest.java delete mode 100644 airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandlerTest.java create mode 100644 airbyte-webapp/src/core/api/hooks/users.ts create mode 100644 airbyte-webapp/src/core/services/auth/AuthContext.ts create mode 100644 airbyte-webapp/src/core/services/auth/DefaultAuthService.tsx rename airbyte-webapp/src/{services => core/services/auth}/KeycloakAuthService.tsx (65%) create mode 100644 airbyte-webapp/src/core/services/auth/OSSAuthService.tsx rename airbyte-webapp/src/{packages/cloud/lib => core/services}/auth/RequestAuthMiddleware.ts (100%) create mode 100644 airbyte-webapp/src/core/services/auth/index.ts rename airbyte-webapp/src/{packages/cloud/services/auth => core/utils}/freeEmailProviders.ts (99%) delete mode 100644 airbyte-webapp/src/packages/cloud/lib/auth/AuthProviders.ts create mode 100644 airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/LogoutSection.tsx diff --git a/airbyte-api/src/main/openapi/cloud-config.yaml b/airbyte-api/src/main/openapi/cloud-config.yaml index 1dc9d22f369..a65d21bfac1 100644 --- a/airbyte-api/src/main/openapi/cloud-config.yaml +++ b/airbyte-api/src/main/openapi/cloud-config.yaml @@ -1006,6 +1006,8 @@ components: enum: # - auth0 - google_identity_platform + - airbyte + - keycloak # WORKSPACE WorkspaceUserRead: type: object diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 09c3b408a82..3e9af588735 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3515,6 +3515,27 @@ paths: $ref: "#/components/schemas/InstanceConfigurationResponse" "401": description: Fetching instance configuration failed. + /v1/instance_configuration/setup: + post: + summary: Setup an instance with user and organization information. + tags: + - instance_configuration + operationId: setupInstanceConfiguration + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceConfigurationSetupRequestBody" + responses: + "200": + description: Successfully setup instance. + content: + application/json: + schema: + $ref: "#/components/schemas/InstanceConfigurationResponse" + "401": + description: Instance setup failed. + /v1/jobs/retry_states/create_or_update: post: summary: Creates or updates a retry state for a job. @@ -7492,6 +7513,10 @@ components: required: - edition - webappUrl + - initialSetupComplete + - defaultUserId + - defaultOrganizationId + - defaultWorkspaceId properties: edition: type: string @@ -7508,6 +7533,43 @@ components: $ref: "#/components/schemas/AuthConfiguration" webappUrl: type: string + initialSetupComplete: + type: boolean + defaultUserId: + type: string + format: uuid + defaultOrganizationId: + type: string + format: uuid + defaultWorkspaceId: + type: string + format: uuid + InstanceConfigurationSetupRequestBody: + type: object + required: + - workspaceId + - email + - anonymousDataCollection + - initialSetupComplete + - displaySetupWizard + properties: + workspaceId: + type: string + format: uuid + email: + type: string + anonymousDataCollection: + type: boolean + initialSetupComplete: + type: boolean + displaySetupWizard: + type: boolean + userName: + description: Optional name of the user to create. Defaults to 'Default User' if not specified. + type: string + organizationName: + description: Optional name of the organization to create. Defaults to 'Default Organization' if not specified. + type: string StreamStatusRead: type: object required: diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java index 9fa87aa361f..6ebfb878d0b 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/Bootloader.java @@ -13,6 +13,7 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.init.PostLoadExecutor; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.OrganizationPersistence; import io.airbyte.db.init.DatabaseInitializationException; import io.airbyte.db.init.DatabaseInitializer; import io.airbyte.db.instance.DatabaseMigrator; @@ -188,7 +189,9 @@ private void createWorkspaceIfNoneExists(final ConfigRepository configRepository .withInitialSetupComplete(false) .withDisplaySetupWizard(true) .withTombstone(false) - .withDefaultGeography(Geography.AUTO); + .withDefaultGeography(Geography.AUTO) + // attach this new workspace to the Default Organization which should always exist at this point. + .withOrganizationId(OrganizationPersistence.DEFAULT_ORGANIZATION_ID); // NOTE: it's safe to use the NoSecrets version since we know that the user hasn't supplied any // secrets yet. configRepository.writeStandardWorkspaceNoSecrets(workspace); diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java index 6dcea132e30..57cfc10e9c8 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java @@ -6,6 +6,9 @@ import io.airbyte.commons.resources.MoreResources; import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.OrganizationPersistence; +import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.config.persistence.WorkspacePersistence; import io.airbyte.db.Database; import io.airbyte.db.check.impl.JobsDatabaseAvailabilityCheck; import io.airbyte.db.factory.DatabaseCheckFactory; @@ -147,4 +150,19 @@ public DatabaseMigrator jobsDatabaseMigrator(@Named("jobsDatabase") final Databa return new JobsDatabaseMigrator(jobsDatabase, jobsFlyway); } + @Singleton + public UserPersistence userPersistence(@Named("configDatabase") final Database configDatabase) { + return new UserPersistence(configDatabase); + } + + @Singleton + public OrganizationPersistence organizationPersistence(@Named("configDatabase") final Database configDatabase) { + return new OrganizationPersistence(configDatabase); + } + + @Singleton + public WorkspacePersistence workspacePersistence(@Named("configDatabase") final Database configDatabase) { + return new WorkspacePersistence(configDatabase); + } + } diff --git a/airbyte-commons-license/build.gradle b/airbyte-commons-license/build.gradle index 7cf60b23f9b..cd78c944d64 100644 --- a/airbyte-commons-license/build.gradle +++ b/airbyte-commons-license/build.gradle @@ -14,6 +14,8 @@ dependencies { annotationProcessor libs.lombok implementation project(':airbyte-commons') + implementation project(':airbyte-commons-micronaut') + implementation project(':airbyte-config:config-models') testAnnotationProcessor platform(libs.micronaut.bom) testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor diff --git a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java index 7f195a7c5fb..091343599b4 100644 --- a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java +++ b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/annotation/RequiresAirbyteProEnabled.java @@ -18,7 +18,7 @@ */ @Documented @Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE}) +@Target({ElementType.TYPE, ElementType.METHOD}) @Inherited @Requires(condition = AirbyteProEnabledCondition.class) public @interface RequiresAirbyteProEnabled { diff --git a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java index f9cb6932ebc..6572a7b1ec8 100644 --- a/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java +++ b/airbyte-commons-license/src/main/java/io/airbyte/commons/license/condition/AirbyteProEnabledCondition.java @@ -4,6 +4,7 @@ package io.airbyte.commons.license.condition; +import io.airbyte.config.Configs.AirbyteEdition; import io.micronaut.context.condition.Condition; import io.micronaut.context.condition.ConditionContext; import lombok.extern.slf4j.Slf4j; @@ -18,10 +19,8 @@ public class AirbyteProEnabledCondition implements Condition { @Override public boolean matches(ConditionContext context) { - log.warn("inside the pro enabled condition!"); - final var edition = context.getProperty("airbyte.edition", String.class).orElse("community"); - log.warn("got edition: " + edition); - return "pro".equals(edition); + final AirbyteEdition edition = context.getBean(AirbyteEdition.class); + return edition.equals(AirbyteEdition.PRO); } } diff --git a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java index 80f9abe3369..7c686a3535f 100644 --- a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java +++ b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbyteConfigurationBeanFactory.java @@ -5,6 +5,7 @@ package io.airbyte.micronaut.config; import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.config.Configs.AirbyteEdition; import io.airbyte.config.Configs.DeploymentMode; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; @@ -33,4 +34,12 @@ public DeploymentMode deploymentMode(@Value("${airbyte.deployment-mode}") final return convertToEnum(deploymentMode, DeploymentMode::valueOf, DeploymentMode.OSS); } + /** + * Fetch the configured edition of the Airbyte instance. Defaults to COMMUNITY. + */ + @Singleton + public AirbyteEdition airbyteEdition(@Value("${airbyte.edition:COMMUNITY}") final String airbyteEdition) { + return convertToEnum(airbyteEdition.toUpperCase(), AirbyteEdition::valueOf, AirbyteEdition.COMMUNITY); + } + } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java new file mode 100644 index 00000000000..34d299af10e --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.handlers; + +import io.airbyte.api.model.generated.AuthConfiguration; +import io.airbyte.api.model.generated.InstanceConfigurationResponse; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.EditionEnum; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.LicenseTypeEnum; +import io.airbyte.api.model.generated.InstanceConfigurationSetupRequestBody; +import io.airbyte.api.model.generated.WorkspaceUpdate; +import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; +import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.license.ActiveAirbyteLicense; +import io.airbyte.config.Configs.AirbyteEdition; +import io.airbyte.config.Organization; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.User; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.OrganizationPersistence; +import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.validation.json.JsonValidationException; +import io.micronaut.context.annotation.Value; +import jakarta.inject.Singleton; +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; + +/** + * InstanceConfigurationHandler. Javadocs suppressed because api docs should be used as source of + * truth. + */ +@SuppressWarnings("MissingJavadocMethod") +@Slf4j +@Singleton +public class InstanceConfigurationHandler { + + private final String webappUrl; + private final AirbyteEdition airbyteEdition; + private final Optional airbyteKeycloakConfiguration; + private final Optional activeAirbyteLicense; + private final ConfigRepository configRepository; + private final WorkspacesHandler workspacesHandler; + private final UserPersistence userPersistence; + private final OrganizationPersistence organizationPersistence; + + // the injected webapp-url value defaults to `null` to preserve backwards compatibility. + // TODO remove the default value once configurations are standardized to always include a + // webapp-url. + public InstanceConfigurationHandler(@Value("${airbyte.webapp-url:null}") final String webappUrl, + final AirbyteEdition airbyteEdition, + final Optional airbyteKeycloakConfiguration, + final Optional activeAirbyteLicense, + final ConfigRepository configRepository, + final WorkspacesHandler workspacesHandler, + final UserPersistence userPersistence, + final OrganizationPersistence organizationPersistence) { + this.webappUrl = webappUrl; + this.airbyteEdition = airbyteEdition; + this.airbyteKeycloakConfiguration = airbyteKeycloakConfiguration; + this.activeAirbyteLicense = activeAirbyteLicense; + this.configRepository = configRepository; + this.workspacesHandler = workspacesHandler; + this.userPersistence = userPersistence; + this.organizationPersistence = organizationPersistence; + } + + public InstanceConfigurationResponse getInstanceConfiguration() throws IOException, ConfigNotFoundException { + final StandardWorkspace defaultWorkspace = getDefaultWorkspace(); + + return new InstanceConfigurationResponse() + .webappUrl(webappUrl) + .edition(Enums.convertTo(airbyteEdition, EditionEnum.class)) + .licenseType(getLicenseType()) + .auth(getAuthConfiguration()) + .initialSetupComplete(defaultWorkspace.getInitialSetupComplete()) + .defaultUserId(getDefaultUserId()) + .defaultOrganizationId(getDefaultOrganizationId()) + .defaultWorkspaceId(defaultWorkspace.getWorkspaceId()); + } + + public InstanceConfigurationResponse setupInstanceConfiguration(final InstanceConfigurationSetupRequestBody requestBody) + throws IOException, JsonValidationException, ConfigNotFoundException { + + // Update the default organization and user with the provided information + updateDefaultOrganization(requestBody); + updateDefaultUser(requestBody); + + // Update the underlying workspace to mark the initial setup as complete + workspacesHandler.updateWorkspace(new WorkspaceUpdate() + .workspaceId(requestBody.getWorkspaceId()) + .email(requestBody.getEmail()) + .displaySetupWizard(requestBody.getDisplaySetupWizard()) + .anonymousDataCollection(requestBody.getAnonymousDataCollection()) + .initialSetupComplete(requestBody.getInitialSetupComplete())); + + // Return the updated instance configuration + return getInstanceConfiguration(); + } + + private LicenseTypeEnum getLicenseType() { + if (airbyteEdition.equals(AirbyteEdition.PRO) && activeAirbyteLicense.isPresent()) { + return Enums.convertTo(activeAirbyteLicense.get().getLicenseType(), LicenseTypeEnum.class); + } else { + return null; + } + } + + private AuthConfiguration getAuthConfiguration() { + if (airbyteEdition.equals(AirbyteEdition.PRO) && airbyteKeycloakConfiguration.isPresent()) { + return new AuthConfiguration() + .clientId(airbyteKeycloakConfiguration.get().getWebClientId()) + .defaultRealm(airbyteKeycloakConfiguration.get().getAirbyteRealm()); + } else { + return null; + } + } + + private UUID getDefaultUserId() throws IOException { + return userPersistence.getDefaultUser().orElseThrow(() -> new IllegalStateException("Default user does not exist.")).getUserId(); + } + + private void updateDefaultUser(final InstanceConfigurationSetupRequestBody requestBody) throws IOException { + final User defaultUser = userPersistence.getDefaultUser().orElseThrow(() -> new IllegalStateException("Default user does not exist.")); + // email is a required request property, so always set it. + defaultUser.setEmail(requestBody.getEmail()); + + // name is currently optional, so only set it if it is provided. + if (requestBody.getUserName() != null) { + defaultUser.setName(requestBody.getUserName()); + } + + userPersistence.writeUser(defaultUser); + } + + private UUID getDefaultOrganizationId() throws IOException, ConfigNotFoundException { + return organizationPersistence.getDefaultOrganization() + .orElseThrow(() -> new IllegalStateException("Default organization does not exist.")) + .getOrganizationId(); + } + + private void updateDefaultOrganization(final InstanceConfigurationSetupRequestBody requestBody) throws IOException { + final Organization defaultOrganization = + organizationPersistence.getDefaultOrganization().orElseThrow(() -> new IllegalStateException("Default organization does not exist.")); + + // email is a required request property, so always set it. + defaultOrganization.setEmail(requestBody.getEmail()); + + // name is currently optional, so only set it if it is provided. + if (requestBody.getOrganizationName() != null) { + defaultOrganization.setName(requestBody.getOrganizationName()); + } + + organizationPersistence.updateOrganization(defaultOrganization); + } + + // Currently, the default workspace is simply the first workspace created by the bootloader. This is + // hacky, but + // historically, the first workspace is used to store instance-level preferences. + // TODO introduce a proper means of persisting instance-level preferences instead of using the first + // workspace as a proxy. + private StandardWorkspace getDefaultWorkspace() throws IOException { + return configRepository.listStandardWorkspaces(true).stream().findFirst() + .orElseThrow(() -> new IllegalStateException("Default workspace does not exist.")); + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandler.java deleted file mode 100644 index e12b5f35bb2..00000000000 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandler.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.micronaut.context.annotation.Value; -import jakarta.inject.Singleton; -import lombok.extern.slf4j.Slf4j; - -/** - * Default InstanceConfigurationHandler that returns the default "community" configuration for - * Airbyte. For Airbyte Pro, this singleton should be replaced with the - * {@link ProInstanceConfigurationHandler} that returns Pro-specific details. - */ -@Slf4j -@Singleton -public class DefaultInstanceConfigurationHandler implements InstanceConfigurationHandler { - - private final String webappUrl; - - // the injected webapp-url value defaults to `null` to preserve backwards compatibility. - // TODO remove the default value once configurations are standardized to always include airbyte.yml - public DefaultInstanceConfigurationHandler(@Value("${airbyte.webapp-url:null}") final String webappUrl) { - this.webappUrl = webappUrl; - } - - @Override - public InstanceConfigurationResponse getInstanceConfiguration() { - return new InstanceConfigurationResponse() - .edition(InstanceConfigurationResponse.EditionEnum.COMMUNITY) - .licenseType(null) - .auth(null) - .webappUrl(webappUrl); - } - -} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/InstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/InstanceConfigurationHandler.java deleted file mode 100644 index 144fb16fc43..00000000000 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/InstanceConfigurationHandler.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import io.airbyte.api.model.generated.InstanceConfigurationResponse; - -/** - * Handles requests for the Instance Configuration API endpoint. - */ -public interface InstanceConfigurationHandler { - - InstanceConfigurationResponse getInstanceConfiguration(); - -} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandler.java deleted file mode 100644 index 06b21065909..00000000000 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandler.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import io.airbyte.api.model.generated.AuthConfiguration; -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.api.model.generated.InstanceConfigurationResponse.LicenseTypeEnum; -import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; -import io.airbyte.commons.license.ActiveAirbyteLicense; -import io.airbyte.commons.license.annotation.RequiresAirbyteProEnabled; -import io.micronaut.context.annotation.Replaces; -import io.micronaut.context.annotation.Value; -import jakarta.inject.Singleton; -import lombok.extern.slf4j.Slf4j; - -/** - * Pro-specific version of the InstanceConfigurationHandler that includes license and auth - * configuration details for the instance. - */ -@Slf4j -@Singleton -@RequiresAirbyteProEnabled -@Replaces(DefaultInstanceConfigurationHandler.class) -public class ProInstanceConfigurationHandler implements InstanceConfigurationHandler { - - private final AirbyteKeycloakConfiguration keycloakConfiguration; - private final ActiveAirbyteLicense activeAirbyteLicense; - private final String webappUrl; - - public ProInstanceConfigurationHandler(final AirbyteKeycloakConfiguration keycloakConfiguration, - final ActiveAirbyteLicense activeAirbyteLicense, - @Value("${airbyte.webapp-url}") final String webappUrl) { - this.keycloakConfiguration = keycloakConfiguration; - this.activeAirbyteLicense = activeAirbyteLicense; - this.webappUrl = webappUrl; - } - - @Override - public InstanceConfigurationResponse getInstanceConfiguration() { - final LicenseTypeEnum licenseTypeEnum = LicenseTypeEnum.fromValue(activeAirbyteLicense.getLicenseType().getValue()); - - return new InstanceConfigurationResponse() - .edition(InstanceConfigurationResponse.EditionEnum.PRO) - .licenseType(licenseTypeEnum) - .auth(getAuthConfiguration()) - .webappUrl(webappUrl); - } - - private AuthConfiguration getAuthConfiguration() { - return new AuthConfiguration() - .clientId(getWebClientId()) - .defaultRealm(getAirbyteRealm()); - } - - private String getAirbyteRealm() { - return keycloakConfiguration.getAirbyteRealm(); - } - - private String getWebClientId() { - return keycloakConfiguration.getWebClientId(); - } - -} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java new file mode 100644 index 00000000000..92b41151958 --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.handlers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.api.model.generated.AuthConfiguration; +import io.airbyte.api.model.generated.InstanceConfigurationResponse; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.EditionEnum; +import io.airbyte.api.model.generated.InstanceConfigurationResponse.LicenseTypeEnum; +import io.airbyte.api.model.generated.InstanceConfigurationSetupRequestBody; +import io.airbyte.api.model.generated.WorkspaceUpdate; +import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; +import io.airbyte.commons.license.ActiveAirbyteLicense; +import io.airbyte.commons.license.AirbyteLicense; +import io.airbyte.commons.license.AirbyteLicense.LicenseType; +import io.airbyte.config.Configs.AirbyteEdition; +import io.airbyte.config.Organization; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.User; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.config.persistence.OrganizationPersistence; +import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class InstanceConfigurationHandlerTest { + + private static final String WEBAPP_URL = "http://localhost:8000"; + private static final String AIRBYTE_REALM = "airbyte"; + private static final String WEB_CLIENT_ID = "airbyte-webapp"; + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID USER_ID = UUID.randomUUID(); + private static final UUID ORGANIZATION_ID = UUID.randomUUID(); + private static final String DEFAULT_ORG_NAME = "Default Org Name"; + private static final String DEFAULT_USER_NAME = "Default User Name"; + + @Mock + private ConfigRepository mConfigRepository; + @Mock + private UserPersistence mUserPersistence; + @Mock + private WorkspacesHandler mWorkspacesHandler; + @Mock + private OrganizationPersistence mOrganizationPersistence; + + private AirbyteKeycloakConfiguration keycloakConfiguration; + private ActiveAirbyteLicense activeAirbyteLicense; + private InstanceConfigurationHandler instanceConfigurationHandler; + + @BeforeEach + void setup() throws IOException { + keycloakConfiguration = new AirbyteKeycloakConfiguration(); + keycloakConfiguration.setAirbyteRealm(AIRBYTE_REALM); + keycloakConfiguration.setWebClientId(WEB_CLIENT_ID); + + activeAirbyteLicense = new ActiveAirbyteLicense(); + activeAirbyteLicense.setLicense(new AirbyteLicense(LicenseType.PRO)); + + when(mUserPersistence.getDefaultUser()).thenReturn( + Optional.of(new User() + .withUserId(USER_ID) + .withName(DEFAULT_USER_NAME))); + + when(mOrganizationPersistence.getDefaultOrganization()).thenReturn( + Optional.of(new Organization() + .withOrganizationId(ORGANIZATION_ID) + .withName(DEFAULT_ORG_NAME) + .withUserId(USER_ID))); + } + + @ParameterizedTest + @CsvSource({ + "true, true", + "true, false", + "false, true", + "false, false" + }) + void testGetInstanceConfiguration(final boolean isPro, final boolean isInitialSetupComplete) + throws IOException, ConfigNotFoundException { + when(mConfigRepository.listStandardWorkspaces(true)).thenReturn( + List.of(new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withInitialSetupComplete(isInitialSetupComplete))); + + instanceConfigurationHandler = new InstanceConfigurationHandler( + WEBAPP_URL, + isPro ? AirbyteEdition.PRO : AirbyteEdition.COMMUNITY, + isPro ? Optional.of(keycloakConfiguration) : Optional.empty(), + isPro ? Optional.of(activeAirbyteLicense) : Optional.empty(), + mConfigRepository, + mWorkspacesHandler, + mUserPersistence, + mOrganizationPersistence); + + final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() + .edition(isPro ? EditionEnum.PRO : EditionEnum.COMMUNITY) + .webappUrl(WEBAPP_URL) + .licenseType(isPro ? LicenseTypeEnum.PRO : null) + .auth(isPro ? new AuthConfiguration() + .clientId(WEB_CLIENT_ID) + .defaultRealm(AIRBYTE_REALM) : null) + .initialSetupComplete(isInitialSetupComplete) + .defaultUserId(USER_ID) + .defaultOrganizationId(ORGANIZATION_ID) + .defaultWorkspaceId(WORKSPACE_ID); + + final InstanceConfigurationResponse actual = instanceConfigurationHandler.getInstanceConfiguration(); + + assertEquals(expected, actual); + } + + @ParameterizedTest + @CsvSource({ + "true, true", + "true, false", + "false, true", + "false, false" + }) + void testSetupInstanceConfiguration(final boolean userNamePresent, final boolean orgNamePresent) + throws IOException, JsonValidationException, ConfigNotFoundException { + when(mConfigRepository.listStandardWorkspaces(true)) + .thenReturn(List.of(new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withInitialSetupComplete(true))); // after the handler's update, the workspace should have initialSetupComplete: true when retrieved + + instanceConfigurationHandler = new InstanceConfigurationHandler( + WEBAPP_URL, + AirbyteEdition.PRO, + Optional.of(keycloakConfiguration), + Optional.of(activeAirbyteLicense), + mConfigRepository, + mWorkspacesHandler, + mUserPersistence, + mOrganizationPersistence); + + final String userName = userNamePresent ? "test user" : DEFAULT_USER_NAME; + final String orgName = orgNamePresent ? "test org" : DEFAULT_ORG_NAME; + final String email = "test@airbyte.com"; + + final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() + .edition(EditionEnum.PRO) + .webappUrl(WEBAPP_URL) + .licenseType(LicenseTypeEnum.PRO) + .auth(new AuthConfiguration() + .clientId(WEB_CLIENT_ID) + .defaultRealm(AIRBYTE_REALM)) + .initialSetupComplete(true) + .defaultUserId(USER_ID) + .defaultOrganizationId(ORGANIZATION_ID) + .defaultWorkspaceId(WORKSPACE_ID); + + final InstanceConfigurationResponse actual = instanceConfigurationHandler.setupInstanceConfiguration( + new InstanceConfigurationSetupRequestBody() + .workspaceId(WORKSPACE_ID) + .email(email) + .initialSetupComplete(true) + .anonymousDataCollection(true) + .displaySetupWizard(true) + .userName(userName) + .organizationName(orgName)); + + assertEquals(expected, actual); + + // verify the user was updated with the email and name from the request + verify(mUserPersistence).writeUser(eq(new User() + .withUserId(USER_ID) + .withEmail(email) + .withName(userName))); + + // verify the organization was updated with the name from the request + verify(mOrganizationPersistence).updateOrganization(eq(new Organization() + .withOrganizationId(ORGANIZATION_ID) + .withName(orgName) + .withEmail(email) + .withUserId(USER_ID))); + + verify(mWorkspacesHandler).updateWorkspace(eq(new WorkspaceUpdate() + .workspaceId(WORKSPACE_ID) + .email(email) + .displaySetupWizard(true) + .anonymousDataCollection(true) + .initialSetupComplete(true))); + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandlerTest.java deleted file mode 100644 index f050bee3ebc..00000000000 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/DefaultInstanceConfigurationHandlerTest.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.api.model.generated.InstanceConfigurationResponse.EditionEnum; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class DefaultInstanceConfigurationHandlerTest { - - private static final String WEBAPP_URL = "http://localhost:8000"; - - private DefaultInstanceConfigurationHandler defaultInstanceConfigurationHandler; - - @BeforeEach - void setup() { - defaultInstanceConfigurationHandler = new DefaultInstanceConfigurationHandler(WEBAPP_URL); - } - - @Test - void testGetInstanceConfiguration() { - final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() - .edition(EditionEnum.COMMUNITY) - .licenseType(null) - .auth(null) - .webappUrl(WEBAPP_URL); - - final InstanceConfigurationResponse actual = defaultInstanceConfigurationHandler.getInstanceConfiguration(); - - assertEquals(expected, actual); - } - -} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandlerTest.java deleted file mode 100644 index acfd9a9cd37..00000000000 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/instance_configuration/ProInstanceConfigurationHandlerTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.commons.server.handlers.instance_configuration; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.api.model.generated.AuthConfiguration; -import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; -import io.airbyte.commons.license.ActiveAirbyteLicense; -import io.airbyte.commons.license.AirbyteLicense; -import io.airbyte.commons.license.AirbyteLicense.LicenseType; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class ProInstanceConfigurationHandlerTest { - - private static final String WEBAPP_URL = "http://localhost:8000"; - private static final String AIRBYTE_REALM = "airbyte"; - private static final String WEB_CLIENT_ID = "airbyte-webapp"; - - private ProInstanceConfigurationHandler proInstanceConfigurationHandler; - - @BeforeEach - void setup() { - final ActiveAirbyteLicense activeAirbyteLicense = new ActiveAirbyteLicense(); - activeAirbyteLicense.setLicense(new AirbyteLicense(LicenseType.PRO)); - - final AirbyteKeycloakConfiguration keycloakConfiguration = new AirbyteKeycloakConfiguration(); - keycloakConfiguration.setAirbyteRealm(AIRBYTE_REALM); - keycloakConfiguration.setWebClientId(WEB_CLIENT_ID); - - proInstanceConfigurationHandler = new ProInstanceConfigurationHandler(keycloakConfiguration, activeAirbyteLicense, WEBAPP_URL); - } - - @Test - void testGetInstanceConfiguration() { - final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() - .edition(InstanceConfigurationResponse.EditionEnum.PRO) - .licenseType(InstanceConfigurationResponse.LicenseTypeEnum.PRO) - .auth(new AuthConfiguration() - .clientId(WEB_CLIENT_ID) - .defaultRealm(AIRBYTE_REALM)) - .webappUrl(WEBAPP_URL); - - final InstanceConfigurationResponse actual = proInstanceConfigurationHandler.getInstanceConfiguration(); - - assertEquals(expected, actual); - } - -} diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index da0959e1a10..eb9a39718bf 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -812,4 +812,14 @@ enum SecretPersistenceType { AWS_SECRET_MANAGER } + /** + * The configured Airbyte edition for the instance. By default, an Airbyte instance is configured as + * Community edition. If configured as Pro edition, the instance will perform a license check and + * activate additional features if valid. + */ + enum AirbyteEdition { + COMMUNITY, + PRO + } + } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java index 21025fa85d1..aa59ced8f1e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/OrganizationPersistence.java @@ -15,6 +15,7 @@ import java.time.OffsetDateTime; import java.util.Optional; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; @@ -25,13 +26,16 @@ * Handle persisting Permission to the Config Database and perform all SQL queries. * */ +@Slf4j public class OrganizationPersistence { private final ExceptionWrappingDatabase database; - public static final String PRIMARY_KEY = "id"; - public static final String USER_KEY = "user_id"; - public static final String WORKSPACE_KEY = "workspace_id"; + /** + * Each installation of Airbyte comes with a default organization. The ID of this organization is + * hardcoded to the 0 UUID so that it can be consistently retrieved. + */ + public static final UUID DEFAULT_ORGANIZATION_ID = UUID.fromString("00000000-0000-0000-0000-000000000000"); public OrganizationPersistence(final Database database) { this.database = new ExceptionWrappingDatabase(database); @@ -95,6 +99,13 @@ public Organization updateOrganization(Organization organization) throws IOExcep return organization; } + /** + * Get the default organization if it exists by looking up the hardcoded default organization id. + */ + public Optional getDefaultOrganization() throws IOException { + return getOrganization(DEFAULT_ORGANIZATION_ID); + } + private void updateOrganizationInDB(final DSLContext ctx, Organization organization) throws IOException { final OffsetDateTime timestamp = OffsetDateTime.now(); @@ -124,6 +135,7 @@ private void insertOrganizationIntoDB(final DSLContext ctx, Organization organiz } ctx.insertInto(ORGANIZATION) .set(ORGANIZATION.ID, organization.getOrganizationId()) + .set(ORGANIZATION.USER_ID, organization.getUserId()) .set(ORGANIZATION.NAME, organization.getName()) .set(ORGANIZATION.EMAIL, organization.getEmail()) .set(ORGANIZATION.CREATED_AT, timestamp) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java index 46e42700b30..6448cbf3816 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/UserPersistence.java @@ -19,6 +19,7 @@ import java.time.OffsetDateTime; import java.util.Optional; import java.util.UUID; +import lombok.extern.slf4j.Slf4j; import org.jooq.Record; import org.jooq.Result; import org.jooq.impl.DSL; @@ -29,10 +30,17 @@ * Perform all SQL queries and handle persisting User to the Config Database. * */ +@Slf4j public class UserPersistence { public static final String PRIMARY_KEY = "id"; + /** + * Each installation of Airbyte comes with a default user. The ID of this user is hardcoded to the 0 + * UUID so that it can be consistently retrieved. + */ + public static final UUID DEFAULT_USER_ID = UUID.fromString("00000000-0000-0000-0000-000000000000"); + private final ExceptionWrappingDatabase database; public UserPersistence(final Database database) { @@ -177,4 +185,11 @@ public Optional getUserByEmail(final String email) throws IOException { return Optional.of(createUserFromRecord(result.get(0))); } + /** + * Get the default user if it exists by looking up the hardcoded default user id. + */ + public Optional getDefaultUser() throws IOException { + return getUser(DEFAULT_USER_ID); + } + } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java index 5a1beae56a2..0493db51289 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java @@ -14,10 +14,13 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; /** * Persistence Interface for Workspace table. */ +@Slf4j public class WorkspacePersistence { private final ExceptionWrappingDatabase database; @@ -45,4 +48,27 @@ public List listWorkspacesByOrganizationId(final ResourcesByO .toList(); } + public void setOrganizationIdIfNull(final UUID workspaceId, final UUID organizationId) throws IOException { + // find the workspace with the given ID and check if its organization ID is null. If so, update it. + // otherwise, log a warning and do nothing. + database.transaction(ctx -> { + final boolean isExistingWorkspace = ctx.fetchExists(ctx.selectFrom(WORKSPACE).where(WORKSPACE.ID.eq(workspaceId))); + if (isExistingWorkspace) { + final boolean isNullOrganizationId = + ctx.fetchExists(ctx.selectFrom(WORKSPACE).where(WORKSPACE.ID.eq(workspaceId)).and(WORKSPACE.ORGANIZATION_ID.isNull())); + if (isNullOrganizationId) { + ctx.update(WORKSPACE) + .set(WORKSPACE.ORGANIZATION_ID, organizationId) + .where(WORKSPACE.ID.eq(workspaceId)) + .execute(); + } else { + log.warn("Workspace with ID {} already has an organization ID set. Skipping update.", workspaceId); + } + } else { + log.warn("Workspace with ID {} does not exist. Skipping update.", workspaceId); + } + return null; + }); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java index c2bcff00396..bcf6246dd16 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/OrganizationPersistenceTest.java @@ -26,6 +26,19 @@ void beforeEach() throws Exception { } } + @Test + void createOrganization() throws Exception { + Organization organization = new Organization() + .withOrganizationId(UUID.randomUUID()) + .withUserId(UUID.randomUUID()) + .withEmail("octavia@airbyte.io") + .withName("new org"); + organizationPersistence.createOrganization(organization); + Optional result = organizationPersistence.getOrganization(organization.getOrganizationId()); + assertTrue(result.isPresent()); + assertEquals(organization, result.get()); + } + @Test void getOrganization() throws Exception { Optional result = organizationPersistence.getOrganization(MockData.ORGANIZATION_ID_1); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java index 81739ff3660..ac09b7c2d4c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/InstanceConfigurationApiController.java @@ -6,7 +6,8 @@ import io.airbyte.api.generated.InstanceConfigurationApi; import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.commons.server.handlers.instance_configuration.InstanceConfigurationHandler; +import io.airbyte.api.model.generated.InstanceConfigurationSetupRequestBody; +import io.airbyte.commons.server.handlers.InstanceConfigurationHandler; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; import io.micronaut.security.annotation.Secured; @@ -26,7 +27,12 @@ public InstanceConfigurationApiController(final InstanceConfigurationHandler ins @Override @Get public InstanceConfigurationResponse getInstanceConfiguration() { - return instanceConfigurationHandler.getInstanceConfiguration(); + return ApiHelper.execute(instanceConfigurationHandler::getInstanceConfiguration); + } + + @Override + public InstanceConfigurationResponse setupInstanceConfiguration(InstanceConfigurationSetupRequestBody instanceConfigurationSetupRequestBody) { + return ApiHelper.execute(() -> instanceConfigurationHandler.setupInstanceConfiguration(instanceConfigurationSetupRequestBody)); } } diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 85e6ea1ce36..98968dc09de 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -28,7 +28,7 @@ micronaut: idle-timeout: ${HTTP_IDLE_TIMEOUT:5m} airbyte: - edition: ${AIRBYTE_EDITION:community} + edition: ${AIRBYTE_EDITION:COMMUNITY} shutdown: delay_ms: 20000 cloud: diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java index a32da5b2e3b..ec2e7066bac 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/InstanceConfigurationApiControllerTest.java @@ -7,7 +7,9 @@ import static org.mockito.Mockito.when; import io.airbyte.api.model.generated.InstanceConfigurationResponse; -import io.airbyte.commons.server.handlers.instance_configuration.InstanceConfigurationHandler; +import io.airbyte.commons.server.handlers.InstanceConfigurationHandler; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Replaces; import io.micronaut.context.annotation.Requires; import io.micronaut.context.env.Environment; @@ -16,6 +18,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.test.annotation.MockBean; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import java.io.IOException; import org.junit.jupiter.api.Test; import org.mockito.Mock; import org.mockito.Mockito; @@ -40,11 +43,19 @@ InstanceConfigurationHandler mmInstanceConfigurationHandler() { static String PATH = "/api/v1/instance_configuration"; @Test - void testGetInstanceConfiguration() { + void testGetInstanceConfiguration() throws ConfigNotFoundException, IOException { when(instanceConfigurationHandler.getInstanceConfiguration()) .thenReturn(new InstanceConfigurationResponse()); testEndpointStatus(HttpRequest.GET(PATH), HttpStatus.OK); } + @Test + void testSetupInstanceConfiguration() throws ConfigNotFoundException, IOException, JsonValidationException { + when(instanceConfigurationHandler.setupInstanceConfiguration(Mockito.any())) + .thenReturn(new InstanceConfigurationResponse()); + + testEndpointStatus(HttpRequest.POST(PATH + "/setup", new InstanceConfigurationResponse()), HttpStatus.OK); + } + } diff --git a/airbyte-webapp/cypress/commands/common.ts b/airbyte-webapp/cypress/commands/common.ts index 004ca97dec7..e5a4b758e60 100644 --- a/airbyte-webapp/cypress/commands/common.ts +++ b/airbyte-webapp/cypress/commands/common.ts @@ -26,10 +26,6 @@ export const clearApp = () => { cy.clearCookies(); }; -export const fillEmail = (email: string) => { - cy.get("input[name=email]").type(email); -}; - // useful for ensuring that a name is unique from one test run to the next export const appendRandomString = (string: string) => { const randomString = Math.random().toString(36).substring(2, 10); diff --git a/airbyte-webapp/cypress/e2e/onboarding.cy.ts b/airbyte-webapp/cypress/e2e/onboarding.cy.ts index 0bbdb861abb..af1e12c50be 100644 --- a/airbyte-webapp/cypress/e2e/onboarding.cy.ts +++ b/airbyte-webapp/cypress/e2e/onboarding.cy.ts @@ -1,10 +1,15 @@ -import { submitButtonClick, fillEmail } from "commands/common"; +import { submitButtonClick } from "commands/common"; const OSS_SECURITY_CHECK_URL = "https://oss.airbyte.com/security-check"; +export const fillSetupForm = () => { + cy.get("input[name=email]").type("test-email-onboarding@test-onboarding-domain.com"); + cy.get("input[name=organizationName]").type("ACME Corp"); +}; + describe("Setup actions", () => { beforeEach(() => { - cy.intercept("POST", "/api/v1/workspaces/get", (req) => { + cy.intercept("GET", "/api/v1/instance_configuration", (req) => { req.continue((res) => { res.body.initialSetupComplete = false; res.send(res.body); @@ -21,7 +26,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); cy.get("[data-testid=securityCheckRunning]").should("be.visible"); cy.get("button[type=submit]").should("be.disabled"); @@ -35,7 +40,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); cy.get("button[type=submit]").should("be.enabled"); }); @@ -50,7 +55,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); cy.get("button[type=submit]").should("be.disabled"); cy.get("[data-testid=advancedOptions]").click(); @@ -69,7 +74,7 @@ describe("Setup actions", () => { cy.visit("/setup"); cy.url().should("include", `/setup`); - fillEmail("test-email-onboarding@test-onboarding-domain.com"); + fillSetupForm(); submitButtonClick(); diff --git a/airbyte-webapp/src/App.tsx b/airbyte-webapp/src/App.tsx index 9725f174d4f..d455f0185bd 100644 --- a/airbyte-webapp/src/App.tsx +++ b/airbyte-webapp/src/App.tsx @@ -8,6 +8,7 @@ import { ApiErrorBoundary } from "components/common/ApiErrorBoundary"; import { config } from "config"; import { QueryProvider, useGetInstanceConfiguration } from "core/api"; import { AnalyticsProvider } from "core/services/analytics"; +import { OSSAuthService } from "core/services/auth"; import { defaultOssFeatures, FeatureService } from "core/services/features"; import { I18nProvider } from "core/services/i18n"; import { BlockerService } from "core/services/navigation"; @@ -19,7 +20,6 @@ import { ModalServiceProvider } from "hooks/services/Modal"; import { NotificationService } from "hooks/services/Notification"; import { AirbyteThemeProvider } from "hooks/theme/useAirbyteTheme"; import { ConnectorBuilderTestInputProvider } from "services/connectorBuilder/ConnectorBuilderTestInputService"; -import { KeycloakAuthService } from "services/KeycloakAuthService"; import LoadingPage from "./components/LoadingPage"; import { ConfigServiceProvider } from "./config"; @@ -33,7 +33,7 @@ const StyleProvider: React.FC> = ({ children }) const Services: React.FC> = ({ children }) => ( - + @@ -45,7 +45,7 @@ const Services: React.FC> = ({ children }) => ( - + ); diff --git a/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx b/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx index 78a2ffa8607..f128de72d10 100644 --- a/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx +++ b/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx @@ -9,8 +9,9 @@ import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; +import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useConfig } from "config"; -import useWorkspace from "hooks/services/useWorkspace"; +import { useSetupInstanceConfiguration } from "core/api"; import { SecurityCheck } from "./SecurityCheck"; @@ -20,6 +21,7 @@ export interface SetupFormValues { email: string; anonymousDataCollection: boolean; securityCheck: SecurityCheckStatus; + organizationName: string; } const SubmissionButton: React.FC = () => { @@ -38,15 +40,17 @@ const setupFormValidationSchema = yup.object().shape({ email: yup.string().email("form.email.error").required("form.empty.error"), anonymousDataCollection: yup.bool().required(), securityCheck: yup.mixed().oneOf(["succeeded", "ignored", "check_failed"]).required(), + organizationName: yup.string().required("form.empty.error"), }); export const SetupForm: React.FC = () => { + const workspaceId = useCurrentWorkspaceId(); const { formatMessage } = useIntl(); - const { setInitialSetupConfig } = useWorkspace(); + const { mutateAsync: setUpInstance } = useSetupInstanceConfiguration(); const config = useConfig(); const onSubmit = async (values: SetupFormValues) => { - await setInitialSetupConfig(values); + await setUpInstance({ ...values, workspaceId, initialSetupComplete: true, displaySetupWizard: false }); }; return ( @@ -67,6 +71,13 @@ export const SetupForm: React.FC = () => { label={formatMessage({ id: "form.yourEmail" })} placeholder={formatMessage({ id: "form.email.placeholder" })} /> + + name="organizationName" + fieldType="input" + type="text" + label={formatMessage({ id: "form.organizationName" })} + placeholder={formatMessage({ id: "form.organizationName.placeholder" })} + /> {config.segment.enabled && ( name="anonymousDataCollection" diff --git a/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts b/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts index 6e7d53fb16d..71c4886b3d2 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts @@ -1,7 +1,7 @@ import { QueryObserverResult, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useCallback } from "react"; -import { useCurrentUser } from "packages/cloud/services/auth/AuthService"; +import { useCurrentUser } from "core/services/auth"; import { SCOPE_USER } from "services/Scope"; import { diff --git a/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts b/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts index 66210df148b..2ab03da2a9f 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts @@ -1,8 +1,8 @@ import { useQueries } from "@tanstack/react-query"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { useCurrentUser } from "core/services/auth"; import { getConnectionListQueryKey, useConnectionListQuery } from "hooks/services/useConnectionHook"; -import { useCurrentUser } from "packages/cloud/services/auth/AuthService"; import { getListCloudWorkspacesAsyncQueryKey, useListCloudWorkspacesAsyncQuery } from "./cloudWorkspaces"; import { getListUsersQueryKey, useListUsersQuery } from "./users"; diff --git a/airbyte-webapp/src/core/api/hooks/index.ts b/airbyte-webapp/src/core/api/hooks/index.ts index 3943c4da46f..5daf8ac10f9 100644 --- a/airbyte-webapp/src/core/api/hooks/index.ts +++ b/airbyte-webapp/src/core/api/hooks/index.ts @@ -12,4 +12,5 @@ export * from "./instanceConfiguration"; export * from "./streams"; export * from "./workspaces"; export * from "./organizations"; +export * from "./users"; export * from "./upgradeConnectorVersion"; diff --git a/airbyte-webapp/src/core/api/hooks/instanceConfiguration.ts b/airbyte-webapp/src/core/api/hooks/instanceConfiguration.ts index 8ed3b6ea491..f20af63c205 100644 --- a/airbyte-webapp/src/core/api/hooks/instanceConfiguration.ts +++ b/airbyte-webapp/src/core/api/hooks/instanceConfiguration.ts @@ -1,12 +1,31 @@ -import { getInstanceConfiguration } from "../generated/AirbyteClient"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; + +import { getInstanceConfiguration, setupInstanceConfiguration } from "../generated/AirbyteClient"; +import { InstanceConfigurationSetupRequestBody } from "../generated/AirbyteClient.schemas"; +import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; +const QUERY_KEY_INSTANCE = "airbyte-instance"; + export function useGetInstanceConfiguration() { // The instance configuration endpoint is not authenticated, so we don't need to pass any auth headers. // But because of how the API client is generated, we still need to pass an empty request options object. const emptyRequestOptions = { middlewares: [] }; - return useSuspenseQuery(["airbyte-instance"], () => getInstanceConfiguration(emptyRequestOptions), { + return useSuspenseQuery([QUERY_KEY_INSTANCE], () => getInstanceConfiguration(emptyRequestOptions), { staleTime: Infinity, // This data should be fetched once and never updated }); } + +export function useSetupInstanceConfiguration() { + const requestOptions = useRequestOptions(); + const queryClient = useQueryClient(); + return useMutation( + (body: InstanceConfigurationSetupRequestBody) => setupInstanceConfiguration(body, requestOptions), + { + onSuccess: (data) => { + queryClient.setQueryData([QUERY_KEY_INSTANCE], data); + }, + } + ); +} diff --git a/airbyte-webapp/src/core/api/hooks/users.ts b/airbyte-webapp/src/core/api/hooks/users.ts new file mode 100644 index 00000000000..9952cd9a6cf --- /dev/null +++ b/airbyte-webapp/src/core/api/hooks/users.ts @@ -0,0 +1,19 @@ +import { getUser } from "../generated/AirbyteClient"; +import { useGetInstanceConfiguration, useSuspenseQuery } from "../index"; +import { UserRead } from "../types/AirbyteClient"; +import { UserRead as CloudUserRead } from "../types/CloudApi"; +import { useRequestOptions } from "../useRequestOptions"; + +const userKeys = { + all: ["users"] as const, + detail: (id: string) => [...userKeys.all, id] as const, +}; + +export const useGetDefaultUser = (): UserRead & CloudUserRead => { + const { defaultUserId: userId } = useGetInstanceConfiguration(); + const requestOptions = useRequestOptions(); + + return useSuspenseQuery(userKeys.detail(userId), () => + getUser({ userId }, requestOptions).then((user) => ({ ...user, defaultWorkspaceId: user.defaultWorkspaceId || "" })) + ); +}; diff --git a/airbyte-webapp/src/core/services/auth/AuthContext.ts b/airbyte-webapp/src/core/services/auth/AuthContext.ts new file mode 100644 index 00000000000..60083d5353d --- /dev/null +++ b/airbyte-webapp/src/core/services/auth/AuthContext.ts @@ -0,0 +1,76 @@ +import React, { useContext } from "react"; +import { Observable } from "rxjs"; + +import { UserRead, UserReadMetadata } from "core/api/types/AirbyteClient"; +import { SignupFormValues } from "packages/cloud/views/auth/SignupPage/components/SignupForm"; + +export type AuthUpdatePassword = (email: string, currentPassword: string, newPassword: string) => Promise; + +export type AuthRequirePasswordReset = (email: string) => Promise; +export type AuthConfirmPasswordReset = (code: string, newPassword: string) => Promise; + +export type AuthLogin = (values: { email: string; password: string }) => Promise; +export type AuthOAuthLogin = (provider: OAuthProviders) => Observable; + +export type AuthSignUp = (form: SignupFormValues) => Promise; + +export type AuthChangeName = (name: string) => Promise; + +export type AuthSendEmailVerification = () => Promise; +export type AuthVerifyEmail = (code: string) => Promise; +export type AuthLogout = () => Promise; + +export type OAuthLoginState = "waiting" | "loading" | "done"; + +export enum AuthProviders { + GoogleIdentityPlatform = "google_identity_platform", +} + +export type OAuthProviders = "github" | "google"; + +// This override is currently needed because the UserRead type is not consistent between OSS and Cloud +export interface CommonUserRead extends Omit { + metadata?: UserReadMetadata; +} + +export interface AuthContextApi { + user: CommonUserRead | null; + inited: boolean; + emailVerified: boolean; + isLoading: boolean; + loggedOut: boolean; + providers: string[] | null; + hasPasswordLogin?: () => boolean; + login?: AuthLogin; + loginWithOAuth?: AuthOAuthLogin; + signUpWithEmailLink?: (form: { name: string; email: string; password: string; news: boolean }) => Promise; + signUp?: AuthSignUp; + updatePassword?: AuthUpdatePassword; + updateName?: AuthChangeName; + requirePasswordReset?: AuthRequirePasswordReset; + confirmPasswordReset?: AuthConfirmPasswordReset; + sendEmailVerification?: AuthSendEmailVerification; + verifyEmail?: AuthVerifyEmail; + logout?: AuthLogout; +} + +// The AuthContext is implemented differently in OSS vs. Cloud, but both implementations use the AuthContextApi interface +export const AuthContext = React.createContext(null); + +export const useCurrentUser = (): CommonUserRead => { + const { user } = useAuthService(); + if (!user) { + throw new Error("useCurrentUser must be used only within authorised flow"); + } + + return user; +}; + +export const useAuthService = (): AuthContextApi => { + const authService = useContext(AuthContext); + if (!authService) { + throw new Error("useAuthService must be used within a AuthenticationService."); + } + + return authService; +}; diff --git a/airbyte-webapp/src/core/services/auth/DefaultAuthService.tsx b/airbyte-webapp/src/core/services/auth/DefaultAuthService.tsx new file mode 100644 index 00000000000..2dea5b110c4 --- /dev/null +++ b/airbyte-webapp/src/core/services/auth/DefaultAuthService.tsx @@ -0,0 +1,24 @@ +import { PropsWithChildren } from "react"; + +import { useGetDefaultUser } from "core/api"; + +import { AuthContext } from "./AuthContext"; + +export const DefaultAuthService: React.FC> = ({ children }) => { + const defaultUser = useGetDefaultUser(); + + return ( + + {children} + + ); +}; diff --git a/airbyte-webapp/src/services/KeycloakAuthService.tsx b/airbyte-webapp/src/core/services/auth/KeycloakAuthService.tsx similarity index 65% rename from airbyte-webapp/src/services/KeycloakAuthService.tsx rename to airbyte-webapp/src/core/services/auth/KeycloakAuthService.tsx index b5d1cc60154..a0b65820f8d 100644 --- a/airbyte-webapp/src/services/KeycloakAuthService.tsx +++ b/airbyte-webapp/src/core/services/auth/KeycloakAuthService.tsx @@ -5,40 +5,34 @@ import { AuthProvider, useAuth } from "react-oidc-context"; import LoadingPage from "components/LoadingPage"; import { useGetInstanceConfiguration } from "core/api"; -import { FeatureItem, useFeature } from "core/services/features"; import { useGetService, useInjectServices } from "core/servicesProvider"; -import { RequestAuthMiddleware } from "packages/cloud/lib/auth/RequestAuthMiddleware"; + +import { RequestAuthMiddleware } from "./RequestAuthMiddleware"; // This wrapper is conditionally present if the KeycloakAuthentication feature is enabled export const KeycloakAuthService: React.FC> = ({ children }) => { const { auth, webappUrl } = useGetInstanceConfiguration(); - const isKeycloakAuthenticationEnabled = useFeature(FeatureItem.KeycloakAuthentication); - - if (isKeycloakAuthenticationEnabled) { - if (!auth) { - throw new Error("Authentication is enabled, but the server returned an invalid auth configuration: ", auth); - } - - const oidcConfig = { - authority: `${webappUrl}/auth/realms/${auth.defaultRealm}`, - client_id: auth.clientId, - redirect_uri: `${window.location.origin}/${window.location.pathname}`, - onSigninCallback: () => { - window.history.replaceState({}, document.title, window.location.pathname); - }, - }; - - return ( - - - {children} - - - ); + if (!auth) { + throw new Error("Authentication is enabled, but the server returned an invalid auth configuration: ", auth); } - return <>{children}; + const oidcConfig = { + authority: `${webappUrl}/auth/realms/${auth.defaultRealm}`, + client_id: auth.clientId, + redirect_uri: `${window.location.origin}/${window.location.pathname}`, + onSigninCallback: () => { + window.history.replaceState({}, document.title, window.location.pathname); + }, + }; + + return ( + + + {children} + + + ); }; // While auth status is loading we want to suspend rendering of the app diff --git a/airbyte-webapp/src/core/services/auth/OSSAuthService.tsx b/airbyte-webapp/src/core/services/auth/OSSAuthService.tsx new file mode 100644 index 00000000000..52e81f70978 --- /dev/null +++ b/airbyte-webapp/src/core/services/auth/OSSAuthService.tsx @@ -0,0 +1,17 @@ +import { PropsWithChildren } from "react"; + +import { FeatureItem, useFeature } from "core/services/features"; + +import { DefaultAuthService } from "./DefaultAuthService"; +import { KeycloakAuthService } from "./KeycloakAuthService"; + +// This wrapper is conditionally present if the KeycloakAuthentication feature is enabled +export const OSSAuthService: React.FC> = ({ children }) => { + const isKeycloakAuthenticationEnabled = useFeature(FeatureItem.KeycloakAuthentication); + + return isKeycloakAuthenticationEnabled ? ( + + ) : ( + + ); +}; diff --git a/airbyte-webapp/src/packages/cloud/lib/auth/RequestAuthMiddleware.ts b/airbyte-webapp/src/core/services/auth/RequestAuthMiddleware.ts similarity index 100% rename from airbyte-webapp/src/packages/cloud/lib/auth/RequestAuthMiddleware.ts rename to airbyte-webapp/src/core/services/auth/RequestAuthMiddleware.ts diff --git a/airbyte-webapp/src/core/services/auth/index.ts b/airbyte-webapp/src/core/services/auth/index.ts new file mode 100644 index 00000000000..2a97d1b9457 --- /dev/null +++ b/airbyte-webapp/src/core/services/auth/index.ts @@ -0,0 +1,3 @@ +export * from "./OSSAuthService"; +export * from "./RequestAuthMiddleware"; +export * from "./AuthContext"; diff --git a/airbyte-webapp/src/packages/cloud/services/auth/freeEmailProviders.ts b/airbyte-webapp/src/core/utils/freeEmailProviders.ts similarity index 99% rename from airbyte-webapp/src/packages/cloud/services/auth/freeEmailProviders.ts rename to airbyte-webapp/src/core/utils/freeEmailProviders.ts index 3c59c72d2c5..71ed49f2405 100644 --- a/airbyte-webapp/src/packages/cloud/services/auth/freeEmailProviders.ts +++ b/airbyte-webapp/src/core/utils/freeEmailProviders.ts @@ -3750,3 +3750,6 @@ export const FREE_EMAIL_SERVICE_PROVIDERS = [ "zzn.com", "zzom.co.uk", ] as const; + +export const isCorporateEmail = (email: string) => + !FREE_EMAIL_SERVICE_PROVIDERS.some((provider) => email?.endsWith(`@${provider}`)); diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index ef2323caf16..e98c959be55 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -48,6 +48,8 @@ "form.error.missing": "missing", "form.yourEmail": "Your email", "form.email.placeholder": "you@company.com", + "form.organizationName": "Organization name", + "form.organizationName.placeholder": "ACME Corp", "form.email.error": "Enter a valid email", "form.empty.error": "Required", "form.selectValue": "Select a value", diff --git a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx index 40afca4e9a3..0f32ffb1b66 100644 --- a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx +++ b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx @@ -8,9 +8,10 @@ import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useCurrentWorkspace, useInvalidateAllWorkspaceScopeOnChange } from "core/api"; import { usePrefetchCloudWorkspaceData } from "core/api/cloud"; import { useAnalyticsIdentifyUser, useAnalyticsRegisterValues } from "core/services/analytics/useAnalyticsService"; +import { useAuthService } from "core/services/auth"; +import { isCorporateEmail } from "core/utils/freeEmailProviders"; import { useBuildUpdateCheck } from "hooks/services/useBuildUpdateCheck"; import { useQuery } from "hooks/useQuery"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import ConnectorBuilderRoutes from "pages/connectorBuilder/ConnectorBuilderRoutes"; import { RoutePaths, DestinationPaths, SourcePaths } from "pages/routePaths"; import { CompleteOauthRequest } from "views/CompleteOauthRequest"; @@ -124,7 +125,8 @@ const CloudWorkspaceDataPrefetcher: React.FC> = ({ ch }; export const Routing: React.FC = () => { - const { user, inited, providers, hasCorporateEmail, loggedOut } = useAuthService(); + const { login, requirePasswordReset } = useAuthService(); + const { user, inited, providers, loggedOut } = useAuthService(); const workspaceId = useCurrentWorkspaceId(); const { pathname } = useLocation(); @@ -144,8 +146,8 @@ export const Routing: React.FC = () => { ); const userTraits = useMemo( - () => (user ? { providers, email: user.email, isCorporate: hasCorporateEmail() } : {}), - [hasCorporateEmail, providers, user] + () => (user ? { providers, email: user.email, isCorporate: isCorporateEmail(user.email) } : {}), + [providers, user] ); useAnalyticsRegisterValues(analyticsContext); @@ -173,9 +175,14 @@ export const Routing: React.FC = () => { }> - } /> + {login && } />} } /> - } /> + {requirePasswordReset && ( + } + /> + )} {/* In case a not logged in user tries to access anything else navigate them to login */} { const { registerNotification } = useNotificationService(); const verifyEmail = () => - sendEmailVerification().then(() => { + sendEmailVerification?.().then(() => { registerNotification({ id: "fcp/verify-email", text: formatMessage({ id: "freeConnectorProgram.enrollmentModal.validationEmailConfirmation" }), diff --git a/airbyte-webapp/src/packages/cloud/lib/auth/AuthProviders.ts b/airbyte-webapp/src/packages/cloud/lib/auth/AuthProviders.ts deleted file mode 100644 index bb7aac02489..00000000000 --- a/airbyte-webapp/src/packages/cloud/lib/auth/AuthProviders.ts +++ /dev/null @@ -1,5 +0,0 @@ -export enum AuthProviders { - GoogleIdentityPlatform = "google_identity_platform", -} - -export type OAuthProviders = "github" | "google"; diff --git a/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts b/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts index ecdead6ba4b..a427a3a8cc8 100644 --- a/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts +++ b/airbyte-webapp/src/packages/cloud/lib/auth/GoogleAuthService.ts @@ -1,5 +1,3 @@ -import type { OAuthProviders } from "./AuthProviders"; - import { Auth, User, @@ -23,6 +21,7 @@ import { reload, } from "firebase/auth"; +import type { OAuthProviders } from "core/services/auth"; import { EmailLinkErrorCodes, ResetPasswordConfirmErrorCodes, diff --git a/airbyte-webapp/src/packages/cloud/services/AppServicesProvider.tsx b/airbyte-webapp/src/packages/cloud/services/AppServicesProvider.tsx index 5669da1085b..087642943c5 100644 --- a/airbyte-webapp/src/packages/cloud/services/AppServicesProvider.tsx +++ b/airbyte-webapp/src/packages/cloud/services/AppServicesProvider.tsx @@ -4,8 +4,8 @@ import { LoadingPage } from "components"; import { MissingConfigError, useConfig } from "config"; import { RequestMiddleware } from "core/request/RequestMiddleware"; +import { RequestAuthMiddleware } from "core/services/auth"; import { ServicesProvider, useGetService, useInjectServices } from "core/servicesProvider"; -import { RequestAuthMiddleware } from "packages/cloud/lib/auth/RequestAuthMiddleware"; import { useAuth } from "packages/firebaseReact"; import { FirebaseSdkProvider } from "./FirebaseSdkProvider"; diff --git a/airbyte-webapp/src/packages/cloud/services/auth/AuthService.tsx b/airbyte-webapp/src/packages/cloud/services/auth/AuthService.tsx index 3d1058912e1..670c661c1b5 100644 --- a/airbyte-webapp/src/packages/cloud/services/auth/AuthService.tsx +++ b/airbyte-webapp/src/packages/cloud/services/auth/AuthService.tsx @@ -1,6 +1,6 @@ import { useQueryClient } from "@tanstack/react-query"; import { User as FirebaseUser, AuthErrorCodes } from "firebase/auth"; -import React, { useCallback, useContext, useMemo, useRef } from "react"; +import React, { useCallback, useMemo, useRef } from "react"; import { useIntl } from "react-intl"; import { useEffectOnce } from "react-use"; import { Observable, Subject } from "rxjs"; @@ -10,36 +10,19 @@ import { UserRead } from "core/api/types/CloudApi"; import { isCommonRequestError } from "core/request/CommonRequestError"; import { Action, Namespace } from "core/services/analytics"; import { useAnalyticsService } from "core/services/analytics"; +import { AuthProviders, AuthContextApi, OAuthLoginState, AuthContext, useAuthService } from "core/services/auth"; import { trackSignup } from "core/utils/fathom"; +import { isCorporateEmail } from "core/utils/freeEmailProviders"; import { useNotificationService } from "hooks/services/Notification"; import useTypesafeReducer from "hooks/useTypesafeReducer"; -import { AuthProviders, OAuthProviders } from "packages/cloud/lib/auth/AuthProviders"; import { GoogleAuthService } from "packages/cloud/lib/auth/GoogleAuthService"; import { SignupFormValues } from "packages/cloud/views/auth/SignupPage/components/SignupForm"; import { useAuth } from "packages/firebaseReact"; import { useInitService } from "services/useInitService"; -import { FREE_EMAIL_SERVICE_PROVIDERS } from "./freeEmailProviders"; import { actions, AuthServiceState, authStateReducer, initialState } from "./reducer"; import { EmailLinkErrorCodes } from "./types"; -export type AuthUpdatePassword = (email: string, currentPassword: string, newPassword: string) => Promise; - -export type AuthRequirePasswordReset = (email: string) => Promise; -export type AuthConfirmPasswordReset = (code: string, newPassword: string) => Promise; - -export type AuthLogin = (values: { email: string; password: string }) => Promise; - -export type AuthSignUp = (form: SignupFormValues) => Promise; - -export type AuthChangeName = (name: string) => Promise; - -export type AuthSendEmailVerification = () => Promise; -export type AuthVerifyEmail = (code: string) => Promise; -export type AuthLogout = () => Promise; - -type OAuthLoginState = "waiting" | "loading" | "done"; - export enum FirebaseAuthMessageId { NetworkFailure = "firebase.auth.error.networkRequestFailed", TooManyRequests = "firebase.auth.error.tooManyRequests", @@ -47,30 +30,6 @@ export enum FirebaseAuthMessageId { DefaultError = "firebase.auth.error.default", } -interface AuthContextApi { - user: UserRead | null; - inited: boolean; - emailVerified: boolean; - isLoading: boolean; - loggedOut: boolean; - providers: string[] | null; - hasPasswordLogin: () => boolean; - hasCorporateEmail: (email?: string) => boolean; - login: AuthLogin; - loginWithOAuth: (provider: OAuthProviders) => Observable; - signUpWithEmailLink: (form: { name: string; email: string; password: string; news: boolean }) => Promise; - signUp: AuthSignUp; - updatePassword: AuthUpdatePassword; - updateName: AuthChangeName; - requirePasswordReset: AuthRequirePasswordReset; - confirmPasswordReset: AuthConfirmPasswordReset; - sendEmailVerification: AuthSendEmailVerification; - verifyEmail: AuthVerifyEmail; - logout: AuthLogout; -} - -export const AuthContext = React.createContext(null); - export const AuthenticationProvider: React.FC> = ({ children }) => { const [state, { loggedIn, emailVerified, authInited, loggedOut, updateUserName }] = useTypesafeReducer< AuthServiceState, @@ -102,7 +61,7 @@ export const AuthenticationProvider: React.FC> companyName: userData.companyName ?? "", news: userData.news ?? false, }); - const isCorporate = ctx.hasCorporateEmail(user.email); + const isCorporate = isCorporateEmail(user.email); analytics.track(Namespace.USER, Action.CREATE, { actionDescription: "New user registered", @@ -182,9 +141,6 @@ export const AuthenticationProvider: React.FC> hasPasswordLogin(): boolean { return !!state.providers?.includes("password"); }, - hasCorporateEmail(email: string | undefined = state.currentUser?.email): boolean { - return !FREE_EMAIL_SERVICE_PROVIDERS.some((provider) => email?.endsWith(`@${provider}`)); - }, async login(values: { email: string; password: string }): Promise { await authService.login(values.email, values.password); @@ -322,24 +278,6 @@ export const AuthenticationProvider: React.FC> return {children}; }; -export const useAuthService = (): AuthContextApi => { - const authService = useContext(AuthContext); - if (!authService) { - throw new Error("useAuthService must be used within a AuthenticationService."); - } - - return authService; -}; - -export const useCurrentUser = (): UserRead => { - const { user } = useAuthService(); - if (!user) { - throw new Error("useCurrentUser must be used only within authorised flow"); - } - - return user; -}; - export const useIsForeignWorkspace = () => { const { user } = useAuthService(); const workspaceUsers = useListUsers(); diff --git a/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx b/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx index 65b802bdd59..054341cdf8e 100644 --- a/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx +++ b/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/LDExperimentService.tsx @@ -9,6 +9,7 @@ import { LoadingPage } from "components"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useConfig } from "config"; import { useAnalyticsService } from "core/services/analytics"; +import { useAuthService } from "core/services/auth"; import { FeatureSet, FeatureItem, useFeatureService } from "core/services/features"; import { useI18nContext } from "core/services/i18n"; import { useDebugVariable } from "core/utils/debug"; @@ -17,7 +18,6 @@ import { rejectAfter } from "core/utils/promises"; import { useAppMonitoringService, AppActionCodes } from "hooks/services/AppMonitoringService"; import { ContextKind, ExperimentProvider, ExperimentService } from "hooks/services/Experiment"; import type { Experiments } from "hooks/services/Experiment/experiments"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { contextReducer } from "./contextReducer"; import { createLDContext, createMultiContext, createUserContext } from "./contexts"; diff --git a/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/contexts.ts b/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/contexts.ts index 1f61e5c8fa6..bcfef2a05ec 100644 --- a/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/contexts.ts +++ b/airbyte-webapp/src/packages/cloud/services/thirdParty/launchdarkly/contexts.ts @@ -1,6 +1,6 @@ import { LDMultiKindContext, LDSingleKindContext } from "launchdarkly-js-client-sdk"; -import { UserRead } from "core/api/types/CloudApi"; +import { CommonUserRead } from "core/services/auth"; import { ContextKind } from "hooks/services/Experiment"; export function createLDContext(kind: ContextKind, key: string): LDSingleKindContext { @@ -10,7 +10,7 @@ export function createLDContext(kind: ContextKind, key: string): LDSingleKindCon }; } -export function createUserContext(user: UserRead | null, locale: string): LDSingleKindContext { +export function createUserContext(user: CommonUserRead | null, locale: string): LDSingleKindContext { const kind = "user"; if (!user) { diff --git a/airbyte-webapp/src/packages/cloud/services/thirdParty/zendesk/ZendeskProvider.tsx b/airbyte-webapp/src/packages/cloud/services/thirdParty/zendesk/ZendeskProvider.tsx index bdc654a5122..688dcd929f6 100644 --- a/airbyte-webapp/src/packages/cloud/services/thirdParty/zendesk/ZendeskProvider.tsx +++ b/airbyte-webapp/src/packages/cloud/services/thirdParty/zendesk/ZendeskProvider.tsx @@ -3,8 +3,7 @@ import { useEffectOnce } from "react-use"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { config } from "config"; - -import { useAuthService } from "../../auth/AuthService"; +import { useAuthService } from "core/services/auth"; import "./zendesk.module.scss"; diff --git a/airbyte-webapp/src/packages/cloud/views/AcceptEmailInvite.tsx b/airbyte-webapp/src/packages/cloud/views/AcceptEmailInvite.tsx index eccfc16d93a..5ae43df09b3 100644 --- a/airbyte-webapp/src/packages/cloud/views/AcceptEmailInvite.tsx +++ b/airbyte-webapp/src/packages/cloud/views/AcceptEmailInvite.tsx @@ -12,7 +12,6 @@ import { Heading } from "components/ui/Heading"; import { isGdprCountry } from "core/utils/dataPrivacy"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { EmailLinkErrorCodes } from "packages/cloud/services/auth/types"; import { Disclaimer } from "./auth/components/Disclaimer"; @@ -33,14 +32,17 @@ const acceptEmailInviteSchema: SchemaOf = yup.objec news: yup.boolean().required(), }); -export const AcceptEmailInvite: React.FC = () => { +interface AcceptEmailInviteProps { + signUpWithEmailLink: (values: AcceptEmailInviteFormValues) => Promise; +} + +export const AcceptEmailInvite: React.FC = ({ signUpWithEmailLink }) => { const { formatMessage } = useIntl(); - const authService = useAuthService(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); const onSubmit = async (values: AcceptEmailInviteFormValues) => { - await authService.signUpWithEmailLink(values); + await signUpWithEmailLink(values); }; const onError = (e: Error, { name, email }: AcceptEmailInviteFormValues) => { diff --git a/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx b/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx index e0f23703a08..2a9ed6f3bce 100644 --- a/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx +++ b/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx @@ -5,9 +5,9 @@ import { useAsync } from "react-use"; import { match, P } from "ts-pattern"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; +import { AuthVerifyEmail, useAuthService } from "core/services/auth"; import { useNotificationService } from "hooks/services/Notification"; import { useQuery } from "hooks/useQuery"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import AuthLayout from "./auth"; import { LoginSignupNavigation } from "./auth/components/LoginSignupNavigation"; @@ -22,9 +22,12 @@ enum FirebaseActionMode { SIGN_IN = "signIn", } -const VerifyEmailAction: React.FC = () => { +interface VerifyEmailActionProps { + verifyEmail: AuthVerifyEmail; +} + +const VerifyEmailAction: React.FC = ({ verifyEmail }) => { const query = useQuery<{ oobCode?: string }>(); - const { verifyEmail } = useAuthService(); const navigate = useNavigate(); const { formatMessage } = useIntl(); const { registerNotification } = useNotificationService(); @@ -35,6 +38,7 @@ const VerifyEmailAction: React.FC = () => { navigate("/"); return; } + // Send verification code to authentication service await verifyEmail(query.oobCode); // Show a notification that the mail got verified successfully @@ -52,16 +56,24 @@ const VerifyEmailAction: React.FC = () => { export const FirebaseActionRoute: React.FC = () => { const { mode } = useQuery<{ mode: FirebaseActionMode }>(); - const { user } = useAuthService(); + const { user, verifyEmail, signUpWithEmailLink, confirmPasswordReset } = useAuthService(); return ( - {match([mode, user]) + {match([mode, user, verifyEmail, signUpWithEmailLink, confirmPasswordReset]) // The verify email route is the only that should work whether the user is logged in or not - .with([FirebaseActionMode.VERIFY_EMAIL, P.any], () => ) + .with([FirebaseActionMode.VERIFY_EMAIL, P.any, P.not(P.nullish), P.any, P.any], ([, , verifyEmail]) => ( + + )) // All other routes will require the user to be logged out - .with([FirebaseActionMode.SIGN_IN, P.nullish], () => ) - .with([FirebaseActionMode.RESET_PASSWORD, P.nullish], () => ) + .with( + [FirebaseActionMode.SIGN_IN, P.nullish, P.any, P.not(P.nullish), P.any], + ([, , , signUpWithEmailLink]) => + ) + .with( + [FirebaseActionMode.RESET_PASSWORD, P.nullish, P.any, P.any, P.not(P.nullish)], + ([, , , , confirmPasswordReset]) => + ) .otherwise(() => ( ))} diff --git a/airbyte-webapp/src/packages/cloud/views/UpcomingFeaturesPage.tsx b/airbyte-webapp/src/packages/cloud/views/UpcomingFeaturesPage.tsx index f796789d91e..f7c078ab42c 100644 --- a/airbyte-webapp/src/packages/cloud/views/UpcomingFeaturesPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/UpcomingFeaturesPage.tsx @@ -2,11 +2,11 @@ import { useIntl } from "react-intl"; import { HeadTitle } from "components/common/HeadTitle"; +import { useCurrentUser } from "core/services/auth"; import { links } from "core/utils/links"; import { useExperiment } from "hooks/services/Experiment"; import styles from "./UpcomingFeaturesPage.module.scss"; -import { useCurrentUser } from "../services/auth/AuthService"; const UpcomingFeaturesPage = () => { const { formatMessage } = useIntl(); const user = useCurrentUser(); diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx index d489a0b9430..4342d7b4342 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx @@ -13,11 +13,11 @@ import { Heading } from "components/ui/Heading"; import { Link } from "components/ui/Link"; import { Text } from "components/ui/Text"; +import { AuthConfirmPasswordReset } from "core/services/auth"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification/NotificationService"; import { useQuery } from "hooks/useQuery"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { ResetPasswordConfirmErrorCodes } from "packages/cloud/services/auth/types"; interface ResetPasswordConfirmFormValues { @@ -38,9 +38,12 @@ const ResetPasswordButton: React.FC = () => { ); }; -export const ResetPasswordConfirmPage: React.FC = () => { +interface ResetPasswordConfirmPageProps { + confirmPasswordReset: AuthConfirmPasswordReset; +} + +export const ResetPasswordConfirmPage: React.FC = ({ confirmPasswordReset }) => { const { formatMessage } = useIntl(); - const { confirmPasswordReset } = useAuthService(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); const navigate = useNavigate(); diff --git a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx index 027cad7b4ef..ba160e2127b 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx @@ -15,11 +15,11 @@ import { Link } from "components/ui/Link"; import { Text } from "components/ui/Text"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; +import { AuthLogin, useAuthService } from "core/services/auth"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification"; import { useQuery } from "hooks/useQuery"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { LoginFormErrorCodes } from "packages/cloud/services/auth/types"; import styles from "./LoginPage.module.scss"; @@ -48,9 +48,13 @@ const LoginButton: React.FC = () => { ); }; -export const LoginPage: React.FC = () => { +interface LoginPageProps { + login: AuthLogin; +} + +export const LoginPage: React.FC = ({ login }) => { + const { loginWithOAuth } = useAuthService(); const { formatMessage } = useIntl(); - const { login } = useAuthService(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); @@ -92,8 +96,12 @@ export const LoginPage: React.FC = () => { - - + {loginWithOAuth && ( + <> + + + + )} defaultValues={{ diff --git a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx index da76c6a831a..ef9a708ece2 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx @@ -4,40 +4,23 @@ import { EMPTY } from "rxjs"; import { TestWrapper } from "test-utils/testutils"; -import type { useExperiment } from "hooks/services/Experiment"; - -const mockUseExperiment = jest.fn, Parameters>(); -jest.doMock("hooks/services/Experiment", () => ({ - useExperiment: mockUseExperiment, -})); +import { OAuthLogin } from "./OAuthLogin"; const mockLoginWithOAuth = jest.fn(); -jest.doMock("packages/cloud/services/auth/AuthService", () => ({ - useAuthService: () => ({ - loginWithOAuth: mockLoginWithOAuth, - }), -})); - -// We need to use require here, so that this really happens after the doMock calls above -// eslint-disable-next-line @typescript-eslint/no-var-requires -const { OAuthLogin } = require("./OAuthLogin"); describe("OAuthLogin", () => { beforeEach(() => { - mockUseExperiment.mockReset(); - mockUseExperiment.mockReturnValue(true); - mockLoginWithOAuth.mockReset(); mockLoginWithOAuth.mockReturnValue(EMPTY); }); it("should call auth service for Google", async () => { - const { getByTestId } = render(, { wrapper: TestWrapper }); + const { getByTestId } = render(, { wrapper: TestWrapper }); await userEvents.click(getByTestId("googleOauthLogin")); expect(mockLoginWithOAuth).toHaveBeenCalledWith("google"); }); it("should call auth service for GitHub", async () => { - const { getByTestId } = render(, { wrapper: TestWrapper }); + const { getByTestId } = render(, { wrapper: TestWrapper }); await userEvents.click(getByTestId("githubOauthLogin")); expect(mockLoginWithOAuth).toHaveBeenCalledWith("github"); }); diff --git a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.tsx b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.tsx index f644e6dd659..8040250d1bd 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.tsx @@ -6,8 +6,7 @@ import { Subscription } from "rxjs"; import { FlexContainer } from "components/ui/Flex"; import { Spinner } from "components/ui/Spinner"; -import { OAuthProviders } from "packages/cloud/lib/auth/AuthProviders"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; +import { OAuthProviders, AuthOAuthLogin } from "core/services/auth"; import githubLogo from "./assets/github-logo.svg"; import googleLogo from "./assets/google-logo.svg"; @@ -31,9 +30,12 @@ const GoogleButton: React.FC<{ onClick: () => void }> = ({ onClick }) => { ); }; -export const OAuthLogin: React.FC = () => { +interface OAuthLoginProps { + loginWithOAuth: AuthOAuthLogin; +} + +export const OAuthLogin: React.FC = ({ loginWithOAuth }) => { const { formatMessage } = useIntl(); - const { loginWithOAuth } = useAuthService(); const stateSubscription = useRef(); const [errorCode, setErrorCode] = useState(); const [isLoading, setLoading] = useState(false); diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx index 02bbe281db9..bedd875f7bf 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx @@ -14,10 +14,10 @@ import { Link } from "components/ui/Link"; import { Text } from "components/ui/Text"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; +import { AuthRequirePasswordReset } from "core/services/auth"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification/NotificationService"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { LoginSignupNavigation } from "../components/LoginSignupNavigation"; @@ -39,9 +39,12 @@ const ResetPasswordButton: React.FC = () => { ); }; -export const ResetPasswordPage: React.FC = () => { +interface ResetPasswordPageProps { + requirePasswordReset: AuthRequirePasswordReset; +} + +export const ResetPasswordPage: React.FC = ({ requirePasswordReset }) => { const { formatMessage } = useIntl(); - const { requirePasswordReset } = useAuthService(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); useTrackPage(PageTrackingCodes.RESET_PASSWORD); diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx index 3608566d937..65c4b19ada9 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx @@ -11,6 +11,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; +import { useAuthService } from "core/services/auth"; import { trackPageview } from "core/utils/fathom"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; @@ -33,6 +34,7 @@ const Detail: React.FC> = ({ children }) => { }; const SignupPage: React.FC = () => { + const { loginWithOAuth, signUp } = useAuthService(); useTrackPage(PageTrackingCodes.SIGNUP); useEffect(() => { @@ -67,14 +69,14 @@ const SignupPage: React.FC = () => { {searchParams.get("method") === "email" ? ( <> - + {signUp && } ) : ( <> - {" "} + {loginWithOAuth && } diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx index 22daa908d92..0d85fa3eff9 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx @@ -10,11 +10,11 @@ import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { AuthSignUp } from "core/services/auth"; import { isGdprCountry } from "core/utils/dataPrivacy"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useExperiment } from "hooks/services/Experiment"; import { useNotificationService } from "hooks/services/Notification"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { SignUpFormErrorCodes } from "packages/cloud/services/auth/types"; import { PasswordFormControl } from "../../components/PasswordFormControl"; @@ -53,9 +53,12 @@ const composeSignUpSchema = (showName: boolean, showCompanyName: boolean): Schem ...(showCompanyName && { companyName: yup.string().required("form.empty.error") }), }); -export const SignupForm: React.FC = () => { +interface SignupFormProps { + signUp: AuthSignUp; +} + +export const SignupForm: React.FC = ({ signUp }) => { const { formatMessage } = useIntl(); - const { signUp } = useAuthService(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/EmailVerificationHint.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/EmailVerificationHint.tsx index 0acca187e91..d8f47594a43 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/EmailVerificationHint.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/EmailVerificationHint.tsx @@ -2,14 +2,14 @@ import { FormattedMessage, useIntl } from "react-intl"; import { Message } from "components/ui/Message"; +import { AuthSendEmailVerification } from "core/services/auth"; import { useNotificationService } from "hooks/services/Notification"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; interface EmailVerificationHintProps { variant: "info" | "warning" | "error"; + sendEmailVerification: AuthSendEmailVerification; } -export const EmailVerificationHint: React.FC = ({ variant }) => { - const { sendEmailVerification } = useAuthService(); +export const EmailVerificationHint: React.FC = ({ sendEmailVerification, variant }) => { const { registerNotification } = useNotificationService(); const { formatMessage } = useIntl(); diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx index 7ae61649a1e..2088ddab083 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx @@ -18,9 +18,9 @@ import { useGetCloudWorkspace, useInvalidateCloudWorkspace } from "core/api/clou import { CloudWorkspaceRead } from "core/api/types/CloudApi"; import { Action, Namespace } from "core/services/analytics"; import { useAnalyticsService } from "core/services/analytics"; +import { useAuthService } from "core/services/auth"; import { links } from "core/utils/links"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { EmailVerificationHint } from "./EmailVerificationHint"; import { LowCreditBalanceHint } from "./LowCreditBalanceHint"; @@ -38,6 +38,7 @@ function hasRecentCreditIncrease(cloudWorkspace: CloudWorkspaceRead): boolean { } export const RemainingCredits: React.FC = () => { + const { sendEmailVerification } = useAuthService(); const retryIntervalId = useRef(); const currentWorkspace = useCurrentWorkspace(); const cloudWorkspace = useGetCloudWorkspace(currentWorkspace.workspaceId); @@ -145,7 +146,9 @@ export const RemainingCredits: React.FC = () => { - {!emailVerified && } + {!emailVerified && sendEmailVerification && ( + + )} diff --git a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx index d42c3fa0a6b..c2e05f92bee 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/CloudMainView.tsx @@ -12,13 +12,14 @@ import { ThemeToggle } from "components/ui/ThemeToggle"; import { useCurrentWorkspace } from "core/api"; import { useGetCloudWorkspaceAsync } from "core/api/cloud"; import { CloudWorkspaceReadWorkspaceTrialStatus as WorkspaceTrialStatus } from "core/api/types/CloudApi"; +import { useAuthService } from "core/services/auth"; import { FeatureItem, useFeature } from "core/services/features"; +import { isCorporateEmail } from "core/utils/freeEmailProviders"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useExperiment } from "hooks/services/Experiment"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; import { useExperimentSpeedyConnection } from "packages/cloud/components/experiments/SpeedyConnection/hooks/useExperimentSpeedyConnection"; import { SpeedyConnectionBanner } from "packages/cloud/components/experiments/SpeedyConnection/SpeedyConnectionBanner"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { RoutePaths } from "pages/routePaths"; import { ResourceNotFoundErrorBoundary } from "views/common/ResourceNotFoundErrorBoundary"; import { StartOverErrorView } from "views/common/StartOverErrorView"; @@ -47,14 +48,14 @@ const CloudMainView: React.FC> = (props) => { // exp-speedy-connection const { isExperimentVariant } = useExperimentSpeedyConnection(); - const { hasCorporateEmail } = useAuthService(); + const { user } = useAuthService(); const isTrial = isNewTrialPolicy ? cloudWorkspace?.workspaceTrialStatus === WorkspaceTrialStatus.in_trial || cloudWorkspace?.workspaceTrialStatus === WorkspaceTrialStatus.pre_trial : Boolean(cloudWorkspace?.trialExpiryTimestamp); - const showExperimentBanner = isExperimentVariant && isTrial && hasCorporateEmail(); + const showExperimentBanner = isExperimentVariant && isTrial && user && isCorporateEmail(user.email); return (
diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx index daf299caefc..cae4614a1b1 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx @@ -1,34 +1,24 @@ -import { useMutation } from "@tanstack/react-query"; import React from "react"; -import { FormattedMessage } from "react-intl"; -import { Box } from "components/ui/Box"; -import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; +import { useAuthService } from "core/services/auth"; import { EmailSection, NameSection, PasswordSection } from "./components"; +import { LogoutSection } from "./components/LogoutSection"; export const AccountSettingsView: React.FC = () => { - const authService = useAuthService(); - const { mutateAsync: logout, isLoading: isLoggingOut } = useMutation(() => authService.logout()); + const { logout, updateName, hasPasswordLogin, updatePassword } = useAuthService(); useTrackPage(PageTrackingCodes.SETTINGS_ACCOUNT); return ( - + {updateName && } - - - - - - + {hasPasswordLogin?.() && updatePassword && } + {logout && } ); }; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx index 14ce85abf77..3232c5261f5 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection.tsx @@ -5,7 +5,7 @@ import * as yup from "yup"; import { Form, FormControl } from "components/forms"; import { Card } from "components/ui/Card"; -import { useCurrentUser } from "packages/cloud/services/auth/AuthService"; +import { useCurrentUser } from "core/services/auth"; const emailFormSchema = yup.object({ email: yup.string().required("form.empty.error"), diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/LogoutSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/LogoutSection.tsx new file mode 100644 index 00000000000..a8ede60ab20 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/LogoutSection.tsx @@ -0,0 +1,22 @@ +import { useMutation } from "@tanstack/react-query"; +import { FormattedMessage } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; + +import { AuthLogout } from "core/services/auth"; + +export const LogoutSection = ({ logout }: { logout: AuthLogout }) => { + const { mutateAsync: doLogout, isLoading: isLoggingOut } = useMutation(() => logout()); + + return ( + + + + + + ); +}; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx index 4277a9c2130..646bdb07aac 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/NameSection.tsx @@ -6,9 +6,9 @@ import { Form, FormControl } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; import { Card } from "components/ui/Card"; +import { AuthChangeName, useCurrentUser } from "core/services/auth"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification"; -import { useAuthService, useCurrentUser } from "packages/cloud/services/auth/AuthService"; const nameFormSchema = yup.object({ name: yup.string().required("form.empty.error"), @@ -18,10 +18,13 @@ interface NameFormValues { name: string; } -export const NameSection: React.FC = () => { +interface NameSectionProps { + updateName: AuthChangeName; +} + +export const NameSection: React.FC = ({ updateName }) => { const { formatMessage } = useIntl(); const user = useCurrentUser(); - const { updateName } = useAuthService(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx index 407feefa51d..a60cfde51a4 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection.tsx @@ -7,9 +7,10 @@ import { Form, FormControl } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; import { Card } from "components/ui/Card"; +import { AuthUpdatePassword, useCurrentUser } from "core/services/auth"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification"; -import { FirebaseAuthMessageId, useAuthService, useCurrentUser } from "packages/cloud/services/auth/AuthService"; +import { FirebaseAuthMessageId } from "packages/cloud/services/auth/AuthService"; import { passwordSchema } from "packages/cloud/views/auth/SignupPage/components/SignupForm"; type AuthErrorCodesNames = (typeof AuthErrorCodes)[keyof typeof AuthErrorCodes]; @@ -46,16 +47,15 @@ export interface PasswordFormValues { passwordConfirmation: string; } -export const PasswordSection: React.FC = () => { +interface PasswordSectionProps { + updatePassword: AuthUpdatePassword; +} + +export const PasswordSection: React.FC = ({ updatePassword }) => { const { formatMessage } = useIntl(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); const { email } = useCurrentUser(); - const { updatePassword, hasPasswordLogin } = useAuthService(); - - if (!hasPasswordLogin()) { - return null; - } const defaultFormValues: PasswordFormValues = { currentPassword: "", diff --git a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx index bd22ec46e96..4650608daa7 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx @@ -11,10 +11,10 @@ import { Table } from "components/ui/Table"; import { useListUsers, useUserHook } from "core/api/cloud"; import { WorkspaceUserRead } from "core/api/types/CloudApi"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; +import { useAuthService } from "core/services/auth"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useModalService } from "hooks/services/Modal"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; -import { useAuthService } from "packages/cloud/services/auth/AuthService"; import styles from "./UsersSettingsView.module.scss"; import { InviteUsersModal } from "../InviteUsersModal"; diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 4e9eac1b578..66cd703fba3 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -3,7 +3,7 @@ import { Navigate, Route, Routes, useLocation, useSearchParams } from "react-rou import { ApiErrorBoundary } from "components/common/ApiErrorBoundary"; -import { useInvalidateAllWorkspaceScopeOnChange, useListWorkspaces } from "core/api"; +import { useGetInstanceConfiguration, useInvalidateAllWorkspaceScopeOnChange, useListWorkspaces } from "core/api"; import { useAnalyticsIdentifyUser, useAnalyticsRegisterValues } from "core/services/analytics"; import { useExperiment } from "hooks/services/Experiment"; import { useApiHealthPoll } from "hooks/services/Health"; @@ -105,6 +105,7 @@ export const AutoSelectFirstWorkspace: React.FC = () => { }; const RoutingWithWorkspace: React.FC<{ element?: JSX.Element }> = ({ element }) => { + const { initialSetupComplete } = useGetInstanceConfiguration(); const workspace = useCurrentWorkspace(); useAddAnalyticsContextForWorkspace(workspace); useApiHealthPoll(); @@ -112,7 +113,7 @@ const RoutingWithWorkspace: React.FC<{ element?: JSX.Element }> = ({ element }) // invalidate everything in the workspace scope when the workspaceId changes useInvalidateAllWorkspaceScopeOnChange(workspace.workspaceId); - return workspace.initialSetupComplete ? element ?? : ; + return initialSetupComplete ? element ?? : ; }; export const Routing: React.FC = () => { diff --git a/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts b/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts index 18dfe89dd6f..d7c8f47c6fd 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts @@ -8,4 +8,8 @@ export const mockProInstanceConfig: InstanceConfigurationResponse = { webappUrl: "http://test-airbyte-webapp-url.com", edition: "pro", licenseType: "pro", + initialSetupComplete: true, + defaultUserId: "00000000-0000-0000-0000-000000000000", + defaultOrganizationId: "00000000-0000-0000-0000-000000000000", + defaultWorkspaceId: "00000000-0000-0000-0000-000000000000", }; From 4a711290e3373d491c03c4d1bddadaee6efbc95f Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Thu, 24 Aug 2023 16:32:42 -0700 Subject: [PATCH 008/201] Copy updates in-app to change "migration guide" to "guide" (#8538) Co-authored-by: Alexandre Cuoci --- airbyte-webapp/src/locales/en.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index e98c959be55..b40ebaa12e1 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -795,11 +795,11 @@ "connector.settings": "Settings", "connector.connections": "Connections", "connector.breakingChange.title": "Action Required", - "connector.breakingChange.pendingUpgrade": "Pending upgrade for {name}", + "connector.breakingChange.pendingUpgrade": "There is a pending upgrade for {name}.", "connector.breakingChange.version": "Version {version}", "connector.breakingChange.deprecatedUpgrade": "Upgrade {name} by {date} to continue syncing with this {type}.", "connector.breakingChange.unsupportedUpgrade": "Upgrade {name} to continue syncing with this {type}.", - "connector.breakingChange.moreInfo": "For more information, see the migration guide.", + "connector.breakingChange.moreInfo": "For more information, see this guide.", "connector.breakingChange.upgradeButton": "Upgrade", "connector.breakingChange.upgradeModal.title.source": "Confirm source upgrade", "connector.breakingChange.upgradeModal.title.destination": "Confirm destination upgrade", @@ -807,7 +807,7 @@ "connector.breakingChange.upgradeModal.text": "{name} is used in {count, plural, one {# connection} other {# connections}}. Upgrading this {type} will affect all associated connections:", "connector.breakingChange.upgradeModal.areYouSure": "Are you sure you want to upgrade this {type}?", "connector.breakingChange.upgradeModal.moreConnections": "+ {count} more connections", - "connector.breakingChange.upgradeToast.success": "{type} upgraded successfully. See the migration guide for any remaining actions.", + "connector.breakingChange.upgradeToast.success": "{type} upgraded successfully. See this guide for any remaining actions.", "connector.breakingChange.upgradeToast.failure": "Failed to upgrade {type}. Please reach out to support for assistance.", "credits.credits": "Credits", From 57a6d4aa4d9181efc3911ea1ef7e492f06a9906a Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Thu, 24 Aug 2023 19:53:50 -0400 Subject: [PATCH 009/201] migration: backfill actor.default_version_id and mark column as non-null (#8502) --- .../io/airbyte/bootloader/BootloaderTest.java | 2 +- .../persistence/WorkspaceFilterTest.java | 21 ++- ...kfillActorDefaultVersionAndSetNonNull.java | 72 ++++++++ .../configs_database/schema_dump.txt | 2 +- ...lActorDefaultVersionAndSetNonNullTest.java | 164 ++++++++++++++++++ .../src/main/kotlin/FlagDefinitions.kt | 2 +- .../reporter/MetricRepositoryTest.java | 30 +++- flags.yml | 2 - 8 files changed, 276 insertions(+), 19 deletions(-) create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.java create mode 100644 airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 3ef79255115..14542ccdf22 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -76,7 +76,7 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.20.001"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.21.001"; private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.001"; private static final String CDK_VERSION = "1.2.3"; diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java index 81b88b67527..f618808987b 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java @@ -6,6 +6,7 @@ import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_VERSION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.WORKSPACE; import static io.airbyte.db.instance.jobs.jooq.generated.Tables.JOBS; @@ -29,6 +30,8 @@ class WorkspaceFilterTest extends BaseConfigDatabaseTest { private static final UUID SRC_DEF_ID = UUID.randomUUID(); private static final UUID DST_DEF_ID = UUID.randomUUID(); + private static final UUID SRC_DEF_VER_ID = UUID.randomUUID(); + private static final UUID DST_DEF_VER_ID = UUID.randomUUID(); private static final UUID ACTOR_ID_0 = UUID.randomUUID(); private static final UUID ACTOR_ID_1 = UUID.randomUUID(); private static final UUID ACTOR_ID_2 = UUID.randomUUID(); @@ -53,6 +56,12 @@ static void setUpAll() throws SQLException { .values(DST_DEF_ID, "dstDef", ActorType.destination) .values(UUID.randomUUID(), "dstDef", ActorType.destination) .execute()); + // create actor_definition_version + database.transaction(ctx -> ctx.insertInto(ACTOR_DEFINITION_VERSION, ACTOR_DEFINITION_VERSION.ID, ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, + ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC) + .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}")) + .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}")) + .execute()); // create workspace database.transaction(ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.SLUG, WORKSPACE.INITIAL_SETUP_COMPLETE) @@ -63,11 +72,13 @@ static void setUpAll() throws SQLException { .execute()); // create actors database.transaction( - ctx -> ctx.insertInto(ACTOR, ACTOR.WORKSPACE_ID, ACTOR.ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE) - .values(WORKSPACE_ID_0, ACTOR_ID_0, SRC_DEF_ID, "ACTOR-0", JSONB.valueOf("{}"), ActorType.source) - .values(WORKSPACE_ID_1, ACTOR_ID_1, SRC_DEF_ID, "ACTOR-1", JSONB.valueOf("{}"), ActorType.source) - .values(WORKSPACE_ID_2, ACTOR_ID_2, DST_DEF_ID, "ACTOR-2", JSONB.valueOf("{}"), ActorType.source) - .values(WORKSPACE_ID_3, ACTOR_ID_3, DST_DEF_ID, "ACTOR-3", JSONB.valueOf("{}"), ActorType.source) + ctx -> ctx + .insertInto(ACTOR, ACTOR.WORKSPACE_ID, ACTOR.ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE) + .values(WORKSPACE_ID_0, ACTOR_ID_0, SRC_DEF_ID, SRC_DEF_VER_ID, "ACTOR-0", JSONB.valueOf("{}"), ActorType.source) + .values(WORKSPACE_ID_1, ACTOR_ID_1, SRC_DEF_ID, SRC_DEF_VER_ID, "ACTOR-1", JSONB.valueOf("{}"), ActorType.source) + .values(WORKSPACE_ID_2, ACTOR_ID_2, DST_DEF_ID, DST_DEF_VER_ID, "ACTOR-2", JSONB.valueOf("{}"), ActorType.source) + .values(WORKSPACE_ID_3, ACTOR_ID_3, DST_DEF_ID, DST_DEF_VER_ID, "ACTOR-3", JSONB.valueOf("{}"), ActorType.source) .execute()); // create connections database.transaction( diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.java new file mode 100644 index 00000000000..b3cced62a6b --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import com.google.common.annotations.VisibleForTesting; +import java.util.UUID; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Sets all actor's default_version_id to its actor_definition's default_version_id, and sets the + * column to be non-null. + */ +public class V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.class); + + private static final Table ACTOR_DEFINITION = DSL.table("actor_definition"); + private static final Table ACTOR = DSL.table("actor"); + + private static final Field ID = DSL.field("id", SQLDataType.UUID); + private static final Field DEFAULT_VERSION_ID = DSL.field("default_version_id", SQLDataType.UUID); + private static final Field ACTOR_DEFINITION_ID = DSL.field("actor_definition_id", SQLDataType.UUID); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + backfillActorDefaultVersionId(ctx); + setNonNull(ctx); + } + + @VisibleForTesting + static void backfillActorDefaultVersionId(final DSLContext ctx) { + final var actorDefinitions = ctx.select(ID, DEFAULT_VERSION_ID) + .from(ACTOR_DEFINITION) + .fetch(); + + for (final var actorDefinition : actorDefinitions) { + final UUID actorDefinitionId = actorDefinition.get(ID); + final UUID defaultVersionId = actorDefinition.get(DEFAULT_VERSION_ID); + + ctx.update(ACTOR) + .set(DEFAULT_VERSION_ID, defaultVersionId) + .where(ACTOR_DEFINITION_ID.eq(actorDefinitionId)) + .execute(); + } + } + + @VisibleForTesting + static void setNonNull(final DSLContext ctx) { + ctx.alterTable(ACTOR) + .alterColumn(DEFAULT_VERSION_ID) + .setNotNull() + .execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index b8e7055617a..bb48f21190e 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -20,7 +20,7 @@ create table "public"."actor" ( "tombstone" boolean not null default false, "created_at" timestamp(6) with time zone not null default current_timestamp, "updated_at" timestamp(6) with time zone not null default current_timestamp, - "default_version_id" uuid, + "default_version_id" uuid not null, constraint "actor_pkey" primary key ("id") ); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java new file mode 100644 index 00000000000..91b58f31561 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.util.Objects; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.exception.DataAccessException; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest extends AbstractConfigsDatabaseTest { + + private static final Table ACTOR = DSL.table("actor"); + private static final Table ACTOR_DEFINITION = DSL.table("actor_definition"); + private static final Table ACTOR_DEFINITION_VERSION = DSL.table("actor_definition_version"); + private static final Table WORKSPACE = DSL.table("workspace"); + + private static final Field ID_COL = DSL.field("id", SQLDataType.UUID); + private static final Field DEFAULT_VERSION_ID_COL = DSL.field("default_version_id", SQLDataType.UUID); + private static final Field ACTOR_DEFINITION_ID_COL = DSL.field("actor_definition_id", SQLDataType.UUID); + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); + private static final UUID ACTOR_ID = UUID.randomUUID(); + private static final UUID VERSION_ID = UUID.randomUUID(); + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_20_001__MakeManualNullableForRemoval(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + private UUID getDefaultVersionIdForActorId(final DSLContext ctx, final UUID actorId) { + final var actor = ctx.select(DEFAULT_VERSION_ID_COL) + .from(ACTOR) + .where(ID_COL.eq(actorId)) + .fetchOne(); + + if (Objects.isNull(actor)) { + return null; + } + + return actor.get(DEFAULT_VERSION_ID_COL); + } + + static void insertDependencies(final DSLContext ctx) { + ctx.insertInto(WORKSPACE) + .columns( + ID_COL, + DSL.field("name"), + DSL.field("slug"), + DSL.field("initial_setup_complete")) + .values( + WORKSPACE_ID, + "name1", + "default", + true) + .execute(); + + ctx.insertInto(ACTOR_DEFINITION) + .columns( + ID_COL, + DSL.field("name"), + DSL.field("actor_type")) + .values( + ACTOR_DEFINITION_ID, + "source def name", + ActorType.source) + .execute(); + + ctx.insertInto(ACTOR_DEFINITION_VERSION) + .columns(ID_COL, ACTOR_DEFINITION_ID_COL, DSL.field("docker_repository"), DSL.field("docker_image_tag"), DSL.field("spec")) + .values(VERSION_ID, ACTOR_DEFINITION_ID, "airbyte/some-source", "1.0.0", JSONB.valueOf("{}")) + .execute(); + + ctx.update(ACTOR_DEFINITION) + .set(DEFAULT_VERSION_ID_COL, VERSION_ID) + .where(ID_COL.eq(ACTOR_DEFINITION_ID)) + .execute(); + } + + @Test + void testBackFillActorDefaultVersionId() { + final DSLContext ctx = getDslContext(); + insertDependencies(ctx); + + ctx.insertInto(ACTOR) + .columns( + ID_COL, + ACTOR_DEFINITION_ID_COL, + DSL.field("workspace_id"), + DSL.field("name"), + DSL.field("configuration"), + DSL.field("actor_type")) + .values( + ACTOR_ID, + ACTOR_DEFINITION_ID, + WORKSPACE_ID, + "My Source", + JSONB.valueOf("{}"), + ActorType.source) + .execute(); + + assertNull(getDefaultVersionIdForActorId(ctx, ACTOR_ID)); + + V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.backfillActorDefaultVersionId(ctx); + + assertEquals(VERSION_ID, getDefaultVersionIdForActorId(ctx, ACTOR_ID)); + } + + @Test + void testActorDefaultVersionIdIsNotNull() { + final DSLContext context = getDslContext(); + + V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull.setNonNull(context); + + final Exception e = Assertions.assertThrows(DataAccessException.class, () -> { + context.insertInto(ACTOR) + .columns( + ID_COL, + ACTOR_DEFINITION_ID_COL, + DSL.field("workspace_id"), + DSL.field("name"), + DSL.field("configuration"), + DSL.field("actor_type")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + UUID.randomUUID(), + "My Source", + JSONB.valueOf("{}"), + ActorType.source) + .execute(); + }); + Assertions.assertTrue(e.getMessage().contains("null value in column \"default_version_id\" of relation \"actor\" violates not-null constraint")); + } + +} diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index 5a3ffb223d0..d912a4898ee 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -61,7 +61,7 @@ object ShouldFailSyncIfHeartbeatFailure : Permanent(key = "heartbeat.fa object ConnectorVersionOverride : Permanent(key = "connectors.versionOverrides", default = "") -object UseActorScopedDefaultVersions : Temporary(key = "connectors.useActorScopedDefaultVersions", default = false) +object UseActorScopedDefaultVersions : Temporary(key = "connectors.useActorScopedDefaultVersions", default = true) object IngestBreakingChanges : Temporary(key = "connectors.ingestBreakingChanges", default = true) diff --git a/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java b/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java index 3078db34019..4d3128e3420 100644 --- a/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java +++ b/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java @@ -11,6 +11,7 @@ import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_CATALOG_FETCH_EVENT; import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION; +import static io.airbyte.db.instance.configs.jooq.generated.Tables.ACTOR_DEFINITION_VERSION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.CONNECTION; import static io.airbyte.db.instance.configs.jooq.generated.Tables.WORKSPACE; import static io.airbyte.db.instance.jobs.jooq.generated.Tables.ATTEMPTS; @@ -58,6 +59,8 @@ class MetricRepositoryTest { private static final UUID SRC_DEF_ID = UUID.randomUUID(); private static final UUID DST_DEF_ID = UUID.randomUUID(); + private static final UUID SRC_DEF_VER_ID = UUID.randomUUID(); + private static final UUID DST_DEF_VER_ID = UUID.randomUUID(); private static MetricRepository db; private static DSLContext ctx; @@ -80,6 +83,12 @@ public static void setUpAll() throws DatabaseInitializationException, IOExceptio .values(UUID.randomUUID(), "dstDef", ActorType.destination) .execute(); + ctx.insertInto(ACTOR_DEFINITION_VERSION, ACTOR_DEFINITION_VERSION.ID, ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, + ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC) + .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}")) + .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}")) + .execute(); + // drop constraints to simplify test set up ctx.alterTable(ACTOR).dropForeignKey(ACTOR__ACTOR_WORKSPACE_ID_FKEY.constraint()).execute(); ctx.alterTable(CONNECTION).dropForeignKey(CONNECTION__CONNECTION_DESTINATION_ID_FKEY.constraint()).execute(); @@ -318,10 +327,11 @@ void shouldReturnNumConnectionsBasic() { final var srcId = UUID.randomUUID(); final var dstId = UUID.randomUUID(); - ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, + ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC_DEF_VER_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DST_DEF_VER_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute(); ctx.insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, @@ -345,10 +355,11 @@ void shouldIgnoreNonRunningConnections() { final var srcId = UUID.randomUUID(); final var dstId = UUID.randomUUID(); - ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, + ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC_DEF_VER_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DST_DEF_VER_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute(); ctx.insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, @@ -374,10 +385,11 @@ void shouldIgnoreDeletedWorkspaces() { final var srcId = UUID.randomUUID(); final var dstId = UUID.randomUUID(); - ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, ACTOR.ACTOR_TYPE, + ctx.insertInto(ACTOR, ACTOR.ID, ACTOR.WORKSPACE_ID, ACTOR.ACTOR_DEFINITION_ID, ACTOR.DEFAULT_VERSION_ID, ACTOR.NAME, ACTOR.CONFIGURATION, + ACTOR.ACTOR_TYPE, ACTOR.TOMBSTONE) - .values(srcId, workspaceId, SRC_DEF_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) - .values(dstId, workspaceId, DST_DEF_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) + .values(srcId, workspaceId, SRC_DEF_ID, SRC_DEF_VER_ID, SRC, JSONB.valueOf("{}"), ActorType.source, false) + .values(dstId, workspaceId, DST_DEF_ID, DST_DEF_VER_ID, DEST, JSONB.valueOf("{}"), ActorType.destination, false) .execute(); ctx.insertInto(CONNECTION, CONNECTION.ID, CONNECTION.NAMESPACE_DEFINITION, CONNECTION.SOURCE_ID, CONNECTION.DESTINATION_ID, diff --git a/flags.yml b/flags.yml index 25b00f31103..d859c0666ee 100644 --- a/flags.yml +++ b/flags.yml @@ -25,8 +25,6 @@ flags: serve: "" - name: platform.add-scheduling-jitter serve: false - - name: connectors.ingestBreakingChanges - serve: false - name: check-replication-progress serve: true - name: use-new-retries From bbd8c94948d561906c6035820ec766eef34a464d Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Fri, 25 Aug 2023 02:10:46 +0200 Subject: [PATCH 010/201] Pass breaking changes for def into getSupportStateUpdate (#8539) --- .../persistence/SupportStateUpdater.java | 24 +++++++++++++++---- .../persistence/SupportStateUpdaterTest.java | 20 ++++++++-------- .../job/tracker/TrackingMetadataTest.java | 10 ++++---- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java index a4aff2ae307..a96a3880b96 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; @@ -181,10 +182,21 @@ private Version getVersionTag(final List actorDefinition * Updates the version support states for all source and destination definitions. */ public void updateSupportStates() throws IOException { + updateSupportStates(LocalDate.now()); + } + + /** + * Updates the version support states for all source and destination definitions based on a + * reference date. + */ + @VisibleForTesting + void updateSupportStates(final LocalDate referenceDate) throws IOException { LOGGER.info("Updating support states for all definitions"); final List sourceDefinitions = configRepository.listPublicSourceDefinitions(false); final List destinationDefinitions = configRepository.listPublicDestinationDefinitions(false); - final List breakingChanges = configRepository.listBreakingChanges(); + final List allBreakingChanges = configRepository.listBreakingChanges(); + final Map> breakingChangesMap = allBreakingChanges.stream() + .collect(Collectors.groupingBy(ActorDefinitionBreakingChange::getActorDefinitionId)); SupportStateUpdate comboSupportStateUpdate = new SupportStateUpdate(List.of(), List.of(), List.of()); @@ -192,10 +204,11 @@ public void updateSupportStates() throws IOException { final List actorDefinitionVersions = configRepository.listActorDefinitionVersionsForDefinition(sourceDefinition.getSourceDefinitionId()); final Version currentDefaultVersion = getVersionTag(actorDefinitionVersions, sourceDefinition.getDefaultVersionId()); + final List breakingChangesForDef = + breakingChangesMap.getOrDefault(sourceDefinition.getSourceDefinitionId(), List.of()); final SupportStateUpdate supportStateUpdate = - getSupportStateUpdate(currentDefaultVersion, LocalDate.now(), breakingChanges, actorDefinitionVersions); - + getSupportStateUpdate(currentDefaultVersion, referenceDate, breakingChangesForDef, actorDefinitionVersions); comboSupportStateUpdate = SupportStateUpdate.merge(comboSupportStateUpdate, supportStateUpdate); } @@ -203,10 +216,11 @@ public void updateSupportStates() throws IOException { final List actorDefinitionVersions = configRepository.listActorDefinitionVersionsForDefinition(destinationDefinition.getDestinationDefinitionId()); final Version currentDefaultVersion = getVersionTag(actorDefinitionVersions, destinationDefinition.getDefaultVersionId()); + final List breakingChangesForDef = + breakingChangesMap.getOrDefault(destinationDefinition.getDestinationDefinitionId(), List.of()); final SupportStateUpdate supportStateUpdate = - getSupportStateUpdate(currentDefaultVersion, LocalDate.now(), breakingChanges, actorDefinitionVersions); - + getSupportStateUpdate(currentDefaultVersion, referenceDate, breakingChangesForDef, actorDefinitionVersions); comboSupportStateUpdate = SupportStateUpdate.merge(comboSupportStateUpdate, supportStateUpdate); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java index d0562d1f097..8ee154d1031 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java @@ -137,37 +137,37 @@ void testUpdateSupportStates() throws IOException { .withDefaultVersionId(SRC_V1_0_0.getVersionId()); final UUID destinationDefinitionId = UUID.randomUUID(); - final ActorDefinitionVersion DEST_V1_1_0 = createActorDefinitionVersion(V1_1_0) + final ActorDefinitionVersion DEST_V0_1_0 = createActorDefinitionVersion(V0_1_0) .withActorDefinitionId(destinationDefinitionId); - final ActorDefinitionVersion DEST_V2_0_0 = createActorDefinitionVersion(V2_0_0) + final ActorDefinitionVersion DEST_V1_0_0 = createActorDefinitionVersion(V1_0_0) .withActorDefinitionId(destinationDefinitionId); final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() .withName("destination") .withDestinationDefinitionId(destinationDefinitionId) - .withDefaultVersionId(DEST_V2_0_0.getVersionId()); + .withDefaultVersionId(DEST_V1_0_0.getVersionId()); final ActorDefinitionBreakingChange SRC_BC_1_0_0 = createBreakingChange(V1_0_0, "2020-01-01"); - final ActorDefinitionBreakingChange DEST_BC_2_0_0 = createBreakingChange(V2_0_0, "2020-02-01") + final ActorDefinitionBreakingChange DEST_BC_1_0_0 = createBreakingChange(V1_0_0, "2020-02-01") .withActorDefinitionId(destinationDefinitionId); when(mConfigRepository.listPublicSourceDefinitions(false)).thenReturn(List.of(sourceDefinition)); when(mConfigRepository.listPublicDestinationDefinitions(false)).thenReturn(List.of(destinationDefinition)); - when(mConfigRepository.listBreakingChanges()).thenReturn(List.of(SRC_BC_1_0_0, DEST_BC_2_0_0)); + when(mConfigRepository.listBreakingChanges()).thenReturn(List.of(SRC_BC_1_0_0, DEST_BC_1_0_0)); when(mConfigRepository.listActorDefinitionVersionsForDefinition(ACTOR_DEFINITION_ID)) .thenReturn(List.of(SRC_V0_1_0, SRC_V1_0_0)); when(mConfigRepository.listActorDefinitionVersionsForDefinition(destinationDefinitionId)) - .thenReturn(List.of(DEST_V1_1_0, DEST_V2_0_0)); + .thenReturn(List.of(DEST_V0_1_0, DEST_V1_0_0)); - supportStateUpdater.updateSupportStates(); + supportStateUpdater.updateSupportStates(LocalDate.parse("2020-01-15")); verify(mConfigRepository).listPublicSourceDefinitions(false); verify(mConfigRepository).listPublicDestinationDefinitions(false); verify(mConfigRepository).listBreakingChanges(); verify(mConfigRepository).listActorDefinitionVersionsForDefinition(ACTOR_DEFINITION_ID); verify(mConfigRepository).listActorDefinitionVersionsForDefinition(destinationDefinitionId); - verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V0_1_0.getVersionId(), DEST_V1_1_0.getVersionId()), - SupportState.UNSUPPORTED); - verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V1_0_0.getVersionId(), DEST_V2_0_0.getVersionId()), + verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V0_1_0.getVersionId()), SupportState.UNSUPPORTED); + verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(DEST_V0_1_0.getVersionId()), SupportState.DEPRECATED); + verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V1_0_0.getVersionId(), DEST_V1_0_0.getVersionId()), SupportState.SUPPORTED); verifyNoMoreInteractions(mConfigRepository); } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java index 08ca67abf92..73f4a32e5ad 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/tracker/TrackingMetadataTest.java @@ -31,6 +31,8 @@ class TrackingMetadataTest { + private static final String UNUSED = "unused"; + @Test void testNulls() { final UUID connectionId = UUID.randomUUID(); @@ -89,21 +91,21 @@ void testGenerateJobAttemptMetadataToleratesNullInputs() { assertTrue(TrackingMetadata.generateJobAttemptMetadata(null).isEmpty()); // There is a job, but it has no attempts. - final Job jobWithNoAttempts = new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(), JobStatus.PENDING, 0L, 0, 0); + final Job jobWithNoAttempts = new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(), JobStatus.PENDING, 0L, 0, 0); assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithNoAttempts).isEmpty()); // There is a job, and it has an attempt, but the attempt has null output. final Attempt mockAttemptWithNullOutput = mock(Attempt.class); when(mockAttemptWithNullOutput.getOutput()).thenReturn(null); final Job jobWithNullOutput = - new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); + new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithNullOutput).isEmpty()); // There is a job, and it has an attempt, but the attempt has empty output. final Attempt mockAttemptWithEmptyOutput = mock(Attempt.class); when(mockAttemptWithEmptyOutput.getOutput()).thenReturn(Optional.empty()); final Job jobWithEmptyOutput = - new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); + new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(mockAttemptWithNullOutput), JobStatus.PENDING, 0L, 0, 0); assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithEmptyOutput).isEmpty()); // There is a job, and it has an attempt, and the attempt has output, but the output has no sync @@ -112,7 +114,7 @@ void testGenerateJobAttemptMetadataToleratesNullInputs() { final JobOutput mockJobOutputWithoutSync = mock(JobOutput.class); when(mockAttemptWithOutput.getOutput()).thenReturn(Optional.of(mockJobOutputWithoutSync)); when(mockJobOutputWithoutSync.getSync()).thenReturn(null); - final Job jobWithoutSyncInfo = new Job(1, JobConfig.ConfigType.SYNC, "unused", null, List.of(mockAttemptWithOutput), JobStatus.PENDING, 0L, 0, 0); + final Job jobWithoutSyncInfo = new Job(1, JobConfig.ConfigType.SYNC, UNUSED, null, List.of(mockAttemptWithOutput), JobStatus.PENDING, 0L, 0, 0); assertTrue(TrackingMetadata.generateJobAttemptMetadata(jobWithoutSyncInfo).isEmpty()); } From 9d0e3f53c8c6c16adff6924d70ae2509e0a5f4d7 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Fri, 25 Aug 2023 10:57:56 +0200 Subject: [PATCH 011/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Hide=20wa?= =?UTF-8?q?rning=20in=20tsconfig=20path=20plugin=20(#8541)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/vite.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/vite.config.ts b/airbyte-webapp/vite.config.ts index 6d782b236d5..0521454caef 100644 --- a/airbyte-webapp/vite.config.ts +++ b/airbyte-webapp/vite.config.ts @@ -25,7 +25,7 @@ export default defineConfig(() => { react(), buildInfo(), compileFormatJsMessages(), - viteTsconfigPaths(), + viteTsconfigPaths({ ignoreConfigErrors: true }), viteYaml(), svgrPlugin({ svgrOptions: { From f20bf829667e20ec881df36c722b3fe0d345bb5d Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Fri, 25 Aug 2023 12:12:08 +0200 Subject: [PATCH 012/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Remove=20?= =?UTF-8?q?unnecessary=20useUnmounts=20(#8542)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connection/CatalogDiffModal/CatalogDiffModal.tsx | 7 ------- .../connection/ConnectionForm/ConnectionFormFields.tsx | 8 +------- .../refreshSourceSchemaWithConfirmationOnDirty.ts | 5 ----- .../ConnectionReplicationPage/ResetWarningModal.tsx | 7 ------- 4 files changed, 1 insertion(+), 26 deletions(-) diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx index 8ce4b516d16..674f6ea6195 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx @@ -1,12 +1,10 @@ import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; -import { useUnmount } from "react-use"; import { Button } from "components/ui/Button"; import { ModalBody, ModalFooter } from "components/ui/Modal"; import { AirbyteCatalog, CatalogDiff } from "core/request/AirbyteClient"; -import { useModalService } from "hooks/services/Modal"; import styles from "./CatalogDiffModal.module.scss"; import { DiffSection } from "./DiffSection"; @@ -20,16 +18,11 @@ interface CatalogDiffModalProps { } export const CatalogDiffModal: React.FC = ({ catalogDiff, catalog, onClose }) => { - const { closeModal } = useModalService(); const { newItems, removedItems, changedItems } = useMemo( () => getSortedDiff(catalogDiff.transforms), [catalogDiff.transforms] ); - useUnmount(() => { - closeModal(); - }); - return ( <> diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.tsx index 9d0c63c8354..87dd90e2f84 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionFormFields.tsx @@ -3,7 +3,7 @@ import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { Field } from "formik"; import React from "react"; import { FormattedMessage } from "react-intl"; -import { useUnmount, useEffectOnce } from "react-use"; +import { useEffectOnce } from "react-use"; import { FormChangeTracker } from "components/common/FormChangeTracker"; import { Button } from "components/ui/Button"; @@ -13,7 +13,6 @@ import { FlexContainer } from "components/ui/Flex"; import { FeatureItem, useFeature } from "core/services/features"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { ConnectionConfigurationFormPreview } from "./ConnectionConfigurationFormPreview"; import styles from "./ConnectionFormFields.module.scss"; @@ -34,14 +33,9 @@ export const ConnectionFormFields: React.FC = ({ isSu const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema); const { mode, formId } = useConnectionFormService(); - const { clearFormChange } = useFormChangeTrackerService(); const refreshSchema = useRefreshSourceSchemaWithConfirmationOnDirty(dirty); - useUnmount(() => { - clearFormChange(formId); - }); - // If the source doesn't select any streams by default, the initial untouched state // won't validate that at least one is selected. In this case, a user could submit the form // without selecting any streams, which would trigger an error and cause a lousy UX. diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts b/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts index fd0945f1b1d..5b7df39924f 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/refreshSourceSchemaWithConfirmationOnDirty.ts @@ -1,5 +1,4 @@ import { useCallback } from "react"; -import { useUnmount } from "react-use"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; @@ -10,10 +9,6 @@ export const useRefreshSourceSchemaWithConfirmationOnDirty = (dirty: boolean) => const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const { formId, refreshSchema } = useConnectionFormService(); - useUnmount(() => { - closeConfirmationModal(); - }); - return useCallback(() => { if (dirty) { openConfirmationModal({ diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx index 3d4d9a6c285..0e553063e3e 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx @@ -1,13 +1,11 @@ import { useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { useUnmount } from "react-use"; import { LabeledSwitch } from "components"; import { Button } from "components/ui/Button"; import { ModalBody, ModalFooter } from "components/ui/Modal"; import { ConnectionStateType } from "core/request/AirbyteClient"; -import { useModalService } from "hooks/services/Modal"; interface ResetWarningModalProps { onClose: (withReset: boolean) => void; @@ -16,15 +14,10 @@ interface ResetWarningModalProps { } export const ResetWarningModal: React.FC = ({ onCancel, onClose, stateType }) => { - const { closeModal } = useModalService(); const { formatMessage } = useIntl(); const [withReset, setWithReset] = useState(true); const requireFullReset = stateType === ConnectionStateType.legacy; - useUnmount(() => { - closeModal(); - }); - return ( <> From 63e9af212046cf58f522d04f6b467ec1f6933bec Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Fri, 25 Aug 2023 13:11:43 +0200 Subject: [PATCH 013/201] align the default sync mode for new streams (#8118) Co-authored-by: josephkmh --- .../server/handlers/ConnectionsHandler.java | 15 +++- .../server/handlers/SchedulerHandler.java | 11 ++- .../WebBackendConnectionsHandler.java | 6 +- .../AutoPropagateSchemaChangeHelper.java | 36 +-------- .../handlers/helpers/CatalogConverter.java | 80 +++++++++++++++++-- .../converters/CatalogConverterTest.java | 45 +++++++++++ .../server/handlers/SchedulerHandlerTest.java | 10 +++ .../AutoPropagateSchemaChangeHelperTest.java | 29 ++++--- .../test/acceptance/BasicAcceptanceTests.java | 2 +- .../cypress/e2e/connection/syncModes.cy.ts | 52 ++++++------ 10 files changed, 208 insertions(+), 78 deletions(-) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java index 1abb20be715..42749957f44 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java @@ -28,6 +28,7 @@ import io.airbyte.api.model.generated.ConnectionUpdate; import io.airbyte.api.model.generated.DestinationRead; import io.airbyte.api.model.generated.DestinationSearch; +import io.airbyte.api.model.generated.DestinationSyncMode; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.ListConnectionsForWorkspacesRequestBody; import io.airbyte.api.model.generated.SourceRead; @@ -728,7 +729,19 @@ public Optional getConnectionAirbyteCatalog(final UUID connectio final ActorDefinitionVersion sourceVersion = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceConnection.getWorkspaceId(), connection.getSourceId()); final io.airbyte.protocol.models.AirbyteCatalog jsonCatalog = Jsons.object(catalog.getCatalog(), io.airbyte.protocol.models.AirbyteCatalog.class); - return Optional.of(CatalogConverter.toApi(jsonCatalog, sourceVersion)); + final StandardDestinationDefinition destination = configRepository.getDestinationDefinitionFromConnection(connectionId); + // Note: we're using the workspace from the source to save an extra db request. + final ActorDefinitionVersion destinationVersion = + actorDefinitionVersionHelper.getDestinationVersion(destination, sourceConnection.getWorkspaceId()); + final List supportedDestinationSyncModes = + Enums.convertListTo(destinationVersion.getSpec().getSupportedDestinationSyncModes(), DestinationSyncMode.class); + final var convertedCatalog = Optional.of(CatalogConverter.toApi(jsonCatalog, sourceVersion)); + if (convertedCatalog.isPresent()) { + convertedCatalog.get().getStreams().forEach((streamAndConfiguration) -> { + CatalogConverter.ensureCompatibleDestinationSyncMode(streamAndConfiguration, supportedDestinationSyncModes); + }); + } + return convertedCatalog; } public ConnectionReadList searchConnections(final ConnectionSearch connectionSearch) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index b8c7e4f6040..798346964ce 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -448,6 +448,11 @@ public void applySchemaChangeForSource(final SourceAutoPropagateChange sourceAut final ConnectionUpdate updateObject = new ConnectionUpdate().connectionId(connectionRead.getConnectionId()); + final UUID destinationDefinitionId = + configRepository.getDestinationDefinitionFromConnection(connectionRead.getConnectionId()).getDestinationDefinitionId(); + final var supportedDestinationSyncModes = + getDestinationSpecification(new DestinationDefinitionIdWithWorkspaceId().destinationDefinitionId(destinationDefinitionId) + .workspaceId(sourceAutoPropagateChange.getWorkspaceId())).getSupportedDestinationSyncModes(); if (shouldAutoPropagate(diff, sourceAutoPropagateChange.getWorkspaceId(), connectionRead)) { applySchemaChange(updateObject.getConnectionId(), @@ -457,7 +462,7 @@ public void applySchemaChangeForSource(final SourceAutoPropagateChange sourceAut sourceAutoPropagateChange.getCatalog(), diff.getTransforms(), sourceAutoPropagateChange.getCatalogId(), - connectionRead.getNonBreakingChangesPreference()); + connectionRead.getNonBreakingChangesPreference(), supportedDestinationSyncModes); connectionsHandler.updateConnection(updateObject); LOGGER.info("Propagating changes for connectionId: '{}', new catalogId '{}'", connectionRead.getConnectionId(), sourceAutoPropagateChange.getCatalogId()); @@ -737,7 +742,8 @@ private void applySchemaChange(final UUID connectionId, final io.airbyte.api.model.generated.AirbyteCatalog newCatalog, final List transformations, final UUID sourceCatalogId, - final NonBreakingChangesPreference nonBreakingChangesPreference) { + final NonBreakingChangesPreference nonBreakingChangesPreference, + final List supportedDestinationSyncModes) { MetricClientFactory.getMetricClient().count(OssMetricsRegistry.SCHEMA_CHANGE_AUTO_PROPAGATED, 1, new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString())); final io.airbyte.api.model.generated.AirbyteCatalog catalog = getUpdatedSchema( @@ -745,6 +751,7 @@ private void applySchemaChange(final UUID connectionId, newCatalog, transformations, nonBreakingChangesPreference, + supportedDestinationSyncModes, featureFlagClient, workspaceId); updateObject.setSyncCatalog(catalog); updateObject.setSourceCatalogId(sourceCatalogId); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java index 88e0d6691f0..7f4243924a3 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WebBackendConnectionsHandler.java @@ -358,14 +358,14 @@ public WebBackendConnectionRead webBackendGetConnection(final WebBackendConnecti final AirbyteCatalog configuredCatalog = connection.getSyncCatalog(); /* * This catalog represents the full catalog that was used to create the configured catalog. It will - * have all streams that were present at the time. It will have no configuration set. + * have all streams that were present at the time. It will have default configuration options set. */ final Optional catalogUsedToMakeConfiguredCatalog = connectionsHandler .getConnectionAirbyteCatalog(webBackendConnectionRequestBody.getConnectionId()); /* - * This catalog represents the full catalog that exists now for the source. It will have no - * configuration set. + * This catalog represents the full catalog that exists now for the source. It will have default + * configuration options set. */ final Optional refreshedCatalog; if (MoreBooleans.isTruthy(webBackendConnectionRequestBody.getWithRefreshedCatalog())) { diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java index fed93cfde99..4fa299bb1fc 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelper.java @@ -11,7 +11,6 @@ import io.airbyte.api.model.generated.NonBreakingChangesPreference; import io.airbyte.api.model.generated.StreamDescriptor; import io.airbyte.api.model.generated.StreamTransform; -import io.airbyte.api.model.generated.SyncMode; import io.airbyte.commons.json.Jsons; import io.airbyte.featureflag.AutoPropagateNewStreams; import io.airbyte.featureflag.FeatureFlagClient; @@ -51,6 +50,7 @@ public static AirbyteCatalog getUpdatedSchema(final AirbyteCatalog oldCatalog, final AirbyteCatalog newCatalog, final List transformations, final NonBreakingChangesPreference nonBreakingChangesPreference, + final List supportedDestinationSyncModes, final FeatureFlagClient featureFlagClient, final UUID workspaceId) { final AirbyteCatalog copiedOldCatalog = Jsons.clone(oldCatalog); @@ -74,7 +74,9 @@ public static AirbyteCatalog getUpdatedSchema(final AirbyteCatalog oldCatalog, // the catalog. streamAndConfigurationToAdd.getConfig() .selected(true); - configureDefaultSyncModesForNewStream(streamAndConfigurationToAdd); + CatalogConverter.configureDefaultSyncModesForNewStream(streamAndConfigurationToAdd.getStream(), + streamAndConfigurationToAdd.getConfig()); + CatalogConverter.ensureCompatibleDestinationSyncMode(streamAndConfigurationToAdd, supportedDestinationSyncModes); } // TODO(mfsiega-airbyte): handle the case where the chosen sync mode isn't actually one of the // supported sync modes. @@ -93,36 +95,6 @@ public static AirbyteCatalog getUpdatedSchema(final AirbyteCatalog oldCatalog, return new AirbyteCatalog().streams(List.copyOf(oldCatalogPerStream.values())); } - private static void configureDefaultSyncModesForNewStream(final AirbyteStreamAndConfiguration streamToAdd) { - // TODO(mfsiega-airbyte): unite this with the default config generation in the CatalogConverter. - final var stream = streamToAdd.getStream(); - final var config = streamToAdd.getConfig(); - final boolean hasSourceDefinedCursor = stream.getSourceDefinedCursor() != null && stream.getSourceDefinedCursor(); - final boolean hasSourceDefinedPrimaryKey = stream.getSourceDefinedPrimaryKey() != null && !stream.getSourceDefinedPrimaryKey().isEmpty(); - final boolean supportsFullRefresh = stream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH); - if (hasSourceDefinedCursor && hasSourceDefinedPrimaryKey) { // Source-defined cursor and primary key - config - .syncMode(SyncMode.INCREMENTAL) - .destinationSyncMode(DestinationSyncMode.APPEND_DEDUP) - .primaryKey(stream.getSourceDefinedPrimaryKey()); - } else if (hasSourceDefinedCursor && supportsFullRefresh) { // Source-defined cursor but no primary key. - // NOTE: we prefer Full Refresh | Overwrite to avoid the risk of an Incremental | Append sync - // blowing up their destination. - config - .syncMode(SyncMode.FULL_REFRESH) - .destinationSyncMode(DestinationSyncMode.OVERWRITE); - } else if (hasSourceDefinedCursor) { // Source-defined cursor but no primary key *and* no full-refresh supported. - // If *only* incremental is supported, we go with it. - config - .syncMode(SyncMode.INCREMENTAL) - .destinationSyncMode(DestinationSyncMode.APPEND); - } else { // No source-defined cursor at all. - config - .syncMode(SyncMode.FULL_REFRESH) - .destinationSyncMode(DestinationSyncMode.OVERWRITE); - } - } - @VisibleForTesting static Map extractStreamAndConfigPerStreamDescriptor(final AirbyteCatalog catalog) { return catalog.getStreams().stream().collect(Collectors.toMap( diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java index d1b1d85be55..ab349611d08 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/CatalogConverter.java @@ -220,6 +220,44 @@ public static io.airbyte.protocol.models.ConfiguredAirbyteCatalog toConfiguredPr .withStreams(streams); } + /** + * Set the default sync modes for an un-configured stream based on the stream properties. + *

+ * The logic is: - source-defined cursor and source-defined primary key -> INCREMENTAL, APPEND-DEDUP + * - source-defined cursor only or nothing defined by the source -> FULL REFRESH, OVERWRITE - + * source-defined cursor and full refresh not available as a sync method -> INCREMENTAL, APPEND + * + * @param streamToConfigure the stream for which we're picking a sync mode + * @param config the config to which we'll write the sync mode + */ + public static void configureDefaultSyncModesForNewStream(final AirbyteStream streamToConfigure, final AirbyteStreamConfiguration config) { + final boolean hasSourceDefinedCursor = streamToConfigure.getSourceDefinedCursor() != null && streamToConfigure.getSourceDefinedCursor(); + final boolean hasSourceDefinedPrimaryKey = + streamToConfigure.getSourceDefinedPrimaryKey() != null && !streamToConfigure.getSourceDefinedPrimaryKey().isEmpty(); + final boolean supportsFullRefresh = streamToConfigure.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH); + if (hasSourceDefinedCursor && hasSourceDefinedPrimaryKey) { // Source-defined cursor and primary key + config + .syncMode(SyncMode.INCREMENTAL) + .destinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .primaryKey(streamToConfigure.getSourceDefinedPrimaryKey()); + } else if (hasSourceDefinedCursor && supportsFullRefresh) { // Source-defined cursor but no primary key. + // NOTE: we prefer Full Refresh | Overwrite to avoid the risk of an Incremental | Append sync + // blowing up their destination. + config + .syncMode(SyncMode.FULL_REFRESH) + .destinationSyncMode(DestinationSyncMode.OVERWRITE); + } else if (hasSourceDefinedCursor) { // Source-defined cursor but no primary key *and* no full-refresh supported. + // If *only* incremental is supported, we go with it. + config + .syncMode(SyncMode.INCREMENTAL) + .destinationSyncMode(DestinationSyncMode.APPEND); + } else { // No source-defined cursor at all. + config + .syncMode(SyncMode.FULL_REFRESH) + .destinationSyncMode(DestinationSyncMode.OVERWRITE); + } + } + @SuppressWarnings("LineLength") private static io.airbyte.api.model.generated.AirbyteStreamConfiguration generateDefaultConfiguration(final io.airbyte.api.model.generated.AirbyteStream stream, final Boolean suggestingStreams, @@ -239,11 +277,7 @@ private static io.airbyte.api.model.generated.AirbyteStreamConfiguration generat result.setSuggested(isSelected); result.setSelected(isSelected); - if (stream.getSupportedSyncModes().size() > 0) { - result.setSyncMode(stream.getSupportedSyncModes().get(0)); - } else { - result.setSyncMode(io.airbyte.api.model.generated.SyncMode.INCREMENTAL); - } + configureDefaultSyncModesForNewStream(stream, result); return result; } @@ -332,4 +366,40 @@ private static String streamDescriptorToStringForFieldSelection(final StreamDesc return String.format("%s/%s", streamDescriptor.getNamespace(), streamDescriptor.getName()); } + /** + * Ensure that the configured sync modes are compatible with the source and the destination. + *

+ * When we discover a new stream -- either during manual or auto schema refresh -- we want to pick + * some default sync modes. This depends both on the source-supported sync modes -- represented in + * the discovered catalog -- and the destination-supported sync modes. The latter is tricky because + * the place where we're generating the default configuration isn't associated with a particular + * destination. + *

+ * A longer-term fix would be to restructure how we generate this default config, but for now we use + * this to ensure that we've chosen defaults that work for the relevant sync. + * + * @param streamAndConfiguration the stream and configuration to check + * @param supportedDestinationSyncModes the sync modes supported by the destination + */ + public static void ensureCompatibleDestinationSyncMode(AirbyteStreamAndConfiguration streamAndConfiguration, + List supportedDestinationSyncModes) { + if (supportedDestinationSyncModes.contains(streamAndConfiguration.getConfig().getDestinationSyncMode())) { + return; + } + final var sourceSupportsFullRefresh = streamAndConfiguration.getStream().getSupportedSyncModes().contains(SyncMode.FULL_REFRESH); + final var destinationSupportsOverwrite = supportedDestinationSyncModes.contains(DestinationSyncMode.OVERWRITE); + if (sourceSupportsFullRefresh && destinationSupportsOverwrite) { + // We prefer to fall back to Full Refresh | Overwrite if possible. + streamAndConfiguration.getConfig().syncMode(SyncMode.FULL_REFRESH).destinationSyncMode(DestinationSyncMode.OVERWRITE); + } else { + // If *that* isn't possible, we pick something that *is* supported. This isn't ideal, but we don't + // have a clean way + // to fail in this case today. + final var supportedSyncMode = streamAndConfiguration.getStream().getSupportedSyncModes().get(0); + final var supportedDestinationSyncMode = supportedDestinationSyncModes.get(0); + LOGGER.warn("Default sync modes are incompatible, so falling back to {} | {}", supportedSyncMode, supportedDestinationSyncMode); + streamAndConfiguration.getConfig().syncMode(supportedSyncMode).destinationSyncMode(supportedDestinationSyncMode); + } + } + } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java index 32eec2e4326..4f288fd605d 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/converters/CatalogConverterTest.java @@ -20,6 +20,7 @@ import io.airbyte.commons.server.helpers.ConnectionHelpers; import io.airbyte.config.DataType; import io.airbyte.config.FieldSelectionData; +import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.validation.json.JsonValidationException; import java.util.List; import org.junit.jupiter.api.Test; @@ -127,4 +128,48 @@ void testConvertToProtocolFieldSelection() throws JsonValidationException { assertEquals(ConnectionHelpers.generateBasicConfiguredAirbyteCatalog(), CatalogConverter.toConfiguredProtocol(catalog)); } + @Test + void testDiscoveredToApiDefaultSyncModesNoSourceCursor() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.FULL_REFRESH, actualSyncMode); + assertEquals(DestinationSyncMode.OVERWRITE, actualDestinationSyncMode); + } + + @Test + void testDiscoveredToApiDefaultSyncModesSourceCursorAndPrimaryKey() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + persistedCatalog.getStreams().get(0).withSourceDefinedCursor(true).withSourceDefinedPrimaryKey(List.of(List.of("unused"))); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.INCREMENTAL, actualSyncMode); + assertEquals(DestinationSyncMode.APPEND_DEDUP, actualDestinationSyncMode); + } + + @Test + void testDiscoveredToApiDefaultSyncModesSourceCursorNoPrimaryKey() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + persistedCatalog.getStreams().get(0).withSourceDefinedCursor(true); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.FULL_REFRESH, actualSyncMode); + assertEquals(DestinationSyncMode.OVERWRITE, actualDestinationSyncMode); + } + + @Test + void testDiscoveredToApiDefaultSyncModesSourceCursorNoFullRefresh() throws JsonValidationException { + final AirbyteCatalog persistedCatalog = CatalogConverter.toProtocol(ConnectionHelpers.generateBasicApiCatalog()); + persistedCatalog.getStreams().get(0).withSourceDefinedCursor(true) + .withSupportedSyncModes(List.of(io.airbyte.protocol.models.SyncMode.INCREMENTAL)); + final var actualStreamConfig = CatalogConverter.toApi(persistedCatalog, null).getStreams().get(0).getConfig(); + final var actualSyncMode = actualStreamConfig.getSyncMode(); + final var actualDestinationSyncMode = actualStreamConfig.getDestinationSyncMode(); + assertEquals(SyncMode.INCREMENTAL, actualSyncMode); + assertEquals(DestinationSyncMode.APPEND, actualDestinationSyncMode); + } + } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java index 95a753e4eca..5bece60a777 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java @@ -120,6 +120,7 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.StreamDescriptor; @@ -214,6 +215,12 @@ class SchedulerHandlerTest { public static final String A_DIFFERENT_STREAM = "aDifferentStream"; private static final ResourceRequirements RESOURCE_REQUIREMENT = new ResourceRequirements().withCpuLimit("1.0").withCpuRequest("0.5"); + private static final StandardDestinationDefinition SOME_DESTINATION_DEFINITION = new StandardDestinationDefinition() + .withDestinationDefinitionId(UUID.randomUUID()); + private static final ActorDefinitionVersion SOME_ACTOR_DEFINITION = new ActorDefinitionVersion().withSpec( + new ConnectorSpecification() + .withSupportedDestinationSyncModes(List.of(DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND, DestinationSyncMode.APPEND_DEDUP)) + .withDocumentationUrl(URI.create("unused"))); private SchedulerHandler schedulerHandler; private ConfigRepository configRepository; @@ -257,6 +264,8 @@ void setup() throws JsonValidationException, ConfigNotFoundException, IOExceptio synchronousSchedulerClient = mock(SynchronousSchedulerClient.class); configRepository = mock(ConfigRepository.class); when(configRepository.getStandardSync(any())).thenReturn(new StandardSync().withStatus(StandardSync.Status.ACTIVE)); + when(configRepository.getStandardDestinationDefinition(any())).thenReturn(SOME_DESTINATION_DEFINITION); + when(configRepository.getDestinationDefinitionFromConnection(any())).thenReturn(SOME_DESTINATION_DEFINITION); secretsRepositoryWriter = mock(SecretsRepositoryWriter.class); jobPersistence = mock(JobPersistence.class); eventRunner = mock(EventRunner.class); @@ -264,6 +273,7 @@ void setup() throws JsonValidationException, ConfigNotFoundException, IOExceptio envVariableFeatureFlags = mock(EnvVariableFeatureFlags.class); webUrlHelper = mock(WebUrlHelper.class); actorDefinitionVersionHelper = mock(ActorDefinitionVersionHelper.class); + when(actorDefinitionVersionHelper.getDestinationVersion(any(), any())).thenReturn(SOME_ACTOR_DEFINITION); streamResetPersistence = mock(StreamResetPersistence.class); oAuthConfigSupplier = mock(OAuthConfigSupplier.class); jobCreator = mock(JobCreator.class); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java index aa578485661..cf684324624 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/AutoPropagateSchemaChangeHelperTest.java @@ -49,6 +49,8 @@ class AutoPropagateSchemaChangeHelperTest { } """; + private static final List SUPPORTED_DESTINATION_SYNC_MODES = List.of( + DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND, DestinationSyncMode.APPEND_DEDUP); private FeatureFlagClient featureFlagClient; @BeforeEach @@ -106,7 +108,8 @@ void applyUpdate() { .transformType(StreamTransform.TransformTypeEnum.UPDATE_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(1); @@ -126,7 +129,8 @@ void applyAddNoFlag() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(2); @@ -151,7 +155,8 @@ void applyAdd() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(2); @@ -182,7 +187,8 @@ void applyAddWithSourceDefinedCursor() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(2); @@ -213,7 +219,8 @@ void applyAddWithSourceDefinedCursorNoPrimaryKey() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(2); @@ -237,7 +244,8 @@ void applyAddWithSourceDefinedCursorNoPrimaryKeyNoFullRefresh() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(2); @@ -258,7 +266,8 @@ void applyRemove() { .transformType(StreamTransform.TransformTypeEnum.REMOVE_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_FULLY, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(0); @@ -277,7 +286,8 @@ void applyAddNotFully() { .transformType(StreamTransform.TransformTypeEnum.ADD_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(1); @@ -297,7 +307,8 @@ void applyRemoveNotFully() { .transformType(StreamTransform.TransformTypeEnum.REMOVE_STREAM); final AirbyteCatalog result = - getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, featureFlagClient, + getUpdatedSchema(oldAirbyteCatalog, newAirbyteCatalog, List.of(transform), NonBreakingChangesPreference.PROPAGATE_COLUMNS, + SUPPORTED_DESTINATION_SYNC_MODES, featureFlagClient, UUID.randomUUID()); Assertions.assertThat(result.getStreams()).hasSize(1); diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index 35a661b0bea..588b0632ddd 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -387,7 +387,7 @@ void testDiscoverSourceSchema() throws ApiException { final AirbyteStreamConfiguration expectedStreamConfig = new AirbyteStreamConfiguration() .syncMode(SyncMode.FULL_REFRESH) .cursorField(Collections.emptyList()) - .destinationSyncMode(DestinationSyncMode.APPEND) + .destinationSyncMode(DestinationSyncMode.OVERWRITE) .primaryKey(Collections.emptyList()) .aliasName(STREAM_NAME.replace(".", "_")) .selected(true) diff --git a/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts b/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts index ac2bbcbb1fb..042ef13070a 100644 --- a/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts +++ b/airbyte-webapp/cypress/e2e/connection/syncModes.cy.ts @@ -57,28 +57,27 @@ const modifyAccountsTableInterceptHandler: RouteHandler = (request) => { }; const saveConnectionAndAssertStreams = ( - ...expectedSyncModes: Array<{ namespace: string; name: string; config: Partial }> + expectedSyncMode: { namespace: string; name: string; config: Partial }, + { expectModal = true }: { expectModal?: boolean } | undefined = {} ) => { replicationPage - .saveChangesAndHandleResetModal({ interceptUpdateHandler: modifyAccountsTableInterceptHandler }) + .saveChangesAndHandleResetModal({ interceptUpdateHandler: modifyAccountsTableInterceptHandler, expectModal }) .then((connection) => { - expectedSyncModes.forEach((expected) => { - const stream = connection.syncCatalog.streams.find( - ({ stream }) => stream?.namespace === expected.namespace && stream.name === expected.name - ); - - expect(stream).to.exist; - expect(stream?.config).to.contain({ - syncMode: expected.config.syncMode, - destinationSyncMode: expected.config.destinationSyncMode, - }); - if (expected.config.cursorField) { - expect(stream?.config?.cursorField).to.eql(expected.config.cursorField); - } - if (expected.config.primaryKey) { - expect(stream?.config?.cursorField).to.eql(expected.config.cursorField); - } + const stream = connection.syncCatalog.streams.find( + ({ stream }) => stream?.namespace === expectedSyncMode.namespace && stream.name === expectedSyncMode.name + ); + + expect(stream).to.exist; + expect(stream?.config).to.contain({ + syncMode: expectedSyncMode.config.syncMode, + destinationSyncMode: expectedSyncMode.config.destinationSyncMode, }); + if (expectedSyncMode.config.cursorField) { + expect(stream?.config?.cursorField).to.eql(expectedSyncMode.config.cursorField); + } + if (expectedSyncMode.config.primaryKey) { + expect(stream?.config?.cursorField).to.eql(expectedSyncMode.config.cursorField); + } }); }; @@ -155,14 +154,17 @@ describe("Connection - sync modes", () => { streamDetails.close(); // Save - saveConnectionAndAssertStreams({ - namespace: "public", - name: "users", - config: { - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, + saveConnectionAndAssertStreams( + { + namespace: "public", + name: "users", + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, }, - }); + { expectModal: false } + ); // Confirm after save usersStreamRow.hasSelectedSyncMode(SyncMode.full_refresh, DestinationSyncMode.overwrite); From df7d0f58a9ae9faa85c2d7029b78cda1bc898146 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Fri, 25 Aug 2023 09:50:38 -0400 Subject: [PATCH 014/201] remove error log when booting without a remote registry url (#8534) --- .../java/io/airbyte/config/specs/RemoteDefinitionsProvider.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java index 9d9002a9fa1..713d1371d3b 100644 --- a/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java +++ b/airbyte-config/specs/src/main/java/io/airbyte/config/specs/RemoteDefinitionsProvider.java @@ -52,7 +52,6 @@ public class RemoteDefinitionsProvider implements DefinitionsProvider { private URI parsedRemoteRegistryBaseUrlOrDefault(final String remoteRegistryBaseUrl) { try { if (remoteRegistryBaseUrl == null || remoteRegistryBaseUrl.isEmpty()) { - LOGGER.error("Remote connector registry base URL cannot be null or empty. Falling back to default"); return new URI(AirbyteCatalogConstants.REMOTE_REGISTRY_BASE_URL); } else { return new URI(remoteRegistryBaseUrl); From ddafb275192c305789d21b94d941436540025405 Mon Sep 17 00:00:00 2001 From: Conor Date: Fri, 25 Aug 2023 11:42:25 -0500 Subject: [PATCH 015/201] add check command to test task (#8521) --- .../airbyte/workers/process/DockerProcessFactoryTest.java | 1 - .../io/airbyte/config/persistence/WorkspacePersistence.java | 6 ++++-- ...1_001__BackfillActorDefaultVersionAndSetNonNullTest.java | 1 + ci/oss/tasks.py | 3 +-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java index 7e42eda6a7e..daaa3478b09 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/process/DockerProcessFactoryTest.java @@ -123,7 +123,6 @@ void testEnvMapSet() throws IOException, WorkerException, InterruptedException { final WorkerConfigs workerConfigs = spy(new WorkerConfigs(new EnvConfigs())); when(workerConfigs.getEnvMap()).thenReturn(Map.of("ENV_VAR_1", "ENV_VALUE_1")); - when(workerConfigs.getEnvMap()).thenReturn(Map.of("ENV_VAR_1", "ENV_VALUE_1")); final DockerProcessFactory processFactory = new DockerProcessFactory( diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java index 0493db51289..836bbfda4f1 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java @@ -48,9 +48,11 @@ public List listWorkspacesByOrganizationId(final ResourcesByO .toList(); } + /** + * Find the workspace with the given ID and check if its organization ID is null. If so, update it. + * Otherwise, log a warning and do nothing. + */ public void setOrganizationIdIfNull(final UUID workspaceId, final UUID organizationId) throws IOException { - // find the workspace with the given ID and check if its organization ID is null. If so, update it. - // otherwise, log a warning and do nothing. database.transaction(ctx -> { final boolean isExistingWorkspace = ctx.fetchExists(ctx.selectFrom(WORKSPACE).where(WORKSPACE.ID.eq(workspaceId))); if (isExistingWorkspace) { diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java index 91b58f31561..b151845c13f 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_21_001__BackfillActorDefaultVersionAndSetNonNullTest.java @@ -69,6 +69,7 @@ private UUID getDefaultVersionIdForActorId(final DSLContext ctx, final UUID acto return actor.get(DEFAULT_VERSION_ID_COL); } + @SuppressWarnings("PMD.AvoidDuplicateLiterals") static void insertDependencies(final DSLContext ctx) { ctx.insertInto(WORKSPACE) .columns( diff --git a/ci/oss/tasks.py b/ci/oss/tasks.py index b0551ac903d..e2ec8f2fbfc 100644 --- a/ci/oss/tasks.py +++ b/ci/oss/tasks.py @@ -48,7 +48,6 @@ async def build_oss_backend_task(settings: OssSettings, ctx: PipelineContext, cl .with_env_variable("VERSION", "dev") .with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte") .with_exec(["./gradlew", ":airbyte-config:specs:downloadConnectorRegistry", "--rerun", "--build-cache", "--no-daemon"]) - .with_exec(["./gradlew", "spotlessCheck"]) .with_exec(gradle_command + ["--scan"] if scan else gradle_command) .with_exec(["rsync", "-az", "/root/.gradle/", "/root/gradle-cache"]) #TODO: Move this to a context manager ) @@ -176,7 +175,7 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set .with_exec(["./run.sh"]) ) - gradle_command = ["./gradlew", "test", "-x", ":airbyte-webapp:test", "-x", "buildDockerImage", "--build-cache", "--no-daemon"] + gradle_command = ["./gradlew", "checK", "test", "-x", ":airbyte-webapp:test", "-x", "buildDockerImage", "--build-cache", "--no-daemon"] result = ( with_gradle(client, ctx, settings, directory=base_dir) From d9034c9dbd06417edf32cac182ada452197f7f34 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Fri, 25 Aug 2023 09:56:58 -0700 Subject: [PATCH 016/201] Add failure type to replication traces (#8536) --- .../java/io/airbyte/commons/temporal/CancellationHandler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java index 25d4c18c285..c79086d7511 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/CancellationHandler.java @@ -4,6 +4,7 @@ package io.airbyte.commons.temporal; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.FAILURE_TYPES_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.TEMPORAL_ACTIVITY_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.TEMPORAL_WORKFLOW_ID_KEY; @@ -67,10 +68,12 @@ public void checkAndHandleCancellation(final Runnable onCancellationCallback) { activityContext.heartbeat(null); } catch (final ActivityCanceledException e) { ApmTraceUtils.addExceptionToTrace(e); + ApmTraceUtils.addTagsToTrace(Map.of(FAILURE_TYPES_KEY, e.getClass().getName())); onCancellationCallback.run(); LOGGER.warn("Job was cancelled.", e); } catch (final ActivityCompletionException e) { ApmTraceUtils.addExceptionToTrace(e); + ApmTraceUtils.addTagsToTrace(Map.of(FAILURE_TYPES_KEY, e.getClass().getName())); // TODO: This is a hack to avoid having to manually destroy pod, it should be revisited if (!e.getWorkflowId().orElse("").toLowerCase().startsWith("sync")) { LOGGER.warn("The job timeout and was not a sync, we will destroy the pods related to it", e); From 6ec4a7345a20b256cd1ae8570bce974c6bec82ea Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Fri, 25 Aug 2023 10:02:21 -0700 Subject: [PATCH 017/201] Exclude tombstoned workspaces from DelinquentBillingCron (#7400) --- .../src/main/openapi/cloud-config.yaml | 4 ++-- .../config/persistence/ConfigRepository.java | 7 ++++-- .../persistence/WorkspaceFilterTest.java | 22 +++++++++---------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/airbyte-api/src/main/openapi/cloud-config.yaml b/airbyte-api/src/main/openapi/cloud-config.yaml index a65d21bfac1..05e78e29874 100644 --- a/airbyte-api/src/main/openapi/cloud-config.yaml +++ b/airbyte-api/src/main/openapi/cloud-config.yaml @@ -239,9 +239,9 @@ paths: application/json: schema: $ref: "#/components/schemas/CloudWorkspaceReadList" - /v1/cloud_workspaces/list_workspaces_by_most_recently_running_jobs: + /v1/cloud_workspaces/list_active_workspaces_by_most_recently_running_jobs: post: - operationId: listWorkspacesByMostRecentlyRunningJobs + operationId: listActiveWorkspacesByMostRecentlyRunningJobs requestBody: content: application/json: diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index c4f052c5524..ad23b7961ae 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -1563,20 +1563,23 @@ public List listWorkspacesDestinationConnections(final Re } /** - * List workspace IDs with most recently running jobs within a given time window (in hours). + * List active workspace IDs with most recently running jobs within a given time window (in hours). * * @param timeWindowInHours - integer, e.g. 24, 48, etc * @return list of workspace IDs * @throws IOException - failed to query data */ - public List listWorkspacesByMostRecentlyRunningJobs(final int timeWindowInHours) throws IOException { + public List listActiveWorkspacesByMostRecentlyRunningJobs(final int timeWindowInHours) throws IOException { final Result> records = database.query(ctx -> ctx.selectDistinct(ACTOR.WORKSPACE_ID) .from(ACTOR) + .join(WORKSPACE) + .on(ACTOR.WORKSPACE_ID.eq(WORKSPACE.ID)) .join(CONNECTION) .on(CONNECTION.SOURCE_ID.eq(ACTOR.ID)) .join(JOBS) .on(CONNECTION.ID.cast(VARCHAR(255)).eq(JOBS.SCOPE)) .where(JOBS.UPDATED_AT.greaterOrEqual(OffsetDateTime.now().minusHours(timeWindowInHours))) + .and(WORKSPACE.TOMBSTONE.isFalse()) .fetch()); return records.stream().map(record -> record.get(ACTOR.WORKSPACE_ID)).collect(Collectors.toList()); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java index f618808987b..2172d6cce88 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java @@ -64,12 +64,13 @@ static void setUpAll() throws SQLException { .execute()); // create workspace - database.transaction(ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.SLUG, WORKSPACE.INITIAL_SETUP_COMPLETE) - .values(WORKSPACE_ID_0, "ws-0", "ws-0", true) - .values(WORKSPACE_ID_1, "ws-1", "ws-1", true) - .values(WORKSPACE_ID_2, "ws-2", "ws-2", true) - .values(WORKSPACE_ID_3, "ws-3", "ws-3", true) - .execute()); + database.transaction( + ctx -> ctx.insertInto(WORKSPACE, WORKSPACE.ID, WORKSPACE.NAME, WORKSPACE.SLUG, WORKSPACE.INITIAL_SETUP_COMPLETE, WORKSPACE.TOMBSTONE) + .values(WORKSPACE_ID_0, "ws-0", "ws-0", true, false) + .values(WORKSPACE_ID_1, "ws-1", "ws-1", true, false) + .values(WORKSPACE_ID_2, "ws-2", "ws-2", true, true) // note that workspace 2 is tombstoned! + .values(WORKSPACE_ID_3, "ws-3", "ws-3", true, true) // note that workspace 3 is tombstoned! + .execute()); // create actors database.transaction( ctx -> ctx @@ -112,8 +113,8 @@ void beforeEach() { } @Test - @DisplayName("Should return a list of workspace IDs with most recently running jobs") - void testListWorkspacesByMostRecentlyRunningJobs() throws IOException { + @DisplayName("Should return a list of active workspace IDs with most recently running jobs") + void testListActiveWorkspacesByMostRecentlyRunningJobs() throws IOException { final int timeWindowInHours = 48; /* * Following function is to filter workspace (IDs) with most recently running jobs within a given @@ -121,17 +122,16 @@ void testListWorkspacesByMostRecentlyRunningJobs() throws IOException { * time window. Step 2: Trace back via CONNECTION table and ACTOR table. Step 3: Return workspace * IDs from ACTOR table. */ - final List actualResult = configRepository.listWorkspacesByMostRecentlyRunningJobs(timeWindowInHours); + final List actualResult = configRepository.listActiveWorkspacesByMostRecentlyRunningJobs(timeWindowInHours); /* * With the test data provided above, expected outputs for each step: Step 1: `jobs` (IDs) OL, 1L, * 2L, 3L, 4L, 5L and 6L. Step 2: `connections` (IDs) CONN_ID_0, CONN_ID_1, CONN_ID_2, CONN_ID_3, * and CONN_ID_4 `actors` (IDs) ACTOR_ID_0, ACTOR_ID_1, and ACTOR_ID_2. Step 3: `workspaces` (IDs) - * WORKSPACE_ID_0, WORKSPACE_ID_1 and WORKSPACE_ID_2. + * WORKSPACE_ID_0, WORKSPACE_ID_1. Note that WORKSPACE_ID_2 is excluded because it is tombstoned. */ final List expectedResult = new ArrayList<>(); expectedResult.add(WORKSPACE_ID_0); expectedResult.add(WORKSPACE_ID_1); - expectedResult.add(WORKSPACE_ID_2); assertTrue(expectedResult.size() == actualResult.size() && expectedResult.containsAll(actualResult) && actualResult.containsAll(expectedResult)); } From 734900f4e5cd6594d5ec2545c35df77b208916f2 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Fri, 25 Aug 2023 20:52:33 +0200 Subject: [PATCH 018/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Refactor?= =?UTF-8?q?=20connections=20to=20new=20API=20service=20(#8505)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.eslintLegacyFolderStructure.js | 1 - .../connection/utils/useStreamsStatuses.ts | 5 +- .../components/StatusCell.test.tsx | 4 +- .../components/StatusCellControl.tsx | 4 +- .../connection/ConnectionForm/formConfig.tsx | 23 +- .../useConnectionStatus.test.ts | 7 +- .../ConnectionStatus/useConnectionStatus.ts | 5 +- .../ConnectionSync/ConnectionSyncContext.tsx | 12 +- .../ExistingConnectorButton.tsx | 4 +- .../CreateConnection/SelectDestination.tsx | 2 +- .../CreateConnectionForm.test.tsx | 1 + .../CreateConnectionForm.tsx | 2 +- .../connector/BreakingChangeBanner.tsx | 5 +- .../DestinationsTable/DestinationsTable.tsx | 4 +- .../cloud/usePrefetchCloudWorkspaceData.ts | 2 +- .../api/hooks/connections.ts} | 231 ++++++++++-------- airbyte-webapp/src/core/api/hooks/index.ts | 7 +- .../domain/connection/ConnectionService.ts | 42 ---- .../connection/WebBackendConnectionService.ts | 28 --- .../src/core/domain/connection/index.ts | 1 - .../ConnectionEditService.test.tsx | 31 +-- .../ConnectionEdit/ConnectionEditService.tsx | 8 +- .../ConnectionForm/ConnectionFormService.tsx | 6 +- .../src/hooks/services/useDestinationHook.tsx | 3 +- .../src/hooks/services/useSourceHook.tsx | 3 +- .../AllConnectionsPage/AllConnectionsPage.tsx | 2 +- .../ConnectionReplicationPage.test.tsx | 27 +- .../ConnectionReplicationPage.tsx | 10 +- .../ConnectionSettingsPage.tsx | 2 +- .../ConnectionSettingsPage/StateBlock.tsx | 2 +- .../DestinationConnectionsPage.tsx | 3 +- .../DestinationSettingsPage.tsx | 2 +- .../source/AllSourcesPage/SourcesTable.tsx | 2 +- .../SourceConnectionsPage.tsx | 3 +- .../SourceSettingsPage/SourceSettingsPage.tsx | 2 +- 35 files changed, 220 insertions(+), 276 deletions(-) rename airbyte-webapp/src/{hooks/services/useConnectionHook.tsx => core/api/hooks/connections.ts} (64%) delete mode 100644 airbyte-webapp/src/core/domain/connection/ConnectionService.ts delete mode 100644 airbyte-webapp/src/core/domain/connection/WebBackendConnectionService.ts diff --git a/airbyte-webapp/.eslintLegacyFolderStructure.js b/airbyte-webapp/.eslintLegacyFolderStructure.js index f1cd97bad69..bfae59d6512 100644 --- a/airbyte-webapp/.eslintLegacyFolderStructure.js +++ b/airbyte-webapp/.eslintLegacyFolderStructure.js @@ -43,7 +43,6 @@ module.exports = [ "src/hooks/services/Modal/types.ts", "src/hooks/services/Modal/ModalService.test.tsx", "src/hooks/services/Modal/index.ts", - "src/hooks/services/useConnectionHook.tsx", "src/hooks/services/useRequestErrorHandler.tsx", "src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx", "src/hooks/services/ConnectionForm/ConnectionFormService.tsx", diff --git a/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts b/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts index 99529c76e90..9e7db41cd1a 100644 --- a/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts +++ b/airbyte-webapp/src/area/connection/utils/useStreamsStatuses.ts @@ -8,11 +8,10 @@ import { useLateMultiplierExperiment, } from "components/connection/StreamStatus/streamStatusUtils"; -import { useListStreamsStatuses } from "core/api"; -import { StreamStatusRead } from "core/request/AirbyteClient"; +import { useListStreamsStatuses, useGetConnection } from "core/api"; +import { StreamStatusRead } from "core/api/types/AirbyteClient"; import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; import { useExperiment } from "hooks/services/Experiment"; -import { useGetConnection } from "hooks/services/useConnectionHook"; import { AirbyteStreamAndConfigurationWithEnforcedStream, diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx index e865363ea6e..4cc164de370 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCell.test.tsx @@ -4,7 +4,7 @@ import { TestWrapper, TestSuspenseBoundary, mockConnection } from "test-utils"; import { StatusCell } from "./StatusCell"; -jest.mock("hooks/services/useConnectionHook", () => ({ +jest.mock("core/api", () => ({ useConnectionList: jest.fn(() => ({ connections: [], })), @@ -18,7 +18,7 @@ jest.mock("hooks/services/useConnectionHook", () => ({ const mockId = "mock-id"; -jest.doMock("hooks/services/useConnectionHook", () => ({ +jest.doMock("core/api", () => ({ useEnableConnection: () => ({ mutateAsync: jest.fn(), isLoading: false, diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx index fbda0dc04f0..df64c1b8b1e 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCellControl.tsx @@ -4,8 +4,8 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { Switch } from "components/ui/Switch"; -import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; -import { useEnableConnection, useSyncConnection } from "hooks/services/useConnectionHook"; +import { useEnableConnection, useSyncConnection } from "core/api"; +import { WebBackendConnectionListItem } from "core/api/types/AirbyteClient"; import styles from "./StatusCellControl.module.scss"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx index a902d84e9c6..f8b8f87abd0 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx @@ -5,15 +5,7 @@ import * as yup from "yup"; import { DropDownOptionDataItem } from "components/ui/DropDown"; import { validateCronExpression, validateCronFrequencyOneHourOrMore } from "area/connection/utils"; -import { useCurrentWorkspace } from "core/api"; -import { SyncSchema } from "core/domain/catalog"; -import { - isDbtTransformation, - isNormalizationTransformation, - isWebhookTransformation, - NormalizationType, -} from "core/domain/connection/operation"; -import { SOURCE_NAMESPACE_TAG } from "core/domain/connector/source"; +import { ConnectionValues, useCurrentWorkspace } from "core/api"; import { ConnectionScheduleData, ConnectionScheduleType, @@ -29,11 +21,18 @@ import { SchemaChange, SyncMode, WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; +import { SyncSchema } from "core/domain/catalog"; +import { + isDbtTransformation, + isNormalizationTransformation, + isWebhookTransformation, + NormalizationType, +} from "core/domain/connection/operation"; +import { SOURCE_NAMESPACE_TAG } from "core/domain/connector/source"; import { FeatureItem, useFeature } from "core/services/features"; import { ConnectionFormMode, ConnectionOrPartialConnection } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperiment } from "hooks/services/Experiment"; -import { ValuesProps } from "hooks/services/useConnectionHook"; import calculateInitialCatalog from "./calculateInitialCatalog"; import { frequencyConfig } from "./frequencyConfig"; @@ -53,7 +52,7 @@ export interface FormikConnectionFormValues { geography: Geography; } -export type ConnectionFormValues = ValuesProps; +export type ConnectionFormValues = ConnectionValues; export const SUPPORTED_MODES: Array<[SyncMode, DestinationSyncMode]> = [ [SyncMode.incremental, DestinationSyncMode.append_dedup], diff --git a/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.test.ts b/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.test.ts index 1c1d73d91b8..3c62348fcd2 100644 --- a/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.test.ts +++ b/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.test.ts @@ -5,7 +5,7 @@ import { mockConnection } from "test-utils"; import { mockAttempt } from "test-utils/mock-data/mockAttempt"; import { mockJob } from "test-utils/mock-data/mockJob"; -import { useListJobsForConnectionStatus } from "core/api"; +import { useListJobsForConnectionStatus, useGetConnection } from "core/api"; import { ConnectionScheduleDataBasicSchedule, ConnectionScheduleDataCron, @@ -17,9 +17,8 @@ import { JobWithAttemptsRead, SchemaChange, WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; -import { useGetConnection } from "hooks/services/useConnectionHook"; import { isConnectionLate, isHandleableScheduledConnection, useConnectionStatus } from "./useConnectionStatus"; import { ConnectionStatusIndicatorStatus } from "../ConnectionStatusIndicator"; @@ -39,7 +38,7 @@ jest.mock("components/connection/StreamStatus/streamStatusUtils", () => ({ }, })); -jest.mock("hooks/services/useConnectionHook"); +jest.mock("core/api"); const mockUseGetConnection = useGetConnection as unknown as jest.Mock; jest.mock("core/api"); diff --git a/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.ts b/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.ts index fc5f1a370cc..5445f65fc20 100644 --- a/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.ts +++ b/airbyte-webapp/src/components/connection/ConnectionStatus/useConnectionStatus.ts @@ -5,7 +5,7 @@ import { useLateMultiplierExperiment, } from "components/connection/StreamStatus/streamStatusUtils"; -import { useListJobsForConnectionStatus } from "core/api"; +import { useListJobsForConnectionStatus, useGetConnection } from "core/api"; import { ConnectionScheduleType, ConnectionStatus, @@ -14,10 +14,9 @@ import { JobStatus, JobWithAttemptsRead, WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { moveTimeToFutureByPeriod } from "core/utils/time"; import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; -import { useGetConnection } from "hooks/services/useConnectionHook"; import { ConnectionStatusIndicatorStatus } from "../ConnectionStatusIndicator"; import { jobStatusesIndicatingFinishedExecution } from "../ConnectionSync/ConnectionSyncContext"; diff --git a/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncContext.tsx b/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncContext.tsx index 1712379d31b..81e0e46b6e4 100644 --- a/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncContext.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncContext.tsx @@ -1,6 +1,13 @@ import { createContext, useCallback, useContext, useMemo } from "react"; -import { useCancelJob, useListJobsForConnectionStatus, useSetConnectionJobsData } from "core/api"; +import { + useResetConnection, + useResetConnectionStream, + useSyncConnection, + useCancelJob, + useListJobsForConnectionStatus, + useSetConnectionJobsData, +} from "core/api"; import { ConnectionStatus, ConnectionStream, @@ -9,10 +16,9 @@ import { JobStatus, JobInfoRead, WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useExperiment } from "hooks/services/Experiment"; -import { useResetConnection, useResetConnectionStream, useSyncConnection } from "hooks/services/useConnectionHook"; interface ConnectionSyncContext { syncConnection: () => Promise; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/ExistingConnectorButton.tsx b/airbyte-webapp/src/components/connection/CreateConnection/ExistingConnectorButton.tsx index a98bbbeeb4d..3ad0df2e3ab 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/ExistingConnectorButton.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/ExistingConnectorButton.tsx @@ -4,9 +4,9 @@ import { ConnectorDefinitionBranding } from "components/ui/ConnectorDefinitionBr import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; +import { useConnectionList } from "core/api"; +import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient"; import { isSource } from "core/domain/connector/source"; -import { DestinationRead, SourceRead } from "core/request/AirbyteClient"; -import { useConnectionList } from "hooks/services/useConnectionHook"; import styles from "./ExistingConnectorButton.module.scss"; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx b/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx index b58a16a6990..a5a4130b965 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx @@ -9,7 +9,7 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; -import { useConnectionList } from "hooks/services/useConnectionHook"; +import { useConnectionList } from "core/api"; import { useDestinationList } from "hooks/services/useDestinationHook"; import { CreateNewDestination, DESTINATION_DEFINITION_PARAM } from "./CreateNewDestination"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx index 12cf6e70410..8170bba6a62 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx @@ -41,6 +41,7 @@ jest.mock("area/workspace/utils", () => ({ jest.mock("core/api", () => ({ useCurrentWorkspace: () => ({}), useInvalidateWorkspaceStateQuery: () => () => null, + useCreateConnection: () => async () => null, })); jest.mock("hooks/domain/connector/useGetSourceFromParams", () => ({ diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx index 97257e1fa84..f0f6ccf4160 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx @@ -12,6 +12,7 @@ import { OperationsSection } from "components/connection/ConnectionForm/Operatio import LoadingSchema from "components/LoadingSchema"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { useCreateConnection } from "core/api"; import { FeatureItem, useFeature } from "core/services/features"; import { useGetDestinationFromSearchParams } from "hooks/domain/connector/useGetDestinationFromParams"; import { useGetSourceFromSearchParams } from "hooks/domain/connector/useGetSourceFromParams"; @@ -22,7 +23,6 @@ import { } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperimentContext } from "hooks/services/Experiment"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; -import { useCreateConnection } from "hooks/services/useConnectionHook"; import { SchemaError as SchemaErrorType, useDiscoverSchema } from "hooks/services/useSourceHook"; import styles from "./CreateConnectionForm.module.scss"; diff --git a/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx b/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx index f33b0014941..aae49579d4f 100644 --- a/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx +++ b/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx @@ -8,13 +8,12 @@ import { Message } from "components/ui/Message"; import { Text } from "components/ui/Text"; import { TextWithHTML } from "components/ui/TextWithHTML"; -import { useUpgradeConnectorVersion } from "core/api"; +import { useConnectionList, useUpgradeConnectorVersion } from "core/api"; +import { ActorDefinitionVersionRead } from "core/api/types/AirbyteClient"; import { getHumanReadableUpgradeDeadline } from "core/domain/connector"; -import { ActorDefinitionVersionRead } from "core/request/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useNotificationService } from "hooks/services/Notification"; -import { useConnectionList } from "hooks/services/useConnectionHook"; import { SourcePaths } from "pages/routePaths"; import styles from "./BreakingChangeBanner.module.scss"; diff --git a/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx b/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx index 5262ef66a29..f0501f001d5 100644 --- a/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx +++ b/airbyte-webapp/src/components/destination/DestinationsTable/DestinationsTable.tsx @@ -5,8 +5,8 @@ import { ImplementationTable } from "components/EntityTable"; import { EntityTableDataItem } from "components/EntityTable/types"; import { getEntityTableData } from "components/EntityTable/utils"; -import { DestinationRead } from "core/request/AirbyteClient"; -import { useConnectionList } from "hooks/services/useConnectionHook"; +import { useConnectionList } from "core/api"; +import { DestinationRead } from "core/api/types/AirbyteClient"; interface DestinationsTableProps { destinations: DestinationRead[]; diff --git a/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts b/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts index 2ab03da2a9f..abd358d2b7d 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/usePrefetchCloudWorkspaceData.ts @@ -1,8 +1,8 @@ import { useQueries } from "@tanstack/react-query"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { getConnectionListQueryKey, useConnectionListQuery } from "core/api"; import { useCurrentUser } from "core/services/auth"; -import { getConnectionListQueryKey, useConnectionListQuery } from "hooks/services/useConnectionHook"; import { getListCloudWorkspacesAsyncQueryKey, useListCloudWorkspacesAsyncQuery } from "./cloudWorkspaces"; import { getListUsersQueryKey, useListUsersQuery } from "./users"; diff --git a/airbyte-webapp/src/hooks/services/useConnectionHook.tsx b/airbyte-webapp/src/core/api/hooks/connections.ts similarity index 64% rename from airbyte-webapp/src/hooks/services/useConnectionHook.tsx rename to airbyte-webapp/src/core/api/hooks/connections.ts index 00a3f899de7..e2de4dabe05 100644 --- a/airbyte-webapp/src/hooks/services/useConnectionHook.tsx +++ b/airbyte-webapp/src/core/api/hooks/connections.ts @@ -1,21 +1,31 @@ import { useIsMutating, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useCallback } from "react"; -import { FormattedMessage, useIntl } from "react-intl"; +import { useIntl } from "react-intl"; import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { useInvalidateWorkspaceStateQuery, useSuspenseQuery } from "core/api"; import { SyncSchema } from "core/domain/catalog"; -import { WebBackendConnectionService } from "core/domain/connection"; -import { ConnectionService } from "core/domain/connection/ConnectionService"; import { getFrequencyFromScheduleData } from "core/services/analytics"; import { Action, Namespace } from "core/services/analytics"; import { useAnalyticsService } from "core/services/analytics"; -import { useInitService } from "services/useInitService"; -import { useAppMonitoringService } from "./AppMonitoringService"; -import { useNotificationService } from "./Notification"; -import { useCurrentWorkspace } from "./useWorkspace"; -import { useConfig } from "../../config"; +import { useInvalidateWorkspaceStateQuery } from "./workspaces"; +import { useAppMonitoringService } from "../../../hooks/services/AppMonitoringService"; +import { useNotificationService } from "../../../hooks/services/Notification"; +import { useCurrentWorkspace } from "../../../hooks/services/useWorkspace"; +import { SCOPE_WORKSPACE } from "../../../services/Scope"; +import { + createOrUpdateStateSafe, + deleteConnection, + getState, + getStateType, + resetConnection, + resetConnectionStream, + syncConnection, + webBackendCreateConnection, + webBackendGetConnection, + webBackendListConnectionsForWorkspace, + webBackendUpdateConnection, +} from "../generated/AirbyteClient"; import { ConnectionScheduleData, ConnectionScheduleType, @@ -31,19 +41,20 @@ import { WebBackendConnectionListRequestBody, WebBackendConnectionRead, WebBackendConnectionReadList, + WebBackendConnectionRequestBody, WebBackendConnectionUpdate, -} from "../../core/request/AirbyteClient"; -import { SCOPE_WORKSPACE } from "../../services/Scope"; -import { useDefaultRequestMiddlewares } from "../../services/useDefaultRequestMiddlewares"; +} from "../types/AirbyteClient"; +import { useRequestOptions } from "../useRequestOptions"; +import { useSuspenseQuery } from "../useSuspenseQuery"; -export const connectionsKeys = { +const connectionsKeys = { all: [SCOPE_WORKSPACE, "connections"] as const, lists: (sourceOrDestinationIds: string[] = []) => [...connectionsKeys.all, "list", ...sourceOrDestinationIds], detail: (connectionId: string) => [...connectionsKeys.all, "details", connectionId] as const, getState: (connectionId: string) => [...connectionsKeys.all, "getState", connectionId] as const, }; -export interface ValuesProps { +export interface ConnectionValues { name?: string; scheduleData: ConnectionScheduleData | undefined; scheduleType: ConnectionScheduleType; @@ -55,7 +66,7 @@ export interface ValuesProps { } interface CreateConnectionProps { - values: ValuesProps; + values: ConnectionValues; source: SourceRead; destination: DestinationRead; sourceDefinition?: Pick; @@ -63,24 +74,8 @@ interface CreateConnectionProps { sourceCatalogId: string | undefined; } -export const useWebConnectionService = () => { - const config = useConfig(); - const middlewares = useDefaultRequestMiddlewares(); - return useInitService( - () => new WebBackendConnectionService(config.apiUrl, middlewares), - [config.apiUrl, middlewares] - ); -}; - -export function useConnectionService() { - const config = useConfig(); - const middlewares = useDefaultRequestMiddlewares(); - return useInitService(() => new ConnectionService(config.apiUrl, middlewares), [config.apiUrl, middlewares]); -} - export const useSyncConnection = () => { - const service = useConnectionService(); - const webConnectionService = useWebConnectionService(); + const requestOptions = useRequestOptions(); const { trackError } = useAppMonitoringService(); const queryClient = useQueryClient(); const analyticsService = useAnalyticsService(); @@ -99,7 +94,7 @@ export const useSyncConnection = () => { frequency: getFrequencyFromScheduleData(connection.scheduleData), }); - return service.sync(connection.connectionId); + return syncConnection({ connectionId: connection.connectionId }, requestOptions); }, { onError: (error: Error) => { @@ -111,36 +106,49 @@ export const useSyncConnection = () => { }); }, onSuccess: async () => { - await webConnectionService - .list({ workspaceId }) - .then((updatedConnections) => queryClient.setQueryData(connectionsKeys.lists(), updatedConnections)); + await webBackendListConnectionsForWorkspace({ workspaceId }, requestOptions).then((updatedConnections) => + queryClient.setQueryData(connectionsKeys.lists(), updatedConnections) + ); }, } ); }; export const useResetConnection = () => { - const service = useConnectionService(); - - const mutation = useMutation(["useResetConnection"], (connectionId: string) => service.reset(connectionId)); + const requestOptions = useRequestOptions(); + const mutation = useMutation(["useResetConnection"], (connectionId: string) => + resetConnection({ connectionId }, requestOptions) + ); const activeMutationsCount = useIsMutating(["useResetConnection"]); return { ...mutation, isLoading: activeMutationsCount > 0 }; }; export const useResetConnectionStream = (connectionId: string) => { - const service = useConnectionService(); + const requestOptions = useRequestOptions(); + return useMutation((streams: ConnectionStream[]) => resetConnectionStream({ connectionId, streams }, requestOptions)); +}; - return useMutation((streams: ConnectionStream[]) => service.resetStream(connectionId, streams)); +export const useGetConnectionQuery = () => { + const requestOptions = useRequestOptions(); + return useMutation((request: WebBackendConnectionRequestBody) => webBackendGetConnection(request, requestOptions)) + .mutateAsync; }; -const useGetConnection = (connectionId: string, options?: { refetchInterval: number }): WebBackendConnectionRead => { - const service = useWebConnectionService(); +export const useGetConnection = ( + connectionId: string, + options?: { refetchInterval: number } +): WebBackendConnectionRead => { + const getConnectionQuery = useGetConnectionQuery(); - return useSuspenseQuery(connectionsKeys.detail(connectionId), () => service.getConnection(connectionId), options); + return useSuspenseQuery( + connectionsKeys.detail(connectionId), + () => getConnectionQuery({ connectionId, withRefreshedCatalog: false }), + options + ); }; -const useCreateConnection = () => { - const service = useWebConnectionService(); +export const useCreateConnection = () => { + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const analyticsService = useAnalyticsService(); const invalidateWorkspaceSummary = useInvalidateWorkspaceStateQuery(); @@ -154,13 +162,16 @@ const useCreateConnection = () => { destinationDefinition, sourceCatalogId, }: CreateConnectionProps) => { - const response = await service.create({ - sourceId: source.sourceId, - destinationId: destination.destinationId, - ...values, - status: "active", - sourceCatalogId, - }); + const response = await webBackendCreateConnection( + { + sourceId: source.sourceId, + destinationId: destination.destinationId, + ...values, + status: "active", + sourceCatalogId, + }, + requestOptions + ); const enabledStreams = values.syncCatalog.streams .map((stream) => stream.config?.selected && stream.stream?.name) @@ -191,49 +202,56 @@ const useCreateConnection = () => { ); }; -const useDeleteConnection = () => { - const service = useConnectionService(); +export const useDeleteConnection = () => { + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const analyticsService = useAnalyticsService(); - return useMutation((connection: WebBackendConnectionRead) => service.delete(connection.connectionId), { - onSuccess: (_data, connection) => { - analyticsService.track(Namespace.CONNECTION, Action.DELETE, { - actionDescription: "Connection deleted", - connector_source: connection.source?.sourceName, - connector_source_definition_id: connection.source?.sourceDefinitionId, - connector_destination: connection.destination?.destinationName, - connector_destination_definition_id: connection.destination?.destinationDefinitionId, - }); + return useMutation( + (connection: WebBackendConnectionRead) => + deleteConnection({ connectionId: connection.connectionId }, requestOptions), + { + onSuccess: (_data, connection) => { + analyticsService.track(Namespace.CONNECTION, Action.DELETE, { + actionDescription: "Connection deleted", + connector_source: connection.source?.sourceName, + connector_source_definition_id: connection.source?.sourceDefinitionId, + connector_destination: connection.destination?.destinationName, + connector_destination_definition_id: connection.destination?.destinationDefinitionId, + }); - queryClient.removeQueries(connectionsKeys.detail(connection.connectionId)); - queryClient.setQueryData(connectionsKeys.lists(), (lst) => ({ - connections: lst?.connections.filter((conn) => conn.connectionId !== connection.connectionId) ?? [], - })); - }, - }); + queryClient.removeQueries(connectionsKeys.detail(connection.connectionId)); + queryClient.setQueryData(connectionsKeys.lists(), (lst) => ({ + connections: lst?.connections.filter((conn) => conn.connectionId !== connection.connectionId) ?? [], + })); + }, + } + ); }; -const useUpdateConnection = () => { - const service = useWebConnectionService(); +export const useUpdateConnection = () => { + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); - return useMutation((connectionUpdate: WebBackendConnectionUpdate) => service.update(connectionUpdate), { - onSuccess: (updatedConnection) => { - queryClient.setQueryData(connectionsKeys.detail(updatedConnection.connectionId), updatedConnection); - // Update the connection inside the connections list response - queryClient.setQueryData(connectionsKeys.lists(), (ls) => ({ - ...ls, - connections: - ls?.connections.map((conn) => { - if (conn.connectionId === updatedConnection.connectionId) { - return updatedConnection; - } - return conn; - }) ?? [], - })); - }, - }); + return useMutation( + (connectionUpdate: WebBackendConnectionUpdate) => webBackendUpdateConnection(connectionUpdate, requestOptions), + { + onSuccess: (updatedConnection) => { + queryClient.setQueryData(connectionsKeys.detail(updatedConnection.connectionId), updatedConnection); + // Update the connection inside the connections list response + queryClient.setQueryData(connectionsKeys.lists(), (ls) => ({ + ...ls, + connections: + ls?.connections.map((conn) => { + if (conn.connectionId === updatedConnection.connectionId) { + return updatedConnection; + } + return conn; + }) ?? [], + })); + }, + } + ); }; /** @@ -298,10 +316,13 @@ export const useConnectionListQuery = ( sourceId?: string[], destinationId?: string[] ): (() => Promise) => { - const service = useWebConnectionService(); + const requestOptions = useRequestOptions(); return async () => { - const { connections } = await service.list({ workspaceId, sourceId, destinationId }); + const { connections } = await webBackendListConnectionsForWorkspace( + { workspaceId, sourceId, destinationId }, + requestOptions + ); const connectionsByConnectorId = new Map(); connections.forEach((connection) => { connectionsByConnectorId.set(connection.source.sourceId, [ @@ -325,7 +346,9 @@ interface ConnectionListTransformed { connectionsByConnectorId: Map; } -const useConnectionList = (payload: Pick = {}) => { +export const useConnectionList = ( + payload: Pick = {} +) => { const workspace = useCurrentWorkspace(); const REFETCH_CONNECTION_LIST_INTERVAL = 60_000; const connectorIds = [ @@ -341,14 +364,19 @@ const useConnectionList = (payload: Pick { - const service = useConnectionService(); +export const useGetConnectionState = (connectionId: string) => { + const requestOptions = useRequestOptions(); + return useSuspenseQuery(connectionsKeys.getState(connectionId), () => getState({ connectionId }, requestOptions)); +}; - return useSuspenseQuery(connectionsKeys.getState(connectionId), () => service.getState(connectionId)); +export const useGetStateTypeQuery = () => { + const requestOptions = useRequestOptions(); + return useMutation((connectionId: string) => getStateType({ connectionId }, requestOptions)).mutateAsync; }; export const useCreateOrUpdateState = () => { - const service = useConnectionService(); + const requestOptions = useRequestOptions(); + const { formatMessage } = useIntl(); const queryClient = useQueryClient(); const analyticsService = useAnalyticsService(); const { trackError } = useAppMonitoringService(); @@ -356,7 +384,7 @@ export const useCreateOrUpdateState = () => { return useMutation( ({ connectionId, connectionState }: ConnectionStateCreateOrUpdate) => - service.createOrUpdateState(connectionId, connectionState), + createOrUpdateStateSafe({ connectionId, connectionState }, requestOptions), { onSuccess: (updatedState) => { analyticsService.track(Namespace.CONNECTION, Action.CREATE_OR_UPDATE_STATE, { @@ -367,7 +395,7 @@ export const useCreateOrUpdateState = () => { queryClient.setQueryData(connectionsKeys.getState(updatedState.connectionId), updatedState); registerNotification({ id: `connection.stateUpdateSuccess.${updatedState.connectionId}`, - text: , + text: formatMessage({ id: "connection.state.success" }), type: "success", }); }, @@ -382,12 +410,3 @@ export const useCreateOrUpdateState = () => { } ); }; - -export { - useConnectionList, - useGetConnection, - useUpdateConnection, - useCreateConnection, - useDeleteConnection, - useGetConnectionState, -}; diff --git a/airbyte-webapp/src/core/api/hooks/index.ts b/airbyte-webapp/src/core/api/hooks/index.ts index 5daf8ac10f9..6c78ae2607e 100644 --- a/airbyte-webapp/src/core/api/hooks/index.ts +++ b/airbyte-webapp/src/core/api/hooks/index.ts @@ -1,4 +1,5 @@ export * from "./actorDefinitionVersions"; +export * from "./connections"; export * from "./connectorBuilderApi"; export * from "./connectorBuilderProject"; export * from "./geographies"; @@ -7,10 +8,10 @@ export * from "./jobs"; export * from "./logs"; export * from "./notifications"; export * from "./operations"; +export * from "./organizations"; export * from "./security"; export * from "./instanceConfiguration"; export * from "./streams"; -export * from "./workspaces"; -export * from "./organizations"; -export * from "./users"; export * from "./upgradeConnectorVersion"; +export * from "./users"; +export * from "./workspaces"; diff --git a/airbyte-webapp/src/core/domain/connection/ConnectionService.ts b/airbyte-webapp/src/core/domain/connection/ConnectionService.ts deleted file mode 100644 index 08af92f052d..00000000000 --- a/airbyte-webapp/src/core/domain/connection/ConnectionService.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { - deleteConnection, - resetConnection, - syncConnection, - getState, - getStateType, - ConnectionStream, - resetConnectionStream, - createOrUpdateStateSafe, - ConnectionState, -} from "../../request/AirbyteClient"; -import { AirbyteRequestService } from "../../request/AirbyteRequestService"; - -export class ConnectionService extends AirbyteRequestService { - public sync(connectionId: string) { - return syncConnection({ connectionId }, this.requestOptions); - } - - public reset(connectionId: string) { - return resetConnection({ connectionId }, this.requestOptions); - } - - public resetStream(connectionId: string, streams: ConnectionStream[]) { - return resetConnectionStream({ connectionId, streams }, this.requestOptions); - } - - public delete(connectionId: string) { - return deleteConnection({ connectionId }, this.requestOptions); - } - - public getState(connectionId: string) { - return getState({ connectionId }, this.requestOptions); - } - - public getStateType(connectionId: string) { - return getStateType({ connectionId }, this.requestOptions); - } - - public createOrUpdateState(connectionId: string, state: ConnectionState) { - return createOrUpdateStateSafe({ connectionId, connectionState: state }, this.requestOptions); - } -} diff --git a/airbyte-webapp/src/core/domain/connection/WebBackendConnectionService.ts b/airbyte-webapp/src/core/domain/connection/WebBackendConnectionService.ts deleted file mode 100644 index 4aa576a1160..00000000000 --- a/airbyte-webapp/src/core/domain/connection/WebBackendConnectionService.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { - WebBackendConnectionCreate, - WebBackendConnectionListRequestBody, - WebBackendConnectionUpdate, - webBackendCreateConnection, - webBackendGetConnection, - webBackendListConnectionsForWorkspace, - webBackendUpdateConnection, -} from "../../request/AirbyteClient"; -import { AirbyteRequestService } from "../../request/AirbyteRequestService"; - -export class WebBackendConnectionService extends AirbyteRequestService { - public getConnection(connectionId: string, withRefreshedCatalog?: boolean) { - return webBackendGetConnection({ connectionId, withRefreshedCatalog }, this.requestOptions); - } - - public list(payload: WebBackendConnectionListRequestBody) { - return webBackendListConnectionsForWorkspace(payload, this.requestOptions); - } - - public update(payload: WebBackendConnectionUpdate) { - return webBackendUpdateConnection(payload, this.requestOptions); - } - - public create(payload: WebBackendConnectionCreate) { - return webBackendCreateConnection(payload, this.requestOptions); - } -} diff --git a/airbyte-webapp/src/core/domain/connection/index.ts b/airbyte-webapp/src/core/domain/connection/index.ts index bdeabee6e47..1598409ec40 100644 --- a/airbyte-webapp/src/core/domain/connection/index.ts +++ b/airbyte-webapp/src/core/domain/connection/index.ts @@ -1,4 +1,3 @@ export * from "./operation"; export * from "./types"; export * from "./utils"; -export * from "./WebBackendConnectionService"; diff --git a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx index 694f545fc53..3eb990968e0 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx @@ -11,7 +11,11 @@ import { mockSourceDefinition, mockSourceDefinitionSpecification } from "test-ut import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; import { TestWrapper } from "test-utils/testutils"; -import { WebBackendConnectionRead, WebBackendConnectionUpdate } from "core/request/AirbyteClient"; +import { + WebBackendConnectionRead, + WebBackendConnectionRequestBody, + WebBackendConnectionUpdate, +} from "core/api/types/AirbyteClient"; import { ConnectionEditServiceProvider, useConnectionEditService } from "./ConnectionEditService"; import { useConnectionFormService } from "../ConnectionForm/ConnectionFormService"; @@ -34,22 +38,11 @@ jest.mock("services/connector/DestinationDefinitionService", () => ({ jest.mock("core/api", () => ({ useCurrentWorkspace: () => mockWorkspace, -})); - -const utils = { - getMockConnectionWithRefreshedCatalog: (): WebBackendConnectionRead => ({ - ...mockConnection, - catalogDiff: mockCatalogDiff, - catalogId: `${mockConnection.catalogId}1`, - }), -}; - -jest.mock("../useConnectionHook", () => ({ useGetConnection: () => mockConnection, - useWebConnectionService: () => ({ - getConnection: (_connectionId: string, withRefreshedCatalog?: boolean) => + useGetConnectionQuery: + () => + async ({ withRefreshedCatalog }: WebBackendConnectionRequestBody) => withRefreshedCatalog ? utils.getMockConnectionWithRefreshedCatalog() : mockConnection, - }), useUpdateConnection: () => ({ mutateAsync: jest.fn(async (connection: WebBackendConnectionUpdate) => { const { sourceCatalogId, ...connectionUpdate } = connection; @@ -59,6 +52,14 @@ jest.mock("../useConnectionHook", () => ({ }), })); +const utils = { + getMockConnectionWithRefreshedCatalog: (): WebBackendConnectionRead => ({ + ...mockConnection, + catalogDiff: mockCatalogDiff, + catalogId: `${mockConnection.catalogId}1`, + }), +}; + describe("ConnectionEditService", () => { const Wrapper: React.FC = ({ children }) => ( diff --git a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx index 3d8e8c3634a..7c68b8515bf 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx @@ -3,16 +3,16 @@ import { useContext, useState, createContext, useCallback } from "react"; import { useIntl } from "react-intl"; import { useAsyncFn } from "react-use"; +import { useGetConnection, useGetConnectionQuery, useUpdateConnection } from "core/api"; import { AirbyteCatalog, ConnectionStatus, WebBackendConnectionRead, WebBackendConnectionUpdate, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { ConnectionFormServiceProvider } from "../ConnectionForm/ConnectionFormService"; import { useNotificationService } from "../Notification/NotificationService"; -import { useGetConnection, useUpdateConnection, useWebConnectionService } from "../useConnectionHook"; import { SchemaError } from "../useSourceHook"; interface ConnectionEditProps { @@ -42,7 +42,7 @@ const getConnectionCatalog = (connection: WebBackendConnectionRead): ConnectionC const useConnectionEdit = ({ connectionId }: ConnectionEditProps): ConnectionEditHook => { const { formatMessage } = useIntl(); const { registerNotification, unregisterNotificationById } = useNotificationService(); - const connectionService = useWebConnectionService(); + const getConnectionQuery = useGetConnectionQuery(); const [connection, setConnection] = useState(useGetConnection(connectionId)); const [catalog, setCatalog] = useState(() => getConnectionCatalog(connection)); const [schemaHasBeenRefreshed, setSchemaHasBeenRefreshed] = useState(false); @@ -50,7 +50,7 @@ const useConnectionEdit = ({ connectionId }: ConnectionEditProps): ConnectionEdi const [{ loading: schemaRefreshing, error: schemaError }, refreshSchema] = useAsyncFn(async () => { unregisterNotificationById("connection.noDiff"); - const refreshedConnection = await connectionService.getConnection(connectionId, true); + const refreshedConnection = await getConnectionQuery({ connectionId, withRefreshedCatalog: true }); if (refreshedConnection.catalogDiff && refreshedConnection.catalogDiff.transforms?.length > 0) { setConnection(refreshedConnection); setSchemaHasBeenRefreshed(true); diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx index 42ee90e1619..d8e4511e257 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx @@ -10,6 +10,7 @@ import { useInitialValues, } from "components/connection/ConnectionForm/formConfig"; +import { ConnectionValues } from "core/api"; import { ConnectionScheduleType, DestinationDefinitionRead, @@ -18,7 +19,7 @@ import { SourceDefinitionRead, SourceDefinitionSpecificationRead, WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { FormError, generateMessageFromError } from "core/utils/errorStatusMessage"; import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; import { useGetDestinationDefinitionSpecification } from "services/connector/DestinationDefinitionSpecificationService"; @@ -26,7 +27,6 @@ import { useSourceDefinition } from "services/connector/SourceDefinitionService" import { useGetSourceDefinitionSpecification } from "services/connector/SourceDefinitionSpecificationService"; import { useUniqueFormId } from "../FormChangeTracker"; -import { ValuesProps } from "../useConnectionHook"; import { SchemaError } from "../useSourceHook"; export type ConnectionFormMode = "create" | "edit" | "readonly"; @@ -47,7 +47,7 @@ export const tidyConnectionFormValues = ( workspaceId: string, validationSchema: ConnectionValidationSchema, operations?: OperationRead[] -): ValuesProps => { +): ConnectionValues => { // TODO (https://github.com/airbytehq/airbyte/issues/17279): We should try to fix the types so we don't need the casting. const formValues: ConnectionFormValues = validationSchema.cast(values, { context: { isRequest: true }, diff --git a/airbyte-webapp/src/hooks/services/useDestinationHook.tsx b/airbyte-webapp/src/hooks/services/useDestinationHook.tsx index 73ec9811d27..f3712881b2d 100644 --- a/airbyte-webapp/src/hooks/services/useDestinationHook.tsx +++ b/airbyte-webapp/src/hooks/services/useDestinationHook.tsx @@ -1,7 +1,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useCallback } from "react"; -import { useSuspenseQuery } from "core/api"; +import { useRemoveConnectionsFromList, useSuspenseQuery } from "core/api"; import { ConnectionConfiguration } from "core/domain/connection"; import { DestinationService } from "core/domain/connector/DestinationService"; import { Action, Namespace } from "core/services/analytics"; @@ -9,7 +9,6 @@ import { useAnalyticsService } from "core/services/analytics"; import { isDefined } from "core/utils/common"; import { useInitService } from "services/useInitService"; -import { useRemoveConnectionsFromList } from "./useConnectionHook"; import { useRequestErrorHandler } from "./useRequestErrorHandler"; import { useCurrentWorkspace } from "./useWorkspace"; import { useConfig } from "../../config"; diff --git a/airbyte-webapp/src/hooks/services/useSourceHook.tsx b/airbyte-webapp/src/hooks/services/useSourceHook.tsx index 191a472a466..9d3def2d175 100644 --- a/airbyte-webapp/src/hooks/services/useSourceHook.tsx +++ b/airbyte-webapp/src/hooks/services/useSourceHook.tsx @@ -2,7 +2,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useCallback, useEffect, useState } from "react"; import { useConfig } from "config"; -import { useSuspenseQuery } from "core/api"; +import { useSuspenseQuery, useRemoveConnectionsFromList } from "core/api"; import { SyncSchema } from "core/domain/catalog"; import { ConnectionConfiguration } from "core/domain/connection"; import { SourceService } from "core/domain/connector/SourceService"; @@ -11,7 +11,6 @@ import { useAnalyticsService } from "core/services/analytics"; import { isDefined } from "core/utils/common"; import { useInitService } from "services/useInitService"; -import { useRemoveConnectionsFromList } from "./useConnectionHook"; import { useRequestErrorHandler } from "./useRequestErrorHandler"; import { useCurrentWorkspace } from "./useWorkspace"; import { SourceRead, SynchronousJobRead, WebBackendConnectionListItem } from "../../core/request/AirbyteClient"; diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx index c85bc950c91..fb1c21b4490 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx @@ -11,8 +11,8 @@ import { Button } from "components/ui/Button"; import { Heading } from "components/ui/Heading"; import { PageHeader } from "components/ui/PageHeader"; +import { useConnectionList } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; -import { useConnectionList } from "hooks/services/useConnectionHook"; import ConnectionsTable from "./ConnectionsTable"; import { ConnectionRoutePaths } from "../../routePaths"; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx index 379da89cda1..dcc5ef6e20f 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx @@ -17,10 +17,10 @@ import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; import { mockWorkspaceId } from "test-utils/mock-data/mockWorkspaceId"; import { TestWrapper, useMockIntersectionObserver } from "test-utils/testutils"; -import { WebBackendConnectionUpdate } from "core/request/AirbyteClient"; +import { useGetConnectionQuery } from "core/api"; +import { WebBackendConnectionUpdate } from "core/api/types/AirbyteClient"; import { defaultOssFeatures, FeatureItem } from "core/services/features"; import { ConnectionEditServiceProvider } from "hooks/services/ConnectionEdit/ConnectionEditService"; -import * as connectionHook from "hooks/services/useConnectionHook"; import { ConnectionReplicationPage } from "./ConnectionReplicationPage"; @@ -48,6 +48,13 @@ jest.mock("area/workspace/utils", () => ({ jest.mock("core/api", () => ({ useCurrentWorkspace: () => mockWorkspace, + useGetConnectionQuery: jest.fn(() => async () => mockConnection), + useGetConnection: () => mockConnection, + useGetStateTypeQuery: () => async () => "global", + useUpdateConnection: () => ({ + mutateAsync: async (connection: WebBackendConnectionUpdate) => connection, + isLoading: false, + }), })); jest.mock("hooks/theme/useAirbyteTheme", () => ({ @@ -79,18 +86,7 @@ describe("ConnectionReplicationPage", () => { }; const setupSpies = (getConnection?: () => Promise) => { - const getConnectionImpl: any = { - getConnection: getConnection ?? (() => new Promise(() => null) as any), - }; - jest.spyOn(connectionHook, "useGetConnection").mockImplementation(() => mockConnection as any); - jest.spyOn(connectionHook, "useWebConnectionService").mockImplementation(() => getConnectionImpl); - jest.spyOn(connectionHook, "useUpdateConnection").mockImplementation( - () => - ({ - mutateAsync: async (connection: WebBackendConnectionUpdate) => connection, - isLoading: false, - } as any) - ); + (useGetConnectionQuery as jest.Mock).mockImplementation(() => getConnection); }; beforeEach(() => { @@ -116,7 +112,8 @@ describe("ConnectionReplicationPage", () => { }); it("should show loading if the schema is refreshing", async () => { - setupSpies(); + // Return pending promise + setupSpies(() => new Promise(() => null)); const renderResult = await render(); await act(async () => { diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx index b7ac7886941..1fa992f8e1d 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.tsx @@ -18,6 +18,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Message } from "components/ui/Message/Message"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { ConnectionValues, useGetStateTypeQuery } from "core/api"; import { toWebBackendConnectionUpdate } from "core/domain/connection"; import { SchemaChange } from "core/request/AirbyteClient"; import { getFrequencyFromScheduleData } from "core/services/analytics"; @@ -32,7 +33,6 @@ import { useConnectionFormService, } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useModalService } from "hooks/services/Modal"; -import { useConnectionService, ValuesProps } from "hooks/services/useConnectionHook"; import styles from "./ConnectionReplicationPage.module.scss"; import { ResetWarningModal } from "./ResetWarningModal"; @@ -88,7 +88,7 @@ const SchemaChangeMessage: React.FC<{ dirty: boolean; schemaChange: SchemaChange export const ConnectionReplicationPage: React.FC = () => { const analyticsService = useAnalyticsService(); - const connectionService = useConnectionService(); + const getStateType = useGetStateTypeQuery(); const workspaceId = useCurrentWorkspaceId(); const { formatMessage } = useIntl(); @@ -106,7 +106,7 @@ export const ConnectionReplicationPage: React.FC = () => { const saveConnection = useCallback( async ( - values: ValuesProps, + values: ConnectionValues, { skipReset, catalogHasChanged }: { skipReset: boolean; catalogHasChanged: boolean } ) => { const connectionAsUpdate = toWebBackendConnectionUpdate(connection); @@ -189,7 +189,7 @@ export const ConnectionReplicationPage: React.FC = () => { // endpoint. try { if (catalogChangesRequireReset) { - const stateType = await connectionService.getStateType(connection.connectionId); + const stateType = await getStateType(connection.connectionId); const result = await openModal({ title: formatMessage({ id: "connection.resetModalTitle" }), size: "md", @@ -223,7 +223,7 @@ export const ConnectionReplicationPage: React.FC = () => { connection.syncCatalog.streams, connection.connectionId, setSubmitError, - connectionService, + getStateType, openModal, formatMessage, saveConnection, diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx index a33a146e0b4..1876f6b8de6 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx @@ -16,6 +16,7 @@ import { Heading } from "components/ui/Heading"; import { ExternalLink } from "components/ui/Link"; import { Spinner } from "components/ui/Spinner"; +import { useDeleteConnection } from "core/api"; import { Geography, WebBackendConnectionUpdate } from "core/request/AirbyteClient"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { FeatureItem, useFeature } from "core/services/features"; @@ -24,7 +25,6 @@ import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useNotificationService } from "hooks/services/Notification"; -import { useDeleteConnection } from "hooks/services/useConnectionHook"; import styles from "./ConnectionSettingsPage.module.scss"; import { SchemaUpdateNotifications } from "./SchemaUpdateNotifications"; diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx index b0ed2645d9e..3c12582d34c 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx @@ -10,10 +10,10 @@ import { Heading } from "components/ui/Heading"; import { Message } from "components/ui/Message"; import { Text } from "components/ui/Text"; +import { useCreateOrUpdateState, useGetConnectionState } from "core/api"; import { AirbyteCatalog, ConnectionState, StreamState } from "core/request/AirbyteClient"; import { haveSameShape } from "core/utils/objects"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; -import { useCreateOrUpdateState, useGetConnectionState } from "hooks/services/useConnectionHook"; import styles from "./StateBlock.module.scss"; diff --git a/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx b/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx index 4b74dd3db65..2ca9a2276b2 100644 --- a/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx @@ -8,9 +8,8 @@ import { DestinationConnectionTable } from "components/destination/DestinationCo import { DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { FlexContainer } from "components/ui/Flex"; -import { useCurrentWorkspace } from "core/api"; +import { useCurrentWorkspace, useConnectionList } from "core/api"; import { useGetDestinationFromParams } from "hooks/domain/connector/useGetDestinationFromParams"; -import { useConnectionList } from "hooks/services/useConnectionHook"; import { useSourceList } from "hooks/services/useSourceHook"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; diff --git a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx index 48ef8c59ba9..baa8b203f3a 100644 --- a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx @@ -4,10 +4,10 @@ import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; import { Text } from "components/ui/Text"; +import { useConnectionList } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useGetDestinationFromParams } from "hooks/domain/connector/useGetDestinationFromParams"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; -import { useConnectionList } from "hooks/services/useConnectionHook"; import { useDeleteDestination, useInvalidateDestination, diff --git a/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx b/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx index b11bf636e4a..5359fb7b4f4 100644 --- a/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx +++ b/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx @@ -5,8 +5,8 @@ import { ImplementationTable } from "components/EntityTable"; import { EntityTableDataItem } from "components/EntityTable/types"; import { getEntityTableData } from "components/EntityTable/utils"; +import { useConnectionList } from "core/api"; import { SourceRead } from "core/request/AirbyteClient"; -import { useConnectionList } from "hooks/services/useConnectionHook"; interface SourcesTableProps { sources: SourceRead[]; diff --git a/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx b/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx index fde59b4dcde..8a5daa61e69 100644 --- a/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx @@ -7,9 +7,8 @@ import { TableItemTitle } from "components/ConnectorBlocks"; import { DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { FlexContainer } from "components/ui/Flex/FlexContainer"; -import { useCurrentWorkspace } from "core/api"; +import { useCurrentWorkspace, useConnectionList } from "core/api"; import { useGetSourceFromParams } from "hooks/domain/connector/useGetSourceFromParams"; -import { useConnectionList } from "hooks/services/useConnectionHook"; import { useDestinationList } from "hooks/services/useDestinationHook"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; diff --git a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx index b2496b24f37..af2e282c6cb 100644 --- a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx @@ -4,10 +4,10 @@ import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; import { Text } from "components/ui/Text"; +import { useConnectionList } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useGetSourceFromParams } from "hooks/domain/connector/useGetSourceFromParams"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; -import { useConnectionList } from "hooks/services/useConnectionHook"; import { useDeleteSource, useInvalidateSource, useUpdateSource } from "hooks/services/useSourceHook"; import { useDeleteModal } from "hooks/useDeleteModal"; import { useSourceDefinition } from "services/connector/SourceDefinitionService"; From 22cbcdee89ac4f8828aa231e6c1487feaa0abe83 Mon Sep 17 00:00:00 2001 From: Charles Date: Fri, 25 Aug 2023 13:06:13 -0700 Subject: [PATCH 019/201] Deprecate `AirbyteApiClient.java` and `retryWithJitter` in favor of `AirbyteApiClient2` (#8329) --- .../airbyte/api/client/AirbyteApiClient.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java index 2962831ec9c..475fdcac4a2 100644 --- a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java +++ b/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java @@ -31,6 +31,8 @@ import org.slf4j.LoggerFactory; /** + * DEPRECATED. USE {@link io.airbyte.api.client2.AirbyteApiClient2}. + *

* This class is meant to consolidate all our API endpoints into a fluent-ish client. Currently, all * open API generators create a separate class per API "root-route". For example, if our API has two * routes "/v1/First/get" and "/v1/Second/get", OpenAPI generates (essentially) the following files: @@ -41,7 +43,10 @@ * ApiClient()).get(), which can get cumbersome if we're interacting with many pieces of the API. *

* This is currently manually maintained. We could look into autogenerating it if needed. + * + * @deprecated Replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ +@Deprecated(forRemoval = true) public class AirbyteApiClient { private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteApiClient.class); @@ -176,16 +181,27 @@ public OrganizationApi getOrganizationApi() { } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Default to 4 retries with a randomised 1 - 10 seconds interval between the first two retries and * an 10-minute wait for the last retry. *

* Exceptions will be swallowed. + * + * @param call method to execute + * @param desc short readable explanation of why this method is executed + * @param type of return type + * @return value returned by method + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ + @Deprecated(forRemoval = true) public static T retryWithJitter(final Callable call, final String desc) { return retryWithJitter(call, desc, DEFAULT_RETRY_INTERVAL_SECS, DEFAULT_FINAL_INTERVAL_SECS, DEFAULT_MAX_RETRIES); } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Provides a simple retry wrapper for api calls. This retry behaviour is slightly different from * generally available retries libraries - the last retry is able to wait an interval inconsistent * with regular intervals/exponential backoff. @@ -200,7 +216,9 @@ public static T retryWithJitter(final Callable call, final String desc) { * @param desc short readable explanation of why this method is executed * @param jitterMaxIntervalSecs upper limit of the randomised retry interval. Minimum value is 1. * @param finalIntervalSecs retry interval before the last retry. + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ + @Deprecated(forRemoval = true) @VisibleForTesting // This is okay since we are logging the stack trace, which PMD is not detecting. @SuppressWarnings("PMD.PreserveStackTrace") @@ -219,14 +237,26 @@ public static T retryWithJitter(final Callable call, } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Default to 4 retries with a randomised 1 - 10 seconds interval between the first two retries and * an 10-minute wait for the last retry. + * + * @param call method to execute + * @param desc description of what is happening + * @param type of return type + * @return value returned by method + * @throws Exception exception while jittering + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ + @Deprecated(forRemoval = true) public static T retryWithJitterThrows(final Callable call, final String desc) throws Exception { return retryWithJitterThrows(call, desc, DEFAULT_RETRY_INTERVAL_SECS, DEFAULT_FINAL_INTERVAL_SECS, DEFAULT_MAX_RETRIES); } /** + * DEPRECATED: Use {@link io.airbyte.api.client2.AirbyteApiClient2} instead. + *

* Provides a simple retry wrapper for api calls. This retry behaviour is slightly different from * generally available retries libraries - the last retry is able to wait an interval inconsistent * with regular intervals/exponential backoff. @@ -239,8 +269,10 @@ public static T retryWithJitterThrows(final Callable call, final String d * @param desc short readable explanation of why this method is executed * @param jitterMaxIntervalSecs upper limit of the randomised retry interval. Minimum value is 1. * @param finalIntervalSecs retry interval before the last retry. + * @deprecated replaced by {@link io.airbyte.api.client2.AirbyteApiClient2} */ @VisibleForTesting + @Deprecated(forRemoval = true) // This is okay since we are logging the stack trace, which PMD is not detecting. @SuppressWarnings("PMD.PreserveStackTrace") public static T retryWithJitterThrows(final Callable call, From f0f15f57c912bdcaa6e677870cfa78faf0d0a8af Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Fri, 25 Aug 2023 13:17:01 -0700 Subject: [PATCH 020/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20Properly?= =?UTF-8?q?=20render=20markdown=20in=20breaking=20change=20messages=20(#85?= =?UTF-8?q?48)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connector/BreakingChangeBanner.module.scss | 16 ++++++++++++++++ .../connector/BreakingChangeBanner.tsx | 6 ++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss b/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss index afde866c9ff..d155836e616 100644 --- a/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss +++ b/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss @@ -17,3 +17,19 @@ width: fit-content; margin-left: variables.$spacing-sm; } + +.breakingChangeMessage { + line-height: 1.2; + font-size: variables.$font-size-md; + + & * { + margin: 0; + font-size: inherit; + line-height: inherit; + } + + // needed to override the Markdown component style + & a { + line-height: inherit; + } +} diff --git a/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx b/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx index aae49579d4f..b8aca1fa533 100644 --- a/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx +++ b/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx @@ -4,9 +4,9 @@ import { useNavigate } from "react-router-dom"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Markdown } from "components/ui/Markdown"; import { Message } from "components/ui/Message"; import { Text } from "components/ui/Text"; -import { TextWithHTML } from "components/ui/TextWithHTML"; import { useConnectionList, useUpgradeConnectorVersion } from "core/api"; import { ActorDefinitionVersionRead } from "core/api/types/AirbyteClient"; @@ -159,9 +159,7 @@ export const BreakingChangeBanner = ({ - - - + ))} From 2d9d46111495a86628eb3cb5ef7b9f63afd865b4 Mon Sep 17 00:00:00 2001 From: Ryan Br Date: Fri, 25 Aug 2023 13:48:21 -0700 Subject: [PATCH 021/201] Rbroughan/destination overrides (#8531) --- .../src/main/kotlin/FlagDefinitions.kt | 2 + .../persistence/job/DefaultJobCreator.java | 38 +++-- .../job/ResourceRequirementsUtils.java | 16 +++ .../job/DefaultJobCreatorTest.java | 135 ++++++++++++++++++ 4 files changed, 182 insertions(+), 9 deletions(-) diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index d912a4898ee..b5409aa25e4 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -128,4 +128,6 @@ object FieldSelectionWorkspaces : EnvVar(envVar = "FIELD_SELECTION_WORKSPACES") object ConnectorOAuthConsentDisabled : Permanent(key = "connectors.oauth.disableOAuthConsent", default = false) object AddSchedulingJitter : Temporary(key = "platform.add-scheduling-jitter", default = false) + + object DestResourceOverrides : Temporary(key = "dest-resource-overrides", default = "") } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java index d2fbaa5da75..104b2e9a6c9 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java @@ -30,6 +30,7 @@ import io.airbyte.featureflag.Destination; import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.FieldSelectionWorkspaces.DestResourceOverrides; import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.Source; import io.airbyte.featureflag.SourceDefinition; @@ -174,7 +175,8 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac final StandardSourceDefinition sourceDefinition, final StandardDestinationDefinition destinationDefinition, final boolean isReset) { - final String variant = getResourceRequirementsVariant(workspaceId, standardSync, sourceDefinition, destinationDefinition); + final var ffContext = buildFeatureFlagContext(workspaceId, standardSync, sourceDefinition, destinationDefinition); + final String variant = featureFlagClient.stringVariation(UseResourceRequirementsVariant.INSTANCE, ffContext); // Note on use of sourceType, throughput is driven by the source, if the source is slow, the rest is // going to be slow. With this in mind, we align the resources given to the orchestrator and the @@ -182,7 +184,8 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac // is slow. final Optional sourceType = getSourceType(sourceDefinition); final ResourceRequirements mergedOrchestratorResourceReq = getOrchestratorResourceRequirements(standardSync, sourceType, variant); - final ResourceRequirements mergedDstResourceReq = getDestinationResourceRequirements(standardSync, destinationDefinition, sourceType, variant); + final ResourceRequirements mergedDstResourceReq = + getDestinationResourceRequirements(standardSync, destinationDefinition, sourceType, variant, ffContext); final var syncResourceRequirements = new SyncResourceRequirements() .withConfigKey(new SyncResourceRequirementsKey().withVariant(variant).withSubType(sourceType.orElse(null))) @@ -204,10 +207,10 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac return syncResourceRequirements; } - private String getResourceRequirementsVariant(final UUID workspaceId, - final StandardSync standardSync, - final StandardSourceDefinition sourceDefinition, - final StandardDestinationDefinition destinationDefinition) { + private Context buildFeatureFlagContext(final UUID workspaceId, + final StandardSync standardSync, + final StandardSourceDefinition sourceDefinition, + final StandardDestinationDefinition destinationDefinition) { final List contextList = new ArrayList<>(); addIfNotNull(contextList, workspaceId, Workspace::new); addIfNotNull(contextList, standardSync.getConnectionId(), Connection::new); @@ -216,7 +219,7 @@ private String getResourceRequirementsVariant(final UUID workspaceId, addIfNotNull(contextList, sourceDefinition != null ? sourceDefinition.getSourceDefinitionId() : null, SourceDefinition::new); addIfNotNull(contextList, standardSync.getDestinationId(), Destination::new); addIfNotNull(contextList, destinationDefinition.getDestinationDefinitionId(), DestinationDefinition::new); - return featureFlagClient.stringVariation(UseResourceRequirementsVariant.INSTANCE, new Multi(contextList)); + return new Multi(contextList); } private static void addIfNotNull(final List contextList, final UUID uuid, final Function supplier) { @@ -247,17 +250,34 @@ private ResourceRequirements getSourceResourceRequirements(final StandardSync st JobType.SYNC); } + private ResourceRequirements getDestinationResourceOverrides(final Context ffCtx) { + final String destOverrides = featureFlagClient.stringVariation(DestResourceOverrides.INSTANCE, ffCtx); + try { + return ResourceRequirementsUtils.parse(destOverrides); + } catch (final Exception e) { + log.warn("Could not parse DESTINATION resource overrides from feature flag string: '{}'", destOverrides); + log.warn("Error parsing DESTINATION resource overrides: {}", e.getMessage()); + return null; + } + } + private ResourceRequirements getDestinationResourceRequirements(final StandardSync standardSync, final StandardDestinationDefinition destinationDefinition, final Optional sourceType, - final String variant) { + final String variant, + final Context ffContext) { final ResourceRequirements defaultDstRssReqs = resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.DESTINATION, sourceType, variant); - return ResourceRequirementsUtils.getResourceRequirements( + + final var mergedRssReqs = ResourceRequirementsUtils.getResourceRequirements( standardSync.getResourceRequirements(), destinationDefinition.getResourceRequirements(), defaultDstRssReqs, JobType.SYNC); + + final var overrides = getDestinationResourceOverrides(ffContext); + + return ResourceRequirementsUtils.getResourceRequirements(overrides, mergedRssReqs); } private Optional getSourceType(final StandardSourceDefinition sourceDefinition) { diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java index 58c04e616c4..10cf5c9f0e7 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/ResourceRequirementsUtils.java @@ -5,6 +5,7 @@ package io.airbyte.persistence.job; import com.google.common.base.Preconditions; +import io.airbyte.commons.json.Jsons; import io.airbyte.config.ActorDefinitionResourceRequirements; import io.airbyte.config.JobTypeResourceLimit; import io.airbyte.config.JobTypeResourceLimit.JobType; @@ -137,4 +138,19 @@ public static Optional getResourceRequirementsForJobType(f : Optional.of(jobTypeResourceRequirement.get(0)); } + /** + * Utility for deserializing from a raw json string. + * + * @param rawOverrides A json string to be parsed. + * @return ResourceRequirements parsed from the string. + */ + public static ResourceRequirements parse(final String rawOverrides) { + if (rawOverrides.isEmpty()) { + return null; + } + + final var json = Jsons.deserialize(rawOverrides); + return Jsons.object(json, ResourceRequirements.class); + } + } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java index 2c2f419902f..a4f40a369b5 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java @@ -41,6 +41,7 @@ import io.airbyte.config.SyncResourceRequirements; import io.airbyte.config.SyncResourceRequirementsKey; import io.airbyte.config.provider.ResourceRequirementsProvider; +import io.airbyte.featureflag.FieldSelectionWorkspaces.DestResourceOverrides; import io.airbyte.featureflag.TestClient; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; @@ -51,12 +52,21 @@ import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.protocol.models.SyncMode; import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.platform.commons.util.StringUtils; +import org.mockito.ArgumentCaptor; +@SuppressWarnings("PMD.AvoidDuplicateLiterals") class DefaultJobCreatorTest { private static final String DEFAULT_VARIANT = "default"; @@ -507,6 +517,131 @@ void testCreateSyncJobSourceAndDestinationResourceReqs() throws IOException { verify(jobPersistence, times(1)).enqueueJob(expectedScope, expectedJobConfig); } + @ParameterizedTest + @MethodSource("resourceOverrideMatrix") + void testDestinationResourceReqsOverrides(final String cpuReqOverride, + final String cpuLimitOverride, + final String memReqOverride, + final String memLimitOverride) + throws IOException { + final var overrides = new HashMap<>(); + if (cpuReqOverride != null) { + overrides.put("cpu_request", cpuReqOverride); + } + if (cpuLimitOverride != null) { + overrides.put("cpu_limit", cpuLimitOverride); + } + if (memReqOverride != null) { + overrides.put("memory_request", memReqOverride); + } + if (memLimitOverride != null) { + overrides.put("memory_limit", memLimitOverride); + } + + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(DestResourceOverrides.INSTANCE.getKey(), Jsons.serialize(overrides)))); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(sourceResourceRequirements)), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withJobSpecific(List.of( + new JobTypeResourceLimit().withJobType(JobType.SYNC).withResourceRequirements(originalReqs)))), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var destConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getDestination(); + + final var expectedCpuReq = StringUtils.isNotBlank(cpuReqOverride) ? cpuReqOverride : originalReqs.getCpuRequest(); + assertEquals(expectedCpuReq, destConfigValues.getCpuRequest()); + + final var expectedCpuLimit = StringUtils.isNotBlank(cpuLimitOverride) ? cpuLimitOverride : originalReqs.getCpuLimit(); + assertEquals(expectedCpuLimit, destConfigValues.getCpuLimit()); + + final var expectedMemReq = StringUtils.isNotBlank(memReqOverride) ? memReqOverride : originalReqs.getMemoryRequest(); + assertEquals(expectedMemReq, destConfigValues.getMemoryRequest()); + + final var expectedMemLimit = StringUtils.isNotBlank(memLimitOverride) ? memLimitOverride : originalReqs.getMemoryLimit(); + assertEquals(expectedMemLimit, destConfigValues.getMemoryLimit()); + } + + private static Stream resourceOverrideMatrix() { + return Stream.of( + Arguments.of("0.7", "0.4", "1000Mi", "2000Mi"), + Arguments.of("0.3", null, "1000Mi", null), + Arguments.of(null, null, null, null), + Arguments.of(null, "0.4", null, null), + Arguments.of("3", "3", "3000Mi", "3000Mi"), + Arguments.of("4", "5", "6000Mi", "7000Mi")); + } + + @ParameterizedTest + @MethodSource("weirdnessOverrideMatrix") + void ignoresOverridesIfJsonStringWeird(final String weirdness) throws IOException { + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(DestResourceOverrides.INSTANCE.getKey(), Jsons.serialize(weirdness)))); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(sourceResourceRequirements)), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withJobSpecific(List.of( + new JobTypeResourceLimit().withJobType(JobType.SYNC).withResourceRequirements(originalReqs)))), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var destConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getDestination(); + + assertEquals(originalReqs.getCpuRequest(), destConfigValues.getCpuRequest()); + assertEquals(originalReqs.getCpuLimit(), destConfigValues.getCpuLimit()); + assertEquals(originalReqs.getMemoryRequest(), destConfigValues.getMemoryRequest()); + assertEquals(originalReqs.getMemoryLimit(), destConfigValues.getMemoryLimit()); + } + + private static Stream weirdnessOverrideMatrix() { + return Stream.of( + Arguments.of("0.7"), + Arguments.of("0.5, 1, 1000Mi, 2000Mi"), + Arguments.of("cat burglar"), + Arguments.of("{ \"cpu_limit\": \"2\", \"cpu_request\": \"1\" "), + Arguments.of("null"), + Arguments.of("undefined"), + Arguments.of(""), + Arguments.of("{}")); + } + @Test void testCreateResetConnectionJob() throws IOException { final Optional expectedSourceType = Optional.empty(); From 87f9936209483cbcdb7840f6575e46d650e5c32b Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Fri, 25 Aug 2023 23:37:33 +0200 Subject: [PATCH 022/201] Change column type of actor_definition_breaking_change.message from VARCHAR(256) to CLOB (aka 'text') (#8547) --- .../io/airbyte/bootloader/BootloaderTest.java | 2 +- ...reakingChangesMessageColumnToClobType.java | 41 +++++++ .../configs_database/schema_dump.txt | 2 +- ...ingChangesMessageColumnToClobTypeTest.java | 113 ++++++++++++++++++ 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobType.java create mode 100644 airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 14542ccdf22..300d558e952 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -76,7 +76,7 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.21.001"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.23.002"; private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.001"; private static final String CDK_VERSION = "1.2.3"; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobType.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobType.java new file mode 100644 index 00000000000..97586d012b5 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobType.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import com.google.common.annotations.VisibleForTesting; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Changes the message column type of actor_definition_breaking_change to CLOB, which will set\ it + * to 'text' in the db. We want to be able to handle large messages. + */ +public class V0_50_23_002__SetBreakingChangesMessageColumnToClobType extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_23_002__SetBreakingChangesMessageColumnToClobType.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + alterMessageColumnType(ctx); + } + + @VisibleForTesting + static void alterMessageColumnType(final DSLContext ctx) { + ctx.alterTable("actor_definition_breaking_change") + .alter("message").set(SQLDataType.CLOB).execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index bb48f21190e..88fd7e8a718 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -66,7 +66,7 @@ create table "public"."actor_definition_breaking_change" ( "version" varchar(256) not null, "migration_documentation_url" varchar(256) not null, "upgrade_deadline" date not null, - "message" varchar(256) not null, + "message" text not null, "created_at" timestamp(6) with time zone not null default current_timestamp, "updated_at" timestamp(6) with time zone not null default current_timestamp, constraint "actor_definition_breaking_change_pkey" diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java new file mode 100644 index 00000000000..dd73e44beb4 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static io.airbyte.db.instance.configs.migrations.V0_50_23_002__SetBreakingChangesMessageColumnToClobType.alterMessageColumnType; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_32_8_001__AirbyteConfigDatabaseDenormalization.ActorType; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.sql.Date; +import java.sql.Timestamp; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.exception.DataAccessException; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest extends AbstractConfigsDatabaseTest { + + private static final Table ACTOR_DEFINITION_BREAKING_CHANGE = DSL.table("actor_definition_breaking_change"); + private static final Table ACTOR_DEFINITION = DSL.table("actor_definition"); + private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_23_002__SetBreakingChangesMessageColumnToClobTypeTest.java", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_21_001__BackfillActorDefaultVersionAndSetNonNull(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + @Test + void testInsertThrowsBeforeMigration() { + final DSLContext ctx = getDslContext(); + insertActorDefinitionDependency(ctx); + final Throwable exception = assertThrows(DataAccessException.class, () -> insertBreakingChange(ctx)); + assertTrue(exception.getMessage().contains("value too long for type character varying(256)")); + } + + @Test + void testInsertSucceedsAfterMigration() { + final DSLContext ctx = getDslContext(); + insertActorDefinitionDependency(ctx); + alterMessageColumnType(ctx); + assertDoesNotThrow(() -> insertBreakingChange(ctx)); + } + + private static void insertActorDefinitionDependency(final DSLContext ctx) { + ctx.insertInto(ACTOR_DEFINITION) + .columns( + DSL.field("id"), + DSL.field("name"), + DSL.field("actor_type")) + .values( + ACTOR_DEFINITION_ID, + "source def name", + ActorType.source) + .onConflict( + DSL.field("id")) + .doNothing() + .execute(); + } + + private static void insertBreakingChange(final DSLContext ctx) { + final String message = + "This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early."; + + ctx.insertInto(ACTOR_DEFINITION_BREAKING_CHANGE) + .columns( + DSL.field("actor_definition_id"), + DSL.field("version"), + DSL.field("upgrade_deadline"), + DSL.field("message"), + DSL.field("migration_documentation_url"), + DSL.field("created_at"), + DSL.field("updated_at")) + .values( + ACTOR_DEFINITION_ID, + "3.0.0", + Date.valueOf("2023-11-01"), + message, + "https://docs.airbyte.com/integrations/destinations/snowflake-migrations#3.0.0", + Timestamp.valueOf("2023-08-25 16:33:42.701943875"), + Timestamp.valueOf("2023-08-25 16:33:42.701943875")) + .onConflict( + DSL.field("actor_definition_id"), + DSL.field("version")) + .doUpdate() + .set(DSL.field("upgrade_deadline"), Date.valueOf("2023-11-01")) + .set(DSL.field("message"), message) + .set(DSL.field("migration_documentation_url"), "https://docs.airbyte.com/integrations/destinations/snowflake-migrations#3.0.0") + .set(DSL.field("updated_at"), Timestamp.valueOf("2023-08-25 16:33:42.701943875")) + .execute(); + + } + +} From da7056ec008a6d5bc98b899c87c677d3392c01c9 Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Fri, 25 Aug 2023 14:39:14 -0700 Subject: [PATCH 023/201] Remove libs.javax.ws.rs.api dependency from Airbyte Server (#8546) --- airbyte-server/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 55eae2fbc30..ed6fd241185 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -65,7 +65,6 @@ dependencies { implementation libs.bundles.datadog implementation libs.sentry.java implementation libs.swagger.annotations - implementation libs.javax.ws.rs.api implementation libs.google.cloud.storage runtimeOnly libs.javax.databind From fb720c4895b51eecec4ab82be1bfe9ed43aef048 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Fri, 25 Aug 2023 14:51:55 -0700 Subject: [PATCH 024/201] Run tests in parallel. (#8050) Enable running tests in parallel. Remove some unnecessary test setup and tests. --- .../state/listener/TestStateListener.java | 3 +- .../acceptance/AdvancedAcceptanceTests.java | 23 ++++-------- .../acceptance/VersioningAcceptanceTests.java | 36 ++++++------------- .../resources/junit-platform.properties | 4 +-- 4 files changed, 21 insertions(+), 45 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java index e46958c6f56..6d3f8a28257 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/scheduling/state/listener/TestStateListener.java @@ -9,6 +9,7 @@ import java.util.Queue; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; /** * Workflow state change listener for testing. Used to verify the behavior of event signals in @@ -25,7 +26,7 @@ public static void reset() { @Override public Queue events(final UUID testId) { if (!events.containsKey(testId)) { - return new LinkedList<>(); + return new ConcurrentLinkedQueue<>(); } return events.get(testId); diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java index 05e1a99b3c7..79dc8c4edfd 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/AdvancedAcceptanceTests.java @@ -46,13 +46,10 @@ import io.airbyte.test.utils.TestConnectionCreate; import java.io.IOException; import java.net.URISyntaxException; -import java.sql.SQLException; import java.util.List; import java.util.UUID; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -121,21 +118,14 @@ static void end() { testHarness.stopDbAndContainers(); } - @BeforeEach - void setup() throws URISyntaxException, IOException, SQLException { - testHarness.setup(); - } - - @AfterEach - void tearDown() { - testHarness.cleanup(); - } - @SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") - @Test + @RetryingTest(3) void testManualSync() throws Exception { + testHarness.setup(); + final UUID sourceId = testHarness.createPostgresSource().getSourceId(); final UUID destinationId = testHarness.createPostgresDestination().getDestinationId(); + final SourceDiscoverSchemaRead discoverResult = testHarness.discoverSourceSchemaWithId(sourceId); final AirbyteCatalog catalog = discoverResult.getCatalog(); final SyncMode syncMode = SyncMode.FULL_REFRESH; @@ -153,8 +143,9 @@ void testManualSync() throws Exception { waitForSuccessfulJob(apiClient.getJobsApi(), connectionSyncRead.getJob()); Asserts.assertSourceAndDestinationDbRawRecordsInSync(testHarness.getSourceDatabase(), testHarness.getDestinationDatabase(), PUBLIC_SCHEMA_NAME, conn.getNamespaceFormat(), false, false); - Asserts.assertStreamStatuses(apiClient, workspaceId, conn.getConnectionId(), connectionSyncRead, StreamStatusRunState.COMPLETE, - StreamStatusJobType.SYNC); + Asserts.assertStreamStatuses(apiClient, workspaceId, connectionId, connectionSyncRead, StreamStatusRunState.COMPLETE, StreamStatusJobType.SYNC); + + testHarness.cleanup(); } @RetryingTest(3) diff --git a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java index 3b610c56fdd..45d34baf122 100644 --- a/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java +++ b/airbyte-tests/src/test-acceptance/java/io/airbyte/test/acceptance/VersioningAcceptanceTests.java @@ -16,16 +16,13 @@ import io.airbyte.api.client2.model.generated.SourceDefinitionCreate; import io.airbyte.api.client2.model.generated.SourceDefinitionIdRequestBody; import io.airbyte.api.client2.model.generated.SourceDefinitionRead; -import io.airbyte.test.utils.AcceptanceTestHarness; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.sql.SQLException; +import java.time.Duration; import java.util.UUID; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; +import okhttp3.OkHttpClient; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.params.ParameterizedTest; @@ -37,30 +34,17 @@ class VersioningAcceptanceTests { private static AirbyteApiClient2 apiClient2; private static UUID workspaceId; - private static AcceptanceTestHarness testHarness; - @BeforeAll - static void init() throws URISyntaxException, IOException, InterruptedException { - testHarness = new AcceptanceTestHarness(null, workspaceId); - RetryPolicy policy = RetryPolicy.ofDefaults(); - apiClient2 = new AirbyteApiClient2("http://localhost:8001/api", policy); - - workspaceId = apiClient2.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); - } + static void init() throws IOException { + RetryPolicy policy = RetryPolicy.builder() + .handle(Throwable.class) + .withMaxAttempts(5) + .withBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10)).build(); - @AfterAll - static void afterAll() { - testHarness.stopDbAndContainers(); - } - - @BeforeEach - void setup() throws SQLException, URISyntaxException, IOException { - testHarness.setup(); - } + OkHttpClient client = new OkHttpClient.Builder().readTimeout(Duration.ofSeconds(20)).build(); + apiClient2 = new AirbyteApiClient2("http://localhost:8001/api", policy, client); - @AfterEach - void tearDown() { - testHarness.cleanup(); + workspaceId = apiClient2.getWorkspaceApi().listWorkspaces().getWorkspaces().get(0).getWorkspaceId(); } @ParameterizedTest diff --git a/airbyte-tests/src/test-acceptance/resources/junit-platform.properties b/airbyte-tests/src/test-acceptance/resources/junit-platform.properties index 19747320c8f..9e63e3e8955 100644 --- a/airbyte-tests/src/test-acceptance/resources/junit-platform.properties +++ b/airbyte-tests/src/test-acceptance/resources/junit-platform.properties @@ -1,2 +1,2 @@ -#junit.jupiter.execution.parallel.enabled=true -#junit.jupiter.execution.parallel.mode.classes.default=concurrent +junit.jupiter.execution.parallel.enabled=true +junit.jupiter.execution.parallel.mode.classes.default=concurrent From 3396f7e464325f5396c5098cc70862b2e5e56663 Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Fri, 25 Aug 2023 15:01:18 -0700 Subject: [PATCH 025/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=A8=20Adjust=20?= =?UTF-8?q?state=20update=20warning=20to=20be=20more=20clear=20(#8530)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/locales/en.json | 3 ++- .../connections/ConnectionSettingsPage/StateBlock.tsx | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index b40ebaa12e1..81da61743e3 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -596,7 +596,8 @@ "connection.state.title": "Connection state", "connection.state.noIncremental": "This connection contains no incremental streams, so it cannot have state.", - "connection.state.warning": "Updates to connection state should be handled with extreme care. Make changes only as directed by the Airbyte team.", + "connection.state.warning": "Updates to connection state should be handled with extreme care.", + "connection.state.warning.secondary": "Updates may break your syncs, requiring a reset to fix.

Make changes only as directed by the Airbyte team.", "connection.state.update": "Update state", "connection.state.confirm.title": "Confirm connection state update", "connection.state.confirm.message": "Are you sure you want to modify the state of this connection?", diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx index 3c12582d34c..b5b146e8688 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx @@ -146,7 +146,13 @@ export const StateBlock: React.FC = ({ connectionId, syncCatalo {errorMessage ? ( ) : ( - } /> + } + secondaryText={ +
}} /> + } + /> )} + + + + ); + } + + // It's a good enough proxy to check if the first definition is a source or destination + const titleKey = isSourceDefinition(definitions[0]) + ? "sources.suggestedSources" + : "destinations.suggestedDestinations"; + + return ( +

+ +
+ + + + +
+ +
+ {definitions.map((definition) => ( + onConnectorButtonClick(definition)} + key={isSourceDefinition(definition) ? definition.sourceDefinitionId : definition.destinationDefinitionId} + /> + ))} +
+
+ ); +}; + +export const SuggestedConnectors = React.memo(SuggestedConnectorsUnmemoized); diff --git a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/index.ts b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/index.ts new file mode 100644 index 00000000000..0fb6a034d5b --- /dev/null +++ b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/index.ts @@ -0,0 +1 @@ +export * from "./SuggestedConnectors"; diff --git a/airbyte-webapp/src/area/connector/components/index.ts b/airbyte-webapp/src/area/connector/components/index.ts new file mode 100644 index 00000000000..0fb6a034d5b --- /dev/null +++ b/airbyte-webapp/src/area/connector/components/index.ts @@ -0,0 +1 @@ +export * from "./SuggestedConnectors"; diff --git a/airbyte-webapp/src/area/connector/utils/index.ts b/airbyte-webapp/src/area/connector/utils/index.ts index 7f6a03c157a..3515cab93f4 100644 --- a/airbyte-webapp/src/area/connector/utils/index.ts +++ b/airbyte-webapp/src/area/connector/utils/index.ts @@ -1,2 +1,4 @@ export { ConnectorIds } from "./constants"; export { SvgIcon } from "./SvgIcon"; +export * from "./useSuggestedDestinations"; +export * from "./useSuggestedSources"; diff --git a/airbyte-webapp/src/area/connector/utils/useSuggestedDestinations.ts b/airbyte-webapp/src/area/connector/utils/useSuggestedDestinations.ts new file mode 100644 index 00000000000..9824a516184 --- /dev/null +++ b/airbyte-webapp/src/area/connector/utils/useSuggestedDestinations.ts @@ -0,0 +1,11 @@ +import { useMemo } from "react"; + +import { useExperiment } from "hooks/services/Experiment/ExperimentService"; + +export const useSuggestedDestinations = () => { + const suggestedDestinationConnectors = useExperiment("connector.suggestedDestinationConnectors", ""); + return useMemo( + () => suggestedDestinationConnectors.split(",").map((id) => id.trim()), + [suggestedDestinationConnectors] + ); +}; diff --git a/airbyte-webapp/src/area/connector/utils/useSuggestedSources.ts b/airbyte-webapp/src/area/connector/utils/useSuggestedSources.ts new file mode 100644 index 00000000000..dfd65e33836 --- /dev/null +++ b/airbyte-webapp/src/area/connector/utils/useSuggestedSources.ts @@ -0,0 +1,8 @@ +import { useMemo } from "react"; + +import { useExperiment } from "hooks/services/Experiment/ExperimentService"; + +export const useSuggestedSources = () => { + const suggestedSourceConnectors = useExperiment("connector.suggestedSourceConnectors", ""); + return useMemo(() => suggestedSourceConnectors.split(",").map((id) => id.trim()), [suggestedSourceConnectors]); +}; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx index 24fa5b8c602..01fb3db6b0d 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx @@ -9,6 +9,7 @@ import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { Icon } from "components/ui/Icon"; +import { useSuggestedDestinations } from "area/connector/utils"; import { useAvailableDestinationDefinitions } from "hooks/domain/connector/useAvailableDestinationDefinitions"; import { AppActionCodes, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; @@ -22,6 +23,8 @@ export const CreateNewDestination: React.FC = () => { const [searchParams, setSearchParams] = useSearchParams(); const selectedDestinationDefinitionId = searchParams.get(DESTINATION_DEFINITION_PARAM); + const suggestedDestinationDefinitionIds = useSuggestedDestinations(); + const destinationDefinitions = useAvailableDestinationDefinitions(); const { trackAction } = useAppMonitoringService(); const { mutateAsync: createDestination } = useCreateDestination(); @@ -84,6 +87,7 @@ export const CreateNewDestination: React.FC = () => { onSelectConnectorDefinition={(destinationDefinitionId) => onSelectDestinationDefinitionId(destinationDefinitionId) } + suggestedConnectorDefinitionIds={suggestedDestinationDefinitionIds} /> ); }; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx index 9f5bae672c7..493b950e3aa 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx @@ -7,6 +7,7 @@ import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { Icon } from "components/ui/Icon"; +import { useSuggestedSources } from "area/connector/utils"; import { useAvailableSourceDefinitions } from "hooks/domain/connector/useAvailableSourceDefinitions"; import { AppActionCodes, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; @@ -20,7 +21,7 @@ export const SOURCE_DEFINITION_PARAM = "sourceDefinitionId"; export const CreateNewSource: React.FC = () => { const [searchParams, setSearchParams] = useSearchParams(); const selectedSourceDefinitionId = searchParams.get(SOURCE_DEFINITION_PARAM); - + const suggestedSourceDefinitionIds = useSuggestedSources(); const sourceDefinitions = useAvailableSourceDefinitions(); const { trackAction } = useAppMonitoringService(); const { mutateAsync: createSource } = useCreateSource(); @@ -79,6 +80,7 @@ export const CreateNewSource: React.FC = () => { connectorDefinitions={sourceDefinitions} connectorType="source" onSelectConnectorDefinition={(sourceDefinitionId) => onSelectSourceDefinitionId(sourceDefinitionId)} + suggestedConnectorDefinitionIds={suggestedSourceDefinitionIds} /> ); }; diff --git a/airbyte-webapp/src/components/source/SelectConnector/FilterReleaseStage.tsx b/airbyte-webapp/src/components/source/SelectConnector/FilterReleaseStage.tsx index 70dea20c0ed..346f20b1620 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/FilterReleaseStage.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/FilterReleaseStage.tsx @@ -37,6 +37,9 @@ export const FilterReleaseStage: React.FC = ({ return ( + + + {availableReleaseStages.map((stage) => { const id = `filter-release-stage-${stage}`; const isChecked = selectedReleaseStages.includes(stage); diff --git a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.module.scss b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.module.scss index d3ee07e22ec..8ea0a4f23fe 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.module.scss +++ b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.module.scss @@ -4,17 +4,16 @@ @use "scss/z-indices"; .selectConnector { - container-type: inline-size; position: relative; z-index: z-indices.$base; display: grid; // The width of the center column includes horizontal page padding on both sides. - grid-template-columns: 1fr min(100%, calc(variables.$page-width + 2 * variables.$spacing-md)) 1fr; + grid-template-columns: 1fr min(100%, calc(variables.$page-width + 2 * variables.$spacing-xl)) 1fr; &__header { grid-column: 2 / 3; - padding: variables.$spacing-md variables.$spacing-md variables.$spacing-md; + padding: variables.$spacing-md variables.$spacing-xl variables.$spacing-md; position: sticky; top: 0; background-color: colors.$backdrop; @@ -35,9 +34,17 @@ grid-column: 3 / 4; } + &__suggestedConnectors { + grid-column: 2 / 3; + + &:not(:empty) { + margin: variables.$spacing-sm 0 variables.$spacing-md; + } + } + &__grid { grid-column: 2 / 3; - padding-inline: variables.$spacing-md; + padding-inline: variables.$spacing-xl; margin-top: calc( variables.$spacing-xl - variables.$spacing-md ); // When not scrolled, we want xl space between the sticky header and the grid diff --git a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.test.tsx b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.test.tsx index 9f1bf8134b7..0753720532d 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.test.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.test.tsx @@ -28,6 +28,7 @@ describe(`${SelectConnector.name}`, () => { connectorType="source" connectorDefinitions={[mockSourceDefinition]} onSelectConnectorDefinition={jest.fn()} + suggestedConnectorDefinitionIds={[]} /> ); diff --git a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx index cb59b13df58..c06ff5ea104 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx @@ -8,6 +8,7 @@ import { FlexContainer } from "components/ui/Flex"; import { SearchInput } from "components/ui/SearchInput"; import { Text } from "components/ui/Text"; +import { SuggestedConnectors } from "area/connector/components"; import { useCurrentWorkspace } from "core/api"; import { ConnectorDefinition } from "core/domain/connector"; import { isSourceDefinition } from "core/domain/connector/source"; @@ -26,6 +27,7 @@ interface SelectConnectorProps { connectorType: "source" | "destination"; connectorDefinitions: ConnectorDefinition[]; onSelectConnectorDefinition: (id: string) => void; + suggestedConnectorDefinitionIds: string[]; } const RELEASE_STAGES: ReleaseStage[] = ["generally_available", "beta", "alpha", "custom"]; @@ -35,6 +37,7 @@ export const SelectConnector: React.FC = ({ connectorType, connectorDefinitions, onSelectConnectorDefinition, + suggestedConnectorDefinitionIds, }) => { const { formatMessage } = useIntl(); const { email } = useCurrentWorkspace(); @@ -142,8 +145,18 @@ export const SelectConnector: React.FC = ({
+
+ {suggestedConnectorDefinitionIds.length > 0 && ( +
+ +
+ )} +
; - "connector.shortSetupGuides": boolean; "authPage.rightSideUrl": string | undefined; - "authPage.signup.hideName": boolean; "authPage.signup.hideCompanyName": boolean; - "onboarding.speedyConnection": boolean; - "connection.onboarding.sources": string; - "connection.onboarding.destinations": string; + "authPage.signup.hideName": boolean; + "autopropagation.enabled": boolean; + "billing.newTrialPolicy": boolean; "connection.autoDetectSchemaChanges": boolean; "connection.columnSelection": boolean; - "connection.streamCentricUI.v2": boolean; - "connection.streamCentricUI.lateMultiplier": number; + "connection.onboarding.destinations": string; + "connection.onboarding.sources": string; + "connection.searchableJobLogs": boolean; "connection.streamCentricUI.errorMultiplier": number; + "connection.streamCentricUI.lateMultiplier": number; "connection.streamCentricUI.numberOfLogsToLoad": number; - "connection.searchableJobLogs": boolean; - "connector.showRequestSchemabutton": boolean; + "connection.streamCentricUI.v2": boolean; "connection.syncCatalog.simplifiedCatalogRow": boolean; - "upcomingFeaturesPage.url": string; - "billing.newTrialPolicy": boolean; - "connector.allowlistIpBanner": boolean; - "settings.emailNotifications": boolean; "connector.airbyteCloudIpAddresses": string; + "connector.allowlistIpBanner": boolean; + "connector.orderOverwrite": Record; + "connector.shortSetupGuides": boolean; + "connector.showRequestSchemabutton": boolean; + "connector.suggestedSourceConnectors": string; + "connector.suggestedDestinationConnectors": string; "connector.updateMethodSelection": boolean; + "onboarding.speedyConnection": boolean; + "settings.emailNotifications": boolean; + "upcomingFeaturesPage.url": string; "workspaces.newWorkspacesUI": boolean; } diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 81da61743e3..e61494c0b6d 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -340,6 +340,7 @@ "sources.newSource": "New source", "sources.newSourceTitle": "New Source", "sources.selectSourceTitle": "Select the type of source you want to connect", + "sources.suggestedSources": "Suggested sources", "sources.status": "Status", "sources.schema": "Schema", "sources.schemaSelectAll": "select all", @@ -420,6 +421,7 @@ "destination.destinationSettings": "Destination Settings", "destinations.newDestination": "New destination", "destinations.selectDestinationTitle": "Select the type of destination you want to connect", + "destinations.suggestedDestinations": "Suggested destinations", "destinations.description": "Destinations are where you send or push your data to.", "destinations.noDestinations": "Destination list is empty", "destinations.noSources": "No sources yet", @@ -772,6 +774,9 @@ "connector.additionalInfo.description": "e.g. links or documentation", "connector.email": "Your email address to get notified when it’s supported", "connector.email.placeholder": "richard@piedpiper.com", + "connector.filterBy": "Filter by:", + "connector.showSuggestedConnectors": "Show suggested connectors", + "connector.hideSuggestedConnectors": "Hide suggested connectors", "connector.source": "Source", "connector.destination": "Destination", "connector.releaseStage.alpha": "Alpha", diff --git a/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx b/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx index bb2d8cd6619..bd88d41ba8c 100644 --- a/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx +++ b/airbyte-webapp/src/pages/destination/SelectDestinationPage/SelectDestinationPage.tsx @@ -7,11 +7,13 @@ import { SelectConnector } from "components/source/SelectConnector"; import { Box } from "components/ui/Box"; import { Heading } from "components/ui/Heading"; +import { useSuggestedDestinations } from "area/connector/utils"; import { useAvailableDestinationDefinitions } from "hooks/domain/connector/useAvailableDestinationDefinitions"; export const SelectDestinationPage: React.FC = () => { const navigate = useNavigate(); const destinationDefinitions = useAvailableDestinationDefinitions(); + const suggestedDestinationDefinitionIds = useSuggestedDestinations(); return ( <> @@ -28,6 +30,7 @@ export const SelectDestinationPage: React.FC = () => { connectorType="destination" connectorDefinitions={destinationDefinitions} onSelectConnectorDefinition={(id) => navigate(`./${id}`)} + suggestedConnectorDefinitionIds={suggestedDestinationDefinitionIds} /> diff --git a/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx b/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx index da263d28fa2..9bcf68a4c92 100644 --- a/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx +++ b/airbyte-webapp/src/pages/source/SelectSourcePage/SelectSourcePage.tsx @@ -7,16 +7,18 @@ import { SelectConnector } from "components/source/SelectConnector"; import { Box } from "components/ui/Box"; import { Heading } from "components/ui/Heading"; +import { useSuggestedSources } from "area/connector/utils"; import { useAvailableSourceDefinitions } from "hooks/domain/connector/useAvailableSourceDefinitions"; export const SelectSourcePage: React.FC = () => { const navigate = useNavigate(); const sourceDefinitions = useAvailableSourceDefinitions(); + const suggestedSourceDefinitionIds = useSuggestedSources(); return ( <> - + @@ -28,6 +30,7 @@ export const SelectSourcePage: React.FC = () => { connectorType="source" connectorDefinitions={sourceDefinitions} onSelectConnectorDefinition={(id) => navigate(`./${id}`)} + suggestedConnectorDefinitionIds={suggestedSourceDefinitionIds} /> From 5e13496d956c0fb3210df4acb357dd975c8ac35b Mon Sep 17 00:00:00 2001 From: Joey Marshment-Howell Date: Mon, 28 Aug 2023 18:11:34 +0200 Subject: [PATCH 031/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Fix=20loc?= =?UTF-8?q?al=20storage=20hook=20(#8564)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/SuggestedConnectors/SuggestedConnectors.tsx | 4 ++-- airbyte-webapp/src/core/utils/useLocalStorage.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx index d674d25df1f..b88e443d518 100644 --- a/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx +++ b/airbyte-webapp/src/area/connector/components/SuggestedConnectors/SuggestedConnectors.tsx @@ -11,7 +11,7 @@ import { Text } from "components/ui/Text"; import { ConnectorDefinition } from "core/domain/connector"; import { isSourceDefinition } from "core/domain/connector/source"; -import { useLocalStorageFixed } from "core/utils/useLocalStorageFixed"; +import { useLocalStorage } from "core/utils/useLocalStorage"; import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; import { useSourceDefinitionList } from "services/connector/SourceDefinitionService"; @@ -29,7 +29,7 @@ export const SuggestedConnectorsUnmemoized: React.FC = const { formatMessage } = useIntl(); const { sourceDefinitionMap } = useSourceDefinitionList(); const { destinationDefinitionMap } = useDestinationDefinitionList(); - const [showSuggestedConnectors, setShowSuggestedConnectors] = useLocalStorageFixed( + const [showSuggestedConnectors, setShowSuggestedConnectors] = useLocalStorage( "airbyte_connector-grid-show-suggested-connectors", true ); diff --git a/airbyte-webapp/src/core/utils/useLocalStorage.ts b/airbyte-webapp/src/core/utils/useLocalStorage.ts index 1f42976dd99..6adec8766e3 100644 --- a/airbyte-webapp/src/core/utils/useLocalStorage.ts +++ b/airbyte-webapp/src/core/utils/useLocalStorage.ts @@ -17,6 +17,7 @@ interface AirbyteLocalStorage { allowlistIpsOpen: boolean; airbyteTheme: Theme; "airbyte_connector-grid-release-stage-filter": ReleaseStage[]; + "airbyte_connector-grid-show-suggested-connectors": boolean; } /* From e88422ae1e197116cfed899a266be9cfe7e247a2 Mon Sep 17 00:00:00 2001 From: Chandler Prall Date: Mon, 28 Aug 2023 10:30:28 -0600 Subject: [PATCH 032/201] =?UTF-8?q?=F0=9F=AA=9F=C2=A0=F0=9F=90=9B=20scroll?= =?UTF-8?q?=20to=20the=20end=20of=20the=20log=20list=20(#8560)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vladimir --- airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx b/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx index deebdd0cc75..ccce9fc7ba2 100644 --- a/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx +++ b/airbyte-webapp/src/components/NewJobItem/VirtualLogs.tsx @@ -62,7 +62,7 @@ const VirtualLogsUnmemoized: React.FC = ({ logLines, searchTer if (listRef.current === null || !logLines.length) { return; } - listRef?.current.scrollToItem(logLines.length); + listRef?.current.scrollToItem(logLines.length, "end"); }, [logLines.length]); useEffect(() => { From ec53cc9f8267b9fda31219ecc10827f6965c6f3f Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Mon, 28 Aug 2023 10:25:12 -0700 Subject: [PATCH 033/201] Refactor cloud-auth/support to airbyte-common-server (#8495) Co-authored-by: Joey Marshment-Howell Co-authored-by: Ben Church Co-authored-by: Evan Tahler Co-authored-by: Ella Rohm-Ensing Co-authored-by: Lake Mossman --- .../AirbyteHttpRequestFieldExtractor.java | 63 +++++++++ .../support/AuthNettyServerCustomizer.java | 82 +++++++++++ .../server/support/AuthenticationFields.java | 89 ++++++++++++ .../support/AuthenticationHttpHeaders.java | 86 ++++++++++++ .../server/support/AuthenticationId.java | 60 ++++++++ .../support/AuthorizationServerHandler.java | 87 ++++++++++++ .../AirbyteHttpRequestFieldExtractorTest.java | 130 ++++++++++++++++++ .../AuthNettyServerCustomizerTest.java | 49 +++++++ .../src/main/resources/application.yml | 2 + .../src/main/resources/application.yml | 2 + .../src/main/resources/application.yml | 3 + 11 files changed, 653 insertions(+) create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthNettyServerCustomizer.java create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthorizationServerHandler.java create mode 100644 airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractorTest.java create mode 100644 airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthNettyServerCustomizerTest.java diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java new file mode 100644 index 00000000000..5658b863e9c --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractor.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.micronaut.core.util.StringUtils; +import jakarta.inject.Singleton; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; + +/** + * Utility class that facilitates the extraction of values from HTTP request POST bodies. + */ +@Singleton +@Slf4j +public class AirbyteHttpRequestFieldExtractor { + + /** + * Extracts the requested ID from the HTTP request, if present. + * + * @param content The raw HTTP request as a string. + * @param idFieldName The name of the field/header that contains the ID. + * @return An {@link Optional} that may or may not contain the ID value extracted from the raw HTTP + * request. + */ + public Optional extractId(final String content, final String idFieldName) { + try { + final JsonNode json = Jsons.deserialize(content); + if (json != null) { + + final Optional idValue = extract(json, idFieldName); + + if (idValue.isEmpty()) { + log.debug("No match for field name '{}' in content '{}'.", idFieldName, content); + } else { + log.debug("Found '{}' for field '{}'", idValue, idFieldName); + return idValue; + } + } + } catch (final RuntimeException e) { + log.debug("Unable to extract ID field '{}' from content '{}'.", idFieldName, content, e); + } + + return Optional.empty(); + } + + private Optional extract(JsonNode jsonNode, String idFieldName) { + if (idFieldName.equals(AuthenticationFields.WORKSPACE_IDS_FIELD_NAME)) { + log.debug("Try to extract list of ids for field {}", idFieldName); + return Optional.ofNullable(jsonNode.get(idFieldName)) + .map(Jsons::serialize) + .filter(StringUtils::hasText); + } else { + return Optional.ofNullable(jsonNode.get(idFieldName)) + .map(JsonNode::asText) + .filter(StringUtils::hasText); + } + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthNettyServerCustomizer.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthNettyServerCustomizer.java new file mode 100644 index 00000000000..977cc457d0c --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthNettyServerCustomizer.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import io.micronaut.context.annotation.Value; +import io.micronaut.context.event.BeanCreatedEvent; +import io.micronaut.context.event.BeanCreatedEventListener; +import io.micronaut.http.server.netty.NettyServerCustomizer; +import io.micronaut.http.server.netty.NettyServerCustomizer.Registry; +import io.netty.channel.Channel; +import io.netty.handler.codec.http.HttpObjectAggregator; +import io.netty.handler.codec.http.HttpRequestDecoder; +import jakarta.inject.Singleton; +import lombok.extern.slf4j.Slf4j; + +/** + * Custom Netty customizer that registers the {@link AuthorizationServerHandler} with the Netty + * stream pipeline.
+ *
+ * This customizer registers the handler as the first in the pipeline to ensure that it can read + * and, if necessary, modify the incoming HTTP request to include a header that can be used to + * determine authorization. + */ +@Singleton +@Slf4j +public class AuthNettyServerCustomizer implements BeanCreatedEventListener { + + private final AuthorizationServerHandler authorizationServerHandler; + + private final Integer aggregatorMaxContentLength; + + public AuthNettyServerCustomizer(final AuthorizationServerHandler authorizationServerHandler, + @Value("${micronaut.server.netty.aggregator.max-content-length}") final Integer aggregatorMaxContentLength) { + this.authorizationServerHandler = authorizationServerHandler; + this.aggregatorMaxContentLength = aggregatorMaxContentLength; + } + + @Override + public Registry onCreated(final BeanCreatedEvent event) { + final NettyServerCustomizer.Registry registry = event.getBean(); + registry.register(new Customizer(null)); // + return registry; + } + + /** + * Custom {@link NettyServerCustomizer} that registers the {@link AuthorizationServerHandler} as the + * first handler in the Netty pipeline. + */ + private class Customizer implements NettyServerCustomizer { + + private final Channel channel; + + Customizer(final Channel channel) { + this.channel = channel; + } + + @Override + public NettyServerCustomizer specializeForChannel(final Channel channel, final ChannelRole role) { + return new Customizer(channel); + } + + @Override + public void onStreamPipelineBuilt() { + /* + * Register the handlers in reverse order so that the final order is: 1. Decoder 2. Aggregator 3. + * Authorization Handler + * + * This is to ensure that the full HTTP request with content is provided to the authorization + * handler. + */ + log.info("register authorizationServerHandler"); + channel.pipeline() + .addFirst("authorizationServerHandler", authorizationServerHandler) + .addFirst("aggregator", new HttpObjectAggregator(aggregatorMaxContentLength)) + .addFirst("decoder", new HttpRequestDecoder()); + } + + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java new file mode 100644 index 00000000000..36b192cb057 --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationFields.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +/** + * Collection of constants used to identify and extract ID values from a raw HTTP request for + * authentication purposes. + */ +public final class AuthenticationFields { + + /** + * Name of the field in HTTP request bodies that contains the connection ID value. + */ + public static final String CONNECTION_ID_FIELD_NAME = "connectionId"; + + /** + * Name of the field in HTTP request bodies that contains the destination ID value. + */ + public static final String DESTINATION_ID_FIELD_NAME = "destinationId"; + + /** + * Name of the field in HTTP request bodies that contains the job ID value. + */ + public static final String JOB_ID_FIELD_NAME = "id"; + + /** + * Alternative name of the field in HTTP request bodies that contains the job ID value. + */ + public static final String JOB_ID_ALT_FIELD_NAME = "jobId"; + + /** + * Name of the field in HTTP request bodies that contains the operation ID value. + */ + public static final String OPERATION_ID_FIELD_NAME = "operationId"; + + /** + * Name of the field in HTTP request bodies that contains the source ID value. + */ + public static final String SOURCE_ID_FIELD_NAME = "sourceId"; + + /** + * Name of the field in HTTP request bodies that contains the source definition ID value. + */ + public static final String SOURCE_DEFINITION_ID_FIELD_NAME = "sourceDefinitionId"; + + /** + * Name of the field in HTTP request bodies that contains the Airbyte-assigned user ID value. + */ + public static final String AIRBYTE_USER_ID_FIELD_NAME = "userId"; + + /** + * Name of the field in HTTP request bodies that contains the resource creator's Airbyte-assigned + * user ID value. + */ + public static final String CREATOR_USER_ID_FIELD_NAME = "creatorUserId"; + + /** + * Name of the field in HTTP request bodies that contains the external auth ID value. + */ + public static final String EXTERNAL_AUTH_ID_FIELD_NAME = "authUserId"; + + /** + * Name of the field in HTTP request bodies that contains the email value. + */ + public static final String EMAIL_FIELD_NAME = "email"; + + /** + * Name of the field in HTTP request bodies that contains the workspace ID value. + */ + public static final String WORKSPACE_ID_FIELD_NAME = "workspaceId"; + + /** + * Name of the field in HTTP request bodies that contains the workspace IDs value. + */ + public static final String WORKSPACE_IDS_FIELD_NAME = "workspaceIds"; + + /** + * Name of the field in HTTP request bodies that contains the config ID value - this is equivalent + * to the connection ID. + */ + public static final String CONFIG_ID_FIELD_NAME = "configId"; + + public static final String ORGANIZATION_ID_FIELD_NAME = "organizationId"; + + private AuthenticationFields() {} + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java new file mode 100644 index 00000000000..6faecd212fd --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHttpHeaders.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import io.micronaut.http.HttpHeaders; + +/** + * Collection of HTTP headers that are used to perform authentication/authorization. + */ +public final class AuthenticationHttpHeaders { + + /** + * Prefix that denotes an internal Airbyte header. + */ + public static final String AIRBYTE_HEADER_PREFIX = "X-Airbyte-"; + + /** + * Authorization header. + */ + public static final String AUTHORIZATION_HEADER = HttpHeaders.AUTHORIZATION; + + /** + * HTTP header that contains the connection ID for authorization purposes. + */ + public static final String CONNECTION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Connection-Id"; + + /** + * HTTP header that contains the destination ID for authorization purposes. + */ + public static final String DESTINATION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Destination-Id"; + + /** + * HTTP header that contains the job ID for authorization purposes. + */ + public static final String JOB_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Job-Id"; + + /** + * HTTP header that contains the operation ID for authorization purposes. + */ + public static final String OPERATION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Operation-Id"; + + /** + * HTTP header that contains the source ID for authorization purposes. + */ + public static final String SOURCE_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Source-Id"; + + /** + * HTTP header that contains the source definition ID for authorization purposes. + */ + public static final String SOURCE_DEFINITION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Source-Definition-Id"; + + /** + * HTTP header that contains the Airbyte-assigned user ID for authorization purposes. + */ + public static final String AIRBYTE_USER_ID_HEADER = AIRBYTE_HEADER_PREFIX + "User-Id"; + + /** + * HTTP header that contains the resource creator's Airbyte-assigned user ID for authorization + * purposes. + */ + public static final String CREATOR_USER_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Creator-User-Id"; + + /** + * HTTP header that contains the external auth ID for authorization purposes. + */ + public static final String EXTERNAL_AUTH_ID_HEADER = AIRBYTE_HEADER_PREFIX + "External-Auth-Id"; + + /** + * HTTP header that contains the auth user ID for authorization purposes. + */ + public static final String EMAIL_HEADER = AIRBYTE_HEADER_PREFIX + "Email"; + + /** + * HTTP header that contains the workspace ID for authorization purposes. + */ + public static final String WORKSPACE_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Workspace-Id"; + public static final String WORKSPACE_IDS_HEADER = AIRBYTE_HEADER_PREFIX + "Workspace-Ids"; + public static final String CONFIG_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Config-Id"; + + public static final String ORGANIZATION_ID_HEADER = AIRBYTE_HEADER_PREFIX + "Organization-Id"; + + private AuthenticationHttpHeaders() {} + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java new file mode 100644 index 00000000000..97fd401f35c --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationId.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.AIRBYTE_USER_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONFIG_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CREATOR_USER_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.EMAIL_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.EXTERNAL_AUTH_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; + +/** + * Enumeration of the ID values that are used to perform authentication. These values are used to + * fetch roles associated with an authenticated user. + */ +public enum AuthenticationId { + + EXTERNAL_AUTH_ID(AuthenticationFields.EXTERNAL_AUTH_ID_FIELD_NAME, EXTERNAL_AUTH_ID_HEADER), + CONNECTION_ID(AuthenticationFields.CONNECTION_ID_FIELD_NAME, CONNECTION_ID_HEADER), + DESTINATION_ID_(AuthenticationFields.DESTINATION_ID_FIELD_NAME, DESTINATION_ID_HEADER), + EMAIL(AuthenticationFields.EMAIL_FIELD_NAME, EMAIL_HEADER), + JOB_ID(AuthenticationFields.JOB_ID_FIELD_NAME, JOB_ID_HEADER), + JOB_ID_ALT(AuthenticationFields.JOB_ID_ALT_FIELD_NAME, JOB_ID_HEADER), + OPERATION_ID(AuthenticationFields.OPERATION_ID_FIELD_NAME, OPERATION_ID_HEADER), + SOURCE_ID(AuthenticationFields.SOURCE_ID_FIELD_NAME, SOURCE_ID_HEADER), + SOURCE_DEFINITION_ID(AuthenticationFields.SOURCE_DEFINITION_ID_FIELD_NAME, SOURCE_DEFINITION_ID_HEADER), + AIRBYTE_USER_ID(AuthenticationFields.AIRBYTE_USER_ID_FIELD_NAME, AIRBYTE_USER_ID_HEADER), + CREATOR_USER_ID(AuthenticationFields.CREATOR_USER_ID_FIELD_NAME, CREATOR_USER_ID_HEADER), + WORKSPACE_ID(AuthenticationFields.WORKSPACE_ID_FIELD_NAME, WORKSPACE_ID_HEADER), + WORKSPACE_IDS(AuthenticationFields.WORKSPACE_IDS_FIELD_NAME, WORKSPACE_IDS_HEADER), + CONFIG_ID(AuthenticationFields.CONFIG_ID_FIELD_NAME, CONFIG_ID_HEADER), + ORGANIZATION_ID(AuthenticationFields.ORGANIZATION_ID_FIELD_NAME, ORGANIZATION_ID_HEADER); + + private final String fieldName; + private final String httpHeader; + + AuthenticationId(final String fieldName, final String httpHeader) { + this.fieldName = fieldName; + this.httpHeader = httpHeader; + } + + public String getFieldName() { + return fieldName; + } + + public String getHttpHeader() { + return httpHeader; + } + +} diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthorizationServerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthorizationServerHandler.java new file mode 100644 index 00000000000..93a6231a825 --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthorizationServerHandler.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandler.Sharable; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpHeaders; +import jakarta.inject.Singleton; +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; + +/** + * Custom Netty {@link ChannelDuplexHandler} that intercepts all operations to ensure that headers + * required for authorization are populated prior to performing the security check. + */ +@Singleton +@Sharable +@Slf4j +public class AuthorizationServerHandler extends ChannelDuplexHandler { + + private final AirbyteHttpRequestFieldExtractor airbyteHttpRequestFieldExtractor; + + public AuthorizationServerHandler(final AirbyteHttpRequestFieldExtractor airbyteHttpRequestFieldExtractor) { + this.airbyteHttpRequestFieldExtractor = airbyteHttpRequestFieldExtractor; + } + + @Override + public void channelRead( + final ChannelHandlerContext context, + final Object message) { + + Object updatedMessage = message; + + if (FullHttpRequest.class.isInstance(message)) { + final FullHttpRequest fullHttpRequest = FullHttpRequest.class.cast(message); + updatedMessage = updateHeaders(fullHttpRequest); + } + + context.fireChannelRead(updatedMessage); + } + + /** + * Checks the payload of the raw HTTP request for ID fields that should be copied to an HTTP header + * in order to facilitate authorization via Micronaut Security. + * + * @param httpRequest The raw HTTP request as a {@link FullHttpRequest}. + * @return The potentially modified raw HTTP request as a {@link FullHttpRequest}. + */ + protected FullHttpRequest updateHeaders(final FullHttpRequest httpRequest) { + for (final AuthenticationId authenticationId : AuthenticationId.values()) { + final String contentAsString = StandardCharsets.UTF_8.decode(httpRequest.content().nioBuffer()).toString(); + log.debug("Checking HTTP request '{}' for field '{}'...", contentAsString, authenticationId.getFieldName()); + final Optional id = + airbyteHttpRequestFieldExtractor.extractId(contentAsString, authenticationId.getFieldName()); + if (id.isPresent()) { + log.debug("Found field '{}' with value '{}' in HTTP request body.", authenticationId.getFieldName(), id.get()); + addHeaderToRequest(authenticationId.getHttpHeader(), id.get(), httpRequest); + } else { + log.debug("Field '{}' not found in content.", authenticationId.getFieldName()); + } + } + + return httpRequest; + } + + /** + * Adds the provided header and value to the HTTP request represented by the {@link FullHttpRequest} + * if the header is not already present. + * + * @param headerName The name of the header. + * @param headerValue The value of the header. + * @param httpRequest The current HTTP request. + */ + protected void addHeaderToRequest(final String headerName, final Object headerValue, final FullHttpRequest httpRequest) { + final HttpHeaders httpHeaders = httpRequest.headers(); + if (!httpHeaders.contains(headerName)) { + log.debug("Adding HTTP header '{}' with value '{}' to request...", headerName, headerValue); + httpHeaders.add(headerName, headerValue.toString()); + } + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractorTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractorTest.java new file mode 100644 index 00000000000..3b6d89c8034 --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AirbyteHttpRequestFieldExtractorTest.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationFields.WORKSPACE_IDS_FIELD_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.commons.json.Jsons; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link AirbyteHttpRequestFieldExtractor} class. + */ +class AirbyteHttpRequestFieldExtractorTest { + + private static final String OTHER_ID = "other_id"; + private static final String SOME_ID = "some_id"; + + private AirbyteHttpRequestFieldExtractor airbyteHttpRequestFieldExtractor; + + @BeforeEach + void setup() { + airbyteHttpRequestFieldExtractor = new AirbyteHttpRequestFieldExtractor(); + } + + @Test + void testExtractionUUID() { + final UUID match = UUID.randomUUID(); + final UUID other = UUID.randomUUID(); + final String idFieldName = SOME_ID; + final Map content = Map.of(idFieldName, match.toString(), OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isPresent()); + assertEquals(match, UUID.fromString(extractedId.get())); + } + + @Test + void testExtractionNonUUID() { + final Long match = 12345L; + final Long other = Long.MAX_VALUE; + final String idFieldName = SOME_ID; + final Map content = Map.of(idFieldName, match.toString(), OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isPresent()); + assertEquals(match, Long.valueOf(extractedId.get())); + } + + @Test + void testExtractionWithEmptyContent() { + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId("", SOME_ID); + assertTrue(extractedId.isEmpty()); + } + + @Test + void testExtractionWithMissingField() { + final UUID match = UUID.randomUUID(); + final UUID other = UUID.randomUUID(); + final String idFieldName = SOME_ID; + final Map content = Map.of(idFieldName, match.toString(), OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, "unknownFieldId"); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testExtractionWithNoMatch() { + final String idFieldName = SOME_ID; + final Map content = Map.of(OTHER_ID, UUID.randomUUID().toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testExtractionWithJsonParsingException() { + final String idFieldName = SOME_ID; + final String contentAsString = "{ \"someInvalidJson\" : \" ], \"foo\" }"; + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testWorkspaceIdsExtractionWithNullWorkspaceIds() { + final String idFieldName = WORKSPACE_IDS_FIELD_NAME; + final UUID other = UUID.randomUUID(); + + final Map content = Map.of(OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertTrue(extractedId.isEmpty()); + } + + @Test + void testWorkspaceIdsExtraction() { + final String idFieldName = WORKSPACE_IDS_FIELD_NAME; + final List valueList = List.of(UUID.randomUUID(), UUID.randomUUID(), UUID.randomUUID()); + final String valueString = String.format("[\"%s\",\"%s\",\"%s\"]", valueList.get(0), valueList.get(1), valueList.get(2)); + final UUID other = UUID.randomUUID(); + + final Map content = Map.of(WORKSPACE_IDS_FIELD_NAME, valueList, OTHER_ID, other.toString()); + final String contentAsString = Jsons.serialize(content); + + final Optional extractedId = airbyteHttpRequestFieldExtractor.extractId(contentAsString, idFieldName); + + assertEquals(extractedId, Optional.of(valueString)); + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthNettyServerCustomizerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthNettyServerCustomizerTest.java new file mode 100644 index 00000000000..4cc2c2f5851 --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthNettyServerCustomizerTest.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.micronaut.context.event.BeanCreatedEvent; +import io.micronaut.http.server.netty.NettyServerCustomizer; +import io.micronaut.http.server.netty.NettyServerCustomizer.Registry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +/** + * Test suite for the {@link AuthNettyServerCustomizer} class. + */ +class AuthNettyServerCustomizerTest { + + private static final Integer MAX_CONTENT_LENGTH = 1024; + + private AuthorizationServerHandler authorizationServerHandler; + + private AuthNettyServerCustomizer customizer; + + @BeforeEach + void setup() { + authorizationServerHandler = Mockito.mock(AuthorizationServerHandler.class); + customizer = new AuthNettyServerCustomizer(authorizationServerHandler, MAX_CONTENT_LENGTH); + } + + @Test + void testCustomizerRegisteredOnCreation() { + final BeanCreatedEvent event = mock(BeanCreatedEvent.class); + final Registry registry = mock(Registry.class); + when(event.getBean()).thenReturn(registry); + + final Registry result = customizer.onCreated(event); + assertEquals(registry, result); + verify(registry, times(1)).register(any(NettyServerCustomizer.class)); + } + +} diff --git a/airbyte-connector-builder-server/src/main/resources/application.yml b/airbyte-connector-builder-server/src/main/resources/application.yml index a576a82a7a4..159af1e34d9 100644 --- a/airbyte-connector-builder-server/src/main/resources/application.yml +++ b/airbyte-connector-builder-server/src/main/resources/application.yml @@ -11,6 +11,8 @@ micronaut: netty: access-logger: enabled: ${HTTP_ACCESS_LOG_ENABLED:true} + aggregator: + max-content-length: 1048576 endpoints: v1/manifest_template: enable: true diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 98968dc09de..fc5fcbaad57 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -23,6 +23,8 @@ micronaut: allowed-origins-regex: - ^.*$ netty: + aggregator: + max-content-length: 1048576 access-logger: enabled: ${HTTP_ACCESS_LOG_ENABLED:true} idle-timeout: ${HTTP_IDLE_TIMEOUT:5m} diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index f263acd7a64..f8194ee2fd3 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -2,6 +2,9 @@ micronaut: application: name: airbyte-workers server: + netty: + aggregator: + max-content-length: 1048576 port: 9000 caches: # used by RecordMetricActivity to cache repeated calls to workspaceApi.getWorkspaceByConnectionId. From 7042fc28872dbffd0d31957f54baf86dca875208 Mon Sep 17 00:00:00 2001 From: Jack Keller Date: Mon, 28 Aug 2023 13:28:53 -0400 Subject: [PATCH 034/201] Add image short name to Kubernetes pod labels (#8506) --- .../workers/process/KubeProcessFactory.java | 9 +++++++-- .../io/airbyte/workers/process/Metadata.java | 1 + .../airbyte/workers/sync/LauncherWorker.java | 20 +++++++++++-------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java index c4761917a58..b813dd9e1ef 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java @@ -6,6 +6,7 @@ import autovalue.shaded.org.jetbrains.annotations.NotNull; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.helper.DockerImageNameHelper; import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.workers.config.WorkerConfigs; @@ -126,7 +127,9 @@ public Process create( final WorkerConfigs workerConfigs = workerConfigsProvider.getConfig(resourceType); - final var allLabels = getLabels(jobId, attempt, connectionId, workspaceId, customLabels, workerConfigs.getWorkerKubeLabels()); + final String shortImageName = imageName != null ? DockerImageNameHelper.extractShortImageName(imageName) : null; + + final var allLabels = getLabels(jobId, attempt, connectionId, workspaceId, shortImageName, customLabels, workerConfigs.getWorkerKubeLabels()); // If using isolated pool, check workerConfigs has isolated pool set. If not set, fall back to use // regular node pool. @@ -180,6 +183,7 @@ public static Map getLabels(final String jobId, final int attemptId, final UUID connectionId, final UUID workspaceId, + final String imageName, final Map customLabels, final Map envLabels) { final Map allLabels = new HashMap<>(); @@ -193,7 +197,8 @@ public static Map getLabels(final String jobId, Metadata.ATTEMPT_LABEL_KEY, String.valueOf(attemptId), Metadata.CONNECTION_ID_LABEL_KEY, String.valueOf(connectionId), Metadata.WORKSPACE_LABEL_KEY, String.valueOf(workspaceId), - Metadata.WORKER_POD_LABEL_KEY, Metadata.WORKER_POD_LABEL_VALUE); + Metadata.WORKER_POD_LABEL_KEY, Metadata.WORKER_POD_LABEL_VALUE, + Metadata.IMAGE_NAME, imageName); allLabels.putAll(generalKubeLabels); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java index 7d42a39b2a1..19f0ddf41a5 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/Metadata.java @@ -19,6 +19,7 @@ public final class Metadata { static final String WORKER_POD_LABEL_KEY = "airbyte"; static final String WORKER_POD_LABEL_VALUE = "job-pod"; public static final String CONNECTION_ID_LABEL_KEY = "connection_id"; + static final String IMAGE_NAME = "image_name"; /** * These are more readable forms of {@link io.airbyte.config.JobTypeResourceLimit.JobType}. diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java index e8fd9598322..0bb07ead93c 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java @@ -13,6 +13,7 @@ import com.google.common.base.Stopwatch; import datadog.trace.api.Trace; import io.airbyte.commons.constants.WorkerConstants; +import io.airbyte.commons.helper.DockerImageNameHelper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.temporal.TemporalUtils; @@ -172,14 +173,6 @@ public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException OrchestratorConstants.PORT3, OrchestratorConstants.PORT3, OrchestratorConstants.PORT4, OrchestratorConstants.PORT4); - final var allLabels = KubeProcessFactory.getLabels( - jobRunConfig.getJobId(), - Math.toIntExact(jobRunConfig.getAttemptId()), - connectionId, - workspaceId, - generateMetadataLabels(), - Collections.emptyMap()); - final var podNameAndJobPrefix = podNamePrefix + "-job-" + jobRunConfig.getJobId() + "-attempt-"; final var podName = podNameAndJobPrefix + jobRunConfig.getAttemptId(); final var mainContainerInfo = new KubeContainerInfo(containerOrchestratorConfig.containerOrchestratorImage(), @@ -192,6 +185,17 @@ public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException final String schedulerName = featureFlagClient.stringVariation(UseCustomK8sScheduler.INSTANCE, new Connection(connectionId)); + final String shortImageName = + mainContainerInfo.image() != null ? DockerImageNameHelper.extractShortImageName(mainContainerInfo.image()) : null; + final var allLabels = KubeProcessFactory.getLabels( + jobRunConfig.getJobId(), + Math.toIntExact(jobRunConfig.getAttemptId()), + connectionId, + workspaceId, + shortImageName, + generateMetadataLabels(), + Collections.emptyMap()); + // Use the configuration to create the process. process = new AsyncOrchestratorPodProcess( kubePodInfo, From f18f93a59aae75f8e0a87479bcbcd921e627d579 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Mon, 28 Aug 2023 19:48:36 +0200 Subject: [PATCH 035/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20Add=20dar?= =?UTF-8?q?k=20mode=20to=20Storybook=20(#8562)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/.storybook/main.ts | 1 + airbyte-webapp/.storybook/preview.ts | 11 +- airbyte-webapp/package.json | 17 +- airbyte-webapp/pnpm-lock.yaml | 4059 ++++++++++++++++++++------ 4 files changed, 3178 insertions(+), 910 deletions(-) diff --git a/airbyte-webapp/.storybook/main.ts b/airbyte-webapp/.storybook/main.ts index ba47e941ac3..769c459b231 100644 --- a/airbyte-webapp/.storybook/main.ts +++ b/airbyte-webapp/.storybook/main.ts @@ -5,6 +5,7 @@ const config: StorybookConfig = { framework: "@storybook/react-vite", stories: ["../src/**/*.stories.@(ts|tsx)", "../src/**/*.docs.mdx"], addons: [ + "storybook-dark-mode", "@storybook/addon-links", "@storybook/addon-essentials", { diff --git a/airbyte-webapp/.storybook/preview.ts b/airbyte-webapp/.storybook/preview.ts index 1e86bf0d54b..d18c59948f0 100644 --- a/airbyte-webapp/.storybook/preview.ts +++ b/airbyte-webapp/.storybook/preview.ts @@ -4,5 +4,14 @@ import "../public/index.css"; import "../src/scss/global.scss"; import "../src/dayjs-setup"; -export const parameters = {}; +export const parameters = { + darkMode: { + stylePreview: true, + darkClass: ["airbyteThemeDark"], + lightClass: ["airbyteThemeLight"], + }, + backgrounds: { + disable: true, + }, +}; export const decorators = [withProviders]; diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 5a22a46ac25..5fe1a9ffcd6 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -144,13 +144,13 @@ "@babel/preset-typescript": "^7.21.0", "@formatjs/icu-messageformat-parser": "^2.4.0", "@modyfi/vite-plugin-yaml": "^1.0.4", - "@storybook/addon-actions": "^7.0.2", - "@storybook/addon-docs": "^7.0.2", - "@storybook/addon-essentials": "^7.0.2", - "@storybook/addon-links": "^7.0.2", - "@storybook/react": "^7.0.2", - "@storybook/react-vite": "^7.0.2", - "@storybook/theming": "^7.0.2", + "@storybook/addon-actions": "^7.3.2", + "@storybook/addon-docs": "^7.3.2", + "@storybook/addon-essentials": "^7.3.2", + "@storybook/addon-links": "^7.3.2", + "@storybook/react": "^7.3.2", + "@storybook/react-vite": "^7.3.2", + "@storybook/theming": "^7.3.2", "@testing-library/jest-dom": "^6.0.0", "@testing-library/react": "^14.0.0", "@testing-library/user-event": "^14.4.3", @@ -208,7 +208,8 @@ "prettier": "^2.8.4", "react-select-event": "^5.5.0", "start-server-and-test": "^2.0.0", - "storybook": "^7.0.2", + "storybook": "^7.3.2", + "storybook-dark-mode": "^3.0.1", "stylelint": "^14.9.1", "stylelint-config-css-modules": "^4.2.0", "stylelint-config-prettier-scss": "^0.0.1", diff --git a/airbyte-webapp/pnpm-lock.yaml b/airbyte-webapp/pnpm-lock.yaml index 14ee3a97a37..c3dbd990ef2 100644 --- a/airbyte-webapp/pnpm-lock.yaml +++ b/airbyte-webapp/pnpm-lock.yaml @@ -288,26 +288,26 @@ devDependencies: specifier: ^1.0.4 version: 1.0.4(rollup@2.79.1)(vite@4.2.0) '@storybook/addon-actions': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) '@storybook/addon-docs': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) '@storybook/addon-essentials': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) '@storybook/addon-links': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0) '@storybook/react': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) '@storybook/react-vite': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2)(vite@4.2.0) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0)(rollup@2.79.1)(typescript@5.0.2)(vite@4.2.0) '@storybook/theming': - specifier: ^7.0.2 - version: 7.0.2(react-dom@18.2.0)(react@18.2.0) + specifier: ^7.3.2 + version: 7.3.2(react-dom@18.2.0)(react@18.2.0) '@testing-library/jest-dom': specifier: ^6.0.0 version: 6.0.0(@types/jest@29.5.3)(jest@29.6.2) @@ -480,8 +480,11 @@ devDependencies: specifier: ^2.0.0 version: 2.0.0 storybook: - specifier: ^7.0.2 - version: 7.0.2 + specifier: ^7.3.2 + version: 7.3.2 + storybook-dark-mode: + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0) stylelint: specifier: ^14.9.1 version: 14.16.1 @@ -538,6 +541,14 @@ packages: '@jridgewell/gen-mapping': 0.1.1 '@jridgewell/trace-mapping': 0.3.17 + /@ampproject/remapping@2.2.1: + resolution: {integrity: sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.19 + dev: true + /@apidevtools/json-schema-ref-parser@9.0.6: resolution: {integrity: sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==} dependencies: @@ -576,8 +587,8 @@ packages: '@types/json-schema': 7.0.11 dev: true - /@aw-web-design/x-default-browser@1.4.88: - resolution: {integrity: sha512-AkEmF0wcwYC2QkhK703Y83fxWARttIWXDmQN8+cof8FmFZ5BRhnNXGymeb1S73bOCLfWjYELxtujL56idCN/XA==} + /@aw-web-design/x-default-browser@1.4.126: + resolution: {integrity: sha512-Xk1sIhyNC/esHGGVjL/niHLowM0csl/kFO5uawBy4IrWwy0o1G8LGt3jP6nmWGz+USxeeqbihAmp/oVZju6wug==} hasBin: true dependencies: default-browser-id: 3.0.0 @@ -601,8 +612,8 @@ packages: resolution: {integrity: sha512-sEnuDPpOJR/fcafHMjpcpGN5M2jbUGUHwmuWKM/YdPzeEDJg8bgmbcWQFUfE32MQjti1koACvoPVsDe8Uq+idg==} engines: {node: '>=6.9.0'} - /@babel/compat-data@7.21.4: - resolution: {integrity: sha512-/DYyDpeCfaVinT40FPGdkkb+lYSKvsVuMjDAG7jPOWWiM1ibOaB9CXJAlc4d1QpP/U2q2P9jbrSlClKSErd55g==} + /@babel/compat-data@7.22.9: + resolution: {integrity: sha512-5UamI7xkUcJ3i9qVDS+KFDEK8/7oJ55/sJMB1Ge7IEapr7KfdfV/HErR+koZwOfd+SgtFKOKRhRakdg++DcJpQ==} engines: {node: '>=6.9.0'} dev: true @@ -628,6 +639,29 @@ packages: transitivePeerDependencies: - supports-color + /@babel/core@7.22.11: + resolution: {integrity: sha512-lh7RJrtPdhibbxndr6/xx0w8+CVlY5FJZiaSz908Fpy+G0xkBFTvwLcKJFF4PJxVfGhVWNebikpWGnOoC71juQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@ampproject/remapping': 2.2.1 + '@babel/code-frame': 7.22.10 + '@babel/generator': 7.22.10 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helpers': 7.22.11 + '@babel/parser': 7.22.11 + '@babel/template': 7.22.5 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + convert-source-map: 1.9.0 + debug: 4.3.4(supports-color@5.5.0) + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: true + /@babel/eslint-parser@7.19.1(@babel/core@7.21.3)(eslint@8.36.0): resolution: {integrity: sha512-AqNf2QWt1rtu2/1rLswy6CDP7H9Oh3mMhk177Y67Rg8d7RD9WfOLLv8CGn6tisFvS2htm86yIe1yLF6I1UDaGQ==} engines: {node: ^10.13.0 || ^12.13.0 || >=14.0.0} @@ -671,7 +705,7 @@ packages: resolution: {integrity: sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.10 + '@babel/types': 7.22.11 dev: true /@babel/helper-builder-binary-assignment-operator-visitor@7.18.9: @@ -682,6 +716,13 @@ packages: '@babel/types': 7.20.7 dev: true + /@babel/helper-builder-binary-assignment-operator-visitor@7.22.10: + resolution: {integrity: sha512-Av0qubwDQxC56DoUReVDeLfMEjYYSN1nZrTUrWkXd7hpU73ymRANkbuDm3yni9npkn+RXy9nNbEJZEzXr7xrfQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-compilation-targets@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-4tGORmfQcrc+bvrjb5y3dG9Mx1IOZjsHqQVUz7XCNHO+iTmqxWnVg3KRygjGmpRLJGdQSKuvFinbIb0CnZwHAQ==} engines: {node: '>=6.9.0'} @@ -695,18 +736,15 @@ packages: lru-cache: 5.1.1 semver: 6.3.0 - /@babel/helper-compilation-targets@7.21.4(@babel/core@7.21.3): - resolution: {integrity: sha512-Fa0tTuOXZ1iL8IeDFUWCzjZcn+sJGd9RZdH9esYVjEejGmzf+FFYQpMi/kZUk2kPy/q1H3/GPw7np8qar/stfg==} + /@babel/helper-compilation-targets@7.22.10: + resolution: {integrity: sha512-JMSwHD4J7SLod0idLq5PKgI+6g/hLD/iuWBq08ZX49xE14VpVEojJ5rHWptpirV2j020MvypRLAXAO50igCJ5Q==} engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.0.0 dependencies: - '@babel/compat-data': 7.21.4 - '@babel/core': 7.21.3 - '@babel/helper-validator-option': 7.21.0 - browserslist: 4.21.4 + '@babel/compat-data': 7.22.9 + '@babel/helper-validator-option': 7.22.5 + browserslist: 4.21.10 lru-cache: 5.1.1 - semver: 6.3.0 + semver: 6.3.1 dev: true /@babel/helper-create-class-features-plugin@7.21.0(@babel/core@7.21.3): @@ -728,6 +766,24 @@ packages: - supports-color dev: true + /@babel/helper-create-class-features-plugin@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-y1grdYL4WzmUDBRGK0pDbIoFd7UZKoDurDzWEoNMYoj1EL+foGRQNyPWDcC+YyegN5y1DUsFFmzjGijB3nSVAQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-member-expression-to-functions': 7.22.5 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-replace-supers': 7.22.9(@babel/core@7.22.11) + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + semver: 6.3.1 + dev: true + /@babel/helper-create-regexp-features-plugin@7.20.5(@babel/core@7.21.3): resolution: {integrity: sha512-m68B1lkg3XDGX5yCvGO0kPx3v9WIYLnzjKfPcQiwntEQa5ZeRkPmo2X/ISJc8qxWGfwUr+kvZAeEzAwLec2r2w==} engines: {node: '>=6.9.0'} @@ -739,6 +795,18 @@ packages: regexpu-core: 5.2.2 dev: true + /@babel/helper-create-regexp-features-plugin@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-+svjVa/tFwsNSG4NEy1h85+HQ5imbT92Q5/bgtS7P0GTQlP8WuFdqsiABmQouhiFGyV66oGxZFpeYHza1rNsKw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + regexpu-core: 5.3.2 + semver: 6.3.1 + dev: true + /@babel/helper-define-polyfill-provider@0.3.3(@babel/core@7.21.3): resolution: {integrity: sha512-z5aQKU4IzbqCC1XH0nAqfsFLMVSo22SBKUc0BxGrLkolTdPTructy0ToNnlO2zA4j9Q/7pjMZf0DSY+DSTYzww==} peerDependencies: @@ -755,10 +823,30 @@ packages: - supports-color dev: true + /@babel/helper-define-polyfill-provider@0.4.2(@babel/core@7.22.11): + resolution: {integrity: sha512-k0qnnOqHn5dK9pZpfD5XXZ9SojAITdCKRn2Lp6rnDGzIbaP0rHyMPk/4wsSxVBVz4RfN0q6VpXWP2pDGIoQ7hw==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + debug: 4.3.4(supports-color@5.5.0) + lodash.debounce: 4.0.8 + resolve: 1.22.4 + transitivePeerDependencies: + - supports-color + dev: true + /@babel/helper-environment-visitor@7.18.9: resolution: {integrity: sha512-3r/aACDJ3fhQ/EVgFy0hpj8oHyHpQc+LPtJoY9SzTThAsStm4Ptegq92vqKoE3vD706ZVFWITnMnxucw+S9Ipg==} engines: {node: '>=6.9.0'} + /@babel/helper-environment-visitor@7.22.5: + resolution: {integrity: sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==} + engines: {node: '>=6.9.0'} + dev: true + /@babel/helper-explode-assignable-expression@7.18.6: resolution: {integrity: sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==} engines: {node: '>=6.9.0'} @@ -773,12 +861,27 @@ packages: '@babel/template': 7.20.7 '@babel/types': 7.21.3 + /@babel/helper-function-name@7.22.5: + resolution: {integrity: sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.5 + '@babel/types': 7.22.11 + dev: true + /@babel/helper-hoist-variables@7.18.6: resolution: {integrity: sha512-UlJQPkFqFULIcyW5sbzgbkxn2FKRgwWiRexcuaR8RNJRy8+LLveqPjwZV/bwrLZCN0eUHD/x8D0heK1ozuoo6Q==} engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.21.3 + /@babel/helper-hoist-variables@7.22.5: + resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-member-expression-to-functions@7.21.0: resolution: {integrity: sha512-Muu8cdZwNN6mRRNG6lAYErJ5X3bRevgYR2O8wN0yn7jJSnGDu6eG59RfT29JHxGUovyfrh6Pj0XzmR7drNVL3Q==} engines: {node: '>=6.9.0'} @@ -786,6 +889,13 @@ packages: '@babel/types': 7.21.4 dev: true + /@babel/helper-member-expression-to-functions@7.22.5: + resolution: {integrity: sha512-aBiH1NKMG0H2cGZqspNvsaBe6wNGjbJjuLy29aU+eDZjSbbN53BaxlpB02xm9v34pLTZ1nIQPFYn2qMZoa5BQQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-module-imports@7.18.6: resolution: {integrity: sha512-0NFvs3VkuSYbFi1x2Vd6tKrywq+z/cLeYC/RJNFrIX/30Bf5aiGYbtvGXolEktzJH8o5E5KJ3tT+nkxuuZFVlA==} engines: {node: '>=6.9.0'} @@ -796,7 +906,7 @@ packages: resolution: {integrity: sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg==} engines: {node: '>=6.9.0'} dependencies: - '@babel/types': 7.22.10 + '@babel/types': 7.22.11 dev: true /@babel/helper-module-transforms@7.21.2: @@ -814,6 +924,34 @@ packages: transitivePeerDependencies: - supports-color + /@babel/helper-module-transforms@7.22.9(@babel/core@7.21.3): + resolution: {integrity: sha512-t+WA2Xn5K+rTeGtC8jCsdAH52bjggG5TKRuRrAGNM/mjIbO4GxvlLMFOEz9wXY5I2XQ60PMFsAG2WIcG82dQMQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-module-imports': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/helper-validator-identifier': 7.22.5 + dev: true + + /@babel/helper-module-transforms@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-t+WA2Xn5K+rTeGtC8jCsdAH52bjggG5TKRuRrAGNM/mjIbO4GxvlLMFOEz9wXY5I2XQ60PMFsAG2WIcG82dQMQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-module-imports': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/helper-validator-identifier': 7.22.5 + dev: true + /@babel/helper-optimise-call-expression@7.18.6: resolution: {integrity: sha512-HP59oD9/fEHQkdcbgFCnbmgH5vIQTJbxh2yf+CdM89/glUNnuzr87Q8GIjGEnOktTROemO0Pe0iPAYbqZuOUiA==} engines: {node: '>=6.9.0'} @@ -821,6 +959,13 @@ packages: '@babel/types': 7.21.3 dev: true + /@babel/helper-optimise-call-expression@7.22.5: + resolution: {integrity: sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-plugin-utils@7.20.2: resolution: {integrity: sha512-8RvlJG2mj4huQ4pZ+rU9lqKi9ZKiRmuvGuM2HlWmkmgOhbs6zEAw6IEiJ5cQqGbDzGZOhwuOQNtZMi/ENLjZoQ==} engines: {node: '>=6.9.0'} @@ -845,6 +990,18 @@ packages: - supports-color dev: true + /@babel/helper-remap-async-to-generator@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-8WWC4oR4Px+tr+Fp0X3RHDVfINGpF3ad1HIbrc8A77epiR6eMMc6jsgozkzT2uDiOOdoS9cLIQ+XD2XvI2WSmQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-wrap-function': 7.22.10 + dev: true + /@babel/helper-replace-supers@7.20.7: resolution: {integrity: sha512-vujDMtB6LVfNW13jhlCrp48QNslK6JXi7lQG736HVbHz/mbf4Dc7tIRh1Xf5C0rF7BP8iiSxGMCmY6Ci1ven3A==} engines: {node: '>=6.9.0'} @@ -859,12 +1016,31 @@ packages: - supports-color dev: true + /@babel/helper-replace-supers@7.22.9(@babel/core@7.22.11): + resolution: {integrity: sha512-LJIKvvpgPOPUThdYqcX6IXRuIcTkcAub0IaDRGCZH0p5GPUp7PhRU9QVgFcDDd51BaPkk77ZjqFwh6DZTAEmGg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-member-expression-to-functions': 7.22.5 + '@babel/helper-optimise-call-expression': 7.22.5 + dev: true + /@babel/helper-simple-access@7.20.2: resolution: {integrity: sha512-+0woI/WPq59IrqDYbVGfshjT5Dmk/nnbdpcF8SnMhhXObpTq2KNBdLFRFrkVdbDOyUmHBCxzm5FHV1rACIkIbA==} engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.21.3 + /@babel/helper-simple-access@7.22.5: + resolution: {integrity: sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-skip-transparent-expression-wrappers@7.20.0: resolution: {integrity: sha512-5y1JYeNKfvnT8sZcK9DVRtpTbGiomYIHviSP3OQWmDPU3DeH4a1ZlT/N2lyQ5P8egjcRaT/Y9aNqUxK0WsnIIg==} engines: {node: '>=6.9.0'} @@ -872,12 +1048,26 @@ packages: '@babel/types': 7.21.3 dev: true + /@babel/helper-skip-transparent-expression-wrappers@7.22.5: + resolution: {integrity: sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-split-export-declaration@7.18.6: resolution: {integrity: sha512-bde1etTx6ZyTmobl9LLMMQsaizFVZrquTEHOqKeQESMKo4PlObf+8+JA25ZsIpZhT/WEd39+vOdLXAFG/nELpA==} engines: {node: '>=6.9.0'} dependencies: '@babel/types': 7.21.3 + /@babel/helper-split-export-declaration@7.22.6: + resolution: {integrity: sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/helper-string-parser@7.19.4: resolution: {integrity: sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==} engines: {node: '>=6.9.0'} @@ -905,6 +1095,11 @@ packages: resolution: {integrity: sha512-rmL/B8/f0mKS2baE9ZpyTcTavvEuWhTTW8amjzXNvYG4AwBsqTLikfXsEofsJEfKHf+HQVQbFOHy6o+4cnC/fQ==} engines: {node: '>=6.9.0'} + /@babel/helper-validator-option@7.22.5: + resolution: {integrity: sha512-R3oB6xlIVKUnxNUxbmgq7pKjxpru24zlimpE8WK47fACIlM0II/Hm1RS8IaOI7NgCr6LNS+jl5l75m20npAziw==} + engines: {node: '>=6.9.0'} + dev: true + /@babel/helper-wrap-function@7.20.5: resolution: {integrity: sha512-bYMxIWK5mh+TgXGVqAtnu5Yn1un+v8DDZtqyzKRLUzrh70Eal2O3aZ7aPYiMADO4uKlkzOiRiZ6GX5q3qxvW9Q==} engines: {node: '>=6.9.0'} @@ -917,6 +1112,15 @@ packages: - supports-color dev: true + /@babel/helper-wrap-function@7.22.10: + resolution: {integrity: sha512-OnMhjWjuGYtdoO3FmsEFWvBStBAe2QOgwOLsLNDjN+aaiMD8InJk1/O3HSD8lkqTjCgg5YI34Tz15KNNA3p+nQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-function-name': 7.22.5 + '@babel/template': 7.22.5 + '@babel/types': 7.22.11 + dev: true + /@babel/helpers@7.21.0: resolution: {integrity: sha512-XXve0CBtOW0pd7MRzzmoyuSj0e3SEzj8pgyFxnTT1NJZL38BD1MK7yYrm8yefRPIDvNNe14xR4FdbHwpInD4rA==} engines: {node: '>=6.9.0'} @@ -927,6 +1131,17 @@ packages: transitivePeerDependencies: - supports-color + /@babel/helpers@7.22.11: + resolution: {integrity: sha512-vyOXC8PBWaGc5h7GMsNx68OH33cypkEDJCHvYVVgVbbxJDROYVtexSk0gK5iCF1xNjRIN2s8ai7hwkWDq5szWg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.5 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + transitivePeerDependencies: + - supports-color + dev: true + /@babel/highlight@7.18.6: resolution: {integrity: sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==} engines: {node: '>=6.9.0'} @@ -967,6 +1182,14 @@ packages: '@babel/types': 7.22.10 dev: true + /@babel/parser@7.22.11: + resolution: {integrity: sha512-R5zb8eJIBPJriQtbH/htEQy4k7E2dHWlD2Y2VT07JCzwYZHBxV5ZYtM0UhXSNMT74LyxuM+b1jdL7pSesXbC/g==} + engines: {node: '>=6.0.0'} + hasBin: true + dependencies: + '@babel/types': 7.22.11 + dev: true + /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-Dgxsyg54Fx1d4Nge8UnvTrED63vrwOdPmyvPzlNN/boaliRP54pm3pGzZD1SJUwrBA+Cs/xdG8kXX6Mn/RfISQ==} engines: {node: '>=6.9.0'} @@ -977,6 +1200,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-NP1M5Rf+u2Gw9qfSO4ihjcTGW5zXTi36ITLd4/EoAcEhIZ0yjMqmftDNl3QC19CX7olhrjpyU454g/2W7X0jvQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-sbr9+wNE5aXMBBFBICk01tt7sBf2Oc9ikRFEcem/ZORup9IMUdNhW7/wVLEbbtlWOsEubJet46mHAL2C8+2jKQ==} engines: {node: '>=6.9.0'} @@ -989,6 +1222,18 @@ packages: '@babel/plugin-proposal-optional-chaining': 7.20.7(@babel/core@7.21.3) dev: true + /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-31Bb65aZaUwqCbWMnZPduIZxCBngHFlzyN6Dq6KAJjtx+lx6ohKHubc61OomYi7XwVD4Ol0XCVz4h+pYFR048g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.13.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-transform-optional-chaining': 7.22.12(@babel/core@7.22.11) + dev: true + /@babel/plugin-proposal-async-generator-functions@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-xMbiLsn/8RK7Wq7VeVytytS2L6qE69bXPB10YCmMdDZbKF4okCqY74pI/jJQ/8U0b/F6NrT2+14b8/P9/3AMGA==} engines: {node: '>=6.9.0'} @@ -1031,20 +1276,6 @@ packages: - supports-color dev: true - /@babel/plugin-proposal-class-static-block@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-XP5G9MWNUskFuP30IfFSEFB0Z6HzLIUcjYM4bYOPHXl7eiJ9HFv8tWj6TXTN5QODiEhDZAeI4hLok2iHFFV4hw==} - engines: {node: '>=6.9.0'} - peerDependencies: - '@babel/core': ^7.12.0 - dependencies: - '@babel/core': 7.21.3 - '@babel/helper-create-class-features-plugin': 7.21.0(@babel/core@7.21.3) - '@babel/helper-plugin-utils': 7.20.2 - '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.21.3) - transitivePeerDependencies: - - supports-color - dev: true - /@babel/plugin-proposal-decorators@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-JB45hbUweYpwAGjkiM7uCyXMENH2lG+9r3G2E+ttc2PRXAoEkpfd/KW5jDg4j8RS6tLtTG1jZi9LbHZVSfs1/A==} engines: {node: '>=6.9.0'} @@ -1171,8 +1402,8 @@ packages: '@babel/core': ^7.0.0-0 dependencies: '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-skip-transparent-expression-wrappers': 7.20.0 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.21.3) dev: true @@ -1204,19 +1435,13 @@ packages: - supports-color dev: true - /@babel/plugin-proposal-private-property-in-object@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-ha4zfehbJjc5MmXBlHec1igel5TJXXLDDRbuJ4+XT2TJcyD9/V1919BA8gMvsdHcNMBy4WBUBiRb3nw/EQUtBw==} + /@babel/plugin-proposal-private-property-in-object@7.21.0-placeholder-for-preset-env.2(@babel/core@7.22.11): + resolution: {integrity: sha512-SOSkfJDddaM7mak6cPEpswyTRnuRltl429hMraQEglW+OkovnCzsiszTmsrlY//qLFjCpQDFRvjdm2wA5pPm9w==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-annotate-as-pure': 7.18.6 - '@babel/helper-create-class-features-plugin': 7.21.0(@babel/core@7.21.3) - '@babel/helper-plugin-utils': 7.20.2 - '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.21.3) - transitivePeerDependencies: - - supports-color + '@babel/core': 7.22.11 dev: true /@babel/plugin-proposal-unicode-property-regex@7.18.6(@babel/core@7.21.3): @@ -1239,6 +1464,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.22.11): + resolution: {integrity: sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-bigint@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==} peerDependencies: @@ -1257,6 +1491,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.22.11): + resolution: {integrity: sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-class-static-block@7.14.5(@babel/core@7.21.3): resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} engines: {node: '>=6.9.0'} @@ -1267,6 +1510,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-class-static-block@7.14.5(@babel/core@7.22.11): + resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-decorators@7.19.0(@babel/core@7.21.3): resolution: {integrity: sha512-xaBZUEDntt4faL1yN8oIFlhfXeQAWJW7CLKYsHTUqriCUbj8xOra8bfxxKGi/UwExPFBuPdH4XfHc9rGQhrVkQ==} engines: {node: '>=6.9.0'} @@ -1286,6 +1539,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-dynamic-import@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-export-namespace-from@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==} peerDependencies: @@ -1295,6 +1557,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-export-namespace-from@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-flow@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-LUbR+KNTBWCUAqRG9ex5Gnzu2IOkt8jRJbHHXFT9q+L9zm7M/QQbEqXyw1n1pohYvOyWC8CjeyjrSaIwiYjK7A==} engines: {node: '>=6.9.0'} @@ -1325,6 +1596,26 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-import-assertions@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-rdV97N7KqsRzeNGoWUOK6yUsWarLjE5Su/Snk9IYPU9CwkWHs4t+rTGOvffTR8XGkJMTAdLfO0xVnXm8wugIJg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-syntax-import-attributes@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-KwvoWDeNKPETmozyFE0P2rOLqh39EoQHNjqizrI5B8Vt0ZNS7M56s7dAiAqbYfiAYOuIzIh96z3iR2ktgu3tEg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.21.3): resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} peerDependencies: @@ -1334,6 +1625,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.22.11): + resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} peerDependencies: @@ -1343,6 +1643,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-jsx@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-6mmljtAedFGTWu2p/8WIORGwy+61PLgOMPOdazc7YoJ9ZCWUyFy3A6CpPkRKLKD1ToAesxX8KGEViAiLo9N+7Q==} engines: {node: '>=6.9.0'} @@ -1371,6 +1680,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.22.11): + resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} peerDependencies: @@ -1380,17 +1698,17 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.21.3): - resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} + /@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 + '@babel/core': 7.22.11 '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.21.3): - resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.21.3): + resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: @@ -1398,6 +1716,33 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.22.11): + resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + + /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.21.3): + resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + + /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} peerDependencies: @@ -1407,6 +1752,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.21.3): resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} peerDependencies: @@ -1416,6 +1770,15 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-private-property-in-object@7.14.5(@babel/core@7.21.3): resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} engines: {node: '>=6.9.0'} @@ -1426,6 +1789,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-private-property-in-object@7.14.5(@babel/core@7.22.11): + resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.21.3): resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} engines: {node: '>=6.9.0'} @@ -1436,6 +1809,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.22.11): + resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.20.2 + dev: true + /@babel/plugin-syntax-typescript@7.20.0(@babel/core@7.21.3): resolution: {integrity: sha512-rd9TkG+u1CExzS4SM1BlMEhMXwFLKVjOAFFCDx9PbX5ycJWDoWMcwdJH9RhkPu1dOgn5TrxLot/Gx6lWFuAUNQ==} engines: {node: '>=6.9.0'} @@ -1456,6 +1839,17 @@ packages: '@babel/helper-plugin-utils': 7.22.5 dev: true + /@babel/plugin-syntax-unicode-sets-regex@7.18.6(@babel/core@7.22.11): + resolution: {integrity: sha512-727YkEAPwSIQTv5im8QHz3upqp92JTWhidIC81Tdx4VJYIte/VndKf1qKrfnnhPLiPghStWfvC/iFaMCQu7Nqg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-arrow-functions@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-3poA5E7dzDomxj9WXWwuD6A5F3kc7VXwIJO+E+J8qtDtS+pXPAhrgEyh+9GBwBgPq1Z+bB+/JD60lp5jsN7JPQ==} engines: {node: '>=6.9.0'} @@ -1466,6 +1860,29 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-arrow-functions@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-26lTNXoVRdAnsaDXPpvCNUq+OVWEVC6bx7Vvz9rC53F2bagUWW4u4ii2+h8Fejfh7RYqPxn+libeFBBck9muEw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-async-generator-functions@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-0pAlmeRJn6wU84zzZsEOx1JV1Jf8fqO9ok7wofIJwUnplYo247dcd24P+cMJht7ts9xkzdtB0EPHmOb7F+KzXw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.9(@babel/core@7.22.11) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-async-to-generator@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-Uo5gwHPT9vgnSXQxqGtpdufUiWp96gk7yiP4Mp5bm1QMkEmLXBO7PAGYbKoJ6DhAwiNkcHFBol/x5zZZkL/t0Q==} engines: {node: '>=6.9.0'} @@ -1480,6 +1897,18 @@ packages: - supports-color dev: true + /@babel/plugin-transform-async-to-generator@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-b1A8D8ZzE/VhNDoV1MSJTnpKkCG5bJo+19R4o4oy03zM7ws8yEMK755j61Dc3EyvdysbqH5BOOTquJ7ZX9C6vQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-imports': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.9(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-block-scoped-functions@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-ExUcOqpPWnliRcPqves5HJcJOvHvIIWfuS4sroBUenPuMdmW+SMHDakmtS7qOo13sVppmUijqeTv7qqGsvURpQ==} engines: {node: '>=6.9.0'} @@ -1490,6 +1919,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-block-scoped-functions@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-tdXZ2UdknEKQWKJP1KMNmuF5Lx3MymtMN/pvA+p/VEkhK8jVcQ1fzSy8KM9qRYhAf2/lV33hoMPKI/xaI9sADA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-block-scoping@7.20.11(@babel/core@7.21.3): resolution: {integrity: sha512-tA4N427a7fjf1P0/2I4ScsHGc5jcHPbb30xMbaTke2gxDuWpUfXDuX1FEymJwKk4tuGUvGcejAR6HdZVqmmPyw==} engines: {node: '>=6.9.0'} @@ -1500,14 +1939,37 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-block-scoping@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-Mdrbunoh9SxwFZapeHVrwFmri16+oYotcZysSzhNIVDwIAb1UV+kvnxULSYq9J3/q5MDG+4X6w8QVgD1zhBXNQ==} + /@babel/plugin-transform-block-scoping@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-1+kVpGAOOI1Albt6Vse7c8pHzcZQdQKW+wJH+g8mCaszOdDVwRXa/slHPqIw+oJAJANTKDMuM2cBdV0Dg618Vg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-class-properties@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-nDkQ0NfkOhPTq8YCLiWNxp1+f9fCobEjCb0n8WdbNUBc4IB5V7P1QnX9IjpSoquKrXF5SKojHleVNs2vGeHCHQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-class-static-block@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-GMM8gGmqI7guS/llMFk1bJDkKfn3v3C4KHK9Yg1ey5qcHcOlKb0QvcMrgzvxo+T03/4szNh5lghY+fEC98Kq9g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.12.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.22.11) dev: true /@babel/plugin-transform-classes@7.20.7(@babel/core@7.21.3): @@ -1530,24 +1992,22 @@ packages: - supports-color dev: true - /@babel/plugin-transform-classes@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-RZhbYTCEUAe6ntPehC4hlslPWosNHDox+vAs4On/mCLRLfoDVHf6hVEd7kuxr1RnHwJmxFfUM3cZiZRmPxJPXQ==} + /@babel/plugin-transform-classes@7.22.6(@babel/core@7.22.11): + resolution: {integrity: sha512-58EgM6nuPNG6Py4Z3zSuu0xWu2VfodiMi72Jt5Kj2FECmaYk1RrTXA45z6KBFsu9tRgwQDwIiY4FXTt+YsSFAQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-annotate-as-pure': 7.18.6 - '@babel/helper-compilation-targets': 7.21.4(@babel/core@7.21.3) - '@babel/helper-environment-visitor': 7.18.9 - '@babel/helper-function-name': 7.21.0 - '@babel/helper-optimise-call-expression': 7.18.6 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-replace-supers': 7.20.7 - '@babel/helper-split-export-declaration': 7.18.6 + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.9(@babel/core@7.22.11) + '@babel/helper-split-export-declaration': 7.22.6 globals: 11.12.0 - transitivePeerDependencies: - - supports-color dev: true /@babel/plugin-transform-computed-properties@7.20.7(@babel/core@7.21.3): @@ -1561,6 +2021,17 @@ packages: '@babel/template': 7.20.7 dev: true + /@babel/plugin-transform-computed-properties@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-4GHWBgRf0krxPX+AaPtgBAlTgTeZmqDynokHOX7aqqAB4tHs3U2Y02zH6ETFdLZGcg9UQSD1WCmkVrE9ErHeOg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/template': 7.22.5 + dev: true + /@babel/plugin-transform-destructuring@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-Xwg403sRrZb81IVB79ZPqNQME23yhugYVqgTxAhT99h485F4f+GMELFhhOsscDUB7HCswepKeCKLn/GZvUKoBA==} engines: {node: '>=6.9.0'} @@ -1571,14 +2042,14 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-destructuring@7.21.3(@babel/core@7.21.3): - resolution: {integrity: sha512-bp6hwMFzuiE4HqYEyoGJ/V2LeIWn+hLVKc4pnj++E5XQptwhtcGmSayM029d/j2X1bPKGTlsyPwAubuU22KhMA==} + /@babel/plugin-transform-destructuring@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-dPJrL0VOyxqLM9sritNbMSGx/teueHF/htMKrPT7DNxccXxRDPYqlgPFFdr8u+F+qUZOkZoXue/6rL5O5GduEw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 dev: true /@babel/plugin-transform-dotall-regex@7.18.6(@babel/core@7.21.3): @@ -1592,6 +2063,17 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-dotall-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-5/Yk9QxCQCl+sOIB1WelKnVRxTJDSAIxtJLL2/pqL14ZVlbH0fUQUZa/T5/UnQtBNgghR7mfB8ERBKyKPCi7Vw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-duplicate-keys@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-d2bmXCtZXYc59/0SanQKbiWINadaJXqtvIQIzd4+hNwkWBgyCd5F/2t1kXoUdvPMrxzPvhK6EMQRROxsue+mfw==} engines: {node: '>=6.9.0'} @@ -1602,6 +2084,27 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-duplicate-keys@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-dEnYD+9BBgld5VBXHnF/DbYGp3fqGMsyxKbtD1mDyIA7AkTSpKXFhCVuj/oQVOoALfBs77DudA0BE4d5mcpmqw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-dynamic-import@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-g/21plo58sfteWjaO0ZNVb+uEOkJNjAaHhbejrnBmu011l/eNDScmkbjCC3l4FKb10ViaGU4aOkFznSu2zRHgA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-exponentiation-operator@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-wzEtc0+2c88FVR34aQmiz56dxEkxr2g8DQb/KfaFa1JYXOFVsbhvAonFN6PwVWj++fKmku8NP80plJ5Et4wqHw==} engines: {node: '>=6.9.0'} @@ -1613,6 +2116,28 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-exponentiation-operator@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-vIpJFNM/FjZ4rh1myqIya9jXwrwwgFRHPjT3DkUA9ZLHuzox8jiXkOLvwm1H+PQIP3CqfC++WPKeuDi0Sjdj1g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-builder-binary-assignment-operator-visitor': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-export-namespace-from@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-xa7aad7q7OiT8oNZ1mU7NrISjlSkVdMbNxn9IuLZyL9AJEhs1Apba3I+u5riX1dIkdptP5EKDG5XDPByWxtehw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-flow-strip-types@7.19.0(@babel/core@7.21.3): resolution: {integrity: sha512-sgeMlNaQVbCSpgLSKP4ZZKfsJVnFnNQlUSk6gPYzR/q7tzCgQF2t8RBKAP6cKJeZdveei7Q7Jm527xepI8lNLg==} engines: {node: '>=6.9.0'} @@ -1624,6 +2149,17 @@ packages: '@babel/plugin-syntax-flow': 7.18.6(@babel/core@7.21.3) dev: true + /@babel/plugin-transform-flow-strip-types@7.22.5(@babel/core@7.21.3): + resolution: {integrity: sha512-tujNbZdxdG0/54g/oua8ISToaXTFBf8EnSb5PgQSciIXWOWKX3S4+JR7ZE9ol8FZwf9kxitzkGQ+QWeov/mCiA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-flow': 7.22.5(@babel/core@7.21.3) + dev: true + /@babel/plugin-transform-for-of@7.18.8(@babel/core@7.21.3): resolution: {integrity: sha512-yEfTRnjuskWYo0k1mHUqrVWaZwrdq8AYbfrpqULOJOaucGSp4mNMVps+YtA8byoevxS/urwU75vyhQIxcCgiBQ==} engines: {node: '>=6.9.0'} @@ -1634,14 +2170,14 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-for-of@7.21.0(@babel/core@7.21.3): - resolution: {integrity: sha512-LlUYlydgDkKpIY7mcBWvyPPmMcOphEyYA27Ef4xpbh1IiDNLr0kZsos2nf92vz3IccvJI25QUwp86Eo5s6HmBQ==} + /@babel/plugin-transform-for-of@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-3kxQjX1dU9uudwSshyLeEipvrLjBCVthCgeTp6CzE/9JYrlAIaeekVxRpCWsDDfYTfRZRoCeZatCQvwo+wvK8A==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 dev: true /@babel/plugin-transform-function-name@7.18.9(@babel/core@7.21.3): @@ -1656,6 +2192,29 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-function-name@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-UIzQNMS0p0HHiQm3oelztj+ECwFnj+ZRV4KnguvlsD2of1whUeM6o7wGNj6oLwcDoAXQ8gEqfgC24D+VdIcevg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-json-strings@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-CxT5tCqpA9/jXFlme9xIBCc5RPtdDq3JpkkhgHQqtDdiTnTI0jtZ0QzXhr5DILeYifDPp2wvY2ad+7+hLMW5Pw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-literals@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-IFQDSRoTPnrAIrI5zoZv73IFeZu2dhu6irxQjY9rNjTT53VmKg9fenjvoiOWOkJ6mm4jKVPtdMzBY98Fp4Z4cg==} engines: {node: '>=6.9.0'} @@ -1666,6 +2225,27 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-fTLj4D79M+mepcw3dgFBTIDYpbcB9Sm0bpm4ppXPaO+U+PKFFyV9MGRvS0gvGw62sd10kT5lRMKXAADb9pWy8g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-logical-assignment-operators@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-qQwRTP4+6xFCDV5k7gZBF3C31K34ut0tbEcTKxlX/0KXxm9GLcO14p570aWxFvVzx6QAfPgq7gaeIHXJC8LswQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-member-expression-literals@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-qSF1ihLGO3q+/g48k85tUjD033C29TNTVB2paCwZPVmOsjn9pClvYYrM2VeJpBY2bcNkuny0YUyTNRyRxJ54KA==} engines: {node: '>=6.9.0'} @@ -1676,6 +2256,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-member-expression-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-RZEdkNtzzYCFl9SE9ATaUMTj2hqMb4StarOJLrZRbqqU4HSBE7UlBw9WBWQiDzrJZJdUWiMTVDI6Gv/8DPvfew==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-modules-amd@7.20.11(@babel/core@7.21.3): resolution: {integrity: sha512-NuzCt5IIYOW0O30UvqktzHYR2ud5bOWbY0yaxWZ6G+aFzOMJvrs5YHNikrbdaT15+KNO31nPOy5Fim3ku6Zb5g==} engines: {node: '>=6.9.0'} @@ -1689,6 +2279,17 @@ packages: - supports-color dev: true + /@babel/plugin-transform-modules-amd@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-R+PTfLTcYEmb1+kK7FNkhQ1gP4KgjpSO6HfH9+f8/yfp2Nt3ggBjiVpRwmwTlfqZLafYKJACy36yDXlEmI9HjQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-modules-commonjs@7.20.11(@babel/core@7.21.3): resolution: {integrity: sha512-S8e1f7WQ7cimJQ51JkAaDrEtohVEitXjgCGAS2N8S31Y42E+kWwfSz83LYz57QdBm7q9diARVqanIaH2oVgQnw==} engines: {node: '>=6.9.0'} @@ -1703,18 +2304,28 @@ packages: - supports-color dev: true - /@babel/plugin-transform-modules-commonjs@7.21.2(@babel/core@7.21.3): - resolution: {integrity: sha512-Cln+Yy04Gxua7iPdj6nOV96smLGjpElir5YwzF0LBPKoPlLDNJePNlrGGaybAJkd0zKRnOVXOgizSqPYMNYkzA==} + /@babel/plugin-transform-modules-commonjs@7.22.11(@babel/core@7.21.3): + resolution: {integrity: sha512-o2+bg7GDS60cJMgz9jWqRUsWkMzLCxp+jFDeDUT5sjRlAxcJWZ2ylNdI7QQ2+CH5hWu7OnN+Cv3htt7AkSf96g==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: '@babel/core': 7.21.3 - '@babel/helper-module-transforms': 7.21.2 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-simple-access': 7.20.2 - transitivePeerDependencies: - - supports-color + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.21.3) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + dev: true + + /@babel/plugin-transform-modules-commonjs@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-o2+bg7GDS60cJMgz9jWqRUsWkMzLCxp+jFDeDUT5sjRlAxcJWZ2ylNdI7QQ2+CH5hWu7OnN+Cv3htt7AkSf96g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-simple-access': 7.22.5 dev: true /@babel/plugin-transform-modules-systemjs@7.20.11(@babel/core@7.21.3): @@ -1732,6 +2343,19 @@ packages: - supports-color dev: true + /@babel/plugin-transform-modules-systemjs@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-rIqHmHoMEOhI3VkVf5jQ15l539KrwhzqcBO6wdCNWPWc/JWt9ILNYNUssbRpeq0qWns8svuw8LnMNCvWBIJ8wA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-identifier': 7.22.5 + dev: true + /@babel/plugin-transform-modules-umd@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-dcegErExVeXcRqNtkRU/z8WlBLnvD4MRnHgNs3MytRO1Mn1sHRyhbcpYbVMGclAqOjdW+9cfkdZno9dFdfKLfQ==} engines: {node: '>=6.9.0'} @@ -1745,6 +2369,17 @@ packages: - supports-color dev: true + /@babel/plugin-transform-modules-umd@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-+S6kzefN/E1vkSsKx8kmQuqeQsvCKCd1fraCM7zXm4SFoggI099Tr4G8U81+5gtMdUeMQ4ipdQffbKLX0/7dBQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-module-transforms': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-named-capturing-groups-regex@7.20.5(@babel/core@7.21.3): resolution: {integrity: sha512-mOW4tTzi5iTLnw+78iEq3gr8Aoq4WNRGpmSlrogqaiCBoR1HFhpU4JkpQFOHfeYx3ReVIFWOQJS4aZBRvuZ6mA==} engines: {node: '>=6.9.0'} @@ -1756,6 +2391,17 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-named-capturing-groups-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-YgLLKmS3aUBhHaxp5hi1WJTgOUb/NCuDHzGT9z9WTt3YG+CPRhJs6nprbStx6DnWM4dh6gt7SU3sZodbZ08adQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-new-target@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-DjwFA/9Iu3Z+vrAn+8pBUGcjhxKguSMlsFqeCKbhb9BAV756v0krzVK04CRDi/4aqmk8BsHb4a/gFcaA5joXRw==} engines: {node: '>=6.9.0'} @@ -1766,19 +2412,99 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-object-super@7.18.6(@babel/core@7.21.3): - resolution: {integrity: sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==} + /@babel/plugin-transform-new-target@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-AsF7K0Fx/cNKVyk3a+DW0JLo+Ua598/NxMRvxDnkpCIGFh43+h/v2xyhRUYf6oD8gE4QtL83C7zZVghMjHd+iw==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-nullish-coalescing-operator@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-YZWOw4HxXrotb5xsjMJUDlLgcDXSfO9eCmdl1bgW4+/lAGdkjaEvOnQ4p5WKKdUgSzO39dgPl0pTnfxm0OAXcg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-numeric-separator@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-3dzU4QGPsILdJbASKhF/V2TVP+gJya1PsueQCxIPCEcerqF21oEcrob4mzjsp2Py/1nLfF5m+xYNMDpmA8vffg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-object-rest-spread@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-nX8cPFa6+UmbepISvlf5jhQyaC7ASs/7UxHmMkuJ/k5xSHvDPPaibMo+v3TXwU/Pjqhep/nFNpd3zn4YR59pnw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.22.9 + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-transform-parameters': 7.22.5(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-object-super@7.18.6(@babel/core@7.21.3): + resolution: {integrity: sha512-uvGz6zk+pZoS1aTZrOvrbj6Pp/kK2mp45t2B+bTDre2UgsZZ8EZLSJtUg7m/no0zOJUWgFONpB7Zv9W2tSaFlA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.21.3 + '@babel/helper-plugin-utils': 7.20.2 '@babel/helper-replace-supers': 7.20.7 transitivePeerDependencies: - supports-color dev: true + /@babel/plugin-transform-object-super@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-klXqyaT9trSjIUrcsYIfETAzmOEZL3cBYqOYLJxBHfMFFggmXOv+NYSX/Jbs9mzMVESw/WycLFPRx8ba/b2Ipw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.9(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-optional-catch-binding@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-rli0WxesXUeCJnMYhzAglEjLWVDF6ahb45HuprcmQuLidBJFWjNnOzssk2kuc6e33FlLaiZhG/kUIzUMWdBKaQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.22.11) + dev: true + + /@babel/plugin-transform-optional-chaining@7.22.12(@babel/core@7.22.11): + resolution: {integrity: sha512-7XXCVqZtyFWqjDsYDY4T45w4mlx1rf7aOgkc/Ww76xkgBiOlmjPkx36PBLHa1k1rwWvVgYMPsbuVnIamx2ZQJw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.22.11) + dev: true + /@babel/plugin-transform-parameters@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-WiWBIkeHKVOSYPO0pWkxGPfKeWrCJyD3NJ53+Lrp/QMSZbsVPovrVl2aWZ19D/LTVnaDv5Ap7GJ/B2CTOZdrfA==} engines: {node: '>=6.9.0'} @@ -1789,14 +2515,38 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true - /@babel/plugin-transform-parameters@7.21.3(@babel/core@7.21.3): - resolution: {integrity: sha512-Wxc+TvppQG9xWFYatvCGPvZ6+SIUxQ2ZdiBP+PHYMIjnPXD+uThCshaz4NZOnODAtBjjcVQQ/3OKs9LW28purQ==} + /@babel/plugin-transform-parameters@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-AVkFUBurORBREOmHRKo06FjHYgjrabpdqRSwq6+C7R5iTCZOsM4QbcB27St0a4U6fffyAOqh3s/qEfybAhfivg==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-private-methods@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-PPjh4gyrQnGe97JTalgRGMuU4icsZFnWkzicB/fUtzlKUqvsWBKEpPPfr5a2JiyirZkHxnAqkQMO5Z5B2kK3fA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-private-property-in-object@7.22.11(@babel/core@7.22.11): + resolution: {integrity: sha512-sSCbqZDBKHetvjSwpyWzhuHkmW5RummxJBVbYLkGkaiTOWGxml7SXt0iWa03bzxFIx7wOj3g/ILRd0RcJKBeSQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.22.11(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.22.11) dev: true /@babel/plugin-transform-property-literals@7.18.6(@babel/core@7.21.3): @@ -1809,6 +2559,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-property-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-TiOArgddK3mK/x1Qwf5hay2pxI6wCZnvQqrFSqbtg1GLl2JcNMitVH/YnqjP+M31pLUeTfzY1HAXFDnUBV30rQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-react-display-name@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-TV4sQ+T013n61uMoygyMRm+xf04Bd5oqFpv2jAEQwSZ8NwQA7zeRPg1LMVg2PWi3zWBz+CLKD+v5bcpZ/BS0aA==} engines: {node: '>=6.9.0'} @@ -1874,7 +2634,7 @@ packages: '@babel/helper-module-imports': 7.22.5 '@babel/helper-plugin-utils': 7.22.5 '@babel/plugin-syntax-jsx': 7.22.5(@babel/core@7.21.3) - '@babel/types': 7.22.10 + '@babel/types': 7.22.11 dev: true /@babel/plugin-transform-react-pure-annotations@7.18.6(@babel/core@7.21.3): @@ -1899,6 +2659,17 @@ packages: regenerator-transform: 0.15.1 dev: true + /@babel/plugin-transform-regenerator@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-F28b1mDt8KcT5bUyJc/U9nwzw6cV+UmTeRlXYIl2TNqMMJif0Jeey9/RQ3C4NOd2zp0/TRsDns9ttj2L523rsw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + regenerator-transform: 0.15.2 + dev: true + /@babel/plugin-transform-reserved-words@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-oX/4MyMoypzHjFrT1CdivfKZ+XvIPMFXwwxHp/r0Ddy2Vuomt4HDFGmft1TAY2yiTKiNSsh3kjBAzcM8kSdsjA==} engines: {node: '>=6.9.0'} @@ -1909,6 +2680,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-reserved-words@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-DTtGKFRQUDm8svigJzZHzb/2xatPc6TzNvAIJ5GqOKDsGFYgAskjRulbR/vGsPKq3OPqtexnz327qYpP57RFyA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-runtime@7.19.6(@babel/core@7.21.3): resolution: {integrity: sha512-PRH37lz4JU156lYFW1p8OxE5i7d6Sl/zV58ooyr+q1J1lnQPyg5tIiXlIwNVhJaY4W3TmOtdc8jqdXQcB1v5Yw==} engines: {node: '>=6.9.0'} @@ -1936,6 +2717,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-shorthand-properties@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-vM4fq9IXHscXVKzDv5itkO1X52SmdFBFcMIBZ2FRn2nqVYqw6dBexUgMvAjHW+KXpPPViD/Yo3GrDEBaRC0QYA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-spread@7.20.7(@babel/core@7.21.3): resolution: {integrity: sha512-ewBbHQ+1U/VnH1fxltbJqDeWBU1oNLG8Dj11uIv3xVf7nrQu0bPGe5Rf716r7K5Qz+SqtAOVswoVunoiBtGhxw==} engines: {node: '>=6.9.0'} @@ -1947,6 +2738,17 @@ packages: '@babel/helper-skip-transparent-expression-wrappers': 7.20.0 dev: true + /@babel/plugin-transform-spread@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-5ZzDQIGyvN4w8+dMmpohL6MBo+l2G7tfC/O2Dg7/hjpgeWvUx8FzfeOKxGog9IimPa4YekaQ9PlDqTLOljkcxg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + dev: true + /@babel/plugin-transform-sticky-regex@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-kfiDrDQ+PBsQDO85yj1icueWMfGfJFKN1KCkndygtu/C9+XUfydLC8Iv5UYJqRwy4zk8EcplRxEOeLyjq1gm6Q==} engines: {node: '>=6.9.0'} @@ -1957,6 +2759,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-sticky-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-zf7LuNpHG0iEeiyCNwX4j3gDg1jgt1k3ZdXBKbZSoA3BbGQGvMiSvfbZRR3Dr3aeJe3ooWFZxOOG3IRStYp2Bw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-template-literals@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-S8cOWfT82gTezpYOiVaGHrCbhlHgKhQt8XH5ES46P2XWmX92yisoZywf5km75wv5sYcXDUCLMmMxOLCtthDgMA==} engines: {node: '>=6.9.0'} @@ -1967,6 +2779,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-template-literals@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-5ciOehRNf+EyUeewo8NkbQiUs4d6ZxiHo6BcBcnFlgiJfu16q0bQUw9Jvo0b0gBKFG1SMhDSjeKXSYuJLeFSMA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-typeof-symbol@7.18.9(@babel/core@7.21.3): resolution: {integrity: sha512-SRfwTtF11G2aemAZWivL7PD+C9z52v9EvMqH9BuYbabyPuKUvSWks3oCg6041pT925L4zVFqaVBeECwsmlguEw==} engines: {node: '>=6.9.0'} @@ -1977,6 +2799,16 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-typeof-symbol@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-bYkI5lMzL4kPii4HHEEChkD0rkc+nvnlR6+o/qdqR6zrm0Sv/nodmyLhlq2DO0YKLUNd2VePmPRjJXSBh9OIdA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-typescript@7.21.3(@babel/core@7.21.3): resolution: {integrity: sha512-RQxPz6Iqt8T0uw/WsJNReuBpWpBqs/n7mNo18sKLoTbMp+UrEekhH+pKSVC7gWz+DNjo9gryfV8YzCiT45RgMw==} engines: {node: '>=6.9.0'} @@ -2002,6 +2834,27 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-unicode-escapes@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-lRfaRKGZCBqDlRU3UIFovdp9c9mEvlylmpod0/OatICsSfuQ9YFthRo1tpTkGsklEefZdqlEFdY4A2dwTb6ohg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-unicode-property-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-HCCIb+CbJIAE6sXn5CjFQXMwkCClcOfPCzTlilJ8cUatfzwHlWQkbtV0zD338u9dZskwvuOYTuuaMaA8J5EI5A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/plugin-transform-unicode-regex@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-gE7A6Lt7YLnNOL3Pb9BNeZvi+d8l7tcRrG4+pwJjK9hD2xX4mEvjlQW60G9EEmfXVYRPv9VRQcyegIVHCql/AA==} engines: {node: '>=6.9.0'} @@ -2013,6 +2866,28 @@ packages: '@babel/helper-plugin-utils': 7.20.2 dev: true + /@babel/plugin-transform-unicode-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-028laaOKptN5vHJf9/Arr/HiJekMd41hOEZYvNsrsXqJ7YPYuX2bQxh31fkZzGmq3YqHRJzYFFAVYvKfMPKqyg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + + /@babel/plugin-transform-unicode-sets-regex@7.22.5(@babel/core@7.22.11): + resolution: {integrity: sha512-lhMfi4FC15j13eKrh3DnYHjpGj6UKQHtNKTbtc1igvAhRy4+kLhV07OpLcsN0VgDEw/MjAvJO4BdMJsHwMhzCg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-create-regexp-features-plugin': 7.22.9(@babel/core@7.22.11) + '@babel/helper-plugin-utils': 7.22.5 + dev: true + /@babel/preset-env@7.20.2(@babel/core@7.21.3): resolution: {integrity: sha512-1G0efQEWR1EHkKvKHqbG+IN/QdgwfByUpM5V5QroDzGV2t3S/WXNQd693cHiHTlCFMpr9B6FkPFXDA2lQcKoDg==} engines: {node: '>=6.9.0'} @@ -2099,102 +2974,107 @@ packages: - supports-color dev: true - /@babel/preset-env@7.21.4(@babel/core@7.21.3): - resolution: {integrity: sha512-2W57zHs2yDLm6GD5ZpvNn71lZ0B/iypSdIeq25OurDKji6AdzV07qp4s3n1/x5BqtiGaTrPN3nerlSCaC5qNTw==} + /@babel/preset-env@7.22.10(@babel/core@7.22.11): + resolution: {integrity: sha512-riHpLb1drNkpLlocmSyEg4oYJIQFeXAK/d7rI6mbD0XsvoTOOweXDmQPG/ErxsEhWk3rl3Q/3F6RFQlVFS8m0A==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: - '@babel/compat-data': 7.21.4 - '@babel/core': 7.21.3 - '@babel/helper-compilation-targets': 7.21.4(@babel/core@7.21.3) - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-validator-option': 7.21.0 - '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-async-generator-functions': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-class-properties': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-class-static-block': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-proposal-dynamic-import': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-export-namespace-from': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-proposal-json-strings': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-logical-assignment-operators': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-nullish-coalescing-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-numeric-separator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-object-rest-spread': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-catch-binding': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-chaining': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-proposal-private-methods': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-private-property-in-object': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-proposal-unicode-property-regex': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.21.3) - '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.21.3) - '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.21.3) - '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-import-assertions': 7.20.0(@babel/core@7.21.3) - '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.21.3) - '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.21.3) - '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.21.3) - '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.21.3) - '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.21.3) - '@babel/plugin-transform-arrow-functions': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-async-to-generator': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-block-scoped-functions': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-block-scoping': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-transform-classes': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-transform-computed-properties': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-destructuring': 7.21.3(@babel/core@7.21.3) - '@babel/plugin-transform-dotall-regex': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-duplicate-keys': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-exponentiation-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-for-of': 7.21.0(@babel/core@7.21.3) - '@babel/plugin-transform-function-name': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-literals': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-member-expression-literals': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-modules-amd': 7.20.11(@babel/core@7.21.3) - '@babel/plugin-transform-modules-commonjs': 7.21.2(@babel/core@7.21.3) - '@babel/plugin-transform-modules-systemjs': 7.20.11(@babel/core@7.21.3) - '@babel/plugin-transform-modules-umd': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-named-capturing-groups-regex': 7.20.5(@babel/core@7.21.3) - '@babel/plugin-transform-new-target': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-object-super': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-parameters': 7.21.3(@babel/core@7.21.3) - '@babel/plugin-transform-property-literals': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-regenerator': 7.20.5(@babel/core@7.21.3) - '@babel/plugin-transform-reserved-words': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-shorthand-properties': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-spread': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-sticky-regex': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-transform-template-literals': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-typeof-symbol': 7.18.9(@babel/core@7.21.3) - '@babel/plugin-transform-unicode-escapes': 7.18.10(@babel/core@7.21.3) - '@babel/plugin-transform-unicode-regex': 7.18.6(@babel/core@7.21.3) - '@babel/preset-modules': 0.1.5(@babel/core@7.21.3) - '@babel/types': 7.21.4 - babel-plugin-polyfill-corejs2: 0.3.3(@babel/core@7.21.3) - babel-plugin-polyfill-corejs3: 0.6.0(@babel/core@7.21.3) - babel-plugin-polyfill-regenerator: 0.4.1(@babel/core@7.21.3) - core-js-compat: 3.27.1 - semver: 6.3.0 + '@babel/compat-data': 7.22.9 + '@babel/core': 7.22.11 + '@babel/helper-compilation-targets': 7.22.10 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.22.5 + '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-proposal-private-property-in-object': 7.21.0-placeholder-for-preset-env.2(@babel/core@7.22.11) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.22.11) + '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.22.11) + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.22.11) + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-import-assertions': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-syntax-import-attributes': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.22.11) + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.22.11) + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.22.11) + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.22.11) + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.22.11) + '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.22.11) + '@babel/plugin-syntax-unicode-sets-regex': 7.18.6(@babel/core@7.22.11) + '@babel/plugin-transform-arrow-functions': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-async-generator-functions': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-async-to-generator': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-block-scoped-functions': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-block-scoping': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-class-properties': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-class-static-block': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-classes': 7.22.6(@babel/core@7.22.11) + '@babel/plugin-transform-computed-properties': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-destructuring': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-dotall-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-duplicate-keys': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-dynamic-import': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-exponentiation-operator': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-export-namespace-from': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-for-of': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-function-name': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-json-strings': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-logical-assignment-operators': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-member-expression-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-modules-amd': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-modules-commonjs': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-modules-systemjs': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-modules-umd': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-named-capturing-groups-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-new-target': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-nullish-coalescing-operator': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-numeric-separator': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-object-rest-spread': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-object-super': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-optional-catch-binding': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-optional-chaining': 7.22.12(@babel/core@7.22.11) + '@babel/plugin-transform-parameters': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-private-methods': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-private-property-in-object': 7.22.11(@babel/core@7.22.11) + '@babel/plugin-transform-property-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-regenerator': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-reserved-words': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-shorthand-properties': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-spread': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-sticky-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-template-literals': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-typeof-symbol': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-escapes': 7.22.10(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-property-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-regex': 7.22.5(@babel/core@7.22.11) + '@babel/plugin-transform-unicode-sets-regex': 7.22.5(@babel/core@7.22.11) + '@babel/preset-modules': 0.1.6-no-external-plugins(@babel/core@7.22.11) + '@babel/types': 7.22.11 + babel-plugin-polyfill-corejs2: 0.4.5(@babel/core@7.22.11) + babel-plugin-polyfill-corejs3: 0.8.3(@babel/core@7.22.11) + babel-plugin-polyfill-regenerator: 0.5.2(@babel/core@7.22.11) + core-js-compat: 3.32.1 + semver: 6.3.1 transitivePeerDependencies: - supports-color dev: true - /@babel/preset-flow@7.18.6(@babel/core@7.21.3): - resolution: {integrity: sha512-E7BDhL64W6OUqpuyHnSroLnqyRTcG6ZdOBl1OKI/QK/HJfplqK/S3sq1Cckx7oTodJ5yOXyfw7rEADJ6UjoQDQ==} + /@babel/preset-flow@7.22.5(@babel/core@7.21.3): + resolution: {integrity: sha512-ta2qZ+LSiGCrP5pgcGt8xMnnkXQrq8Sa4Ulhy06BOlF5QbLw9q5hIx7bn5MrsvyTGAfh6kTOo07Q+Pfld/8Y5Q==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 dependencies: '@babel/core': 7.21.3 - '@babel/helper-plugin-utils': 7.20.2 - '@babel/helper-validator-option': 7.21.0 - '@babel/plugin-transform-flow-strip-types': 7.19.0(@babel/core@7.21.3) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.22.5 + '@babel/plugin-transform-flow-strip-types': 7.22.5(@babel/core@7.21.3) dev: true /@babel/preset-modules@0.1.5(@babel/core@7.21.3): @@ -2210,6 +3090,17 @@ packages: esutils: 2.0.3 dev: true + /@babel/preset-modules@0.1.6-no-external-plugins(@babel/core@7.22.11): + resolution: {integrity: sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==} + peerDependencies: + '@babel/core': ^7.0.0-0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/types': 7.22.11 + esutils: 2.0.3 + dev: true + /@babel/preset-react@7.18.6(@babel/core@7.21.3): resolution: {integrity: sha512-zXr6atUmyYdiWRVLOZahakYmOBHtWc2WGCkP8PYTgZi0iJXDY2CN180TdrIW4OGOAdLc7TifzDIvtx6izaRIzg==} engines: {node: '>=6.9.0'} @@ -2239,8 +3130,8 @@ packages: - supports-color dev: true - /@babel/register@7.18.9(@babel/core@7.21.3): - resolution: {integrity: sha512-ZlbnXDcNYHMR25ITwwNKT88JiaukkdVj/nG7r3wnuXkOTHc60Uy05PwMCPre0hSkY68E6zK3xz+vUJSP2jWmcw==} + /@babel/register@7.22.5(@babel/core@7.21.3): + resolution: {integrity: sha512-vV6pm/4CijSQ8Y47RH5SopXzursN35RQINfGJkmOlcpAtGuf94miFvIPhCKGQN7WGIcsgG1BHEX2KVdTYwTwUQ==} engines: {node: '>=6.9.0'} peerDependencies: '@babel/core': ^7.0.0-0 @@ -2249,10 +3140,14 @@ packages: clone-deep: 4.0.1 find-cache-dir: 2.1.0 make-dir: 2.1.0 - pirates: 4.0.5 + pirates: 4.0.6 source-map-support: 0.5.21 dev: true + /@babel/regjsgen@0.8.0: + resolution: {integrity: sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==} + dev: true + /@babel/runtime@7.20.7: resolution: {integrity: sha512-UF0tvkUtxwAgZ5W/KrkHf0Rn0fdnLDU9ScxBrEVNUprE/MzirjK4MJUX1/BVDv00Sv8cljtukVK1aky++X1SjQ==} engines: {node: '>=6.9.0'} @@ -2265,6 +3160,13 @@ packages: dependencies: regenerator-runtime: 0.14.0 + /@babel/runtime@7.22.11: + resolution: {integrity: sha512-ee7jVNlWN09+KftVOu9n7S8gQzD/Z6hN/I8VBRXW4P1+Xe7kJGXMwu8vds4aGIMHZnNbdpSWCfZZtinytpcAvA==} + engines: {node: '>=6.9.0'} + dependencies: + regenerator-runtime: 0.14.0 + dev: true + /@babel/template@7.20.7: resolution: {integrity: sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw==} engines: {node: '>=6.9.0'} @@ -2273,6 +3175,15 @@ packages: '@babel/parser': 7.21.3 '@babel/types': 7.21.3 + /@babel/template@7.22.5: + resolution: {integrity: sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/code-frame': 7.22.10 + '@babel/parser': 7.22.11 + '@babel/types': 7.22.11 + dev: true + /@babel/traverse@7.20.12(supports-color@5.5.0): resolution: {integrity: sha512-MsIbFN0u+raeja38qboyF8TIT7K0BFzz/Yd/77ta4MsUsmP2RAnidIlwq7d5HFQrH/OZJecGV6B71C4zAgpoSQ==} engines: {node: '>=6.9.0'} @@ -2308,17 +3219,35 @@ packages: transitivePeerDependencies: - supports-color - /@babel/types@7.20.7: - resolution: {integrity: sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg==} + /@babel/traverse@7.22.11: + resolution: {integrity: sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ==} engines: {node: '>=6.9.0'} dependencies: - '@babel/helper-string-parser': 7.19.4 - '@babel/helper-validator-identifier': 7.19.1 - to-fast-properties: 2.0.0 - dev: true - - /@babel/types@7.21.3: - resolution: {integrity: sha512-sBGdETxC+/M4o/zKC0sl6sjWv62WFR/uzxrJ6uYyMLZOUlPnwzw0tKgVHOXxaAd5l2g8pEDM5RZ495GPQI77kg==} + '@babel/code-frame': 7.22.10 + '@babel/generator': 7.22.10 + '@babel/helper-environment-visitor': 7.22.5 + '@babel/helper-function-name': 7.22.5 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/parser': 7.22.11 + '@babel/types': 7.22.11 + debug: 4.3.4(supports-color@5.5.0) + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + dev: true + + /@babel/types@7.20.7: + resolution: {integrity: sha512-69OnhBxSSgK0OzTJai4kyPDiKTIe3j+ctaHdIGVbRahTLAT7L3R9oeXHC2aVSuGYt3cVnoAMDmOCgJ2yaiLMvg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.19.4 + '@babel/helper-validator-identifier': 7.19.1 + to-fast-properties: 2.0.0 + dev: true + + /@babel/types@7.21.3: + resolution: {integrity: sha512-sBGdETxC+/M4o/zKC0sl6sjWv62WFR/uzxrJ6uYyMLZOUlPnwzw0tKgVHOXxaAd5l2g8pEDM5RZ495GPQI77kg==} engines: {node: '>=6.9.0'} dependencies: '@babel/helper-string-parser': 7.19.4 @@ -2343,6 +3272,15 @@ packages: to-fast-properties: 2.0.0 dev: true + /@babel/types@7.22.11: + resolution: {integrity: sha512-siazHiGuZRz9aB9NpHy9GOs9xiQPKnMzgdr493iI1M67vRXpnEq8ZOOKzezC5q7zwuQ6sDhdSp4SD9ixKSqKZg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.22.5 + '@babel/helper-validator-identifier': 7.22.5 + to-fast-properties: 2.0.0 + dev: true + /@base2/pretty-print-object@1.0.1: resolution: {integrity: sha512-4iri8i1AqYHJE2DstZYkyEprg6Pq6sKx3xn5FpySk9sNhH7qN2LLlHJCfDTZRILNwQNPD7mATWM0TBui7uC1pA==} dev: true @@ -2598,6 +3536,15 @@ packages: react: '>=16.8.0' dependencies: react: 18.2.0 + dev: false + + /@emotion/use-insertion-effect-with-fallbacks@1.0.1(react@18.2.0): + resolution: {integrity: sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw==} + peerDependencies: + react: '>=16.8.0' + dependencies: + react: 18.2.0 + dev: true /@emotion/utils@1.2.0: resolution: {integrity: sha512-sn3WH53Kzpw8oQ5mgMmIzzyAaH2ZqFEbozVVBSYp538E06OSE6ytOp7pRAjNQR+Q/orwqdQYJSe2m3hCOeznkw==} @@ -2616,6 +3563,15 @@ packages: dev: true optional: true + /@esbuild/android-arm64@0.18.20: + resolution: {integrity: sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + requiresBuild: true + dev: true + optional: true + /@esbuild/android-arm@0.15.18: resolution: {integrity: sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==} engines: {node: '>=12'} @@ -2634,6 +3590,15 @@ packages: dev: true optional: true + /@esbuild/android-arm@0.18.20: + resolution: {integrity: sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + requiresBuild: true + dev: true + optional: true + /@esbuild/android-x64@0.17.12: resolution: {integrity: sha512-m4OsaCr5gT+se25rFPHKQXARMyAehHTQAz4XX1Vk3d27VtqiX0ALMBPoXZsGaB6JYryCLfgGwUslMqTfqeLU0w==} engines: {node: '>=12'} @@ -2643,6 +3608,15 @@ packages: dev: true optional: true + /@esbuild/android-x64@0.18.20: + resolution: {integrity: sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + requiresBuild: true + dev: true + optional: true + /@esbuild/darwin-arm64@0.17.12: resolution: {integrity: sha512-O3GCZghRIx+RAN0NDPhyyhRgwa19MoKlzGonIb5hgTj78krqp9XZbYCvFr9N1eUxg0ZQEpiiZ4QvsOQwBpP+lg==} engines: {node: '>=12'} @@ -2652,6 +3626,15 @@ packages: dev: true optional: true + /@esbuild/darwin-arm64@0.18.20: + resolution: {integrity: sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + /@esbuild/darwin-x64@0.17.12: resolution: {integrity: sha512-5D48jM3tW27h1qjaD9UNRuN+4v0zvksqZSPZqeSWggfMlsVdAhH3pwSfQIFJwcs9QJ9BRibPS4ViZgs3d2wsCA==} engines: {node: '>=12'} @@ -2661,6 +3644,15 @@ packages: dev: true optional: true + /@esbuild/darwin-x64@0.18.20: + resolution: {integrity: sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: true + optional: true + /@esbuild/freebsd-arm64@0.17.12: resolution: {integrity: sha512-OWvHzmLNTdF1erSvrfoEBGlN94IE6vCEaGEkEH29uo/VoONqPnoDFfShi41Ew+yKimx4vrmmAJEGNoyyP+OgOQ==} engines: {node: '>=12'} @@ -2670,6 +3662,15 @@ packages: dev: true optional: true + /@esbuild/freebsd-arm64@0.18.20: + resolution: {integrity: sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/freebsd-x64@0.17.12: resolution: {integrity: sha512-A0Xg5CZv8MU9xh4a+7NUpi5VHBKh1RaGJKqjxe4KG87X+mTjDE6ZvlJqpWoeJxgfXHT7IMP9tDFu7IZ03OtJAw==} engines: {node: '>=12'} @@ -2679,6 +3680,15 @@ packages: dev: true optional: true + /@esbuild/freebsd-x64@0.18.20: + resolution: {integrity: sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-arm64@0.17.12: resolution: {integrity: sha512-cK3AjkEc+8v8YG02hYLQIQlOznW+v9N+OI9BAFuyqkfQFR+DnDLhEM5N8QRxAUz99cJTo1rLNXqRrvY15gbQUg==} engines: {node: '>=12'} @@ -2688,6 +3698,15 @@ packages: dev: true optional: true + /@esbuild/linux-arm64@0.18.20: + resolution: {integrity: sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-arm@0.17.12: resolution: {integrity: sha512-WsHyJ7b7vzHdJ1fv67Yf++2dz3D726oO3QCu8iNYik4fb5YuuReOI9OtA+n7Mk0xyQivNTPbl181s+5oZ38gyA==} engines: {node: '>=12'} @@ -2697,6 +3716,15 @@ packages: dev: true optional: true + /@esbuild/linux-arm@0.18.20: + resolution: {integrity: sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-ia32@0.17.12: resolution: {integrity: sha512-jdOBXJqcgHlah/nYHnj3Hrnl9l63RjtQ4vn9+bohjQPI2QafASB5MtHAoEv0JQHVb/xYQTFOeuHnNYE1zF7tYw==} engines: {node: '>=12'} @@ -2706,6 +3734,15 @@ packages: dev: true optional: true + /@esbuild/linux-ia32@0.18.20: + resolution: {integrity: sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-loong64@0.15.18: resolution: {integrity: sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==} engines: {node: '>=12'} @@ -2724,6 +3761,15 @@ packages: dev: true optional: true + /@esbuild/linux-loong64@0.18.20: + resolution: {integrity: sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-mips64el@0.17.12: resolution: {integrity: sha512-o8CIhfBwKcxmEENOH9RwmUejs5jFiNoDw7YgS0EJTF6kgPgcqLFjgoc5kDey5cMHRVCIWc6kK2ShUePOcc7RbA==} engines: {node: '>=12'} @@ -2733,6 +3779,15 @@ packages: dev: true optional: true + /@esbuild/linux-mips64el@0.18.20: + resolution: {integrity: sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-ppc64@0.17.12: resolution: {integrity: sha512-biMLH6NR/GR4z+ap0oJYb877LdBpGac8KfZoEnDiBKd7MD/xt8eaw1SFfYRUeMVx519kVkAOL2GExdFmYnZx3A==} engines: {node: '>=12'} @@ -2742,6 +3797,15 @@ packages: dev: true optional: true + /@esbuild/linux-ppc64@0.18.20: + resolution: {integrity: sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-riscv64@0.17.12: resolution: {integrity: sha512-jkphYUiO38wZGeWlfIBMB72auOllNA2sLfiZPGDtOBb1ELN8lmqBrlMiucgL8awBw1zBXN69PmZM6g4yTX84TA==} engines: {node: '>=12'} @@ -2751,6 +3815,15 @@ packages: dev: true optional: true + /@esbuild/linux-riscv64@0.18.20: + resolution: {integrity: sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-s390x@0.17.12: resolution: {integrity: sha512-j3ucLdeY9HBcvODhCY4b+Ds3hWGO8t+SAidtmWu/ukfLLG/oYDMaA+dnugTVAg5fnUOGNbIYL9TOjhWgQB8W5g==} engines: {node: '>=12'} @@ -2760,6 +3833,15 @@ packages: dev: true optional: true + /@esbuild/linux-s390x@0.18.20: + resolution: {integrity: sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/linux-x64@0.17.12: resolution: {integrity: sha512-uo5JL3cgaEGotaqSaJdRfFNSCUJOIliKLnDGWaVCgIKkHxwhYMm95pfMbWZ9l7GeW9kDg0tSxcy9NYdEtjwwmA==} engines: {node: '>=12'} @@ -2769,6 +3851,15 @@ packages: dev: true optional: true + /@esbuild/linux-x64@0.18.20: + resolution: {integrity: sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: true + optional: true + /@esbuild/netbsd-x64@0.17.12: resolution: {integrity: sha512-DNdoRg8JX+gGsbqt2gPgkgb00mqOgOO27KnrWZtdABl6yWTST30aibGJ6geBq3WM2TIeW6COs5AScnC7GwtGPg==} engines: {node: '>=12'} @@ -2778,6 +3869,15 @@ packages: dev: true optional: true + /@esbuild/netbsd-x64@0.18.20: + resolution: {integrity: sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/openbsd-x64@0.17.12: resolution: {integrity: sha512-aVsENlr7B64w8I1lhHShND5o8cW6sB9n9MUtLumFlPhG3elhNWtE7M1TFpj3m7lT3sKQUMkGFjTQBrvDDO1YWA==} engines: {node: '>=12'} @@ -2787,6 +3887,15 @@ packages: dev: true optional: true + /@esbuild/openbsd-x64@0.18.20: + resolution: {integrity: sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + requiresBuild: true + dev: true + optional: true + /@esbuild/sunos-x64@0.17.12: resolution: {integrity: sha512-qbHGVQdKSwi0JQJuZznS4SyY27tYXYF0mrgthbxXrZI3AHKuRvU+Eqbg/F0rmLDpW/jkIZBlCO1XfHUBMNJ1pg==} engines: {node: '>=12'} @@ -2796,6 +3905,15 @@ packages: dev: true optional: true + /@esbuild/sunos-x64@0.18.20: + resolution: {integrity: sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + requiresBuild: true + dev: true + optional: true + /@esbuild/win32-arm64@0.17.12: resolution: {integrity: sha512-zsCp8Ql+96xXTVTmm6ffvoTSZSV2B/LzzkUXAY33F/76EajNw1m+jZ9zPfNJlJ3Rh4EzOszNDHsmG/fZOhtqDg==} engines: {node: '>=12'} @@ -2805,6 +3923,15 @@ packages: dev: true optional: true + /@esbuild/win32-arm64@0.18.20: + resolution: {integrity: sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@esbuild/win32-ia32@0.17.12: resolution: {integrity: sha512-FfrFjR4id7wcFYOdqbDfDET3tjxCozUgbqdkOABsSFzoZGFC92UK7mg4JKRc/B3NNEf1s2WHxJ7VfTdVDPN3ng==} engines: {node: '>=12'} @@ -2814,6 +3941,15 @@ packages: dev: true optional: true + /@esbuild/win32-ia32@0.18.20: + resolution: {integrity: sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@esbuild/win32-x64@0.17.12: resolution: {integrity: sha512-JOOxw49BVZx2/5tW3FqkdjSD/5gXYeVGPDcB0lvap0gLQshkh1Nyel1QazC+wNxus3xPlsYAgqU1BUmrmCvWtw==} engines: {node: '>=12'} @@ -2823,6 +3959,15 @@ packages: dev: true optional: true + /@esbuild/win32-x64@0.18.20: + resolution: {integrity: sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: true + optional: true + /@eslint-community/eslint-utils@4.3.0(eslint@8.36.0): resolution: {integrity: sha512-v3oplH6FYCULtFuCeqyuTd9D2WKO937Dxdq+GmHOLL72TTRriLxz2VLlNfkZRsvj6PKnOPAtuT6dwrs/pA5DvA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -3337,6 +4482,12 @@ packages: resolution: {integrity: sha512-EvYTiXet5XqweYGClEmpu3BoxmsQ4hkj3QaYA6qEnigCWffTP3vNRwBReTdrwDwo7OoJ3wM8Uoe9Uk4n+d4hfg==} dev: false + /@floating-ui/core@1.4.1: + resolution: {integrity: sha512-jk3WqquEJRlcyu7997NtR5PibI+y5bi+LS3hPmguVClypenMsCY3CBa3LAQnozRCtCrYWSEtAdiskpamuJRFOQ==} + dependencies: + '@floating-ui/utils': 0.1.1 + dev: true + /@floating-ui/dom@1.1.0: resolution: {integrity: sha512-TSogMPVxbRe77QCj1dt8NmRiJasPvuc+eT5jnJ6YpLqgOD2zXc5UA3S1qwybN+GVCDNdKfpKy1oj8RpzLJvh6A==} dependencies: @@ -3349,6 +4500,13 @@ packages: '@floating-ui/core': 1.2.6 dev: false + /@floating-ui/dom@1.5.1: + resolution: {integrity: sha512-KwvVcPSXg6mQygvA1TjbN/gh///36kKtllIF8SUm0qpFj8+rvYrpvlYdL1JoA71SHpDqgSSdGOSoQ0Mp3uY5aw==} + dependencies: + '@floating-ui/core': 1.4.1 + '@floating-ui/utils': 0.1.1 + dev: true + /@floating-ui/react-dom@1.2.1(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-YCLlqibZtgUhxUpxkSp1oekvYgH/jI4KdZEJv85E62twlZHN43xdlQNe6JcF4ROD3/Zu6juNHN+aOygN+6yZjg==} peerDependencies: @@ -3371,6 +4529,17 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: false + /@floating-ui/react-dom@2.0.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-5qhlDvjaLmAst/rKb3VdlCinwTF4EYMiVxuuc/HVUjs46W0zgtbMmAZ1UTsDrRTxRmUEzl92mOtWbeeXL26lSQ==} + peerDependencies: + react: '>=16.8.0' + react-dom: '>=16.8.0' + dependencies: + '@floating-ui/dom': 1.5.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + /@floating-ui/react@0.19.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-JyNk4A0Ezirq8FlXECvRtQOX/iBe5Ize0W/pLkrZjfHW9GUV7Xnq6zm6fyZuQzaHHqEnVizmvlA96e1/CkZv+w==} peerDependencies: @@ -3384,6 +4553,10 @@ packages: tabbable: 6.1.2 dev: false + /@floating-ui/utils@0.1.1: + resolution: {integrity: sha512-m0G6wlnhm/AX0H12IOWtK8gASEMffnX08RtKkCgTdHb9JpHKGloI7icFfLg9ZmQeavcvR0PKmzxClyuFPSjKWw==} + dev: true + /@formatjs/ecma402-abstract@1.14.3: resolution: {integrity: sha512-SlsbRC/RX+/zg4AApWIFNDdkLtFbkq3LNoZWXZCE/nHVKqoIJyaoQyge/I0Y38vLxowUn9KTtXgusLD91+orbg==} dependencies: @@ -3638,6 +4811,18 @@ packages: - encoding dev: true + /@isaacs/cliui@8.0.2: + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + dependencies: + string-width: 5.1.2 + string-width-cjs: /string-width@4.2.3 + strip-ansi: 7.1.0 + strip-ansi-cjs: /strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: /wrap-ansi@7.0.0 + dev: true + /@istanbuljs/load-nyc-config@1.1.0: resolution: {integrity: sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==} engines: {node: '>=8'} @@ -3833,6 +5018,13 @@ packages: '@sinclair/typebox': 0.27.8 dev: true + /@jest/schemas@29.6.3: + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@sinclair/typebox': 0.27.8 + dev: true + /@jest/source-map@29.6.0: resolution: {integrity: sha512-oA+I2SHHQGxDCZpbrsCQSoMLb3Bz547JnM+jUr9qEbuw0vQlWZfpPS7CO9J7XiwKicEz9OFn/IYoLkkiUD7bzA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3908,6 +5100,29 @@ packages: - supports-color dev: true + /@jest/transform@29.6.4: + resolution: {integrity: sha512-8thgRSiXUqtr/pPGY/OsyHuMjGyhVnWrFAwoxmIemlBuiMyU1WFs0tXoNxzcr4A4uErs/ABre76SGmrr5ab/AA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@babel/core': 7.21.3 + '@jest/types': 29.6.3 + '@jridgewell/trace-mapping': 0.3.19 + babel-plugin-istanbul: 6.1.1 + chalk: 4.1.2 + convert-source-map: 2.0.0 + fast-json-stable-stringify: 2.1.0 + graceful-fs: 4.2.11 + jest-haste-map: 29.6.4 + jest-regex-util: 29.6.3 + jest-util: 29.6.3 + micromatch: 4.0.5 + pirates: 4.0.6 + slash: 3.0.0 + write-file-atomic: 4.0.2 + transitivePeerDependencies: + - supports-color + dev: true + /@jest/types@29.5.0: resolution: {integrity: sha512-qbu7kN6czmVRc3xWFQcAN03RAUamgppVUdXrvl1Wr3jlNF93o9mJbGcDWrwGB6ht44u7efB1qCFgVQmca24Uog==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3932,6 +5147,18 @@ packages: chalk: 4.1.2 dev: true + /@jest/types@29.6.3: + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.4 + '@types/istanbul-reports': 3.0.1 + '@types/node': 18.16.3 + '@types/yargs': 17.0.24 + chalk: 4.1.2 + dev: true + /@joshwooding/vite-plugin-react-docgen-typescript@0.2.1(typescript@5.0.2)(vite@4.2.0): resolution: {integrity: sha512-ou4ZJSXMMWHqGS4g8uNRbC5TiTWxAgQZiVucoUrOCWuPrTbkpJbmVyIi9jU72SBry7gQtuMEDp4YR8EEXAg7VQ==} peerDependencies: @@ -4047,12 +5274,12 @@ packages: '@types/whatwg-streams': 0.0.7 dev: false - /@mdx-js/react@2.2.1(react@18.2.0): - resolution: {integrity: sha512-YdXcMcEnqZhzql98RNrqYo9cEhTTesBiCclEtoiQUbJwx87q9453GTapYU6kJ8ZZ2ek1Vp25SiAXEFy5O/eAPw==} + /@mdx-js/react@2.3.0(react@18.2.0): + resolution: {integrity: sha512-zQH//gdOmuu7nt2oJR29vFhDv88oGPmVw6BggmrHeMI+xgEkp1B2dX9/bMBSYtK0dyLX/aOmesKS09g222K1/g==} peerDependencies: react: '>=16' dependencies: - '@types/mdx': 2.0.3 + '@types/mdx': 2.0.7 '@types/react': 18.2.20 react: 18.2.0 dev: true @@ -4140,8 +5367,8 @@ packages: tslib: 2.5.2 dev: false - /@ndelangen/get-tarball@3.0.7: - resolution: {integrity: sha512-NqGfTZIZpRFef1GoVaShSSRwDC3vde3ThtTeqFdcYd6ipKqnfEVhjK2hUeHjCQUcptyZr2TONqcloFXM+5QBrQ==} + /@ndelangen/get-tarball@3.0.9: + resolution: {integrity: sha512-9JKTEik4vq+yGosHYhZ1tiH/3WpUS0Nh0kej4Agndhox8pAdWhEx5knFVRcb/ya9knCRCs1rPxNrSXTDdfVqpA==} dependencies: gunzip-maybe: 1.4.2 pump: 3.0.0 @@ -4271,6 +5498,13 @@ packages: - supports-color dev: true + /@pkgjs/parseargs@0.11.0: + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + requiresBuild: true + dev: true + optional: true + /@pkgr/utils@2.3.1: resolution: {integrity: sha512-wfzX8kc1PMyUILA+1Z/EqoE4UCXGy0iRGMhPwdfae1+f0OXlLqCk+By+aMzgJBzR9AzS4CDizioG6Ss1gvAFJw==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} @@ -4329,18 +5563,578 @@ packages: resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} dev: false - /@remix-run/router@1.7.2: - resolution: {integrity: sha512-7Lcn7IqGMV+vizMPoEl5F0XDshcdDYtMI6uJLQdQz5CfZAwy3vvGKYSUk789qndt5dEC4HfSjviSYlSoHGL2+A==} - engines: {node: '>=14'} - dev: false + /@radix-ui/number@1.0.1: + resolution: {integrity: sha512-T5gIdVO2mmPW3NNhjNgEP3cqMXjXL9UbO0BzWcXfvdBs+BohbQxvd/K5hSVKmn9/lbTdsQVKbUcP5WLCwvUbBg==} + dependencies: + '@babel/runtime': 7.22.11 + dev: true - /@rollup/plugin-commonjs@22.0.2(rollup@2.79.1): - resolution: {integrity: sha512-//NdP6iIwPbMTcazYsiBMbJW7gfmpHom33u1beiIoHDEM0Q9clvtQB1T0efvMqHeKsGohiHo97BCPCkBXdscwg==} - engines: {node: '>= 12.0.0'} - peerDependencies: - rollup: ^2.68.0 + /@radix-ui/primitive@1.0.1: + resolution: {integrity: sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==} dependencies: - '@rollup/pluginutils': 3.1.0(rollup@2.79.1) + '@babel/runtime': 7.22.11 + dev: true + + /@radix-ui/react-arrow@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-collection@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-3SzW+0PW7yBBoQlT8wNcGtaxaD0XSu0uLUFgrtHY08Acx05TaHaOmVLR73c0j/cqpDy53KBMO7s0dx2wmOIDIA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-slot': 1.0.2(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-compose-refs@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-context@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-direction@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-RXcvnXgyvYvBEOhCBuddKecVkoMiI10Jcm5cTI7abJRAHYfFxeu+FBQs/DvdxSYucxR5mna0dNsL6QFlds5TMA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-dismissable-layer@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-7UpBa/RKMoHJYjie1gkF1DlK8l1fdU/VKDpoS3rCCo8YBJR294GwcEHyxHw72yvphJ7ld0AXEcSLAzY2F/WyCg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-escape-keydown': 1.0.3(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-focus-guards@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-focus-scope@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-upXdPfqI4islj2CslyfUBNlaJCPybbqRHAi1KER7Isel9Q2AtSJ0zRBZv8mWQiFXD2nyAJ4BhC3yXgZ6kMBSrQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-id@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-popper@1.1.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-1CnGGfFi/bbqtJZZ0P/NQY20xdG3E0LALJaLUEoKwPLwl6PPPfbeiCqMVQnhoFRAxjJj4RpBRJzDmUgsex2tSg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@floating-ui/react-dom': 2.0.2(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-arrow': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-rect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-size': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/rect': 1.0.1 + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-portal@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-xLYZeHrWoPmA5mEKEfZZevoVRK/Q43GfzRXkWV6qawIWWK8t6ifIiLQdd7rmQ4Vk1bmI21XhqF9BN3jWf+phpA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-primitive@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-slot': 1.0.2(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-roving-focus@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-2mUg5Mgcu001VkGy+FfzZyzbmuUWzgWkj3rvv4yu+mLw03+mTzbxZHvfcGyFp2b8EkQeMkpRQ5FiA2Vr2O6TeQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-collection': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-id': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-select@1.2.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-zI7McXr8fNaSrUY9mZe4x/HC0jTLY9fWNhO1oLWYMQGDXuV4UCivIGTxwioSzO0ZCYX9iSLyWmAh/1TOmX3Cnw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/number': 1.0.1 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-collection': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-dismissable-layer': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-focus-guards': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-focus-scope': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-id': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-popper': 1.1.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-portal': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-slot': 1.0.2(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-use-previous': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-visually-hidden': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + aria-hidden: 1.2.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-remove-scroll: 2.5.5(@types/react@18.2.20)(react@18.2.0) + dev: true + + /@radix-ui/react-separator@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-itYmTy/kokS21aiV5+Z56MZB54KrhPgn6eHDKkFeOLR34HMN2s8PaN47qZZAGnvupcjxHaFZnW4pQEh0BvvVuw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-slot@1.0.2(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-compose-refs': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-toggle-group@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-Uaj/M/cMyiyT9Bx6fOZO0SAG4Cls0GptBWiBmBxofmDbNVnYYoyRWj/2M/6VCi/7qcXFWnHhRUfdfZFvvkuu8A==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-roving-focus': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-toggle': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-toggle@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-Pkqg3+Bc98ftZGsl60CLANXQBBQ4W3mTFS9EJvNxKMZ7magklKV69/id1mlAlOFDDfHvlCms0fx8fA4CMKDJHg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-use-controllable-state': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-toolbar@1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-tBgmM/O7a07xbaEkYJWYTXkIdU/1pW4/KZORR43toC/4XWyBCURK0ei9kMUdp+gTPPKBgYLxXmRSH1EVcIDp8Q==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/primitive': 1.0.1 + '@radix-ui/react-context': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-direction': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-roving-focus': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-separator': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-toggle-group': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/react-use-callback-ref@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-use-controllable-state@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-use-escape-keydown@1.0.3(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-callback-ref': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-use-layout-effect@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-use-previous@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-cV5La9DPwiQ7S0gf/0qiD6YgNqM5Fk97Kdrlc5yBcrF3jyEZQwm7vYFqMo4IfeHgJXsRaMvLABFtd0OVEmZhDw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-use-rect@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/rect': 1.0.1 + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-use-size@1.0.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-use-layout-effect': 1.0.1(@types/react@18.2.20)(react@18.2.0) + '@types/react': 18.2.20 + react: 18.2.0 + dev: true + + /@radix-ui/react-visually-hidden@1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 + react-dom: ^16.8 || ^17.0 || ^18.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + dependencies: + '@babel/runtime': 7.22.11 + '@radix-ui/react-primitive': 1.0.3(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@types/react': 18.2.20 + '@types/react-dom': 18.2.7 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@radix-ui/rect@1.0.1: + resolution: {integrity: sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==} + dependencies: + '@babel/runtime': 7.22.11 + dev: true + + /@remix-run/router@1.7.2: + resolution: {integrity: sha512-7Lcn7IqGMV+vizMPoEl5F0XDshcdDYtMI6uJLQdQz5CfZAwy3vvGKYSUk789qndt5dEC4HfSjviSYlSoHGL2+A==} + engines: {node: '>=14'} + dev: false + + /@rollup/plugin-commonjs@22.0.2(rollup@2.79.1): + resolution: {integrity: sha512-//NdP6iIwPbMTcazYsiBMbJW7gfmpHom33u1beiIoHDEM0Q9clvtQB1T0efvMqHeKsGohiHo97BCPCkBXdscwg==} + engines: {node: '>= 12.0.0'} + peerDependencies: + rollup: ^2.68.0 + dependencies: + '@rollup/pluginutils': 3.1.0(rollup@2.79.1) commondir: 1.0.1 estree-walker: 2.0.2 glob: 7.2.3 @@ -4362,16 +6156,23 @@ packages: rollup: 2.79.1 dev: true - /@rollup/pluginutils@4.2.1: - resolution: {integrity: sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ==} - engines: {node: '>= 8.0.0'} + /@rollup/pluginutils@5.0.2(rollup@2.79.1): + resolution: {integrity: sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0 + peerDependenciesMeta: + rollup: + optional: true dependencies: + '@types/estree': 1.0.0 estree-walker: 2.0.2 picomatch: 2.3.1 + rollup: 2.79.1 dev: true - /@rollup/pluginutils@5.0.2(rollup@2.79.1): - resolution: {integrity: sha512-pTd9rIsP92h+B6wWwFbW8RkZv4hiR/xKsqre4SIuAOaOEQRxi0lqLke9k2/7WegC85GgUs9pjmOjCUi3In4vwA==} + /@rollup/pluginutils@5.0.4(rollup@2.79.1): + resolution: {integrity: sha512-0KJnIoRI8A+a1dqOYLxH8vBf8bphDmty5QvIm2hqm7oFCFYKCAZWWd2hXgMibaPsNDhI0AtpYfQZJG47pt/k4g==} engines: {node: '>=14.0.0'} peerDependencies: rollup: ^1.20.0||^2.0.0||^3.0.0 @@ -4379,7 +6180,7 @@ packages: rollup: optional: true dependencies: - '@types/estree': 1.0.0 + '@types/estree': 1.0.1 estree-walker: 2.0.2 picomatch: 2.3.1 rollup: 2.79.1 @@ -4468,7 +6269,7 @@ packages: fast-memoize: 2.5.2 immer: 9.0.21 lodash: 4.17.21 - tslib: 2.5.2 + tslib: 2.6.2 urijs: 1.19.11 dev: true @@ -4519,7 +6320,7 @@ packages: stacktracey: 2.1.8 strip-ansi: 6.0.1 text-table: 0.2.0 - tslib: 2.5.2 + tslib: 2.6.2 yargs: 17.3.1 transitivePeerDependencies: - encoding @@ -4550,7 +6351,7 @@ packages: nimma: 0.2.2 pony-cause: 1.1.1 simple-eval: 1.0.0 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4562,7 +6363,7 @@ packages: '@stoplight/json': 3.20.2 '@stoplight/spectral-core': 1.18.0 '@types/json-schema': 7.0.11 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4581,7 +6382,7 @@ packages: ajv-errors: 3.0.0(ajv@8.12.0) ajv-formats: 2.1.1(ajv@8.12.0) lodash: 4.17.21 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4593,7 +6394,7 @@ packages: '@stoplight/json': 3.20.2 '@stoplight/types': 13.15.0 '@stoplight/yaml': 4.2.3 - tslib: 2.5.2 + tslib: 2.6.2 dev: true /@stoplight/spectral-ref-resolver@1.0.2: @@ -4604,7 +6405,7 @@ packages: '@stoplight/json-ref-resolver': 3.1.5 '@stoplight/spectral-runtime': 1.1.2 dependency-graph: 0.11.0 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4617,7 +6418,7 @@ packages: '@stoplight/json-ref-resolver': 3.1.5 '@stoplight/spectral-runtime': 1.1.2 dependency-graph: 0.11.0 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4640,7 +6441,7 @@ packages: '@types/node': 18.16.3 pony-cause: 1.1.1 rollup: 2.79.1 - tslib: 2.5.2 + tslib: 2.6.2 validate-npm-package-name: 3.0.0 transitivePeerDependencies: - encoding @@ -4662,7 +6463,7 @@ packages: ast-types: 0.14.2 astring: 1.8.4 reserved: 0.1.2 - tslib: 2.5.2 + tslib: 2.6.2 validate-npm-package-name: 3.0.0 transitivePeerDependencies: - encoding @@ -4685,7 +6486,7 @@ packages: ajv-formats: 2.1.1(ajv@8.12.0) json-schema-traverse: 1.0.0 lodash: 4.17.21 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4700,7 +6501,7 @@ packages: abort-controller: 3.0.0 lodash: 4.17.21 node-fetch: 2.6.9 - tslib: 2.5.2 + tslib: 2.6.2 transitivePeerDependencies: - encoding dev: true @@ -4740,11 +6541,11 @@ packages: '@stoplight/ordered-object-literal': 1.0.4 '@stoplight/types': 13.15.0 '@stoplight/yaml-ast-parser': 0.0.48 - tslib: 2.5.2 + tslib: 2.6.2 dev: true - /@storybook/addon-actions@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-rcj39u9MrmzsrDWYt1zsoVxrogZ1Amrv9xkEofEY/QKUr2R3xpHhTALveY9BKIlG1GoE8zLlLoP2k4nz3sNNwQ==} + /@storybook/addon-actions@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-TsTOHGmwBHRsWS9kaG/bu6haP2dMeiETeGwOgfB5qmukodenXlmi1RujtUdJCNwW3APa0utEFYFKtZVEu9f7WQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4754,28 +6555,31 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 dequal: 2.0.3 lodash: 4.17.21 polished: 4.2.2 prop-types: 15.8.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) - react-inspector: 6.0.1(react@18.2.0) - telejson: 7.0.4 + react-inspector: 6.0.2(react@18.2.0) + telejson: 7.2.0 ts-dedent: 2.2.0 - uuid-browser: 3.1.0 + uuid: 9.0.0 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-backgrounds@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-yRNHQ4PPRJ+HIORQPhDGxn5xolw1xW0ByQZoNRpMD+AMEyfUNFdWbCsRQAOWjNhawxVMHM7EeA2Exrb41zhEjA==} + /@storybook/addon-backgrounds@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-tcQSt6mjAR1h1XiMFlg9OvpAwvBCjFrtpr9qnVaOZD15EIu/TRoumkJOVA7J5sWuQ6kGJXx1t8FfhQfAqvJ9iw==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4785,22 +6589,25 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 memoizerific: 1.11.3 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-controls@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-dMpRtj5cmfC9vEMve5ncvbWCEC+WD9YuzJ+grdc48E/Hd//p+O2FE6klSkrz5FAjrc+rHINixdyssekpEL6nYQ==} + /@storybook/addon-controls@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-n9ZoxlV8c9VLNfpFY1HpcRxjUFmHPmcFnW0UzFfGknIArPKFxzw9S/zCJ7CSH9Mf7+NJtYAUzCXlSU/YzT1eZQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4810,97 +6617,100 @@ packages: react-dom: optional: true dependencies: - '@storybook/blocks': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-common': 7.0.2 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/node-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/blocks': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-common': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/node-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 lodash: 4.17.21 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/addon-docs@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-q3rDWoZEym6Lkmhqc/HBNfLDAmTY8l0WINGUZo/nF98eP5iu4B7Nk7V6BRGYGQt6Y6ZyIQ8WKH0e/eJww2zIog==} + /@storybook/addon-docs@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-g4B+gM7xzRvUeiUcijPyxwDG/LlgHrfQx1chzY7oiFIImGXyewZ+CtGCjhrSdJGhXSj/69oqoz26RQ1VhSlrXg==} peerDependencies: - '@storybook/mdx1-csf': '>=1.0.0-0' react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 - peerDependenciesMeta: - '@storybook/mdx1-csf': - optional: true dependencies: - '@babel/core': 7.21.3 - '@babel/plugin-transform-react-jsx': 7.20.7(@babel/core@7.21.3) - '@jest/transform': 29.5.0 - '@mdx-js/react': 2.2.1(react@18.2.0) - '@storybook/blocks': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/csf-plugin': 7.0.2 - '@storybook/csf-tools': 7.0.2 + '@jest/transform': 29.6.4 + '@mdx-js/react': 2.3.0(react@18.2.0) + '@storybook/blocks': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/csf-plugin': 7.3.2 + '@storybook/csf-tools': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/mdx2-csf': 1.0.0 - '@storybook/node-logger': 7.0.2 - '@storybook/postinstall': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/react-dom-shim': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 - fs-extra: 11.1.0 + '@storybook/mdx2-csf': 1.1.0 + '@storybook/node-logger': 7.3.2 + '@storybook/postinstall': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/react-dom-shim': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 + fs-extra: 11.1.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) remark-external-links: 8.0.0 remark-slug: 6.1.0 ts-dedent: 2.2.0 transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/addon-essentials@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-LAsWsXa/Pp2B4Ve2WVgc990FtsiHpFDRsq7S3V7xRrZP8DYRbtJIVdszPMDS5uKC+yzbswFEXz08lqbGvq8zgQ==} + /@storybook/addon-essentials@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-MI5wi5k/nDgAqnsS4/uibcQhMk3/mVkAAWNO+Epmg5UMCCmDch8SoX9BprEHARwwsVwXChiHAx99fXF/XacWFQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/addon-actions': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-backgrounds': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-controls': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-docs': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-highlight': 7.0.2 - '@storybook/addon-measure': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-outline': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-toolbars': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/addon-viewport': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-common': 7.0.2 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/node-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 + '@storybook/addon-actions': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-backgrounds': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-controls': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-docs': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-highlight': 7.3.2 + '@storybook/addon-measure': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-outline': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-toolbars': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/addon-viewport': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-common': 7.3.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/node-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 transitivePeerDependencies: - - '@storybook/mdx1-csf' + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/addon-highlight@7.0.2: - resolution: {integrity: sha512-9BkL1OOanguuy73S6nLK0isUb045tOkFONd/PQldOJ0PV3agCvKxKHyzlBz7Hsba8KZhY5jQs+nVW2NiREyGYg==} + /@storybook/addon-highlight@7.3.2: + resolution: {integrity: sha512-Zdq//ZqOYpm+xXHt00l0j/baVuZDSkpP6Xbd3jqXV1ToojAjANlk0CAzHCJxZBiyeSCj7Qxtj9LvTqD+IU/bMA==} dependencies: - '@storybook/core-events': 7.0.2 + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/preview-api': 7.0.2 + '@storybook/preview-api': 7.3.2 dev: true - /@storybook/addon-links@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-lPtfy2MqrcI9YjupBM2eRKGPdFKVPCz7WgO/JQQakGugORJTEGCyJrNJNtWY9jDenv8ynLZ40OxtPBZi54Sr6Q==} + /@storybook/addon-links@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-xpOpb33KscvmM2Sl9nFqU3DCk3tGaoqtFKkDOzf/QlZsMq9CCn4zPNGMfOFqifBEnDGDADHbp+Uxst5i535vdQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4910,22 +6720,22 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/router': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/router': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 prop-types: 15.8.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 dev: true - /@storybook/addon-measure@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-cf/d5MXpHAjyUiDIVfc8pLn79CPHgnryDmNNlSiP2zEFKcivrRWiu8Rmrad8pGqLkuAh+PXLKCGn9uiqDvg7QQ==} + /@storybook/addon-measure@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-bEoH3zuKA9b5RA0LBQzdSnoaxEKHa5rZDoAuMbKiEYotTqO7PfP2j/hil31F95UgmH7wPnSkRSqsBsUtWJz3Jg==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4935,19 +6745,23 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) + tiny-invariant: 1.3.1 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-outline@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-thVISO4NM22xlETisBvAPvz2yFD3qLGOjgzBmj8l8r9Rv0IEdwdPrwm5j0WTv8OtbhC4A8lPpvMsn5FhY5mDXg==} + /@storybook/addon-outline@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-DA/O5b4bznV2JsC/o0/JkP2tZLLPftRaz2HHCG+z0mwzNv2pl8lvIl4RpIVJWt1iO0K17kT43ToYYjknMUdJnA==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4957,20 +6771,23 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) ts-dedent: 2.2.0 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-toolbars@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-tAxZ2+nUYsJdT1sx3BrmoMAZFM19+OzWJY6qSnbEq5zoRgvGZaXGR6tLMKydDoHQBU9Ta9YHGo7N7u7h1C23yg==} + /@storybook/addon-toolbars@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-hd+5Ax7p3vmsNNuO3t4pcmB2pxp58i9k12ygD66NLChSNafHxediLqdYJDTRuono2No1InV1HMZghlXXucCCHQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -4980,17 +6797,20 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) react: 18.2.0 react-dom: 18.2.0(react@18.2.0) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/addon-viewport@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-TaHJWIIazPM/TerRbka9RqjMPNpwaRsGRdVRBtVoVosy1FzsEjAdQSO7RBMe4G03m5CacSqdsDiJCblI2AXaew==} + /@storybook/addon-viewport@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-G7i67xL35WE6qSmEoctavZUoPd2VDTaAqkRwrGa4oDQs5wed76PgIL2S5IybzbypSzPIXauiNQiBBd2RRMrLFg==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5000,182 +6820,210 @@ packages: react-dom: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) memoizerific: 1.11.3 prop-types: 15.8.1 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/blocks@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-JzHmU8jZLzeQ6bunzci8j/2Ji18GBTyhrPFLk5RjEbMNGWpGjvER/yR127tZOdbPguVNr4iVbRfGzd1wGHlrzA==} + /@storybook/addons@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-qYwHniTJzfR7jKh5juYCjU9ukG7l1YAAt7BpnouItgRutxU/+UoC2iAFooQW+i74SxDoovqnEp9TkG7TAFOLxQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 - '@storybook/docs-tools': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@storybook/api@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-HAiaEl9uFQJM3AC5LhdnUbqr+7BVMaCNzhbUg1sWfO7sTFXPO0P1BAz9UuDKPlndwaVGcGpypRw9P/bdpuWLfA==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + react: + optional: true + react-dom: + optional: true + dependencies: + '@storybook/client-logger': 7.3.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@storybook/blocks@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-j/PRnvGLn0Y3VAu/t6RrU7pjenb7II7Cl/SnFW8LzjMBKXBrkFaq8BRbglzDAUtGdAa9HmJBosogenoZ9iWoBw==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/components': 7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 + '@storybook/docs-tools': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/manager-api': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/preview-api': 7.0.2 - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/manager-api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/preview-api': 7.3.2 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 '@types/lodash': 4.14.191 color-convert: 2.0.1 dequal: 2.0.3 lodash: 4.17.21 - markdown-to-jsx: 7.1.8(react@18.2.0) + markdown-to-jsx: 7.3.2(react@18.2.0) memoizerific: 1.11.3 polished: 4.2.2 react: 18.2.0 react-colorful: 5.6.1(react-dom@18.2.0)(react@18.2.0) react-dom: 18.2.0(react@18.2.0) - telejson: 7.0.4 + telejson: 7.2.0 + tocbot: 4.21.1 ts-dedent: 2.2.0 util-deprecate: 1.0.2 transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + - encoding - supports-color dev: true - /@storybook/builder-manager@7.0.2: - resolution: {integrity: sha512-Oej/n8D7eaWgmWF7nN2hXLRM53lcYOdh6umSN8Mh/LcYUfxB+dvUBFzUjoLE0xjhW6xRinrKrENT5LcP/f/HBQ==} + /@storybook/builder-manager@7.3.2: + resolution: {integrity: sha512-M0zdzpnZSg6Gd/QiIbOJkVoifAADpMT85NOC5zuAg3h3o29hedVBAigv/CE2nSbuwZtqPifjxs1AUh7wgtmj8A==} dependencies: '@fal-works/esbuild-plugin-global-externals': 2.1.2 - '@storybook/core-common': 7.0.2 - '@storybook/manager': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@types/ejs': 3.1.1 + '@storybook/core-common': 7.3.2 + '@storybook/manager': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@types/ejs': 3.1.2 '@types/find-cache-dir': 3.2.1 - '@yarnpkg/esbuild-plugin-pnp': 3.0.0-rc.15(esbuild@0.17.12) + '@yarnpkg/esbuild-plugin-pnp': 3.0.0-rc.15(esbuild@0.18.20) browser-assert: 1.2.1 - ejs: 3.1.8 - esbuild: 0.17.12 + ejs: 3.1.9 + esbuild: 0.18.20 esbuild-plugin-alias: 0.2.1 express: 4.18.2 find-cache-dir: 3.3.2 - fs-extra: 11.1.0 + fs-extra: 11.1.1 process: 0.11.10 util: 0.12.5 transitivePeerDependencies: + - encoding - supports-color dev: true - /@storybook/builder-vite@7.0.2(typescript@5.0.2)(vite@4.2.0): - resolution: {integrity: sha512-G6CD2Gf2zwzRslvNvqgz4FeADVEA9XA4Mw6+NM6Twc+Wy/Ah482dvHS9ApSgirtGyBKjOfdHn1xQT4Z+kzbJnw==} + /@storybook/builder-vite@7.3.2(typescript@5.0.2)(vite@4.2.0): + resolution: {integrity: sha512-9xB3Z6QfDBX6Daj+LFldhavA8O7JU2E1dL6IHfaTLIamFH884Sl5Svq3GS3oh4/EbB/GifpVKEiwlvJaINCj+A==} peerDependencies: '@preact/preset-vite': '*' - '@storybook/mdx1-csf': '>=1.0.0-next.1' typescript: '>= 4.3.x' vite: ^3.0.0 || ^4.0.0 vite-plugin-glimmerx: '*' peerDependenciesMeta: '@preact/preset-vite': optional: true - '@storybook/mdx1-csf': - optional: true typescript: optional: true vite-plugin-glimmerx: optional: true dependencies: - '@storybook/channel-postmessage': 7.0.2 - '@storybook/channel-websocket': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-common': 7.0.2 - '@storybook/csf-plugin': 7.0.2 - '@storybook/mdx2-csf': 1.0.0 - '@storybook/node-logger': 7.0.2 - '@storybook/preview': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/csf-plugin': 7.3.2 + '@storybook/mdx2-csf': 1.1.0 + '@storybook/node-logger': 7.3.2 + '@storybook/preview': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 + '@types/find-cache-dir': 3.2.1 browser-assert: 1.2.1 es-module-lexer: 0.9.3 express: 4.18.2 - fs-extra: 11.1.0 - glob: 8.1.0 - glob-promise: 6.0.2(glob@8.1.0) - magic-string: 0.27.0 + find-cache-dir: 3.3.2 + fs-extra: 11.1.1 + magic-string: 0.30.3 remark-external-links: 8.0.0 remark-slug: 6.1.0 - rollup: 3.19.1 + rollup: 3.28.1 typescript: 5.0.2 vite: 4.2.0(@types/node@18.16.3)(sass@1.59.3) transitivePeerDependencies: + - encoding - supports-color dev: true - /@storybook/channel-postmessage@7.0.2: - resolution: {integrity: sha512-SZ/KqnZcx10W9hJbrzBKcP9dmgaeTaXugUhcgw1IkmjKWdsKazqFZCPwQWZZKAmhO4wYbyYOhkz3wfSIeB4mFw==} - dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/global': 5.0.0 - qs: 6.11.0 - telejson: 7.0.4 + /@storybook/channels@7.0.2: + resolution: {integrity: sha512-qkI8mFy9c8mxN2f01etayKhCaauL6RAsxRzbX1/pKj6UqhHWqqUbtHwymrv4hG5qDYjV1e9pd7ae5eNF8Kui0g==} dev: true - /@storybook/channel-websocket@7.0.2: - resolution: {integrity: sha512-YU3lFId6Nsi75ddA+3qfbnLfNUPswboYyx+SALhaLuXqz7zqfzX4ezMgxeS/h0gRlUJ7nf2/yJ5qie/kZaizjw==} + /@storybook/channels@7.3.2: + resolution: {integrity: sha512-GG5+qzv2OZAzXonqUpJR81f2pjKExj7v5MoFJhKYgb3Y+jVYlUzBHBjhQZhuQczP4si418/jvjimvU1PZ4hqcg==} dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 '@storybook/global': 5.0.0 - telejson: 7.0.4 - dev: true - - /@storybook/channels@7.0.2: - resolution: {integrity: sha512-qkI8mFy9c8mxN2f01etayKhCaauL6RAsxRzbX1/pKj6UqhHWqqUbtHwymrv4hG5qDYjV1e9pd7ae5eNF8Kui0g==} + qs: 6.11.2 + telejson: 7.2.0 + tiny-invariant: 1.3.1 dev: true - /@storybook/cli@7.0.2: - resolution: {integrity: sha512-xMM2QdXNGg09wuXzAGroKrbsnaHSFPmtmefX1XGALhHuKVwxOoC2apWMpek6gY/9vh5EIRTog2Dvfd2BzNrT6Q==} + /@storybook/cli@7.3.2: + resolution: {integrity: sha512-RnqE/6KSelL9TQ44uCIU5xvUhY9zXM2Upanr0hao72x44rvlGQbV262pHdkVIYsn0wi8QzYtnoxQPLSqUfUDfA==} hasBin: true dependencies: - '@babel/core': 7.21.3 - '@babel/preset-env': 7.20.2(@babel/core@7.21.3) - '@ndelangen/get-tarball': 3.0.7 - '@storybook/codemod': 7.0.2 - '@storybook/core-common': 7.0.2 - '@storybook/core-server': 7.0.2 - '@storybook/csf-tools': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@storybook/telemetry': 7.0.2 - '@storybook/types': 7.0.2 + '@babel/core': 7.22.11 + '@babel/preset-env': 7.22.10(@babel/core@7.22.11) + '@babel/types': 7.22.11 + '@ndelangen/get-tarball': 3.0.9 + '@storybook/codemod': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/core-server': 7.3.2 + '@storybook/csf-tools': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@storybook/telemetry': 7.3.2 + '@storybook/types': 7.3.2 '@types/semver': 7.3.13 - boxen: 5.1.2 + '@yarnpkg/fslib': 2.10.3 + '@yarnpkg/libzip': 2.3.0 chalk: 4.1.2 commander: 6.2.1 cross-spawn: 7.0.3 detect-indent: 6.1.0 - envinfo: 7.8.1 + envinfo: 7.10.0 execa: 5.1.1 express: 4.18.2 find-up: 5.0.0 - fs-extra: 11.1.0 + fs-extra: 11.1.1 get-npm-tarball-url: 2.0.3 get-port: 5.1.1 - giget: 1.0.0 + giget: 1.1.2 globby: 11.1.0 - jscodeshift: 0.14.0(@babel/preset-env@7.20.2) + jscodeshift: 0.14.0(@babel/preset-env@7.22.10) leven: 3.1.0 + ora: 5.4.1 prettier: 2.8.4 prompts: 2.4.2 puppeteer-core: 2.1.1 read-pkg-up: 7.0.1 semver: 7.3.8 - shelljs: 0.8.5 - simple-update-notifier: 1.1.0 + simple-update-notifier: 2.0.0 strip-json-comments: 3.1.1 tempy: 1.0.1 ts-dedent: 2.2.0 @@ -5193,22 +7041,29 @@ packages: '@storybook/global': 5.0.0 dev: true - /@storybook/codemod@7.0.2: - resolution: {integrity: sha512-D9PdByxJlFiaDJcLkM+RN1DHCj4VfQIlSZkADOcNtI4o9H064oiMloWDGZiR1i1FCYMSXuWmW6tMsuCVebA+Nw==} + /@storybook/client-logger@7.3.2: + resolution: {integrity: sha512-T7q/YS5lPUE6xjz9EUwJ/v+KCd5KU9dl1MQ9RcH7IpM73EtQZeNSuM9/P96uKXZTf0wZOUBTXVlTzKr66ZB/RQ==} dependencies: - '@babel/core': 7.21.3 - '@babel/preset-env': 7.21.4(@babel/core@7.21.3) - '@babel/types': 7.21.3 - '@storybook/csf': 0.1.0 - '@storybook/csf-tools': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/global': 5.0.0 + dev: true + + /@storybook/codemod@7.3.2: + resolution: {integrity: sha512-B2P91aYhlxdk7zeQOq0VBnDox2HEcboP2unSh6Vcf4V8j2FCdPvBIM7ZkT9p15FHfyOHvvrtf56XdBIyD8/XJA==} + dependencies: + '@babel/core': 7.22.11 + '@babel/preset-env': 7.22.10(@babel/core@7.22.11) + '@babel/types': 7.22.11 + '@storybook/csf': 0.1.1 + '@storybook/csf-tools': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@storybook/types': 7.3.2 + '@types/cross-spawn': 6.0.2 cross-spawn: 7.0.3 globby: 11.1.0 - jscodeshift: 0.14.0(@babel/preset-env@7.21.4) + jscodeshift: 0.14.0(@babel/preset-env@7.22.10) lodash: 4.17.21 prettier: 2.8.4 - recast: 0.23.1 + recast: 0.23.4 transitivePeerDependencies: - supports-color dev: true @@ -5220,7 +7075,7 @@ packages: react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: '@storybook/client-logger': 7.0.2 - '@storybook/csf': 0.1.0 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) '@storybook/types': 7.0.2 @@ -5231,36 +7086,64 @@ packages: util-deprecate: 1.0.2 dev: true - /@storybook/core-client@7.0.2: - resolution: {integrity: sha512-tr6Uv41YD2O0xiUrtgujiY1QxuznhbyUI0BRsSh49e8cx3QoW7FgPy7IVZHgb17DXKZ/wY/hgdyTTB87H6IbLA==} + /@storybook/components@7.3.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-hsa1OJx4yEtLHTzrCxq8G9U5MTbcTuItj9yp1gsW9RTNc/V1n/rReQv4zE/k+//2hDsLrS62o3yhZ9VksRhLNw==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 + '@radix-ui/react-select': 1.2.2(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@radix-ui/react-toolbar': 1.0.4(@types/react-dom@18.2.7)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/csf': 0.1.1 + '@storybook/global': 5.0.0 + '@storybook/icons': 1.1.6(react-dom@18.2.0)(react@18.2.0) + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 + memoizerific: 1.11.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + use-resize-observer: 9.1.0(react-dom@18.2.0)(react@18.2.0) + util-deprecate: 1.0.2 + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' dev: true - /@storybook/core-common@7.0.2: - resolution: {integrity: sha512-DayFPTCj695tnEKLuDlogclBim8mzdrbj9U1xzFm23BUReheGSGdLl2zrb3mP1l9Zj4xJ/Ctst1KN9SFbW84vw==} + /@storybook/core-client@7.3.2: + resolution: {integrity: sha512-K2jCnjZiUUskFjKUj7m1FTCphIwBv0KPOE5JCd0UR7un1P1G1kdXMctADE6fHosrW73xRrad9CBSyyetUVQQOA==} dependencies: - '@storybook/node-logger': 7.0.2 - '@storybook/types': 7.0.2 - '@types/node': 16.18.11 + '@storybook/client-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 + dev: true + + /@storybook/core-common@7.3.2: + resolution: {integrity: sha512-W+X7JXV0UmHuUl9xSF/xzz1+P7VM8xHt7ORfp8yrtJRwLHURqHvFFQC+NUHBKno1Ydtt/Uch7QNOWUlQKmiWEw==} + dependencies: + '@storybook/node-logger': 7.3.2 + '@storybook/types': 7.3.2 + '@types/find-cache-dir': 3.2.1 + '@types/node': 16.18.46 + '@types/node-fetch': 2.6.4 '@types/pretty-hrtime': 1.0.1 chalk: 4.1.2 - esbuild: 0.17.12 - esbuild-register: 3.4.2(esbuild@0.17.12) - file-system-cache: 2.0.2 + esbuild: 0.18.20 + esbuild-register: 3.4.2(esbuild@0.18.20) + file-system-cache: 2.3.0 + find-cache-dir: 3.3.2 find-up: 5.0.0 - fs-extra: 11.1.0 - glob: 8.1.0 - glob-promise: 6.0.2(glob@8.1.0) - handlebars: 4.7.7 + fs-extra: 11.1.1 + glob: 10.3.3 + handlebars: 4.7.8 lazy-universal-dotenv: 4.0.0 + node-fetch: 2.6.9 picomatch: 2.3.1 pkg-dir: 5.0.0 pretty-hrtime: 1.0.3 resolve-from: 5.0.0 ts-dedent: 2.2.0 transitivePeerDependencies: + - encoding - supports-color dev: true @@ -5268,51 +7151,55 @@ packages: resolution: {integrity: sha512-1DCHCwHRL3+rlvnVVc/BCfReP31XaT2WYgcLeGTmkX1E43Po1MkgcM7PnJPSaa9POvSqZ+6YLZv5Bs1SXbufow==} dev: true - /@storybook/core-server@7.0.2: - resolution: {integrity: sha512-7ipGws8YffVaiwkc+D0+MfZc/Sy52aKenG3nDJdK4Ajmp5LPAlelb/sxIhfRvoHDbDsy2FQNz++Mb55Yh03KkA==} + /@storybook/core-events@7.3.2: + resolution: {integrity: sha512-DCrM3s+sxLKS8vl0zB+1tZEtcl5XQTOGl46XgRRV/SIBabFbsC0l5pQPswWkTUsIqdREtiT0YUHcXB1+YDyFvA==} + dev: true + + /@storybook/core-server@7.3.2: + resolution: {integrity: sha512-TLMEptmfqYLu4bayRV5m8T3R50uR07Fwja1n/8CCmZOGWjnr5kXMFRkD7+hj7wm82yoidfd23bmVcRU9mlG+tg==} dependencies: - '@aw-web-design/x-default-browser': 1.4.88 + '@aw-web-design/x-default-browser': 1.4.126 '@discoveryjs/json-ext': 0.5.7 - '@storybook/builder-manager': 7.0.2 - '@storybook/core-common': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 - '@storybook/csf-tools': 7.0.2 + '@storybook/builder-manager': 7.3.2 + '@storybook/channels': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 + '@storybook/csf-tools': 7.3.2 '@storybook/docs-mdx': 0.1.0 '@storybook/global': 5.0.0 - '@storybook/manager': 7.0.2 - '@storybook/node-logger': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/telemetry': 7.0.2 - '@storybook/types': 7.0.2 - '@types/detect-port': 1.3.2 - '@types/node': 16.18.11 - '@types/node-fetch': 2.6.2 + '@storybook/manager': 7.3.2 + '@storybook/node-logger': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/telemetry': 7.3.2 + '@storybook/types': 7.3.2 + '@types/detect-port': 1.3.3 + '@types/node': 16.18.46 '@types/pretty-hrtime': 1.0.1 '@types/semver': 7.3.13 - better-opn: 2.1.1 - boxen: 5.1.2 + better-opn: 3.0.2 chalk: 4.1.2 cli-table3: 0.6.3 compression: 1.7.4 detect-port: 1.5.1 express: 4.18.2 - fs-extra: 11.1.0 + fs-extra: 11.1.1 globby: 11.1.0 ip: 2.0.0 lodash: 4.17.21 - node-fetch: 2.6.9 - open: 8.4.0 + open: 8.4.2 pretty-hrtime: 1.0.3 prompts: 2.4.2 read-pkg-up: 7.0.1 semver: 7.3.8 serve-favicon: 2.5.0 - telejson: 7.0.4 + telejson: 7.2.0 + tiny-invariant: 1.3.1 ts-dedent: 2.2.0 + util: 0.12.5 util-deprecate: 1.0.2 watchpack: 2.4.0 - ws: 8.12.0 + ws: 8.13.0 transitivePeerDependencies: - bufferutil - encoding @@ -5320,33 +7207,33 @@ packages: - utf-8-validate dev: true - /@storybook/csf-plugin@7.0.2: - resolution: {integrity: sha512-aGuo+G6G5IwSGkmc+OUA796sOfvJMaQj8QS/Zh5F0nL4ZlQvghHpXON8cRHHvmXHQqUo07KLiy7CZh2I2oq4iQ==} + /@storybook/csf-plugin@7.3.2: + resolution: {integrity: sha512-uXJLJkRQeXnI2jHRdHfjJCbtEDohqzCrADh1xDfjqy/MQ/Sh2iFnRBCbEXsrxROBMh7Ow88/hJdy+vX0ZQh9fA==} dependencies: - '@storybook/csf-tools': 7.0.2 - unplugin: 0.10.2 + '@storybook/csf-tools': 7.3.2 + unplugin: 1.4.0 transitivePeerDependencies: - supports-color dev: true - /@storybook/csf-tools@7.0.2: - resolution: {integrity: sha512-sOp355yQSpYiMqNSopmFYWZkPPRJdGgy4tpxGGLxpOZMygK3j1wQ/WQtl2Z0h61KP0S0dl6hrs0pHQz3A/eVrw==} + /@storybook/csf-tools@7.3.2: + resolution: {integrity: sha512-54UaOsx9QZxiuMSpX01kSAEYuZYaB72Zz8ihlVrKZbIPTSJ6SYcM/jzNCGf1Rz7AjgU2UjXCSs5zBq5t37Nuqw==} dependencies: - '@babel/generator': 7.21.3 - '@babel/parser': 7.21.3 - '@babel/traverse': 7.21.3 - '@babel/types': 7.21.4 - '@storybook/csf': 0.1.0 - '@storybook/types': 7.0.2 - fs-extra: 11.1.0 - recast: 0.23.1 + '@babel/generator': 7.22.10 + '@babel/parser': 7.22.11 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + '@storybook/csf': 0.1.1 + '@storybook/types': 7.3.2 + fs-extra: 11.1.1 + recast: 0.23.4 ts-dedent: 2.2.0 transitivePeerDependencies: - supports-color dev: true - /@storybook/csf@0.1.0: - resolution: {integrity: sha512-uk+jMXCZ8t38jSTHk2o5btI+aV2Ksbvl6DoOv3r6VaCM1KZqeuMwtwywIQdflkA8/6q/dKT8z8L+g8hC4GC3VQ==} + /@storybook/csf@0.1.1: + resolution: {integrity: sha512-4hE3AlNVxR60Wc5KSC68ASYzUobjPqtSKyhV6G+ge0FIXU55N5nTY7dXGRZHQGDBPq+XqchMkIdlkHPRs8nTHg==} dependencies: type-fest: 2.19.0 dev: true @@ -5355,17 +7242,17 @@ packages: resolution: {integrity: sha512-JDaBR9lwVY4eSH5W8EGHrhODjygPd6QImRbwjAuJNEnY0Vw4ie3bPkeGfnacB3OBW6u/agqPv2aRlR46JcAQLg==} dev: true - /@storybook/docs-tools@7.0.2: - resolution: {integrity: sha512-w4D5BURrYjLbLGG9VKAaKU2dSdukszxRE3HWkJyhQU9R1JHvS3n8ntcMqYPqRfoHCOeBLBxP0edDYcAfzGNDYQ==} + /@storybook/docs-tools@7.3.2: + resolution: {integrity: sha512-MSmAiL/lg+B14CIKD6DvkBPdTDfGBSSt3bE+vW2uW9ohNJB5eWePZLQZUe34uZuunn3uqyTAgbEF7KjrtGZ/MQ==} dependencies: - '@babel/core': 7.21.3 - '@storybook/core-common': 7.0.2 - '@storybook/preview-api': 7.0.2 - '@storybook/types': 7.0.2 + '@storybook/core-common': 7.3.2 + '@storybook/preview-api': 7.3.2 + '@storybook/types': 7.3.2 '@types/doctrine': 0.0.3 doctrine: 3.0.0 lodash: 4.17.21 transitivePeerDependencies: + - encoding - supports-color dev: true @@ -5373,20 +7260,31 @@ packages: resolution: {integrity: sha512-FcOqPAXACP0I3oJ/ws6/rrPT9WGhu915Cg8D02a9YxLo0DE9zI+a9A5gRGvmQ09fiWPukqI8ZAEoQEdWUKMQdQ==} dev: true - /@storybook/manager-api@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-PbLj9Rc5uCMPfMdaXv1wE3koA3+d0rmZ3BJI8jeq+mfZEvpvfI4OOpRioT1q04CkkVomFOVFTyO0Q/o6Rb5N7g==} + /@storybook/icons@1.1.6(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-co5gDCYPojRAc5lRMnWxbjrR1V37/rTmAo9Vok4a1hDpHZIwkGTWesdzvYivSQXYFxZTpxdM1b5K3W87brnahw==} + engines: {node: '>=14.0.0'} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /@storybook/manager-api@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-EEosLcc+CPLjorLf2+rGLBW0sH0SHVcB1yClLIzKM5Wt8Cl/0l19wNtGMooE/28SDLA4DPIl4WDnP83wRE1hsg==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 - '@storybook/router': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/theming': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/router': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 dequal: 2.0.3 lodash: 4.17.21 memoizerific: 1.11.3 @@ -5394,57 +7292,51 @@ packages: react-dom: 18.2.0(react@18.2.0) semver: 7.3.8 store2: 2.14.2 - telejson: 7.0.4 + telejson: 7.2.0 ts-dedent: 2.2.0 dev: true - /@storybook/manager@7.0.2: - resolution: {integrity: sha512-jsFsFKG0rPNYfuRm/WSXGMBy8vnALyFWU330ObDmfU0JID3SeLlVqAOZT1GlwI6vupYpWodsN6qPZKRmC8onRw==} + /@storybook/manager@7.3.2: + resolution: {integrity: sha512-nA3XcnD36WUjgMCtID2M4DWYZh6MnabItXvKXGbNUkI8SVaIekc5nEgeplFyqutL11eKz3Es/FwwEP+mePbWfw==} dev: true - /@storybook/mdx2-csf@1.0.0: - resolution: {integrity: sha512-dBAnEL4HfxxJmv7LdEYUoZlQbWj9APZNIbOaq0tgF8XkxiIbzqvgB0jhL/9UOrysSDbQWBiCRTu2wOVxedGfmw==} + /@storybook/mdx2-csf@1.1.0: + resolution: {integrity: sha512-TXJJd5RAKakWx4BtpwvSNdgTDkKM6RkXU8GK34S/LhidQ5Pjz3wcnqb0TxEkfhK/ztbP8nKHqXFwLfa2CYkvQw==} dev: true - /@storybook/node-logger@7.0.2: - resolution: {integrity: sha512-UENpXxB1yDqP7JXaODJo+pbGt5y3NFBNurBr4+pI4bMAC4ARjpgRE4wp6fxUKFPu9MAR10oCdcLEHkaVUAjYRg==} - dependencies: - '@types/npmlog': 4.1.4 - chalk: 4.1.2 - npmlog: 5.0.1 - pretty-hrtime: 1.0.3 + /@storybook/node-logger@7.3.2: + resolution: {integrity: sha512-XCCYiLa5mQ7KeDQcZ4awlyWDmtxJHLIJeedvXx29JUNztUjgwyon9rlNvxtxtGj6171zgn9MERFh920WyJOOOQ==} dev: true - /@storybook/postinstall@7.0.2: - resolution: {integrity: sha512-Hhiu3+N3ZDcbrhOCBJTDJbn/mC4l0v3ziyAP3yalq/2ZR9R5kfsEHHakKmswsKKV+ey0gNGijFTy3soU5oSs+A==} + /@storybook/postinstall@7.3.2: + resolution: {integrity: sha512-23/QUseeVaYjqexq4O1f1g/Fxq+pNGD+/wbXLPkdwNydutGwMZ3XAD8jcm+zeOmkbUPN8jQzKUXqO2OE/GgvHg==} dev: true - /@storybook/preview-api@7.0.2: - resolution: {integrity: sha512-QAlJM/r92+dQe/kB7MTTR9b/1mt9UJjxNjazGdEWipA/nw23kOF3o/hBcvKwBYkit4zGYsX70H+vuzW8hCo/lA==} + /@storybook/preview-api@7.3.2: + resolution: {integrity: sha512-exQrWQQLwf/nXB6OEuQScygN5iO914iNQAvicaJ7mrX9L1ypIq1PpXgJR3mSezBd9dhOMBP/BMy1Zck/wBEL9A==} dependencies: - '@storybook/channel-postmessage': 7.0.2 - '@storybook/channels': 7.0.2 - '@storybook/client-logger': 7.0.2 - '@storybook/core-events': 7.0.2 - '@storybook/csf': 0.1.0 + '@storybook/channels': 7.3.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-events': 7.3.2 + '@storybook/csf': 0.1.1 '@storybook/global': 5.0.0 - '@storybook/types': 7.0.2 + '@storybook/types': 7.3.2 '@types/qs': 6.9.7 dequal: 2.0.3 lodash: 4.17.21 memoizerific: 1.11.3 - qs: 6.11.0 - synchronous-promise: 2.0.16 + qs: 6.11.2 + synchronous-promise: 2.0.17 ts-dedent: 2.2.0 util-deprecate: 1.0.2 dev: true - /@storybook/preview@7.0.2: - resolution: {integrity: sha512-U7MZkDT9bBq7HggLAXmTO9gI4eqhYs26fZS0L6iTE/PCX4Wg2TJBJSq2X8jhDXRqJFOt8SrQ756+V5Vtwrh4Og==} + /@storybook/preview@7.3.2: + resolution: {integrity: sha512-UXgImhD7xa+nYgXRcNFQdTqQT1725mOzWbQUtYPMJXkHO+t251hQrEc81tMzSSPEgPrFY8wndpEqTt8glFm91g==} dev: true - /@storybook/react-dom-shim@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-fMl0aV7mJ3wyQKvt6z+rZuiIiSd9YinS77IJ1ETHqVZ4SxWriOS0GFKP6sZflrlpShoZBh+zl1lDPG7ZZdrQGw==} + /@storybook/react-dom-shim@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-63ysybmpl9UULmLu/aUwWwhjf5QEWTvnMW9r8Z3LF3sW8Z698ZsssdThzNWqw0zlwTlgnQA4ta2Df4/oVXR0+Q==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5453,8 +7345,8 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true - /@storybook/react-vite@7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2)(vite@4.2.0): - resolution: {integrity: sha512-1bDrmGo6imxBzZKJJ+SEHPuDn474JY3Yatm0cPaNVtlYhbnbiTPa3PxhI4U3233l4Qsc6DXNLKvi++j/knXDCw==} + /@storybook/react-vite@7.3.2(react-dom@18.2.0)(react@18.2.0)(rollup@2.79.1)(typescript@5.0.2)(vite@4.2.0): + resolution: {integrity: sha512-lfDrcESQkrqVh1PkFgiJMrfhGYNckQ3rB2ZCvvzruHYWe/9B40EbMxGZauLg7B7M5j2Rzj+sa7Jfr3dasm+GJA==} engines: {node: '>=16'} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5462,26 +7354,27 @@ packages: vite: ^3.0.0 || ^4.0.0 dependencies: '@joshwooding/vite-plugin-react-docgen-typescript': 0.2.1(typescript@5.0.2)(vite@4.2.0) - '@rollup/pluginutils': 4.2.1 - '@storybook/builder-vite': 7.0.2(typescript@5.0.2)(vite@4.2.0) - '@storybook/react': 7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) + '@rollup/pluginutils': 5.0.4(rollup@2.79.1) + '@storybook/builder-vite': 7.3.2(typescript@5.0.2)(vite@4.2.0) + '@storybook/react': 7.3.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2) '@vitejs/plugin-react': 3.1.0(vite@4.2.0) ast-types: 0.14.2 - magic-string: 0.27.0 + magic-string: 0.30.3 react: 18.2.0 react-docgen: 6.0.0-alpha.3 react-dom: 18.2.0(react@18.2.0) vite: 4.2.0(@types/node@18.16.3)(sass@1.59.3) transitivePeerDependencies: - '@preact/preset-vite' - - '@storybook/mdx1-csf' + - encoding + - rollup - supports-color - typescript - vite-plugin-glimmerx dev: true - /@storybook/react@7.0.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2): - resolution: {integrity: sha512-2P7Oju1XKWMyn75dO0vjL4gthzBL/lLiCBRyAHKXZJ1H2eNdWjXkOOtH1HxnbRcXjWSU4tW96dqKY8m0iR9zAA==} + /@storybook/react@7.3.2(react-dom@18.2.0)(react@18.2.0)(typescript@5.0.2): + resolution: {integrity: sha512-VMXy+soLnEW+lN1sfkkMGkmk3gnS3KLfEk0JssSlj+jGA4cPpvO+P1uGNkN8MjdiU9VaWt0aZ7uRdwx0rrfFUw==} engines: {node: '>=16.0.0'} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 @@ -5491,21 +7384,21 @@ packages: typescript: optional: true dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/core-client': 7.0.2 - '@storybook/docs-tools': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-client': 7.3.2 + '@storybook/docs-tools': 7.3.2 '@storybook/global': 5.0.0 - '@storybook/preview-api': 7.0.2 - '@storybook/react-dom-shim': 7.0.2(react-dom@18.2.0)(react@18.2.0) - '@storybook/types': 7.0.2 + '@storybook/preview-api': 7.3.2 + '@storybook/react-dom-shim': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/types': 7.3.2 '@types/escodegen': 0.0.6 '@types/estree': 0.0.51 - '@types/node': 16.18.11 + '@types/node': 16.18.46 acorn: 7.4.1 acorn-jsx: 5.3.2(acorn@7.4.1) acorn-walk: 7.2.0 - escodegen: 2.0.0 - html-tags: 3.2.0 + escodegen: 2.1.0 + html-tags: 3.3.1 lodash: 4.17.21 prop-types: 15.8.1 react: 18.2.0 @@ -5516,33 +7409,33 @@ packages: typescript: 5.0.2 util-deprecate: 1.0.2 transitivePeerDependencies: + - encoding - supports-color dev: true - /@storybook/router@7.0.2(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-ZB2vucfayZUrMLBlXju4v6CNOQQb0YKDLw5RoojdBxOsUFtnp5UiPOE+I8PQR63EBwnRjozeibV1XSM+GlQb5w==} + /@storybook/router@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-J3QPudwCJhdnfqPx9GaNDlnsjJ6JbFta/ypp3EkHntyuuaNBeNP3Aq73DJJY2XMTS2Xdw8tD9Y9Y9gCFHJXMDQ==} peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@storybook/client-logger': 7.0.2 + '@storybook/client-logger': 7.3.2 memoizerific: 1.11.3 - qs: 6.11.0 + qs: 6.11.2 react: 18.2.0 react-dom: 18.2.0(react@18.2.0) dev: true - /@storybook/telemetry@7.0.2: - resolution: {integrity: sha512-s2PIwI9nVYQBf3h40EFHLynYUfdqzRJMXyaCWJdVQuvdQfRkAn3CLXaubK+VdjC869z3ZfW20EMu3Mbgzcc0HA==} + /@storybook/telemetry@7.3.2: + resolution: {integrity: sha512-BmgwaZGoR2ZzGZpcO5ipc4uMd9y28qmu9Ynx054Q3mb86daJrw4CU18TVi5UoFa9qmygQhoHx2gaK2QStNtqCg==} dependencies: - '@storybook/client-logger': 7.0.2 - '@storybook/core-common': 7.0.2 + '@storybook/client-logger': 7.3.2 + '@storybook/core-common': 7.3.2 + '@storybook/csf-tools': 7.3.2 chalk: 4.1.2 detect-package-manager: 2.0.1 - fetch-retry: 5.0.3 - fs-extra: 11.1.0 - isomorphic-unfetch: 3.1.0 - nanoid: 3.3.4 + fetch-retry: 5.0.6 + fs-extra: 11.1.1 read-pkg-up: 7.0.1 transitivePeerDependencies: - encoding @@ -5555,7 +7448,7 @@ packages: react: ^16.8.0 || ^17.0.0 || ^18.0.0 react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 dependencies: - '@emotion/use-insertion-effect-with-fallbacks': 1.0.0(react@18.2.0) + '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) '@storybook/client-logger': 7.0.2 '@storybook/global': 5.0.0 memoizerific: 1.11.3 @@ -5563,6 +7456,20 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true + /@storybook/theming@7.3.2(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-npVsnmNAtqGwl1K7vLC/hcVhL8tBC8G0vdZXEcufF0jHdQmRCUs9ZVrnR6W0LCrtmIHDaDoO7PqJVSzu2wgVxw==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@emotion/use-insertion-effect-with-fallbacks': 1.0.1(react@18.2.0) + '@storybook/client-logger': 7.3.2 + '@storybook/global': 5.0.0 + memoizerific: 1.11.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + /@storybook/types@7.0.2: resolution: {integrity: sha512-0OCt/kAexa8MCcljxA+yZxGMn0n2U2Ync0KxotItqNbKBKVkaLQUls0+IXTWSCpC/QJvNZ049jxUHHanNi/96w==} dependencies: @@ -5572,6 +7479,15 @@ packages: file-system-cache: 2.0.2 dev: true + /@storybook/types@7.3.2: + resolution: {integrity: sha512-1UHC1r2J6H9dEpj4pp9a16P1rTL87V9Yc6TtYBpp7m+cxzyIZBRvu1wZFKmRB51RXE/uDaxGRKzfNRfgTALcIQ==} + dependencies: + '@storybook/channels': 7.3.2 + '@types/babel__core': 7.20.1 + '@types/express': 4.17.17 + file-system-cache: 2.3.0 + dev: true + /@svgr/babel-plugin-add-jsx-attribute@6.5.1(@babel/core@7.21.3): resolution: {integrity: sha512-9PYGcXrAxitycIjRmZB+Q0JaN07GZIWaTBIGQzfaZv+qr1n8X1XUEJ5rZ/vx6OVD9RRYlrNnXWExQXcmZeD/BQ==} engines: {node: '>=10'} @@ -5919,6 +7835,12 @@ packages: '@types/node': 18.16.3 dev: true + /@types/cross-spawn@6.0.2: + resolution: {integrity: sha512-KuwNhp3eza+Rhu8IFI5HUXRP0LIhqH5cAjubUvGXXthh4YYBuP2ntwEX+Cz8GJoZUHlKo247wPWOfA9LYEq4cw==} + dependencies: + '@types/node': 18.16.3 + dev: true + /@types/d3-array@3.0.4: resolution: {integrity: sha512-nwvEkG9vYOc0Ic7G7kwgviY4AQlTfYGIZ0fqB7CQHXGyYM6nO7kJh5EguSNA3jfh4rq7Sb7eMVq8isuvg2/miQ==} dev: false @@ -5967,8 +7889,8 @@ packages: '@types/ms': 0.7.31 dev: false - /@types/detect-port@1.3.2: - resolution: {integrity: sha512-xxgAGA2SAU4111QefXPSp5eGbDm/hW6zhvYl9IeEPZEry9F4d66QAHm5qpUXjb6IsevZV/7emAEx5MhP6O192g==} + /@types/detect-port@1.3.3: + resolution: {integrity: sha512-bV/jQlAJ/nPY3XqSatkGpu+nGzou+uSwrH1cROhn+jBFg47yaNH+blW4C7p9KhopC7QxCv/6M86s37k8dMk0Yg==} dev: true /@types/diff@5.0.2: @@ -5979,8 +7901,12 @@ packages: resolution: {integrity: sha512-w5jZ0ee+HaPOaX25X2/2oGR/7rgAQSYII7X7pp0m9KgBfMP7uKfMfTvcpl5Dj+eDBbpxKGiqE+flqDr6XTd2RA==} dev: true - /@types/ejs@3.1.1: - resolution: {integrity: sha512-RQul5wEfY7BjWm0sYY86cmUN/pcXWGyVxWX93DFFJvcrxax5zKlieLwA3T77xJGwNcZW0YW6CYG70p1m8xPFmA==} + /@types/ejs@3.1.2: + resolution: {integrity: sha512-ZmiaE3wglXVWBM9fyVC17aGPkLo/UgaOjEiI2FXQfyczrCefORPxIe+2dVmnmk3zkVIbizjrlQzmPGhSYGXG5g==} + dev: true + + /@types/emscripten@1.39.7: + resolution: {integrity: sha512-tLqYV94vuqDrXh515F/FOGtBcRMTPGvVV1LzLbtYDcQmmhtpf/gLYf+hikBbQk8MzOHNz37wpFfJbYAuSn8HqA==} dev: true /@types/es-aggregate-error@1.0.2: @@ -6009,12 +7935,21 @@ packages: resolution: {integrity: sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA==} dev: true - /@types/express-serve-static-core@4.17.33: - resolution: {integrity: sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==} + /@types/express-serve-static-core@4.17.33: + resolution: {integrity: sha512-TPBqmR/HRYI3eC2E5hmiivIzv+bidAfXofM+sbonAGvyDhySGw9/PQZFt2BLOrjUUR++4eJVpx6KnLQK1Fk9tA==} + dependencies: + '@types/node': 18.16.3 + '@types/qs': 6.9.7 + '@types/range-parser': 1.2.4 + dev: true + + /@types/express-serve-static-core@4.17.36: + resolution: {integrity: sha512-zbivROJ0ZqLAtMzgzIUC4oNqDG9iF0lSsAqpOD9kbs5xcIM3dTiyuHvBc7R8MtWBp3AAWGaovJa+wzWPjLYW7Q==} dependencies: '@types/node': 18.16.3 '@types/qs': 6.9.7 '@types/range-parser': 1.2.4 + '@types/send': 0.17.1 dev: true /@types/express@4.17.16: @@ -6026,6 +7961,15 @@ packages: '@types/serve-static': 1.15.0 dev: true + /@types/express@4.17.17: + resolution: {integrity: sha512-Q4FmmuLGBG58btUnfS1c1r/NQdlp3DMfGDGig8WhfpA2YRUtEkxAjkZb0yvplJGYdF1fsQ81iMDcH24sSCNC/Q==} + dependencies: + '@types/body-parser': 1.19.2 + '@types/express-serve-static-core': 4.17.36 + '@types/qs': 6.9.7 + '@types/serve-static': 1.15.2 + dev: true + /@types/find-cache-dir@3.2.1: resolution: {integrity: sha512-frsJrz2t/CeGifcu/6uRo4b+SzAwT4NYCVPu1GN8IB9XTzrpPkGuV0tmh9mN+/L0PklAlsC3u5Fxt0ju00LXIw==} dev: true @@ -6041,13 +7985,6 @@ packages: '@types/node': 18.16.3 dev: true - /@types/glob@8.1.0: - resolution: {integrity: sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==} - dependencies: - '@types/minimatch': 5.1.2 - '@types/node': 18.16.3 - dev: true - /@types/graceful-fs@4.1.6: resolution: {integrity: sha512-Sig0SNORX9fdW+bQuTEovKj3uHcUL6LQKbCrrqb1X7J6/ReAbhCXRAhc+SMejhLELFj2QcyuxmUooZ4bt5ReSw==} dependencies: @@ -6066,6 +8003,10 @@ packages: '@types/react': 18.2.20 hoist-non-react-statics: 3.3.2 + /@types/http-errors@2.0.1: + resolution: {integrity: sha512-/K3ds8TRAfBvi5vfjuz8y6+GiAYBZ0x4tXv1Av6CWBWn0IlADc+ZX9pMq7oU0fNQPnBwIZl3rmeLp6SBApbxSQ==} + dev: true + /@types/istanbul-lib-coverage@2.0.4: resolution: {integrity: sha512-z/QT1XN4K4KYuslS23k62yDIDLwLFkzxOuMplDtObz0+y7VqJCaO2o+SPwHCvLFZh7xazvvoor2tA/hPz9ee7g==} dev: true @@ -6130,14 +8071,18 @@ packages: resolution: {integrity: sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==} dev: false - /@types/mdx@2.0.3: - resolution: {integrity: sha512-IgHxcT3RC8LzFLhKwP3gbMPeaK7BM9eBH46OdapPA7yvuIUJ8H6zHZV53J8hGZcTSnt95jANt+rTBNUUc22ACQ==} + /@types/mdx@2.0.7: + resolution: {integrity: sha512-BG4tyr+4amr3WsSEmHn/fXPqaCba/AYZ7dsaQTiavihQunHSIxk+uAtqsjvicNpyHN6cm+B9RVrUOtW9VzIKHw==} dev: true /@types/mime-types@2.1.1: resolution: {integrity: sha512-vXOTGVSLR2jMw440moWTC7H19iUyLtP3Z1YTj7cSsubOICinjMxFeb/V57v9QdyyPGbbWolUFSSmSiRSn94tFw==} dev: true + /@types/mime@1.3.2: + resolution: {integrity: sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==} + dev: true + /@types/mime@3.0.1: resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} dev: true @@ -6159,13 +8104,21 @@ packages: dependencies: '@types/node': 18.16.3 form-data: 3.0.1 + dev: false + + /@types/node-fetch@2.6.4: + resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==} + dependencies: + '@types/node': 18.16.3 + form-data: 3.0.1 + dev: true /@types/node@14.18.43: resolution: {integrity: sha512-n3eFEaoem0WNwLux+k272P0+aq++5o05bA9CfiwKPdYPB5ZambWKdWoeHy7/OJiizMhzg27NLaZ6uzjLTzXceQ==} dev: true - /@types/node@16.18.11: - resolution: {integrity: sha512-3oJbGBUWuS6ahSnEq1eN2XrCyf4YsWI8OyCvo7c64zQJNplk3mO84t53o8lfTk+2ji59g5ycfc6qQ3fdHliHuA==} + /@types/node@16.18.46: + resolution: {integrity: sha512-Mnq3O9Xz52exs3mlxMcQuA7/9VFe/dXcrgAyfjLkABIqxXKOgBRjyazTxUbjsxDa4BP7hhPliyjVTP9RDP14xg==} dev: true /@types/node@18.16.3: @@ -6175,10 +8128,6 @@ packages: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} dev: true - /@types/npmlog@4.1.4: - resolution: {integrity: sha512-WKG4gTr8przEZBiJ5r3s8ZIAoMXNbOgQ+j/d5O4X3x6kZJRLNvyUJuUK/KoG3+8BaOHPhp2m7WC6JKKeovDSzQ==} - dev: true - /@types/parse-json@4.0.0: resolution: {integrity: sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==} @@ -6305,6 +8254,13 @@ packages: /@types/semver@7.3.13: resolution: {integrity: sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==} + /@types/send@0.17.1: + resolution: {integrity: sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==} + dependencies: + '@types/mime': 1.3.2 + '@types/node': 18.16.3 + dev: true + /@types/serve-static@1.15.0: resolution: {integrity: sha512-z5xyF6uh8CbjAu9760KDKsH2FcDxZ2tFCsA4HIMWE6IkiYMXfVoa+4f9KX+FN0ZLsaMw1WNG2ETLA6N+/YA+cg==} dependencies: @@ -6312,6 +8268,14 @@ packages: '@types/node': 18.16.3 dev: true + /@types/serve-static@1.15.2: + resolution: {integrity: sha512-J2LqtvFYCzaj8pVYKw8klQXrLLk7TBZmQ4ShlcdkELFKGwGMfevMLneMMRkMgZxotOD9wg497LpC7O8PcvAmfw==} + dependencies: + '@types/http-errors': 2.0.1 + '@types/mime': 3.0.1 + '@types/node': 18.16.3 + dev: true + /@types/sinonjs__fake-timers@8.1.1: resolution: {integrity: sha512-0kSuKjAS0TrGLJ0M/+8MaFkGsQhZpB6pxOmvS3K8FYI72K//YmdfoW9X2qPsAKh1mkwxGD5zib9s1FIFed6E8g==} dev: true @@ -6615,14 +8579,30 @@ packages: resolution: {integrity: sha512-N8tkAACJx2ww8vFMneJmaAgmjAG1tnVBZJRLRcx061tmsLRZHSEZSLuGWnwPtunsSLvSqXQ2wfp7Mgqg1I+2dQ==} dev: false - /@yarnpkg/esbuild-plugin-pnp@3.0.0-rc.15(esbuild@0.17.12): + /@yarnpkg/esbuild-plugin-pnp@3.0.0-rc.15(esbuild@0.18.20): resolution: {integrity: sha512-kYzDJO5CA9sy+on/s2aIW0411AklfCi8Ck/4QDivOqsMKpStZA2SsR+X27VTggGwpStWaLrjJcDcdDMowtG8MA==} engines: {node: '>=14.15.0'} peerDependencies: esbuild: '>=0.10.0' dependencies: - esbuild: 0.17.12 - tslib: 2.5.2 + esbuild: 0.18.20 + tslib: 2.6.2 + dev: true + + /@yarnpkg/fslib@2.10.3: + resolution: {integrity: sha512-41H+Ga78xT9sHvWLlFOZLIhtU6mTGZ20pZ29EiZa97vnxdohJD2AF42rCoAoWfqUz486xY6fhjMH+DYEM9r14A==} + engines: {node: '>=12 <14 || 14.2 - 14.9 || >14.10.0'} + dependencies: + '@yarnpkg/libzip': 2.3.0 + tslib: 1.14.1 + dev: true + + /@yarnpkg/libzip@2.3.0: + resolution: {integrity: sha512-6xm38yGVIa6mKm/DUCF2zFFJhERh/QWp1ufm4cNUvxsONBmfPg8uZ9pZBdOmF6qFGr/HlT6ABBkCSx/dlEtvWg==} + engines: {node: '>=12 <14 || 14.2 - 14.9 || >14.10.0'} + dependencies: + '@types/emscripten': 1.39.7 + tslib: 1.14.1 dev: true /abab@2.0.6: @@ -6687,6 +8667,12 @@ packages: hasBin: true dev: true + /acorn@8.10.0: + resolution: {integrity: sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: true + /acorn@8.8.1: resolution: {integrity: sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==} engines: {node: '>=0.4.0'} @@ -6778,12 +8764,6 @@ packages: resolution: {integrity: sha512-nqLm4HxOTpeLOxcmB3QWmV5TcDFhW9y/fyQ+hivtDFcK4OQ+pQ5fzPnXHM1Mfcm0VkLtvVi1TCPr++Qy0Q/3EQ==} dev: false - /ansi-align@3.0.1: - resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} - dependencies: - string-width: 4.2.3 - dev: true - /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -6842,22 +8822,10 @@ packages: resolution: {integrity: sha512-jlpIfsOoNoafl92Sz//64uQHGSyMrD2vYG5d8o2a4qGvyNCvXur7bzIsWtAC/6flI2RYAp3kv8rsfBtaLm7w0g==} dev: true - /aproba@2.0.0: - resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==} - dev: true - /arch@2.2.0: resolution: {integrity: sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==} dev: true - /are-we-there-yet@2.0.0: - resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==} - engines: {node: '>=10'} - dependencies: - delegates: 1.0.0 - readable-stream: 3.6.0 - dev: true - /arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} dev: true @@ -6880,7 +8848,6 @@ packages: engines: {node: '>=10'} dependencies: tslib: 2.5.2 - dev: false /aria-query@5.1.3: resolution: {integrity: sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ==} @@ -7004,21 +8971,21 @@ packages: resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} engines: {node: '>=4'} dependencies: - tslib: 2.5.2 + tslib: 2.6.2 dev: true /ast-types@0.14.2: resolution: {integrity: sha512-O0yuUDnZeQDL+ncNGlJ78BiO4jnYI3bvMsD5prT0/nsgijG/LpNBIr63gTjVTNsiGkgQhiyCShTgxt8oXOrklA==} engines: {node: '>=4'} dependencies: - tslib: 2.5.2 + tslib: 2.6.2 dev: true /ast-types@0.15.2: resolution: {integrity: sha512-c27loCv9QkZinsa5ProX751khO9DJl/AcB5c2KNtA6NRvHKS0PgLfcftz72KVq504vB0Gku5s2kUZzDBvQWvHg==} engines: {node: '>=4'} dependencies: - tslib: 2.5.2 + tslib: 2.6.2 dev: true /ast-types@0.16.1: @@ -7175,6 +9142,19 @@ packages: - supports-color dev: true + /babel-plugin-polyfill-corejs2@0.4.5(@babel/core@7.22.11): + resolution: {integrity: sha512-19hwUH5FKl49JEsvyTcoHakh6BE0wgXLLptIyKZ3PijHc/Ci521wygORCUCCred+E/twuqRyAkE02BAWPmsHOg==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/compat-data': 7.22.9 + '@babel/core': 7.22.11 + '@babel/helper-define-polyfill-provider': 0.4.2(@babel/core@7.22.11) + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: true + /babel-plugin-polyfill-corejs3@0.6.0(@babel/core@7.21.3): resolution: {integrity: sha512-+eHqR6OPcBhJOGgsIar7xoAB1GcSwVUA3XjAd7HJNzOXT4wv6/H7KIdA/Nc60cvUlDbKApmqNvD1B1bzOt4nyA==} peerDependencies: @@ -7187,6 +9167,18 @@ packages: - supports-color dev: true + /babel-plugin-polyfill-corejs3@0.8.3(@babel/core@7.22.11): + resolution: {integrity: sha512-z41XaniZL26WLrvjy7soabMXrfPWARN25PZoriDEiLMxAp50AUW3t35BGQUMg5xK3UrpVTtagIDklxYa+MhiNA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-define-polyfill-provider': 0.4.2(@babel/core@7.22.11) + core-js-compat: 3.32.1 + transitivePeerDependencies: + - supports-color + dev: true + /babel-plugin-polyfill-regenerator@0.4.1(@babel/core@7.21.3): resolution: {integrity: sha512-NtQGmyQDXjQqQ+IzRkBVwEOz9lQ4zxAQZgoAYEtU9dJjnl1Oc98qnN7jcp+bE7O7aYzVpavXE3/VKXNzUbh7aw==} peerDependencies: @@ -7198,6 +9190,17 @@ packages: - supports-color dev: true + /babel-plugin-polyfill-regenerator@0.5.2(@babel/core@7.22.11): + resolution: {integrity: sha512-tAlOptU0Xj34V1Y2PNTL4Y0FOJMDB6bZmoW39FeCQIhigGLkqu3Fj6uiXpxIf6Ij274ENdYx64y6Au+ZKlb1IA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.22.11 + '@babel/helper-define-polyfill-provider': 0.4.2(@babel/core@7.22.11) + transitivePeerDependencies: + - supports-color + dev: true + /babel-plugin-styled-components@2.0.7(styled-components@5.3.9): resolution: {integrity: sha512-i7YhvPgVqRKfoQ66toiZ06jPNA3p6ierpfUuEWxNF+fV27Uv5gxBkf8KZLHUCc1nFA9j6+80pYoIpqCeyW3/bA==} peerDependencies: @@ -7299,11 +9302,11 @@ packages: tweetnacl: 0.14.5 dev: true - /better-opn@2.1.1: - resolution: {integrity: sha512-kIPXZS5qwyKiX/HcRvDYfmBQUa8XP17I0mYZZ0y4UhpYOSvtsLHDYqmomS+Mj20aDvD3knEiQ0ecQy2nhio3yA==} - engines: {node: '>8.0.0'} + /better-opn@3.0.2: + resolution: {integrity: sha512-aVNobHnJqLiUelTaHat9DZ1qM2w0C0Eym4LPI/3JxOnSokGVdsl1T1kN7TFvsEAD8G47A6VKQ0TVHqbBnYMJlQ==} + engines: {node: '>=12.0.0'} dependencies: - open: 7.4.2 + open: 8.4.2 dev: true /big-integer@1.6.51: @@ -7320,7 +9323,7 @@ packages: dependencies: buffer: 5.7.1 inherits: 2.0.4 - readable-stream: 3.6.0 + readable-stream: 3.6.2 dev: true /blob-util@2.0.2: @@ -7351,20 +9354,6 @@ packages: - supports-color dev: true - /boxen@5.1.2: - resolution: {integrity: sha512-9gYgQKXx+1nP8mP7CzFyaUARhg7D3n1dF/FnErWmu9l6JvGpNUN278h0aSb+QjoiKSWG+iZ3uHrcqk0qrY9RQQ==} - engines: {node: '>=10'} - dependencies: - ansi-align: 3.0.1 - camelcase: 6.3.0 - chalk: 4.1.2 - cli-boxes: 2.2.1 - string-width: 4.2.3 - type-fest: 0.20.2 - widest-line: 3.1.0 - wrap-ansi: 7.0.0 - dev: true - /bplist-parser@0.2.0: resolution: {integrity: sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw==} engines: {node: '>= 5.10.0'} @@ -7401,6 +9390,17 @@ packages: pako: 0.2.9 dev: true + /browserslist@4.21.10: + resolution: {integrity: sha512-bipEBdZfVH5/pwrvqc+Ub0kUPVfGUhlKxbvfD+z1BDnPEO/X98ruXGA1WP5ASpAFKan7Qr6j736IacbZQuAlKQ==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + dependencies: + caniuse-lite: 1.0.30001524 + electron-to-chromium: 1.4.503 + node-releases: 2.0.13 + update-browserslist-db: 1.0.11(browserslist@4.21.10) + dev: true + /browserslist@4.21.4: resolution: {integrity: sha512-CBHJJdDmgjl3daYjN5Cp5kbTf1mUhZoS+beLklHIvkOWscs83YAhLlF3Wsh/lciQYAcbBJgTOD44VtG31ZM4Hw==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -7451,8 +9451,8 @@ packages: engines: {node: '>= 0.8'} dev: true - /c8@7.12.0: - resolution: {integrity: sha512-CtgQrHOkyxr5koX1wEUmN/5cfDa2ckbHRA4Gy5LAL0zaCFtVWJS5++n+w4/sr2GWGerBxgTjpKeDclk/Qk6W/A==} + /c8@7.14.0: + resolution: {integrity: sha512-i04rtkkcNcCf7zsQcSv/T9EbUn4RXQ6mropeMcjFOsQXQ0iGLAr/xT6TImQg4+U9hmNpN9XdvPkjUL1IzbgxJw==} engines: {node: '>=10.12.0'} hasBin: true dependencies: @@ -7461,8 +9461,8 @@ packages: find-up: 5.0.0 foreground-child: 2.0.0 istanbul-lib-coverage: 3.2.0 - istanbul-lib-report: 3.0.0 - istanbul-reports: 3.1.5 + istanbul-lib-report: 3.0.1 + istanbul-reports: 3.1.6 rimraf: 3.0.2 test-exclude: 6.0.0 v8-to-istanbul: 9.1.0 @@ -7521,6 +9521,10 @@ packages: /caniuse-lite@1.0.30001445: resolution: {integrity: sha512-8sdQIdMztYmzfTMO6KfLny878Ln9c2M0fc7EH60IjlP4Dc4PiCy7K2Vl3ITmWgOyPgVQKa5x+UP/KqFsxj4mBg==} + /caniuse-lite@1.0.30001524: + resolution: {integrity: sha512-Jj917pJtYg9HSJBF95HVX3Cdr89JUyLT4IZ8SvM5aDRni95swKgYi3TgYLH5hnGfPE/U1dg6IfZ50UsIlLkwSA==} + dev: true + /caseless@0.12.0: resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} dev: true @@ -7591,7 +9595,7 @@ packages: normalize-path: 3.0.0 readdirp: 3.6.0 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 /chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} @@ -7620,11 +9624,6 @@ packages: engines: {node: '>=6'} dev: true - /cli-boxes@2.2.1: - resolution: {integrity: sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==} - engines: {node: '>=6'} - dev: true - /cli-cursor@3.1.0: resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} engines: {node: '>=8'} @@ -7632,6 +9631,11 @@ packages: restore-cursor: 3.1.0 dev: true + /cli-spinners@2.9.0: + resolution: {integrity: sha512-4/aL9X3Wh0yiMQlE+eeRhWP6vclO3QRtw1JHKIT0FFUs5FjpFmESqtMvYZ0+lbzBw900b95mS0hohy+qn2VK/g==} + engines: {node: '>=6'} + dev: true + /cli-table3@0.6.3: resolution: {integrity: sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==} engines: {node: 10.* || >= 12.*} @@ -7722,11 +9726,6 @@ packages: /color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - /color-support@1.1.3: - resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} - hasBin: true - dev: true - /colord@2.9.3: resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} dev: true @@ -7735,6 +9734,10 @@ packages: resolution: {integrity: sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==} dev: true + /colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + dev: true + /combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -7809,7 +9812,7 @@ packages: dependencies: buffer-from: 1.1.2 inherits: 2.0.4 - readable-stream: 2.3.7 + readable-stream: 2.3.8 typedarray: 0.0.6 dev: true @@ -7817,10 +9820,6 @@ packages: resolution: {integrity: sha512-JsPKdmh8ZkmnHxDk55FZ1TqVLvEQTvoByJZRN9jzI0UjxK/QgAmsphz7PGtqgPieQZ/CQcHWXCR7ATDNhGe+YA==} dev: true - /console-control-strings@1.1.0: - resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} - dev: true - /content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -7874,6 +9873,12 @@ packages: browserslist: 4.21.4 dev: true + /core-js-compat@3.32.1: + resolution: {integrity: sha512-GSvKDv4wE0bPnQtjklV101juQ85g6H3rm5PDP20mqlS5j0kXF3pP97YvAu5hl+uFHqMictp3b2VxOHljWMAtuA==} + dependencies: + browserslist: 4.21.10 + dev: true + /core-util-is@1.0.2: resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} dev: true @@ -8365,10 +10370,6 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - /delegates@1.0.0: - resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} - dev: true - /depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -8398,6 +10399,10 @@ packages: engines: {node: '>=8'} dev: true + /detect-node-es@1.1.0: + resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} + dev: true + /detect-package-manager@2.0.1: resolution: {integrity: sha512-j/lJHyoLlWi6G1LDdLgvUtz60Zo5GEj+sVYtTVXnYLDPuzgC3llMxonXym9zIwhhUII8vjdw0LXxavpLqTbl1A==} engines: {node: '>=12'} @@ -8524,7 +10529,7 @@ packages: dependencies: end-of-stream: 1.4.4 inherits: 2.0.4 - readable-stream: 2.3.7 + readable-stream: 2.3.8 stream-shift: 1.0.1 dev: true @@ -8543,17 +10548,21 @@ packages: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} dev: true - /ejs@3.1.8: - resolution: {integrity: sha512-/sXZeMlhS0ArkfX2Aw780gJzXSMPnKjtspYZv+f3NiKLlubezAHDU5+9xz6gd3/NhG3txQCo6xlglmTS+oTGEQ==} + /ejs@3.1.9: + resolution: {integrity: sha512-rC+QVNMJWv+MtPgkt0y+0rVEIdbtxVADApW9JXrUVlzHetgcyczP/E7DJmWJ4fJCZF2cPcBk0laWO9ZHMG3DmQ==} engines: {node: '>=0.10.0'} hasBin: true dependencies: - jake: 10.8.5 + jake: 10.8.7 dev: true /electron-to-chromium@1.4.284: resolution: {integrity: sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==} + /electron-to-chromium@1.4.503: + resolution: {integrity: sha512-LF2IQit4B0VrUHFeQkWhZm97KuJSGF2WJqq1InpY+ECpFRkXd8yTIaTtJxsO0OKDmiBYwWqcrNaXOurn2T2wiA==} + dev: true + /emittery@0.13.1: resolution: {integrity: sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==} engines: {node: '>=12'} @@ -8600,8 +10609,8 @@ packages: resolution: {integrity: sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==} engines: {node: '>=0.12'} - /envinfo@7.8.1: - resolution: {integrity: sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==} + /envinfo@7.10.0: + resolution: {integrity: sha512-ZtUjZO6l5mwTHvc1L9+1q5p/R3wTopcfqMW8r5t8SJSKqeVI/LtajORwRFEKpEFuekjD0VBjwu1HMxL4UalIRw==} engines: {node: '>=4'} hasBin: true dev: true @@ -8920,13 +10929,13 @@ packages: resolution: {integrity: sha512-jyfL/pwPqaFXyKnj8lP8iLk6Z0m099uXR45aSN8Av1XD4vhvQutxxPzgA2bTcAwQpa1zCXDcWOlhFgyP3GKqhQ==} dev: true - /esbuild-register@3.4.2(esbuild@0.17.12): + /esbuild-register@3.4.2(esbuild@0.18.20): resolution: {integrity: sha512-kG/XyTDyz6+YDuyfB9ZoSIOOmgyFCH+xPRtsCa8W85HLRV5Csp+o3jWVbOSHgSLfyLc5DmP+KFDNwty4mEjC+Q==} peerDependencies: esbuild: '>=0.12 <1' dependencies: debug: 4.3.4(supports-color@5.5.0) - esbuild: 0.17.12 + esbuild: 0.18.20 transitivePeerDependencies: - supports-color dev: true @@ -9027,6 +11036,36 @@ packages: '@esbuild/win32-x64': 0.17.12 dev: true + /esbuild@0.18.20: + resolution: {integrity: sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + optionalDependencies: + '@esbuild/android-arm': 0.18.20 + '@esbuild/android-arm64': 0.18.20 + '@esbuild/android-x64': 0.18.20 + '@esbuild/darwin-arm64': 0.18.20 + '@esbuild/darwin-x64': 0.18.20 + '@esbuild/freebsd-arm64': 0.18.20 + '@esbuild/freebsd-x64': 0.18.20 + '@esbuild/linux-arm': 0.18.20 + '@esbuild/linux-arm64': 0.18.20 + '@esbuild/linux-ia32': 0.18.20 + '@esbuild/linux-loong64': 0.18.20 + '@esbuild/linux-mips64el': 0.18.20 + '@esbuild/linux-ppc64': 0.18.20 + '@esbuild/linux-riscv64': 0.18.20 + '@esbuild/linux-s390x': 0.18.20 + '@esbuild/linux-x64': 0.18.20 + '@esbuild/netbsd-x64': 0.18.20 + '@esbuild/openbsd-x64': 0.18.20 + '@esbuild/sunos-x64': 0.18.20 + '@esbuild/win32-arm64': 0.18.20 + '@esbuild/win32-ia32': 0.18.20 + '@esbuild/win32-x64': 0.18.20 + dev: true + /escalade@3.1.1: resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} engines: {node: '>=6'} @@ -9079,6 +11118,18 @@ packages: source-map: 0.6.1 dev: true + /escodegen@2.1.0: + resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} + engines: {node: '>=6.0'} + hasBin: true + dependencies: + esprima: 4.0.1 + estraverse: 5.3.0 + esutils: 2.0.3 + optionalDependencies: + source-map: 0.6.1 + dev: true + /eslint-config-prettier@8.7.0(eslint@8.36.0): resolution: {integrity: sha512-HHVXLSlVUhMSmyW4ZzEuvjpwqamgmlfkutD53cYXLikh4pt/modINRcCIApJ84czDxM4GZInwUrromsDdTImTA==} hasBin: true @@ -9542,9 +11593,9 @@ packages: resolution: {integrity: sha512-YNF+mZ/Wu2FU/gvmzuWtYc8rloubL7wfXCTgouFrnjGVXPA/EeYYA7pupXWrb3Iv1cTBeSSxxJIbK23l4MRNqg==} engines: {node: '>=8.3.0'} dependencies: - '@babel/traverse': 7.21.3 - '@babel/types': 7.21.3 - c8: 7.12.0 + '@babel/traverse': 7.22.11 + '@babel/types': 7.22.11 + c8: 7.14.0 transitivePeerDependencies: - supports-color dev: true @@ -9823,8 +11874,8 @@ packages: resolution: {integrity: sha512-qu4mXWf4wus4idBIN/kVH+XSer8IZ9CwHP+Pd7DL7TuKNC1hP7ykon4kkBjwJF3EMX2WsFp4hH7gU7CyL7ucXw==} dev: false - /fetch-retry@5.0.3: - resolution: {integrity: sha512-uJQyMrX5IJZkhoEUBQ3EjxkeiZkppBd5jS/fMTJmfZxLSiaQjv2zD0kTvuvkSH89uFvgSlB6ueGpjD3HWN7Bxw==} + /fetch-retry@5.0.6: + resolution: {integrity: sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ==} dev: true /figures@3.2.0: @@ -9848,6 +11899,13 @@ packages: ramda: 0.28.0 dev: true + /file-system-cache@2.3.0: + resolution: {integrity: sha512-l4DMNdsIPsVnKrgEXbJwDJsA5mB8rGwHYERMgqQx/xAUtChPJMre1bXBzDEqqVbWv9AIbFezXMxeEkZDSrXUOQ==} + dependencies: + fs-extra: 11.1.1 + ramda: 0.29.0 + dev: true + /file-uri-to-path@2.0.0: resolution: {integrity: sha512-hjPFI8oE/2iQPVe4gbrJ73Pp+Xfub2+WI2LlXDbsaJBwT5wuMh35WNWVYYTpnz895shtwfyutMFLFywpQAFdLg==} engines: {node: '>= 6'} @@ -9980,8 +12038,8 @@ packages: resolution: {integrity: sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==} dev: true - /flow-parser@0.198.2: - resolution: {integrity: sha512-tCQzqXbRAz0ZadIhAXGwdp/xsusADo8IK9idgc/2qCK5RmazbKDGedyykfRtzWgy7Klt4f4NZxq0o/wFUg6plQ==} + /flow-parser@0.215.1: + resolution: {integrity: sha512-qq3rdRToqwesrddyXf+Ml8Tuf7TdoJS+EMbJgC6fHAVoBCXjb4mHelNd3J+jD8ts0bSHX81FG3LN7Qn/dcl6pA==} engines: {node: '>=0.4.0'} dev: true @@ -10011,6 +12069,14 @@ packages: signal-exit: 3.0.7 dev: true + /foreground-child@3.1.1: + resolution: {integrity: sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==} + engines: {node: '>=14'} + dependencies: + cross-spawn: 7.0.3 + signal-exit: 4.1.0 + dev: true + /forever-agent@0.6.1: resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} dev: true @@ -10125,6 +12191,15 @@ packages: universalify: 2.0.0 dev: true + /fs-extra@11.1.1: + resolution: {integrity: sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==} + engines: {node: '>=14.14'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.0 + dev: true + /fs-extra@8.1.0: resolution: {integrity: sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==} engines: {node: '>=6 <7 || >=8'} @@ -10155,8 +12230,8 @@ packages: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} dev: true - /fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + /fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] requiresBuild: true @@ -10187,21 +12262,6 @@ packages: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} dev: true - /gauge@3.0.2: - resolution: {integrity: sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==} - engines: {node: '>=10'} - dependencies: - aproba: 2.0.0 - color-support: 1.1.3 - console-control-strings: 1.1.0 - has-unicode: 2.0.1 - object-assign: 4.1.1 - signal-exit: 3.0.7 - string-width: 4.2.3 - strip-ansi: 6.0.1 - wide-align: 1.1.5 - dev: true - /gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -10231,6 +12291,11 @@ packages: resolution: {integrity: sha512-2MSPMu7S1iOTL+BOa6K1S62hB2zUAYNF/lV0gSVlOaacd087lc6nR1H1r0e3B1CerTo+RceOmi1iJW+vp21xcQ==} dev: false + /get-nonce@1.0.1: + resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} + engines: {node: '>=6'} + dev: true + /get-npm-tarball-url@2.0.3: resolution: {integrity: sha512-R/PW6RqyaBQNWYaSyfrh54/qtcnOp22FHCCiRhSSZj0FP3KQWCsxxt0DzIdVTbwTqe9CtQfvl/FPD4UIPt4pqw==} engines: {node: '>=12.17'} @@ -10303,16 +12368,16 @@ packages: assert-plus: 1.0.0 dev: true - /giget@1.0.0: - resolution: {integrity: sha512-KWELZn3Nxq5+0So485poHrFriK9Bn3V/x9y+wgqrHkbmnGbjfLmZ685/SVA/ovW+ewoqW0gVI47pI4yW/VNobQ==} + /giget@1.1.2: + resolution: {integrity: sha512-HsLoS07HiQ5oqvObOI+Qb2tyZH4Gj5nYGfF9qQcZNrPw+uEFhdXtgJr01aO2pWadGHucajYDLxxbtQkm97ON2A==} hasBin: true dependencies: - colorette: 2.0.19 + colorette: 2.0.20 defu: 6.1.2 https-proxy-agent: 5.0.1 mri: 1.2.0 - node-fetch-native: 1.0.1 - pathe: 1.1.0 + node-fetch-native: 1.4.0 + pathe: 1.1.1 tar: 6.1.13 transitivePeerDependencies: - supports-color @@ -10349,20 +12414,22 @@ packages: glob: 7.2.3 dev: true - /glob-promise@6.0.2(glob@8.1.0): - resolution: {integrity: sha512-Ni2aDyD1ekD6x8/+K4hDriRDbzzfuK4yKpqSymJ4P7IxbtARiOOuU+k40kbHM0sLIlbf1Qh0qdMkAHMZYE6XJQ==} - engines: {node: '>=16'} - peerDependencies: - glob: ^8.0.3 - dependencies: - '@types/glob': 8.1.0 - glob: 8.1.0 - dev: true - /glob-to-regexp@0.4.1: resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} dev: true + /glob@10.3.3: + resolution: {integrity: sha512-92vPiMb/iqpmEgsOoIDvTjc50wf9CCCvMzsi6W0JLPeUKE8TWP1a73PgqSrqy7iAZxaSD1YdzU7QZR5LF51MJw==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + dependencies: + foreground-child: 3.1.1 + jackspeak: 2.3.0 + minimatch: 9.0.3 + minipass: 7.0.3 + path-scurry: 1.10.1 + dev: true + /glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} dependencies: @@ -10374,17 +12441,6 @@ packages: path-is-absolute: 1.0.1 dev: true - /glob@8.1.0: - resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} - engines: {node: '>=12'} - dependencies: - fs.realpath: 1.0.0 - inflight: 1.0.6 - inherits: 2.0.4 - minimatch: 5.1.6 - once: 1.4.0 - dev: true - /glob@9.3.1: resolution: {integrity: sha512-qERvJb7IGsnkx6YYmaaGvDpf77c951hICMdWaFXyH3PlVob8sbPJJyJX0kWkiCWyXUzoy9UOTNjGg0RbD8bYIw==} engines: {node: '>=16 || 14 >=14.17'} @@ -10509,8 +12565,8 @@ packages: through2: 2.0.5 dev: true - /handlebars@4.7.7: - resolution: {integrity: sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA==} + /handlebars@4.7.8: + resolution: {integrity: sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==} engines: {node: '>=0.4.7'} hasBin: true dependencies: @@ -10563,10 +12619,6 @@ packages: has-symbols: 1.0.3 dev: true - /has-unicode@2.0.1: - resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} - dev: true - /has@1.0.3: resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==} engines: {node: '>= 0.4.0'} @@ -10639,6 +12691,11 @@ packages: engines: {node: '>=8'} dev: true + /html-tags@3.3.1: + resolution: {integrity: sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==} + engines: {node: '>=8'} + dev: true + /htmlparser2@8.0.1: resolution: {integrity: sha512-4lVbmc1diZC7GUJQtRQ5yBAeUCL1exyMwmForWkRLnwyzWBFxN633SALPMGYaWZvKe9j1pRZJpauvmxENSp/EA==} dependencies: @@ -10901,11 +12958,6 @@ packages: engines: {node: '>=12'} dev: false - /interpret@1.4.0: - resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==} - engines: {node: '>= 0.10'} - dev: true - /intl-messageformat@10.2.5: resolution: {integrity: sha512-AievYMN6WLLHwBeCTv4aRKG+w3ZNyZtkObwgsKk3Q7GNTq8zDRvDbJSBQkb2OPeVCcAKcIXvak9FF/bRNavoww==} dependencies: @@ -10919,7 +12971,6 @@ packages: resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} dependencies: loose-envify: 1.4.0 - dev: false /ip@1.1.8: resolution: {integrity: sha512-PuExPYUiu6qMBQb4l06ecm6T6ujzhmh+MeJcW9wa89PoAz5pvd4zPgN5WJV104mb6S2T1AwNIAaB70JNrLQWhg==} @@ -11098,6 +13149,11 @@ packages: is-path-inside: 3.0.3 dev: true + /is-interactive@1.0.0: + resolution: {integrity: sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==} + engines: {node: '>=8'} + dev: true + /is-map@2.0.2: resolution: {integrity: sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==} dev: true @@ -11278,15 +13334,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /isomorphic-unfetch@3.1.0: - resolution: {integrity: sha512-geDJjpoZ8N0kWexiwkX8F9NkTsXhetLPVbZFQ+JTW239QNOwvB0gniuR1Wc6f0AMTn7/mFGyXvHTifrCp/GH8Q==} - dependencies: - node-fetch: 2.6.9 - unfetch: 4.2.0 - transitivePeerDependencies: - - encoding - dev: true - /isstream@0.1.2: resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} dev: true @@ -11309,15 +13356,6 @@ packages: - supports-color dev: true - /istanbul-lib-report@3.0.0: - resolution: {integrity: sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw==} - engines: {node: '>=8'} - dependencies: - istanbul-lib-coverage: 3.2.0 - make-dir: 3.1.0 - supports-color: 7.2.0 - dev: true - /istanbul-lib-report@3.0.1: resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} engines: {node: '>=10'} @@ -11338,14 +13376,6 @@ packages: - supports-color dev: true - /istanbul-reports@3.1.5: - resolution: {integrity: sha512-nUsEMa9pBt/NOHqbcbeJEgqIlY/K7rVWUX6Lql2orY5e9roQOthbR3vtY4zzf2orPELg80fnxxk9zUyPlgwD1w==} - engines: {node: '>=8'} - dependencies: - html-escaper: 2.0.2 - istanbul-lib-report: 3.0.0 - dev: true - /istanbul-reports@3.1.6: resolution: {integrity: sha512-TLgnMkKg3iTDsQ9PbPTdpfAK2DzjF9mqUG7RMgcQl8oFjad8ob4laGxv5XV5U9MAfx8D6tSJiUyuAwzLicaxlg==} engines: {node: '>=8'} @@ -11354,8 +13384,17 @@ packages: istanbul-lib-report: 3.0.1 dev: true - /jake@10.8.5: - resolution: {integrity: sha512-sVpxYeuAhWt0OTWITwT98oyV0GsXyMlXCF+3L1SuafBVUIr/uILGRB+NqwkzhgXKvoJpDIpQvqkUALgdmQsQxw==} + /jackspeak@2.3.0: + resolution: {integrity: sha512-uKmsITSsF4rUWQHzqaRUuyAir3fZfW3f202Ee34lz/gZCi970CPZwyQXLGNgWJvvZbvFyzeyGq0+4fcG/mBKZg==} + engines: {node: '>=14'} + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + dev: true + + /jake@10.8.7: + resolution: {integrity: sha512-ZDi3aP+fG/LchyBzUM804VjddnwfSfsdeYkwt8NcbKRvo4rFkjhs456iLFn3k2ZUWvNe4i48WACDbza8fhq2+w==} engines: {node: '>=10'} hasBin: true dependencies: @@ -11556,7 +13595,7 @@ packages: micromatch: 4.0.5 walker: 1.0.8 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /jest-haste-map@29.6.2: @@ -11575,7 +13614,26 @@ packages: micromatch: 4.0.5 walker: 1.0.8 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 + dev: true + + /jest-haste-map@29.6.4: + resolution: {integrity: sha512-12Ad+VNTDHxKf7k+M65sviyynRoZYuL1/GTuhEVb8RYsNSNln71nANRb/faSyWvx0j+gHcivChXHIoMJrGYjog==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/types': 29.6.3 + '@types/graceful-fs': 4.1.6 + '@types/node': 18.16.3 + anymatch: 3.1.3 + fb-watchman: 2.0.2 + graceful-fs: 4.2.11 + jest-regex-util: 29.6.3 + jest-util: 29.6.3 + jest-worker: 29.6.4 + micromatch: 4.0.5 + walker: 1.0.8 + optionalDependencies: + fsevents: 2.3.3 dev: true /jest-leak-detector@29.6.2: @@ -11661,6 +13719,11 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dev: true + /jest-regex-util@29.6.3: + resolution: {integrity: sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dev: true + /jest-resolve-dependencies@29.6.2: resolution: {integrity: sha512-LGqjDWxg2fuQQm7ypDxduLu/m4+4Lb4gczc13v51VMZbVP5tSBILqVx8qfWcsdP8f0G7aIqByIALDB0R93yL+w==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -11797,6 +13860,18 @@ packages: picomatch: 2.3.1 dev: true + /jest-util@29.6.3: + resolution: {integrity: sha512-QUjna/xSy4B32fzcKTSz1w7YYzgiHrjjJjevdRf61HYk998R5vVMMNmrHESYZVDS5DSWs+1srPLPKxXPkeSDOA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/types': 29.6.3 + '@types/node': 18.16.3 + chalk: 4.1.2 + ci-info: 3.8.0 + graceful-fs: 4.2.11 + picomatch: 2.3.1 + dev: true + /jest-validate@29.6.2: resolution: {integrity: sha512-vGz0yMN5fUFRRbpJDPwxMpgSXW1LDKROHfBopAvDcmD6s+B/s8WJrwi+4bfH4SdInBA5C3P3BI19dBtKzx1Arg==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -11828,17 +13903,27 @@ packages: engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: '@types/node': 18.16.3 - jest-util: 29.5.0 + jest-util: 29.5.0 + merge-stream: 2.0.0 + supports-color: 8.1.1 + dev: true + + /jest-worker@29.6.2: + resolution: {integrity: sha512-l3ccBOabTdkng8I/ORCkADz4eSMKejTYv1vB/Z83UiubqhC1oQ5Li6dWCyqOIvSifGjUBxuvxvlm6KGK2DtuAQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@types/node': 18.16.3 + jest-util: 29.6.2 merge-stream: 2.0.0 supports-color: 8.1.1 dev: true - /jest-worker@29.6.2: - resolution: {integrity: sha512-l3ccBOabTdkng8I/ORCkADz4eSMKejTYv1vB/Z83UiubqhC1oQ5Li6dWCyqOIvSifGjUBxuvxvlm6KGK2DtuAQ==} + /jest-worker@29.6.4: + resolution: {integrity: sha512-6dpvFV4WjcWbDVGgHTWo/aupl8/LbBx2NSKfiwqf79xC/yeJjKHT1+StcKy/2KTmW16hE68ccKVOtXf+WZGz7Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} dependencies: '@types/node': 18.16.3 - jest-util: 29.6.2 + jest-util: 29.6.3 merge-stream: 2.0.0 supports-color: 8.1.1 dev: true @@ -11903,55 +13988,25 @@ packages: resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} dev: true - /jscodeshift@0.14.0(@babel/preset-env@7.20.2): - resolution: {integrity: sha512-7eCC1knD7bLUPuSCwXsMZUH51O8jIcoVyKtI6P0XM0IVzlGjckPy3FIwQlorzbN0Sg79oK+RlohN32Mqf/lrYA==} - hasBin: true - peerDependencies: - '@babel/preset-env': ^7.1.6 - dependencies: - '@babel/core': 7.21.3 - '@babel/parser': 7.21.3 - '@babel/plugin-proposal-class-properties': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-nullish-coalescing-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-chaining': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-modules-commonjs': 7.20.11(@babel/core@7.21.3) - '@babel/preset-env': 7.20.2(@babel/core@7.21.3) - '@babel/preset-flow': 7.18.6(@babel/core@7.21.3) - '@babel/preset-typescript': 7.21.0(@babel/core@7.21.3) - '@babel/register': 7.18.9(@babel/core@7.21.3) - babel-core: 7.0.0-bridge.0(@babel/core@7.21.3) - chalk: 4.1.2 - flow-parser: 0.198.2 - graceful-fs: 4.2.11 - micromatch: 4.0.5 - neo-async: 2.6.2 - node-dir: 0.1.17 - recast: 0.21.5 - temp: 0.8.4 - write-file-atomic: 2.4.3 - transitivePeerDependencies: - - supports-color - dev: true - - /jscodeshift@0.14.0(@babel/preset-env@7.21.4): + /jscodeshift@0.14.0(@babel/preset-env@7.22.10): resolution: {integrity: sha512-7eCC1knD7bLUPuSCwXsMZUH51O8jIcoVyKtI6P0XM0IVzlGjckPy3FIwQlorzbN0Sg79oK+RlohN32Mqf/lrYA==} hasBin: true peerDependencies: '@babel/preset-env': ^7.1.6 dependencies: '@babel/core': 7.21.3 - '@babel/parser': 7.21.3 + '@babel/parser': 7.22.11 '@babel/plugin-proposal-class-properties': 7.18.6(@babel/core@7.21.3) '@babel/plugin-proposal-nullish-coalescing-operator': 7.18.6(@babel/core@7.21.3) - '@babel/plugin-proposal-optional-chaining': 7.20.7(@babel/core@7.21.3) - '@babel/plugin-transform-modules-commonjs': 7.20.11(@babel/core@7.21.3) - '@babel/preset-env': 7.21.4(@babel/core@7.21.3) - '@babel/preset-flow': 7.18.6(@babel/core@7.21.3) + '@babel/plugin-proposal-optional-chaining': 7.21.0(@babel/core@7.21.3) + '@babel/plugin-transform-modules-commonjs': 7.22.11(@babel/core@7.21.3) + '@babel/preset-env': 7.22.10(@babel/core@7.22.11) + '@babel/preset-flow': 7.22.5(@babel/core@7.21.3) '@babel/preset-typescript': 7.21.0(@babel/core@7.21.3) - '@babel/register': 7.18.9(@babel/core@7.21.3) + '@babel/register': 7.22.5(@babel/core@7.21.3) babel-core: 7.0.0-bridge.0(@babel/core@7.21.3) chalk: 4.1.2 - flow-parser: 0.198.2 + flow-parser: 0.215.1 graceful-fs: 4.2.11 micromatch: 4.0.5 neo-async: 2.6.2 @@ -12448,6 +14503,11 @@ packages: dependencies: js-tokens: 4.0.0 + /lru-cache@10.0.1: + resolution: {integrity: sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g==} + engines: {node: 14 || >=16.14} + dev: true + /lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} dependencies: @@ -12487,19 +14547,26 @@ packages: '@jridgewell/sourcemap-codec': 1.4.14 dev: true + /magic-string@0.30.3: + resolution: {integrity: sha512-B7xGbll2fG/VjP+SWg4sX3JynwIU0mjoTc6MPpKNuIvftk6u6vqhDnk1R80b8C2GBR6ywqy+1DcKBrevBg+bmw==} + engines: {node: '>=12'} + dependencies: + '@jridgewell/sourcemap-codec': 1.4.15 + dev: true + /make-dir@2.1.0: resolution: {integrity: sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==} engines: {node: '>=6'} dependencies: pify: 4.0.1 - semver: 5.7.1 + semver: 5.7.2 dev: true /make-dir@3.1.0: resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==} engines: {node: '>=8'} dependencies: - semver: 6.3.0 + semver: 6.3.1 dev: true /make-dir@4.0.0: @@ -12541,8 +14608,8 @@ packages: resolution: {integrity: sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==} dev: false - /markdown-to-jsx@7.1.8(react@18.2.0): - resolution: {integrity: sha512-rRSa1aFmFnpDRFAhv5vIkWM4nPaoB9vnzIjuIKa1wGupfn2hdCNeaQHKpu4/muoc8n8J7yowjTP2oncA4/Rbgg==} + /markdown-to-jsx@7.3.2(react@18.2.0): + resolution: {integrity: sha512-B+28F5ucp83aQm+OxNrPkS8z0tMKaeHiy0lHJs3LqCyDQFtWuenaIrkaVTgAm1pf1AU85LXltva86hlaT17i8Q==} engines: {node: '>= 10'} peerDependencies: react: '>= 0.14.0' @@ -13109,6 +15176,13 @@ packages: brace-expansion: 2.0.1 dev: true + /minimatch@9.0.3: + resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + brace-expansion: 2.0.1 + dev: true + /minimist-options@4.1.0: resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} engines: {node: '>= 6'} @@ -13141,6 +15215,11 @@ packages: engines: {node: '>=8'} dev: true + /minipass@7.0.3: + resolution: {integrity: sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==} + engines: {node: '>=16 || 14 >=14.17'} + dev: true + /minizlib@2.1.2: resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} engines: {node: '>= 8'} @@ -13289,8 +15368,8 @@ packages: http2-client: 1.3.5 dev: true - /node-fetch-native@1.0.1: - resolution: {integrity: sha512-VzW+TAk2wE4X9maiKMlT+GsPU4OMmR1U9CrHSmd3DFLn2IcZ9VJ6M6BBugGfYUnPCLSYxXdZy17M0BEJyhUTwg==} + /node-fetch-native@1.4.0: + resolution: {integrity: sha512-F5kfEj95kX8tkDhUCYdV8dg3/8Olx/94zB8+ZNthFs6Bz31UpUi8Xh40TN3thLwXgrwXry1pEg9lJ++tLWTcqA==} dev: true /node-fetch@2.6.7: @@ -13327,6 +15406,10 @@ packages: es6-promise: 3.3.1 dev: true + /node-releases@2.0.13: + resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==} + dev: true + /node-releases@2.0.8: resolution: {integrity: sha512-dFSmB8fFHEH/s81Xi+Y/15DQY6VHW81nXRj86EMSL3lmuTmK1e+aT4wrFCkTbm+gSwkw4KpX+rT/pMM2c1mF+A==} @@ -13376,15 +15459,6 @@ packages: path-key: 3.1.1 dev: true - /npmlog@5.0.1: - resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==} - dependencies: - are-we-there-yet: 2.0.0 - console-control-strings: 1.1.0 - gauge: 3.0.2 - set-blocking: 2.0.0 - dev: true - /nwsapi@2.2.2: resolution: {integrity: sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==} dev: true @@ -13535,16 +15609,17 @@ packages: format-util: 1.0.5 dev: true - /open@7.4.2: - resolution: {integrity: sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==} - engines: {node: '>=8'} + /open@8.4.0: + resolution: {integrity: sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==} + engines: {node: '>=12'} dependencies: + define-lazy-prop: 2.0.0 is-docker: 2.2.1 is-wsl: 2.2.0 dev: true - /open@8.4.0: - resolution: {integrity: sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==} + /open@8.4.2: + resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} engines: {node: '>=12'} dependencies: define-lazy-prop: 2.0.0 @@ -13586,6 +15661,21 @@ packages: word-wrap: 1.2.3 dev: true + /ora@5.4.1: + resolution: {integrity: sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==} + engines: {node: '>=10'} + dependencies: + bl: 4.1.0 + chalk: 4.1.2 + cli-cursor: 3.1.0 + cli-spinners: 2.9.0 + is-interactive: 1.0.0 + is-unicode-supported: 0.1.0 + log-symbols: 4.1.0 + strip-ansi: 6.0.1 + wcwidth: 1.0.1 + dev: true + /orval@6.15.0(openapi-types@12.1.3)(typescript@5.0.2): resolution: {integrity: sha512-SJx5PlwER6BzBMpnLkvFZkx50+rTvi2tvXgZyjMVjyxskgknb28BhFvYhMlKryT2UL0ihcQWHMnjgov4mScBwQ==} hasBin: true @@ -13797,6 +15887,14 @@ packages: /path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + /path-scurry@1.10.1: + resolution: {integrity: sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + lru-cache: 10.0.1 + minipass: 7.0.3 + dev: true + /path-scurry@1.6.1: resolution: {integrity: sha512-OW+5s+7cw6253Q4E+8qQ/u1fVvcJQCJo/VFD8pje+dbJCF1n5ZRMV2AEHbGp+5Q7jxQIYJxkHopnj6nzdGeZLA==} engines: {node: '>=14'} @@ -13813,8 +15911,8 @@ packages: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} - /pathe@1.1.0: - resolution: {integrity: sha512-ODbEPR0KKHqECXW1GoxdDb+AZvULmXjVPy4rt+pGo2+TnjJTIPJQSVS6N63n8T2Ip+syHhbn52OewKicV0373w==} + /pathe@1.1.1: + resolution: {integrity: sha512-d+RQGp0MAYTIaDBIMmOfMwz3E+LOZnxx1HZd5R18mmCZY0QBlK0LDZfPc8FW8Ed2DlvsuE6PRjroDY+wg4+j/Q==} dev: true /pause-stream@0.0.11: @@ -13971,7 +16069,7 @@ packages: resolution: {integrity: sha512-Sz2Lkdxz6F2Pgnpi9U5Ng/WdWAUZxmHrNPoVlm3aAemxoy2Qy7LGjQg4uf8qKelDAUW94F4np3iH2YPf2qefcQ==} engines: {node: '>=10'} dependencies: - '@babel/runtime': 7.22.10 + '@babel/runtime': 7.22.11 dev: true /pony-cause@1.1.1: @@ -14377,6 +16475,13 @@ packages: side-channel: 1.0.4 dev: true + /qs@6.11.2: + resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} + engines: {node: '>=0.6'} + dependencies: + side-channel: 1.0.4 + dev: true + /query-string@6.14.1: resolution: {integrity: sha512-XDxAeVmpfu1/6IjyT/gXHOl+S0vQ9owggJ30hhWKdHAsNPOcasn5o9BW0eejZqL2e4vMjhAxoW3jVHcD6mbcYw==} engines: {node: '>=6'} @@ -14410,6 +16515,10 @@ packages: resolution: {integrity: sha512-9QnLuG/kPVgWvMQ4aODhsBUFKOUmnbUnsSXACv+NCQZcHbeb+v8Lodp8OVxtRULN1/xOyYLLaL6npE6dMq5QTA==} dev: true + /ramda@0.29.0: + resolution: {integrity: sha512-BBea6L67bYLtdbOqfp8f58fPMqEwx0doL+pAi8TZyp2YWz8R9G8z9x75CZI8W+ftqhFHCpEX2cRnUUXK130iKA==} + dev: true + /range-parser@1.2.1: resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} engines: {node: '>= 0.6'} @@ -14498,14 +16607,14 @@ packages: hasBin: true dependencies: '@babel/core': 7.21.3 - '@babel/generator': 7.21.3 + '@babel/generator': 7.22.10 ast-types: 0.14.2 commander: 2.20.3 doctrine: 3.0.0 estree-to-babel: 3.2.1 neo-async: 2.6.2 node-dir: 0.1.17 - resolve: 1.22.1 + resolve: 1.22.4 strip-indent: 3.0.0 transitivePeerDependencies: - supports-color @@ -14576,8 +16685,8 @@ packages: react: 18.2.0 dev: false - /react-inspector@6.0.1(react@18.2.0): - resolution: {integrity: sha512-cxKSeFTf7jpSSVddm66sKdolG90qURAX3g1roTeaN6x0YEbtWc8JpmFN9+yIqLNH2uEkYerWLtJZIXRIFuBKrg==} + /react-inspector@6.0.2(react@18.2.0): + resolution: {integrity: sha512-x+b7LxhmHXjHoU/VrFAzw5iutsILRoYyDq97EDYdFpPLcvqtEzk4ZSZSQjnFPbr5T57tLXnHcqFYoN1pI6u8uQ==} peerDependencies: react: ^16.8.4 || ^17.0.0 || ^18.0.0 dependencies: @@ -14753,6 +16862,41 @@ packages: engines: {node: '>=0.10.0'} dev: true + /react-remove-scroll-bar@2.3.4(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-63C4YQBUt0m6ALadE9XV56hV8BgJWDmmTPY758iIJjfQKt2nYwoUrPk0LXRXcB/yIj82T1/Ixfdpdk68LwIB0A==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + react: 18.2.0 + react-style-singleton: 2.2.1(@types/react@18.2.20)(react@18.2.0) + tslib: 2.6.2 + dev: true + + /react-remove-scroll@2.5.5(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + react: 18.2.0 + react-remove-scroll-bar: 2.3.4(@types/react@18.2.20)(react@18.2.0) + react-style-singleton: 2.2.1(@types/react@18.2.20)(react@18.2.0) + tslib: 2.6.2 + use-callback-ref: 1.3.0(@types/react@18.2.20)(react@18.2.0) + use-sidecar: 1.1.2(@types/react@18.2.20)(react@18.2.0) + dev: true + /react-resize-detector@7.1.2(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-zXnPJ2m8+6oq9Nn8zsep/orts9vQv3elrpA+R8XTcW7DVVUJ9vwDwMXaBtykAYjMnkCIaOoK9vObyR7ZgFNlOw==} peerDependencies: @@ -14875,6 +17019,23 @@ packages: lodash: 4.17.21 dev: false + /react-style-singleton@2.2.1(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + get-nonce: 1.0.1 + invariant: 2.2.4 + react: 18.2.0 + tslib: 2.6.2 + dev: true + /react-transition-group@2.9.0(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg==} peerDependencies: @@ -15053,8 +17214,8 @@ packages: string_decoder: 0.10.31 dev: true - /readable-stream@2.3.7: - resolution: {integrity: sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==} + /readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} dependencies: core-util-is: 1.0.3 inherits: 2.0.4 @@ -15065,8 +17226,8 @@ packages: util-deprecate: 1.0.2 dev: true - /readable-stream@3.6.0: - resolution: {integrity: sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==} + /readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} dependencies: inherits: 2.0.4 @@ -15097,18 +17258,18 @@ packages: ast-types: 0.15.2 esprima: 4.0.1 source-map: 0.6.1 - tslib: 2.5.2 + tslib: 2.6.2 dev: true - /recast@0.23.1: - resolution: {integrity: sha512-RokaBcoxSjXUDzz1TXSZmZsSW6ZpLmlA3GGqJ8uuTrQ9hZhEz+4Tpsc+gRvYRJ2BU4H+ZyUlg91eSGDw7bwy7g==} + /recast@0.23.4: + resolution: {integrity: sha512-qtEDqIZGVcSZCHniWwZWbRy79Dc6Wp3kT/UmDA2RJKBPg7+7k51aQBZirHmUGn5uvHf2rg8DkjizrN26k61ATw==} engines: {node: '>= 4'} dependencies: assert: 2.0.0 ast-types: 0.16.1 esprima: 4.0.1 source-map: 0.6.1 - tslib: 2.5.2 + tslib: 2.6.2 dev: true /recharts-scale@0.4.5: @@ -15139,13 +17300,6 @@ packages: victory-vendor: 36.6.8 dev: false - /rechoir@0.6.2: - resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} - engines: {node: '>= 0.10'} - dependencies: - resolve: 1.22.1 - dev: true - /redent@3.0.0: resolution: {integrity: sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==} engines: {node: '>=8'} @@ -15188,6 +17342,12 @@ packages: '@babel/runtime': 7.22.10 dev: true + /regenerator-transform@0.15.2: + resolution: {integrity: sha512-hfMp2BoF0qOk3uc5V20ALGDS2ddjQaLrdl7xrGXvAIow7qeWRM2VA2HuCHkUKk9slq3VwEwLNK3DFBqDfPGYtg==} + dependencies: + '@babel/runtime': 7.22.11 + dev: true + /regexp.prototype.flags@1.4.3: resolution: {integrity: sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==} engines: {node: '>= 0.4'} @@ -15218,6 +17378,18 @@ packages: unicode-match-property-value-ecmascript: 2.1.0 dev: true + /regexpu-core@5.3.2: + resolution: {integrity: sha512-RAM5FlZz+Lhmo7db9L298p2vHP5ZywrVXmVXpmAD9GuL5MPH6t9ROw1iA/wfHkQ76Qe7AaPF0nGuim96/IrQMQ==} + engines: {node: '>=4'} + dependencies: + '@babel/regjsgen': 0.8.0 + regenerate: 1.4.2 + regenerate-unicode-properties: 10.1.0 + regjsparser: 0.9.1 + unicode-match-property-ecmascript: 2.0.0 + unicode-match-property-value-ecmascript: 2.1.0 + dev: true + /regjsgen@0.7.1: resolution: {integrity: sha512-RAt+8H2ZEzHeYWxZ3H2z6tF18zyyOnlcdaafLrm21Bguj7uZy6ULibiAFdXEtKQY4Sy7wDTwDiOazasMLc4KPA==} dev: true @@ -15446,7 +17618,7 @@ packages: engines: {node: '>=10.0.0'} hasBin: true optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /rollup@3.19.1: @@ -15454,7 +17626,15 @@ packages: engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 + dev: true + + /rollup@3.28.1: + resolution: {integrity: sha512-R9OMQmIHJm9znrU3m3cpE8uhN0fGdXiawME7aZIpQqvpS/85+Vt1Hq1/yVIcYfOmaQiHjvXkQAoJukvLpau6Yw==} + engines: {node: '>=14.18.0', npm: '>=8.0.0'} + hasBin: true + optionalDependencies: + fsevents: 2.3.3 dev: true /rtl-css-js@1.16.1: @@ -15554,12 +17734,18 @@ packages: hasBin: true dev: true + /semver@5.7.2: + resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} + hasBin: true + requiresBuild: true + dev: true + /semver@6.3.0: resolution: {integrity: sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==} hasBin: true - /semver@7.0.0: - resolution: {integrity: sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==} + /semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true dev: true @@ -15622,10 +17808,6 @@ packages: - supports-color dev: true - /set-blocking@2.0.0: - resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} - dev: true - /set-harmonic-interval@1.0.1: resolution: {integrity: sha512-AhICkFV84tBP1aWqPwLZqFvAwqEoVA9kxNMniGEUvzOlm4vLmOFLiTT3UZ6bziJTy4bOVpzWGTfSCbmaayGx8g==} engines: {node: '>=6.9'} @@ -15658,16 +17840,6 @@ packages: engines: {node: '>=8'} dev: true - /shelljs@0.8.5: - resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} - engines: {node: '>=4'} - hasBin: true - dependencies: - glob: 7.2.3 - interpret: 1.4.0 - rechoir: 0.6.2 - dev: true - /should-equal@2.0.0: resolution: {integrity: sha512-ZP36TMrK9euEuWQYBig9W55WPC7uo37qzAEmbjHz4gfyuXrEUgF8cUvQVO+w+d3OMfPvSRQJ22lSm8MQJ43LTA==} dependencies: @@ -15718,6 +17890,11 @@ packages: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} dev: true + /signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + dev: true + /simple-eval@1.0.0: resolution: {integrity: sha512-kpKJR+bqTscgC0xuAl2xHN6bB12lHjC2DCUfqjAx19bQyO3R2EVLOurm3H9AUltv/uFVcSCVNc6faegR+8NYLw==} engines: {node: '>=12'} @@ -15725,11 +17902,11 @@ packages: jsep: 1.3.8 dev: true - /simple-update-notifier@1.1.0: - resolution: {integrity: sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg==} - engines: {node: '>=8.10.0'} + /simple-update-notifier@2.0.0: + resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==} + engines: {node: '>=10'} dependencies: - semver: 7.0.0 + semver: 7.5.4 dev: true /sisteransi@1.0.5: @@ -16018,11 +18195,34 @@ packages: resolution: {integrity: sha512-siT1RiqlfQnGqgT/YzXVUNsom9S0H1OX+dpdGN1xkyYATo4I6sep5NmsRD/40s3IIOvlCq6akxkqG82urIZW1w==} dev: true - /storybook@7.0.2: - resolution: {integrity: sha512-/XBLhT9Vb14yNBcA9rlW15y+C6IsCA3kx5PKvK9kL10sKCi8invcY94UfCSisXe8HqsO3u6peumo2xpYucKMjw==} + /storybook-dark-mode@3.0.1(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-3V6XBhkUq63BF6KzyDBbfV5/8sYtF4UtVccH1tK+Lrd4p0tF8k7yHOvVDhFL9hexnKXcLEnbC+42YDTPvjpK+A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + react: + optional: true + react-dom: + optional: true + dependencies: + '@storybook/addons': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/api': 7.3.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/components': 7.0.2(react-dom@18.2.0)(react@18.2.0) + '@storybook/core-events': 7.0.2 + '@storybook/global': 5.0.0 + '@storybook/theming': 7.3.2(react-dom@18.2.0)(react@18.2.0) + fast-deep-equal: 3.1.3 + memoizerific: 1.11.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: true + + /storybook@7.3.2: + resolution: {integrity: sha512-Vf1C5pfF5NHQsb+33NeBd3gLGhcwbT+v6WqqIdARV7LSByqKiWNgJl2ATgzm1b4ERJo8sHU+EiJZIovFWEElkg==} hasBin: true dependencies: - '@storybook/cli': 7.0.2 + '@storybook/cli': 7.3.2 transitivePeerDependencies: - bufferutil - encoding @@ -16162,6 +18362,13 @@ packages: ansi-regex: 6.0.1 dev: true + /strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + dependencies: + ansi-regex: 6.0.1 + dev: true + /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -16454,8 +18661,8 @@ packages: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} dev: true - /synchronous-promise@2.0.16: - resolution: {integrity: sha512-qImOD23aDfnIDNqlG1NOehdB9IYsn1V9oByPjKY1nakv2MQYCEMyX033/q+aEtYCpmYK1cv2+NTmlH+ra6GA5A==} + /synchronous-promise@2.0.17: + resolution: {integrity: sha512-AsS729u2RHUfEra9xJrE39peJcc2stq2+poBXX8bcM08Y6g9j/i/PUzwNQqkaJde7Ntg1TO7bSREbR5sdosQ+g==} dev: true /synckit@0.8.5: @@ -16503,7 +18710,7 @@ packages: end-of-stream: 1.4.4 fs-constants: 1.0.0 inherits: 2.0.4 - readable-stream: 3.6.0 + readable-stream: 3.6.2 dev: true /tar@6.1.13: @@ -16518,8 +18725,8 @@ packages: yallist: 4.0.0 dev: true - /telejson@7.0.4: - resolution: {integrity: sha512-J4QEuCnYGXAI9KSN7RXK0a0cOW2ONpjc4IQbInGZ6c3stvplLAYyZjTnScrRd8deXVjNCFV1wXcLC7SObDuQYA==} + /telejson@7.2.0: + resolution: {integrity: sha512-1QTEcJkJEhc8OnStBx/ILRu5J2p0GjvWsBx56bmZRqnrkdBMUe+nX92jxV+p3dB4CP6PZCdJMQJwCggkNBMzkQ==} dependencies: memoizerific: 1.11.3 dev: true @@ -16576,7 +18783,7 @@ packages: /through2@2.0.5: resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} dependencies: - readable-stream: 2.3.7 + readable-stream: 2.3.8 xtend: 4.0.2 dev: true @@ -16620,6 +18827,10 @@ packages: dependencies: is-number: 7.0.0 + /tocbot@4.21.1: + resolution: {integrity: sha512-IfajhBTeg0HlMXu1f+VMbPef05QpDTsZ9X2Yn1+8npdaXsXg/+wrm9Ze1WG5OS1UDC3qJ5EQN/XOZ3gfXjPFCw==} + dev: true + /toggle-selection@1.0.6: resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} dev: false @@ -16772,6 +18983,10 @@ packages: /tslib@2.5.2: resolution: {integrity: sha512-5svOrSA2w3iGFDs1HibEVBGbDrAY82bFQ3HZ3ixB+88nsbsWQoKqDRb5UBYAUPEzbBn6dAp5gRNXglySbx1MlA==} + /tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + dev: true + /tsutils@3.21.0(typescript@5.0.2): resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==} engines: {node: '>= 6'} @@ -16932,10 +19147,6 @@ packages: react-lifecycles-compat: 3.0.4 dev: false - /unfetch@4.2.0: - resolution: {integrity: sha512-F9p7yYCn6cIW9El1zi0HI6vqpeIvBsr3dSuRO6Xuppb1u5rXpCPmMvLSyECLhybr9isec8Ohl0hPekMVrEinDA==} - dev: true - /unicode-canonical-property-names-ecmascript@2.0.0: resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} engines: {node: '>=4'} @@ -17074,13 +19285,13 @@ packages: engines: {node: '>= 0.8'} dev: true - /unplugin@0.10.2: - resolution: {integrity: sha512-6rk7GUa4ICYjae5PrAllvcDeuT8pA9+j5J5EkxbMFaV+SalHhxZ7X2dohMzu6C3XzsMT+6jwR/+pwPNR3uK9MA==} + /unplugin@1.4.0: + resolution: {integrity: sha512-5x4eIEL6WgbzqGtF9UV8VEC/ehKptPXDS6L2b0mv4FRMkJxRtjaJfOWDd6a8+kYbqsjklix7yWP0N3SUepjXcg==} dependencies: - acorn: 8.8.2 + acorn: 8.10.0 chokidar: 3.5.3 webpack-sources: 3.2.3 - webpack-virtual-modules: 0.4.6 + webpack-virtual-modules: 0.5.0 dev: true /untildify@4.0.0: @@ -17098,6 +19309,17 @@ packages: escalade: 3.1.1 picocolors: 1.0.0 + /update-browserslist-db@1.0.11(browserslist@4.21.10): + resolution: {integrity: sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + dependencies: + browserslist: 4.21.10 + escalade: 3.1.1 + picocolors: 1.0.0 + dev: true + /uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} dependencies: @@ -17122,6 +19344,21 @@ packages: querystring: 0.2.0 dev: false + /use-callback-ref@1.3.0(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + react: 18.2.0 + tslib: 2.6.2 + dev: true + /use-isomorphic-layout-effect@1.1.2(@types/react@18.2.20)(react@18.2.0): resolution: {integrity: sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==} peerDependencies: @@ -17146,6 +19383,22 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true + /use-sidecar@1.1.2(@types/react@18.2.20)(react@18.2.0): + resolution: {integrity: sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': ^16.9.0 || ^17.0.0 || ^18.0.0 + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + dependencies: + '@types/react': 18.2.20 + detect-node-es: 1.1.0 + react: 18.2.0 + tslib: 2.6.2 + dev: true + /use-sync-external-store@1.2.0(react@18.2.0): resolution: {integrity: sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==} peerDependencies: @@ -17182,10 +19435,6 @@ packages: engines: {node: '>= 0.4.0'} dev: true - /uuid-browser@3.1.0: - resolution: {integrity: sha512-dsNgbLaTrd6l3MMxTtouOCFw4CBFc/3a+GgYA2YyrJvyQ1u6q4pcu3ktLoUZ/VN/Aw9WsauazbgsgdfVWgAKQg==} - dev: true - /uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true @@ -17193,7 +19442,6 @@ packages: /uuid@9.0.0: resolution: {integrity: sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==} hasBin: true - dev: false /uvu@0.5.6: resolution: {integrity: sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==} @@ -17407,7 +19655,7 @@ packages: rollup: 3.19.1 sass: 1.59.3 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /vm2@3.9.19: @@ -17524,8 +19772,8 @@ packages: engines: {node: '>=10.13.0'} dev: true - /webpack-virtual-modules@0.4.6: - resolution: {integrity: sha512-5tyDlKLqPfMqjT3Q9TAqf2YqjwmnUleZwzJi1A5qXnlBCdj2AtOJ6wAWdglTIDOPgOiOrXeBeFcsQ8+aGQ6QbA==} + /webpack-virtual-modules@0.5.0: + resolution: {integrity: sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw==} dev: true /websocket-driver@0.7.4: @@ -17618,19 +19866,6 @@ packages: isexe: 2.0.0 dev: true - /wide-align@1.1.5: - resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} - dependencies: - string-width: 4.2.3 - dev: true - - /widest-line@3.1.0: - resolution: {integrity: sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==} - engines: {node: '>=8'} - dependencies: - string-width: 4.2.3 - dev: true - /word-wrap@1.2.3: resolution: {integrity: sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==} engines: {node: '>=0.10.0'} @@ -17657,6 +19892,15 @@ packages: string-width: 4.2.3 strip-ansi: 6.0.1 + /wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 + dev: true + /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} dev: true @@ -17704,6 +19948,19 @@ packages: optional: true dev: true + /ws@8.13.0: + resolution: {integrity: sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: true + /xml-name-validator@4.0.0: resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==} engines: {node: '>=12'} From 880d2c1c4ed4737b0aa54533ed6ec5f6913b922f Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Mon, 28 Aug 2023 12:34:44 -0700 Subject: [PATCH 036/201] Replication worker shutdown improvement. (#8535) --- .../commons/temporal/TemporalUtils.java | 4 ++- .../general/BufferedReplicationWorker.java | 30 ++++++++++++++++++- .../general/DefaultReplicationWorker.java | 30 ++++++++++++++++++- .../internal/HeartbeatTimeoutChaperone.java | 9 +++++- .../metrics/lib/OssMetricsRegistry.java | 3 ++ 5 files changed, 72 insertions(+), 4 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java index 3ea1869cb4b..93a10e27ef8 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java @@ -344,7 +344,7 @@ public T withBackgroundHeartbeat(final AtomicReference afterCancel try { // Making sure the heartbeat executor is terminated to avoid heartbeat attempt after we're done with // the activity. - if (scheduledExecutor.awaitTermination(10, TimeUnit.SECONDS)) { + if (scheduledExecutor.awaitTermination(HEARTBEAT_SHUTDOWN_GRACE_PERIOD.toSeconds(), TimeUnit.SECONDS)) { log.info("Temporal heartbeating stopped."); } else { // Heartbeat thread failed to stop, we may leak a thread if this happens. @@ -354,6 +354,8 @@ public T withBackgroundHeartbeat(final AtomicReference afterCancel } catch (InterruptedException e) { // We got interrupted while attempting to shutdown the executor. Not much more we can do. log.info("Interrupted while stopping the Temporal heartbeating, continuing the shutdown."); + // Preserve the interrupt status + Thread.currentThread().interrupt(); } } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java index a60019974e6..e61698816bb 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java @@ -13,8 +13,10 @@ import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncInput; import io.airbyte.metrics.lib.ApmTraceUtils; +import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; @@ -89,6 +91,7 @@ public class BufferedReplicationWorker implements ReplicationWorker { private static final int sourceMaxBufferSize = 1000; private static final int destinationMaxBufferSize = 1000; private static final int observabilityMetricsPeriodInSeconds = 1; + private static final int executorShutdownGracePeriodInSeconds = 10; public BufferedReplicationWorker(final String jobId, final int attempt, @@ -174,6 +177,20 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo } finally { executors.shutdownNow(); scheduledExecutors.shutdownNow(); + + try { + // Best effort to mark as complete when the Worker is actually done. + executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); + scheduledExecutors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); + if (!executors.isTerminated() || !scheduledExecutors.isTerminated()) { + final MetricClient metricClient = MetricClientFactory.getMetricClient(); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_EXECUTOR_SHUTDOWN_ERROR, 1, + new MetricAttribute(MetricTags.IMPLEMENTATION, "buffered")); + } + } catch (final InterruptedException e) { + // Preserve the interrupt status + Thread.currentThread().interrupt(); + } } if (!cancelled) { @@ -248,13 +265,19 @@ private ReplicationContext getReplicationContext(final StandardSyncInput syncInp @Override public void cancel() { + boolean wasInterrupted = false; + cancelled = true; replicationWorkerHelper.markCancelled(); LOGGER.info("Cancelling replication worker..."); + executors.shutdownNow(); + scheduledExecutors.shutdownNow(); try { - executors.awaitTermination(10, TimeUnit.SECONDS); + executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); + scheduledExecutors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); } catch (final InterruptedException e) { + wasInterrupted = true; ApmTraceUtils.addExceptionToTrace(e); LOGGER.error("Unable to cancel due to interruption.", e); } @@ -276,6 +299,11 @@ public void cancel() { } replicationWorkerHelper.endOfReplication(); + + if (wasInterrupted) { + // Preserve the interrupt flag if we were interrupted + Thread.currentThread().interrupt(); + } } /** diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index b01a346d41f..905392da9b1 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -13,6 +13,11 @@ import io.airbyte.config.ReplicationOutput; import io.airbyte.config.StandardSyncInput; import io.airbyte.metrics.lib.ApmTraceUtils; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.workers.RecordSchemaValidator; @@ -79,6 +84,8 @@ public class DefaultReplicationWorker implements ReplicationWorker { private final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone; private final ReplicationFeatureFlagReader replicationFeatureFlagReader; + private static final int executorShutdownGracePeriodInSeconds = 10; + public DefaultReplicationWorker(final String jobId, final int attempt, final AirbyteSource source, @@ -219,6 +226,18 @@ private void replicate(final Path jobRoot, LOGGER.error("Sync worker failed.", e); } finally { executors.shutdownNow(); + + try { + // Best effort to mark as complete when the Worker is actually done. + if (!executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS)) { + final MetricClient metricClient = MetricClientFactory.getMetricClient(); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_EXECUTOR_SHUTDOWN_ERROR, 1, + new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); + } + } catch (final InterruptedException e) { + // Preserve the interrupt status + Thread.currentThread().interrupt(); + } } } @@ -344,13 +363,17 @@ private static Runnable readFromSrcAndWriteToDstRunnable(final AirbyteSource sou @Override public void cancel() { + boolean wasInterrupted = false; + // Resources are closed in the opposite order they are declared. LOGGER.info("Cancelling replication worker..."); + executors.shutdownNow(); try { - executors.awaitTermination(10, TimeUnit.SECONDS); + executors.awaitTermination(executorShutdownGracePeriodInSeconds, TimeUnit.SECONDS); } catch (final InterruptedException e) { ApmTraceUtils.addExceptionToTrace(e); LOGGER.error("Unable to cancel due to interruption.", e); + wasInterrupted = true; } cancelled.set(true); replicationWorkerHelper.markCancelled(); @@ -372,6 +395,11 @@ public void cancel() { } replicationWorkerHelper.endOfReplication(); + + if (wasInterrupted) { + // Preserve the interrupt flag if we were interrupted + Thread.currentThread().interrupt(); + } } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java index af64df475df..065f38fd36b 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java @@ -21,6 +21,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -154,7 +155,13 @@ void monitor() { @Override public void close() throws Exception { if (lazyExecutorService != null) { - lazyExecutorService.shutdown(); + lazyExecutorService.shutdownNow(); + try { + lazyExecutorService.awaitTermination(10, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + // Propagate the status if we were interrupted + Thread.currentThread().interrupt(); + } } } diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java index 7b346f8f4c3..520649e65f9 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java @@ -214,6 +214,9 @@ public enum OssMetricsRegistry implements MetricsRegistry { REPLICATION_WORKER_CREATED(MetricEmittingApps.WORKER, "replication_worker_created", "number of replication worker created"), + REPLICATION_WORKER_EXECUTOR_SHUTDOWN_ERROR(MetricEmittingApps.WORKER, + "replication_worker_executor_shutdown_error", + "number of failure to shutdown executors"), REPLICATION_MADE_PROGRESS(MetricEmittingApps.WORKER, "replication_made_progress", "Count of replication runs that made progress. To be faceted by attributes."), From d94f71404b4210c121ab94766a5940a8615f4e4c Mon Sep 17 00:00:00 2001 From: Flutra Osmani <129116758+flutra-osmani@users.noreply.github.com> Date: Mon, 28 Aug 2023 13:10:56 -0700 Subject: [PATCH 037/201] write catalog schema with canonical ordering (#8463) --- .../server/handlers/SourceHandler.java | 8 ++- .../config/persistence/ConfigRepository.java | 61 +++++++++++++------ .../ConfigRepositoryE2EReadWriteTest.java | 58 +++++++++++++++++- .../src/main/kotlin/FlagDefinitions.kt | 2 + 4 files changed, 109 insertions(+), 20 deletions(-) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java index 68de335ff1d..f6fcfbcb651 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceHandler.java @@ -41,6 +41,7 @@ import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.config.persistence.split_secrets.SecretCoordinate; import io.airbyte.featureflag.CanonicalCatalogSchema; +import io.airbyte.featureflag.CatalogCanonicalJson; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.Source; import io.airbyte.persistence.job.factory.OAuthConfigSupplier; @@ -394,13 +395,18 @@ private boolean shouldWriteCanonicalActorCatalog(final SourceDiscoverSchemaWrite return request.getSourceId() != null && featureFlagClient.boolVariation(CanonicalCatalogSchema.INSTANCE, new Source(request.getSourceId())); } + private boolean shouldWriteCatalogInCanonicalJson(SourceDiscoverSchemaWriteRequestBody request) { + return request.getSourceId() != null && featureFlagClient.boolVariation(CatalogCanonicalJson.INSTANCE, new Source(request.getSourceId())); + } + private UUID writeCanonicalActorCatalog(final AirbyteCatalog persistenceCatalog, final SourceDiscoverSchemaWriteRequestBody request) throws IOException { return configRepository.writeCanonicalActorCatalogFetchEvent( persistenceCatalog, request.getSourceId(), request.getConnectorVersion(), - request.getConfigurationHash()); + request.getConfigurationHash(), + shouldWriteCatalogInCanonicalJson(request)); } private UUID writeActorCatalog(final AirbyteCatalog persistenceCatalog, final SourceDiscoverSchemaWriteRequestBody request) throws IOException { diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index ad23b7961ae..1195e8c7690 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -2413,38 +2413,63 @@ private UUID getOrInsertActorCatalog(final AirbyteCatalog airbyteCatalog, * @param airbyteCatalog the catalog to be cached * @param context - db context * @param timestamp - timestamp + * @param writeCatalogInCanonicalJson - should we write the catalog in canonical json * @return the db identifier for the cached catalog. */ private UUID getOrInsertCanonicalActorCatalog(final AirbyteCatalog airbyteCatalog, final DSLContext context, - final OffsetDateTime timestamp) { - final HashFunction hashFunction = Hashing.murmur3_32_fixed(); + final OffsetDateTime timestamp, + final boolean writeCatalogInCanonicalJson) { + + final String canonicalCatalogHash = generateCanonicalHash(airbyteCatalog); + UUID catalogId = lookupCatalogId(canonicalCatalogHash, airbyteCatalog, context); + if (catalogId != null) { + return catalogId; + } + + final String oldCatalogHash = generateOldHash(airbyteCatalog); + catalogId = lookupCatalogId(oldCatalogHash, airbyteCatalog, context); + if (catalogId != null) { + return catalogId; + } + + final String catalogHash = writeCatalogInCanonicalJson ? canonicalCatalogHash : oldCatalogHash; + + return insertCatalog(airbyteCatalog, catalogHash, context, timestamp); + } + private String generateCanonicalHash(AirbyteCatalog airbyteCatalog) { + final HashFunction hashFunction = Hashing.murmur3_32_fixed(); try { - final String catalogHash = hashFunction.hashBytes(Jsons.canonicalJsonSerialize(airbyteCatalog) + return hashFunction.hashBytes(Jsons.canonicalJsonSerialize(airbyteCatalog) .getBytes(Charsets.UTF_8)).toString(); - - final UUID catalogId = findAndReturnCatalogId(catalogHash, airbyteCatalog, context); - if (catalogId != null) { - return catalogId; - } - } catch (final IOException e) { + } catch (IOException e) { LOGGER.error("Failed to serialize AirbyteCatalog to canonical JSON", e); + return null; } + } - // Fallback to the old json when canonical json serialization failed - final String oldCatalogHash = hashFunction.hashBytes(Jsons.serialize(airbyteCatalog).getBytes(Charsets.UTF_8)).toString(); + private String generateOldHash(AirbyteCatalog airbyteCatalog) { + final HashFunction hashFunction = Hashing.murmur3_32_fixed(); + return hashFunction.hashBytes(Jsons.serialize(airbyteCatalog).getBytes(Charsets.UTF_8)).toString(); + } - final UUID oldCatalogId = findAndReturnCatalogId(oldCatalogHash, airbyteCatalog, context); - if (oldCatalogId != null) { - return oldCatalogId; + private UUID lookupCatalogId(String catalogHash, AirbyteCatalog airbyteCatalog, DSLContext context) { + if (catalogHash == null) { + return null; } + return findAndReturnCatalogId(catalogHash, airbyteCatalog, context); + } + private UUID insertCatalog(AirbyteCatalog airbyteCatalog, + String catalogHash, + DSLContext context, + OffsetDateTime timestamp) { final UUID catalogId = UUID.randomUUID(); context.insertInto(ACTOR_CATALOG) .set(ACTOR_CATALOG.ID, catalogId) .set(ACTOR_CATALOG.CATALOG, JSONB.valueOf(Jsons.serialize(airbyteCatalog))) - .set(ACTOR_CATALOG.CATALOG_HASH, oldCatalogHash) + .set(ACTOR_CATALOG.CATALOG_HASH, catalogHash) .set(ACTOR_CATALOG.CREATED_AT, timestamp) .set(ACTOR_CATALOG.MODIFIED_AT, timestamp).execute(); return catalogId; @@ -2611,6 +2636,7 @@ public UUID writeActorCatalogFetchEvent(final AirbyteCatalog catalog, * @param catalog - catalog that was fetched. * @param actorId - actor the catalog was fetched by * @param connectorVersion - version of the connector when catalog was fetched + * @param writeCatalogInCanonicalJson - should we write the catalog in canonical json * @param configurationHash - hash of the config of the connector when catalog was fetched * @return The identifier (UUID) of the fetch event inserted in the database * @throws IOException - error while interacting with db @@ -2618,12 +2644,13 @@ public UUID writeActorCatalogFetchEvent(final AirbyteCatalog catalog, public UUID writeCanonicalActorCatalogFetchEvent(final AirbyteCatalog catalog, final UUID actorId, final String connectorVersion, - final String configurationHash) + final String configurationHash, + final boolean writeCatalogInCanonicalJson) throws IOException { final OffsetDateTime timestamp = OffsetDateTime.now(); final UUID fetchEventID = UUID.randomUUID(); return database.transaction(ctx -> { - final UUID catalogId = getOrInsertCanonicalActorCatalog(catalog, ctx, timestamp); + final UUID catalogId = getOrInsertCanonicalActorCatalog(catalog, ctx, timestamp, writeCatalogInCanonicalJson); ctx.insertInto(ACTOR_CATALOG_FETCH_EVENT) .set(ACTOR_CATALOG_FETCH_EVENT.ID, fetchEventID) .set(ACTOR_CATALOG_FETCH_EVENT.ACTOR_ID, actorId) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 960db980cd0..7e8ee2d3b63 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -169,12 +169,12 @@ void testReadActorCatalog() throws IOException, JsonValidationException, SQLExce final AirbyteCatalog firstCatalog = CatalogHelpers.createAirbyteCatalog("product", Field.of("label", JsonSchemaType.STRING), Field.of("size", JsonSchemaType.NUMBER), Field.of("color", JsonSchemaType.STRING), Field.of("price", JsonSchemaType.NUMBER)); - configRepository.writeCanonicalActorCatalogFetchEvent(firstCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH); + configRepository.writeCanonicalActorCatalogFetchEvent(firstCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH, false); final AirbyteCatalog secondCatalog = CatalogHelpers.createAirbyteCatalog("product", Field.of("size", JsonSchemaType.NUMBER), Field.of("label", JsonSchemaType.STRING), Field.of("color", JsonSchemaType.STRING), Field.of("price", JsonSchemaType.NUMBER)); - configRepository.writeCanonicalActorCatalogFetchEvent(secondCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, otherConfigHash); + configRepository.writeCanonicalActorCatalogFetchEvent(secondCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, otherConfigHash, false); final String expectedCatalog = "{" @@ -202,6 +202,60 @@ void testReadActorCatalog() throws IOException, JsonValidationException, SQLExce assertEquals(expectedCatalog, Jsons.serialize(catalogResult.get().getCatalog())); } + @Test + void testWriteCanonicalActorCatalog() throws IOException, JsonValidationException, SQLException { + final String canonicalConfigHash = "8ad32981"; + final StandardWorkspace workspace = MockData.standardWorkspaces().get(0); + + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withSourceDefinitionId(UUID.randomUUID()) + .withSourceType(SourceType.DATABASE) + .withName("sourceDefinition"); + final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) + .withVersionId(sourceDefinition.getDefaultVersionId()); + configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + + final SourceConnection source = new SourceConnection() + .withSourceDefinitionId(sourceDefinition.getSourceDefinitionId()) + .withSourceId(UUID.randomUUID()) + .withName("SomeConnector") + .withWorkspaceId(workspace.getWorkspaceId()) + .withConfiguration(Jsons.deserialize("{}")); + configRepository.writeSourceConnectionNoSecrets(source); + + final AirbyteCatalog firstCatalog = CatalogHelpers.createAirbyteCatalog("product", + Field.of("label", JsonSchemaType.STRING), Field.of("size", JsonSchemaType.NUMBER), + Field.of("color", JsonSchemaType.STRING), Field.of("price", JsonSchemaType.NUMBER)); + configRepository.writeCanonicalActorCatalogFetchEvent(firstCatalog, source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH, true); + + String expectedCatalog = + "{" + + "\"streams\":[" + + "{" + + "\"default_cursor_field\":[]," + + "\"json_schema\":{" + + "\"properties\":{" + + "\"color\":{\"type\":\"string\"}," + + "\"label\":{\"type\":\"string\"}," + + "\"price\":{\"type\":\"number\"}," + + "\"size\":{\"type\":\"number\"}" + + "}," + + "\"type\":\"object\"" + + "}," + + "\"name\":\"product\"," + + "\"source_defined_primary_key\":[]," + + "\"supported_sync_modes\":[\"full_refresh\"]" + + "}" + + "]" + + "}"; + + final Optional catalogResult = configRepository.getActorCatalog(source.getSourceId(), DOCKER_IMAGE_TAG, CONFIG_HASH); + assertTrue(catalogResult.isPresent()); + assertEquals(catalogResult.get().getCatalogHash(), canonicalConfigHash); + assertEquals(expectedCatalog, Jsons.canonicalJsonSerialize(catalogResult.get().getCatalog())); + } + @Test void testSimpleInsertActorCatalog() throws IOException, JsonValidationException, SQLException { final String otherConfigHash = "OtherConfigHash"; diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index b5409aa25e4..57e9c643ba3 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -41,6 +41,8 @@ object AutoPropagateNewStreams : Temporary(key = "autopropagate-new-str object CanonicalCatalogSchema : Temporary(key = "canonical-catalog-schema", default = false) +object CatalogCanonicalJson : Temporary(key = "catalog-canonical-json", default = false) + object CheckConnectionUseApiEnabled : Temporary(key = "check-connection-use-api", default = false) object CheckConnectionUseChildWorkflowEnabled : Temporary(key = "check-connection-use-child-workflow", default = false) From 19a254589a02c522879f9a5a9e0e2b30f3cb2153 Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Mon, 28 Aug 2023 22:35:31 +0200 Subject: [PATCH 038/201] introduce another retry around an api request (#8571) --- .../test/utils/AcceptanceTestHarness.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java index 306c58ab404..85465d3a714 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java +++ b/airbyte-test-utils/src/main/java/io/airbyte/test/utils/AcceptanceTestHarness.java @@ -785,14 +785,16 @@ public SourceDefinitionRead createE2eSourceDefinition(final UUID workspaceId) { return sourceDefinitionRead; } - public DestinationDefinitionRead createE2eDestinationDefinition(final UUID workspaceId) throws ApiException { - return apiClient.getDestinationDefinitionApi().createCustomDestinationDefinition(new CustomDestinationDefinitionCreate() - .workspaceId(workspaceId) - .destinationDefinition(new DestinationDefinitionCreate() - .name("E2E Test Destination") - .dockerRepository("airbyte/destination-e2e-test") - .dockerImageTag(DESTINATION_E2E_TEST_CONNECTOR_VERSION) - .documentationUrl(URI.create("https://example.com")))); + public DestinationDefinitionRead createE2eDestinationDefinition(final UUID workspaceId) throws Exception { + return AirbyteApiClient.retryWithJitterThrows(() -> apiClient.getDestinationDefinitionApi() + .createCustomDestinationDefinition(new CustomDestinationDefinitionCreate() + .workspaceId(workspaceId) + .destinationDefinition(new DestinationDefinitionCreate() + .name("E2E Test Destination") + .dockerRepository("airbyte/destination-e2e-test") + .dockerImageTag(DESTINATION_E2E_TEST_CONNECTOR_VERSION) + .documentationUrl(URI.create("https://example.com")))), + "create destination definition", 10, 60, 3); } public SourceRead createPostgresSource() { From 124acd018cfdbd2c2edc42fdc50978b9763014bd Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 29 Aug 2023 14:47:17 +0200 Subject: [PATCH 039/201] =?UTF-8?q?=F0=9F=AA=9F=F0=9F=94=A7=20Memoize=20no?= =?UTF-8?q?tification=20service=20(#8581)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Notification/NotificationService.tsx | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx b/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx index 1aec7291daf..e034d5a49ac 100644 --- a/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx +++ b/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx @@ -1,5 +1,5 @@ import { motion, AnimatePresence } from "framer-motion"; -import React, { useContext, useMemo } from "react"; +import React, { useCallback, useContext, useMemo } from "react"; import { Toast } from "components/ui/Toast"; @@ -17,7 +17,7 @@ export const NotificationService = React.memo(({ children }: { children: React.R typeof actions >(notificationServiceReducer, initialState, actions); - const notificationService: NotificationServiceApi = useMemo( + const baseNotificationService: NotificationServiceApi = useMemo( () => ({ addNotification, deleteNotificationById, @@ -27,15 +27,24 @@ export const NotificationService = React.memo(({ children }: { children: React.R [] ); - const registerNotification = (notification: Notification) => { - addNotification({ ...notification, timeout: notification.timeout ?? notification.type !== "error" }); - }; + const registerNotification = useCallback( + (notification: Notification) => { + addNotification({ ...notification, timeout: notification.timeout ?? notification.type !== "error" }); + }, + [addNotification] + ); + + const notificationService = useMemo( + () => ({ + ...baseNotificationService, + addNotification: registerNotification, + }), + [baseNotificationService, registerNotification] + ); return ( <> - - {children} - + {children} {state.notifications From 8f17aa4c8b96fe5242f94128541a055bd7bc35c8 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 29 Aug 2023 15:48:56 +0200 Subject: [PATCH 040/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Make=20po?= =?UTF-8?q?llUntil=20test=20use=20fake=20timers=20(#8554)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lake Mossman --- .../src/core/utils/pollUntil.test.ts | 75 ++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/airbyte-webapp/src/core/utils/pollUntil.test.ts b/airbyte-webapp/src/core/utils/pollUntil.test.ts index 796e84a3e6a..9ecdc97fdc2 100644 --- a/airbyte-webapp/src/core/utils/pollUntil.test.ts +++ b/airbyte-webapp/src/core/utils/pollUntil.test.ts @@ -5,63 +5,70 @@ const fourZerosAndThenSeven = () => { let _callCount = 0; return () => Promise.resolve([0, 0, 0, 0, 7][_callCount++]); }; -// eslint-disable-next-line -const truthyResponse = (x: any) => !!x; + +const truthyResponse = (x: unknown) => !!x; describe("pollUntil", () => { + beforeAll(() => { + jest.useFakeTimers({ doNotFake: ["nextTick"] }); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + describe("when maxTimeoutMs is not provided", () => { it("calls the provided apiFn until condition returns true and resolves to its final return value", () => { const pollableFn = fourZerosAndThenSeven(); - - return expect(pollUntil(pollableFn, truthyResponse, { intervalMs: 1 })).resolves.toBe(7); + const result = pollUntil(pollableFn, truthyResponse, { intervalMs: 1 }); + jest.advanceTimersByTime(10); + return expect(result).resolves.toBe(7); }); }); describe("when condition returns true before maxTimeoutMs is reached", () => { it("calls the provided apiFn until condition returns true and resolves to its final return value", () => { const pollableFn = fourZerosAndThenSeven(); - - return expect(pollUntil(pollableFn, truthyResponse, { intervalMs: 1, maxTimeoutMs: 100 })).resolves.toBe(7); + const result = pollUntil(pollableFn, truthyResponse, { intervalMs: 1, maxTimeoutMs: 100 }); + jest.advanceTimersByTime(10); + return expect(result).resolves.toBe(7); }); }); describe("when maxTimeoutMs is reached before condition returns true", () => { it("resolves to false", () => { const pollableFn = fourZerosAndThenSeven(); - - return expect(pollUntil(pollableFn, truthyResponse, { intervalMs: 100, maxTimeoutMs: 1 })).resolves.toBe(false); + const result = pollUntil(pollableFn, truthyResponse, { intervalMs: 100, maxTimeoutMs: 1 }); + jest.advanceTimersByTime(100); + return expect(result).resolves.toBe(false); }); - // Because the timing of the polling depends on both the provided `intervalMs` and the - // execution time of `apiFn`, the timing of polling iterations isn't entirely - // deterministic; it's precise enough for its job, but it's difficult to make precise - // test assertions about polling behavior without long intervalMs/maxTimeoutMs bogging - // down the test suite. it("calls its apiFn arg no more than (maxTimeoutMs / intervalMs) times", async () => { let _callCount = 0; - let lastCalledValue = 999; - const pollableFn = () => - Promise.resolve([1, 2, 3, 4, 5][_callCount++]).then((val) => { - lastCalledValue = val; - return val; - }); + const pollableFn = jest.fn(() => { + return Promise.resolve([1, 2, 3, 4, 5][_callCount++]); + }); + + const polling = pollUntil(pollableFn, (_) => false, { intervalMs: 20, maxTimeoutMs: 78 }); + + // Advance the timer by 20ms each. Make sure to wait one more tick (which isn't using fake timers) + // so rxjs will actually call pollableFn again (and thus we get the right count on the that function). + // Without waiting a tick after each advance timer, we'd effectively just advance by 80ms and + // not call the pollableFn multiple times, because the maxTimeout logic would be triggered which + // would cause the subsequent pollableFn calls to not be properly processed. + jest.advanceTimersByTime(20); + await new Promise(process.nextTick); + jest.advanceTimersByTime(20); + await new Promise(process.nextTick); + jest.advanceTimersByTime(20); + await new Promise(process.nextTick); + jest.advanceTimersByTime(20); + await new Promise(process.nextTick); - await pollUntil(pollableFn, (_) => false, { intervalMs: 20, maxTimeoutMs: 78 }); + const result = await polling; - // In theory, this is what just happened: - // | time elapsed | value (source) | - // |--------------+-----------------| - // | 0ms | 1 (poll) | - // | 20ms | 2 (poll) | - // | 40ms | 3 (poll) | - // | 60ms | 4 (poll) | - // | 78ms | false (timeout) | - // - // In practice, since the polling intervalMs isn't started until after `apiFn` - // resolves to a value, the actual call counts are slightly nondeterministic. We - // could ignore that fact with a slow enough intervalMs, but who wants slow tests? - expect(lastCalledValue > 2).toBe(true); - expect(lastCalledValue <= 4).toBe(true); + expect(result).toBe(false); + expect(pollableFn).toHaveBeenCalledTimes(4); }); }); }); From 489830bf63aac788b21d4938995e6372570b7690 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Tue, 29 Aug 2023 10:04:58 -0400 Subject: [PATCH 041/201] =?UTF-8?q?=F0=9F=AA=9F=20Remove=20the=20terminolo?= =?UTF-8?q?gy=20"GA"=20from=20credits=20banners=20(#8507)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/packages/cloud/locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json index e09c022504a..0f13d79c6e9 100644 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ b/airbyte-webapp/src/packages/cloud/locales/en.json @@ -169,7 +169,7 @@ "credits.whatAre": "What are credits?", "credits.usage": "Usage", "credits.noData": "You have no credits usage data for this period. Sync a connection to get started!", - "credits.creditsProblem": "You’re out of credits! To set up connections and run syncs with GA connectors, add credits.", + "credits.creditsProblem": "You’re out of credits! To set up connections and run syncs, add credits.", "credits.emailVerificationRequired": "You need to verify your email address before you can buy credits.", "credits.emailVerification.resendConfirmation": "We sent you a new verification link.", "credits.emailVerification.resend": "Send verification link again", From 172fbe99f9e14ae39765b0e20e71e44d911eecbe Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 29 Aug 2023 18:28:54 +0300 Subject: [PATCH 042/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Do=20not?= =?UTF-8?q?=20track=20dirty=20changes=20in=20`?= =?UTF-8?q?`=20(#8563)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../TransformationHookForm.tsx | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/airbyte-webapp/src/components/connection/TransformationHookForm/TransformationHookForm.tsx b/airbyte-webapp/src/components/connection/TransformationHookForm/TransformationHookForm.tsx index 9ecfee2bd6f..af63faa938b 100644 --- a/airbyte-webapp/src/components/connection/TransformationHookForm/TransformationHookForm.tsx +++ b/airbyte-webapp/src/components/connection/TransformationHookForm/TransformationHookForm.tsx @@ -7,7 +7,6 @@ import { FlexContainer, FlexItem } from "components/ui/Flex"; import { ModalBody, ModalFooter } from "components/ui/Modal"; import { useOperationsCheck } from "core/api"; -import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { dbtOperationReadOrCreateSchema } from "./schema"; import { DbtOperationReadOrCreate } from "./types"; @@ -30,26 +29,17 @@ interface TransformationHookFormProps { export const TransformationHookForm: React.FC = ({ transformation, onDone, onCancel }) => { const { formatMessage } = useIntl(); const operationCheck = useOperationsCheck(); - const { clearFormChange } = useFormChangeTrackerService(); - const formId = useUniqueFormId(); const onSubmit = async (values: DbtOperationReadOrCreate) => { await operationCheck(values); - clearFormChange(formId); onDone(values); }; - const onFormCancel = () => { - clearFormChange(formId); - onCancel(); - }; - return ( onSubmit={onSubmit} schema={dbtOperationReadOrCreateSchema} defaultValues={transformation} - trackDirtyChanges > @@ -96,7 +86,7 @@ export const TransformationHookForm: React.FC = ({ - + ); From 189b17330a361f4320cef08e5e082d6c29d54e6a Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Tue, 29 Aug 2023 18:26:10 +0200 Subject: [PATCH 043/201] remove ingestBreakingChanges feature flag (#8584) --- .../DestinationDefinitionsHandler.java | 5 +--- .../handlers/SourceDefinitionsHandler.java | 5 +--- .../DestinationDefinitionsHandlerTest.java | 3 -- .../SourceDefinitionsHandlerTest.java | 3 -- .../config/init/ApplyDefinitionsHelper.java | 5 +--- .../init/ApplyDefinitionsHelperTest.java | 28 ------------------- .../src/main/kotlin/FlagDefinitions.kt | 2 -- 7 files changed, 3 insertions(+), 48 deletions(-) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java index dd609bc6bca..57c24398710 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java @@ -40,7 +40,6 @@ import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.Workspace; @@ -286,9 +285,7 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe actorDefinitionHandlerHelper.getBreakingChanges(newVersion, ActorType.DESTINATION); configRepository.writeDestinationDefinitionAndDefaultVersion(newDestination, newVersion, breakingChangesForDef); - if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { - configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); - } + configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { final StandardDestinationDefinition updatedDestinationDefinition = configRepository .getStandardDestinationDefinition(destinationDefinitionUpdate.getDestinationDefinitionId()); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java index f37c177e521..aed724d27f5 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java @@ -40,7 +40,6 @@ import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.SourceDefinition; @@ -285,10 +284,8 @@ public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate final List breakingChangesForDef = actorDefinitionHandlerHelper.getBreakingChanges(newVersion, ActorType.SOURCE); configRepository.writeSourceDefinitionAndDefaultVersion(newSource, newVersion, breakingChangesForDef); + configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); - if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { - configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); - } if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { final StandardSourceDefinition updatedSourceDefinition = configRepository.getStandardSourceDefinition(newSource.getSourceDefinitionId()); supportStateUpdater.updateSupportStatesForSourceDefinition(updatedSourceDefinition); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java index dc7149efa39..dfc2f9b0d19 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java @@ -58,7 +58,6 @@ import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.TestClient; @@ -126,8 +125,6 @@ void setUp() { destinationHandler, supportStateUpdater, featureFlagClient); - - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); } private StandardDestinationDefinition generateDestinationDefinition() { diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java index 2ddd94abd13..dc2607815d3 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java @@ -58,7 +58,6 @@ import io.airbyte.config.specs.RemoteDefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.HideActorDefinitionFromList; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.SourceDefinition; @@ -117,8 +116,6 @@ void setUp() { sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, uuidSupplier, actorDefinitionHandlerHelper, remoteDefinitionsProvider, sourceHandler, supportStateUpdater, featureFlagClient); - - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); } private StandardSourceDefinition generateSourceDefinition() { diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java index 9b0f38d91fc..7d677d36973 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java @@ -20,7 +20,6 @@ import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.Workspace; import io.airbyte.persistence.job.JobPersistence; @@ -105,9 +104,7 @@ public void apply(final boolean updateAll) throws JsonValidationException, IOExc for (final ConnectorRegistryDestinationDefinition def : protocolCompatibleDestinationDefinitions) { applyDestinationDefinition(actorDefinitionIdsToDefaultVersionsMap, def, actorDefinitionIdsInUse, updateAll); } - if (featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))) { - configRepository.writeActorDefinitionBreakingChanges(allBreakingChanges); - } + configRepository.writeActorDefinitionBreakingChanges(allBreakingChanges); if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { supportStateUpdater.updateSupportStates(); } diff --git a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java index 3a607431c01..8f858baa85e 100644 --- a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java +++ b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java @@ -5,7 +5,6 @@ package io.airbyte.config.init; import static io.airbyte.featureflag.ContextKt.ANONYMOUS; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -27,7 +26,6 @@ import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.IngestBreakingChanges; import io.airbyte.featureflag.RunSupportStateUpdater; import io.airbyte.featureflag.TestClient; import io.airbyte.featureflag.Workspace; @@ -110,7 +108,6 @@ void setup() { applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobPersistence, configRepository, featureFlagClient, supportStateUpdater); - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); when(featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(true); // Default calls to empty. @@ -279,31 +276,6 @@ void testTurnOffRunSupportStateUpdaterFeatureFlag() throws JsonValidationExcepti verifyNoMoreInteractions(configRepository, supportStateUpdater); } - @Test - void testTurnOffIngestBreakingChangesFeatureFlag() throws JsonValidationException, IOException, ConfigNotFoundException { - when(featureFlagClient.boolVariation(IngestBreakingChanges.INSTANCE, new Workspace(ANONYMOUS))).thenReturn(false); - - when(definitionsProvider.getSourceDefinitions()).thenReturn(List.of(SOURCE_POSTGRES_2)); - when(definitionsProvider.getDestinationDefinitions()).thenReturn(List.of(DESTINATION_S3_2)); - - applyDefinitionsHelper.apply(true); - verifyConfigRepositoryGetInteractions(); - - verify(configRepository).writeSourceDefinitionAndDefaultVersion( - ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), - ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), - ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( - ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), - ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), - ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository, never()).writeActorDefinitionBreakingChanges(any()); - - verify(supportStateUpdater).updateSupportStates(); - - verifyNoMoreInteractions(configRepository, supportStateUpdater); - } - private static List getExpectedBreakingChanges() { final List breakingChanges = new ArrayList<>(); breakingChanges.addAll(ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index 57e9c643ba3..552aad3e99d 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -65,8 +65,6 @@ object ConnectorVersionOverride : Permanent(key = "connectors.versionOve object UseActorScopedDefaultVersions : Temporary(key = "connectors.useActorScopedDefaultVersions", default = true) -object IngestBreakingChanges : Temporary(key = "connectors.ingestBreakingChanges", default = true) - object RunSupportStateUpdater : Temporary(key = "connectors.runSupportStateUpdater", default = true) object RefreshSchemaPeriod : Temporary(key = "refreshSchema.period.hours", default = 24) From 4d9c39a5ae72f9ca257e901c19e2fd9de3c7787a Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Tue, 29 Aug 2023 09:33:29 -0700 Subject: [PATCH 044/201] Remove heartbeat from launcher worker (#8577) --- .../KubeOrchestratorHandleFactory.java | 6 - .../workers/sync/DbtLauncherWorker.java | 7 - .../airbyte/workers/sync/LauncherWorker.java | 274 ++++++++---------- .../sync/NormalizationLauncherWorker.java | 7 - .../sync/ReplicationLauncherWorker.java | 7 - .../sync/DbtTransformationActivityImpl.java | 5 +- .../sync/NormalizationActivityImpl.java | 5 +- 7 files changed, 127 insertions(+), 184 deletions(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java index 84af91329f1..3f5f485c0ea 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/orchestrator/KubeOrchestratorHandleFactory.java @@ -6,7 +6,6 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.functional.CheckedSupplier; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.commons.workers.config.WorkerConfigsProvider; import io.airbyte.commons.workers.config.WorkerConfigsProvider.ResourceType; @@ -45,18 +44,15 @@ public class KubeOrchestratorHandleFactory implements OrchestratorHandleFactory private final ContainerOrchestratorConfig containerOrchestratorConfig; private final WorkerConfigsProvider workerConfigsProvider; private final FeatureFlagClient featureFlagClient; - private final TemporalUtils temporalUtils; private final Integer serverPort; public KubeOrchestratorHandleFactory(@Named("containerOrchestratorConfig") final ContainerOrchestratorConfig containerOrchestratorConfig, final WorkerConfigsProvider workerConfigsProvider, final FeatureFlagClient featureFlagClient, - final TemporalUtils temporalUtils, @Value("${micronaut.server.port}") final Integer serverPort) { this.containerOrchestratorConfig = containerOrchestratorConfig; this.workerConfigsProvider = workerConfigsProvider; this.featureFlagClient = featureFlagClient; - this.temporalUtils = temporalUtils; this.serverPort = serverPort; } @@ -78,9 +74,7 @@ public CheckedSupplier, Exception> destinationLauncherConfig, jobRunConfig, syncInput.getSyncResourceRequirements() != null ? syncInput.getSyncResourceRequirements().getOrchestrator() : null, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient); } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java index 1b70c483805..890b11d0a6c 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/DbtLauncherWorker.java @@ -8,17 +8,14 @@ import static io.airbyte.workers.process.Metadata.SYNC_STEP_KEY; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.OperatorDbtInput; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; -import io.temporal.activity.ActivityExecutionContext; import java.util.Map; import java.util.UUID; -import java.util.function.Supplier; /** * Dbt Launcher Worker. @@ -35,9 +32,7 @@ public DbtLauncherWorker(final UUID connectionId, final JobRunConfig jobRunConfig, final WorkerConfigs workerConfigs, final ContainerOrchestratorConfig containerOrchestratorConfig, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final FeatureFlagClient featureFlagClient) { super( connectionId, @@ -50,9 +45,7 @@ public DbtLauncherWorker(final UUID connectionId, containerOrchestratorConfig, workerConfigs.getResourceRequirements(), Void.class, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient, // Custom connector does not use Dbt at this moment, thus this flag for runnning job under diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java index 0bb07ead93c..86bacc48fb8 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java @@ -16,7 +16,6 @@ import io.airbyte.commons.helper.DockerImageNameHelper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.temporal.sync.OrchestratorConstants; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.ResourceRequirements; @@ -39,7 +38,6 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.KubernetesClientException; import io.micronaut.core.util.StringUtils; -import io.temporal.activity.ActivityExecutionContext; import java.nio.file.Path; import java.time.Duration; import java.util.Collections; @@ -49,8 +47,6 @@ import java.util.UUID; import java.util.concurrent.CancellationException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.slf4j.Logger; @@ -85,9 +81,7 @@ public abstract class LauncherWorker implements Worker outputClass; - private final Supplier activityContext; private final Integer serverPort; - private final TemporalUtils temporalUtils; private final WorkerConfigs workerConfigs; private final boolean isCustomConnector; @@ -104,9 +98,7 @@ public LauncherWorker(final UUID connectionId, final ContainerOrchestratorConfig containerOrchestratorConfig, final ResourceRequirements resourceRequirements, final Class outputClass, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final WorkerConfigs workerConfigs, final FeatureFlagClient featureFlagClient, final boolean isCustomConnector) { @@ -120,9 +112,7 @@ public LauncherWorker(final UUID connectionId, this.containerOrchestratorConfig = containerOrchestratorConfig; this.resourceRequirements = resourceRequirements; this.outputClass = outputClass; - this.activityContext = activityContext; this.serverPort = serverPort; - this.temporalUtils = temporalUtils; this.workerConfigs = workerConfigs; this.featureFlagClient = featureFlagClient; this.isCustomConnector = isCustomConnector; @@ -134,154 +124,140 @@ public LauncherWorker(final UUID connectionId, @Trace(operationName = WORKER_OPERATION_NAME) @Override public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException { - final AtomicBoolean isCanceled = new AtomicBoolean(false); - final AtomicReference cancellationCallback = new AtomicReference<>(null); - return temporalUtils.withBackgroundHeartbeat(cancellationCallback, () -> { - try { - // Assemble configuration. - final Map envMap = System.getenv().entrySet().stream() - .filter(entry -> OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - // Manually add the worker environment to the env var map - envMap.put(WorkerConstants.WORKER_ENVIRONMENT, containerOrchestratorConfig.workerEnvironment().name()); - - // Merge in the env from the ContainerOrchestratorConfig - containerOrchestratorConfig.environmentVariables().entrySet().stream().forEach(e -> envMap.putIfAbsent(e.getKey(), e.getValue())); - - // Allow for the override of the socat pod CPU resources as part of the concurrent source read - // experimentation - final String socatResources = featureFlagClient.stringVariation(ConcurrentSocatResources.INSTANCE, new Connection(connectionId)); - if (StringUtils.isNotEmpty(socatResources)) { - LOGGER.info("Overriding Socat CPU limit and request to {}.", socatResources); - envMap.put(SOCAT_KUBE_CPU_LIMIT, socatResources); - envMap.put(SOCAT_KUBE_CPU_REQUEST, socatResources); - } - - final Map fileMap = new HashMap<>(additionalFileMap); - fileMap.putAll(Map.of( - OrchestratorConstants.INIT_FILE_APPLICATION, application, - OrchestratorConstants.INIT_FILE_JOB_RUN_CONFIG, Jsons.serialize(jobRunConfig), - OrchestratorConstants.INIT_FILE_INPUT, Jsons.serialize(input), - // OrchestratorConstants.INIT_FILE_ENV_MAP might be duplicated since the pod env contains everything - OrchestratorConstants.INIT_FILE_ENV_MAP, Jsons.serialize(envMap))); - - final Map portMap = Map.of( - serverPort, serverPort, - OrchestratorConstants.PORT1, OrchestratorConstants.PORT1, - OrchestratorConstants.PORT2, OrchestratorConstants.PORT2, - OrchestratorConstants.PORT3, OrchestratorConstants.PORT3, - OrchestratorConstants.PORT4, OrchestratorConstants.PORT4); - - final var podNameAndJobPrefix = podNamePrefix + "-job-" + jobRunConfig.getJobId() + "-attempt-"; - final var podName = podNameAndJobPrefix + jobRunConfig.getAttemptId(); - final var mainContainerInfo = new KubeContainerInfo(containerOrchestratorConfig.containerOrchestratorImage(), - containerOrchestratorConfig.containerOrchestratorImagePullPolicy()); - final var kubePodInfo = new KubePodInfo(containerOrchestratorConfig.namespace(), - podName, - mainContainerInfo); - - ApmTraceUtils.addTagsToTrace(connectionId, jobRunConfig.getJobId(), jobRoot); - - final String schedulerName = featureFlagClient.stringVariation(UseCustomK8sScheduler.INSTANCE, new Connection(connectionId)); - - final String shortImageName = - mainContainerInfo.image() != null ? DockerImageNameHelper.extractShortImageName(mainContainerInfo.image()) : null; - final var allLabels = KubeProcessFactory.getLabels( - jobRunConfig.getJobId(), - Math.toIntExact(jobRunConfig.getAttemptId()), - connectionId, - workspaceId, - shortImageName, - generateMetadataLabels(), - Collections.emptyMap()); - - // Use the configuration to create the process. - process = new AsyncOrchestratorPodProcess( - kubePodInfo, - containerOrchestratorConfig.documentStoreClient(), - containerOrchestratorConfig.kubernetesClient(), - containerOrchestratorConfig.secretName(), - containerOrchestratorConfig.secretMountPath(), - containerOrchestratorConfig.dataPlaneCredsSecretName(), - containerOrchestratorConfig.dataPlaneCredsSecretMountPath(), - containerOrchestratorConfig.googleApplicationCredentials(), - envMap, - workerConfigs.getWorkerKubeAnnotations(), - serverPort, - containerOrchestratorConfig.serviceAccount(), - schedulerName.isBlank() ? null : schedulerName); - - // Define what to do on cancellation. - cancellationCallback.set(() -> { - // When cancelled, try to set to true. - // Only proceed if value was previously false, so we only have one cancellation going. at a time - if (!isCanceled.getAndSet(true)) { - log.info("Trying to cancel async pod process."); - process.destroy(); - } - }); - - // only kill running pods and create process if it is not already running. - if (process.getDocStoreStatus().equals(AsyncKubePodStatus.NOT_STARTED)) { - log.info("Creating " + podName + " for attempt number: " + jobRunConfig.getAttemptId()); - killRunningPodsForConnection(); - - // custom connectors run in an isolated node pool from airbyte-supported connectors - // to reduce the blast radius of any problems with custom connector code. - final var nodeSelectors = - isCustomConnector ? workerConfigs.getWorkerIsolatedKubeNodeSelectors().orElse(workerConfigs.getworkerKubeNodeSelectors()) - : workerConfigs.getworkerKubeNodeSelectors(); - - try { - process.create( - allLabels, - resourceRequirements, - fileMap, - portMap, - nodeSelectors); - } catch (final KubernetesClientException e) { - ApmTraceUtils.addExceptionToTrace(e); - throw new WorkerException( - "Failed to create pod " + podName + ", pre-existing pod exists which didn't advance out of the NOT_STARTED state.", e); - } - } - - // this waitFor can resume if the activity is re-run - process.waitFor(); + try { + // Assemble configuration. + final Map envMap = System.getenv().entrySet().stream() + .filter(entry -> OrchestratorConstants.ENV_VARS_TO_TRANSFER.contains(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // Manually add the worker environment to the env var map + envMap.put(WorkerConstants.WORKER_ENVIRONMENT, containerOrchestratorConfig.workerEnvironment().name()); + + // Merge in the env from the ContainerOrchestratorConfig + containerOrchestratorConfig.environmentVariables().entrySet().stream().forEach(e -> envMap.putIfAbsent(e.getKey(), e.getValue())); + + // Allow for the override of the socat pod CPU resources as part of the concurrent source read + // experimentation + final String socatResources = featureFlagClient.stringVariation(ConcurrentSocatResources.INSTANCE, new Connection(connectionId)); + if (StringUtils.isNotEmpty(socatResources)) { + LOGGER.info("Overriding Socat CPU limit and request to {}.", socatResources); + envMap.put(SOCAT_KUBE_CPU_LIMIT, socatResources); + envMap.put(SOCAT_KUBE_CPU_REQUEST, socatResources); + } - if (cancelled.get()) { - final CancellationException e = new CancellationException(); + final Map fileMap = new HashMap<>(additionalFileMap); + fileMap.putAll(Map.of( + OrchestratorConstants.INIT_FILE_APPLICATION, application, + OrchestratorConstants.INIT_FILE_JOB_RUN_CONFIG, Jsons.serialize(jobRunConfig), + OrchestratorConstants.INIT_FILE_INPUT, Jsons.serialize(input), + // OrchestratorConstants.INIT_FILE_ENV_MAP might be duplicated since the pod env contains everything + OrchestratorConstants.INIT_FILE_ENV_MAP, Jsons.serialize(envMap))); + + final Map portMap = Map.of( + serverPort, serverPort, + OrchestratorConstants.PORT1, OrchestratorConstants.PORT1, + OrchestratorConstants.PORT2, OrchestratorConstants.PORT2, + OrchestratorConstants.PORT3, OrchestratorConstants.PORT3, + OrchestratorConstants.PORT4, OrchestratorConstants.PORT4); + + final var podNameAndJobPrefix = podNamePrefix + "-job-" + jobRunConfig.getJobId() + "-attempt-"; + final var podName = podNameAndJobPrefix + jobRunConfig.getAttemptId(); + final var mainContainerInfo = new KubeContainerInfo(containerOrchestratorConfig.containerOrchestratorImage(), + containerOrchestratorConfig.containerOrchestratorImagePullPolicy()); + final var kubePodInfo = new KubePodInfo(containerOrchestratorConfig.namespace(), + podName, + mainContainerInfo); + + ApmTraceUtils.addTagsToTrace(connectionId, jobRunConfig.getJobId(), jobRoot); + + final String schedulerName = featureFlagClient.stringVariation(UseCustomK8sScheduler.INSTANCE, new Connection(connectionId)); + + final String shortImageName = + mainContainerInfo.image() != null ? DockerImageNameHelper.extractShortImageName(mainContainerInfo.image()) : null; + final var allLabels = KubeProcessFactory.getLabels( + jobRunConfig.getJobId(), + Math.toIntExact(jobRunConfig.getAttemptId()), + connectionId, + workspaceId, + shortImageName, + generateMetadataLabels(), + Collections.emptyMap()); + + // Use the configuration to create the process. + process = new AsyncOrchestratorPodProcess( + kubePodInfo, + containerOrchestratorConfig.documentStoreClient(), + containerOrchestratorConfig.kubernetesClient(), + containerOrchestratorConfig.secretName(), + containerOrchestratorConfig.secretMountPath(), + containerOrchestratorConfig.dataPlaneCredsSecretName(), + containerOrchestratorConfig.dataPlaneCredsSecretMountPath(), + containerOrchestratorConfig.googleApplicationCredentials(), + envMap, + workerConfigs.getWorkerKubeAnnotations(), + serverPort, + containerOrchestratorConfig.serviceAccount(), + schedulerName.isBlank() ? null : schedulerName); + + // only kill running pods and create process if it is not already running. + if (process.getDocStoreStatus().equals(AsyncKubePodStatus.NOT_STARTED)) { + log.info("Creating " + podName + " for attempt number: " + jobRunConfig.getAttemptId()); + killRunningPodsForConnection(); + + // custom connectors run in an isolated node pool from airbyte-supported connectors + // to reduce the blast radius of any problems with custom connector code. + final var nodeSelectors = + isCustomConnector ? workerConfigs.getWorkerIsolatedKubeNodeSelectors().orElse(workerConfigs.getworkerKubeNodeSelectors()) + : workerConfigs.getworkerKubeNodeSelectors(); + + try { + process.create( + allLabels, + resourceRequirements, + fileMap, + portMap, + nodeSelectors); + } catch (final KubernetesClientException e) { ApmTraceUtils.addExceptionToTrace(e); - throw e; + throw new WorkerException( + "Failed to create pod " + podName + ", pre-existing pod exists which didn't advance out of the NOT_STARTED state.", e); } + } - final int asyncProcessExitValue = process.exitValue(); - if (asyncProcessExitValue != 0) { - final WorkerException e = new WorkerException("Orchestrator process exited with non-zero exit code: " + asyncProcessExitValue); - ApmTraceUtils.addTagsToTrace(Map.of(PROCESS_EXIT_VALUE_KEY, asyncProcessExitValue)); - ApmTraceUtils.addExceptionToTrace(e); - throw e; - } + // this waitFor can resume if the activity is re-run + process.waitFor(); - final var output = process.getOutput(); + if (cancelled.get()) { + final CancellationException e = new CancellationException(); + ApmTraceUtils.addExceptionToTrace(e); + throw e; + } - return output.map(s -> Jsons.deserialize(s, outputClass)).orElse(null); - } catch (final Exception e) { + final int asyncProcessExitValue = process.exitValue(); + if (asyncProcessExitValue != 0) { + final WorkerException e = new WorkerException("Orchestrator process exited with non-zero exit code: " + asyncProcessExitValue); + ApmTraceUtils.addTagsToTrace(Map.of(PROCESS_EXIT_VALUE_KEY, asyncProcessExitValue)); ApmTraceUtils.addExceptionToTrace(e); - if (cancelled.get()) { - try { - log.info("Destroying process due to cancellation."); - process.destroy(); - } catch (final Exception e2) { - log.error("Failed to destroy process on cancellation.", e2); - } - throw new WorkerException("Launcher " + application + " was cancelled.", e); - } else { - throw new WorkerException("Running the launcher " + application + " failed", e); + throw e; + } + + final var output = process.getOutput(); + + return output.map(s -> Jsons.deserialize(s, outputClass)).orElse(null); + } catch (final Exception e) { + ApmTraceUtils.addExceptionToTrace(e); + if (cancelled.get()) { + try { + log.info("Destroying process due to cancellation."); + process.destroy(); + } catch (final Exception e2) { + log.error("Failed to destroy process on cancellation.", e2); } + throw new WorkerException("Launcher " + application + " was cancelled.", e); + } else { + throw new WorkerException("Running the launcher " + application + " failed", e); } - }, activityContext); + } } private Map generateMetadataLabels() { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java index aa4f8a48a5c..d6eb84cce8e 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/NormalizationLauncherWorker.java @@ -8,7 +8,6 @@ import static io.airbyte.workers.process.Metadata.SYNC_STEP_KEY; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; @@ -16,10 +15,8 @@ import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; -import io.temporal.activity.ActivityExecutionContext; import java.util.Map; import java.util.UUID; -import java.util.function.Supplier; /** * Normalization Launcher Worker. @@ -36,9 +33,7 @@ public NormalizationLauncherWorker(final UUID connectionId, final JobRunConfig jobRunConfig, final WorkerConfigs workerConfigs, final ContainerOrchestratorConfig containerOrchestratorConfig, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final FeatureFlagClient featureFlagClient) { super( connectionId, @@ -51,9 +46,7 @@ public NormalizationLauncherWorker(final UUID connectionId, containerOrchestratorConfig, workerConfigs.getResourceRequirements(), NormalizationSummary.class, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient, // Normalization process will happen only on a fixed set of connectors, diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java index 2992443ebab..bf35d9ab236 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/ReplicationLauncherWorker.java @@ -8,7 +8,6 @@ import static io.airbyte.workers.process.Metadata.SYNC_STEP_KEY; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.commons.workers.config.WorkerConfigs; import io.airbyte.config.ReplicationOutput; import io.airbyte.config.ResourceRequirements; @@ -17,10 +16,8 @@ import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.ContainerOrchestratorConfig; -import io.temporal.activity.ActivityExecutionContext; import java.util.Map; import java.util.UUID; -import java.util.function.Supplier; /** * Launches a container-orchestrator container/pod to manage the message passing for the replication @@ -41,9 +38,7 @@ public ReplicationLauncherWorker(final UUID connectionId, final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig, final ResourceRequirements resourceRequirements, - final Supplier activityContext, final Integer serverPort, - final TemporalUtils temporalUtils, final WorkerConfigs workerConfigs, final FeatureFlagClient featureFlagClient) { super( @@ -58,9 +53,7 @@ public ReplicationLauncherWorker(final UUID connectionId, containerOrchestratorConfig, resourceRequirements, ReplicationOutput.class, - activityContext, serverPort, - temporalUtils, workerConfigs, featureFlagClient, sourceLauncherConfig.getIsCustomConnector() || destinationLauncherConfig.getIsCustomConnector()); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index 7d61aec1be3..e2146836c61 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -128,7 +128,7 @@ public Void run(final JobRunConfig jobRunConfig, final WorkerConfigs workerConfigs = workerConfigsProvider.getConfig(ResourceType.DEFAULT); workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig, - () -> context, input.getConnectionId(), input.getWorkspaceId()); + input.getConnectionId(), input.getWorkspaceId()); } else { workerFactory = getLegacyWorkerFactory(destinationLauncherConfig, jobRunConfig, resourceRequirements); } @@ -170,7 +170,6 @@ private CheckedSupplier, Exception> getContainerL final WorkerConfigs workerConfigs, final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig, - final Supplier activityContext, final UUID connectionId, final UUID workspaceId) { @@ -181,9 +180,7 @@ private CheckedSupplier, Exception> getContainerL jobRunConfig, workerConfigs, containerOrchestratorConfig.get(), - activityContext, serverPort, - temporalUtils, featureFlagClient); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index c5b6d81e351..2678a2d3909 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -156,7 +156,7 @@ public NormalizationSummary normalize(final JobRunConfig jobRunConfig, if (containerOrchestratorConfig.isPresent()) { final WorkerConfigs workerConfigs = workerConfigsProvider.getConfig(ResourceType.DEFAULT); workerFactory = getContainerLauncherWorkerFactory(workerConfigs, destinationLauncherConfig, jobRunConfig, - () -> context, input.getConnectionId(), input.getWorkspaceId()); + input.getConnectionId(), input.getWorkspaceId()); } else { workerFactory = getLegacyWorkerFactory(destinationLauncherConfig, jobRunConfig); } @@ -263,7 +263,6 @@ private CheckedSupplier, Except final WorkerConfigs workerConfigs, final IntegrationLauncherConfig destinationLauncherConfig, final JobRunConfig jobRunConfig, - final Supplier activityContext, final UUID connectionId, final UUID workspaceId) { return () -> new NormalizationLauncherWorker( @@ -273,9 +272,7 @@ private CheckedSupplier, Except jobRunConfig, workerConfigs, containerOrchestratorConfig.get(), - activityContext, serverPort, - temporalUtils, featureFlagClient); } From b0ea9338b95b776141f529b10f8342b81c941f5d Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 29 Aug 2023 09:42:41 -0700 Subject: [PATCH 045/201] [copy] Add period to `connectionForm.breakingChange.deprecatedNoDeadline.message` (#8585) --- airbyte-webapp/src/locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index e61494c0b6d..c5c5d6d5fcd 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -221,7 +221,7 @@ "connectionForm.update.success": "Connection update succeeded", "connectionForm.update.failed": "Connection update failed", - "connectionForm.breakingChange.deprecatedNoDeadline.message": "A pending upgrade to {actor_name} requires your attention", + "connectionForm.breakingChange.deprecatedNoDeadline.message": "A pending upgrade to {actor_name} requires your attention.", "connectionForm.breakingChange.deprecated.message": "There is a new version available for the {actor_definition_name} {actor_type} connector which contains breaking changes. To ensure continuous syncing, please upgrade the {actor_name} {actor_type} by {upgrade_deadline}, otherwise the {connection_name} connection will automatically be paused on that date to prevent failure or unexpected behavior.", "connectionForm.breakingChange.unsupported.message": "There is a new version available for the {actor_definition_name} {actor_type} connector which contains breaking changes. To prevent failure or unexpected behavior, the {connection_name} connection was automatically paused. Upgrade the {actor_name} {actor_type} to resume syncing.", "connectionForm.breakingChange.source.buttonLabel": "Go to Source", From 1618866dac04ac00acd09033d193fa371675fa49 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Tue, 29 Aug 2023 12:53:36 -0400 Subject: [PATCH 046/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20?= =?UTF-8?q?=F0=9F=8F=81=20(flagged)=20Organization=20and=20Workspace=20mem?= =?UTF-8?q?bers=20visible=20in=20OSS=20settings=20(#8348)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Chandler Prall --- .../src/core/api/hooks/organizations.ts | 12 ++- .../src/core/api/hooks/workspaces.tsx | 9 +++ .../services/analytics/pageTrackingCodes.tsx | 2 + .../hooks/services/Experiment/experiments.ts | 1 + airbyte-webapp/src/locales/en.json | 16 +++- .../components/RoleToolTip.tsx | 39 ---------- .../src/pages/SettingsPage/SettingsPage.tsx | 22 ++++++ .../OrganizationAccessManagementPage.tsx | 17 ++++ .../WorkspaceAccessManagementPage.tsx | 15 ++++ .../components/AccessManagementCard.tsx | 20 +++++ .../AccessManagementPageContent.tsx | 38 +++++++++ .../components/AccessManagementTable.tsx | 45 +++++++++++ .../components/RoleToolTip.tsx | 35 +++++++++ .../components/useGetAccessManagementData.tsx | 77 +++++++++++++++++++ airbyte-webapp/src/services/Scope.ts | 1 + 15 files changed, 307 insertions(+), 42 deletions(-) delete mode 100644 airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleToolTip.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx diff --git a/airbyte-webapp/src/core/api/hooks/organizations.ts b/airbyte-webapp/src/core/api/hooks/organizations.ts index 239b6e3d440..94e905560c8 100644 --- a/airbyte-webapp/src/core/api/hooks/organizations.ts +++ b/airbyte-webapp/src/core/api/hooks/organizations.ts @@ -1,8 +1,8 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { SCOPE_USER } from "services/Scope"; +import { SCOPE_ORGANIZATION, SCOPE_USER } from "services/Scope"; -import { getOrganization, updateOrganization } from "../generated/AirbyteClient"; +import { getOrganization, listUsersInOrganization, updateOrganization } from "../generated/AirbyteClient"; import { OrganizationUpdateRequestBody } from "../generated/AirbyteClient.schemas"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; @@ -10,6 +10,7 @@ import { useSuspenseQuery } from "../useSuspenseQuery"; export const organizationKeys = { all: [SCOPE_USER, "organizations"] as const, detail: (organizationId: string) => [...organizationKeys.all, "details", organizationId] as const, + listUsers: (organizationId: string) => [SCOPE_ORGANIZATION, "users", "list", organizationId] as const, }; export const useOrganization = (organizationId: string) => { @@ -32,3 +33,10 @@ export const useUpdateOrganization = () => { } ); }; + +export const useListUsersInOrganization = (organizationId: string) => { + const requestOptions = useRequestOptions(); + const queryKey = [organizationKeys.listUsers(organizationId)]; + + return useSuspenseQuery(queryKey, () => listUsersInOrganization({ organizationId }, requestOptions)); +}; diff --git a/airbyte-webapp/src/core/api/hooks/workspaces.tsx b/airbyte-webapp/src/core/api/hooks/workspaces.tsx index cdc3377c1a4..49b2b9410b0 100644 --- a/airbyte-webapp/src/core/api/hooks/workspaces.tsx +++ b/airbyte-webapp/src/core/api/hooks/workspaces.tsx @@ -8,6 +8,7 @@ import { createWorkspace, deleteWorkspace, getWorkspace, + listUsersInWorkspace, listWorkspaces, updateWorkspace, updateWorkspaceName, @@ -21,6 +22,7 @@ export const workspaceKeys = { all: [SCOPE_USER, "workspaces"] as const, lists: () => [...workspaceKeys.all, "list"] as const, list: (filters: string) => [...workspaceKeys.lists(), { filters }] as const, + listUsers: (workspaceId: string) => [SCOPE_WORKSPACE, "users", "list", workspaceId] as const, detail: (workspaceId: string) => [...workspaceKeys.all, "details", workspaceId] as const, state: (workspaceId: string) => [...workspaceKeys.all, "state", workspaceId] as const, }; @@ -133,6 +135,13 @@ export const useGetWorkspace = ( return useSuspenseQuery(queryKey, queryFn, options); }; +export const useListUsersInWorkspace = (workspaceId: string) => { + const requestOptions = useRequestOptions(); + const queryKey = workspaceKeys.listUsers(workspaceId); + + return useSuspenseQuery(queryKey, () => listUsersInWorkspace({ workspaceId }, requestOptions)); +}; + export const useUpdateWorkspace = () => { const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); diff --git a/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx b/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx index d2c9b5fc91c..ceecdce01cb 100644 --- a/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx +++ b/airbyte-webapp/src/core/services/analytics/pageTrackingCodes.tsx @@ -27,6 +27,8 @@ export enum PageTrackingCodes { SETTINGS_ACCESS_MANAGEMENT = "Settings.AccessManagement", SETTINGS_METRICS = "Settings.Metrics", SETTINGS_DATA_RESIDENCY = "Settings.DataResidency", + SETTINGS_WORKSPACE_ACCESS_MANAGEMENT = "Settings.WorkspaceAccessManagement", + SETTINGS_ORGANIZATION_ACCESS_MANAGEMENT = "Settings.OrganizationAccessManagement", CREDITS = "Credits", WORKSPACES = "Workspaces", PREFERENCES = "Preferences", diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts index 644be736c7a..709e358461b 100644 --- a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts +++ b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts @@ -34,4 +34,5 @@ export interface Experiments { "settings.emailNotifications": boolean; "upcomingFeaturesPage.url": string; "workspaces.newWorkspacesUI": boolean; + "settings.accessManagement": boolean; } diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index c5c5d6d5fcd..25a78f10f25 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -20,8 +20,11 @@ "user.roleLabel": "Role: {role}", "role.admin": "Admin", - "role.member": "Member", + "role.reader": "Reader", "role.editor": "Editor", + "role.admin.description": "Can view and modify all aspects of the {resourceType}, including its settings, users, and roles", + "role.editor.description": "Can view and modify the {resourceType}", + "role.reader.description": "Has read access to the {resourceType}", "sidebar.homepage": "Homepage", "sidebar.sources": "Sources", @@ -757,6 +760,17 @@ "settings.defaultDataResidencyDescription": "Choose the default preferred data processing location for all of your connections. The default data residency setting only affects new connections. Existing connections will retain their data residency setting. Learn more.", "settings.defaultDataResidencyUpdateError": "There was an error updating the default data residency for this workspace.", "settings.defaultDataResidencyUpdateSuccess": "Data residency preference has been updated!", + "settings.accessManagement": "Access management", + "settings.accessManagement.resource.description": "The following users have access to {resourceName}:", + "settings.accessManagement.organization": "Organization users", + "settings.accessManagement.workspace": "Workspace users", + "settings.accessManagement.instance": "Instance users", + "settings.accessManagement.table.column.fullname": "Full Name", + "settings.accessManagement.table.column.email": "Email", + "settings.accessManagement.table.column.role": "Role", + "settings.accessManagement.table.column.action": "Action", + "settings.accessManagement.button.addNewUser": "New user", + "settings.accessManagement.user.remove": "Remove", "connector.connectorCount": "{count, plural, one {# connector} other {# connectors}}", "connector.noSearchResults": "No connectors match your search.", diff --git a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx deleted file mode 100644 index 1bcfe77f130..00000000000 --- a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/components/RoleToolTip.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import React from "react"; -import { FormattedMessage } from "react-intl"; -import styled from "styled-components"; - -import { InfoTooltip } from "components/ui/Tooltip"; - -const LineBlock = styled.div` - text-transform: none; - font-weight: 500; - font-size: 11px; - line-height: 13px; - letter-spacing: 0.3px; - min-width: 230px; - color: ${({ theme }) => theme.whiteColor}; - margin-bottom: 5px; - - &:last-child { - margin-bottom: 0; - } -`; - -const RoleToolTip: React.FC = () => { - return ( - - - - - - - - - - - - - ); -}; - -export default RoleToolTip; diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx index e545c765aab..98b9bae8666 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx @@ -16,6 +16,8 @@ import { useGetConnectorsOutOfDate } from "hooks/services/useConnector"; import { GeneralOrganizationSettingsPage } from "./GeneralOrganizationSettingsPage"; import { GeneralWorkspaceSettingsPage } from "./GeneralWorkspaceSettingsPage"; +import { OrganizationAccessManagementPage } from "./pages/AccessManagementPage/OrganizationAccessManagementPage"; +import { WorkspaceAccessManagementPage } from "./pages/AccessManagementPage/WorkspaceAccessManagementPage"; import { AccountPage } from "./pages/AccountPage"; import { ConfigurationsPage } from "./pages/ConfigurationsPage"; import { DestinationsPage, SourcesPage } from "./pages/ConnectorsPage"; @@ -40,6 +42,7 @@ export const SettingsRoute = { DataResidency: "data-residency", Workspace: "workspace", Organization: "organization", + AccessManagement: "access-management", } as const; const SettingsPage: React.FC = ({ pageConfig }) => { @@ -48,6 +51,7 @@ const SettingsPage: React.FC = ({ pageConfig }) => { const { pathname } = useLocation(); const { countNewSourceVersion, countNewDestinationVersion } = useGetConnectorsOutOfDate(); const newWorkspacesUI = useExperiment("workspaces.newWorkspacesUI", false); + const isAccessManagementEnabled = useExperiment("settings.accessManagement", false); const menuItems: CategoryItem[] = pageConfig?.menuConfig || [ { @@ -99,6 +103,15 @@ const SettingsPage: React.FC = ({ pageConfig }) => { name: , component: MetricsPage, }, + ...(isAccessManagementEnabled && !pageConfig + ? [ + { + path: `${SettingsRoute.Workspace}/${SettingsRoute.AccessManagement}`, + name: , + component: WorkspaceAccessManagementPage, + }, + ] + : []), ], }, ...(newWorkspacesUI && organizationId @@ -111,6 +124,15 @@ const SettingsPage: React.FC = ({ pageConfig }) => { name: , component: GeneralOrganizationSettingsPage, }, + ...(isAccessManagementEnabled && !pageConfig + ? [ + { + path: `${SettingsRoute.Organization}/${SettingsRoute.AccessManagement}`, + name: , + component: OrganizationAccessManagementPage, + }, + ] + : []), ], }, ] diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx new file mode 100644 index 00000000000..915f8371a11 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx @@ -0,0 +1,17 @@ +import React from "react"; + +import { useCurrentWorkspace, useOrganization } from "core/api"; +import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; + +import { AccessManagementPageContent } from "./components/AccessManagementPageContent"; +import { useGetOrganizationAccessUsers } from "./components/useGetAccessManagementData"; + +export const OrganizationAccessManagementPage: React.FC = () => { + useTrackPage(PageTrackingCodes.SETTINGS_ORGANIZATION_ACCESS_MANAGEMENT); + const workspace = useCurrentWorkspace(); + const organizationId = workspace.organizationId ?? ""; + const { organizationName } = useOrganization(organizationId); + const organizationAccessUsers = useGetOrganizationAccessUsers(); + + return ; +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx new file mode 100644 index 00000000000..b3f78d563d1 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx @@ -0,0 +1,15 @@ +import React from "react"; + +import { useCurrentWorkspace } from "core/api"; +import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; + +import { AccessManagementPageContent } from "./components/AccessManagementPageContent"; +import { useGetWorkspaceAccessUsers } from "./components/useGetAccessManagementData"; + +export const WorkspaceAccessManagementPage: React.FC = () => { + useTrackPage(PageTrackingCodes.SETTINGS_WORKSPACE_ACCESS_MANAGEMENT); + const { name } = useCurrentWorkspace(); + const workspaceAccessUsers = useGetWorkspaceAccessUsers(); + + return ; +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx new file mode 100644 index 00000000000..73e600e0f0c --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx @@ -0,0 +1,20 @@ +import { FormattedMessage } from "react-intl"; + +import { Card } from "components/ui/Card"; + +import { OrganizationUserRead, WorkspaceUserRead } from "core/request/AirbyteClient"; + +import { AccessManagementTable } from "./AccessManagementTable"; +import { ResourceType, tableTitleDictionary } from "./useGetAccessManagementData"; + +interface AccessManagementCardProps { + users: OrganizationUserRead[] | WorkspaceUserRead[]; + resourceType: ResourceType; +} +export const AccessManagementCard: React.FC = ({ users, resourceType }) => { + return ( + }> + + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx new file mode 100644 index 00000000000..69fba0350d9 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx @@ -0,0 +1,38 @@ +import { FormattedMessage } from "react-intl"; + +import { HeadTitle } from "components/common/HeadTitle"; +import { PageContainer } from "components/PageContainer"; +import { Box } from "components/ui/Box"; +import { FlexContainer } from "components/ui/Flex"; +import { Text } from "components/ui/Text"; + +import { AccessManagementCard } from "./AccessManagementCard"; +import { AccessUsers } from "./useGetAccessManagementData"; + +interface AccessManagementContentProps { + resourceName: string; + accessUsers: AccessUsers; +} +export const AccessManagementPageContent: React.FC = ({ resourceName, accessUsers }) => { + return ( + + + + + + + + + {Object.keys(accessUsers).map((key) => { + const resourceType = key as keyof typeof accessUsers; + const users = accessUsers[resourceType]; + + return ( + users && + users.length > 0 && + ); + })} + + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx new file mode 100644 index 00000000000..8e4a678f0df --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx @@ -0,0 +1,45 @@ +import { createColumnHelper } from "@tanstack/react-table"; +import { useMemo } from "react"; +import { FormattedMessage } from "react-intl"; + +import { Table } from "components/ui/Table"; + +import { OrganizationUserRead, WorkspaceUserRead } from "core/request/AirbyteClient"; + +import { RoleToolTip } from "./RoleToolTip"; +import { ResourceType, permissionStringDictionary } from "./useGetAccessManagementData"; + +export const AccessManagementTable: React.FC<{ + users: WorkspaceUserRead[] | OrganizationUserRead[]; + tableResourceType: ResourceType; +}> = ({ users, tableResourceType }) => { + const columnHelper = createColumnHelper(); + + const columns = useMemo( + () => [ + columnHelper.accessor("name", { + header: () => , + cell: (props) => props.cell.getValue(), + sortingFn: "alphanumeric", + }), + columnHelper.accessor("email", { + header: () => , + cell: (props) => props.cell.getValue(), + sortingFn: "alphanumeric", + }), + columnHelper.accessor("permissionType", { + header: () => ( + <> + + + + ), + cell: (props) => , + sortingFn: "alphanumeric", + }), + ], + [columnHelper, tableResourceType] + ); + + return ; +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleToolTip.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleToolTip.tsx new file mode 100644 index 00000000000..faec73228d1 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleToolTip.tsx @@ -0,0 +1,35 @@ +import React from "react"; +import { FormattedMessage } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { Text } from "components/ui/Text"; +import { InfoTooltip } from "components/ui/Tooltip"; + +import { + ResourceType, + permissionDescriptionDictionary, + permissionStringDictionary, + permissionsByResourceType, +} from "./useGetAccessManagementData"; + +export const RoleToolTip: React.FC<{ resourceType: ResourceType }> = ({ resourceType }) => { + return ( + + {permissionsByResourceType[resourceType].map((permission) => { + return ( + + + + + + + + + ); + })} + + ); +}; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx new file mode 100644 index 00000000000..e57ed1172de --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx @@ -0,0 +1,77 @@ +import { useCurrentWorkspace, useListUsersInOrganization, useListUsersInWorkspace } from "core/api"; +import { OrganizationUserRead, PermissionType, WorkspaceUserRead } from "core/request/AirbyteClient"; + +export type ResourceType = "workspace" | "organization" | "instance"; + +export const permissionStringDictionary: Record = { + instance_admin: "role.admin", + organization_admin: "role.admin", + organization_editor: "role.editor", + organization_reader: "role.reader", + workspace_admin: "role.admin", + workspace_owner: "role.admin", + workspace_editor: "role.editor", + workspace_reader: "role.reader", +}; + +interface PermissionDescription { + id: string; + values: Record<"resourceType", ResourceType>; +} + +export const permissionDescriptionDictionary: Record = { + instance_admin: { id: "role.admin.description", values: { resourceType: "instance" } }, + organization_admin: { id: "role.admin.description", values: { resourceType: "organization" } }, + organization_editor: { id: "role.editor.description", values: { resourceType: "organization" } }, + organization_reader: { id: "role.reader.description", values: { resourceType: "organization" } }, + workspace_admin: { id: "role.admin.description", values: { resourceType: "workspace" } }, + workspace_owner: { id: "role.admin.description", values: { resourceType: "workspace" } }, // is not and should not be referenced in code. required by types but will be deprecated soon. + workspace_editor: { id: "role.editor.description", values: { resourceType: "workspace" } }, + workspace_reader: { id: "role.reader.description", values: { resourceType: "workspace" } }, +}; + +export const tableTitleDictionary: Record = { + workspace: "settings.accessManagement.workspace", + organization: "settings.accessManagement.organization", + instance: "settings.accessManagement.instance", +}; + +export const permissionsByResourceType: Record = { + workspace: [PermissionType.workspace_admin, PermissionType.workspace_editor, PermissionType.workspace_reader], + organization: [ + PermissionType.organization_admin, + PermissionType.organization_editor, + PermissionType.organization_reader, + ], + instance: [PermissionType.instance_admin], +}; + +export interface AccessUsers { + workspace?: WorkspaceUserRead[]; + organization?: OrganizationUserRead[]; + instance?: WorkspaceUserRead[]; +} + +export const useGetWorkspaceAccessUsers = (): AccessUsers => { + const workspace = useCurrentWorkspace(); + const workspaceUsers = useListUsersInWorkspace(workspace.workspaceId).users; + const organizationUsers = useListUsersInOrganization(workspace.organizationId ?? "").users; + const instanceUsers: WorkspaceUserRead[] = []; // todo: temporary hacky workaround until we have an endpoint + + return { + workspace: workspaceUsers, + organization: organizationUsers, + instance: instanceUsers, + }; +}; + +export const useGetOrganizationAccessUsers = (): AccessUsers => { + const workspace = useCurrentWorkspace(); + const organizationUsers = useListUsersInOrganization(workspace.organizationId ?? "").users; + const instanceUsers: WorkspaceUserRead[] = []; // todo: temporary hacky workaround until we have an endpoint + + return { + organization: organizationUsers, + instance: instanceUsers, + }; +}; diff --git a/airbyte-webapp/src/services/Scope.ts b/airbyte-webapp/src/services/Scope.ts index a63568636c8..df0e412fdfe 100644 --- a/airbyte-webapp/src/services/Scope.ts +++ b/airbyte-webapp/src/services/Scope.ts @@ -1,2 +1,3 @@ export const SCOPE_WORKSPACE = "scope:workspace"; export const SCOPE_USER = "scope:user"; +export const SCOPE_ORGANIZATION = "scope:organization"; From 235b3218e78d0ba2f2b2fa8458804b02c43d7177 Mon Sep 17 00:00:00 2001 From: Jose Pefaur Date: Tue, 29 Aug 2023 12:22:32 -0500 Subject: [PATCH 047/201] delete feature flag logic for different state messages processing (#8551) --- .../general/BufferedReplicationWorker.java | 5 +- .../general/DefaultReplicationWorker.java | 5 +- .../general/ReplicationWorkerFactory.java | 15 ++--- .../general/ReplicationWorkerHelper.java | 62 +------------------ .../workers/internal/NamespacingMapper.java | 51 +-------------- .../BufferedReplicationWorkerTest.java | 3 +- .../general/DefaultReplicationWorkerTest.java | 3 +- .../general/ReplicationWorkerHelperTest.java | 5 +- .../general/ReplicationWorkerTest.java | 2 + ...feredReplicationWorkerPerformanceTest.java | 6 +- ...faultReplicationWorkerPerformanceTest.java | 6 +- .../ReplicationWorkerPerformanceTest.java | 8 +-- .../internal/NamespacingMapperTest.java | 23 +++---- .../src/main/kotlin/FlagDefinitions.kt | 2 - 14 files changed, 37 insertions(+), 159 deletions(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java index e61698816bb..c9549bf2971 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java @@ -106,14 +106,13 @@ public BufferedReplicationWorker(final String jobId, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, - final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final VoidCallable onReplicationRunning) { this.jobId = jobId; this.attempt = attempt; this.source = source; this.destination = destination; this.replicationWorkerHelper = new ReplicationWorkerHelper(airbyteMessageDataExtractor, fieldSelector, mapper, messageTracker, syncPersistence, - replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning, useNewStateMessageProcessing); + replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning); this.replicationFeatureFlagReader = replicationFeatureFlagReader; this.recordSchemaValidator = recordSchemaValidator; this.syncPersistence = syncPersistence; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index 905392da9b1..6091ffc8932 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -99,12 +99,11 @@ public DefaultReplicationWorker(final String jobId, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, - final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final VoidCallable onReplicationRunning) { this.jobId = jobId; this.attempt = attempt; this.replicationWorkerHelper = new ReplicationWorkerHelper(airbyteMessageDataExtractor, fieldSelector, mapper, messageTracker, syncPersistence, - replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning, useNewStateMessageProcessing); + replicationAirbyteMessageEventPublishingHelper, new ThreadedTimeTracker(), onReplicationRunning); this.source = source; this.destination = destination; this.syncPersistence = syncPersistence; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index a6df0c58a8f..267a5da3b6a 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -26,7 +26,6 @@ import io.airbyte.featureflag.Source; import io.airbyte.featureflag.SourceDefinition; import io.airbyte.featureflag.SourceType; -import io.airbyte.featureflag.UseNewStateMessageProcessing; import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; @@ -267,7 +266,6 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou final VoidCallable onReplicationRunning) { final Context flagContext = getFeatureFlagContext(syncInput); final String workerImpl = featureFlagClient.stringVariation(ReplicationWorkerImpl.INSTANCE, flagContext); - final boolean useNewStateMessageProcessing = featureFlagClient.boolVariation(UseNewStateMessageProcessing.INSTANCE, flagContext); return buildReplicationWorkerInstance( workerImpl, jobRunConfig.getJobId(), @@ -276,8 +274,7 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou new NamespacingMapper( syncInput.getNamespaceDefinition(), syncInput.getNamespaceFormat(), - syncInput.getPrefix(), - useNewStateMessageProcessing), + syncInput.getPrefix()), destination, messageTracker, syncPersistence, @@ -287,8 +284,7 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationEventPublishingHelper, - onReplicationRunning, - useNewStateMessageProcessing); + onReplicationRunning); } private static Context getFeatureFlagContext(final StandardSyncInput syncInput) { @@ -327,20 +323,19 @@ private static ReplicationWorker buildReplicationWorkerInstance(final String wor final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final VoidCallable onReplicationRunning) { if ("buffered".equals(workerImpl)) { MetricClientFactory.getMetricClient() .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, onReplicationRunning, useNewStateMessageProcessing); + messageEventPublishingHelper, onReplicationRunning); } else { MetricClientFactory.getMetricClient() .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, onReplicationRunning, useNewStateMessageProcessing); + messageEventPublishingHelper, onReplicationRunning); } } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java index a91c6bcca78..b520ba18be8 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java @@ -21,7 +21,6 @@ import io.airbyte.config.SyncStats; import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.config.WorkerSourceConfig; -import io.airbyte.featureflag.UseNewStateMessageProcessing; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteTraceMessage; @@ -75,7 +74,6 @@ class ReplicationWorkerHelper { private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; private final ThreadedTimeTracker timeTracker; private final VoidCallable onReplicationRunning; - private final boolean useNewStateMessageProcessing; private long recordsRead; private StreamDescriptor currentDestinationStream = null; private ReplicationContext replicationContext = null; @@ -96,8 +94,7 @@ public ReplicationWorkerHelper( final SyncPersistence syncPersistence, final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, final ThreadedTimeTracker timeTracker, - final VoidCallable onReplicationRunning, - final boolean useNewStateMessageProcessing) { + final VoidCallable onReplicationRunning) { this.airbyteMessageDataExtractor = airbyteMessageDataExtractor; this.fieldSelector = fieldSelector; this.mapper = mapper; @@ -106,7 +103,6 @@ public ReplicationWorkerHelper( this.replicationAirbyteMessageEventPublishingHelper = replicationAirbyteMessageEventPublishingHelper; this.timeTracker = timeTracker; this.onReplicationRunning = onReplicationRunning; - this.useNewStateMessageProcessing = useNewStateMessageProcessing; this.recordsRead = 0L; } @@ -226,11 +222,7 @@ AirbyteMessage internalProcessMessageFromSource(final AirbyteMessage sourceRawMe return sourceRawMessage; } - /** - * If you are making changes in this method please read the documentation in - * {@link #processMessageFromSource} first. - */ - Optional processMessageFromSourceNew(final AirbyteMessage sourceRawMessage) { + public Optional processMessageFromSource(final AirbyteMessage sourceRawMessage) { final AirbyteMessage processedMessage = internalProcessMessageFromSource(sourceRawMessage); // internally we always want to deal with the state message we got from the // source, so we only modify the state message after processing it, right before we send it to the @@ -239,49 +231,6 @@ Optional processMessageFromSourceNew(final AirbyteMessage source } - /** - * If you are making changes in this method please read the documentation in - * {@link #processMessageFromSource} first. - */ - private Optional processMessageFromSourceOld(final AirbyteMessage airbyteMessage) { - fieldSelector.filterSelectedFields(airbyteMessage); - fieldSelector.validateSchema(airbyteMessage); - - final AirbyteMessage message = mapper.mapMessage(airbyteMessage); - - messageTracker.acceptFromSource(message); - - if (shouldPublishMessage(airbyteMessage)) { - replicationAirbyteMessageEventPublishingHelper - .publishStatusEvent(new ReplicationAirbyteMessageEvent(AirbyteMessageOrigin.SOURCE, message, replicationContext)); - } - - recordsRead += 1; - - if (recordsRead % 5000 == 0) { - LOGGER.info("Records read: {} ({})", recordsRead, - FileUtils.byteCountToDisplaySize(messageTracker.getSyncStatsTracker().getTotalBytesEmitted())); - } - - return Optional.of(message); - } - - /** - * The behavior of this method depends on the value of {@link #useNewStateMessageProcessing}, which - * is decided by the {@link UseNewStateMessageProcessing} feature flag. The feature flag is being - * used to test a new version of this method which is meant to fix the issue described here - * https://github.com/airbytehq/airbyte/issues/29478. {@link #processMessageFromSourceNew} is the - * new version of the method that is meant to fix the issue. {@link #processMessageFromSourceOld} is - * the original version of the method where the issue is present. - */ - public Optional processMessageFromSource(final AirbyteMessage airbyteMessage) { - if (useNewStateMessageProcessing) { - return processMessageFromSourceNew(airbyteMessage); - } else { - return processMessageFromSourceOld(airbyteMessage); - } - } - @VisibleForTesting void internalProcessMessageFromDestination(final AirbyteMessage destinationRawMessage) { LOGGER.info("State in ReplicationWorkerHelper from destination: {}", destinationRawMessage); @@ -310,13 +259,8 @@ void internalProcessMessageFromDestination(final AirbyteMessage destinationRawMe } } - /** - * The value of {@link #useNewStateMessageProcessing} is decided by the - * {@link UseNewStateMessageProcessing} feature flag. The feature flag is being used to test a fix - * the issue described here https://github.com/airbytehq/airbyte/issues/29478. - */ public void processMessageFromDestination(final AirbyteMessage destinationRawMessage) { - final AirbyteMessage message = useNewStateMessageProcessing ? mapper.revertMap(destinationRawMessage) : destinationRawMessage; + final AirbyteMessage message = mapper.revertMap(destinationRawMessage); internalProcessMessageFromDestination(message); } diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java index cb4eefcddb7..a6ae8fbe6c1 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/NamespacingMapper.java @@ -7,7 +7,6 @@ import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.json.Jsons; import io.airbyte.config.JobSyncConfig.NamespaceDefinitionType; -import io.airbyte.featureflag.UseNewStateMessageProcessing; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; @@ -36,7 +35,6 @@ public class NamespacingMapper implements AirbyteMapper { private final String namespaceFormat; private final String streamPrefix; private final Map destinationToSourceNamespaceAndStreamName; - private final boolean useNewNamespaceMapping; @VisibleForTesting record NamespaceAndStreamName(String namespace, String streamName) {} @@ -44,9 +42,8 @@ record NamespaceAndStreamName(String namespace, String streamName) {} public NamespacingMapper( final NamespaceDefinitionType namespaceDefinition, final String namespaceFormat, - final String streamPrefix, - final boolean useNewNamespaceMapping) { - this(namespaceDefinition, namespaceFormat, streamPrefix, new HashMap<>(), useNewNamespaceMapping); + final String streamPrefix) { + this(namespaceDefinition, namespaceFormat, streamPrefix, new HashMap<>()); } @VisibleForTesting @@ -54,13 +51,11 @@ public NamespacingMapper( final NamespaceDefinitionType namespaceDefinition, final String namespaceFormat, final String streamPrefix, - final Map destinationToSourceNamespaceAndStreamName, - final boolean useNewNamespaceMapping) { + final Map destinationToSourceNamespaceAndStreamName) { this.namespaceDefinition = namespaceDefinition; this.namespaceFormat = namespaceFormat; this.streamPrefix = streamPrefix; this.destinationToSourceNamespaceAndStreamName = destinationToSourceNamespaceAndStreamName; - this.useNewNamespaceMapping = useNewNamespaceMapping; } @Override @@ -85,48 +80,8 @@ public ConfiguredAirbyteCatalog mapCatalog(final ConfiguredAirbyteCatalog inputC return catalog; } - /** - * The behavior of this method depends on the value of {@link #useNewNamespaceMapping}, which is - * decided by the {@link UseNewStateMessageProcessing} feature flag. The feature flag is being used - * to test a new version of this method which is meant to fix the issue described here - * https://github.com/airbytehq/airbyte/issues/29478. {@link #mapMessageNew} is the new version of - * the method that is meant to fix the issue. {@link #mapMessageOld} is the original version of the - * method where the issue is present. - */ @Override public AirbyteMessage mapMessage(final AirbyteMessage message) { - if (useNewNamespaceMapping) { - return mapMessageNew(message); - } else { - return mapMessageOld(message); - } - } - - /** - * If you are making changes in this method please read the documentation in {@link #mapMessage} - * first. - */ - AirbyteMessage mapMessageOld(final AirbyteMessage message) { - if (message.getType() == Type.RECORD) { - // Default behavior if namespaceDefinition is not set is to follow SOURCE - if (namespaceDefinition != null) { - if (namespaceDefinition.equals(NamespaceDefinitionType.DESTINATION)) { - message.getRecord().withNamespace(null); - } else if (namespaceDefinition.equals(NamespaceDefinitionType.CUSTOMFORMAT)) { - message.getRecord().withNamespace(formatNamespace(message.getRecord().getNamespace(), namespaceFormat)); - } - } - message.getRecord().setStream(transformStreamName(message.getRecord().getStream(), streamPrefix)); - return message; - } - return message; - } - - /** - * If you are making changes in this method please read the documentation in {@link #mapMessage} - * first. - */ - AirbyteMessage mapMessageNew(final AirbyteMessage message) { if (message.getType() == Type.RECORD) { final AirbyteRecordMessage recordMessage = message.getRecord(); diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java index c2b89c7f9c1..6275401f663 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/BufferedReplicationWorkerTest.java @@ -37,8 +37,7 @@ ReplicationWorker getDefaultReplicationWorker(final boolean fieldSelectionEnable new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning, - false); + onReplicationRunning); } // BufferedReplicationWorkerTests. diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java index a075f282f9b..3423b90f2e4 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/DefaultReplicationWorkerTest.java @@ -31,8 +31,7 @@ ReplicationWorker getDefaultReplicationWorker(final boolean fieldSelectionEnable new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning, - false); + onReplicationRunning); } // DefaultReplicationWorkerTests. diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java index 168da046cb5..f47ce1a8a0b 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerHelperTest.java @@ -34,8 +34,7 @@ void setUp() { null, null, null, - null, - true)); + null)); } @Test @@ -46,7 +45,7 @@ void testMessageIsMappedAfterProcessing() { doReturn(sourceRawMessage).when(replicationWorkerHelper).internalProcessMessageFromSource(sourceRawMessage); when(mapper.mapMessage(sourceRawMessage)).thenReturn(mappedSourceMessage); - final Optional processedMessageFromSource = replicationWorkerHelper.processMessageFromSourceNew(sourceRawMessage); + final Optional processedMessageFromSource = replicationWorkerHelper.processMessageFromSource(sourceRawMessage); assertEquals(Optional.of(mappedSourceMessage), processedMessageFromSource); } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java index 94e18f571c2..663a78ad252 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/ReplicationWorkerTest.java @@ -208,6 +208,8 @@ void setup() throws Exception { when(mapper.mapMessage(RECORD_MESSAGE2)).thenReturn(RECORD_MESSAGE2); when(mapper.mapMessage(RECORD_MESSAGE3)).thenReturn(RECORD_MESSAGE3); when(mapper.mapMessage(CONFIG_MESSAGE)).thenReturn(CONFIG_MESSAGE); + when(mapper.revertMap(STATE_MESSAGE)).thenReturn(STATE_MESSAGE); + when(mapper.revertMap(CONFIG_MESSAGE)).thenReturn(CONFIG_MESSAGE); when(heartbeatMonitor.isBeating()).thenReturn(Optional.of(true)); } diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java index 921cfa46854..a4b96f1db53 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/BufferedReplicationWorkerPerformanceTest.java @@ -4,7 +4,6 @@ package io.airbyte.workers.general.performance; -import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.workers.RecordSchemaValidator; import io.airbyte.workers.general.BufferedReplicationWorker; import io.airbyte.workers.general.ReplicationFeatureFlagReader; @@ -38,11 +37,10 @@ public ReplicationWorker getReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final FeatureFlagClient featureFlagClient) { + final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper) { return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, () -> {}, false); + messageEventPublishingHelper, () -> {}); } public static void main(final String[] args) throws IOException, InterruptedException { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java index be9e9b52398..4ba578546b5 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/DefaultReplicationWorkerPerformanceTest.java @@ -4,7 +4,6 @@ package io.airbyte.workers.general.performance; -import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.workers.RecordSchemaValidator; import io.airbyte.workers.general.DefaultReplicationWorker; import io.airbyte.workers.general.ReplicationFeatureFlagReader; @@ -38,11 +37,10 @@ public ReplicationWorker getReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final FeatureFlagClient featureFlagClient) { + final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper) { return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, - messageEventPublishingHelper, /* we don't care about the onReplicationRunning callback here */ () -> {}, false); + messageEventPublishingHelper, /* we don't care about the onReplicationRunning callback here */ () -> {}); } public static void main(final String[] args) throws IOException, InterruptedException { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java index 1ed8c7eb0f9..aa68cacacae 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/general/performance/ReplicationWorkerPerformanceTest.java @@ -81,8 +81,7 @@ public abstract ReplicationWorker getReplicationWorker(final String jobId, final HeartbeatTimeoutChaperone srcHeartbeatTimeoutChaperone, final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, - final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final FeatureFlagClient featureFlagClient); + final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper); /** * Hook up the DefaultReplicationWorker to a test harness with an insanely quick Source @@ -117,7 +116,7 @@ public void executeOneSync() throws InterruptedException { final var syncPersistence = mock(SyncPersistence.class); final var connectorConfigUpdater = mock(ConnectorConfigUpdater.class); final var metricReporter = new WorkerMetricReporter(new NotImplementedMetricClient(), "test-image:0.01"); - final var dstNamespaceMapper = new NamespacingMapper(NamespaceDefinitionType.DESTINATION, "", "", false); + final var dstNamespaceMapper = new NamespacingMapper(NamespaceDefinitionType.DESTINATION, "", ""); final var validator = new RecordSchemaValidator(Map.of( new AirbyteStreamNameNamespacePair("s1", null), CatalogHelpers.fieldsToJsonSchema(io.airbyte.protocol.models.Field.of("data", JsonSchemaType.STRING)))); @@ -172,8 +171,7 @@ public void executeOneSync() throws InterruptedException { heartbeatTimeoutChaperone, new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, - replicationAirbyteMessageEventPublishingHelper, - featureFlagClient); + replicationAirbyteMessageEventPublishingHelper); final AtomicReference output = new AtomicReference<>(); final Thread workerThread = new Thread(() -> { try { diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java index 6c3661edf4c..71eb02d3e94 100644 --- a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/NamespacingMapperTest.java @@ -64,7 +64,7 @@ void setUp() { @Test void testSourceNamespace() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); final ConfiguredAirbyteCatalog expectedCatalog = CatalogHelpers.createConfiguredAirbyteCatalog( @@ -90,7 +90,7 @@ void testSourceNamespace() { @Test void testEmptySourceNamespace() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); assertEquals(originalCatalog, CATALOG); @@ -117,7 +117,7 @@ void testEmptySourceNamespace() { @Test void testDestinationNamespace() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.DESTINATION, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.DESTINATION, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); final ConfiguredAirbyteCatalog expectedCatalog = CatalogHelpers.createConfiguredAirbyteCatalog( @@ -143,8 +143,7 @@ void testCustomFormatWithVariableNamespace() { NamespaceDefinitionType.CUSTOMFORMAT, "${SOURCE_NAMESPACE}_suffix", OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - false); + destinationToSourceNamespaceAndStreamName); final String expectedNamespace = INPUT_NAMESPACE + "_suffix"; final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); @@ -172,8 +171,7 @@ void testCustomFormatWithoutVariableNamespace() { NamespaceDefinitionType.CUSTOMFORMAT, NAMESPACE_FORMAT, OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - false); + destinationToSourceNamespaceAndStreamName); final String expectedNamespace = NAMESPACE_FORMAT; final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); @@ -201,8 +199,7 @@ void testEmptyCustomFormatWithVariableNamespace() { NamespaceDefinitionType.CUSTOMFORMAT, "${SOURCE_NAMESPACE}", OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - false); + destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); assertEquals(originalCatalog, CATALOG); @@ -232,8 +229,7 @@ void testMapStateMessage() { NamespaceDefinitionType.CUSTOMFORMAT, NAMESPACE_FORMAT, OUTPUT_PREFIX, - destinationToSourceNamespaceAndStreamName, - true); + destinationToSourceNamespaceAndStreamName); final AirbyteMessage originalMessage = Jsons.clone(stateMessage); final AirbyteMessage expectedMessage = Jsons.clone(stateMessage); @@ -253,7 +249,7 @@ void testMapStateMessage() { @Test void testEmptyPrefix() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, null, destinationToSourceNamespaceAndStreamName, false); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, null, destinationToSourceNamespaceAndStreamName); final ConfiguredAirbyteCatalog originalCatalog = Jsons.clone(CATALOG); final ConfiguredAirbyteCatalog expectedCatalog = CatalogHelpers.createConfiguredAirbyteCatalog( @@ -280,7 +276,7 @@ void testEmptyPrefix() { @Test void testRevertMapStateMessage() { final NamespacingMapper mapper = - new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName, true); + new NamespacingMapper(NamespaceDefinitionType.SOURCE, null, OUTPUT_PREFIX, destinationToSourceNamespaceAndStreamName); final AirbyteMessage originalMessage = Jsons.clone(stateMessage); originalMessage.getState().getStream().getStreamDescriptor().withNamespace(DESTINATION_NAMESPACE); @@ -296,7 +292,6 @@ void testRevertMapStateMessage() { expectedMessage.getState().getStream().getStreamDescriptor().withName(STREAM_NAME); assertEquals(expectedMessage, actualMessage); - } } diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index 552aad3e99d..d3a1960b064 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -101,8 +101,6 @@ object UseCustomK8sScheduler : Temporary(key = "platform.use-custom-k8s- object HideActorDefinitionFromList : Permanent(key = "connectors.hideActorDefinitionFromList", default = false) -object UseNewStateMessageProcessing : Temporary(key = "platform.use-new-state-message-processing", default = false) - // NOTE: this is deprecated in favor of FieldSelectionEnabled and will be removed once that flag is fully deployed. object FieldSelectionWorkspaces : EnvVar(envVar = "FIELD_SELECTION_WORKSPACES") { override fun enabled(ctx: Context): Boolean { From f64a522605e901bad8961685fb6dfc6da74773b5 Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Tue, 29 Aug 2023 10:58:07 -0700 Subject: [PATCH 048/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Default?= =?UTF-8?q?=20to=20YAML=20view=20if=20resolve=20call=20fails=20(#8568)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/connectorBuilder/ConnectorBuilderStateService.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index a9f86430b3a..caa0bad23fd 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -441,7 +441,7 @@ export function useInitializedBuilderProject() { // could not resolve manifest, use default form values return [ DEFAULT_BUILDER_FORM_VALUES, - false, + true, convertJsonToYaml(builderProject.declarativeManifest?.manifest ?? DEFAULT_JSON_MANIFEST_VALUES), ]; } From c4607eb2ef524150608dbcee94309fc14422ff36 Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Tue, 29 Aug 2023 10:58:18 -0700 Subject: [PATCH 049/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Fix=20(la?= =?UTF-8?q?test)=20overlap=20in=20connector=20version=20upgrade=20input=20?= =?UTF-8?q?(#8555)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../VersionCell/VersionCell.module.scss | 20 +++-- .../components/VersionCell/VersionCell.tsx | 84 ++++++++++++------- 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.module.scss index 518fe2b1d89..dba5b7e2c9f 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.module.scss +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.module.scss @@ -5,16 +5,26 @@ } .inputField { - display: inline-block; position: relative; background: colors.$foreground; .inputContainer { - padding-bottom: 0; // overwrite default FormControl style + // overwrite default FormControl style + padding-bottom: 0; .versionInput { - max-width: 145px; - margin-right: 19px; + max-width: 165px; + overflow: hidden; + text-overflow: ellipsis; + + &:not(:focus-within) { + // space for (latest) text + padding-right: 50px; + } + + &.noLatest:not(:focus-within) { + padding-right: 0; + } } } @@ -25,7 +35,7 @@ content: attr(data-before); color: colors.$grey-400; top: 10px; - right: 22px; + right: 8px; } &:focus-within::after { diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx index d76ff01eaf3..b2a447b7030 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx @@ -1,4 +1,6 @@ +import classNames from "classnames"; import React from "react"; +import { useWatch } from "react-hook-form"; import { useIntl } from "react-intl"; import * as yup from "yup"; @@ -37,17 +39,9 @@ export const VersionCell: React.FC = ({ latestVersion, releaseStage, }) => { - const { formatMessage } = useIntl(); const { feedbackList } = useUpdatingState(); const feedback = feedbackList[connectorDefinitionId]; - const inputLatestNote = - feedback !== "success" && releaseStage !== ReleaseStage.custom - ? formatMessage({ - id: "admin.latestNote", - }) - : undefined; - return ( defaultValues={{ @@ -57,26 +51,60 @@ export const VersionCell: React.FC = ({ schema={versionCellFormSchema} onSubmit={onChange} > - - - - -
- -
- - + ); }; + +const VersionFormContent = ({ + feedback, + releaseStage, + connectorDefinitionId, + currentVersion, + latestVersion, +}: { + feedback: string; + releaseStage?: ReleaseStage; + connectorDefinitionId: string; + currentVersion: string; + latestVersion?: string; +}) => { + const { formatMessage } = useIntl(); + const value = useWatch({ name: "version" }); + + const inputLatestNote = + value === latestVersion && releaseStage !== ReleaseStage.custom + ? formatMessage({ + id: "admin.latestNote", + }) + : undefined; + + return ( + + + + +
+ +
+ + + ); +}; From e7d72703a82383c2af22da0703524f88777e5a3b Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Tue, 29 Aug 2023 12:42:46 -0700 Subject: [PATCH 050/201] Small heartbeat related cleanup (#8596) --- .../commons/temporal/TemporalUtils.java | 18 ------------ .../temporal/CancellationHandlerTest.java | 14 +++++---- .../commons/temporal/TemporalUtilsTest.java | 29 ++++++++++++------- ...artbeatWorkflow.java => TestWorkflow.java} | 18 +++++++----- 4 files changed, 36 insertions(+), 43 deletions(-) rename airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/{HeartbeatWorkflow.java => TestWorkflow.java} (70%) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java index 93a10e27ef8..39f8d1f2e5a 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalUtils.java @@ -275,24 +275,6 @@ protected NamespaceInfo getNamespaceInfo(final WorkflowServiceStubs temporalServ .getNamespaceInfo(); } - /** - * Run a callable. If while it is running the temporal activity is cancelled, the provided callback - * is triggered. - * - * It manages this by regularly calling back to temporal in order to check whether the activity has - * been cancelled. If it is cancelled it calls the callback. - * - * @param callable callable to run with cancellation - * @param activityContext context used to check whether the activity has been cancelled - * @param type of variable returned by the callable - * @return if the callable succeeds without being cancelled, returns the value returned by the - * callable - */ - public T withBackgroundHeartbeat(final Callable callable, - final Supplier activityContext) { - return withBackgroundHeartbeat(null, callable, activityContext); - } - /** * Run a callable. If while it is running the temporal activity is cancelled, the provided callback * is triggered. diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java index 27082940882..89fe7bdeed3 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/CancellationHandlerTest.java @@ -4,7 +4,9 @@ package io.airbyte.commons.temporal; -import io.airbyte.commons.temporal.stubs.HeartbeatWorkflow; +import io.airbyte.commons.temporal.stubs.TestWorkflow; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestActivityImplTest; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestWorkflowImpl; import io.temporal.activity.Activity; import io.temporal.activity.ActivityExecutionContext; import io.temporal.client.WorkflowClient; @@ -23,23 +25,23 @@ void testCancellationHandler() { final Worker worker = testEnv.newWorker("task-queue"); - worker.registerWorkflowImplementationTypes(HeartbeatWorkflow.HeartbeatWorkflowImpl.class); + worker.registerWorkflowImplementationTypes(TestWorkflowImpl.class); final WorkflowClient client = testEnv.getWorkflowClient(); - worker.registerActivitiesImplementations(new HeartbeatWorkflow.HeartbeatActivityImpl(() -> { + worker.registerActivitiesImplementations(new TestActivityImplTest(() -> { final ActivityExecutionContext context = Activity.getExecutionContext(); new CancellationHandler.TemporalCancellationHandler(context).checkAndHandleCancellation(() -> {}); })); testEnv.start(); - final HeartbeatWorkflow heartbeatWorkflow = client.newWorkflowStub( - HeartbeatWorkflow.class, + final TestWorkflow testWorkflow = client.newWorkflowStub( + TestWorkflow.class, WorkflowOptions.newBuilder() .setTaskQueue("task-queue") .build()); - Assertions.assertDoesNotThrow(heartbeatWorkflow::execute); + Assertions.assertDoesNotThrow(testWorkflow::execute); } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java index f246d27e756..410a319ae0c 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalUtilsTest.java @@ -10,10 +10,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.airbyte.commons.concurrency.VoidCallable; -import io.airbyte.commons.temporal.stubs.HeartbeatWorkflow; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestActivityImplTest; +import io.airbyte.commons.temporal.stubs.TestWorkflow.TestWorkflowImpl; import io.temporal.activity.Activity; import io.temporal.activity.ActivityCancellationType; import io.temporal.activity.ActivityExecutionContext; @@ -188,14 +191,16 @@ void testHeartbeatWithContext() throws InterruptedException { final Worker worker = testEnv.newWorker(TASK_QUEUE); - worker.registerWorkflowImplementationTypes(HeartbeatWorkflow.HeartbeatWorkflowImpl.class); + worker.registerWorkflowImplementationTypes(TestWorkflowImpl.class); final WorkflowClient client = testEnv.getWorkflowClient(); final CountDownLatch latch = new CountDownLatch(2); + final Runnable cancellationCallback = mock(Runnable.class); - worker.registerActivitiesImplementations(new HeartbeatWorkflow.HeartbeatActivityImpl(() -> { + worker.registerActivitiesImplementations(new TestActivityImplTest(() -> { final ActivityExecutionContext context = Activity.getExecutionContext(); temporalUtils.withBackgroundHeartbeat( + new AtomicReference<>(cancellationCallback), // TODO (itaseski) figure out how to decrease heartbeat intervals using reflection () -> { latch.await(); @@ -209,16 +214,18 @@ void testHeartbeatWithContext() throws InterruptedException { testEnv.start(); - final HeartbeatWorkflow heartbeatWorkflow = client.newWorkflowStub( - HeartbeatWorkflow.class, + final io.airbyte.commons.temporal.stubs.TestWorkflow testWorkflow = client.newWorkflowStub( + io.airbyte.commons.temporal.stubs.TestWorkflow.class, WorkflowOptions.newBuilder() .setTaskQueue(TASK_QUEUE) .build()); // use async execution to avoid blocking the test thread - WorkflowClient.start(heartbeatWorkflow::execute); + WorkflowClient.start(testWorkflow::execute); assertTrue(latch.await(25, TimeUnit.SECONDS)); + // The activity is expected to succeed, we should never call the cancellation callback. + verify(cancellationCallback, never()).run(); } @@ -229,12 +236,12 @@ void testHeartbeatWithContextAndCallbackRef() throws InterruptedException { final Worker worker = testEnv.newWorker(TASK_QUEUE); - worker.registerWorkflowImplementationTypes(HeartbeatWorkflow.HeartbeatWorkflowImpl.class); + worker.registerWorkflowImplementationTypes(TestWorkflowImpl.class); final WorkflowClient client = testEnv.getWorkflowClient(); final CountDownLatch latch = new CountDownLatch(2); - worker.registerActivitiesImplementations(new HeartbeatWorkflow.HeartbeatActivityImpl(() -> { + worker.registerActivitiesImplementations(new TestActivityImplTest(() -> { final ActivityExecutionContext context = Activity.getExecutionContext(); temporalUtils.withBackgroundHeartbeat( // TODO (itaseski) figure out how to decrease heartbeat intervals using reflection @@ -251,14 +258,14 @@ void testHeartbeatWithContextAndCallbackRef() throws InterruptedException { testEnv.start(); - final HeartbeatWorkflow heartbeatWorkflow = client.newWorkflowStub( - HeartbeatWorkflow.class, + final io.airbyte.commons.temporal.stubs.TestWorkflow testWorkflow = client.newWorkflowStub( + io.airbyte.commons.temporal.stubs.TestWorkflow.class, WorkflowOptions.newBuilder() .setTaskQueue(TASK_QUEUE) .build()); // use async execution to avoid blocking the test thread - WorkflowClient.start(heartbeatWorkflow::execute); + WorkflowClient.start(testWorkflow::execute); assertTrue(latch.await(25, TimeUnit.SECONDS)); diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/HeartbeatWorkflow.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/TestWorkflow.java similarity index 70% rename from airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/HeartbeatWorkflow.java rename to airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/TestWorkflow.java index 22dd8f3bb3b..a9a616de438 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/HeartbeatWorkflow.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/stubs/TestWorkflow.java @@ -14,15 +14,17 @@ import io.temporal.workflow.WorkflowMethod; import java.time.Duration; -// todo (cgardens) - don't know what this is meant to do. needs javadocs. +/** + * Workflow used for testing cancellations and heartbeats. + */ @SuppressWarnings("MissingJavadocType") @WorkflowInterface -public interface HeartbeatWorkflow { +public interface TestWorkflow { @WorkflowMethod void execute(); - class HeartbeatWorkflowImpl implements HeartbeatWorkflow { + class TestWorkflowImpl implements TestWorkflow { private final ActivityOptions options = ActivityOptions.newBuilder() .setScheduleToCloseTimeout(Duration.ofDays(1)) @@ -30,28 +32,28 @@ class HeartbeatWorkflowImpl implements HeartbeatWorkflow { .setRetryOptions(TemporalUtils.NO_RETRY) .build(); - private final HeartbeatActivity heartbeatActivity = Workflow.newActivityStub(HeartbeatActivity.class, options); + private final TestHeartbeatActivity testHeartbeatActivity = Workflow.newActivityStub(TestHeartbeatActivity.class, options); @Override public void execute() { - heartbeatActivity.heartbeat(); + testHeartbeatActivity.heartbeat(); } } @ActivityInterface - interface HeartbeatActivity { + interface TestHeartbeatActivity { @ActivityMethod void heartbeat(); } - class HeartbeatActivityImpl implements HeartbeatActivity { + class TestActivityImplTest implements TestHeartbeatActivity { private final Runnable runnable; - public HeartbeatActivityImpl(final Runnable runnable) { + public TestActivityImplTest(final Runnable runnable) { this.runnable = runnable; } From 40a9c8ead04c885849bf142addc0a57c3d28217e Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Tue, 29 Aug 2023 14:56:42 -0700 Subject: [PATCH 051/201] Revert "Revert "remove outdated check flags (#8424)" (#8461)" (#8575) --- airbyte-commons-worker/build.gradle | 2 +- .../src/main/kotlin/FlagDefinitions.kt | 4 - .../workers/config/ActivityBeanFactory.java | 3 - .../SubmitCheckConnectionActivity.java | 27 ------ .../SubmitCheckConnectionActivityImpl.java | 96 ------------------- .../ConnectionManagerWorkflowImpl.java | 76 +++------------ .../FeatureFlagFetchActivityImpl.java | 22 +---- .../ConnectionManagerWorkflowTest.java | 57 +---------- .../FeatureFlagFetchActivityTest.java | 19 +--- 9 files changed, 21 insertions(+), 285 deletions(-) delete mode 100644 airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivity.java delete mode 100644 airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivityImpl.java diff --git a/airbyte-commons-worker/build.gradle b/airbyte-commons-worker/build.gradle index a85a86d98d6..0b044fb71d4 100644 --- a/airbyte-commons-worker/build.gradle +++ b/airbyte-commons-worker/build.gradle @@ -74,7 +74,7 @@ dependencies { } test { - maxHeapSize = '2g' + maxHeapSize = '4g' useJUnitPlatform { excludeTags("cloud-storage") diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index d3a1960b064..e11eddc5506 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -43,10 +43,6 @@ object CanonicalCatalogSchema : Temporary(key = "canonical-catalog-sche object CatalogCanonicalJson : Temporary(key = "catalog-canonical-json", default = false) -object CheckConnectionUseApiEnabled : Temporary(key = "check-connection-use-api", default = false) - -object CheckConnectionUseChildWorkflowEnabled : Temporary(key = "check-connection-use-child-workflow", default = false) - object ShouldRunOnGkeDataplane : Temporary(key = "should-run-on-gke-dataplane", default = false) object ShouldRunOnExpandedGkeDataplane : Temporary(key = "should-run-on-expanded-gke-dataplane", default = false) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java index be88c70cb4b..53e51dbed39 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java @@ -8,7 +8,6 @@ import io.airbyte.commons.temporal.config.WorkerMode; import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.temporal.check.connection.CheckConnectionActivity; -import io.airbyte.workers.temporal.check.connection.SubmitCheckConnectionActivity; import io.airbyte.workers.temporal.discover.catalog.DiscoverCatalogActivity; import io.airbyte.workers.temporal.scheduling.activities.AppendToAttemptLogActivity; import io.airbyte.workers.temporal.scheduling.activities.AutoDisableConnectionActivity; @@ -70,7 +69,6 @@ public List connectionManagerActivities( final WorkflowConfigActivity workflowConfigActivity, final RouteToSyncTaskQueueActivity routeToTaskQueueActivity, final FeatureFlagFetchActivity featureFlagFetchActivity, - final SubmitCheckConnectionActivity submitCheckConnectionActivity, final CheckRunProgressActivity checkRunProgressActivity, final RetryStatePersistenceActivity retryStatePersistenceActivity, final AppendToAttemptLogActivity appendToAttemptLogActivity) { @@ -84,7 +82,6 @@ public List connectionManagerActivities( workflowConfigActivity, routeToTaskQueueActivity, featureFlagFetchActivity, - submitCheckConnectionActivity, checkRunProgressActivity, retryStatePersistenceActivity, appendToAttemptLogActivity); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivity.java deleted file mode 100644 index f691afd754b..00000000000 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivity.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.temporal.check.connection; - -import io.airbyte.config.ConnectorJobOutput; -import io.temporal.activity.ActivityInterface; -import java.util.UUID; - -/** - * Temporal activity to submit a check_connection request to airbyte server. - */ -@ActivityInterface -public interface SubmitCheckConnectionActivity { - - /** - * Submits an API request to airbyte server to run check connection for source. - */ - ConnectorJobOutput submitCheckConnectionToSource(final UUID sourceId); - - /** - * Submits an API request to airbyte server to run check connection for destination. - */ - ConnectorJobOutput submitCheckConnectionToDestination(final UUID destinationId); - -} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivityImpl.java deleted file mode 100644 index 42b2ed6b344..00000000000 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/SubmitCheckConnectionActivityImpl.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.workers.temporal.check.connection; - -import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; - -import datadog.trace.api.Trace; -import io.airbyte.api.client.AirbyteApiClient; -import io.airbyte.api.client.generated.DestinationApi; -import io.airbyte.api.client.generated.SourceApi; -import io.airbyte.api.client.model.generated.CheckConnectionRead; -import io.airbyte.api.client.model.generated.CheckConnectionRead.StatusEnum; -import io.airbyte.api.client.model.generated.DestinationIdRequestBody; -import io.airbyte.api.client.model.generated.SourceIdRequestBody; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.config.ConnectorJobOutput; -import io.airbyte.config.ConnectorJobOutput.OutputType; -import io.airbyte.config.FailureReason; -import io.airbyte.config.FailureReason.FailureType; -import io.airbyte.config.StandardCheckConnectionOutput; -import io.airbyte.config.StandardCheckConnectionOutput.Status; -import io.airbyte.metrics.lib.MetricClientFactory; -import io.airbyte.metrics.lib.OssMetricsRegistry; -import jakarta.inject.Singleton; -import java.util.UUID; -import lombok.extern.slf4j.Slf4j; - -/** - * - * Implementation of SubmitCheckConnectionActivity. - */ -@Slf4j -@Singleton -public class SubmitCheckConnectionActivityImpl implements SubmitCheckConnectionActivity { - - private final SourceApi sourceApi; - private final DestinationApi destinationApi; - private final EnvVariableFeatureFlags envVariableFeatureFlags; - - public SubmitCheckConnectionActivityImpl(final SourceApi sourceApi, - final DestinationApi destinationApi, - final EnvVariableFeatureFlags envVariableFeatureFlags) { - this.sourceApi = sourceApi; - this.destinationApi = destinationApi; - this.envVariableFeatureFlags = envVariableFeatureFlags; - } - - @Override - @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) - public ConnectorJobOutput submitCheckConnectionToSource(final UUID sourceId) { - MetricClientFactory.getMetricClient().count(OssMetricsRegistry.ACTIVITY_SUBMIT_CHECK_SOURCE_CONNECTION, 1); - - ConnectorJobOutput jobOutput = new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION); - try { - CheckConnectionRead checkResult = AirbyteApiClient.retryWithJitter( - () -> sourceApi.checkConnectionToSource(new SourceIdRequestBody().sourceId(sourceId)), - "Trigger check connection to source"); - jobOutput.withCheckConnection(convertApiOutputToStandardOutput(checkResult)); - } catch (Exception ex) { - jobOutput.withFailureReason(new FailureReason().withFailureType(FailureType.SYSTEM_ERROR).withInternalMessage(ex.getMessage())); - throw ex; - } - return jobOutput; - } - - @Override - @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) - public ConnectorJobOutput submitCheckConnectionToDestination(final UUID destinationId) { - MetricClientFactory.getMetricClient().count(OssMetricsRegistry.ACTIVITY_SUBMIT_CHECK_DESTINATION_CONNECTION, 1); - - ConnectorJobOutput jobOutput = new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION); - try { - CheckConnectionRead checkResult = AirbyteApiClient.retryWithJitter( - () -> destinationApi.checkConnectionToDestination(new DestinationIdRequestBody().destinationId(destinationId)), - "Trigger check connection to destination"); - jobOutput.withCheckConnection(convertApiOutputToStandardOutput(checkResult)); - } catch (Exception ex) { - jobOutput.withFailureReason(new FailureReason().withFailureType(FailureType.SYSTEM_ERROR).withInternalMessage(ex.getMessage())); - throw ex; - } - return jobOutput; - } - - private StandardCheckConnectionOutput convertApiOutputToStandardOutput(final CheckConnectionRead apiOutput) { - StandardCheckConnectionOutput output = new StandardCheckConnectionOutput().withMessage(apiOutput.getMessage()); - if (StatusEnum.SUCCEEDED.equals(apiOutput.getStatus())) { - output.withStatus(Status.SUCCEEDED); - } else { - output.withStatus(Status.FAILED); - } - return output; - } - -} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java index 5b9fa785f38..454d8c4eacc 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowImpl.java @@ -31,8 +31,6 @@ import io.airbyte.config.StandardSyncOutput; import io.airbyte.config.StandardSyncSummary; import io.airbyte.config.StandardSyncSummary.ReplicationStatus; -import io.airbyte.featureflag.CheckConnectionUseApiEnabled; -import io.airbyte.featureflag.CheckConnectionUseChildWorkflowEnabled; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricTags; @@ -43,9 +41,6 @@ import io.airbyte.workers.models.JobInput; import io.airbyte.workers.models.SyncJobCheckConnectionInputs; import io.airbyte.workers.temporal.annotations.TemporalActivityStub; -import io.airbyte.workers.temporal.check.connection.CheckConnectionActivity; -import io.airbyte.workers.temporal.check.connection.CheckConnectionActivity.CheckConnectionInput; -import io.airbyte.workers.temporal.check.connection.SubmitCheckConnectionActivity; import io.airbyte.workers.temporal.scheduling.activities.AppendToAttemptLogActivity; import io.airbyte.workers.temporal.scheduling.activities.AppendToAttemptLogActivity.LogInput; import io.airbyte.workers.temporal.scheduling.activities.AppendToAttemptLogActivity.LogLevel; @@ -115,15 +110,12 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow { private static final String GENERATE_CHECK_INPUT_TAG = "generate_check_input"; - private static final String CHECK_WITH_API_TAG = "check_with_api"; - private static final String CHECK_WITH_CHILD_WORKFLOW_TAG = "check_with_child_workflow"; private static final String SYNC_TASK_QUEUE_ROUTE_RENAME_TAG = "sync_task_queue_route_rename"; private static final String CHECK_RUN_PROGRESS_TAG = "check_run_progress"; private static final String NEW_RETRIES_TAG = "new_retries"; private static final String APPEND_ATTEMPT_LOG_TAG = "append_attempt_log"; private static final String DONT_FAIL_FOR_BACKOFF_SCHEDULE_CONFLICT_TAG = "dont_fail_for_backoff_schedule_conflict"; private static final int GENERATE_CHECK_INPUT_CURRENT_VERSION = 1; - private static final int CHECK_WITH_CHILD_WORKFLOW_CURRENT_VERSION = 1; private static final int SYNC_TASK_QUEUE_ROUTE_RENAME_CURRENT_VERSION = 1; private static final int CHECK_RUN_PROGRESS_VERSION = 1; private static final int NEW_RETRIES_VERSION = 1; @@ -136,7 +128,6 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow private static final String GET_FEATURE_FLAGS_TAG = "get_feature_flags"; private static final int GET_FEATURE_FLAGS_CURRENT_VERSION = 1; - private static final int CHECK_WITH_API_CURRENT_VERSION = 1; @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") private GenerateInputActivity getSyncInputActivity; @@ -147,10 +138,6 @@ public class ConnectionManagerWorkflowImpl implements ConnectionManagerWorkflow @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") private AutoDisableConnectionActivity autoDisableConnectionActivity; @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") - private CheckConnectionActivity checkActivity; - @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") - private SubmitCheckConnectionActivity submitCheckActivity; - @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") private StreamResetActivity streamResetActivity; @TemporalActivityStub(activityOptionsBeanName = "shortActivityOptions") private RecordMetricActivity recordMetricActivity; @@ -302,7 +289,7 @@ private CancellationScope generateSyncWorkflowRunnable(final ConnectionUpdaterIn StandardSyncOutput standardSyncOutput = null; try { - final SyncCheckConnectionResult syncCheckConnectionResult = checkConnections(getJobRunConfig(), jobInputs, featureFlags); + final SyncCheckConnectionResult syncCheckConnectionResult = checkConnections(getJobRunConfig(), jobInputs); if (syncCheckConnectionResult.isFailed()) { final StandardSyncOutput checkFailureOutput = syncCheckConnectionResult.buildFailureOutput(); workflowState.setFailed(getFailStatus(checkFailureOutput)); @@ -517,10 +504,6 @@ private boolean shouldRunCheckInputGeneration() { return generateCheckInputVersion >= GENERATE_CHECK_INPUT_CURRENT_VERSION; } - private ConnectorJobOutput getCheckResponse(final CheckConnectionInput checkInput) { - return runMandatoryActivityWithOutput(checkActivity::runWithJobOutput, checkInput); - } - private SyncJobCheckConnectionInputs getCheckConnectionInputFromSync(final JobInput jobInputs) { final StandardSyncInput syncInput = jobInputs.getSyncInput(); final JsonNode sourceConfig = syncInput.getSourceConfiguration(); @@ -546,8 +529,7 @@ private SyncJobCheckConnectionInputs getCheckConnectionInputFromSync(final JobIn } private SyncCheckConnectionResult checkConnections(final JobRunConfig jobRunConfig, - @Nullable final JobInput jobInputs, - final Map featureFlags) { + @Nullable final JobInput jobInputs) { final SyncCheckConnectionResult checkConnectionResult = new SyncCheckConnectionResult(jobRunConfig); final JobCheckFailureInput jobStateInput = @@ -576,29 +558,11 @@ private SyncCheckConnectionResult checkConnections(final JobRunConfig jobRunConf } else { log.info("SOURCE CHECK: Starting"); final ConnectorJobOutput sourceCheckResponse; - final int checkWithApiVersion = - Workflow.getVersion(CHECK_WITH_API_TAG, Workflow.DEFAULT_VERSION, CHECK_WITH_API_CURRENT_VERSION); - final int checkWithChildWorkflowVersion = - Workflow.getVersion(CHECK_WITH_CHILD_WORKFLOW_TAG, Workflow.DEFAULT_VERSION, CHECK_WITH_CHILD_WORKFLOW_CURRENT_VERSION); - if (checkWithApiVersion >= CHECK_WITH_API_CURRENT_VERSION && featureFlags.get(CheckConnectionUseApiEnabled.INSTANCE.getKey())) { - sourceCheckResponse = runMandatoryActivityWithOutput(submitCheckActivity::submitCheckConnectionToSource, - checkInputs.getSourceCheckConnectionInput().getActorId()); - } else if (checkWithChildWorkflowVersion >= CHECK_WITH_CHILD_WORKFLOW_CURRENT_VERSION - && featureFlags.get(CheckConnectionUseChildWorkflowEnabled.INSTANCE.getKey())) { - // Retrieve source definition; - - sourceCheckResponse = runCheckInChildWorkflow(jobRunConfig, sourceLauncherConfig, new StandardCheckConnectionInput() - .withActorType(ActorType.SOURCE) - .withActorId(checkInputs.getSourceCheckConnectionInput().getActorId()) - .withConnectionConfiguration(checkInputs.getSourceCheckConnectionInput().getConnectionConfiguration()) - .withResourceRequirements(checkInputs.getSourceCheckConnectionInput().getResourceRequirements())); - } else { - final CheckConnectionInput checkSourceInput = new CheckConnectionInput( - jobRunConfig, - sourceLauncherConfig, - checkInputs.getSourceCheckConnectionInput()); - sourceCheckResponse = getCheckResponse(checkSourceInput); - } + sourceCheckResponse = runCheckInChildWorkflow(jobRunConfig, sourceLauncherConfig, new StandardCheckConnectionInput() + .withActorType(ActorType.SOURCE) + .withActorId(checkInputs.getSourceCheckConnectionInput().getActorId()) + .withConnectionConfiguration(checkInputs.getSourceCheckConnectionInput().getConnectionConfiguration()) + .withResourceRequirements(checkInputs.getSourceCheckConnectionInput().getResourceRequirements())); if (SyncCheckConnectionResult.isOutputFailed(sourceCheckResponse)) { checkConnectionResult.setFailureOrigin(FailureReason.FailureOrigin.SOURCE); @@ -614,27 +578,11 @@ private SyncCheckConnectionResult checkConnections(final JobRunConfig jobRunConf } else { log.info("DESTINATION CHECK: Starting"); final ConnectorJobOutput destinationCheckResponse; - final int checkWithApiVersion = - Workflow.getVersion(CHECK_WITH_API_TAG, Workflow.DEFAULT_VERSION, CHECK_WITH_API_CURRENT_VERSION); - final int checkWithChildWorkflowVersion = - Workflow.getVersion(CHECK_WITH_CHILD_WORKFLOW_TAG, Workflow.DEFAULT_VERSION, CHECK_WITH_CHILD_WORKFLOW_CURRENT_VERSION); - if (checkWithApiVersion >= CHECK_WITH_API_CURRENT_VERSION && featureFlags.get(CheckConnectionUseApiEnabled.INSTANCE.getKey())) { - destinationCheckResponse = runMandatoryActivityWithOutput(submitCheckActivity::submitCheckConnectionToDestination, - checkInputs.getDestinationCheckConnectionInput().getActorId()); - } else if (checkWithChildWorkflowVersion >= CHECK_WITH_CHILD_WORKFLOW_CURRENT_VERSION - && featureFlags.get(CheckConnectionUseChildWorkflowEnabled.INSTANCE.getKey())) { - destinationCheckResponse = runCheckInChildWorkflow(jobRunConfig, checkInputs.getDestinationLauncherConfig(), - new StandardCheckConnectionInput() - .withActorType(ActorType.DESTINATION) - .withActorId(checkInputs.getDestinationCheckConnectionInput().getActorId()) - .withConnectionConfiguration(checkInputs.getDestinationCheckConnectionInput().getConnectionConfiguration())); - } else { - final CheckConnectionInput checkDestinationInput = new CheckConnectionInput( - jobRunConfig, - checkInputs.getDestinationLauncherConfig(), - checkInputs.getDestinationCheckConnectionInput()); - destinationCheckResponse = getCheckResponse(checkDestinationInput); - } + destinationCheckResponse = runCheckInChildWorkflow(jobRunConfig, checkInputs.getDestinationLauncherConfig(), + new StandardCheckConnectionInput() + .withActorType(ActorType.DESTINATION) + .withActorId(checkInputs.getDestinationCheckConnectionInput().getActorId()) + .withConnectionConfiguration(checkInputs.getDestinationCheckConnectionInput().getConnectionConfiguration())); if (SyncCheckConnectionResult.isOutputFailed(destinationCheckResponse)) { checkConnectionResult.setFailureOrigin(FailureReason.FailureOrigin.DESTINATION); checkConnectionResult.setFailureOutput(destinationCheckResponse); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityImpl.java index 4383cf1a0fc..7860ffcbaf0 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityImpl.java @@ -8,15 +8,8 @@ import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.ConnectionIdRequestBody; import io.airbyte.api.client.model.generated.WorkspaceRead; -import io.airbyte.featureflag.CheckConnectionUseApiEnabled; -import io.airbyte.featureflag.CheckConnectionUseChildWorkflowEnabled; -import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.Flag; -import io.airbyte.featureflag.Workspace; import jakarta.inject.Singleton; import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.UUID; import lombok.extern.slf4j.Slf4j; @@ -28,12 +21,9 @@ public class FeatureFlagFetchActivityImpl implements FeatureFlagFetchActivity { private final WorkspaceApi workspaceApi; - private final FeatureFlagClient featureFlagClient; - public FeatureFlagFetchActivityImpl(final WorkspaceApi workspaceApi, - final FeatureFlagClient featureFlagClient) { + public FeatureFlagFetchActivityImpl(final WorkspaceApi workspaceApi) { this.workspaceApi = workspaceApi; - this.featureFlagClient = featureFlagClient; } /** @@ -53,17 +43,9 @@ public UUID getWorkspaceId(final UUID connectionId) { @Override public FeatureFlagFetchOutput getFeatureFlags(final FeatureFlagFetchInput input) { - final UUID workspaceId = getWorkspaceId(input.getConnectionId()); - // No feature flags are currently in use. // To get value for a feature flag with the workspace context, add it to the workspaceFlags list. - final List> workspaceFlags = List.of(CheckConnectionUseApiEnabled.INSTANCE, CheckConnectionUseChildWorkflowEnabled.INSTANCE); - final Map featureFlags = new HashMap<>(); - for (final Flag flag : workspaceFlags) { - featureFlags.put(flag.getKey(), featureFlagClient.boolVariation(flag, new Workspace(workspaceId))); - } - - return new FeatureFlagFetchOutput(featureFlags); + return new FeatureFlagFetchOutput(new HashMap<>()); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java index 9cc367db42f..eb433783862 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/ConnectionManagerWorkflowTest.java @@ -22,22 +22,15 @@ import io.airbyte.commons.temporal.scheduling.state.listener.WorkflowStateChangedListener.ChangedStateEvent; import io.airbyte.commons.temporal.scheduling.state.listener.WorkflowStateChangedListener.StateField; import io.airbyte.commons.version.Version; -import io.airbyte.config.ConnectorJobOutput; -import io.airbyte.config.ConnectorJobOutput.OutputType; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.FailureReason.FailureType; import io.airbyte.config.StandardCheckConnectionInput; -import io.airbyte.config.StandardCheckConnectionOutput; -import io.airbyte.config.StandardCheckConnectionOutput.Status; import io.airbyte.config.StandardSyncInput; -import io.airbyte.featureflag.CheckConnectionUseApiEnabled; -import io.airbyte.featureflag.CheckConnectionUseChildWorkflowEnabled; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.models.JobInput; import io.airbyte.workers.models.SyncJobCheckConnectionInputs; -import io.airbyte.workers.temporal.check.connection.SubmitCheckConnectionActivity; import io.airbyte.workers.temporal.scheduling.activities.AppendToAttemptLogActivity; import io.airbyte.workers.temporal.scheduling.activities.AppendToAttemptLogActivity.LogOutput; import io.airbyte.workers.temporal.scheduling.activities.AutoDisableConnectionActivity; @@ -141,8 +134,6 @@ class ConnectionManagerWorkflowTest { private final ConfigFetchActivity mConfigFetchActivity = mock(ConfigFetchActivity.class, Mockito.withSettings().withoutAnnotations()); - private final SubmitCheckConnectionActivity mSubmitCheckConnectionActivity = - mock(SubmitCheckConnectionActivity.class, Mockito.withSettings().withoutAnnotations()); private static final GenerateInputActivityImpl mGenerateInputActivityImpl = mock(GenerateInputActivityImpl.class, Mockito.withSettings().withoutAnnotations()); private static final JobCreationAndStatusUpdateActivity mJobCreationAndStatusUpdateActivity = @@ -187,7 +178,6 @@ static Stream getMaxAttemptForResetRetry() { @BeforeEach void setUp() throws Exception { Mockito.reset(mConfigFetchActivity); - Mockito.reset(mSubmitCheckConnectionActivity); Mockito.reset(mGenerateInputActivityImpl); Mockito.reset(mJobCreationAndStatusUpdateActivity); Mockito.reset(mAutoDisableConnectionActivity); @@ -195,7 +185,6 @@ void setUp() throws Exception { Mockito.reset(mRecordMetricActivity); Mockito.reset(mWorkflowConfigActivity); Mockito.reset(mRouteToSyncTaskQueueActivity); - Mockito.reset(mFeatureFlagFetchActivity); Mockito.reset(mCheckRunProgressActivity); Mockito.reset(mRetryStatePersistenceActivity); Mockito.reset(mAppendToAttemptLogActivity); @@ -226,14 +215,6 @@ void setUp() throws Exception { new IntegrationLauncherConfig().withDockerImage(SOURCE_DOCKER_IMAGE), new IntegrationLauncherConfig(), new StandardSyncInput())); - - when(mSubmitCheckConnectionActivity.submitCheckConnectionToSource(Mockito.any())) - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.SUCCEEDED).withMessage("check worked"))); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToDestination(Mockito.any())) - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.SUCCEEDED).withMessage("check worked"))); - when(mAutoDisableConnectionActivity.autoDisableFailingConnection(Mockito.any())) .thenReturn(new AutoDisableConnectionOutput(false)); @@ -246,11 +227,8 @@ void setUp() throws Exception { .thenReturn(new RouteToSyncTaskQueueOutput(TemporalJobType.SYNC.name())); when(mRouteToSyncTaskQueueActivity.routeToCheckConnection(Mockito.any())) .thenReturn(new RouteToSyncTaskQueueOutput(TemporalJobType.CHECK_CONNECTION.name())); - when(mFeatureFlagFetchActivity.getFeatureFlags(Mockito.any())) - .thenReturn(new FeatureFlagFetchOutput(Map.of(CheckConnectionUseApiEnabled.INSTANCE.getKey(), false, - CheckConnectionUseChildWorkflowEnabled.INSTANCE.getKey(), true))); - + .thenReturn(new FeatureFlagFetchOutput(Map.of())); when(mCheckRunProgressActivity.checkProgress(Mockito.any())) .thenReturn(new CheckRunProgressActivity.Output(false)); // false == complete failure final var manager = RetryManager.builder().totalCompleteFailureLimit(1).build(); // just run once @@ -1042,9 +1020,6 @@ void testSourceCheckFailuresRecorded() throws Exception { .thenReturn(new JobCreationOutput(JOB_ID)); when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) .thenReturn(new AttemptNumberCreationOutput(ATTEMPT_ID)); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToSource((Mockito.any()))) - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.FAILED).withMessage(FAILED_CHECK_MESSAGE))); final Worker checkWorker = testEnv.newWorker(TemporalJobType.CHECK_CONNECTION.name()); checkWorker.registerWorkflowImplementationTypes(CheckConnectionFailedWorkflow.class); @@ -1085,9 +1060,6 @@ void testSourceCheckInChildWorkflowFailuresRecorded() throws Exception { .thenReturn(new JobCreationOutput(JOB_ID)); when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) .thenReturn(new AttemptNumberCreationOutput(ATTEMPT_ID)); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToSource((Mockito.any()))) - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.FAILED).withMessage(FAILED_CHECK_MESSAGE))); final Worker checkWorker = testEnv.newWorker(TemporalJobType.CHECK_CONNECTION.name()); checkWorker.registerWorkflowImplementationTypes(CheckConnectionFailedWorkflow.class); testEnv.start(); @@ -1126,9 +1098,6 @@ void testSourceCheckFailureReasonsRecorded() throws Exception { .thenReturn(new JobCreationOutput(JOB_ID)); when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) .thenReturn(new AttemptNumberCreationOutput(ATTEMPT_ID)); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToSource((Mockito.any()))) - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withFailureReason(new FailureReason().withFailureType(FailureType.SYSTEM_ERROR))); final Worker checkWorker = testEnv.newWorker(TemporalJobType.CHECK_CONNECTION.name()); checkWorker.registerWorkflowImplementationTypes(CheckConnectionSystemErrorWorkflow.class); testEnv.start(); @@ -1167,14 +1136,6 @@ void testDestinationCheckFailuresRecorded() throws Exception { .thenReturn(new JobCreationOutput(JOB_ID)); when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) .thenReturn(new AttemptNumberCreationOutput(ATTEMPT_ID)); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToSource(Mockito.any())) - // First call (source) succeeds - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.SUCCEEDED).withMessage("all good"))); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToDestination(Mockito.any())) - // Second call (destination) fails - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.FAILED).withMessage(FAILED_CHECK_MESSAGE))); final Worker checkWorker = testEnv.newWorker(TemporalJobType.CHECK_CONNECTION.name()); checkWorker.registerWorkflowImplementationTypes(CheckConnectionSourceSuccessOnlyWorkflow.class); @@ -1214,14 +1175,6 @@ void testDestinationCheckFailureReasonsRecorded() throws Exception { .thenReturn(new JobCreationOutput(JOB_ID)); when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) .thenReturn(new AttemptNumberCreationOutput(ATTEMPT_ID)); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToSource(Mockito.any())) - // First call (source) succeeds - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.SUCCEEDED).withMessage("all good"))); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToDestination(Mockito.any())) - // Second call (destination) fails - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withFailureReason(new FailureReason().withFailureType(FailureType.SYSTEM_ERROR))); final Worker checkWorker = testEnv.newWorker(TemporalJobType.CHECK_CONNECTION.name()); checkWorker.registerWorkflowImplementationTypes(CheckConnectionDestinationSystemErrorWorkflow.class); @@ -1276,10 +1229,6 @@ void testSourceCheckSkippedWhenReset() throws Exception { when(mJobCreationAndStatusUpdateActivity.createNewAttemptNumber(Mockito.any())) .thenReturn(new AttemptNumberCreationOutput(ATTEMPT_ID)); mockResetJobInput(jobRunConfig); - when(mSubmitCheckConnectionActivity.submitCheckConnectionToDestination(Mockito.any())) - // only call destination because because source check is skipped - .thenReturn(new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION) - .withCheckConnection(new StandardCheckConnectionOutput().withStatus(Status.FAILED).withMessage(FAILED_CHECK_MESSAGE))); final Worker checkWorker = testEnv.newWorker(TemporalJobType.CHECK_CONNECTION.name()); checkWorker.registerWorkflowImplementationTypes(CheckConnectionFailedWorkflow.class); @@ -1996,7 +1945,7 @@ private void setup final Worker managerWorker = testEnv.newWorker(TemporalJobType.CONNECTION_UPDATER.name()); managerWorker.registerWorkflowImplementationTypes(temporalProxyHelper.proxyWorkflowClass(ConnectionManagerWorkflowImpl.class)); - managerWorker.registerActivitiesImplementations(mConfigFetchActivity, mSubmitCheckConnectionActivity, mGenerateInputActivityImpl, + managerWorker.registerActivitiesImplementations(mConfigFetchActivity, mGenerateInputActivityImpl, mJobCreationAndStatusUpdateActivity, mAutoDisableConnectionActivity, mRecordMetricActivity, mWorkflowConfigActivity, mRouteToSyncTaskQueueActivity, mFeatureFlagFetchActivity, mCheckRunProgressActivity, mRetryStatePersistenceActivity, mAppendToAttemptLogActivity); @@ -2096,7 +2045,7 @@ private void setupSimpleConnectionManagerWorkflow() { final Worker managerWorker = testEnv.newWorker(TemporalJobType.CONNECTION_UPDATER.name()); managerWorker.registerWorkflowImplementationTypes(temporalProxyHelper.proxyWorkflowClass(ConnectionManagerWorkflowImpl.class)); - managerWorker.registerActivitiesImplementations(mConfigFetchActivity, mSubmitCheckConnectionActivity, mGenerateInputActivityImpl, + managerWorker.registerActivitiesImplementations(mConfigFetchActivity, mGenerateInputActivityImpl, mJobCreationAndStatusUpdateActivity, mAutoDisableConnectionActivity, mRecordMetricActivity, mWorkflowConfigActivity, mRouteToSyncTaskQueueActivity, mFeatureFlagFetchActivity, mCheckRunProgressActivity, mRetryStatePersistenceActivity, mAppendToAttemptLogActivity); diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityTest.java index e6fb2b5e10a..49e92af8ec9 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/scheduling/activities/FeatureFlagFetchActivityTest.java @@ -5,16 +5,9 @@ package io.airbyte.workers.temporal.scheduling.activities; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import io.airbyte.api.client.generated.WorkspaceApi; -import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.api.client.model.generated.ConnectionIdRequestBody; -import io.airbyte.api.client.model.generated.WorkspaceRead; -import io.airbyte.featureflag.CheckConnectionUseApiEnabled; -import io.airbyte.featureflag.CheckConnectionUseChildWorkflowEnabled; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.TestClient; import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.Assertions; @@ -24,20 +17,15 @@ class FeatureFlagFetchActivityTest { private static final UUID CONNECTION_ID = UUID.randomUUID(); - private static final UUID WORKSPACE_ID = UUID.randomUUID(); FeatureFlagFetchActivity featureFlagFetchActivity; FeatureFlagClient featureFlagClient; @BeforeEach - void setUp() throws ApiException { + void setUp() { final WorkspaceApi workspaceApi = mock(WorkspaceApi.class); - featureFlagClient = mock(TestClient.class); - featureFlagFetchActivity = new FeatureFlagFetchActivityImpl(workspaceApi, featureFlagClient); - - when(workspaceApi.getWorkspaceByConnectionId(new ConnectionIdRequestBody().connectionId(CONNECTION_ID))) - .thenReturn(new WorkspaceRead().workspaceId(WORKSPACE_ID)); + featureFlagFetchActivity = new FeatureFlagFetchActivityImpl(workspaceApi); } @Test @@ -45,8 +33,7 @@ void testGetFeatureFlags() { final FeatureFlagFetchActivity.FeatureFlagFetchInput input = new FeatureFlagFetchActivity.FeatureFlagFetchInput(CONNECTION_ID); final FeatureFlagFetchActivity.FeatureFlagFetchOutput output = featureFlagFetchActivity.getFeatureFlags(input); - Assertions.assertEquals(output.getFeatureFlags(), Map.of(CheckConnectionUseApiEnabled.INSTANCE.getKey(), false, - CheckConnectionUseChildWorkflowEnabled.INSTANCE.getKey(), false)); + Assertions.assertEquals(output.getFeatureFlags(), Map.of()); } From a92d6c93c8d5850fc3414f8daabbd7c8544fd87d Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Tue, 29 Aug 2023 15:20:48 -0700 Subject: [PATCH 052/201] Add TraceTags to RetryStatePersistenceActivityImpl (#8599) --- .../activities/RetryStatePersistenceActivityImpl.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/RetryStatePersistenceActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/RetryStatePersistenceActivityImpl.java index 8652924ac97..276c3951c8b 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/RetryStatePersistenceActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/scheduling/activities/RetryStatePersistenceActivityImpl.java @@ -4,6 +4,11 @@ package io.airbyte.workers.temporal.scheduling.activities; +import static io.airbyte.metrics.lib.ApmTraceConstants.ACTIVITY_TRACE_OPERATION_NAME; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.WORKSPACE_ID_KEY; + +import datadog.trace.api.Trace; import io.airbyte.api.client.generated.WorkspaceApi; import io.airbyte.api.client.invoker.generated.ApiException; import io.airbyte.api.client.model.generated.ConnectionIdRequestBody; @@ -14,9 +19,11 @@ import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.UseNewRetries; import io.airbyte.featureflag.Workspace; +import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.workers.helpers.RetryStateClient; import jakarta.inject.Singleton; import java.util.List; +import java.util.Map; import java.util.UUID; /** @@ -38,9 +45,12 @@ public RetryStatePersistenceActivityImpl(final RetryStateClient client, this.workspaceApi = workspaceApi; } + @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) @Override public HydrateOutput hydrateRetryState(final HydrateInput input) { + ApmTraceUtils.addTagsToTrace(Map.of(CONNECTION_ID_KEY, input.getConnectionId())); final var workspaceId = getWorkspaceId(input.getConnectionId()); + ApmTraceUtils.addTagsToTrace(Map.of(WORKSPACE_ID_KEY, workspaceId)); final var enabled = featureFlagClient.boolVariation(UseNewRetries.INSTANCE, new Multi(List.of( new Connection(input.getConnectionId()), From c7fbb072c60f53f6c46228530a6649aeeff81973 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Tue, 29 Aug 2023 20:21:04 -0400 Subject: [PATCH 053/201] fix: don't show breaking change banners if an LD override is present (#8601) --- airbyte-api/src/main/openapi/config.yaml | 4 +- .../ActorDefinitionVersionHandler.java | 22 ++++--- .../ActorDefinitionVersionHandlerTest.java | 35 +++++------ .../ActorDefinitionVersionHelper.java | 63 +++++++++++++++---- .../ActorDefinitionVersionHelperTest.java | 38 +++++++---- .../src/core/domain/connector/connector.ts | 2 +- 6 files changed, 107 insertions(+), 57 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 3e9af588735..bb2b6619d4f 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -4924,13 +4924,13 @@ components: - dockerRepository - dockerImageTag - supportState - - isActorDefaultVersion + - isOverrideApplied properties: dockerRepository: type: string dockerImageTag: type: string - isActorDefaultVersion: + isOverrideApplied: type: boolean supportState: $ref: "#/components/schemas/SupportState" diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java index 5390ea678ab..3bdf66b8c06 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java @@ -19,6 +19,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.validation.json.JsonValidationException; @@ -58,10 +59,10 @@ public ActorDefinitionVersionRead getActorDefinitionVersionForSourceId(final Sou throws JsonValidationException, ConfigNotFoundException, IOException { final SourceConnection sourceConnection = configRepository.getSourceConnection(sourceIdRequestBody.getSourceId()); final StandardSourceDefinition sourceDefinition = configRepository.getSourceDefinitionFromSource(sourceConnection.getSourceId()); - final ActorDefinitionVersion actorDefinitionVersion = - actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, sourceConnection.getWorkspaceId(), sourceConnection.getSourceId()); - final boolean isActorDefaultVersion = actorDefinitionVersion.getVersionId().equals(sourceConnection.getDefaultVersionId()); - return createActorDefinitionVersionRead(actorDefinitionVersion, isActorDefaultVersion); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, sourceConnection.getWorkspaceId(), + sourceConnection.getSourceId()); + return createActorDefinitionVersionRead(versionWithOverrideStatus); } public ActorDefinitionVersionRead getActorDefinitionVersionForDestinationId(final DestinationIdRequestBody destinationIdRequestBody) @@ -69,20 +70,21 @@ public ActorDefinitionVersionRead getActorDefinitionVersionForDestinationId(fina final DestinationConnection destinationConnection = configRepository.getDestinationConnection(destinationIdRequestBody.getDestinationId()); final StandardDestinationDefinition destinationDefinition = configRepository.getDestinationDefinitionFromDestination(destinationConnection.getDestinationId()); - final ActorDefinitionVersion actorDefinitionVersion = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, - destinationConnection.getWorkspaceId(), destinationConnection.getDestinationId()); - final boolean isActorDefaultVersion = actorDefinitionVersion.getVersionId().equals(destinationConnection.getDefaultVersionId()); - return createActorDefinitionVersionRead(actorDefinitionVersion, isActorDefaultVersion); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, + destinationConnection.getWorkspaceId(), destinationConnection.getDestinationId()); + return createActorDefinitionVersionRead(versionWithOverrideStatus); } @VisibleForTesting - ActorDefinitionVersionRead createActorDefinitionVersionRead(final ActorDefinitionVersion actorDefinitionVersion, final boolean isActorDefault) + ActorDefinitionVersionRead createActorDefinitionVersionRead(final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus) throws IOException { + final ActorDefinitionVersion actorDefinitionVersion = versionWithOverrideStatus.actorDefinitionVersion(); final ActorDefinitionVersionRead advRead = new ActorDefinitionVersionRead() .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) .supportState(toApiSupportState(actorDefinitionVersion.getSupportState())) - .isActorDefaultVersion(isActorDefault); + .isOverrideApplied(versionWithOverrideStatus.isOverrideApplied()); final List breakingChanges = configRepository.listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java index 695af43beab..e730c67f8e7 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java @@ -25,6 +25,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.validation.json.JsonValidationException; @@ -71,7 +72,7 @@ private ActorDefinitionVersion createActorDefinitionVersion() { @ParameterizedTest @CsvSource({"true", "false"}) - void testGetActorDefinitionVersionForSource(final boolean isSourceVersionDefault) + void testGetActorDefinitionVersionForSource(final boolean isOverrideApplied) throws JsonValidationException, ConfigNotFoundException, IOException { final UUID sourceId = UUID.randomUUID(); final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersion(); @@ -79,22 +80,18 @@ void testGetActorDefinitionVersionForSource(final boolean isSourceVersionDefault .withSourceId(sourceId) .withWorkspaceId(WORKSPACE_ID); - if (isSourceVersionDefault) { - sourceConnection.withDefaultVersionId(actorDefinitionVersion.getVersionId()); - } - when(mConfigRepository.getSourceConnection(sourceId)) .thenReturn(sourceConnection); when(mConfigRepository.getSourceDefinitionFromSource(sourceId)) .thenReturn(SOURCE_DEFINITION); - when(mActorDefinitionVersionHelper.getSourceVersion(SOURCE_DEFINITION, WORKSPACE_ID, sourceId)) - .thenReturn(actorDefinitionVersion); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(SOURCE_DEFINITION, WORKSPACE_ID, sourceId)) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, isOverrideApplied)); final SourceIdRequestBody sourceIdRequestBody = new SourceIdRequestBody().sourceId(sourceId); final ActorDefinitionVersionRead actorDefinitionVersionRead = actorDefinitionVersionHandler.getActorDefinitionVersionForSourceId(sourceIdRequestBody); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() - .isActorDefaultVersion(isSourceVersionDefault) + .isOverrideApplied(isOverrideApplied) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()); @@ -102,7 +99,7 @@ void testGetActorDefinitionVersionForSource(final boolean isSourceVersionDefault assertEquals(expectedRead, actorDefinitionVersionRead); verify(mConfigRepository).getSourceConnection(sourceId); verify(mConfigRepository).getSourceDefinitionFromSource(sourceId); - verify(mActorDefinitionVersionHelper).getSourceVersion(SOURCE_DEFINITION, WORKSPACE_ID, sourceId); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(SOURCE_DEFINITION, WORKSPACE_ID, sourceId); verify(mConfigRepository).listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); verifyNoMoreInteractions(mConfigRepository); verifyNoMoreInteractions(mActorDefinitionVersionHelper); @@ -110,7 +107,7 @@ void testGetActorDefinitionVersionForSource(final boolean isSourceVersionDefault @ParameterizedTest @CsvSource({"true", "false"}) - void testGetActorDefinitionVersionForDestination(final boolean isDestinationVersionDefault) + void testGetActorDefinitionVersionForDestination(final boolean isOverrideApplied) throws JsonValidationException, ConfigNotFoundException, IOException { final UUID destinationId = UUID.randomUUID(); final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersion(); @@ -118,22 +115,18 @@ void testGetActorDefinitionVersionForDestination(final boolean isDestinationVers .withDestinationId(destinationId) .withWorkspaceId(WORKSPACE_ID); - if (isDestinationVersionDefault) { - destinationConnection.withDefaultVersionId(actorDefinitionVersion.getVersionId()); - } - when(mConfigRepository.getDestinationConnection(destinationId)) .thenReturn(destinationConnection); when(mConfigRepository.getDestinationDefinitionFromDestination(destinationId)) .thenReturn(DESTINATION_DEFINITION); - when(mActorDefinitionVersionHelper.getDestinationVersion(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId)) - .thenReturn(actorDefinitionVersion); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId)) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, isOverrideApplied)); final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(destinationId); final ActorDefinitionVersionRead actorDefinitionVersionRead = actorDefinitionVersionHandler.getActorDefinitionVersionForDestinationId(destinationIdRequestBody); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() - .isActorDefaultVersion(isDestinationVersionDefault) + .isOverrideApplied(isOverrideApplied) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()); @@ -141,7 +134,7 @@ void testGetActorDefinitionVersionForDestination(final boolean isDestinationVers assertEquals(expectedRead, actorDefinitionVersionRead); verify(mConfigRepository).getDestinationConnection(destinationId); verify(mConfigRepository).getDestinationDefinitionFromDestination(destinationId); - verify(mActorDefinitionVersionHelper).getDestinationVersion(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId); verify(mConfigRepository).listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); verifyNoMoreInteractions(mConfigRepository); verifyNoMoreInteractions(mActorDefinitionVersionHelper); @@ -166,11 +159,13 @@ void testCreateActorDefinitionVersionReadWithBreakingChange() throws IOException final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersion().withSupportState(SupportState.DEPRECATED); when(mConfigRepository.listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion)).thenReturn(breakingChanges); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, false); final ActorDefinitionVersionRead actorDefinitionVersionRead = - actorDefinitionVersionHandler.createActorDefinitionVersionRead(actorDefinitionVersion, true); + actorDefinitionVersionHandler.createActorDefinitionVersionRead(versionWithOverrideStatus); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() - .isActorDefaultVersion(true) + .isOverrideApplied(false) .supportState(io.airbyte.api.model.generated.SupportState.DEPRECATED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java index e21fafa21e3..326003f160e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ActorDefinitionVersionHelper.java @@ -33,6 +33,15 @@ @Singleton public class ActorDefinitionVersionHelper { + /** + * A wrapper class for returning the actor definition version and whether an override was applied. + * + * @param actorDefinitionVersion - actor definition version to use + * @param isOverrideApplied - true if the version is the result of an override being applied, + * otherwise false + */ + public record ActorDefinitionVersionWithOverrideStatus(ActorDefinitionVersion actorDefinitionVersion, boolean isOverrideApplied) {} + private static final Logger LOGGER = LoggerFactory.getLogger(ActorDefinitionVersionHelper.class); private final ConfigRepository configRepository; @@ -90,16 +99,16 @@ private ActorDefinitionVersion getDefaultDestinationVersion(final StandardDestin } /** - * Get the actor definition version to use for a source. + * Get the actor definition version to use for a source, and whether an override was applied. * * @param sourceDefinition source definition * @param workspaceId workspace id * @param actorId source id - * @return actor definition version + * @return actor definition version with override status */ - public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition sourceDefinition, - final UUID workspaceId, - @Nullable final UUID actorId) + public ActorDefinitionVersionWithOverrideStatus getSourceVersionWithOverrideStatus(final StandardSourceDefinition sourceDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion defaultVersion = getDefaultSourceVersion(sourceDefinition, workspaceId, actorId); @@ -110,7 +119,22 @@ public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition so actorId, defaultVersion); - return versionOverride.orElse(defaultVersion); + return new ActorDefinitionVersionWithOverrideStatus(versionOverride.orElse(defaultVersion), versionOverride.isPresent()); + } + + /** + * Get the actor definition version to use for a source. + * + * @param sourceDefinition source definition + * @param workspaceId workspace id + * @param actorId source id + * @return actor definition version + */ + public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition sourceDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) + throws ConfigNotFoundException, IOException, JsonValidationException { + return getSourceVersionWithOverrideStatus(sourceDefinition, workspaceId, actorId).actorDefinitionVersion(); } /** @@ -126,16 +150,16 @@ public ActorDefinitionVersion getSourceVersion(final StandardSourceDefinition so } /** - * Get the actor definition version to use for a destination. + * Get the actor definition version to use for a destination, and whether an override was applied. * * @param destinationDefinition destination definition * @param workspaceId workspace id * @param actorId destination id - * @return actor definition version + * @return actor definition version with override status */ - public ActorDefinitionVersion getDestinationVersion(final StandardDestinationDefinition destinationDefinition, - final UUID workspaceId, - @Nullable final UUID actorId) + public ActorDefinitionVersionWithOverrideStatus getDestinationVersionWithOverrideStatus(final StandardDestinationDefinition destinationDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion defaultVersion = getDefaultDestinationVersion(destinationDefinition, workspaceId, actorId); @@ -146,7 +170,22 @@ public ActorDefinitionVersion getDestinationVersion(final StandardDestinationDef actorId, defaultVersion); - return versionOverride.orElse(defaultVersion); + return new ActorDefinitionVersionWithOverrideStatus(versionOverride.orElse(defaultVersion), versionOverride.isPresent()); + } + + /** + * Get the actor definition version to use for a destination. + * + * @param destinationDefinition destination definition + * @param workspaceId workspace id + * @param actorId destination id + * @return actor definition version + */ + public ActorDefinitionVersion getDestinationVersion(final StandardDestinationDefinition destinationDefinition, + final UUID workspaceId, + @Nullable final UUID actorId) + throws ConfigNotFoundException, IOException, JsonValidationException { + return getDestinationVersionWithOverrideStatus(destinationDefinition, workspaceId, actorId).actorDefinitionVersion(); } /** diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java index fe27b66c362..f31e1138c6d 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionHelperTest.java @@ -5,6 +5,7 @@ package io.airbyte.config.persistence; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -20,6 +21,7 @@ import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; import io.airbyte.config.persistence.version_overrides.DefinitionVersionOverrideProvider; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.featureflag.TestClient; @@ -94,8 +96,10 @@ void testGetSourceVersion() throws ConfigNotFoundException, IOException, JsonVal .withSourceDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -107,8 +111,10 @@ void testGetSourceVersionFromActorDefault() throws ConfigNotFoundException, IOEx when(mFeatureFlagClient.boolVariation(UseActorScopedDefaultVersions.INSTANCE, new Workspace(WORKSPACE_ID))).thenReturn(true); when(mConfigRepository.getSourceConnection(ACTOR_ID)).thenReturn(new SourceConnection().withDefaultVersionId(DEFAULT_VERSION_ID)); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -120,8 +126,10 @@ void testGetSourceVersionWithOverride() throws ConfigNotFoundException, IOExcept .withSourceDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getSourceVersion(sourceDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(OVERRIDDEN_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(OVERRIDDEN_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertTrue(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -165,8 +173,10 @@ void testGetDestinationVersion() throws ConfigNotFoundException, IOException, Js .withDestinationDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -178,8 +188,10 @@ void testGetDestinationVersionFromActorDefault() throws ConfigNotFoundException, when(mFeatureFlagClient.boolVariation(UseActorScopedDefaultVersions.INSTANCE, new Workspace(WORKSPACE_ID))).thenReturn(true); when(mConfigRepository.getDestinationConnection(ACTOR_ID)).thenReturn(new DestinationConnection().withDefaultVersionId(DEFAULT_VERSION_ID)); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(DEFAULT_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(DEFAULT_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertFalse(versionWithOverrideStatus.isOverrideApplied()); } @Test @@ -191,8 +203,10 @@ void testGetDestinationVersionWithOverride() throws ConfigNotFoundException, IOE .withDestinationDefinitionId(ACTOR_DEFINITION_ID) .withDefaultVersionId(DEFAULT_VERSION_ID); - final ActorDefinitionVersion actual = actorDefinitionVersionHelper.getDestinationVersion(destinationDefinition, WORKSPACE_ID, ACTOR_ID); - assertEquals(OVERRIDDEN_VERSION, actual); + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, ACTOR_ID); + assertEquals(OVERRIDDEN_VERSION, versionWithOverrideStatus.actorDefinitionVersion()); + assertTrue(versionWithOverrideStatus.isOverrideApplied()); } @Test diff --git a/airbyte-webapp/src/core/domain/connector/connector.ts b/airbyte-webapp/src/core/domain/connector/connector.ts index a1d516d4caf..5993d03b404 100644 --- a/airbyte-webapp/src/core/domain/connector/connector.ts +++ b/airbyte-webapp/src/core/domain/connector/connector.ts @@ -36,7 +36,7 @@ export const shouldDisplayBreakingChangeBanner = (actorDefinitionVersion: ActorD // This is important as it catches the case where a user has been explicitly pinned to a previous version // e.g. Prereleases, PbA Users etc.. - const actorNotOverriden = actorDefinitionVersion.isActorDefaultVersion; + const actorNotOverriden = !actorDefinitionVersion.isOverrideApplied; return hasUpcomingBreakingChanges && actorNotOverriden; }; From 7060fc8658f07b108fb039f9edd9a0483b97483b Mon Sep 17 00:00:00 2001 From: Vladimir Date: Wed, 30 Aug 2023 16:49:31 +0300 Subject: [PATCH 054/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Extend=20?= =?UTF-8?q?``=20to=20support=20object=20as=20an=20option=20va?= =?UTF-8?q?lue=20(#8591)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: lmossman --- .../components/ui/ListBox/ListBox.stories.tsx | 66 ++++++++++++------- .../src/components/ui/ListBox/ListBox.tsx | 5 +- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx b/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx index 589fe4e28ce..f8d3ff16fe4 100644 --- a/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx +++ b/airbyte-webapp/src/components/ui/ListBox/ListBox.stories.tsx @@ -1,10 +1,11 @@ import { faEdit } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { Meta, StoryObj, StoryFn } from "@storybook/react"; +import { Meta, StoryFn } from "@storybook/react"; +import { useState } from "react"; import { FlexContainer } from "components/ui/Flex"; -import { ListBox } from "./ListBox"; +import { ListBox, ListBoxProps } from "./ListBox"; export default { title: "Ui/ListBox", @@ -30,7 +31,10 @@ export default { }, } as Meta; -type Story = StoryObj; +const Template: StoryFn = (args: Omit, "onSelect">) => { + const [selectedValue, setSelectedValue] = useState(args.selectedValue); + return ; +}; const options = [ { @@ -48,26 +52,44 @@ const options = [ }, ]; -export const Primary: Story = { - args: { - options, - }, +export const Primary = Template.bind({}); +Primary.args = { + options, + selectedValue: 1, }; -export const Placement: Story = { - args: { - options, - adaptiveWidth: false, - }, - decorators: [ - (Story: StoryFn) => ( - - - - ), +export const Placement = Template.bind({}); +Placement.args = { + options, + adaptiveWidth: false, +}; +Placement.decorators = [ + (Story: StoryFn) => ( + + + + ), +]; + +export const ValueAsObject = Template.bind({}); +ValueAsObject.args = { + options: [ + { + label: "Basic", + value: { scheduleType: "basic" }, + }, + { + label: "Manual", + value: { scheduleType: "manual" }, + }, + { + label: "Cron", + value: { scheduleType: "cron" }, + }, ], + selectedValue: { scheduleType: "basic" }, }; diff --git a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx index 3a70a67a954..977f3c33c62 100644 --- a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx +++ b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx @@ -2,6 +2,7 @@ import { Placement } from "@floating-ui/react-dom"; import { Listbox } from "@headlessui/react"; import { Float } from "@headlessui-float/react"; import classNames from "classnames"; +import isEqual from "lodash/isEqual"; import React from "react"; import { useIntl } from "react-intl"; @@ -88,7 +89,7 @@ export const ListBox = ({ adaptiveWidth = true, footerOption, }: ListBoxProps) => { - const selectedOption = options.find((option) => option.value === selectedValue); + const selectedOption = options.find((option) => isEqual(option.value, selectedValue)); const onOnSelect = (value: T) => { onSelect(value); @@ -96,7 +97,7 @@ export const ListBox = ({ return (
- + Date: Wed, 30 Aug 2023 12:28:35 -0400 Subject: [PATCH 055/201] =?UTF-8?q?=F0=9F=AA=9F=20Remove=20enrollment=20ma?= =?UTF-8?q?terials=20for=20Free=20Connector=20Program=20(#8511)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vladimir --- .../api/hooks/cloud/freeConnectorProgram.ts | 50 -- .../EnrollmentModal.module.scss | 53 -- .../EnrollmentModal/EnrollmentModal.tsx | 155 ---- .../EnrollmentModal/cards.svg | 24 - .../EnrollmentModal/connectorGrid.svg | 439 ------------ .../EnrollmentModal/free-alpha-beta-pills.svg | 34 - .../EnrollmentModal/free.svg | 13 - .../EnrollmentModal/index.ts | 1 - .../EnrollmentModal/mail.svg | 14 - .../useShowEnrollmentModal.tsx | 44 -- .../InlineEnrollmentCallout.module.scss | 18 - .../InlineEnrollmentCallout.tsx | 52 -- .../LargeEnrollmentCallout.module.scss | 30 - .../LargeEnrollmentCallout.tsx | 45 -- .../connectors-badges.svg | 22 - .../longtail-connectors.svg | 667 ------------------ .../src/packages/cloud/locales/en.json | 15 - .../views/billing/BillingPage/BillingPage.tsx | 4 - .../useBillingPageBanners.test.tsx | 110 +-- .../BillingPage/useBillingPageBanners.tsx | 13 +- 20 files changed, 38 insertions(+), 1765 deletions(-) delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg diff --git a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts index f63848448a8..dd1c4ab90cd 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts @@ -1,70 +1,21 @@ import { useQuery } from "@tanstack/react-query"; -import { useState } from "react"; -import { useIntl } from "react-intl"; -import { useSearchParams } from "react-router-dom"; -import { useEffectOnce } from "react-use"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { FeatureItem, useFeature } from "core/services/features"; -import { pollUntil } from "core/utils/pollUntil"; -import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; -import { useNotificationService } from "hooks/services/Notification"; import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { webBackendGetFreeConnectorProgramInfoForWorkspace } from "../../generated/CloudApi"; -export const STRIPE_SUCCESS_QUERY = "fcpEnrollmentSuccess"; - export const useFreeConnectorProgram = () => { const workspaceId = useCurrentWorkspaceId(); const middlewares = useDefaultRequestMiddlewares(); const requestOptions = { middlewares }; const freeConnectorProgramEnabled = useFeature(FeatureItem.FreeConnectorProgram); - const [searchParams, setSearchParams] = useSearchParams(); - const [userDidEnroll, setUserDidEnroll] = useState(false); - const { formatMessage } = useIntl(); - const { registerNotification } = useNotificationService(); - const { trackError } = useAppMonitoringService(); - - const removeStripeSuccessQuery = () => { - const { [STRIPE_SUCCESS_QUERY]: _, ...unrelatedSearchParams } = Object.fromEntries(searchParams); - setSearchParams(unrelatedSearchParams, { replace: true }); - }; - - useEffectOnce(() => { - if (searchParams.has(STRIPE_SUCCESS_QUERY)) { - pollUntil( - () => webBackendGetFreeConnectorProgramInfoForWorkspace({ workspaceId }, requestOptions), - ({ hasPaymentAccountSaved }) => hasPaymentAccountSaved, - { intervalMs: 1000, maxTimeoutMs: 10000 } - ).then((maybeFcpInfo) => { - if (maybeFcpInfo) { - removeStripeSuccessQuery(); - setUserDidEnroll(true); - registerNotification({ - id: "fcp/enrollment-success", - text: formatMessage({ id: "freeConnectorProgram.enroll.success" }), - type: "success", - }); - } else { - trackError(new Error("Unable to confirm Free Connector Program enrollment before timeout"), { workspaceId }); - registerNotification({ - id: "fcp/enrollment-failure", - text: formatMessage({ id: "freeConnectorProgram.enroll.failure" }), - type: "error", - }); - } - }); - } - }); const programStatusQuery = useQuery(["freeConnectorProgramInfo", workspaceId], () => webBackendGetFreeConnectorProgramInfoForWorkspace({ workspaceId }, requestOptions).then( ({ hasPaymentAccountSaved, hasEligibleConnections, hasNonEligibleConnections }) => { - const userIsEligibleToEnroll = !hasPaymentAccountSaved; - return { - showEnrollmentUi: freeConnectorProgramEnabled && userIsEligibleToEnroll, isEnrolled: freeConnectorProgramEnabled && hasPaymentAccountSaved, hasEligibleConnections: freeConnectorProgramEnabled && hasEligibleConnections, hasNonEligibleConnections: freeConnectorProgramEnabled && hasNonEligibleConnections, @@ -75,6 +26,5 @@ export const useFreeConnectorProgram = () => { return { programStatusQuery, - userDidEnroll, }; }; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss deleted file mode 100644 index 62aba2c1d09..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss +++ /dev/null @@ -1,53 +0,0 @@ -@use "scss/colors"; -@use "scss/variables"; - -.header { - overflow: hidden; - position: relative; - background: colors.$blue-100; - height: 120px; - border-top-left-radius: variables.$border-radius-lg; - border-top-right-radius: variables.$border-radius-lg; -} - -.headerBackgroundImageContainer { - position: absolute; - left: 50%; - transform: translateX(-50%); -} - -.pill { - z-index: 2; - - // HACK: the hardcoded top margin is to vertically center the image, which was exported - // from Figma with an off-center vertical alignment - margin-top: 2em; -} - -.contentWrapper { - width: variables.$width-modal-sm; - padding: 0 variables.$spacing-xl variables.$spacing-2xl; -} - -.contentHeader { - margin: variables.$spacing-xl 0; -} - -.iconContainer { - flex: 0 0 82px; -} - -.resendEmailLink { - text-decoration: underline; - cursor: pointer; - padding: 0; - color: colors.$dark-blue; - border: none; - background-color: transparent; - font-size: inherit; - - &:hover, - &:active { - color: colors.$blue; - } -} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx deleted file mode 100644 index cdf1d5a96d8..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx +++ /dev/null @@ -1,155 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Heading } from "components/ui/Heading"; -import { Message } from "components/ui/Message"; -import { ModalFooter } from "components/ui/Modal/ModalFooter"; -import { Text } from "components/ui/Text"; - -import { STRIPE_SUCCESS_QUERY } from "core/api/cloud"; -import { StripeCheckoutSessionCreate, StripeCheckoutSessionRead } from "core/api/types/CloudApi"; - -import { ReactComponent as CardSVG } from "./cards.svg"; -import { ReactComponent as ConnectorGridSvg } from "./connectorGrid.svg"; -import styles from "./EnrollmentModal.module.scss"; -import { ReactComponent as FreeAlphaBetaPillsSVG } from "./free-alpha-beta-pills.svg"; -import { ReactComponent as FreeSVG } from "./free.svg"; -import { ReactComponent as MailSVG } from "./mail.svg"; - -interface EnrollmentModalContentProps { - closeModal: () => void; - createCheckout: (p: StripeCheckoutSessionCreate) => Promise; - workspaceId: string; - emailVerified: boolean; - sendEmailVerification: () => void; -} - -// we have to pass the email verification data and functions in as props, rather than -// directly using useAuthService(), because the modal renders outside of the -// AuthenticationProvider context. -export const EnrollmentModalContent: React.FC = ({ - closeModal, - createCheckout, - workspaceId, - emailVerified, - sendEmailVerification, -}) => { - const isMountedRef = useRef(false); - const [isLoading, setIsLoading] = useState(false); - - const startStripeCheckout = async () => { - setIsLoading(true); - // Use the current URL as a success URL but attach the STRIPE_SUCCESS_QUERY to it - const successUrl = new URL(window.location.href); - successUrl.searchParams.set(STRIPE_SUCCESS_QUERY, "true"); - const { stripeUrl } = await createCheckout({ - workspaceId, - successUrl: successUrl.href, - cancelUrl: window.location.href, - stripeMode: "setup", - }); - - // Forward to stripe as soon as we created a checkout session successfully - if (isMountedRef.current) { - window.location.assign(stripeUrl); - } - }; - - // If the user closes the modal while the request is processing, we don't want to redirect them - useEffect(() => { - isMountedRef.current = true; - return () => { - isMountedRef.current = false; - }; - }, []); - - const EnrollmentEmailVerificationWarning = () => { - const WarningContent = () => ( - - ( - - ), - }} - /> - - } - /> - ); - - return <>{!emailVerified && }; - }; - - return ( - <> - -
- -
- - -
- - - - - - - - - - {content}, - p2: (content: React.ReactNode) => {content}, - }} - /> - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - ); -}; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg deleted file mode 100644 index a86f40a6581..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg deleted file mode 100644 index da8c9978e4e..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg +++ /dev/null @@ -1,439 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg deleted file mode 100644 index 6531acb626e..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg deleted file mode 100644 index 55c619b2c1d..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts deleted file mode 100644 index eb0c733b211..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./useShowEnrollmentModal"; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg deleted file mode 100644 index d3c1f86a68a..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx deleted file mode 100644 index c3642c65df5..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import { useIntl } from "react-intl"; - -import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { useStripeCheckout } from "core/api/cloud"; -import { useAuthService } from "core/services/auth"; -import { useModalService } from "hooks/services/Modal"; -import { useNotificationService } from "hooks/services/Notification"; - -import { EnrollmentModalContent } from "./EnrollmentModal"; - -export const useShowEnrollmentModal = () => { - const { openModal, closeModal } = useModalService(); - const { mutateAsync: createCheckout } = useStripeCheckout(); - const workspaceId = useCurrentWorkspaceId(); - const { emailVerified, sendEmailVerification } = useAuthService(); - const { formatMessage } = useIntl(); - const { registerNotification } = useNotificationService(); - - const verifyEmail = () => - sendEmailVerification?.().then(() => { - registerNotification({ - id: "fcp/verify-email", - text: formatMessage({ id: "freeConnectorProgram.enrollmentModal.validationEmailConfirmation" }), - type: "info", - }); - }); - - return { - showEnrollmentModal: () => { - openModal({ - title: null, - content: () => ( - - ), - }); - }, - }; -}; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss deleted file mode 100644 index 98fb1eeeeb0..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss +++ /dev/null @@ -1,18 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.container { - width: 100%; -} - -.withMargin { - margin-top: variables.$spacing-lg; - margin-bottom: variables.$spacing-lg; -} - -.enrollLink { - all: unset; - text-decoration: underline; - cursor: pointer; - color: colors.$blue; -} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx deleted file mode 100644 index 5eb53293429..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx +++ /dev/null @@ -1,52 +0,0 @@ -import classNames from "classnames"; -import React, { PropsWithChildren } from "react"; -import { FormattedMessage } from "react-intl"; - -import { Message } from "components/ui/Message"; -import { Text } from "components/ui/Text"; - -import { useFreeConnectorProgram } from "core/api/cloud"; - -import { useShowEnrollmentModal } from "./EnrollmentModal"; -import styles from "./InlineEnrollmentCallout.module.scss"; - -export const EnrollLink: React.FC> = ({ children }) => { - const { showEnrollmentModal } = useShowEnrollmentModal(); - - return ( - - ); -}; - -interface InlineEnrollmentCalloutProps { - withMargin?: boolean; -} - -export const InlineEnrollmentCallout: React.FC = ({ withMargin }) => { - const { userDidEnroll, programStatusQuery } = useFreeConnectorProgram(); - const { showEnrollmentUi } = programStatusQuery.data || {}; - - if (userDidEnroll || !showEnrollmentUi) { - return null; - } - return ( - - {content}, - }} - /> - - } - className={classNames(styles.container, { [styles.withMargin]: withMargin })} - /> - ); -}; - -export default InlineEnrollmentCallout; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss deleted file mode 100644 index 3959159143b..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss +++ /dev/null @@ -1,30 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.container { - padding: variables.$spacing-lg variables.$spacing-xl; - width: 100%; - border-radius: variables.$border-radius-md; - background-color: colors.$blue-200; - background-image: url("./longtail-connectors.svg"); - background-repeat: no-repeat; - background-position-y: center; - background-position-x: -50px; -} - -.title { - text-transform: uppercase; - color: colors.$white; -} - -.description { - color: colors.$white; -} - -.flexRow { - width: 100%; -} - -.enrollButton { - margin-left: auto; -} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx deleted file mode 100644 index 161ae02dc4f..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Heading } from "components/ui/Heading"; -import { Text } from "components/ui/Text"; - -import { useBillingPageBanners } from "packages/cloud/views/billing/BillingPage/useBillingPageBanners"; - -import { ReactComponent as ConnectorsBadges } from "./connectors-badges.svg"; -import { useShowEnrollmentModal } from "./EnrollmentModal"; -import styles from "./LargeEnrollmentCallout.module.scss"; - -export const LargeEnrollmentCallout: React.FC = () => { - const { showEnrollmentModal } = useShowEnrollmentModal(); - const { showFCPBanner } = useBillingPageBanners(); - - if (!showFCPBanner) { - return null; - } - - return ( -
- - - - - - - - - - - - - - -
- ); -}; - -export default LargeEnrollmentCallout; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg deleted file mode 100644 index 71eb4f43730..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg deleted file mode 100644 index 92bfc42f8ad..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg +++ /dev/null @@ -1,667 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json index 0f13d79c6e9..9d612a83631 100644 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ b/airbyte-webapp/src/packages/cloud/locales/en.json @@ -195,21 +195,6 @@ "sidebar.billing": "Billing", "freeConnectorProgram.title": "Free Connector Program", - "freeConnectorProgram.enrollNow": "Enroll now!", - "freeConnectorProgram.enroll.description": "Enroll in the Free Connector Program to use Alpha and Beta connectors for free.", - "freeConnectorProgram.enroll.success": "Successfully enrolled in the Free Connector Program", - "freeConnectorProgram.enroll.failure": "Unable to verify that payment details were saved successfully. Please contact support for additional help.", - "freeConnectorProgram.enrollmentModal.title": "Free connector program", - "freeConnectorProgram.enrollmentModal.free": "Alpha and Beta Connectors are free while you're in the program.The whole Connection is free until both Connectors have moved into General Availability (GA)", - "freeConnectorProgram.enrollmentModal.emailNotification": "We will email you before your connection will start being charged.", - "freeConnectorProgram.enrollmentModal.cardOnFile": "When both Connectors are in GA, the Connection will no longer be free. You'll need to have a credit card on file to enroll so Airbyte can handle a Connection's transition to paid service.", - "freeConnectorProgram.enrollmentModal.unvalidatedEmailWarning": "You need to verify your email address before you can enroll in the Free Connector Program. Re-send verification email.", - "freeConnectorProgram.enrollmentModal.validationEmailConfirmation": "Verification email sent", - "freeConnectorProgram.enrollmentModal.cancelButtonText": "Cancel", - "freeConnectorProgram.enrollmentModal.enrollButtonText": "Enroll now!", - "freeConnectorProgram.enrollmentModal.unvalidatedEmailButtonText": "Resend email validation", - "freeConnectorProgram.releaseStageBadge.free": "Free", - "freeConnectorProgram.joinTheFreeConnectorProgram": "Join the Free Connector Program to use Alpha and Beta connectors for free Enroll now", "experiment.speedyConnection": "Set up your first connection in the next and get 100 additional credits for your trial", diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx index 26f8fb51a41..3fd44c99cb2 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx @@ -12,9 +12,7 @@ import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; -import { FeatureItem, useFeature } from "core/services/features"; import { links } from "core/utils/links"; -import LargeEnrollmentCallout from "packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout"; import styles from "./BillingPage.module.scss"; import { CreditsUsage } from "./components/CreditsUsage"; @@ -44,7 +42,6 @@ const StripePortalLink: React.FC = () => { }; export const BillingPage: React.FC = () => { useTrackPage(PageTrackingCodes.CREDITS); - const fcpEnabled = useFeature(FeatureItem.FreeConnectorProgram); return ( { > - {fcpEnabled && } diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx index d2d93c0a581..aa0b9e0b9ab 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx @@ -66,313 +66,275 @@ describe("useBillingPageBanners", () => { }; }); describe("enrolled in FCP", () => { - it("should show an info variant banner and no FCP materials", () => { + it("should show an info variant banner", () => { mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show FCP enrollment materials + an info variant banner", () => { + it("should an info variant banner", () => { mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("out of trial with 0 connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banners + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banners if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banners + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banners + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banners if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banners + no FCP enrollment materials if 0 credits", () => { + it("should show error banners if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); }); describe("out of trial with only eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info variant banner + no FCP banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error variant banner + no FCP banner if < 20 credits", () => { + it("should show error variant banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error variant banner + no FCP banner if user is in 0 credits state", () => { + it("should show error variant banner if user is in 0 credits state", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show FCP banner + info variant banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); it("should show FCP banner + warning variant banner if user has < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); it("should show FCP banner + error variant banner if user has 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("out of trial with only non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); }); describe("out of trial with eligible and non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + FCP enrollment materials if more than 20 credits", () => { + it("should show info banner if more than 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show warning banner + FCP enrollment materials if fewer than 20 credits", () => { + it("should show warning banner if fewer than 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show error banner + FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("credit purchased with only eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info variant banner + no FCP banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show info variant banner + no FCP banner if < 20 credits", () => { + it("should show info variant banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show info variant banner + no FCP banner if user is in 0 credits state", () => { + it("should show info variant banner if user is in 0 credits state", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show FCP banner + info variant banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show FCP banner + info variant banner if user has < 20 credits", () => { + it("should show info variant banner if user has < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show FCP banner + error variant banner if user has 0 credits", () => { + it("should show error variant banner if user has 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("credit purchased with only non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); }); diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx index 3c56c1a9dd6..2575e9abcd2 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx @@ -8,9 +8,8 @@ import { LOW_BALANCE_CREDIT_THRESHOLD } from "./components/LowCreditBalanceHint/ export const useBillingPageBanners = () => { const currentWorkspace = useCurrentWorkspace(); const cloudWorkspace = useGetCloudWorkspace(currentWorkspace.workspaceId); - const { programStatusQuery, userDidEnroll } = useFreeConnectorProgram(); - const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled, showEnrollmentUi } = - programStatusQuery.data || {}; + const { programStatusQuery } = useFreeConnectorProgram(); + const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled } = programStatusQuery.data || {}; const isNewTrialPolicyEnabled = useExperiment("billing.newTrialPolicy", false); const isPreTrial = isNewTrialPolicyEnabled @@ -40,15 +39,7 @@ export const useBillingPageBanners = () => { return "info"; }; - const calculateShowFcpBanner = () => { - if (!isEnrolled && !userDidEnroll && showEnrollmentUi && (hasEligibleConnections || isPreTrial)) { - return true; - } - return false; - }; - return { bannerVariant: calculateVariant(), - showFCPBanner: calculateShowFcpBanner(), }; }; From d9787c76d036d2f4b99c8d4aa173418c06d16330 Mon Sep 17 00:00:00 2001 From: xiaohansong Date: Wed, 30 Aug 2023 17:22:25 +0000 Subject: [PATCH 056/201] Bump helm chart version reference to 0.48.3 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 9842d7f4450..d427f9a893e 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 1d7436fc847..653f2092199 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 9101ccfe450..8716c256b06 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 6955b7637b5..d04358ad322 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index e93c6f400ba..35d224470ee 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 53689226d5f..049dddba7d4 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index c70a72efd7b..fc37406d40e 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index bcbafe8f88c..b1525b3b33e 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 8b9d10ef99f..372bb06b05e 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 9aedbdad197..8dad3ac4b2c 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 6ead9e68741..ccd1b5c5fc7 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 30000504d59..62bf04188ab 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index ba6a363cf01..05f47993bf7 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 -digest: sha256:218bc8598e8ca8cf247268a15069a87117ffb4302ec14649e05559b4de095432 -generated: "2023-08-23T23:57:17.03291188Z" + version: 0.48.3 +digest: sha256:0d9988bebcad0eb5d2b6499b87b5b182f9fb211b1f8df1a77c1fe2980b155360 +generated: "2023-08-30T17:22:11.936258038Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index e8d8668a357..305c3261462 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.2 +version: 0.48.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 + version: 0.48.3 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.2 \ No newline at end of file + version: 0.48.3 \ No newline at end of file From 291739831840887920a85d31950461980884f5b3 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Wed, 30 Aug 2023 19:59:34 +0200 Subject: [PATCH 057/201] remove error swallowing from definitionsupdater (#8608) --- .../airbyte/cron/jobs/DefinitionsUpdater.java | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java b/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java index 1e907be004d..cd68389f9aa 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java +++ b/airbyte-cron/src/main/java/io/airbyte/cron/jobs/DefinitionsUpdater.java @@ -9,13 +9,16 @@ import datadog.trace.api.Trace; import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.init.ApplyDefinitionsHelper; +import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; +import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Requires; import io.micronaut.scheduling.annotation.Scheduled; import jakarta.inject.Singleton; +import java.io.IOException; import lombok.extern.slf4j.Slf4j; /** @@ -47,23 +50,11 @@ public DefinitionsUpdater(final ApplyDefinitionsHelper applyDefinitionsHelper, @Trace(operationName = SCHEDULED_TRACE_OPERATION_NAME) @Scheduled(fixedRate = "30s", initialDelay = "1m") - void updateDefinitions() { + void updateDefinitions() throws JsonValidationException, ConfigNotFoundException, IOException { log.info("Updating definitions..."); metricClient.count(OssMetricsRegistry.CRON_JOB_RUN_BY_CRON_TYPE, 1, new MetricAttribute(MetricTags.CRON_TYPE, "definitions_updater")); - - try { - try { - applyDefinitionsHelper.apply(deploymentMode == DeploymentMode.CLOUD); - - log.info("Done applying remote connector definitions"); - } catch (final Exception e) { - log.error("Error while applying remote definitions", e); - } - - } catch (final Exception e) { - log.error("Error when retrieving remote definitions", e); - } - + applyDefinitionsHelper.apply(deploymentMode == DeploymentMode.CLOUD); + log.info("Done applying remote connector definitions"); } } From 558e44c31f58b273683e4779b750d37ac8da31c5 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 30 Aug 2023 15:37:45 -0400 Subject: [PATCH 058/201] =?UTF-8?q?Revert=20"=F0=9F=AA=9F=20Remove=20enrol?= =?UTF-8?q?lment=20materials=20for=20Free=20Connector=20Program"=20(#8616)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/hooks/cloud/freeConnectorProgram.ts | 50 ++ .../EnrollmentModal.module.scss | 53 ++ .../EnrollmentModal/EnrollmentModal.tsx | 155 ++++ .../EnrollmentModal/cards.svg | 24 + .../EnrollmentModal/connectorGrid.svg | 439 ++++++++++++ .../EnrollmentModal/free-alpha-beta-pills.svg | 34 + .../EnrollmentModal/free.svg | 13 + .../EnrollmentModal/index.ts | 1 + .../EnrollmentModal/mail.svg | 14 + .../useShowEnrollmentModal.tsx | 44 ++ .../InlineEnrollmentCallout.module.scss | 18 + .../InlineEnrollmentCallout.tsx | 52 ++ .../LargeEnrollmentCallout.module.scss | 30 + .../LargeEnrollmentCallout.tsx | 45 ++ .../connectors-badges.svg | 22 + .../longtail-connectors.svg | 667 ++++++++++++++++++ .../src/packages/cloud/locales/en.json | 15 + .../views/billing/BillingPage/BillingPage.tsx | 4 + .../useBillingPageBanners.test.tsx | 110 ++- .../BillingPage/useBillingPageBanners.tsx | 13 +- 20 files changed, 1765 insertions(+), 38 deletions(-) create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg create mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg diff --git a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts index dd1c4ab90cd..f63848448a8 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts @@ -1,21 +1,70 @@ import { useQuery } from "@tanstack/react-query"; +import { useState } from "react"; +import { useIntl } from "react-intl"; +import { useSearchParams } from "react-router-dom"; +import { useEffectOnce } from "react-use"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { FeatureItem, useFeature } from "core/services/features"; +import { pollUntil } from "core/utils/pollUntil"; +import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; +import { useNotificationService } from "hooks/services/Notification"; import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { webBackendGetFreeConnectorProgramInfoForWorkspace } from "../../generated/CloudApi"; +export const STRIPE_SUCCESS_QUERY = "fcpEnrollmentSuccess"; + export const useFreeConnectorProgram = () => { const workspaceId = useCurrentWorkspaceId(); const middlewares = useDefaultRequestMiddlewares(); const requestOptions = { middlewares }; const freeConnectorProgramEnabled = useFeature(FeatureItem.FreeConnectorProgram); + const [searchParams, setSearchParams] = useSearchParams(); + const [userDidEnroll, setUserDidEnroll] = useState(false); + const { formatMessage } = useIntl(); + const { registerNotification } = useNotificationService(); + const { trackError } = useAppMonitoringService(); + + const removeStripeSuccessQuery = () => { + const { [STRIPE_SUCCESS_QUERY]: _, ...unrelatedSearchParams } = Object.fromEntries(searchParams); + setSearchParams(unrelatedSearchParams, { replace: true }); + }; + + useEffectOnce(() => { + if (searchParams.has(STRIPE_SUCCESS_QUERY)) { + pollUntil( + () => webBackendGetFreeConnectorProgramInfoForWorkspace({ workspaceId }, requestOptions), + ({ hasPaymentAccountSaved }) => hasPaymentAccountSaved, + { intervalMs: 1000, maxTimeoutMs: 10000 } + ).then((maybeFcpInfo) => { + if (maybeFcpInfo) { + removeStripeSuccessQuery(); + setUserDidEnroll(true); + registerNotification({ + id: "fcp/enrollment-success", + text: formatMessage({ id: "freeConnectorProgram.enroll.success" }), + type: "success", + }); + } else { + trackError(new Error("Unable to confirm Free Connector Program enrollment before timeout"), { workspaceId }); + registerNotification({ + id: "fcp/enrollment-failure", + text: formatMessage({ id: "freeConnectorProgram.enroll.failure" }), + type: "error", + }); + } + }); + } + }); const programStatusQuery = useQuery(["freeConnectorProgramInfo", workspaceId], () => webBackendGetFreeConnectorProgramInfoForWorkspace({ workspaceId }, requestOptions).then( ({ hasPaymentAccountSaved, hasEligibleConnections, hasNonEligibleConnections }) => { + const userIsEligibleToEnroll = !hasPaymentAccountSaved; + return { + showEnrollmentUi: freeConnectorProgramEnabled && userIsEligibleToEnroll, isEnrolled: freeConnectorProgramEnabled && hasPaymentAccountSaved, hasEligibleConnections: freeConnectorProgramEnabled && hasEligibleConnections, hasNonEligibleConnections: freeConnectorProgramEnabled && hasNonEligibleConnections, @@ -26,5 +75,6 @@ export const useFreeConnectorProgram = () => { return { programStatusQuery, + userDidEnroll, }; }; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss new file mode 100644 index 00000000000..62aba2c1d09 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss @@ -0,0 +1,53 @@ +@use "scss/colors"; +@use "scss/variables"; + +.header { + overflow: hidden; + position: relative; + background: colors.$blue-100; + height: 120px; + border-top-left-radius: variables.$border-radius-lg; + border-top-right-radius: variables.$border-radius-lg; +} + +.headerBackgroundImageContainer { + position: absolute; + left: 50%; + transform: translateX(-50%); +} + +.pill { + z-index: 2; + + // HACK: the hardcoded top margin is to vertically center the image, which was exported + // from Figma with an off-center vertical alignment + margin-top: 2em; +} + +.contentWrapper { + width: variables.$width-modal-sm; + padding: 0 variables.$spacing-xl variables.$spacing-2xl; +} + +.contentHeader { + margin: variables.$spacing-xl 0; +} + +.iconContainer { + flex: 0 0 82px; +} + +.resendEmailLink { + text-decoration: underline; + cursor: pointer; + padding: 0; + color: colors.$dark-blue; + border: none; + background-color: transparent; + font-size: inherit; + + &:hover, + &:active { + color: colors.$blue; + } +} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx new file mode 100644 index 00000000000..cdf1d5a96d8 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx @@ -0,0 +1,155 @@ +import React, { useEffect, useRef, useState } from "react"; +import { FormattedMessage } from "react-intl"; + +import { Button } from "components/ui/Button"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; +import { Message } from "components/ui/Message"; +import { ModalFooter } from "components/ui/Modal/ModalFooter"; +import { Text } from "components/ui/Text"; + +import { STRIPE_SUCCESS_QUERY } from "core/api/cloud"; +import { StripeCheckoutSessionCreate, StripeCheckoutSessionRead } from "core/api/types/CloudApi"; + +import { ReactComponent as CardSVG } from "./cards.svg"; +import { ReactComponent as ConnectorGridSvg } from "./connectorGrid.svg"; +import styles from "./EnrollmentModal.module.scss"; +import { ReactComponent as FreeAlphaBetaPillsSVG } from "./free-alpha-beta-pills.svg"; +import { ReactComponent as FreeSVG } from "./free.svg"; +import { ReactComponent as MailSVG } from "./mail.svg"; + +interface EnrollmentModalContentProps { + closeModal: () => void; + createCheckout: (p: StripeCheckoutSessionCreate) => Promise; + workspaceId: string; + emailVerified: boolean; + sendEmailVerification: () => void; +} + +// we have to pass the email verification data and functions in as props, rather than +// directly using useAuthService(), because the modal renders outside of the +// AuthenticationProvider context. +export const EnrollmentModalContent: React.FC = ({ + closeModal, + createCheckout, + workspaceId, + emailVerified, + sendEmailVerification, +}) => { + const isMountedRef = useRef(false); + const [isLoading, setIsLoading] = useState(false); + + const startStripeCheckout = async () => { + setIsLoading(true); + // Use the current URL as a success URL but attach the STRIPE_SUCCESS_QUERY to it + const successUrl = new URL(window.location.href); + successUrl.searchParams.set(STRIPE_SUCCESS_QUERY, "true"); + const { stripeUrl } = await createCheckout({ + workspaceId, + successUrl: successUrl.href, + cancelUrl: window.location.href, + stripeMode: "setup", + }); + + // Forward to stripe as soon as we created a checkout session successfully + if (isMountedRef.current) { + window.location.assign(stripeUrl); + } + }; + + // If the user closes the modal while the request is processing, we don't want to redirect them + useEffect(() => { + isMountedRef.current = true; + return () => { + isMountedRef.current = false; + }; + }, []); + + const EnrollmentEmailVerificationWarning = () => { + const WarningContent = () => ( + + ( + + ), + }} + /> + + } + /> + ); + + return <>{!emailVerified && }; + }; + + return ( + <> + +
+ +
+ + +
+ + + + + + + + + + {content}, + p2: (content: React.ReactNode) => {content}, + }} + /> + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + ); +}; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg new file mode 100644 index 00000000000..a86f40a6581 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg new file mode 100644 index 00000000000..da8c9978e4e --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg @@ -0,0 +1,439 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg new file mode 100644 index 00000000000..6531acb626e --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg new file mode 100644 index 00000000000..55c619b2c1d --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts new file mode 100644 index 00000000000..eb0c733b211 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts @@ -0,0 +1 @@ +export * from "./useShowEnrollmentModal"; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg new file mode 100644 index 00000000000..d3c1f86a68a --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx new file mode 100644 index 00000000000..c3642c65df5 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx @@ -0,0 +1,44 @@ +import { useIntl } from "react-intl"; + +import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { useStripeCheckout } from "core/api/cloud"; +import { useAuthService } from "core/services/auth"; +import { useModalService } from "hooks/services/Modal"; +import { useNotificationService } from "hooks/services/Notification"; + +import { EnrollmentModalContent } from "./EnrollmentModal"; + +export const useShowEnrollmentModal = () => { + const { openModal, closeModal } = useModalService(); + const { mutateAsync: createCheckout } = useStripeCheckout(); + const workspaceId = useCurrentWorkspaceId(); + const { emailVerified, sendEmailVerification } = useAuthService(); + const { formatMessage } = useIntl(); + const { registerNotification } = useNotificationService(); + + const verifyEmail = () => + sendEmailVerification?.().then(() => { + registerNotification({ + id: "fcp/verify-email", + text: formatMessage({ id: "freeConnectorProgram.enrollmentModal.validationEmailConfirmation" }), + type: "info", + }); + }); + + return { + showEnrollmentModal: () => { + openModal({ + title: null, + content: () => ( + + ), + }); + }, + }; +}; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss new file mode 100644 index 00000000000..98fb1eeeeb0 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss @@ -0,0 +1,18 @@ +@use "scss/variables"; +@use "scss/colors"; + +.container { + width: 100%; +} + +.withMargin { + margin-top: variables.$spacing-lg; + margin-bottom: variables.$spacing-lg; +} + +.enrollLink { + all: unset; + text-decoration: underline; + cursor: pointer; + color: colors.$blue; +} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx new file mode 100644 index 00000000000..5eb53293429 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx @@ -0,0 +1,52 @@ +import classNames from "classnames"; +import React, { PropsWithChildren } from "react"; +import { FormattedMessage } from "react-intl"; + +import { Message } from "components/ui/Message"; +import { Text } from "components/ui/Text"; + +import { useFreeConnectorProgram } from "core/api/cloud"; + +import { useShowEnrollmentModal } from "./EnrollmentModal"; +import styles from "./InlineEnrollmentCallout.module.scss"; + +export const EnrollLink: React.FC> = ({ children }) => { + const { showEnrollmentModal } = useShowEnrollmentModal(); + + return ( + + ); +}; + +interface InlineEnrollmentCalloutProps { + withMargin?: boolean; +} + +export const InlineEnrollmentCallout: React.FC = ({ withMargin }) => { + const { userDidEnroll, programStatusQuery } = useFreeConnectorProgram(); + const { showEnrollmentUi } = programStatusQuery.data || {}; + + if (userDidEnroll || !showEnrollmentUi) { + return null; + } + return ( + + {content}, + }} + /> + + } + className={classNames(styles.container, { [styles.withMargin]: withMargin })} + /> + ); +}; + +export default InlineEnrollmentCallout; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss new file mode 100644 index 00000000000..3959159143b --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss @@ -0,0 +1,30 @@ +@use "scss/variables"; +@use "scss/colors"; + +.container { + padding: variables.$spacing-lg variables.$spacing-xl; + width: 100%; + border-radius: variables.$border-radius-md; + background-color: colors.$blue-200; + background-image: url("./longtail-connectors.svg"); + background-repeat: no-repeat; + background-position-y: center; + background-position-x: -50px; +} + +.title { + text-transform: uppercase; + color: colors.$white; +} + +.description { + color: colors.$white; +} + +.flexRow { + width: 100%; +} + +.enrollButton { + margin-left: auto; +} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx new file mode 100644 index 00000000000..161ae02dc4f --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx @@ -0,0 +1,45 @@ +import React from "react"; +import { FormattedMessage } from "react-intl"; + +import { Button } from "components/ui/Button"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Heading } from "components/ui/Heading"; +import { Text } from "components/ui/Text"; + +import { useBillingPageBanners } from "packages/cloud/views/billing/BillingPage/useBillingPageBanners"; + +import { ReactComponent as ConnectorsBadges } from "./connectors-badges.svg"; +import { useShowEnrollmentModal } from "./EnrollmentModal"; +import styles from "./LargeEnrollmentCallout.module.scss"; + +export const LargeEnrollmentCallout: React.FC = () => { + const { showEnrollmentModal } = useShowEnrollmentModal(); + const { showFCPBanner } = useBillingPageBanners(); + + if (!showFCPBanner) { + return null; + } + + return ( +
+ + + + + + + + + + + + + + +
+ ); +}; + +export default LargeEnrollmentCallout; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg new file mode 100644 index 00000000000..71eb4f43730 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg new file mode 100644 index 00000000000..92bfc42f8ad --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg @@ -0,0 +1,667 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json index 9d612a83631..0f13d79c6e9 100644 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ b/airbyte-webapp/src/packages/cloud/locales/en.json @@ -195,6 +195,21 @@ "sidebar.billing": "Billing", "freeConnectorProgram.title": "Free Connector Program", + "freeConnectorProgram.enrollNow": "Enroll now!", + "freeConnectorProgram.enroll.description": "Enroll in the Free Connector Program to use Alpha and Beta connectors for free.", + "freeConnectorProgram.enroll.success": "Successfully enrolled in the Free Connector Program", + "freeConnectorProgram.enroll.failure": "Unable to verify that payment details were saved successfully. Please contact support for additional help.", + "freeConnectorProgram.enrollmentModal.title": "Free connector program", + "freeConnectorProgram.enrollmentModal.free": "Alpha and Beta Connectors are free while you're in the program.The whole Connection is free until both Connectors have moved into General Availability (GA)", + "freeConnectorProgram.enrollmentModal.emailNotification": "We will email you before your connection will start being charged.", + "freeConnectorProgram.enrollmentModal.cardOnFile": "When both Connectors are in GA, the Connection will no longer be free. You'll need to have a credit card on file to enroll so Airbyte can handle a Connection's transition to paid service.", + "freeConnectorProgram.enrollmentModal.unvalidatedEmailWarning": "You need to verify your email address before you can enroll in the Free Connector Program. Re-send verification email.", + "freeConnectorProgram.enrollmentModal.validationEmailConfirmation": "Verification email sent", + "freeConnectorProgram.enrollmentModal.cancelButtonText": "Cancel", + "freeConnectorProgram.enrollmentModal.enrollButtonText": "Enroll now!", + "freeConnectorProgram.enrollmentModal.unvalidatedEmailButtonText": "Resend email validation", + "freeConnectorProgram.releaseStageBadge.free": "Free", + "freeConnectorProgram.joinTheFreeConnectorProgram": "Join the Free Connector Program to use Alpha and Beta connectors for free Enroll now", "experiment.speedyConnection": "Set up your first connection in the next and get 100 additional credits for your trial", diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx index 3fd44c99cb2..26f8fb51a41 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx @@ -12,7 +12,9 @@ import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; +import { FeatureItem, useFeature } from "core/services/features"; import { links } from "core/utils/links"; +import LargeEnrollmentCallout from "packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout"; import styles from "./BillingPage.module.scss"; import { CreditsUsage } from "./components/CreditsUsage"; @@ -42,6 +44,7 @@ const StripePortalLink: React.FC = () => { }; export const BillingPage: React.FC = () => { useTrackPage(PageTrackingCodes.CREDITS); + const fcpEnabled = useFeature(FeatureItem.FreeConnectorProgram); return ( { > + {fcpEnabled && } diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx index aa0b9e0b9ab..d2d93c0a581 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx @@ -66,275 +66,313 @@ describe("useBillingPageBanners", () => { }; }); describe("enrolled in FCP", () => { - it("should show an info variant banner", () => { + it("should show an info variant banner and no FCP materials", () => { mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should an info variant banner", () => { + it("should show FCP enrollment materials + an info variant banner", () => { mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("out of trial with 0 connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { + it("should show info banner + no FCP enrollment materials if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banners if < 20 credits", () => { + it("should show warning banners + no FCP enrollment materials if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner if 0 credits", () => { + it("should show error banners + no FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { + it("should show info banner + no FCP enrollment materials if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banners if < 20 credits", () => { + it("should show warning banners + no FCP enrollment materials if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banners if 0 credits", () => { + it("should show error banners + no FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(false); }); }); }); describe("out of trial with only eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { + it("should show info variant banner + no FCP banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error variant banner if < 20 credits", () => { + it("should show error variant banner + no FCP banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error variant banner if user is in 0 credits state", () => { + it("should show error variant banner + no FCP banner if user is in 0 credits state", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { + it("should show FCP banner + info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(true); }); it("should show FCP banner + warning variant banner if user has < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(true); }); it("should show FCP banner + error variant banner if user has 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("out of trial with only non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { + it("should show info banner + no FCP enrollment materials if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner if < 20 credits", () => { + it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner if 0 credits", () => { + it("should show error banner + no FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { + it("should show info banner + no FCP enrollment materials if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner if < 20 credits", () => { + it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner if 0 credits", () => { + it("should show error banner + no FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(false); }); }); }); describe("out of trial with eligible and non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { + it("should show info banner + no FCP enrollment materials if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner if < 20 credits", () => { + it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner if 0 credits", () => { + it("should show error banner + no FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner if more than 20 credits", () => { + it("should show info banner + FCP enrollment materials if more than 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(true); }); - it("should show warning banner if fewer than 20 credits", () => { + it("should show warning banner + FCP enrollment materials if fewer than 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(true); }); - it("should show error banner if 0 credits", () => { + it("should show error banner + FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("credit purchased with only eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { + it("should show info variant banner + no FCP banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show info variant banner if < 20 credits", () => { + it("should show info variant banner + no FCP banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show info variant banner if user is in 0 credits state", () => { + it("should show info variant banner + no FCP banner if user is in 0 credits state", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { + it("should show FCP banner + info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(true); }); - it("should show info variant banner if user has < 20 credits", () => { + it("should show FCP banner + info variant banner if user has < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(true); }); - it("should show error variant banner if user has 0 credits", () => { + it("should show FCP banner + error variant banner if user has 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("credit purchased with only non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { + it("should show info banner + no FCP enrollment materials if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner if < 20 credits", () => { + it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner if 0 credits", () => { + it("should show error banner + no FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { + it("should show info banner + no FCP enrollment materials if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner if < 20 credits", () => { + it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); + expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner if 0 credits", () => { + it("should show error banner + no FCP enrollment materials if 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); + expect(result.current.showFCPBanner).toEqual(false); }); }); }); diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx index 2575e9abcd2..3c56c1a9dd6 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx @@ -8,8 +8,9 @@ import { LOW_BALANCE_CREDIT_THRESHOLD } from "./components/LowCreditBalanceHint/ export const useBillingPageBanners = () => { const currentWorkspace = useCurrentWorkspace(); const cloudWorkspace = useGetCloudWorkspace(currentWorkspace.workspaceId); - const { programStatusQuery } = useFreeConnectorProgram(); - const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled } = programStatusQuery.data || {}; + const { programStatusQuery, userDidEnroll } = useFreeConnectorProgram(); + const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled, showEnrollmentUi } = + programStatusQuery.data || {}; const isNewTrialPolicyEnabled = useExperiment("billing.newTrialPolicy", false); const isPreTrial = isNewTrialPolicyEnabled @@ -39,7 +40,15 @@ export const useBillingPageBanners = () => { return "info"; }; + const calculateShowFcpBanner = () => { + if (!isEnrolled && !userDidEnroll && showEnrollmentUi && (hasEligibleConnections || isPreTrial)) { + return true; + } + return false; + }; + return { bannerVariant: calculateVariant(), + showFCPBanner: calculateShowFcpBanner(), }; }; From e089ec1ebd1c9a14f6afc23ae69ad4beee995daf Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Wed, 30 Aug 2023 13:38:39 -0700 Subject: [PATCH 059/201] Comment out webapp:validateLinks so we can continue release (#8621) --- airbyte-webapp/build.gradle | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index a335aeed378..1e010600a99 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -138,22 +138,22 @@ tasks.register("licenseCheck", PnpmTask) { // as long as the inputs have not changed outputs.upToDateWhen { true } } - -tasks.register("validateLinks", PnpmTask) { - dependsOn tasks.named("pnpmInstall") - - args = ['run', 'validate-links'] - - inputs.file 'scripts/validate-links.ts' - inputs.file 'src/core/utils/links.ts' - - // Since the output of this task depends on availability of URLs - // we never want to treat it as "up-to-date" on CI and always want to run it - // but running locally we treat it as up-to-date just depending on its inputs - outputs.upToDateWhen { - System.getenv("CI") === null - } -} +// +//tasks.register("validateLinks", PnpmTask) { +// dependsOn tasks.named("pnpmInstall") +// +// args = ['run', 'validate-links'] +// +// inputs.file 'scripts/validate-links.ts' +// inputs.file 'src/core/utils/links.ts' +// +// // Since the output of this task depends on availability of URLs +// // we never want to treat it as "up-to-date" on CI and always want to run it +// // but running locally we treat it as up-to-date just depending on its inputs +// outputs.upToDateWhen { +// System.getenv("CI") === null +// } +//} tasks.register("buildStorybook", PnpmTask) { dependsOn tasks.named("pnpmInstall") @@ -189,7 +189,7 @@ tasks.register("copyNginx", Copy) { // Those tasks should be run as part of the "check" task tasks.named("check") { - dependsOn tasks.named("validateLinks"), tasks.named("licenseCheck"), tasks.named("test") + dependsOn /*tasks.named("validateLinks"),*/ tasks.named("licenseCheck"), tasks.named("test") } From 45d0c8150e82450d388b6b7cc7a81807c0aab0ad Mon Sep 17 00:00:00 2001 From: xiaohansong Date: Wed, 30 Aug 2023 21:10:41 +0000 Subject: [PATCH 060/201] Bump Airbyte version from 0.50.23 to 0.50.24 --- .bumpversion.cfg | 2 +- .env | 2 +- airbyte-connector-builder-server/Dockerfile | 4 ++-- airbyte-container-orchestrator/Dockerfile | 2 +- airbyte-proxy/Dockerfile | 2 +- airbyte-server/Dockerfile | 2 +- airbyte-webapp/package.json | 2 +- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.yaml | 2 +- charts/airbyte/README.md | 20 +++++++++---------- 21 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 023eabca9a5..5bf8e02886e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.50.23 +current_version = 0.50.24 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index 2169a04ebe1..de28947e9be 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.50.23 +VERSION=0.50.24 # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index 99e44391147..79e04ff511d 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -10,7 +10,7 @@ ENV PIP=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/pip COPY requirements.txt requirements.txt RUN ${PIP} install -r requirements.txt -ARG VERSION=0.50.23 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-connector-builder-server ENV VERSION ${VERSION} @@ -23,5 +23,5 @@ ADD airbyte-app.tar /app # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] -LABEL io.airbyte.version=0.50.23 +LABEL io.airbyte.version=0.50.24 LABEL io.airbyte.name=airbyte/connector-builder-server \ No newline at end of file diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index f6ba103be2e..633d9fae785 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -10,7 +10,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.50.23 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-proxy/Dockerfile b/airbyte-proxy/Dockerfile index 177ea6521e2..4542e765f64 100644 --- a/airbyte-proxy/Dockerfile +++ b/airbyte-proxy/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:latest -ARG VERSION=0.50.23 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-proxy ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 6d684fde080..4bdb76fb63e 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,7 +3,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 5005 -ARG VERSION=0.50.23 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 5fe1a9ffcd6..773ec7f1dca 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.50.23", + "version": "0.50.24", "private": true, "engines": { "node": "18.15.0", diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index d427f9a893e..4704907211c 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 653f2092199..3a0e77fc264 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 8716c256b06..a138e8023ff 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index d04358ad322..25b659be96a 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 35d224470ee..25c932f8143 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 049dddba7d4..3b0e6f7947c 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index fc37406d40e..de7c526f27a 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index b1525b3b33e..60c77eae12d 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 372bb06b05e..10a31932e23 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 8dad3ac4b2c..faa05dd3bc9 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index ccd1b5c5fc7..fde817f71a4 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 62bf04188ab..93e39fe0667 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 305c3261462..bf327a22d50 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.3 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.23 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 48933dc0120..e87bc3d30de 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,6 @@ # airbyte -![Version: 0.50.23](https://img.shields.io/badge/Version-0.50.23-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.50.23](https://img.shields.io/badge/AppVersion-0.50.23-informational?style=flat-square) +![Version: 0.50.24](https://img.shields.io/badge/Version-0.50.24-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.50.24](https://img.shields.io/badge/AppVersion-0.50.24-informational?style=flat-square) Helm chart to deploy airbyte @@ -8,15 +8,15 @@ Helm chart to deploy airbyte | Repository | Name | Version | |------------|------|---------| -| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | cron | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | metrics | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | server | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | temporal | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | webapp | 0.50.23 | -| https://airbytehq.github.io/helm-charts/ | worker | 0.50.23 | +| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | cron | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | metrics | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | server | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | temporal | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | webapp | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | worker | 0.50.24 | | https://charts.bitnami.com/bitnami | common | 1.x.x | ## Values From 728b1648b5b5dd618d4faed0659529e9b22f67c0 Mon Sep 17 00:00:00 2001 From: xiaohansong Date: Wed, 30 Aug 2023 21:26:22 +0000 Subject: [PATCH 061/201] Bump helm chart version reference to 0.48.4 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 4704907211c..0e39d6a0096 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 3a0e77fc264..26589ed8573 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index a138e8023ff..4f1f7de0ff1 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 25b659be96a..e1864712dbc 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 25c932f8143..e8065bbbb11 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 3b0e6f7947c..f72cc8762b7 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index de7c526f27a..5700ab9e81a 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 60c77eae12d..b404497c300 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 10a31932e23..0a52367e813 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index faa05dd3bc9..b5f9a09a85b 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index fde817f71a4..e3c7501a97f 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 93e39fe0667..32c4bd27453 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 05f47993bf7..030b4cbf57f 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 -digest: sha256:0d9988bebcad0eb5d2b6499b87b5b182f9fb211b1f8df1a77c1fe2980b155360 -generated: "2023-08-30T17:22:11.936258038Z" + version: 0.48.4 +digest: sha256:6b96b3584fa1574e01e37d11686769482028c67bb9e032610a93473fb61f3602 +generated: "2023-08-30T21:26:08.620169815Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index bf327a22d50..8a3ced9c0da 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.3 +version: 0.48.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 + version: 0.48.4 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.3 \ No newline at end of file + version: 0.48.4 \ No newline at end of file From 5cb7fc9d59f2ecb356e1c820f8ee6cd41e33dbe0 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 30 Aug 2023 15:02:05 -0700 Subject: [PATCH 062/201] Increase timeout (#8624) --- .../java/io/airbyte/workers/config/ActivityBeanFactory.java | 2 +- airbyte-workers/src/main/resources/application.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java index 53e51dbed39..d51e8888cb9 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/ActivityBeanFactory.java @@ -137,7 +137,7 @@ public ActivityOptions checkActivityOptions(@Property(name = "airbyte.activity.c @Singleton @Named("discoveryActivityOptions") public ActivityOptions discoveryActivityOptions(@Property(name = "airbyte.activity.discovery-timeout", - defaultValue = "5") final Integer discoveryTimeoutMinutes) { + defaultValue = "30") final Integer discoveryTimeoutMinutes) { return ActivityOptions.newBuilder() .setScheduleToCloseTimeout(Duration.ofMinutes(discoveryTimeoutMinutes)) .setRetryOptions(TemporalUtils.NO_RETRY) diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index f8194ee2fd3..f07dace84be 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -21,7 +21,7 @@ airbyte: max-delay: ${ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS:600} max-timeout: ${ACTIVITY_MAX_TIMEOUT_SECOND:120} check-timeout: ${ACTIVITY_CHECK_TIMEOUT:5} - discovery-timeout: ${ACTIVITY_DISCOVERY_TIMEOUT:5} + discovery-timeout: ${ACTIVITY_DISCOVERY_TIMEOUT:30} acceptance: test: enabled: ${ACCEPTANCE_TEST_ENABLED:false} From 648b44f346de1d8daebbf658ae62e08bac2fc320 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Wed, 30 Aug 2023 15:03:28 -0700 Subject: [PATCH 063/201] Mark DestroyWorflowThreadError as non error from traces (#8619) --- .../tracing/TemporalSdkInterceptor.java | 30 +++++++++++++++- .../tracing/TemporalSdkInterceptorTest.java | 35 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/tracing/TemporalSdkInterceptor.java b/airbyte-workers/src/main/java/io/airbyte/workers/tracing/TemporalSdkInterceptor.java index d8009a0f8e6..2b35cfcec6a 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/tracing/TemporalSdkInterceptor.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/tracing/TemporalSdkInterceptor.java @@ -43,6 +43,11 @@ public class TemporalSdkInterceptor implements TraceInterceptor { */ static final String ERROR_MESSAGE_TAG = DDTags.ERROR_MSG; + /** + * The {@code error.type} tag name. + */ + static final String ERROR_TYPE_TAG = "error.type"; + /** * Error message tag key name that contains the Temporal exit error message. */ @@ -53,6 +58,11 @@ public class TemporalSdkInterceptor implements TraceInterceptor { */ static final String EXIT_ERROR_MESSAGE = "exit"; + /** + * Temporal destroy workflow thread error type. + */ + static final String DESTROY_WORKFLOW_ERROR_TYPE = "DestroyWorkflowThreadError"; + @Override public Collection onTraceComplete(final Collection trace) { final var filtered = new ArrayList(); @@ -66,7 +76,7 @@ public Collection onTraceComplete(final Collection Date: Wed, 30 Aug 2023 15:17:28 -0700 Subject: [PATCH 064/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Fix=20Upg?= =?UTF-8?q?rade=20All=20connectors=20behavior=20(#8558)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/ui/Message/Message.tsx | 4 +- .../src/components/ui/Toast/Toast.module.scss | 5 ++ .../src/components/ui/Toast/Toast.tsx | 1 + .../src/hooks/services/useConnector.tsx | 68 +++++++++++++----- airbyte-webapp/src/locales/en.json | 5 +- .../pages/ConnectorsPage/DestinationsPage.tsx | 72 +++++++++++-------- .../pages/ConnectorsPage/SourcesPage.tsx | 70 ++++++++++-------- .../components/ConnectorsView.tsx | 5 +- .../components/ConnectorsViewContext.tsx | 1 - .../components/UpgradeAllButton.module.scss | 3 - .../components/UpgradeAllButton.tsx | 61 +++++----------- .../components/VersionCell/VersionCell.tsx | 49 +++++-------- .../connector/DestinationDefinitionService.ts | 5 ++ .../connector/SourceDefinitionService.ts | 5 ++ 14 files changed, 194 insertions(+), 160 deletions(-) delete mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.module.scss diff --git a/airbyte-webapp/src/components/ui/Message/Message.tsx b/airbyte-webapp/src/components/ui/Message/Message.tsx index dc0db610984..465c3841aad 100644 --- a/airbyte-webapp/src/components/ui/Message/Message.tsx +++ b/airbyte-webapp/src/components/ui/Message/Message.tsx @@ -24,6 +24,7 @@ export interface MessageProps { "data-testid"?: string; hideIcon?: boolean; iconOverride?: keyof typeof ICON_MAPPING; + textClassName?: string; } const ICON_MAPPING = { @@ -65,6 +66,7 @@ export const Message: React.FC> = ({ childrenClassName, children, iconOverride, + textClassName, }) => { const mainMessage = (
> = ({ />
)} -
+
{text && {text}} {secondaryText && ( diff --git a/airbyte-webapp/src/components/ui/Toast/Toast.module.scss b/airbyte-webapp/src/components/ui/Toast/Toast.module.scss index 312faf5835b..9d9c14183d5 100644 --- a/airbyte-webapp/src/components/ui/Toast/Toast.module.scss +++ b/airbyte-webapp/src/components/ui/Toast/Toast.module.scss @@ -38,3 +38,8 @@ width: 0; } } + +.toastText { + max-height: 150px; + overflow-y: auto; +} diff --git a/airbyte-webapp/src/components/ui/Toast/Toast.tsx b/airbyte-webapp/src/components/ui/Toast/Toast.tsx index 17eeeeb405c..1c804d45a0d 100644 --- a/airbyte-webapp/src/components/ui/Toast/Toast.tsx +++ b/airbyte-webapp/src/components/ui/Toast/Toast.tsx @@ -16,6 +16,7 @@ export const Toast: React.FC = ({ timeout, ...props }) => { className={classNames(props.className, styles.toastContainer, { [styles["toastContainer--timeout"]]: timeout, })} + textClassName={styles.toastText} />
); diff --git a/airbyte-webapp/src/hooks/services/useConnector.tsx b/airbyte-webapp/src/hooks/services/useConnector.tsx index 24fc361aebd..0676acddbba 100644 --- a/airbyte-webapp/src/hooks/services/useConnector.tsx +++ b/airbyte-webapp/src/hooks/services/useConnector.tsx @@ -1,5 +1,6 @@ import { useMutation } from "@tanstack/react-query"; import { useMemo } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; import { ConnectionConfiguration } from "area/connector/types"; import { useCurrentWorkspaceId } from "area/workspace/utils"; @@ -20,14 +21,40 @@ import { import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { useInitService } from "services/useInitService"; +import { useAppMonitoringService } from "./AppMonitoringService"; +import { useNotificationService } from "./Notification"; import { CheckConnectionRead } from "../../core/request/AirbyteClient"; export const useUpdateAllConnectors = (connectorType: "sources" | "destinations") => { + const { formatMessage } = useIntl(); const workspaceId = useCurrentWorkspaceId(); const { updateAllSourceVersions } = useUpdateSourceDefinitions(); const { updateAllDestinationVersions } = useUpdateDestinationDefinitions(); - return useMutation(["updateAllConnectors", workspaceId], async () => - connectorType === "sources" ? updateAllSourceVersions() : updateAllDestinationVersions() + const { registerNotification } = useNotificationService(); + const { trackError } = useAppMonitoringService(); + + return useMutation( + ["updateAllConnectors", workspaceId], + async () => (connectorType === "sources" ? updateAllSourceVersions() : updateAllDestinationVersions()), + { + onSuccess: () => { + registerNotification({ + id: `workspace.connectorUpdateSuccess.${connectorType}.${workspaceId}`, + text: , + type: "success", + }); + }, + onError: (error: Error) => { + trackError(error); + registerNotification({ + id: `workspace.connectorUpdateError.${connectorType}.${workspaceId}`, + text: + formatMessage({ id: "admin.upgradeAll.error" }, { type: connectorType }) + + (error.message ? `: ${error.message}` : ""), + type: "error", + }); + }, + } ); }; @@ -36,19 +63,23 @@ export const useUpdateSourceDefinitions = () => { const { sourceDefinitions: latestSourceDefinitions } = useLatestSourceDefinitionList(); const { mutateAsync: updateSourceDefinition } = useUpdateSourceDefinition(); - const newSourceDefinitions = useMemo( + const sourceDefinitionsWithUpdates = useMemo( () => - latestSourceDefinitions.filter( - (latestDefinition) => - sourceDefinitions.find((definition) => definition.sourceDefinitionId === latestDefinition.sourceDefinitionId) - ?.dockerImageTag !== latestDefinition.dockerImageTag - ), + latestSourceDefinitions.filter((latestDefinition) => { + const matchingExistingDefinition = sourceDefinitions.find( + (definition) => definition.sourceDefinitionId === latestDefinition.sourceDefinitionId + ); + return ( + matchingExistingDefinition !== undefined && + matchingExistingDefinition.dockerImageTag !== latestDefinition.dockerImageTag + ); + }), [sourceDefinitions, latestSourceDefinitions] ); const updateAllSourceVersions = async () => { await Promise.all( - newSourceDefinitions?.map((item) => + sourceDefinitionsWithUpdates?.map((item) => updateSourceDefinition({ sourceDefinitionId: item.sourceDefinitionId, dockerImageTag: item.dockerImageTag, @@ -65,20 +96,23 @@ export const useUpdateDestinationDefinitions = () => { const { destinationDefinitions: latestDestinationDefinitions } = useLatestDestinationDefinitionList(); const { mutateAsync: updateDestinationDefinition } = useUpdateDestinationDefinition(); - const newDestinationDefinitions = useMemo( + const newDestinationDefinitionsWithUpdates = useMemo( () => - latestDestinationDefinitions.filter( - (latestDefinition) => - destinationDefinitions.find( - (definition) => definition.destinationDefinitionId === latestDefinition.destinationDefinitionId - )?.dockerImageTag !== latestDefinition.dockerImageTag - ), + latestDestinationDefinitions.filter((latestDefinition) => { + const matchingExistingDefinition = destinationDefinitions.find( + (definition) => definition.destinationDefinitionId === latestDefinition.destinationDefinitionId + ); + return ( + matchingExistingDefinition !== undefined && + matchingExistingDefinition.dockerImageTag !== latestDefinition.dockerImageTag + ); + }), [destinationDefinitions, latestDestinationDefinitions] ); const updateAllDestinationVersions = async () => { await Promise.all( - newDestinationDefinitions?.map((item) => + newDestinationDefinitionsWithUpdates?.map((item) => updateDestinationDefinition({ destinationDefinitionId: item.destinationDefinitionId, dockerImageTag: item.dockerImageTag, diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 25a78f10f25..356c792ffb0 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -694,7 +694,10 @@ "admin.downloadServerLogs": "Download Server Logs", "admin.downloadSchedulerLogs": "Download Scheduler Logs", "admin.upgradeAll": "Upgrade all", - "admin.upgraded": "Upgraded!", + "admin.upgradeAll.success": "All {type} upgraded successfully", + "admin.upgradeAll.error": "Error upgrading all {type}", + "admin.upgradeConnector.success": "Successfully upgraded {name} to version {version}", + "admin.upgradeConnector.error": "Error upgrading {name} to version {version}", "settings.userSettings": "User settings", "settings.workspaceSettings": "Workspace settings", diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx index 94f767a8eb1..eed312ba69c 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx @@ -1,9 +1,10 @@ -import React, { useCallback, useMemo, useRef, useState } from "react"; -import { useIntl } from "react-intl"; +import React, { useCallback, useMemo, useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; import { DestinationDefinitionRead } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useAvailableDestinationDefinitions } from "hooks/domain/connector/useAvailableDestinationDefinitions"; +import { useNotificationService } from "hooks/services/Notification"; import { useUpdateDestinationDefinition } from "services/connector/DestinationDefinitionService"; import ConnectorsView from "./components/ConnectorsView"; @@ -16,13 +17,20 @@ const DestinationsPage: React.FC = () => { const destinationDefinitions = useAvailableDestinationDefinitions(); const { destinations } = useDestinationList(); - const [feedbackList, setFeedbackList] = useState>({}); - const feedbackListRef = useRef(feedbackList); - feedbackListRef.current = feedbackList; - const { mutateAsync: updateDestinationDefinition } = useUpdateDestinationDefinition(); const [updatingDefinitionId, setUpdatingDefinitionId] = useState(); + const { registerNotification } = useNotificationService(); + + const idToDestinationDefinition = useMemo( + () => + destinationDefinitions.reduce((map, destinationDefinition) => { + map.set(destinationDefinition.destinationDefinitionId, destinationDefinition); + return map; + }, new Map()), + [destinationDefinitions] + ); + const onUpdateVersion = useCallback( async ({ id, version }: { id: string; version: string }) => { try { @@ -31,34 +39,43 @@ const DestinationsPage: React.FC = () => { destinationDefinitionId: id, dockerImageTag: version, }); - setFeedbackList({ ...feedbackListRef.current, [id]: "success" }); - } catch (e) { - const messageId = e.status === 422 ? "form.imageCannotFound" : "form.someError"; - setFeedbackList({ - ...feedbackListRef.current, - [id]: formatMessage({ id: messageId }), + registerNotification({ + id: `destination.update.success.${id}.${version}`, + text: ( + + ), + type: "success", + }); + } catch (error) { + registerNotification({ + id: `destination.update.error.${id}.${version}`, + text: + formatMessage( + { id: "admin.upgradeConnector.error" }, + { name: idToDestinationDefinition.get(id)?.name, version } + ) + (error.message ? `: ${error.message}` : ""), + type: "error", }); } finally { setUpdatingDefinitionId(undefined); } }, - [formatMessage, updateDestinationDefinition] + [formatMessage, idToDestinationDefinition, registerNotification, updateDestinationDefinition] ); - const usedDestinationDefinitions = useMemo(() => { - const destinationDefinitionMap = new Map(); - destinations.forEach((destination) => { - const destinationDefinition = destinationDefinitions.find( - (destinationDefinition) => destinationDefinition.destinationDefinitionId === destination.destinationDefinitionId - ); - - if (destinationDefinition) { - destinationDefinitionMap.set(destinationDefinition.destinationDefinitionId, destinationDefinition); - } - }); - - return Array.from(destinationDefinitionMap.values()).sort((a, b) => a.name.localeCompare(b.name)); - }, [destinations, destinationDefinitions]); + const usedDestinationDefinitions: DestinationDefinitionRead[] = useMemo(() => { + const usedDestinationDefinitionIds = new Set( + destinations.map((destination) => destination.destinationDefinitionId) + ); + return destinationDefinitions + .filter((destinationDefinition) => + usedDestinationDefinitionIds.has(destinationDefinition.destinationDefinitionId) + ) + .sort((a, b) => a.name.localeCompare(b.name)); + }, [destinationDefinitions, destinations]); return ( { usedConnectorsDefinitions={usedDestinationDefinitions} connectorsDefinitions={destinationDefinitions} updatingDefinitionId={updatingDefinitionId} - feedbackList={feedbackList} /> ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx index 917eb2aad9c..7a670e69302 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx @@ -1,10 +1,11 @@ -import React, { useCallback, useMemo, useRef, useState } from "react"; -import { useIntl } from "react-intl"; +import React, { useCallback, useMemo, useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; import { useListBuilderProjects } from "core/api"; import { SourceDefinitionRead } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useAvailableSourceDefinitions } from "hooks/domain/connector/useAvailableSourceDefinitions"; +import { useNotificationService } from "hooks/services/Notification"; import { useSourceList } from "hooks/services/useSourceHook"; import { useUpdateSourceDefinition } from "services/connector/SourceDefinitionService"; @@ -13,10 +14,6 @@ import ConnectorsView, { ConnectorsViewProps } from "./components/ConnectorsView const SourcesPage: React.FC = () => { useTrackPage(PageTrackingCodes.SETTINGS_SOURCE); - const [feedbackList, setFeedbackList] = useState>({}); - const feedbackListRef = useRef(feedbackList); - feedbackListRef.current = feedbackList; - const { formatMessage } = useIntl(); const { sources } = useSourceList(); const sourceDefinitions = useAvailableSourceDefinitions(); @@ -24,6 +21,17 @@ const SourcesPage: React.FC = () => { const { mutateAsync: updateSourceDefinition } = useUpdateSourceDefinition(); const [updatingDefinitionId, setUpdatingDefinitionId] = useState(); + const { registerNotification } = useNotificationService(); + + const idToSourceDefinition = useMemo( + () => + sourceDefinitions.reduce((map, sourceDefinition) => { + map.set(sourceDefinition.sourceDefinitionId, sourceDefinition); + return map; + }, new Map()), + [sourceDefinitions] + ); + const onUpdateVersion = useCallback( async ({ id, version }: { id: string; version: string }) => { try { @@ -32,34 +40,39 @@ const SourcesPage: React.FC = () => { sourceDefinitionId: id, dockerImageTag: version, }); - setFeedbackList({ ...feedbackListRef.current, [id]: "success" }); - } catch (e) { - const messageId = e.status === 422 ? "form.imageCannotFound" : "form.someError"; - setFeedbackList({ - ...feedbackListRef.current, - [id]: formatMessage({ id: messageId }), + registerNotification({ + id: `source.update.success.${id}.${version}`, + text: ( + + ), + type: "success", + }); + } catch (error) { + registerNotification({ + id: `source.update.error.${id}.${version}`, + text: + formatMessage( + { id: "admin.upgradeConnector.error" }, + { name: idToSourceDefinition.get(id)?.name, version } + ) + (error.message ? `: ${error.message}` : ""), + type: "error", }); } finally { setUpdatingDefinitionId(undefined); } }, - [formatMessage, updateSourceDefinition] + [formatMessage, idToSourceDefinition, registerNotification, updateSourceDefinition] ); - const usedSourcesDefinitions: SourceDefinitionRead[] = useMemo(() => { - const sourceDefinitionMap = new Map(); - sources.forEach((source) => { - const sourceDefinition = sourceDefinitions.find( - (sourceDefinition) => sourceDefinition.sourceDefinitionId === source.sourceDefinitionId - ); - - if (sourceDefinition) { - sourceDefinitionMap.set(source.sourceDefinitionId, sourceDefinition); - } - }); - - return Array.from(sourceDefinitionMap.values()).sort((a, b) => a.name.localeCompare(b.name)); - }, [sources, sourceDefinitions]); + const usedSourceDefinitions: SourceDefinitionRead[] = useMemo(() => { + const usedSourceDefinitionIds = new Set(sources.map((source) => source.sourceDefinitionId)); + return sourceDefinitions + .filter((sourceDefinition) => usedSourceDefinitionIds.has(sourceDefinition.sourceDefinitionId)) + .sort((a, b) => a.name.localeCompare(b.name)); + }, [sourceDefinitions, sources]); const ConnectorsViewComponent = WithBuilderProjects; @@ -67,9 +80,8 @@ const SourcesPage: React.FC = () => { ); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx index bf36c492742..67bd2d272d9 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx @@ -30,7 +30,6 @@ export interface ConnectorsViewProps { connectorsDefinitions: SourceDefinitionRead[] | DestinationDefinitionRead[]; updatingDefinitionId?: string; onUpdateVersion: (values: ConnectorVersionFormValues) => Promise; - feedbackList: Record; connectorBuilderProjects?: BuilderProject[]; } @@ -57,7 +56,6 @@ const columnHelper = createColumnHelper(); const ConnectorsView: React.FC = ({ type, onUpdateVersion, - feedbackList, usedConnectorsDefinitions, updatingDefinitionId, connectorsDefinitions, @@ -161,9 +159,8 @@ const ConnectorsView: React.FC = ({ setUpdatingAll: setUpdatingAllConnectors, updatingAll: updatingAllConnectors, updatingDefinitionId, - feedbackList, }), - [feedbackList, updatingDefinitionId, updatingAllConnectors] + [updatingDefinitionId, updatingAllConnectors] ); const usedDefinitionColumns = useMemo( diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsViewContext.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsViewContext.tsx index c3bf6d91a85..a769605c499 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsViewContext.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsViewContext.tsx @@ -4,7 +4,6 @@ interface Context { setUpdatingAll: (isUpdating: boolean) => void; updatingAll: boolean; updatingDefinitionId?: string; - feedbackList: Record; } export const useUpdatingState = (): Context => { diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.module.scss deleted file mode 100644 index 38388dba1e4..00000000000 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.module.scss +++ /dev/null @@ -1,3 +0,0 @@ -.updateButton { - min-width: 120px; -} diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx index e30fbf186cc..7d442a8c718 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/UpgradeAllButton.tsx @@ -2,36 +2,12 @@ import { faRedoAlt } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React from "react"; import { FormattedMessage } from "react-intl"; -import styled from "styled-components"; import { Button } from "components/ui/Button"; import { useGetConnectorsOutOfDate, useUpdateAllConnectors } from "hooks/services/useConnector"; import { useUpdatingState } from "./ConnectorsViewContext"; -import styles from "./UpgradeAllButton.module.scss"; - -const TryArrow = styled(FontAwesomeIcon)` - margin: 0 10px -1px 0; - font-size: 14px; -`; - -const UpdateButtonContent = styled.div` - position: relative; - display: inline-block; - margin-left: 5px; -`; - -const ErrorBlock = styled.div` - color: ${({ theme }) => theme.dangerColor}; - font-size: 11px; - position: absolute; - font-weight: normal; - bottom: -17px; - line-height: 11px; - right: 0; - left: -46px; -`; interface UpdateAllButtonProps { connectorType: "sources" | "destinations"; @@ -42,33 +18,28 @@ const UpgradeAllButton: React.FC = ({ connectorType }) => const { hasNewSourceVersion, hasNewDestinationVersion } = useGetConnectorsOutOfDate(); const hasNewVersion = connectorType === "sources" ? hasNewSourceVersion : hasNewDestinationVersion; - const { mutateAsync, isError, isLoading, isSuccess } = useUpdateAllConnectors(connectorType); + const { mutateAsync, isLoading } = useUpdateAllConnectors(connectorType); const handleUpdateAllConnectors = async () => { // Since we want to display the loading state on each connector row that is being updated, we "share" the react-query loading state via context here - setUpdatingAll(true); - await mutateAsync(); - setUpdatingAll(false); + try { + setUpdatingAll(true); + await mutateAsync(); + } finally { + setUpdatingAll(false); + } }; return ( - - {isError && ( - - - - )} - - + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx index b2a447b7030..831ddba6863 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx @@ -5,14 +5,12 @@ import { useIntl } from "react-intl"; import * as yup from "yup"; import { Form, FormControl } from "components/forms"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { FlexContainer } from "components/ui/Flex"; import { ReleaseStage } from "core/request/AirbyteClient"; import { SubmissionButton } from "./SubmissionButton"; import styles from "./VersionCell.module.scss"; -import { VersionChangeResult } from "./VersionChangeResult"; -import { useUpdatingState } from "../ConnectorsViewContext"; const versionCellFormSchema = yup.object().shape({ id: yup.string().trim().required("form.empty.error"), @@ -38,38 +36,30 @@ export const VersionCell: React.FC = ({ currentVersion, latestVersion, releaseStage, -}) => { - const { feedbackList } = useUpdatingState(); - const feedback = feedbackList[connectorDefinitionId]; - - return ( - - defaultValues={{ - id: connectorDefinitionId, - version: latestVersion || currentVersion, - }} - schema={versionCellFormSchema} - onSubmit={onChange} - > - - - ); -}; +}) => ( + + defaultValues={{ + id: connectorDefinitionId, + version: latestVersion || currentVersion, + }} + schema={versionCellFormSchema} + onSubmit={onChange} + > + + +); const VersionFormContent = ({ - feedback, releaseStage, connectorDefinitionId, currentVersion, latestVersion, }: { - feedback: string; releaseStage?: ReleaseStage; connectorDefinitionId: string; currentVersion: string; @@ -87,9 +77,6 @@ const VersionFormContent = ({ return ( - - -
{ export const useUpdateDestinationDefinition = () => { const service = useGetDestinationDefinitionService(); const queryClient = useQueryClient(); + const { trackError } = useAppMonitoringService(); return useMutation< DestinationDefinitionRead, @@ -135,5 +137,8 @@ export const useUpdateDestinationDefinition = () => { queryClient.invalidateQueries(connectorDefinitionKeys.count()); }, + onError: (error: Error) => { + trackError(error); + }, }); }; diff --git a/airbyte-webapp/src/services/connector/SourceDefinitionService.ts b/airbyte-webapp/src/services/connector/SourceDefinitionService.ts index f04ff424cdd..2586ceb29e7 100644 --- a/airbyte-webapp/src/services/connector/SourceDefinitionService.ts +++ b/airbyte-webapp/src/services/connector/SourceDefinitionService.ts @@ -6,6 +6,7 @@ import { useConfig } from "config"; import { useSuspenseQuery } from "core/api"; import { SourceDefinitionService } from "core/domain/connector/SourceDefinitionService"; import { isDefined } from "core/utils/common"; +import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { useInitService } from "services/useInitService"; @@ -123,6 +124,7 @@ export const useCreateSourceDefinition = () => { export const useUpdateSourceDefinition = () => { const service = useGetSourceDefinitionService(); const queryClient = useQueryClient(); + const { trackError } = useAppMonitoringService(); return useMutation< SourceDefinitionRead, @@ -148,5 +150,8 @@ export const useUpdateSourceDefinition = () => { queryClient.invalidateQueries(connectorDefinitionKeys.count()); }, + onError: (error: Error) => { + trackError(error); + }, }); }; From 442e6460c6b7f5ed4a186986828feff40eab5ad2 Mon Sep 17 00:00:00 2001 From: Ryan Br Date: Wed, 30 Aug 2023 15:36:36 -0700 Subject: [PATCH 065/201] Add metrics for processing state messages from the source/destination. (#8615) --- .../general/ReplicationWorkerFactory.java | 2 +- .../general/ReplicationWorkerHelper.java | 34 +++++++++++++++++++ .../metrics/lib/OssMetricsRegistry.java | 6 ++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index 267a5da3b6a..f1bf177b949 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -138,7 +138,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, final var airbyteDestination = airbyteIntegrationLauncherFactory.createAirbyteDestination(destinationLauncherConfig, syncInput.getSyncResourceRequirements(), syncInput.getCatalog()); - // TODO MetricClient should be injectable + // TODO MetricClient should be injectable (please) MetricClientFactory.initialize(MetricEmittingApps.WORKER); final MetricClient metricClient = MetricClientFactory.getMetricClient(); final WorkerMetricReporter metricReporter = new WorkerMetricReporter(metricClient, sourceLauncherConfig.getDockerImage()); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java index b520ba18be8..def3e58da2c 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java @@ -21,6 +21,11 @@ import io.airbyte.config.SyncStats; import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.config.WorkerSourceConfig; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteTraceMessage; @@ -70,6 +75,7 @@ class ReplicationWorkerHelper { private final FieldSelector fieldSelector; private final AirbyteMapper mapper; private final MessageTracker messageTracker; + private final MetricClient metricClient; private final SyncPersistence syncPersistence; private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; private final ThreadedTimeTracker timeTracker; @@ -79,6 +85,7 @@ class ReplicationWorkerHelper { private ReplicationContext replicationContext = null; private ReplicationFeatureFlags replicationFeatureFlags = null; // NOPMD - keeping this as a placeholder private WorkerDestinationConfig destinationConfig = null; + private MetricAttribute[] metricAttrs = new MetricAttribute[0]; // We expect the number of operations on failures to be low, so synchronizedList should be // performant enough. @@ -104,6 +111,7 @@ public ReplicationWorkerHelper( this.timeTracker = timeTracker; this.onReplicationRunning = onReplicationRunning; this.recordsRead = 0L; + this.metricClient = MetricClientFactory.getMetricClient(); } public void markCancelled() { @@ -118,6 +126,7 @@ public void initialize(final ReplicationContext replicationContext, final Replic this.replicationContext = replicationContext; this.replicationFeatureFlags = replicationFeatureFlags; this.timeTracker.trackReplicationStartTime(); + this.metricAttrs = toConnectionAttrs(replicationContext); } public void startDestination(final AirbyteDestination destination, final StandardSyncInput syncInput, final Path jobRoot) { @@ -219,6 +228,10 @@ AirbyteMessage internalProcessMessageFromSource(final AirbyteMessage sourceRawMe FileUtils.byteCountToDisplaySize(messageTracker.getSyncStatsTracker().getTotalBytesEmitted())); } + if (sourceRawMessage.getType() == Type.STATE) { + metricClient.count(OssMetricsRegistry.STATE_PROCESSED_FROM_SOURCE, 1, metricAttrs); + } + return sourceRawMessage; } @@ -250,6 +263,8 @@ void internalProcessMessageFromDestination(final AirbyteMessage destinationRawMe messageTracker.acceptFromDestination(destinationRawMessage); if (destinationRawMessage.getType() == Type.STATE) { syncPersistence.persist(replicationContext.connectionId(), destinationRawMessage.getState()); + + metricClient.count(OssMetricsRegistry.STATE_PROCESSED_FROM_DESTINATION, 1, metricAttrs); } if (shouldPublishMessage(destinationRawMessage)) { @@ -390,4 +405,23 @@ static FailureReason getFailureReason(final Throwable ex, final long jobId, fina } } + private MetricAttribute[] toConnectionAttrs(final ReplicationContext ctx) { + if (ctx == null) { + return new MetricAttribute[0]; + } + + final var attrs = new ArrayList(); + if (ctx.connectionId() != null) { + attrs.add(new MetricAttribute(MetricTags.CONNECTION_ID, ctx.connectionId().toString())); + } + if (ctx.jobId() != null) { + attrs.add(new MetricAttribute(MetricTags.JOB_ID, ctx.jobId().toString())); + } + if (ctx.attempt() != null) { + attrs.add(new MetricAttribute(MetricTags.ATTEMPT_NUMBER, ctx.attempt().toString())); + } + + return attrs.toArray(new MetricAttribute[0]); + } + } diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java index 520649e65f9..74d27ab29e2 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java @@ -273,6 +273,12 @@ public enum OssMetricsRegistry implements MetricsRegistry { "state_timestamp_metric_tracker_error", "number of syncs where the state timestamp metric tracker ran out of memory or " + "was unable to match destination state message to source state message"), + STATE_PROCESSED_FROM_DESTINATION(MetricEmittingApps.WORKER, + "state_processed_from_destination", + "counter for number of state messages received from destination"), + STATE_PROCESSED_FROM_SOURCE(MetricEmittingApps.WORKER, + "state_processed_from_source", + "counter for number of state messages received from source"), // TEMPORARY, delete after the migration. STATS_TRACKER_IMPLEMENTATION(MetricEmittingApps.WORKER, "stats_tracker_implementation", From c9142adcdd33845ea9170ca0462249cc612039f4 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Wed, 30 Aug 2023 15:42:31 -0700 Subject: [PATCH 066/201] Add actual error to replication traces (#8623) --- .../io/airbyte/metrics/lib/ApmTraceConstants.java | 5 +++++ .../workers/temporal/TemporalAttemptExecution.java | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java index b938fae66b5..7ce444e875b 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceConstants.java @@ -79,6 +79,11 @@ public static final class Tags { */ public static final String DOCKER_IMAGE_KEY = "docker_image"; + /** + * Name of the APM trace tag that holds the actual type of the error. + */ + public static final String ERROR_ACTUAL_TYPE_KEY = "error.actual_type"; + /** * Name of the APM trace tag that holds the failure origin(s) associated with the trace. */ diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index 2347e581340..23411c7449d 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -4,6 +4,8 @@ package io.airbyte.workers.temporal; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ERROR_ACTUAL_TYPE_KEY; + import com.google.common.annotations.VisibleForTesting; import io.airbyte.api.client.AirbyteApiClient; import io.airbyte.api.client.invoker.generated.ApiException; @@ -12,11 +14,13 @@ import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.helpers.LogConfigs; +import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.Worker; import io.temporal.activity.Activity; import io.temporal.activity.ActivityExecutionContext; import java.nio.file.Path; +import java.util.Map; import java.util.Optional; import java.util.function.Consumer; import java.util.function.Supplier; @@ -131,12 +135,21 @@ public OUTPUT get() { return worker.run(input, jobRoot); } catch (final Exception e) { + addActualRootCauseToTrace(e); throw Activity.wrap(e); } finally { mdcSetter.accept(null); } } + private void addActualRootCauseToTrace(final Exception e) { + Throwable inner = e; + while (inner.getCause() != null) { + inner = inner.getCause(); + } + ApmTraceUtils.addTagsToTrace(Map.of(ERROR_ACTUAL_TYPE_KEY, e.getClass().getName())); + } + private void saveWorkflowIdForCancellation(final AirbyteApiClient airbyteApiClient) throws ApiException { // If the jobId is not a number, it means the job is a synchronous job. No attempt is created for // it, and it cannot be cancelled, so do not save the workflowId. See From 125061f8806b972ed6b142d221dc6aa3a847446a Mon Sep 17 00:00:00 2001 From: Ryan Br Date: Wed, 30 Aug 2023 15:49:49 -0700 Subject: [PATCH 067/201] =?UTF-8?q?Make=20noisy=20logs=20debug=20so=20they?= =?UTF-8?q?=20don't=20show=20up=20in=20customer=20facing=20logs=20w?= =?UTF-8?q?=E2=80=A6=20(#8626)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../io/airbyte/workers/general/ReplicationWorkerHelper.java | 2 +- .../main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java index def3e58da2c..88bae872865 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java @@ -246,7 +246,7 @@ public Optional processMessageFromSource(final AirbyteMessage so @VisibleForTesting void internalProcessMessageFromDestination(final AirbyteMessage destinationRawMessage) { - LOGGER.info("State in ReplicationWorkerHelper from destination: {}", destinationRawMessage); + LOGGER.debug("State in ReplicationWorkerHelper from destination: {}", destinationRawMessage); final StreamDescriptor previousStream = currentDestinationStream; currentDestinationStream = airbyteMessageDataExtractor.extractStreamDescriptor(destinationRawMessage, previousStream); if (currentDestinationStream != null) { diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java index b6610c50a93..5b97ccf303c 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/DogStatsDMetricClient.java @@ -82,7 +82,7 @@ public void count(final MetricsRegistry metric, final long amt, final MetricAttr return; } - log.info("publishing count, name: {}, value: {}, attributes: {}", metric, amt, attributes); + log.debug("publishing count, name: {}, value: {}, attributes: {}", metric, amt, attributes); statsDClient.count(metric.getMetricName(), amt, toTags(attributes)); } } From 87dfa2ad33c99a0d5746b74487bee7dd283ad460 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Wed, 30 Aug 2023 20:31:48 -0500 Subject: [PATCH 068/201] Return normalization info as versioned info to correctly display it in the UI (#8629) Co-authored-by: lmossman --- airbyte-api/src/main/openapi/config.yaml | 8 ++- .../ActorDefinitionVersionHandler.java | 3 + .../ActorDefinitionVersionHandlerTest.java | 58 ++++++++++++++++++- .../ConnectionForm/OperationsSection.tsx | 2 +- .../OperationsSectionHookForm.tsx | 2 +- .../ConnectionForm/formConfig.test.ts | 8 +-- .../connection/ConnectionForm/formConfig.tsx | 12 ++-- .../CreateConnectionForm.test.tsx | 2 + .../ConnectionEditService.test.tsx | 2 + .../ConnectionFormService.test.tsx | 2 + .../ConnectionForm/ConnectionFormService.tsx | 15 ++++- .../ConnectionReplicationPage.test.tsx | 2 + .../ConnectionTransformationPage.tsx | 9 +-- .../test-utils/mock-data/mockDestination.ts | 21 ++++++- 14 files changed, 123 insertions(+), 23 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index bb2b6619d4f..9cee8047300 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -4923,6 +4923,8 @@ components: required: - dockerRepository - dockerImageTag + - supportsDbt + - normalizationConfig - supportState - isOverrideApplied properties: @@ -4930,6 +4932,10 @@ components: type: string dockerImageTag: type: string + supportsDbt: + type: boolean + normalizationConfig: + $ref: "#/components/schemas/NormalizationDestinationDefinitionConfig" isOverrideApplied: type: boolean supportState: @@ -6699,7 +6705,7 @@ components: actorType: $ref: "#/components/schemas/ActorType" NormalizationDestinationDefinitionConfig: - description: describes a normalization config for destination definition + description: describes a normalization config for destination definition version type: object required: - supported diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java index 3bdf66b8c06..a57bbd815d3 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.time.LocalDate; import java.util.List; +import java.util.Objects; /** * ActorDefinitionVersionHandler. Javadocs suppressed because api docs should be used as source of @@ -83,6 +84,8 @@ ActorDefinitionVersionRead createActorDefinitionVersionRead(final ActorDefinitio final ActorDefinitionVersionRead advRead = new ActorDefinitionVersionRead() .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(Objects.requireNonNullElse(actorDefinitionVersion.getSupportsDbt(), false)) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(actorDefinitionVersion.getNormalizationConfig())) .supportState(toApiSupportState(actorDefinitionVersion.getSupportState())) .isOverrideApplied(versionWithOverrideStatus.isOverrideApplied()); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java index e730c67f8e7..2262f87b9e4 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java @@ -15,11 +15,13 @@ import io.airbyte.api.model.generated.ActorDefinitionVersionRead; import io.airbyte.api.model.generated.DestinationIdRequestBody; import io.airbyte.api.model.generated.SourceIdRequestBody; +import io.airbyte.commons.server.converters.ApiPojoConverters; import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ActorDefinitionVersion.SupportState; import io.airbyte.config.DestinationConnection; +import io.airbyte.config.NormalizationDestinationDefinitionConfig; import io.airbyte.config.ReleaseStage; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; @@ -70,6 +72,15 @@ private ActorDefinitionVersion createActorDefinitionVersion() { .withDocumentationUrl("https://docs.airbyte.io"); } + private ActorDefinitionVersion createActorDefinitionVersionWithNormalization() { + return createActorDefinitionVersion() + .withSupportsDbt(true) + .withNormalizationConfig(new NormalizationDestinationDefinitionConfig() + .withNormalizationRepository("repository") + .withNormalizationTag("dev") + .withNormalizationIntegrationType("integration-type")); + } + @ParameterizedTest @CsvSource({"true", "false"}) void testGetActorDefinitionVersionForSource(final boolean isOverrideApplied) @@ -94,7 +105,9 @@ void testGetActorDefinitionVersionForSource(final boolean isOverrideApplied) .isOverrideApplied(isOverrideApplied) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) - .dockerImageTag(actorDefinitionVersion.getDockerImageTag()); + .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(false) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(null)); assertEquals(expectedRead, actorDefinitionVersionRead); verify(mConfigRepository).getSourceConnection(sourceId); @@ -129,7 +142,46 @@ void testGetActorDefinitionVersionForDestination(final boolean isOverrideApplied .isOverrideApplied(isOverrideApplied) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) - .dockerImageTag(actorDefinitionVersion.getDockerImageTag()); + .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(false) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(null)); + + assertEquals(expectedRead, actorDefinitionVersionRead); + verify(mConfigRepository).getDestinationConnection(destinationId); + verify(mConfigRepository).getDestinationDefinitionFromDestination(destinationId); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId); + verify(mConfigRepository).listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); + verifyNoMoreInteractions(mConfigRepository); + verifyNoMoreInteractions(mActorDefinitionVersionHelper); + } + + @ParameterizedTest + @CsvSource({"true", "false"}) + void testGetActorDefinitionVersionForDestinationWithNormalization(final boolean isOverrideApplied) + throws JsonValidationException, ConfigNotFoundException, IOException { + final UUID destinationId = UUID.randomUUID(); + final ActorDefinitionVersion actorDefinitionVersion = createActorDefinitionVersionWithNormalization(); + final DestinationConnection destinationConnection = new DestinationConnection() + .withDestinationId(destinationId) + .withWorkspaceId(WORKSPACE_ID); + + when(mConfigRepository.getDestinationConnection(destinationId)) + .thenReturn(destinationConnection); + when(mConfigRepository.getDestinationDefinitionFromDestination(destinationId)) + .thenReturn(DESTINATION_DEFINITION); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(DESTINATION_DEFINITION, WORKSPACE_ID, destinationId)) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(actorDefinitionVersion, isOverrideApplied)); + + final DestinationIdRequestBody destinationIdRequestBody = new DestinationIdRequestBody().destinationId(destinationId); + final ActorDefinitionVersionRead actorDefinitionVersionRead = + actorDefinitionVersionHandler.getActorDefinitionVersionForDestinationId(destinationIdRequestBody); + final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() + .isOverrideApplied(isOverrideApplied) + .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) + .dockerRepository(actorDefinitionVersion.getDockerRepository()) + .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(actorDefinitionVersion.getSupportsDbt()) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(actorDefinitionVersion.getNormalizationConfig())); assertEquals(expectedRead, actorDefinitionVersionRead); verify(mConfigRepository).getDestinationConnection(destinationId); @@ -169,6 +221,8 @@ void testCreateActorDefinitionVersionReadWithBreakingChange() throws IOException .supportState(io.airbyte.api.model.generated.SupportState.DEPRECATED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) + .supportsDbt(false) + .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(null)) .breakingChanges(new ActorDefinitionVersionBreakingChanges() .minUpgradeDeadline(LocalDate.parse("2023-01-01")) .upcomingBreakingChanges(List.of( diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSection.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSection.tsx index 4df672a53b1..489a2d8dd93 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSection.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSection.tsx @@ -24,7 +24,7 @@ export const OperationsSection: React.FC = ({ const { formatMessage } = useIntl(); const { - destDefinition: { normalizationConfig, supportsDbt }, + destDefinitionVersion: { normalizationConfig, supportsDbt }, } = useConnectionFormService(); const supportsNormalization = normalizationConfig.supported; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx index f636d8f3e4d..6d7048c3213 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx @@ -21,7 +21,7 @@ import { TransformationFieldHookForm } from "./TransformationFieldHookForm"; export const OperationsSectionHookForm: React.FC = () => { const { formatMessage } = useIntl(); const { - destDefinition: { normalizationConfig, supportsDbt }, + destDefinitionVersion: { normalizationConfig, supportsDbt }, } = useConnectionFormService(); const supportsNormalization = normalizationConfig.supported; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.test.ts b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.test.ts index 304c219d310..f34babb0e2a 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.test.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.test.ts @@ -2,8 +2,8 @@ import { renderHook } from "@testing-library/react"; import { mockConnection } from "test-utils/mock-data/mockConnection"; import { - mockDestinationDefinition, mockDestinationDefinitionSpecification, + mockDestinationDefinitionVersion, } from "test-utils/mock-data/mockDestination"; import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; import { TestWrapper as wrapper } from "test-utils/testutils"; @@ -194,7 +194,7 @@ describe("#mapFormPropsToOperation", () => { describe("#useInitialValues", () => { it("should generate initial values w/ no 'not create' mode", () => { const { result } = renderHook(() => - useInitialValues(mockConnection, mockDestinationDefinition, mockDestinationDefinitionSpecification) + useInitialValues(mockConnection, mockDestinationDefinitionVersion, mockDestinationDefinitionSpecification) ); expect(result.current).toMatchSnapshot(); expect(result.current.name).toBeDefined(); @@ -202,7 +202,7 @@ describe("#useInitialValues", () => { it("should generate initial values w/ 'not create' mode: false", () => { const { result } = renderHook(() => - useInitialValues(mockConnection, mockDestinationDefinition, mockDestinationDefinitionSpecification, false) + useInitialValues(mockConnection, mockDestinationDefinitionVersion, mockDestinationDefinitionSpecification, false) ); expect(result.current).toMatchSnapshot(); expect(result.current.name).toBeDefined(); @@ -210,7 +210,7 @@ describe("#useInitialValues", () => { it("should generate initial values w/ 'not create' mode: true", () => { const { result } = renderHook(() => - useInitialValues(mockConnection, mockDestinationDefinition, mockDestinationDefinitionSpecification, true) + useInitialValues(mockConnection, mockDestinationDefinitionVersion, mockDestinationDefinitionSpecification, true) ); expect(result.current).toMatchSnapshot(); expect(result.current.name).toBeUndefined(); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx index be017739cb4..acfd23e8bfc 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx @@ -9,9 +9,9 @@ import { validateCronExpression, validateCronFrequencyOneHourOrMore } from "area import { isDbtTransformation, isNormalizationTransformation, isWebhookTransformation } from "area/connection/utils"; import { ConnectionValues, useCurrentWorkspace } from "core/api"; import { + ActorDefinitionVersionRead, ConnectionScheduleData, ConnectionScheduleType, - DestinationDefinitionRead, DestinationDefinitionSpecificationRead, DestinationSyncMode, Geography, @@ -315,7 +315,7 @@ export const getInitialNormalization = ( export const useInitialValues = ( connection: ConnectionOrPartialConnection, - destDefinition: DestinationDefinitionRead, + destDefinitionVersion: ActorDefinitionVersionRead, destDefinitionSpecification: DestinationDefinitionSpecificationRead, isNotCreateMode?: boolean ): FormikConnectionFormValues => { @@ -382,11 +382,11 @@ export const useInitialValues = ( const operations = connection.operations ?? []; - if (destDefinition.supportsDbt) { + if (destDefinitionVersion.supportsDbt) { initialValues.transformations = getInitialTransformations(operations); } - if (destDefinition.normalizationConfig?.supported) { + if (destDefinitionVersion.normalizationConfig?.supported) { initialValues.normalization = getInitialNormalization(operations, isNotCreateMode); } @@ -405,8 +405,8 @@ export const useInitialValues = ( connection.scheduleType, connection.source.name, defaultNonBreakingChangesPreference, - destDefinition.supportsDbt, - destDefinition.normalizationConfig, + destDefinitionVersion.supportsDbt, + destDefinitionVersion.normalizationConfig, initialSchema, isNotCreateMode, workspace, diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx index 8170bba6a62..6b383efdb2f 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx @@ -8,6 +8,7 @@ import { mockConnection } from "test-utils/mock-data/mockConnection"; import { mockDestinationDefinition, mockDestinationDefinitionSpecification, + mockDestinationDefinitionVersion, } from "test-utils/mock-data/mockDestination"; import { mockSourceDefinition, mockSourceDefinitionSpecification } from "test-utils/mock-data/mockSource"; import { mockTheme } from "test-utils/mock-data/mockTheme"; @@ -42,6 +43,7 @@ jest.mock("core/api", () => ({ useCurrentWorkspace: () => ({}), useInvalidateWorkspaceStateQuery: () => () => null, useCreateConnection: () => async () => null, + useDestinationDefinitionVersion: () => mockDestinationDefinitionVersion, })); jest.mock("hooks/domain/connector/useGetSourceFromParams", () => ({ diff --git a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx index 3eb990968e0..63ae4a6571c 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.test.tsx @@ -6,6 +6,7 @@ import { mockConnection } from "test-utils/mock-data/mockConnection"; import { mockDestinationDefinition, mockDestinationDefinitionSpecification, + mockDestinationDefinitionVersion, } from "test-utils/mock-data/mockDestination"; import { mockSourceDefinition, mockSourceDefinitionSpecification } from "test-utils/mock-data/mockSource"; import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; @@ -50,6 +51,7 @@ jest.mock("core/api", () => ({ }), isLoading: false, }), + useDestinationDefinitionVersion: () => mockDestinationDefinitionVersion, })); const utils = { diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx index eb332d4a919..5d0eabca877 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.test.tsx @@ -5,6 +5,7 @@ import { mockConnection } from "test-utils/mock-data/mockConnection"; import { mockDestinationDefinition, mockDestinationDefinitionSpecification, + mockDestinationDefinitionVersion, } from "test-utils/mock-data/mockDestination"; import { mockSourceDefinition, mockSourceDefinitionSpecification } from "test-utils/mock-data/mockSource"; import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; @@ -36,6 +37,7 @@ jest.mock("services/connector/DestinationDefinitionService", () => ({ jest.mock("core/api", () => ({ useCurrentWorkspace: () => mockWorkspace, + useDestinationDefinitionVersion: () => mockDestinationDefinitionVersion, })); describe("ConnectionFormService", () => { diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx index d8e4511e257..b0553c19f24 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx @@ -10,8 +10,9 @@ import { useInitialValues, } from "components/connection/ConnectionForm/formConfig"; -import { ConnectionValues } from "core/api"; +import { ConnectionValues, useDestinationDefinitionVersion } from "core/api"; import { + ActorDefinitionVersionRead, ConnectionScheduleType, DestinationDefinitionRead, DestinationDefinitionSpecificationRead, @@ -68,6 +69,7 @@ interface ConnectionFormHook { sourceDefinition: SourceDefinitionRead; sourceDefinitionSpecification: SourceDefinitionSpecificationRead; destDefinition: DestinationDefinitionRead; + destDefinitionVersion: ActorDefinitionVersionRead; destDefinitionSpecification: DestinationDefinitionSpecificationRead; initialValues: FormikConnectionFormValues; schemaError?: SchemaError; @@ -88,18 +90,24 @@ const useConnectionForm = ({ }: ConnectionServiceProps): ConnectionFormHook => { const { source: { sourceDefinitionId }, - destination: { destinationDefinitionId }, + destination: { destinationId, destinationDefinitionId }, } = connection; const sourceDefinition = useSourceDefinition(sourceDefinitionId); const sourceDefinitionSpecification = useGetSourceDefinitionSpecification(sourceDefinitionId, connection.sourceId); const destDefinition = useDestinationDefinition(destinationDefinitionId); + const destDefinitionVersion = useDestinationDefinitionVersion(destinationId); const destDefinitionSpecification = useGetDestinationDefinitionSpecification( destinationDefinitionId, connection.destinationId ); - const initialValues = useInitialValues(connection, destDefinition, destDefinitionSpecification, mode !== "create"); + const initialValues = useInitialValues( + connection, + destDefinitionVersion, + destDefinitionSpecification, + mode !== "create" + ); const { formatMessage } = useIntl(); const [submitError, setSubmitError] = useState(null); const formId = useUniqueFormId(); @@ -131,6 +139,7 @@ const useConnectionForm = ({ sourceDefinition, sourceDefinitionSpecification, destDefinition, + destDefinitionVersion, destDefinitionSpecification, initialValues, schemaError, diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx index dcc5ef6e20f..17fec665458 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx @@ -10,6 +10,7 @@ import { mockConnection } from "test-utils/mock-data/mockConnection"; import { mockDestinationDefinition, mockDestinationDefinitionSpecification, + mockDestinationDefinitionVersion, } from "test-utils/mock-data/mockDestination"; import { mockSourceDefinition, mockSourceDefinitionSpecification } from "test-utils/mock-data/mockSource"; import { mockTheme } from "test-utils/mock-data/mockTheme"; @@ -55,6 +56,7 @@ jest.mock("core/api", () => ({ mutateAsync: async (connection: WebBackendConnectionUpdate) => connection, isLoading: false, }), + useDestinationDefinitionVersion: () => mockDestinationDefinitionVersion, })); jest.mock("hooks/theme/useAirbyteTheme", () => ({ diff --git a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx index 9d7cf82a8f7..46209b17b91 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionTransformationPage/ConnectionTransformationPage.tsx @@ -16,12 +16,13 @@ import { DbtCloudTransformations } from "./DbtCloudTransformations"; import { NormalizationForm } from "./NormalizationForm"; export const ConnectionTransformationPage: React.FC = () => { - const { destDefinition } = useConnectionFormService(); + const { destDefinitionVersion } = useConnectionFormService(); useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_TRANSFORMATION); - const supportsNormalization = destDefinition.normalizationConfig.supported; - const supportsDbt = useFeature(FeatureItem.AllowCustomDBT) && destDefinition.supportsDbt; - const supportsCloudDbtIntegration = useFeature(FeatureItem.AllowDBTCloudIntegration) && destDefinition.supportsDbt; + const supportsNormalization = destDefinitionVersion.normalizationConfig.supported; + const supportsDbt = useFeature(FeatureItem.AllowCustomDBT) && destDefinitionVersion.supportsDbt; + const supportsCloudDbtIntegration = + useFeature(FeatureItem.AllowDBTCloudIntegration) && destDefinitionVersion.supportsDbt; const noSupportedTransformations = !supportsNormalization && !supportsDbt && !supportsCloudDbtIntegration; return ( diff --git a/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts b/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts index b468e87f0e4..942dba6077a 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts @@ -1,5 +1,10 @@ import { ConnectorIds } from "area/connector/utils"; -import { DestinationDefinitionSpecificationRead, DestinationDefinitionRead } from "core/request/AirbyteClient"; +import { + DestinationDefinitionSpecificationRead, + DestinationDefinitionRead, + ActorDefinitionVersionRead, + SupportState, +} from "core/request/AirbyteClient"; export const mockDestinationDefinition: DestinationDefinitionRead = { destinationDefinitionId: ConnectorIds.Destinations.Postgres, @@ -18,6 +23,20 @@ export const mockDestinationDefinition: DestinationDefinitionRead = { }, }; +export const mockDestinationDefinitionVersion: ActorDefinitionVersionRead = { + dockerRepository: "airbyte/destination-postgres", + dockerImageTag: "0.3.26", + supportsDbt: true, + normalizationConfig: { + supported: true, + normalizationRepository: "airbyte/normalization", + normalizationTag: "0.2.25", + normalizationIntegrationType: "postgres", + }, + isOverrideApplied: false, + supportState: SupportState.supported, +}; + export const mockDestinationDefinitionSpecification: DestinationDefinitionSpecificationRead = { destinationDefinitionId: ConnectorIds.Destinations.Postgres, documentationUrl: "https://docs.airbyte.io/integrations/destinations/postgres", From 81224d711b68a9d079f888c69f5e32152e64bac6 Mon Sep 17 00:00:00 2001 From: Chandler Prall Date: Wed, 30 Aug 2023 21:03:00 -0600 Subject: [PATCH 069/201] =?UTF-8?q?=F0=9F=AA=9F=C2=A0=F0=9F=8E=89=20fronte?= =?UTF-8?q?nd=20RBAC=20engine=20&=20react=20hooks=20(#8533)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Teal Larson --- airbyte-webapp/.storybook/withProvider.tsx | 3 +- airbyte-webapp/src/core/api/hooks/index.ts | 1 + .../src/core/api/hooks/organizations.ts | 1 + .../src/core/api/hooks/permissions.ts | 15 ++ .../src/core/api/hooks/workspaces.tsx | 6 +- airbyte-webapp/src/core/utils/rbac/index.ts | 1 + .../src/core/utils/rbac/intent.test.ts | 84 +++++++ airbyte-webapp/src/core/utils/rbac/intent.ts | 66 ++++++ .../src/core/utils/rbac/rbac.docs.mdx | 105 +++++++++ .../src/core/utils/rbac/rbac.docs.tsx | 221 ++++++++++++++++++ .../src/core/utils/rbac/rbac.test.ts | 179 ++++++++++++++ airbyte-webapp/src/core/utils/rbac/rbac.ts | 38 +++ .../utils/rbac/rbacPermissionsQuery.test.ts | 200 ++++++++++++++++ .../core/utils/rbac/rbacPermissionsQuery.ts | 81 +++++++ .../src/test-utils/mock-data/mockUser.ts | 8 + 15 files changed, 1004 insertions(+), 5 deletions(-) create mode 100644 airbyte-webapp/src/core/api/hooks/permissions.ts create mode 100644 airbyte-webapp/src/core/utils/rbac/index.ts create mode 100644 airbyte-webapp/src/core/utils/rbac/intent.test.ts create mode 100644 airbyte-webapp/src/core/utils/rbac/intent.ts create mode 100644 airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx create mode 100644 airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx create mode 100644 airbyte-webapp/src/core/utils/rbac/rbac.test.ts create mode 100644 airbyte-webapp/src/core/utils/rbac/rbac.ts create mode 100644 airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.test.ts create mode 100644 airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts create mode 100644 airbyte-webapp/src/test-utils/mock-data/mockUser.ts diff --git a/airbyte-webapp/.storybook/withProvider.tsx b/airbyte-webapp/.storybook/withProvider.tsx index 56dc8c6f90c..d0535512278 100644 --- a/airbyte-webapp/.storybook/withProvider.tsx +++ b/airbyte-webapp/.storybook/withProvider.tsx @@ -1,4 +1,5 @@ import React from "react"; +import { Decorator } from "@storybook/react"; import { MemoryRouter } from "react-router-dom"; import { IntlProvider } from "react-intl"; @@ -32,7 +33,7 @@ const queryClient = new QueryClient({ }, }); -export const withProviders = (getStory) => ( +export const withProviders = (getStory: Parameters[0]) => ( diff --git a/airbyte-webapp/src/core/api/hooks/index.ts b/airbyte-webapp/src/core/api/hooks/index.ts index 6c78ae2607e..bff7582f306 100644 --- a/airbyte-webapp/src/core/api/hooks/index.ts +++ b/airbyte-webapp/src/core/api/hooks/index.ts @@ -12,6 +12,7 @@ export * from "./organizations"; export * from "./security"; export * from "./instanceConfiguration"; export * from "./streams"; +export * from "./permissions"; export * from "./upgradeConnectorVersion"; export * from "./users"; export * from "./workspaces"; diff --git a/airbyte-webapp/src/core/api/hooks/organizations.ts b/airbyte-webapp/src/core/api/hooks/organizations.ts index 94e905560c8..f41fe4ee314 100644 --- a/airbyte-webapp/src/core/api/hooks/organizations.ts +++ b/airbyte-webapp/src/core/api/hooks/organizations.ts @@ -11,6 +11,7 @@ export const organizationKeys = { all: [SCOPE_USER, "organizations"] as const, detail: (organizationId: string) => [...organizationKeys.all, "details", organizationId] as const, listUsers: (organizationId: string) => [SCOPE_ORGANIZATION, "users", "list", organizationId] as const, + workspaces: (organizationIds: string[]) => [...organizationKeys.all, "workspaces", organizationIds] as const, }; export const useOrganization = (organizationId: string) => { diff --git a/airbyte-webapp/src/core/api/hooks/permissions.ts b/airbyte-webapp/src/core/api/hooks/permissions.ts new file mode 100644 index 00000000000..953b8fbd6d0 --- /dev/null +++ b/airbyte-webapp/src/core/api/hooks/permissions.ts @@ -0,0 +1,15 @@ +import { SCOPE_USER } from "services/Scope"; + +import { listPermissionsByUser } from "../generated/AirbyteClient"; +import { useRequestOptions } from "../useRequestOptions"; +import { useSuspenseQuery } from "../useSuspenseQuery"; + +export const permissionKeys = { + all: [SCOPE_USER, "permissions"] as const, + listByUser: (userId: string) => [...permissionKeys.all, "listByUser", userId] as const, +}; + +export const useListPermissions = (userId: string) => { + const requestOptions = useRequestOptions(); + return useSuspenseQuery(permissionKeys.listByUser(userId), () => listPermissionsByUser({ userId }, requestOptions)); +}; diff --git a/airbyte-webapp/src/core/api/hooks/workspaces.tsx b/airbyte-webapp/src/core/api/hooks/workspaces.tsx index 49b2b9410b0..ccd7e941720 100644 --- a/airbyte-webapp/src/core/api/hooks/workspaces.tsx +++ b/airbyte-webapp/src/core/api/hooks/workspaces.tsx @@ -14,7 +14,7 @@ import { updateWorkspaceName, webBackendGetWorkspaceState, } from "../generated/AirbyteClient"; -import { WorkspaceReadList, WorkspaceUpdate, WorkspaceUpdateName } from "../types/AirbyteClient"; +import { WorkspaceRead, WorkspaceReadList, WorkspaceUpdate, WorkspaceUpdateName } from "../types/AirbyteClient"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; @@ -125,9 +125,7 @@ export const useGetWorkspaceQuery = (workspaceId: string) => { export const useGetWorkspace = ( workspaceId: string, - options?: { - staleTime: number; - } + options?: Parameters>[2] ) => { const queryKey = getWorkspaceQueryKey(workspaceId); const queryFn = useGetWorkspaceQuery(workspaceId); diff --git a/airbyte-webapp/src/core/utils/rbac/index.ts b/airbyte-webapp/src/core/utils/rbac/index.ts new file mode 100644 index 00000000000..654450b87e8 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/index.ts @@ -0,0 +1 @@ +export * from "./rbac"; diff --git a/airbyte-webapp/src/core/utils/rbac/intent.test.ts b/airbyte-webapp/src/core/utils/rbac/intent.test.ts new file mode 100644 index 00000000000..6e288879bb5 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/intent.test.ts @@ -0,0 +1,84 @@ +import { renderHook } from "@testing-library/react"; + +import { Intent, useIntent } from "./intent"; +import { useRbac } from "./rbac"; + +jest.mock("./rbac", () => ({ + useRbac: jest.fn(), +})); +const mockUseRbac = useRbac as unknown as jest.Mock; + +describe("useIntent", () => { + it("maps intent to query", () => { + mockUseRbac.mockClear(); + renderHook(() => useIntent(Intent.ListOrganizationMembers, undefined)); + expect(mockUseRbac).toHaveBeenCalledTimes(1); + expect(mockUseRbac).toHaveBeenCalledWith({ + resourceType: "ORGANIZATION", + role: "READER", + }); + }); + + describe("applies overriding details", () => { + it("overrides the organizationId", () => { + mockUseRbac.mockClear(); + renderHook(() => useIntent(Intent.ListOrganizationMembers, { organizationId: "some-other-org" })); + expect(mockUseRbac).toHaveBeenCalledTimes(1); + expect(mockUseRbac).toHaveBeenCalledWith({ + resourceType: "ORGANIZATION", + role: "READER", + resourceId: "some-other-org", + }); + }); + + it("overrides the workspaceId", () => { + mockUseRbac.mockClear(); + renderHook(() => useIntent(Intent.ListWorkspaceMembers, { workspaceId: "some-other-workspace" })); + expect(mockUseRbac).toHaveBeenCalledTimes(1); + expect(mockUseRbac).toHaveBeenCalledWith({ + resourceType: "WORKSPACE", + role: "READER", + resourceId: "some-other-workspace", + }); + }); + + it("does not override a resourceId with that of a mismatched resource", () => { + mockUseRbac.mockClear(); + renderHook(() => + // @ts-expect-error we're testing invalid object shapes + useIntent(Intent.ListOrganizationMembers, { workspaceId: "some-other-organization" }, mockUseRbac) + ); + expect(mockUseRbac).toHaveBeenCalledTimes(1); + expect(mockUseRbac).toHaveBeenCalledWith({ + resourceType: "ORGANIZATION", + role: "READER", + }); + + mockUseRbac.mockClear(); + // @ts-expect-error we're testing invalid object shapes + renderHook(() => useIntent(Intent.ListWorkspaceMembers, { organizationId: "some-other-workspace" }, mockUseRbac)); + expect(mockUseRbac).toHaveBeenCalledTimes(1); + expect(mockUseRbac).toHaveBeenCalledWith({ + resourceType: "WORKSPACE", + role: "READER", + }); + }); + }); + + // eslint-disable-next-line jest/expect-expect + it("intent meta property enforcement", () => { + const processIntent = useIntent; // avoid react rules of hooks warnings 🤡 + + // @TODO: if we have any instance-level intents, add checks here to exclude organizationId and workspaceId + + processIntent(Intent.ListOrganizationMembers); + processIntent(Intent.ListOrganizationMembers, { organizationId: "org" }); + // @ts-expect-error workspaceId is not valid for ListOrganizationMembers + processIntent(Intent.ListOrganizationMembers, { workspaceId: "workspace" }); + + processIntent(Intent.ListWorkspaceMembers); + processIntent(Intent.ListWorkspaceMembers, { workspaceId: "workspace" }); + // @ts-expect-error workspaceId is not valid for ListWorkspaceMembers + processIntent(Intent.ListWorkspaceMembers, { organizationId: "organizationId" }); + }); +}); diff --git a/airbyte-webapp/src/core/utils/rbac/intent.ts b/airbyte-webapp/src/core/utils/rbac/intent.ts new file mode 100644 index 00000000000..b75547acf38 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/intent.ts @@ -0,0 +1,66 @@ +import { useRbac } from "./rbac"; +import { RbacQuery, RbacQueryWithoutResourceId, RbacResource } from "./rbacPermissionsQuery"; + +export enum Intent { + // instance + + // organization + "ListOrganizationMembers" = "ListOrganizationMembers", + + // workspace + "ListWorkspaceMembers" = "ListWorkspaceMembers", +} + +const intentToRbacQuery = { + [Intent.ListOrganizationMembers]: { resourceType: "ORGANIZATION", role: "READER" }, + + [Intent.ListWorkspaceMembers]: { resourceType: "WORKSPACE", role: "READER" }, +} as const; + +interface OrganizationIntentMeta { + organizationId?: string; +} +interface WorkspaceIntentMeta { + workspaceId?: string; +} + +// Utility types to enforce shape of any meta object; i.e. organzaition-focused intents shouldn't receive workspaceIds +// provides proper autocomplete hinting for `meta` and to future proof changes to Intent:Query mapping, +// e.g. changing Intent X from resource:"WORKSPACE" to resource:"ORGANIZATION" should catch instances of invalid meta +type MapIntentToResource = (typeof intentToRbacQuery)[I]["resourceType"]; +type IntentMetaForResource = R extends "INSTANCE" + ? Record + : R extends "ORGANIZATION" + ? OrganizationIntentMeta + : R extends "WORKSPACE" + ? WorkspaceIntentMeta + : never; + +/* +Given the React context + overrides provided in optional `details`, +determine if the user has the required permissions to perform the given `intent`. +*/ +export const useIntent = (intent: I, meta?: IntentMetaForResource>) => { + let query: RbacQuery | RbacQueryWithoutResourceId = intentToRbacQuery[intent]; + if (isOrganizationIntentMeta(query.resourceType, meta) && meta?.organizationId) { + query = { ...query, resourceId: meta?.organizationId }; + } else if (isWorkspaceIntentMeta(query.resourceType, meta) && meta?.workspaceId) { + query = { ...query, resourceId: meta?.workspaceId }; + } + + return useRbac(query); +}; + +function isOrganizationIntentMeta( + resource: RbacResource, + _meta: OrganizationIntentMeta | WorkspaceIntentMeta | undefined +): _meta is OrganizationIntentMeta | undefined { + return resource === "ORGANIZATION"; +} + +function isWorkspaceIntentMeta( + resource: RbacResource, + _meta: OrganizationIntentMeta | WorkspaceIntentMeta | undefined +): _meta is WorkspaceIntentMeta | undefined { + return resource === "WORKSPACE"; +} diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx b/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx new file mode 100644 index 00000000000..f6deac62dd4 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx @@ -0,0 +1,105 @@ +import { Canvas, Controls, Meta, Story } from "@storybook/addon-docs"; +import { FlexContainer, FlexItem } from "../../../components/ui/Flex"; +import { PermissionTestView } from "./rbac.docs.tsx"; + + + +# RBAC + +## Intents + +Use intents to ask if a user can see something or perform an action. Intents better reflect our intention in the code and map to a specific permission. This allows us to change the related permission in one place, and to stay focused on the task at hand while developing instead of recalling specific roles and permissions. + +(based on https://docs.google.com/spreadsheets/d/1rC7yuWVmDuRBgDKun2KNQ5wXH2jfGj0nqiCdZTDWEVk/edit#gid=855220348) + +| Intent | Query | +| -------------------- | -------------------------------------------------------- | +| **ReadWorkspace** | `{resource: "WORKSPACE", role: "READER", resourceId}` | +| **CreateWorkspace** | `{resource: "ORGANIZATION", role: "EDITOR", resourceId}` | +| **UpdateConnection** | `{resource: "WORKSPACE", role: "EDITOR", resourceId}` | +| **ReadBilling** | `{resource: "ORGANIZATION", role: "ADMIN", resourceId}` | + +### Usage + +The `useIntent` hook is provided to make inquiries. If an intent for your use case does not yet exist it's almost certainly the right decision to create it. + +```typescript +const canReadWorkspace = useIntent(Intent.ReadWorkspace); +``` + +#### Meta details + +By default, `useIntent` will locate the necessary resource IDs from available React context. If those values are not available or must be overridden, an object with those extra values can be passed. + +```typescript +const canThatUserReadThatWorkspace = useIntent(Intent.ReadWorkspace, { + userId: "some-user", + workspaceId: "some-workspace", +}); +``` + +```typescript +interface MetaDetails { + userId?: string; + organizationId?: string; + workspaceId?: string; +} +``` + +### Direct RBAC querying + +If for some reason an intent does not make sense for your use case, `useRbac` is available to pass a specific query to. Similar to `useIntent`, this will use avaiable React context to fill in any un-provided query meta. + +`useRbac(permissions: RbacPermission[], query: RbacQuery | RbacQueryWithoutResourceId)` + +--- + +Alternatively, if you want or need to bypass React context altogether, `useRbacPermissionsQuery` will do just that. + +`useRbacPermissionsQuery(permissions: RbacPermission[], query: RbacQuery)` + +## Interactive Demo + +This calls the same function as the webapp and shows how different user permissions affect the result. Organization+Workspace relationship relies on a webapp backend server running locally. + + + +## Engine implementation notes + +Need a way for code to ask if a user can `ADMIN|WRITE|READ` a given `INSTANCE|ORGANIZATION|WORKSPACE`. An RBAC query should be of the form: + +```typescript +query = { role, resource, resourceId }; // does user have Role at Resource +``` + +Any granted role also satisfies a lower role (`Admin->Editor->Reader`), and any Resource+Role copies that role to lower resources (`Instace->Organization->Workspace`). Together, these rules transform e.g. Organization Editor into: + +| _resource_ | _role_ | _reason_ | +| --------------------------------- | ---------- | ------------------------------------------- | +| **Organization** (by id) | **Editor** | granted permission | +| | **Reader** | editor bestows reader | +| **Workspace** (when owned by org) | **Editor** | organization bestows workspace, copies role | +| | **Reader** | editor bestows reader | + +A static mapping of one permission to all of the Permissions it can be satisfied by could be used; alternatively, we can encode the hierarchy of resources and another of roles, and if both hierarchies are satisfied then 🌟. + +The following code is shortened from our implementation of this approach, with the full function performing additional assertions and enabling Organization+Workspace relationships. + +```typescript +const RbacRoleHierarchy = ["ADMIN", "EDITOR", "READER"] as const; +const RbacResourceHierarchy = ["INSTANCE", "ORGANIZATION", "WORKSPACE"] as const; + +const doesUserHaveAccess = (permissions: PermissionRead[], query: RbacQuery) => { + const queryRoleHierarchy = RbacRoleHierarchy.indexOf(query.role); + const queryResourceHierarchy = RbacResourceHierarchy.indexOf(query.resource); + + return !!permissions.some((permission) => { + const [permissionResource, permissionRole] = partitionPermissionType(permission.permissionType); + + const permissionRoleHierarchy = RbacRoleHierarchy.indexOf(permissionRole); + const permissionResourceHierarchy = RbacResourceHierarchy.indexOf(permissionResource); + + return permissionRoleHierarchy <= queryRoleHierarchy && permissionResourceHierarchy <= queryResourceHierarchy; + }); +}; +``` diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx b/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx new file mode 100644 index 00000000000..302cbe3fb01 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx @@ -0,0 +1,221 @@ +import React, { Suspense, useState } from "react"; + +import { Button } from "components/ui/Button"; +import { FlexContainer, FlexItem } from "components/ui/Flex"; +import { Icon } from "components/ui/Icon"; +import { Input } from "components/ui/Input"; +import { ListBox } from "components/ui/ListBox"; + +import { PermissionType } from "core/request/AirbyteClient"; + +import { + RbacQuery, + RbacResource, + RbacResourceHierarchy, + RbacRole, + RbacRoleHierarchy, + useRbacPermissionsQuery, +} from "./rbacPermissionsQuery"; +import { withProviders } from "../../../../.storybook/withProvider"; + +const PermissionBuilder: React.FC<{ + resource: RbacResource; + setResource: (resource: RbacResource) => void; + role: RbacRole; + setRole: (role: RbacRole) => void; + id: string; + setId: (id: string) => void; +}> = ({ resource, setResource, role, setRole, id, setId }) => { + return ( +
+ + + + + + + + + { + setId(e.target.value); + }} + /> + + +
+ ); +}; + +interface PermissionQueryResultProps { + resourceType: RbacResource; + resourceId: string; + role: RbacRole; + permissions: RbacQuery[]; +} +const PermissionQueryResult: React.FC = ({ + resourceType, + resourceId, + role, + permissions, +}) => { + const query = resourceType === "INSTANCE" ? { resourceType, role } : { resourceType, role, resourceId }; + + const hasMatchingPermissions = useRbacPermissionsQuery( + permissions.map(({ resourceType, role, resourceId }) => { + return { + permissionId: "", + permissionType: `${resourceType.toLowerCase() as Lowercase}_${ + role.toLowerCase() as Lowercase + }` as PermissionType, + userId: "", + organizationId: resourceType === "ORGANIZATION" ? resourceId || undefined : undefined, + workspaceId: resourceType === "WORKSPACE" ? resourceId || undefined : undefined, + }; + }), + query + ); + + return {hasMatchingPermissions ? "✅" : "❌"}; +}; + +class PermissionQueryResultWithErrorBoundary extends React.Component { + state = { hasError: false }; + + static getDerivedStateFromError() { + return { hasError: true }; + } + + componentDidUpdate(prevProps: PermissionQueryResultProps) { + if (prevProps.resourceType !== this.props.resourceType || prevProps.resourceId !== this.props.resourceId) { + this.setState({ hasError: false }); + } + } + + render() { + const { resourceType, resourceId, role, permissions } = this.props; + + return this.state.hasError ? ( + "Error executing query, are you sure it's an UUID of an existing workspace?" + ) : ( + <> + Is permission granted?{" "} + + + ); + } +} + +const PermisisonTestViewInner = () => { + const [queryResource, setQueryResource] = useState(RbacResourceHierarchy[0]); + const [queryRole, setQueryRole] = useState(RbacRoleHierarchy[0]); + const [queryId, setQueryId] = useState(""); + + const [permissions, setPermissions] = useState([]); + const updatePermission = (index: number, type: "resourceType" | "role" | "resourceId", value: string) => { + const nextPermissions = [...permissions]; + // typescript can't validate that `value` is assignable to `type` + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore-next-line + nextPermissions[index][type] = value; + setPermissions(nextPermissions); + }; + + return ( +
+ +
+
+
+ Query +
+ +
+
 
+
+ + User permissions{" "} +
+ ))} +
+
+ ); +}; + +export const PermissionTestView = () => { + return withProviders(() => ( + Loading organization:workspace mapping
}> + + + )); +}; diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts new file mode 100644 index 00000000000..5444d652e55 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts @@ -0,0 +1,179 @@ +import { renderHook } from "@testing-library/react"; + +import { mockUser } from "test-utils/mock-data/mockUser"; +import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; + +import { useListPermissions, useCurrentWorkspace } from "core/api"; +import { PermissionRead, WorkspaceRead } from "core/request/AirbyteClient"; + +import { useRbac } from "./rbac"; +import { RbacPermission, useRbacPermissionsQuery } from "./rbacPermissionsQuery"; + +jest.mock("core/services/auth", () => ({ + useCurrentUser: () => mockUser, +})); + +jest.mock("./rbacPermissionsQuery", () => ({ + useRbacPermissionsQuery: jest.fn(), +})); +const mockUseRbacPermissionsQuery = useRbacPermissionsQuery as unknown as jest.Mock; + +jest.mock("core/api", () => { + const actual = jest.requireActual("core/api"); + return { + ...actual, + useListPermissions: jest.fn(() => ({ permissions: [] })), + useCurrentWorkspace: jest.fn(() => ({ ...mockWorkspace, organizationId: "test-organization" })), + }; +}); + +const mockUseListPermissions = useListPermissions as unknown as jest.Mock<{ permissions: RbacPermission[] }>; + +const mockUseCurrentWorkspace = useCurrentWorkspace as unknown as jest.Mock; +mockUseCurrentWorkspace.mockImplementation(() => ({ + ...mockWorkspace, + workspaceId: "test-workspace", + organizationId: "test-organization", +})); + +describe("useRbac", () => { + it("passes permissions", () => { + mockUseRbacPermissionsQuery.mockClear(); + + const permissions: PermissionRead[] = [ + { permissionId: "", userId: "", permissionType: "instance_admin" }, + { permissionId: "", userId: "", permissionType: "workspace_reader", organizationId: "work-18" }, + { permissionId: "", userId: "", permissionType: "organization_editor", organizationId: "org-1" }, + ]; + + mockUseListPermissions.mockImplementation(() => ({ + permissions: [...permissions], + })); + + renderHook(() => useRbac({ resourceType: "INSTANCE", role: "ADMIN" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[0]).toEqual(permissions); + }); + + describe("query assembly", () => { + it("no permissions", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [], + })); + + mockUseRbacPermissionsQuery.mockClear(); + renderHook(() => useRbac({ resourceType: "INSTANCE", role: "ADMIN" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ resourceType: "INSTANCE", role: "ADMIN" }); + }); + + it("instance admin does not need to add details to the query", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + mockUseRbacPermissionsQuery.mockClear(); + renderHook(() => useRbac({ resourceType: "INSTANCE", role: "ADMIN" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ resourceType: "INSTANCE", role: "ADMIN" }); + }); + + it("organizationId is found by the context", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + mockUseRbacPermissionsQuery.mockClear(); + renderHook(() => useRbac({ resourceType: "ORGANIZATION", role: "ADMIN" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ + resourceType: "ORGANIZATION", + role: "ADMIN", + resourceId: "test-organization", + }); + }); + + it("organizationId can be provided directly", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + mockUseRbacPermissionsQuery.mockClear(); + renderHook(() => useRbac({ resourceType: "ORGANIZATION", role: "ADMIN", resourceId: "some-other-organization" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ + resourceType: "ORGANIZATION", + role: "ADMIN", + resourceId: "some-other-organization", + }); + }); + + it("workspaceId is found by the context", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + mockUseRbacPermissionsQuery.mockClear(); + renderHook(() => useRbac({ resourceType: "WORKSPACE", role: "ADMIN" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ + resourceType: "WORKSPACE", + role: "ADMIN", + resourceId: "test-workspace", + }); + }); + + it("workspaceId can be provided directly", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + mockUseRbacPermissionsQuery.mockClear(); + renderHook(() => useRbac({ resourceType: "WORKSPACE", role: "ADMIN", resourceId: "some-other-workspace" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ + resourceType: "WORKSPACE", + role: "ADMIN", + resourceId: "some-other-workspace", + }); + }); + }); + + describe("degenerate cases", () => { + let existingWorkspaceMock: typeof mockUseCurrentWorkspace; // override any useCurrentWorkspaceId mock for this set of tests + const consoleError = console.error; // testing for errors in these tests, so we need to silence them + beforeAll(() => { + existingWorkspaceMock = mockUseCurrentWorkspace.getMockImplementation() as typeof mockUseCurrentWorkspace; + mockUseCurrentWorkspace.mockImplementation(() => mockWorkspace); + console.error = () => void 0; + }); + afterAll(() => { + mockUseCurrentWorkspace.mockImplementation(existingWorkspaceMock); + console.error = consoleError; + }); + + it("throws an error when instance query includes a resourceId", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + expect(() => + renderHook(() => useRbac({ resourceType: "INSTANCE", role: "ADMIN", resourceId: "some-workspace" })) + ).toThrowError("Invalid RBAC query: resource INSTANCE with resourceId some-workspace"); + }); + + it("throws an error when non-instance query is missing and cannot find a resourceId", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + expect(() => renderHook(() => useRbac({ resourceType: "ORGANIZATION", role: "ADMIN" }))).toThrowError( + "Invalid RBAC query: resource ORGANIZATION with resourceId undefined" + ); + + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + }); + }); +}); diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.ts b/airbyte-webapp/src/core/utils/rbac/rbac.ts new file mode 100644 index 00000000000..c64e38d4597 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/rbac.ts @@ -0,0 +1,38 @@ +import { useCurrentWorkspace, useListPermissions } from "core/api"; +import { useCurrentUser } from "core/services/auth"; + +import { RbacQuery, RbacQueryWithoutResourceId, useRbacPermissionsQuery } from "./rbacPermissionsQuery"; + +/** + * Takes a list of permissions and a full or partial query, returning a boolean representing if the user has any permissions satisfying the query + */ +export const useRbac = (query: RbacQuery | RbacQueryWithoutResourceId) => { + const { resourceType, role } = query; + let resourceId = "resourceId" in query ? query.resourceId : undefined; + + const queryUsesResourceId = resourceType !== "INSTANCE"; + + const { userId } = useCurrentUser(); + const { permissions } = useListPermissions(userId); + + const contextWorkspace = useCurrentWorkspace(); + if (queryUsesResourceId && !resourceId) { + // attempt to locate this from context + if (resourceType === "WORKSPACE") { + resourceId = contextWorkspace.workspaceId; + } else if (resourceType === "ORGANIZATION") { + resourceId = contextWorkspace.organizationId; + } + } + + // invariant check + if ((!queryUsesResourceId && resourceId) || (queryUsesResourceId && !resourceId)) { + throw new Error(`Invalid RBAC query: resource ${resourceType} with resourceId ${resourceId}`); + } + + // above invariant guarantees resourceId is defined when needed + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const assembledQuery = queryUsesResourceId ? { resourceType, resourceId: resourceId!, role } : { resourceType, role }; + + return useRbacPermissionsQuery(permissions, assembledQuery); +}; diff --git a/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.test.ts b/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.test.ts new file mode 100644 index 00000000000..dbdf24940a0 --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.test.ts @@ -0,0 +1,200 @@ +import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; + +import { RbacPermission, RbacQuery, partitionPermissionType, useRbacPermissionsQuery } from "./rbacPermissionsQuery"; + +jest.mock("core/api", () => { + const actual = jest.requireActual("core/api"); + return { + ...actual, + useGetWorkspace: jest.fn((workspaceId: string) => { + const workspace = { ...mockWorkspace }; + + if (workspaceId === "test-workspace") { + workspace.organizationId = "org-with-test-workspace"; + } else if (workspaceId === "workspace-1" || workspaceId === "workspace-2") { + workspace.organizationId = "org-with-two-workspaces"; + } + + return workspace; + }), + }; +}); + +describe("partitionPermissionType", () => { + it("correctly parses permissions", () => { + expect(partitionPermissionType("instance_admin")).toEqual(["INSTANCE", "ADMIN"]); + + expect(partitionPermissionType("organization_admin")).toEqual(["ORGANIZATION", "ADMIN"]); + expect(partitionPermissionType("organization_reader")).toEqual(["ORGANIZATION", "READER"]); + + expect(partitionPermissionType("workspace_admin")).toEqual(["WORKSPACE", "ADMIN"]); + expect(partitionPermissionType("workspace_reader")).toEqual(["WORKSPACE", "READER"]); + }); + + it("maps workspace_owner to workspace_admin", () => { + expect(partitionPermissionType("workspace_owner")).toEqual(["WORKSPACE", "ADMIN"]); + }); +}); + +describe("useRbacPermissionsQuery", () => { + describe("permission grants", () => { + // title, query, permissions, expectedResult + it.each<[string, RbacQuery, RbacPermission[], boolean | null]>([ + /* NO PERMISSIONS */ + [ + "no permission grants no access to workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "READER" }, + [], + false, + ], + [ + "no permission grants no access to organization", + { resourceType: "ORGANIZATION", resourceId: "test-workspace", role: "READER" }, + [], + false, + ], + ["no permission grants no access to instance", { resourceType: "INSTANCE", role: "ADMIN" }, [], false], + + /* BASIC WORKSPACE PERMISSIONS */ + [ + "workspace_reader permission grants access to the workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "READER" }, + [{ permissionType: "workspace_reader", workspaceId: "test-workspace" }], + true, + ], + [ + "workspace_admin permission grants access to the workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "READER" }, + [{ permissionType: "workspace_admin", workspaceId: "test-workspace" }], + true, + ], + [ + "workspace_admin permission on another workspace does not grant access to the workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "READER" }, + [{ permissionType: "workspace_admin", workspaceId: "an-whole-other-workspace" }], + false, + ], + [ + "workspace_reader permission on a workspace does not satisfy workspace_admin permission on the same workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "ADMIN" }, + [{ permissionType: "workspace_reader", workspaceId: "test-workspace" }], + false, + ], + [ + "workspace_reader AND workspace_admin permission on a workspace does satisfy workspace_admin permission on the same workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "ADMIN" }, + [ + { permissionType: "workspace_reader", workspaceId: "test-workspace" }, + { permissionType: "workspace_admin", workspaceId: "test-workspace" }, + ], + true, + ], + + /* BASIC ORGANIZATION PERMISSIONS */ + [ + "organization_reader permission grants access to the organization", + { resourceType: "ORGANIZATION", resourceId: "test-organization", role: "READER" }, + [{ permissionType: "organization_reader", organizationId: "test-organization" }], + true, + ], + [ + "organization_admin permission grants access to the organization", + { resourceType: "ORGANIZATION", resourceId: "test-organization", role: "READER" }, + [{ permissionType: "organization_admin", organizationId: "test-organization" }], + true, + ], + [ + "organization_admin permission on another organization does not grant access to the organization", + { resourceType: "ORGANIZATION", resourceId: "test-organization", role: "READER" }, + [{ permissionType: "organization_admin", organizationId: "an-whole-other-organization" }], + false, + ], + [ + "organization_reader permission on a organization does not satisfy organization_admin permission on the same organization", + { resourceType: "ORGANIZATION", resourceId: "test-organization", role: "ADMIN" }, + [{ permissionType: "organization_reader", organizationId: "test-organization" }], + false, + ], + [ + "organization_reader AND organization_admin permission on a organization does satisfy organization_admin permission on the same organization", + { resourceType: "ORGANIZATION", resourceId: "test-organization", role: "ADMIN" }, + [ + { permissionType: "organization_reader", organizationId: "test-organization" }, + { permissionType: "organization_admin", organizationId: "test-organization" }, + ], + true, + ], + + /* ORGANIZATION PERMISSIONS INFORMING WORKSPACES */ + [ + "organization_editor permission grants access to read its workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "READER" }, + [{ permissionType: "organization_editor", organizationId: "org-with-test-workspace" }], + true, + ], + [ + "organization_admin permission does not grant access to read external workspaces", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "READER" }, + [{ permissionType: "organization_admin", organizationId: "org-with-two-workspaces" }], + false, + ], + [ + "organization_admin permission does not grant access to read external workspaces but a workspace permission does", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "READER" }, + [ + { permissionType: "organization_admin", organizationId: "org-with-two-workspaces" }, + { permissionType: "workspace_editor", workspaceId: "test-workspace" }, + ], + true, + ], + + /* BASIC INSTANCE PERMISSIONS */ + [ + "instance_admin permission grants access to the instance", + { resourceType: "INSTANCE", role: "ADMIN" }, + [{ permissionType: "instance_admin" }], + true, + ], + [ + "instance_admin permission grants access to an organization", + { resourceType: "ORGANIZATION", resourceId: "test-organization", role: "ADMIN" }, + [{ permissionType: "instance_admin" }], + true, + ], + [ + "instance_admin permission grants access to a workspace", + { resourceType: "WORKSPACE", resourceId: "test-workspace", role: "ADMIN" }, + [{ permissionType: "instance_admin" }], + true, + ], + ])("%s", (_title, query, permissions, expectedResult) => { + expect(useRbacPermissionsQuery(permissions, query)).toBe(expectedResult); + }); + }); + + describe("degenerate cases", () => { + it("returns false when an organization or workspace resource permission is missing an id", () => { + expect( + useRbacPermissionsQuery([{ permissionType: "organization_reader" }], { + resourceType: "INSTANCE", + role: "ADMIN", + }) + ).toBe(false); + + expect( + useRbacPermissionsQuery([{ permissionType: "workspace_reader" }], { + resourceType: "INSTANCE", + role: "ADMIN", + }) + ).toBe(false); + + expect( + useRbacPermissionsQuery([{ permissionType: "organization_admin", organizationId: "test-organization" }], { + resourceType: "WORKSPACE", + role: "EDITOR", + resourceId: "", + }) + ).toBe(false); + }); + }); +}); diff --git a/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts b/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts new file mode 100644 index 00000000000..5afc2d4fffb --- /dev/null +++ b/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts @@ -0,0 +1,81 @@ +import { useGetWorkspace } from "core/api"; +import { PermissionRead } from "core/request/AirbyteClient"; + +export const RbacResourceHierarchy = ["INSTANCE", "ORGANIZATION", "WORKSPACE"] as const; +export const RbacRoleHierarchy = ["ADMIN", "EDITOR", "READER"] as const; +export type RbacResource = (typeof RbacResourceHierarchy)[number]; +export type RbacRole = (typeof RbacRoleHierarchy)[number]; + +export interface RbacQuery { + resourceType: RbacResource; + resourceId?: string; + role: RbacRole; +} +export type RbacQueryWithoutResourceId = Omit; // to allow optionally reading `resourceId` from React context + +// allows for easier object creation as we want to align with PermissionRead but have no use for permissionId or userId when processing permissions +export type RbacPermission = Omit; + +/** + * Accepts a permission type and splits it into its resource and role parts + */ +export const partitionPermissionType = (permissionType: RbacPermission["permissionType"]): [RbacResource, RbacRole] => { + // for legacy support, map workspace_owner to workspace_adin + if (permissionType === "workspace_owner") { + permissionType = "workspace_admin"; + } + + // type guarantees all of the enumerations of PermissionType are handled + type PermissionTypeParts = typeof permissionType extends `${infer Resource}_${infer Role}` ? [Resource, Role] : never; + + const [permissionResource, permissionRole] = permissionType.split("_") as PermissionTypeParts; + const rbacResource: RbacResource = permissionResource.toUpperCase() as Uppercase; + const rbacRole: RbacRole = permissionRole.toUpperCase() as Uppercase; + return [rbacResource, rbacRole]; +}; + +export const useRbacPermissionsQuery = (permissions: RbacPermission[], query: RbacQuery) => { + const queryRoleHierarchy = RbacRoleHierarchy.indexOf(query.role); + const queryResourceHierarchy = RbacResourceHierarchy.indexOf(query.resourceType); + + const owningOrganizationId = useGetWorkspace(query.resourceId ?? "", { + enabled: query.resourceType === "WORKSPACE" && !!query.resourceId, + })?.organizationId; + + return permissions.some((permission) => { + const [permissionResource, permissionRole] = partitionPermissionType(permission.permissionType); + + const permissionRoleHierarchy = RbacRoleHierarchy.indexOf(permissionRole); + const permissionResourceHierarchy = RbacResourceHierarchy.indexOf(permissionResource); + + const { organizationId, workspaceId } = permission; + + if (query.resourceType === "WORKSPACE") { + if (workspaceId && query.resourceId !== workspaceId) { + // workspace permission applies to a different workspace + return false; + } + + // is this permission for an organization + if (organizationId) { + if (!query.resourceId) { + return false; + } + + if (owningOrganizationId !== organizationId) { + // this organization permission does not apply to the workspace request + return false; + } + } + } + + if (query.resourceType === "ORGANIZATION") { + if (organizationId && query.resourceId !== organizationId) { + // organization permission applies to a different organization + return false; + } + } + + return permissionRoleHierarchy <= queryRoleHierarchy && permissionResourceHierarchy <= queryResourceHierarchy; + }); +}; diff --git a/airbyte-webapp/src/test-utils/mock-data/mockUser.ts b/airbyte-webapp/src/test-utils/mock-data/mockUser.ts new file mode 100644 index 00000000000..4741c007d28 --- /dev/null +++ b/airbyte-webapp/src/test-utils/mock-data/mockUser.ts @@ -0,0 +1,8 @@ +import { CommonUserRead } from "core/services/auth"; + +export const mockUser: CommonUserRead = { + userId: "mock-user", + email: "mockUser@airbyte.io", + authProvider: "airbyte", + authUserId: "mock-user", +}; From 0685ea49c116b8479994a724a8ade6f0d9c9612f Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 31 Aug 2023 10:01:50 +0200 Subject: [PATCH 070/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=A7=B9=20Remove=20?= =?UTF-8?q?SlickSlider=20component=20(#8604)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/package.json | 1 - airbyte-webapp/pnpm-lock.yaml | 33 +-------- .../ui/SlickSlider/SlickSlider.module.scss | 46 ------------ .../ui/SlickSlider/SlickSlider.stories.tsx | 43 ----------- .../components/ui/SlickSlider/SlickSlider.tsx | 71 ------------------- .../src/components/ui/SlickSlider/index.ts | 1 - .../src/components/ui/SlickSlider/slider.css | 19 ----- 7 files changed, 1 insertion(+), 213 deletions(-) delete mode 100644 airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.module.scss delete mode 100644 airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.stories.tsx delete mode 100644 airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx delete mode 100644 airbyte-webapp/src/components/ui/SlickSlider/index.ts delete mode 100644 airbyte-webapp/src/components/ui/SlickSlider/slider.css diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 773ec7f1dca..22641ccd481 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -112,7 +112,6 @@ "react-rnd": "^10.4.1", "react-router-dom": "6.14.2", "react-select": "^5.4.0", - "react-slick": "^0.29.0", "react-use": "^17.4.0", "react-virtualized-auto-sizer": "^1.0.17", "react-widgets": "^4.6.1", diff --git a/airbyte-webapp/pnpm-lock.yaml b/airbyte-webapp/pnpm-lock.yaml index c3dbd990ef2..59e6067c593 100644 --- a/airbyte-webapp/pnpm-lock.yaml +++ b/airbyte-webapp/pnpm-lock.yaml @@ -195,9 +195,6 @@ dependencies: react-select: specifier: ^5.4.0 version: 5.7.0(@babel/core@7.21.3)(@types/react@18.2.20)(react-dom@18.2.0)(react@18.2.0) - react-slick: - specifier: ^0.29.0 - version: 0.29.0(react-dom@18.2.0)(react@18.2.0) react-use: specifier: ^17.4.0 version: 17.4.0(react-dom@18.2.0)(react@18.2.0) @@ -10594,10 +10591,6 @@ packages: tapable: 2.2.1 dev: true - /enquire.js@2.1.6: - resolution: {integrity: sha512-/KujNpO+PT63F7Hlpu4h3pE3TokKRHN26JYmQpPyjkRD/N57R7bPDNojMXdi7uveAKjYB7yQnartCxZnFWr0Xw==} - dev: false - /enquirer@2.3.6: resolution: {integrity: sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==} engines: {node: '>=8.6'} @@ -14114,12 +14107,6 @@ packages: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} dev: true - /json2mq@0.2.0: - resolution: {integrity: sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA==} - dependencies: - string-convert: 0.2.1 - dev: false - /json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -14411,6 +14398,7 @@ packages: /lodash.debounce@4.0.8: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + dev: true /lodash.get@4.4.2: resolution: {integrity: sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==} @@ -16983,21 +16971,6 @@ packages: - '@types/react' dev: false - /react-slick@0.29.0(react-dom@18.2.0)(react@18.2.0): - resolution: {integrity: sha512-TGdOKE+ZkJHHeC4aaoH85m8RnFyWqdqRfAGkhd6dirmATXMZWAxOpTLmw2Ll/jPTQ3eEG7ercFr/sbzdeYCJXA==} - peerDependencies: - react: ^0.14.0 || ^15.0.1 || ^16.0.0 || ^17.0.0 || ^18.0.0 - react-dom: ^0.14.0 || ^15.0.1 || ^16.0.0 || ^17.0.0 || ^18.0.0 - dependencies: - classnames: 2.3.2 - enquire.js: 2.1.6 - json2mq: 0.2.0 - lodash.debounce: 4.0.8 - react: 18.2.0 - react-dom: 18.2.0(react@18.2.0) - resize-observer-polyfill: 1.5.1 - dev: false - /react-smooth@2.0.1(prop-types@15.8.1)(react-dom@18.2.0)(react@18.2.0): resolution: {integrity: sha512-Own9TA0GPPf3as4vSwFhDouVfXP15ie/wIHklhyKBH5AN6NFtdk0UpHBnonV11BtqDkAWlt40MOUc+5srmW7NA==} peerDependencies: @@ -18255,10 +18228,6 @@ packages: engines: {node: '>=0.6.19'} dev: true - /string-convert@0.2.1: - resolution: {integrity: sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A==} - dev: false - /string-length@4.0.2: resolution: {integrity: sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==} engines: {node: '>=10'} diff --git a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.module.scss b/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.module.scss deleted file mode 100644 index 4d8b5d326e4..00000000000 --- a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.module.scss +++ /dev/null @@ -1,46 +0,0 @@ -@use "../../../scss/colors"; -@use "../../../scss/variables"; - -.container { - background: colors.$foreground; - display: inline-block; - position: relative; - width: 100%; -} - -.titleContainer { - min-height: variables.$spacing-lg; - margin-bottom: variables.$spacing-md; -} - -%arrow { - position: absolute; - border: none; - background-color: colors.$foreground; - top: 0; - font-size: 11px; - width: 17px; - height: 17px; - color: colors.$dark-blue-900; - display: flex; - align-items: center; - justify-content: center; - transition: variables.$transition; - border-radius: variables.$border-radius-xs; -} - -.leftArrow { - @extend %arrow; - - right: 35px; -} - -.rightArrow { - @extend %arrow; - - right: 0; -} - -.arrowDisabled { - color: colors.$grey-200; -} diff --git a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.stories.tsx b/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.stories.tsx deleted file mode 100644 index 36a26c98242..00000000000 --- a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.stories.tsx +++ /dev/null @@ -1,43 +0,0 @@ -import { ComponentStory, ComponentMeta } from "@storybook/react"; - -import { SlickSlider } from "./SlickSlider"; - -export default { - title: "UI/SlickSlider", - component: SlickSlider, -} as ComponentMeta; - -const cardStyle = { - height: 100, - border: "1px solid blue", - display: "flex", - justifyContent: "center", - alignItems: "center", -}; - -export const Primary: ComponentStory = (args) => ( -
- -
-
1
-
-
-
2
-
-
-
3
-
-
-
4
-
-
-
5
-
- -
-); - -export const WithTitle = Primary.bind({}); -WithTitle.args = { - title: "Test title text", -}; diff --git a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx b/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx deleted file mode 100644 index ec31d79174e..00000000000 --- a/airbyte-webapp/src/components/ui/SlickSlider/SlickSlider.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import { faChevronLeft, faChevronRight } from "@fortawesome/free-solid-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import classnames from "classnames"; -import React, { useMemo } from "react"; -import Slider, { CustomArrowProps, Settings as SliderProps } from "react-slick"; - -import { Text } from "components/ui/Text"; - -import styles from "./SlickSlider.module.scss"; - -import "./slider.css"; - -const PrevArrow = ({ slideCount, currentSlide, className, onClick, ...restProps }: CustomArrowProps) => ( - -); -const NextArrow = ({ slideCount, currentSlide, className, onClick, ...restProps }: CustomArrowProps) => ( - -); - -interface SlickSliderProps { - title?: string; - sliderSettings?: SliderProps; - children: React.ReactNode; -} - -export const SlickSlider: React.FC = ({ title, sliderSettings, children }) => { - const settings: SliderProps = useMemo( - () => ({ - arrows: true, - accessibility: true, - infinite: false, - swipeToSlide: true, - speed: 200, - slidesToShow: 2, - slidesToScroll: 2, - prevArrow: , - nextArrow: , - ...sliderSettings, - }), - [sliderSettings] - ); - - return ( -
-
{title && {title}}
- {children} -
- ); -}; diff --git a/airbyte-webapp/src/components/ui/SlickSlider/index.ts b/airbyte-webapp/src/components/ui/SlickSlider/index.ts deleted file mode 100644 index 8ceacd03c5e..00000000000 --- a/airbyte-webapp/src/components/ui/SlickSlider/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { SlickSlider } from "./SlickSlider"; diff --git a/airbyte-webapp/src/components/ui/SlickSlider/slider.css b/airbyte-webapp/src/components/ui/SlickSlider/slider.css deleted file mode 100644 index cd09efdcb92..00000000000 --- a/airbyte-webapp/src/components/ui/SlickSlider/slider.css +++ /dev/null @@ -1,19 +0,0 @@ -/* Override default Slider styles: - this is one of ways how to override slider style without adding them to global style file -*/ -/* stylelint-disable selector-class-pattern */ -.slick-list { - position: relative; - display: block; - overflow: hidden; - user-select: none; - margin: 0 -9px; /* add space between slides */ -} - -.slick-track { - display: flex; /* make horizontal slider */ -} - -.slick-slide { - margin: 0 10px; /* add space between slides */ -} From f33758e1a25a17967780615167f3df335707a049 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 31 Aug 2023 11:02:46 +0200 Subject: [PATCH 071/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=A8=20Show=20ic?= =?UTF-8?q?on=20of=20selected=20option=20in=20ListBox=20(#8603)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/components/ui/ListBox/ListBox.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx index 977f3c33c62..cf6d3b8a2d9 100644 --- a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx +++ b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx @@ -24,7 +24,10 @@ const DefaultControlButton = ({ selectedOption, isDisabled }: ListBoxControl <> {selectedOption ? ( - {selectedOption.label} + + {selectedOption.icon && {selectedOption.icon}} + {selectedOption.label} + ) : ( From d6ce9be135280f76e2f03d5d691f4727451e068b Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 31 Aug 2023 13:29:09 +0200 Subject: [PATCH 072/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Disable?= =?UTF-8?q?=20flaky=20Cloud=20E2E=20test=20(#8636)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts b/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts index ff2766766f6..afd0cbcd03e 100644 --- a/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts +++ b/airbyte-webapp/cypress/cloud-e2e/feature-flags.cy.ts @@ -1,6 +1,6 @@ import { FeatureItem } from "@src/core/services/features"; -describe("AllowDBTCloudIntegration", () => { +describe.skip("AllowDBTCloudIntegration", () => { beforeEach(() => { cy.login(); cy.selectWorkspace(); From 118e8f7054b4f1ab1aec284d96cf8689ac0cc1b5 Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Thu, 31 Aug 2023 08:09:35 -0700 Subject: [PATCH 073/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=A8=20Add=20not?= =?UTF-8?q?e=20about=20connector=20upgrade=20being=20irreversible=20(#8613?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ella Rohm-Ensing --- .../BreakingChangeBanner.module.scss | 1 + .../connector/BreakingChangeBanner.tsx | 25 +++++++++++++------ airbyte-webapp/src/locales/en.json | 2 ++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss b/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss index d155836e616..1bf334127ed 100644 --- a/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss +++ b/airbyte-webapp/src/components/connector/BreakingChangeBanner.module.scss @@ -1,4 +1,5 @@ @use "scss/variables"; +@use "scss/colors"; .additionalContent { margin-top: variables.$spacing-md; diff --git a/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx b/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx index b8aca1fa533..62ea8339067 100644 --- a/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx +++ b/airbyte-webapp/src/components/connector/BreakingChangeBanner.tsx @@ -43,6 +43,7 @@ export const BreakingChangeBanner = ({ const { mutateAsync: upgradeVersion } = useUpgradeConnectorVersion(connectorType, connectorId, connectorDefinitionId); const { registerNotification } = useNotificationService(); const connectorBreakingChangeDeadlines = useFeature(FeatureItem.ConnectorBreakingChangeDeadlines); + const allowUpdateConnectors = useFeature(FeatureItem.AllowUpdateConnectors); const supportState = actorDefinitionVersion.supportState; @@ -95,15 +96,15 @@ export const BreakingChangeBanner = ({ openConfirmationModal({ title: `connector.breakingChange.upgradeModal.title.${connectorType}`, text: - connections.length === 0 - ? "connector.breakingChange.upgradeModal.areYouSure" - : "connector.breakingChange.upgradeModal.text", + connections.length > 0 + ? "connector.breakingChange.upgradeModal.text" + : "connector.breakingChange.upgradeModal.textNoConnections", textValues: { name: connectorName, count: connections.length, type: connectorType }, submitButtonText: "connector.breakingChange.upgradeModal.confirm", submitButtonVariant: "primary", - additionalContent: - connections.length > 0 ? ( - + additionalContent: ( + + {connections.length > 0 && (
    {connections.slice(0, MAX_CONNECTION_NAMES).map((connection, idx) => ( @@ -133,12 +134,20 @@ export const BreakingChangeBanner = ({ )} + )} + + {!allowUpdateConnectors && ( + <> + {" "} + + )} - - ) : undefined, + + + ), onSubmit: handleSubmitUpgrade, }); }} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 356c792ffb0..efcd9d911c6 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -828,6 +828,8 @@ "connector.breakingChange.upgradeModal.title.destination": "Confirm destination upgrade", "connector.breakingChange.upgradeModal.confirm": "Confirm upgrade", "connector.breakingChange.upgradeModal.text": "{name} is used in {count, plural, one {# connection} other {# connections}}. Upgrading this {type} will affect all associated connections:", + "connector.breakingChange.upgradeModal.textNoConnections": "{name} is used in no connections.", + "connector.breakingChange.upgradeModal.irreversible": "Upgrading is an irreversible action.", "connector.breakingChange.upgradeModal.areYouSure": "Are you sure you want to upgrade this {type}?", "connector.breakingChange.upgradeModal.moreConnections": "+ {count} more connections", "connector.breakingChange.upgradeToast.success": "{type} upgraded successfully. See this guide for any remaining actions.", From 08f8ee741875725f77b1b21cc7cdaed6944c651c Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Thu, 31 Aug 2023 08:09:52 -0700 Subject: [PATCH 074/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Fix=20ses?= =?UTF-8?q?sion=20token=20auth=20conversion=20and=20related=20issues=20(#8?= =?UTF-8?q?605)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../convertManifestToBuilderForm.ts | 3 ++- .../src/components/connectorBuilder/types.ts | 26 +++++++++---------- .../connectorBuilder/useInferredInputs.ts | 6 ++--- .../ConnectorBuilderStateService.tsx | 4 ++- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts b/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts index db80615994c..b28de86c28a 100644 --- a/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts +++ b/airbyte-webapp/src/components/connectorBuilder/convertManifestToBuilderForm.ts @@ -45,6 +45,7 @@ import { DEFAULT_BUILDER_FORM_VALUES, DEFAULT_BUILDER_STREAM_VALUES, extractInterpolatedConfigKey, + getInferredAuthValue, hasIncrementalSyncUserInput, INCREMENTAL_SYNC_USER_INPUT_DATE_FORMAT, incrementalSyncInferredInputs, @@ -691,7 +692,7 @@ function manifestAuthenticatorToBuilder( for (const userInputAuthKey of userInputAuthKeys) { if ( !inferredInputs[userInputAuthKey].as_config_path && - !isInterpolatedConfigKey(Reflect.get(builderAuthenticator, userInputAuthKey)) + !isInterpolatedConfigKey(getInferredAuthValue(builderAuthenticator, userInputAuthKey)) ) { throw new ManifestCompatibilityError( undefined, diff --git a/airbyte-webapp/src/components/connectorBuilder/types.ts b/airbyte-webapp/src/components/connectorBuilder/types.ts index 0e67d621a87..e5e842bffd7 100644 --- a/airbyte-webapp/src/components/connectorBuilder/types.ts +++ b/airbyte-webapp/src/components/connectorBuilder/types.ts @@ -491,25 +491,25 @@ export function hasIncrementalSyncUserInput( ); } +export const getInferredAuthValue = (authenticator: BuilderFormAuthenticator, authKey: string) => { + if (authenticator.type === "SessionTokenAuthenticator") { + return Reflect.get(authenticator.login_requester.authenticator, authKey); + } + return Reflect.get(authenticator, authKey); +}; + export function getInferredInputList( - global: BuilderFormValues["global"], + authenticator: BuilderFormAuthenticator, inferredInputOverrides: BuilderFormValues["inferredInputOverrides"], startDateInput: boolean, endDateInput: boolean ): BuilderFormInput[] { - const authKeyToInferredInput = authTypeToKeyToInferredInput(global.authenticator); + const authKeyToInferredInput = authTypeToKeyToInferredInput(authenticator); const authKeys = Object.keys(authKeyToInferredInput); const inputs = authKeys.flatMap((authKey) => { if ( authKeyToInferredInput[authKey].as_config_path || - extractInterpolatedConfigKey( - Reflect.get( - global.authenticator.type === "SessionTokenAuthenticator" - ? global.authenticator.login_requester.authenticator - : global.authenticator, - authKey - ) - ) === authKeyToInferredInput[authKey].key + extractInterpolatedConfigKey(getInferredAuthValue(authenticator, authKey)) === authKeyToInferredInput[authKey].key ) { return [authKeyToInferredInput[authKey]]; } @@ -738,7 +738,7 @@ export const authenticatorSchema = yup.object({ }), session_token_path: yup.mixed().when("type", { is: SESSION_TOKEN_AUTHENTICATOR, - then: yup.array().of(yup.string()).min(1).required(REQUIRED_ERROR), + then: yup.array().of(yup.string()).min(1, REQUIRED_ERROR).required(REQUIRED_ERROR), otherwise: strip, }), expiration_duration: yup.mixed().when("type", { @@ -978,7 +978,7 @@ function builderAuthenticatorToManifest(globalSettings: BuilderFormValues["globa } if (globalSettings.authenticator.type === "SessionTokenAuthenticator") { const builderLoginRequester = globalSettings.authenticator.login_requester; - const { base, path } = splitUrl(builderLoginRequester.url); + const { base, path } = splitUrl(builderLoginRequester.url ?? ""); return { ...globalSettings.authenticator, login_requester: { @@ -1340,7 +1340,7 @@ export const convertToManifest = (values: BuilderFormValues): ConnectorManifest const orderedInputs = orderInputs( values.inputs, getInferredInputList( - values.global, + values.global.authenticator, values.inferredInputOverrides, hasIncrementalSyncUserInput(values.streams, "start_datetime"), hasIncrementalSyncUserInput(values.streams, "end_datetime") diff --git a/airbyte-webapp/src/components/connectorBuilder/useInferredInputs.ts b/airbyte-webapp/src/components/connectorBuilder/useInferredInputs.ts index 4392aa2eeb8..c7aa9091469 100644 --- a/airbyte-webapp/src/components/connectorBuilder/useInferredInputs.ts +++ b/airbyte-webapp/src/components/connectorBuilder/useInferredInputs.ts @@ -9,13 +9,13 @@ export const useInferredInputs = () => { if (!watch) { throw new Error("rhf context not available"); } - const global = watch("formValues.global"); + const authenticator = watch("formValues.global.authenticator"); const inferredInputOverrides = watch("formValues.inferredInputOverrides"); const streams = watch("formValues.streams"); const startDateInput = hasIncrementalSyncUserInput(streams, "start_datetime"); const endDateInput = hasIncrementalSyncUserInput(streams, "end_datetime"); return useMemo( - () => getInferredInputList(global, inferredInputOverrides, startDateInput, endDateInput), - [endDateInput, global, inferredInputOverrides, startDateInput] + () => getInferredInputList(authenticator, inferredInputOverrides, startDateInput, endDateInput), + [authenticator, endDateInput, inferredInputOverrides, startDateInput] ); }; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index caa0bad23fd..7a7f8408095 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -213,6 +213,7 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< const convertedManifest = removeEmptyProperties(convertToManifest(convertedFormValues)); // set jsonManifest first so that a save isn't triggered setJsonManifest(convertedManifest); + setPersistedState({ name: currentProject.name, manifest: convertedManifest }); setValue("formValues", convertedFormValues, { shouldValidate: true }); setValue("mode", "ui"); } catch (e) { @@ -240,6 +241,7 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< analyticsService, closeConfirmationModal, convertToBuilderFormValues, + currentProject.name, jsonManifest, openConfirmationModal, projectId, @@ -449,7 +451,7 @@ export function useInitializedBuilderProject() { return [convertToBuilderFormValuesSync(resolvedManifest), false, convertJsonToYaml(resolvedManifest)]; } catch (e) { // could not convert to form values, use default form values - return [DEFAULT_BUILDER_FORM_VALUES, true, convertJsonToYaml(DEFAULT_JSON_MANIFEST_VALUES)]; + return [DEFAULT_BUILDER_FORM_VALUES, true, convertJsonToYaml(resolvedManifest)]; } }, [builderProject.declarativeManifest?.manifest, resolvedManifest]); From def1104339c7b7e9e9227b72f1079f1582c1c3f2 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Thu, 31 Aug 2023 09:04:56 -0700 Subject: [PATCH 075/201] Attempt to detect if orchestrator OOMed (#8630) --- .../process/AsyncOrchestratorPodProcess.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java index 369687bbd51..89b8148df84 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java @@ -60,6 +60,7 @@ public class AsyncOrchestratorPodProcess implements KubePod { public static final String NO_OP = "NO_OP"; // TODO Ths frequency should be configured and injected rather hard coded here. public static final long JOB_STATUS_POLLING_FREQUENCY_IN_MILLIS = 5000; + private static final String JAVA_OOM_EXCEPTION_STRING = "java.lang.OutOfMemoryError"; private final KubePodInfo kubePodInfo; private final DocumentStoreClient documentStoreClient; @@ -177,6 +178,12 @@ private int computeExitValue() { } else { // otherwise, the actual pod is terminal when the doc store says it shouldn't be. log.info("The current non terminal state is {}", secondDocStoreStatus); + + if (isOOM(pod)) { + log.warn("Terminating due to OutOfMemoryError"); + return 137; + } + log.warn("State Store missing status, however orchestrator pod {} in terminal. Assume failure.", getInfo().name()); return 3; } @@ -190,6 +197,20 @@ private int computeExitValue() { } } + private boolean isOOM(final Pod pod) { + if (pod == null) { + return false; + } + try { + return kubernetesClient.pods().withName(pod.getFullResourceName()).tailingLines(5).getLog().contains(JAVA_OOM_EXCEPTION_STRING); + } catch (final Exception e) { + // We are trying to detect if the pod OOMed, if we fail to check the logs, we don't want to add more + // exception noise since this is an extra inspection attempt for more precise error logging. + log.info("Failed to retrieve pod logs for additional debugging information.", e); + } + return false; + } + @Override public int exitValue() { final var optionalCached = cachedExitValue.get(); From 7055654e9ed55531ca958f778caa30e43e092942 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 31 Aug 2023 13:01:38 -0400 Subject: [PATCH 076/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20(flagged)?= =?UTF-8?q?=20Allow=20OSS=20users=20to=20delete=20workspaces=20=20(#8600)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Chandler Prall --- .../src/core/api/hooks/workspaces.tsx | 2 +- airbyte-webapp/src/locales/en.json | 5 ++ .../GeneralWorkspaceSettingsPage.tsx | 16 +++-- .../components/DeleteWorkspace.tsx | 65 +++++++++++++++++++ 4 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx diff --git a/airbyte-webapp/src/core/api/hooks/workspaces.tsx b/airbyte-webapp/src/core/api/hooks/workspaces.tsx index ccd7e941720..4f7b8dcf295 100644 --- a/airbyte-webapp/src/core/api/hooks/workspaces.tsx +++ b/airbyte-webapp/src/core/api/hooks/workspaces.tsx @@ -52,7 +52,7 @@ export const useCreateWorkspace = () => { }); }; -export const useDeleteCurrentWorkspace = () => { +export const useDeleteWorkspace = () => { const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index efcd9d911c6..4521eda212f 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -706,6 +706,11 @@ "settings.workspaceSettings.updateWorkspaceNameForm.name.label": "Workspace name", "settings.workspaceSettings.updateWorkspaceNameForm.name.placeholder": "Workspace name", "settings.workspaceSettings.deleteLabel": "Delete your workspace", + "settings.workspaceSettings.deleteModal.confirm": "Delete workspace", + "settings.workspaceSettings.deleteModal.text": "Deleting {workspace} will remove it for all users and cancel all pending syncs.", + "settings.workspaceSettings.deleteModal.proceed": "Are you sure you want to proceed?", + "settings.workspaceSettings.delete.success": "Workspace deleted successfully", + "settings.workspaceSettings.delete.error": "There was an error deleting this workspace", "settings.generalSettings": "General Settings", "settings.organizationSettings": "Organization settings", "settings.organizationSettings.organizationName": "Organization name", diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx index f2f0746f66b..35bb7bb9441 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx @@ -2,15 +2,21 @@ import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; import { Card } from "components/ui/Card"; +import { FlexContainer } from "components/ui/Flex"; import { UpdateWorkspaceNameForm } from "area/workspace/components/UpdateWorkspaceNameForm"; +import { DeleteWorkspace } from "./components/DeleteWorkspace"; + export const GeneralWorkspaceSettingsPage = () => { return ( - }> - - - - + + }> + + + + + + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx b/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx new file mode 100644 index 00000000000..a05a329b33b --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx @@ -0,0 +1,65 @@ +import { FormattedMessage, useIntl } from "react-intl"; +import { useNavigate } from "react-router-dom"; + +import { Box } from "components/ui/Box"; +import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; +import { Text } from "components/ui/Text"; + +import { useCurrentWorkspace, useDeleteWorkspace } from "core/api"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; +import { useNotificationService } from "hooks/services/Notification"; +import { RoutePaths } from "pages/routePaths"; + +export const DeleteWorkspace: React.FC = () => { + const { formatMessage } = useIntl(); + const workspace = useCurrentWorkspace(); + const navigate = useNavigate(); + const { registerNotification } = useNotificationService(); + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + const { mutateAsync: deleteWorkspace, isLoading: isDeletingWorkspace } = useDeleteWorkspace(); + + const onDeleteClick = () => { + openConfirmationModal({ + title: formatMessage({ id: "settings.workspaceSettings.deleteLabel" }), + text: formatMessage({ id: "settings.workspaceSettings.deleteModal.proceed" }), + additionalContent: ( + + + + + + ), + submitButtonText: formatMessage({ id: "settings.workspaceSettings.deleteModal.confirm" }), + onSubmit() { + closeConfirmationModal(); + deleteWorkspace(workspace.workspaceId) + .then(() => { + registerNotification({ + id: "settings.workspace.delete.success", + text: formatMessage({ id: "settings.workspaceSettings.delete.success" }), + type: "success", + }); + navigate(`/${RoutePaths.Workspaces}`); + }) + .catch(() => { + registerNotification({ + id: "settings.workspace.delete.error", + text: formatMessage({ id: "settings.workspaceSettings.delete.error" }), + type: "error", + }); + }); + }, + }); + }; + + return ( + + + + + + ); +}; From 9fac1a19823134a8d5b2d9638341ce130e5c47de Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Thu, 31 Aug 2023 10:09:08 -0700 Subject: [PATCH 077/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=A8=20Center=20?= =?UTF-8?q?the=20Connection=20page=20breaking=20change=20banner=20button?= =?UTF-8?q?=20(#8609)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connections/StreamStatusPage/ErrorMessage.module.scss | 4 ++++ .../src/pages/connections/StreamStatusPage/ErrorMessage.tsx | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.module.scss b/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.module.scss index 9e5984391c4..6168eb34e34 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.module.scss +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.module.scss @@ -3,3 +3,7 @@ .error { flex: 1; } + +.breakingChangeButton { + align-self: center; +} diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.tsx index d7129dd96d5..e8203d37b71 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/ErrorMessage.tsx @@ -157,6 +157,9 @@ export const ErrorMessage: React.FC = () => { actionBtnText: formatMessage({ id: "connectionForm.breakingChange.source.buttonLabel", }), + actionBtnProps: { + className: styles.breakingChangeButton, + }, type: errorType, iconOverride: "warning", } as const; @@ -187,6 +190,9 @@ export const ErrorMessage: React.FC = () => { actionBtnText: formatMessage({ id: "connectionForm.breakingChange.destination.buttonLabel", }), + actionBtnProps: { + className: styles.breakingChangeButton, + }, type: errorType, iconOverride: "warning", } as const; From f0d19b830d4835c324b842eb5b971a292c1a2879 Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Thu, 31 Aug 2023 10:17:49 -0700 Subject: [PATCH 078/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20Add=20sup?= =?UTF-8?q?port=20for=20display=5Ftype=20setting=20on=20oneOfs=20(#8493)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/core/form/schemaToFormBlock.ts | 1 + airbyte-webapp/src/core/form/types.ts | 1 + airbyte-webapp/src/core/jsonSchema/types.ts | 1 + .../src/hooks/services/Experiment/experiments.ts | 1 - .../components/Sections/ConditionSection.tsx | 16 ++-------------- 5 files changed, 5 insertions(+), 15 deletions(-) diff --git a/airbyte-webapp/src/core/form/schemaToFormBlock.ts b/airbyte-webapp/src/core/form/schemaToFormBlock.ts index 6000644d19f..53c97bba5b4 100644 --- a/airbyte-webapp/src/core/form/schemaToFormBlock.ts +++ b/airbyte-webapp/src/core/form/schemaToFormBlock.ts @@ -168,6 +168,7 @@ const defaultFields = [ "always_show", "pattern_descriptor", "group", + "display_type", ] as const; const pickDefaultFields = (schema: AirbyteJSONSchema) => { diff --git a/airbyte-webapp/src/core/form/types.ts b/airbyte-webapp/src/core/form/types.ts index b55962c3ba3..7d0904c20fb 100644 --- a/airbyte-webapp/src/core/form/types.ts +++ b/airbyte-webapp/src/core/form/types.ts @@ -27,6 +27,7 @@ type FormRelevantJSONSchema = Pick< | "pattern_descriptor" | "group" | "readOnly" + | "display_type" >; interface FormItem extends FormRelevantJSONSchema { diff --git a/airbyte-webapp/src/core/jsonSchema/types.ts b/airbyte-webapp/src/core/jsonSchema/types.ts index 959186d5421..40a186b5d5b 100644 --- a/airbyte-webapp/src/core/jsonSchema/types.ts +++ b/airbyte-webapp/src/core/jsonSchema/types.ts @@ -8,6 +8,7 @@ interface AirbyteJSONSchemaProps { group?: string; always_show?: boolean; pattern_descriptor?: string; + display_type?: "dropdown" | "radio"; } /** diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts index 709e358461b..d31dad825b1 100644 --- a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts +++ b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts @@ -29,7 +29,6 @@ export interface Experiments { "connector.showRequestSchemabutton": boolean; "connector.suggestedSourceConnectors": string; "connector.suggestedDestinationConnectors": string; - "connector.updateMethodSelection": boolean; "onboarding.speedyConnection": boolean; "settings.emailNotifications": boolean; "upcomingFeaturesPage.url": string; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx index c393e62897a..0ebb00bf818 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/ConditionSection.tsx @@ -10,15 +10,12 @@ import { RadioButton } from "components/ui/RadioButton"; import { Text } from "components/ui/Text"; import { TextWithHTML } from "components/ui/TextWithHTML"; -import { ConnectorIds } from "area/connector/utils"; import { FormConditionItem } from "core/form/types"; -import { useExperiment } from "hooks/services/Experiment"; import styles from "./ConditionSection.module.scss"; import { FormSection } from "./FormSection"; import { GroupLabel } from "./GroupLabel"; import { SectionContainer } from "./SectionContainer"; -import { useConnectorForm } from "../../connectorFormContext"; import { setDefaultValues } from "../../useBuildForm"; interface ConditionSectionProps { @@ -34,15 +31,6 @@ export const ConditionSection: React.FC = ({ formField, p const { setValue, clearErrors } = useFormContext(); const value = useWatch({ name: path }); - const { selectedConnectorDefinition } = useConnectorForm(); - - const showRadioButtonCards = - useExperiment("connector.updateMethodSelection", true) && - selectedConnectorDefinition && - "sourceDefinitionId" in selectedConnectorDefinition && - selectedConnectorDefinition.sourceDefinitionId === ConnectorIds.Sources.MySql && - path === "connectionConfiguration.replication_method"; - const { conditions, selectionConstValues } = formField; const currentSelectionValue = useWatch({ name: `${path}.${formField.selectionKey}` }); let currentlySelectedCondition: number | undefined = selectionConstValues.indexOf(currentSelectionValue); @@ -89,7 +77,7 @@ export const ConditionSection: React.FC = ({ formField, p key={`form-field-group-${formField.fieldKey}`} label={} control={ - showRadioButtonCards ? undefined : ( + formField.display_type === "radio" ? undefined : ( = ({ formField, p {/* currentlySelectedCondition is only falsy if a malformed config is loaded which doesn't have a valid value for the const selection key. In this case, render the selection group as empty. */} {typeof currentlySelectedCondition !== "undefined" && ( <> - {showRadioButtonCards && ( + {formField.display_type === "radio" && ( {options.map((option) => ( } - /> + > + {isConfirmUpdateOpen && ( + setConfirmUpdateOpen(false)} + /> + )} + ); }; @@ -206,3 +127,163 @@ const Anchor: React.FC> = ({ href, children {children} ); + +interface ConfirmUpdateModalProps { + connectorId: string; + connectorDefinitionId: string; + connectorName: string; + connectorType: "source" | "destination"; + connections: WebBackendConnectionListItem[]; + migrationGuideUrl: string; + onClose: () => void; +} + +const ConfirmUpdateModal = ({ + connectorId, + connectorDefinitionId, + connectorName, + connectorType, + connections, + migrationGuideUrl, + onClose, +}: ConfirmUpdateModalProps) => { + const { formatMessage } = useIntl(); + const navigate = useNavigate(); + const { mutateAsync: upgradeVersion, isLoading } = useUpgradeConnectorVersion( + connectorType, + connectorId, + connectorDefinitionId + ); + const { registerNotification } = useNotificationService(); + const allowUpdateConnectors = useFeature(FeatureItem.AllowUpdateConnectors); + + const [upgradeInputValue, setUpgradeInputValue] = useState(""); + + const handleSubmitUpgrade = () => { + upgradeVersion(connectorId) + .then(() => { + registerNotification({ + type: "success", + id: `connector.upgradeSuccess.${connectorId}`, + text: ( + {node}, + }} + /> + ), + timeout: false, + }); + }) + .catch(() => { + registerNotification({ + type: "error", + id: `connector.upgradeError.${connectorId}`, + text: ( + + ), + }); + }) + .finally(() => { + onClose(); + }); + }; + + return ( + } + onClose={onClose} + size="md" + > + + + + 0 + ? "connector.breakingChange.upgradeModal.text" + : "connector.breakingChange.upgradeModal.textNoConnections" + } + values={{ + name: connectorName, + count: connections.length, + type: connectorType, + }} + /> + + {connections.length > 0 && ( + +
      + {connections.slice(0, MAX_CONNECTION_NAMES).map((connection, idx) => ( +
    • + {connection.name} +
      • + ))} +
    + {connections.length > MAX_CONNECTION_NAMES && ( + + )} +     + )} + + + {node}, + br: () =>
    , + }} + /> +     + {!allowUpdateConnectors && } + + + + setUpgradeInputValue(event.target.value)} + /> +     + } + /> + +     + + + + +     + ); +}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 4521eda212f..66063a08139 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -823,20 +823,22 @@ "connector.settings": "Settings", "connector.connections": "Connections", "connector.breakingChange.title": "Action Required", - "connector.breakingChange.pendingUpgrade": "There is a pending upgrade for {name}.", + "connector.breakingChange.pendingUpgrade": "There is a pending upgrade for {name} with breaking changes.", "connector.breakingChange.version": "Version {version}", "connector.breakingChange.deprecatedUpgrade": "Upgrade {name} by {date} to continue syncing with this {type}.", "connector.breakingChange.unsupportedUpgrade": "Upgrade {name} to continue syncing with this {type}.", "connector.breakingChange.moreInfo": "For more information, see this guide.", "connector.breakingChange.upgradeButton": "Upgrade", - "connector.breakingChange.upgradeModal.title.source": "Confirm source upgrade", - "connector.breakingChange.upgradeModal.title.destination": "Confirm destination upgrade", + "connector.breakingChange.upgradeModal.title": "Confirm {type} upgrade", "connector.breakingChange.upgradeModal.confirm": "Confirm upgrade", "connector.breakingChange.upgradeModal.text": "{name} is used in {count, plural, one {# connection} other {# connections}}. Upgrading this {type} will affect all associated connections:", "connector.breakingChange.upgradeModal.textNoConnections": "{name} is used in no connections.", + "connector.breakingChange.upgradeModal.warning": "Upgrading is a potentially dangerous operation and could cause issues with your syncs and data. Please read the upgrade guide carefully before continuing.", "connector.breakingChange.upgradeModal.irreversible": "Upgrading is an irreversible action.", + "connector.breakingChange.upgradeModal.typeUpgrade": "If you're sure you're ready to upgrade, type upgrade below and click \"Confirm upgrade\".", + "connector.breakingChange.upgradeModal.typeUpgrade.value": "upgrade", "connector.breakingChange.upgradeModal.areYouSure": "Are you sure you want to upgrade this {type}?", - "connector.breakingChange.upgradeModal.moreConnections": "+ {count} more connections", + "connector.breakingChange.upgradeModal.moreConnections": "+ {count, plural, one {# more connection} other {# more connections}}", "connector.breakingChange.upgradeToast.success": "{type} upgraded successfully. See this guide for any remaining actions.", "connector.breakingChange.upgradeToast.failure": "Failed to upgrade {type}. Please reach out to support for assistance.", From 6b4546f40056747b3004ecdc19fc9dd7b0fc26a7 Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Thu, 31 Aug 2023 15:49:01 -0700 Subject: [PATCH 083/201] InstanceConfiguration API fixes: Default Workspace selection and removal of workspaceId from `/setup` call (#8617) --- airbyte-api/src/main/openapi/config.yaml | 4 - .../InstanceConfigurationHandler.java | 36 +++-- .../InstanceConfigurationHandlerTest.java | 145 +++++++++++------- .../persistence/WorkspacePersistence.java | 34 ++-- .../persistence/WorkspacePersistenceTest.java | 29 ++++ .../settings/SetupForm/SetupForm.tsx | 4 +- 6 files changed, 153 insertions(+), 99 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 9cee8047300..ec108c806b3 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -7553,15 +7553,11 @@ components: InstanceConfigurationSetupRequestBody: type: object required: - - workspaceId - email - anonymousDataCollection - initialSetupComplete - displaySetupWizard properties: - workspaceId: - type: string - format: uuid email: type: string anonymousDataCollection: diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java index 34d299af10e..85c88a8b325 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandler.java @@ -18,9 +18,9 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.User; import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.OrganizationPersistence; import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.config.persistence.WorkspacePersistence; import io.airbyte.validation.json.JsonValidationException; import io.micronaut.context.annotation.Value; import jakarta.inject.Singleton; @@ -42,7 +42,7 @@ public class InstanceConfigurationHandler { private final AirbyteEdition airbyteEdition; private final Optional airbyteKeycloakConfiguration; private final Optional activeAirbyteLicense; - private final ConfigRepository configRepository; + private final WorkspacePersistence workspacePersistence; private final WorkspacesHandler workspacesHandler; private final UserPersistence userPersistence; private final OrganizationPersistence organizationPersistence; @@ -54,7 +54,7 @@ public InstanceConfigurationHandler(@Value("${airbyte.webapp-url:null}") final S final AirbyteEdition airbyteEdition, final Optional airbyteKeycloakConfiguration, final Optional activeAirbyteLicense, - final ConfigRepository configRepository, + final WorkspacePersistence workspacePersistence, final WorkspacesHandler workspacesHandler, final UserPersistence userPersistence, final OrganizationPersistence organizationPersistence) { @@ -62,14 +62,15 @@ public InstanceConfigurationHandler(@Value("${airbyte.webapp-url:null}") final S this.airbyteEdition = airbyteEdition; this.airbyteKeycloakConfiguration = airbyteKeycloakConfiguration; this.activeAirbyteLicense = activeAirbyteLicense; - this.configRepository = configRepository; + this.workspacePersistence = workspacePersistence; this.workspacesHandler = workspacesHandler; this.userPersistence = userPersistence; this.organizationPersistence = organizationPersistence; } - public InstanceConfigurationResponse getInstanceConfiguration() throws IOException, ConfigNotFoundException { - final StandardWorkspace defaultWorkspace = getDefaultWorkspace(); + public InstanceConfigurationResponse getInstanceConfiguration() throws IOException { + final UUID defaultOrganizationId = getDefaultOrganizationId(); + final StandardWorkspace defaultWorkspace = getDefaultWorkspace(defaultOrganizationId); return new InstanceConfigurationResponse() .webappUrl(webappUrl) @@ -78,20 +79,23 @@ public InstanceConfigurationResponse getInstanceConfiguration() throws IOExcepti .auth(getAuthConfiguration()) .initialSetupComplete(defaultWorkspace.getInitialSetupComplete()) .defaultUserId(getDefaultUserId()) - .defaultOrganizationId(getDefaultOrganizationId()) + .defaultOrganizationId(defaultOrganizationId) .defaultWorkspaceId(defaultWorkspace.getWorkspaceId()); } public InstanceConfigurationResponse setupInstanceConfiguration(final InstanceConfigurationSetupRequestBody requestBody) throws IOException, JsonValidationException, ConfigNotFoundException { + final UUID defaultOrganizationId = getDefaultOrganizationId(); + final StandardWorkspace defaultWorkspace = getDefaultWorkspace(defaultOrganizationId); + // Update the default organization and user with the provided information updateDefaultOrganization(requestBody); updateDefaultUser(requestBody); // Update the underlying workspace to mark the initial setup as complete workspacesHandler.updateWorkspace(new WorkspaceUpdate() - .workspaceId(requestBody.getWorkspaceId()) + .workspaceId(defaultWorkspace.getWorkspaceId()) .email(requestBody.getEmail()) .displaySetupWizard(requestBody.getDisplaySetupWizard()) .anonymousDataCollection(requestBody.getAnonymousDataCollection()) @@ -136,7 +140,7 @@ private void updateDefaultUser(final InstanceConfigurationSetupRequestBody reque userPersistence.writeUser(defaultUser); } - private UUID getDefaultOrganizationId() throws IOException, ConfigNotFoundException { + private UUID getDefaultOrganizationId() throws IOException { return organizationPersistence.getDefaultOrganization() .orElseThrow(() -> new IllegalStateException("Default organization does not exist.")) .getOrganizationId(); @@ -157,14 +161,12 @@ private void updateDefaultOrganization(final InstanceConfigurationSetupRequestBo organizationPersistence.updateOrganization(defaultOrganization); } - // Currently, the default workspace is simply the first workspace created by the bootloader. This is - // hacky, but - // historically, the first workspace is used to store instance-level preferences. - // TODO introduce a proper means of persisting instance-level preferences instead of using the first - // workspace as a proxy. - private StandardWorkspace getDefaultWorkspace() throws IOException { - return configRepository.listStandardWorkspaces(true).stream().findFirst() - .orElseThrow(() -> new IllegalStateException("Default workspace does not exist.")); + // Historically, instance setup for an OSS installation of Airbyte was stored on the one and only + // workspace that was created for the instance. Now that OSS supports multiple workspaces, we + // use the default Organization ID to select a workspace to use for instance setup. This is a hack. + // TODO persist instance configuration to a separate resource, rather than using a workspace. + private StandardWorkspace getDefaultWorkspace(final UUID organizationId) throws IOException { + return workspacePersistence.getDefaultWorkspaceForOrganization(organizationId); } } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java index 92b41151958..e9fec7a8116 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/InstanceConfigurationHandlerTest.java @@ -5,6 +5,7 @@ package io.airbyte.commons.server.handlers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -24,15 +25,15 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.User; import io.airbyte.config.persistence.ConfigNotFoundException; -import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.OrganizationPersistence; import io.airbyte.config.persistence.UserPersistence; +import io.airbyte.config.persistence.WorkspacePersistence; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.util.List; import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -52,7 +53,7 @@ class InstanceConfigurationHandlerTest { private static final String DEFAULT_USER_NAME = "Default User Name"; @Mock - private ConfigRepository mConfigRepository; + private WorkspacePersistence mWorkspacePersistence; @Mock private UserPersistence mUserPersistence; @Mock @@ -72,17 +73,6 @@ void setup() throws IOException { activeAirbyteLicense = new ActiveAirbyteLicense(); activeAirbyteLicense.setLicense(new AirbyteLicense(LicenseType.PRO)); - - when(mUserPersistence.getDefaultUser()).thenReturn( - Optional.of(new User() - .withUserId(USER_ID) - .withName(DEFAULT_USER_NAME))); - - when(mOrganizationPersistence.getDefaultOrganization()).thenReturn( - Optional.of(new Organization() - .withOrganizationId(ORGANIZATION_ID) - .withName(DEFAULT_ORG_NAME) - .withUserId(USER_ID))); } @ParameterizedTest @@ -92,22 +82,16 @@ void setup() throws IOException { "false, true", "false, false" }) - void testGetInstanceConfiguration(final boolean isPro, final boolean isInitialSetupComplete) - throws IOException, ConfigNotFoundException { - when(mConfigRepository.listStandardWorkspaces(true)).thenReturn( - List.of(new StandardWorkspace() + void testGetInstanceConfiguration(final boolean isPro, final boolean isInitialSetupComplete) throws IOException { + stubGetDefaultUser(); + stubGetDefaultOrganization(); + + when(mWorkspacePersistence.getDefaultWorkspaceForOrganization(ORGANIZATION_ID)).thenReturn( + new StandardWorkspace() .withWorkspaceId(WORKSPACE_ID) - .withInitialSetupComplete(isInitialSetupComplete))); + .withInitialSetupComplete(isInitialSetupComplete)); - instanceConfigurationHandler = new InstanceConfigurationHandler( - WEBAPP_URL, - isPro ? AirbyteEdition.PRO : AirbyteEdition.COMMUNITY, - isPro ? Optional.of(keycloakConfiguration) : Optional.empty(), - isPro ? Optional.of(activeAirbyteLicense) : Optional.empty(), - mConfigRepository, - mWorkspacesHandler, - mUserPersistence, - mOrganizationPersistence); + instanceConfigurationHandler = getInstanceConfigurationHandler(isPro); final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() .edition(isPro ? EditionEnum.PRO : EditionEnum.COMMUNITY) @@ -126,6 +110,25 @@ void testGetInstanceConfiguration(final boolean isPro, final boolean isInitialSe assertEquals(expected, actual); } + @Test + void testSetupInstanceConfigurationAlreadySetup() throws IOException { + stubGetDefaultOrganization(); + + when(mWorkspacePersistence.getDefaultWorkspaceForOrganization(ORGANIZATION_ID)).thenReturn( + new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withInitialSetupComplete(true)); // already setup, should trigger an error + + instanceConfigurationHandler = getInstanceConfigurationHandler(true); + + assertThrows(IllegalStateException.class, () -> instanceConfigurationHandler.setupInstanceConfiguration( + new InstanceConfigurationSetupRequestBody() + .email("test@mail.com") + .displaySetupWizard(false) + .initialSetupComplete(false) + .anonymousDataCollection(false))); + } + @ParameterizedTest @CsvSource({ "true, true", @@ -135,23 +138,20 @@ void testGetInstanceConfiguration(final boolean isPro, final boolean isInitialSe }) void testSetupInstanceConfiguration(final boolean userNamePresent, final boolean orgNamePresent) throws IOException, JsonValidationException, ConfigNotFoundException { - when(mConfigRepository.listStandardWorkspaces(true)) - .thenReturn(List.of(new StandardWorkspace() - .withWorkspaceId(WORKSPACE_ID) - .withInitialSetupComplete(true))); // after the handler's update, the workspace should have initialSetupComplete: true when retrieved - instanceConfigurationHandler = new InstanceConfigurationHandler( - WEBAPP_URL, - AirbyteEdition.PRO, - Optional.of(keycloakConfiguration), - Optional.of(activeAirbyteLicense), - mConfigRepository, - mWorkspacesHandler, - mUserPersistence, - mOrganizationPersistence); + stubGetDefaultOrganization(); + stubGetDefaultUser(); + + // first time default workspace is fetched, initial setup complete is false. + // second time is after the workspace is updated, so initial setup complete is true. + when(mWorkspacePersistence.getDefaultWorkspaceForOrganization(ORGANIZATION_ID)) + .thenReturn( + new StandardWorkspace().withWorkspaceId(WORKSPACE_ID).withInitialSetupComplete(false)) + .thenReturn( + new StandardWorkspace().withWorkspaceId(WORKSPACE_ID).withInitialSetupComplete(true)); + + instanceConfigurationHandler = getInstanceConfigurationHandler(true); - final String userName = userNamePresent ? "test user" : DEFAULT_USER_NAME; - final String orgName = orgNamePresent ? "test org" : DEFAULT_ORG_NAME; final String email = "test@airbyte.com"; final InstanceConfigurationResponse expected = new InstanceConfigurationResponse() @@ -166,15 +166,25 @@ void testSetupInstanceConfiguration(final boolean userNamePresent, final boolean .defaultOrganizationId(ORGANIZATION_ID) .defaultWorkspaceId(WORKSPACE_ID); - final InstanceConfigurationResponse actual = instanceConfigurationHandler.setupInstanceConfiguration( - new InstanceConfigurationSetupRequestBody() - .workspaceId(WORKSPACE_ID) - .email(email) - .initialSetupComplete(true) - .anonymousDataCollection(true) - .displaySetupWizard(true) - .userName(userName) - .organizationName(orgName)); + final InstanceConfigurationSetupRequestBody requestBody = new InstanceConfigurationSetupRequestBody() + .email(email) + .displaySetupWizard(true) + .anonymousDataCollection(true) + .initialSetupComplete(true); + + String expectedUserName = DEFAULT_USER_NAME; + if (userNamePresent) { + expectedUserName = "test user"; + requestBody.setUserName(expectedUserName); + } + + String expectedOrgName = DEFAULT_ORG_NAME; + if (orgNamePresent) { + expectedOrgName = "test org"; + requestBody.setOrganizationName(expectedOrgName); + } + + final InstanceConfigurationResponse actual = instanceConfigurationHandler.setupInstanceConfiguration(requestBody); assertEquals(expected, actual); @@ -182,12 +192,12 @@ void testSetupInstanceConfiguration(final boolean userNamePresent, final boolean verify(mUserPersistence).writeUser(eq(new User() .withUserId(USER_ID) .withEmail(email) - .withName(userName))); + .withName(expectedUserName))); // verify the organization was updated with the name from the request verify(mOrganizationPersistence).updateOrganization(eq(new Organization() .withOrganizationId(ORGANIZATION_ID) - .withName(orgName) + .withName(expectedOrgName) .withEmail(email) .withUserId(USER_ID))); @@ -199,4 +209,31 @@ void testSetupInstanceConfiguration(final boolean userNamePresent, final boolean .initialSetupComplete(true))); } + private void stubGetDefaultUser() throws IOException { + when(mUserPersistence.getDefaultUser()).thenReturn( + Optional.of(new User() + .withUserId(USER_ID) + .withName(DEFAULT_USER_NAME))); + } + + private void stubGetDefaultOrganization() throws IOException { + when(mOrganizationPersistence.getDefaultOrganization()).thenReturn( + Optional.of(new Organization() + .withOrganizationId(ORGANIZATION_ID) + .withName(DEFAULT_ORG_NAME) + .withUserId(USER_ID))); + } + + private InstanceConfigurationHandler getInstanceConfigurationHandler(final boolean isPro) { + return new InstanceConfigurationHandler( + WEBAPP_URL, + isPro ? AirbyteEdition.PRO : AirbyteEdition.COMMUNITY, + isPro ? Optional.of(keycloakConfiguration) : Optional.empty(), + isPro ? Optional.of(activeAirbyteLicense) : Optional.empty(), + mWorkspacePersistence, + mWorkspacesHandler, + mUserPersistence, + mOrganizationPersistence); + } + } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java index 836bbfda4f1..352fd312da6 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java @@ -49,28 +49,20 @@ public List listWorkspacesByOrganizationId(final ResourcesByO } /** - * Find the workspace with the given ID and check if its organization ID is null. If so, update it. - * Otherwise, log a warning and do nothing. + * Fetch the oldest, non-tombstoned Workspace that belongs to the given Organization. */ - public void setOrganizationIdIfNull(final UUID workspaceId, final UUID organizationId) throws IOException { - database.transaction(ctx -> { - final boolean isExistingWorkspace = ctx.fetchExists(ctx.selectFrom(WORKSPACE).where(WORKSPACE.ID.eq(workspaceId))); - if (isExistingWorkspace) { - final boolean isNullOrganizationId = - ctx.fetchExists(ctx.selectFrom(WORKSPACE).where(WORKSPACE.ID.eq(workspaceId)).and(WORKSPACE.ORGANIZATION_ID.isNull())); - if (isNullOrganizationId) { - ctx.update(WORKSPACE) - .set(WORKSPACE.ORGANIZATION_ID, organizationId) - .where(WORKSPACE.ID.eq(workspaceId)) - .execute(); - } else { - log.warn("Workspace with ID {} already has an organization ID set. Skipping update.", workspaceId); - } - } else { - log.warn("Workspace with ID {} does not exist. Skipping update.", workspaceId); - } - return null; - }); + public StandardWorkspace getDefaultWorkspaceForOrganization(final UUID organizationId) throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(WORKSPACE.ORGANIZATION_ID.eq(organizationId)) + .and(WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.CREATED_AT.asc()) + .limit(1) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .findFirst() + .orElseThrow(() -> new RuntimeException("No workspace found for organization: " + organizationId)); } } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java index 2281164ffc8..ee21d13d08f 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java @@ -273,4 +273,33 @@ void testListWorkspacesInOrgWithKeyword() throws Exception { assertEquals(workspace, workspaces.get(0)); } + @Test + void testGetDefaultWorkspaceForOrganization() throws JsonValidationException, IOException { + final StandardWorkspace expectedWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspaceInOrganization1"); + + configRepository.writeStandardWorkspaceNoSecrets(expectedWorkspace); + + final StandardWorkspace tombstonedWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("tombstonedWorkspace") + .withTombstone(true); + + configRepository.writeStandardWorkspaceNoSecrets(tombstonedWorkspace); + + final StandardWorkspace laterWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("laterWorkspace"); + + configRepository.writeStandardWorkspaceNoSecrets(laterWorkspace); + + final StandardWorkspace actualWorkspace = workspacePersistence.getDefaultWorkspaceForOrganization(MockData.ORGANIZATION_ID_1); + + assertEquals(expectedWorkspace, actualWorkspace); + } + } diff --git a/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx b/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx index f128de72d10..4b2b356789c 100644 --- a/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx +++ b/airbyte-webapp/src/components/settings/SetupForm/SetupForm.tsx @@ -9,7 +9,6 @@ import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; -import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useConfig } from "config"; import { useSetupInstanceConfiguration } from "core/api"; @@ -44,13 +43,12 @@ const setupFormValidationSchema = yup.object().shape({ }); export const SetupForm: React.FC = () => { - const workspaceId = useCurrentWorkspaceId(); const { formatMessage } = useIntl(); const { mutateAsync: setUpInstance } = useSetupInstanceConfiguration(); const config = useConfig(); const onSubmit = async (values: SetupFormValues) => { - await setUpInstance({ ...values, workspaceId, initialSetupComplete: true, displaySetupWizard: false }); + await setUpInstance({ ...values, initialSetupComplete: true, displaySetupWizard: false }); }; return ( From 98c2158f4f15aebfce860ac9bfec4ad68478583a Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Thu, 31 Aug 2023 23:51:54 +0000 Subject: [PATCH 084/201] Bump Airbyte version from 0.50.24 to 0.0.99 --- .bumpversion.cfg | 2 +- .env | 2 +- airbyte-connector-builder-server/Dockerfile | 4 ++-- airbyte-container-orchestrator/Dockerfile | 2 +- airbyte-proxy/Dockerfile | 2 +- airbyte-server/Dockerfile | 2 +- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.yaml | 2 +- charts/airbyte/README.md | 20 +++++++++---------- 20 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7bc8e02fe00..0d91c47475d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.50.24 +current_version = 0.0.99-test commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index de28947e9be..f9aa6a82fe7 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.50.24 +VERSION=0.0.99 # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index 79e04ff511d..7e7c985c841 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -10,7 +10,7 @@ ENV PIP=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/pip COPY requirements.txt requirements.txt RUN ${PIP} install -r requirements.txt -ARG VERSION=0.50.24 +ARG VERSION=0.0.99 ENV APPLICATION airbyte-connector-builder-server ENV VERSION ${VERSION} @@ -23,5 +23,5 @@ ADD airbyte-app.tar /app # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] -LABEL io.airbyte.version=0.50.24 +LABEL io.airbyte.version=0.0.99 LABEL io.airbyte.name=airbyte/connector-builder-server \ No newline at end of file diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 633d9fae785..fa11a242d01 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -10,7 +10,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.50.24 +ARG VERSION=0.0.99 ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-proxy/Dockerfile b/airbyte-proxy/Dockerfile index 4542e765f64..a6ed8826364 100644 --- a/airbyte-proxy/Dockerfile +++ b/airbyte-proxy/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:latest -ARG VERSION=0.50.24 +ARG VERSION=0.0.99 ENV APPLICATION airbyte-proxy ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 4bdb76fb63e..56310cbf809 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,7 +3,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 5005 -ARG VERSION=0.50.24 +ARG VERSION=0.0.99 ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 0e39d6a0096..7b8a7a4270a 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 26589ed8573..c0ece8337c1 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 4f1f7de0ff1..bfb46ad463a 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index e1864712dbc..a72c1704b2c 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index e8065bbbb11..61908bd3a33 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index f72cc8762b7..10e51e3b634 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 5700ab9e81a..ddc3b6081b6 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index b404497c300..1b08c62a067 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 0a52367e813..2db99360115 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index b5f9a09a85b..5c207c2e41a 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index e3c7501a97f..1c2bfa01723 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 32c4bd27453..f9e36e2d7d0 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 8a3ced9c0da..1cae5421001 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.4 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: 0.0.99 dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index e87bc3d30de..cb3217ebeda 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,6 @@ # airbyte -![Version: 0.50.24](https://img.shields.io/badge/Version-0.50.24-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.50.24](https://img.shields.io/badge/AppVersion-0.50.24-informational?style=flat-square) +![Version: 0.0.99](https://img.shields.io/badge/Version-0.0.99-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.99](https://img.shields.io/badge/AppVersion-0.0.99-informational?style=flat-square) Helm chart to deploy airbyte @@ -8,15 +8,15 @@ Helm chart to deploy airbyte | Repository | Name | Version | |------------|------|---------| -| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | cron | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | metrics | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | server | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | temporal | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | webapp | 0.50.24 | -| https://airbytehq.github.io/helm-charts/ | worker | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | cron | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | metrics | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | server | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | temporal | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | webapp | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | worker | 0.0.99 | | https://charts.bitnami.com/bitnami | common | 1.x.x | ## Values From c284350ad2003dc754b054fb34ad8362189bd251 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Fri, 1 Sep 2023 00:06:57 +0000 Subject: [PATCH 085/201] Bump helm chart version reference to 0.48.5 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 7b8a7a4270a..acd2e869cdf 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index c0ece8337c1..7b2763ffc95 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index bfb46ad463a..e5ddfe5a591 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index a72c1704b2c..f66b3e6c32e 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 61908bd3a33..e5091cd3a9e 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 10e51e3b634..238065e3b9d 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index ddc3b6081b6..e7f93f322c7 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 1b08c62a067..7782ea7aa45 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 2db99360115..420be52cd7a 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 5c207c2e41a..c4e7c570b9d 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 1c2bfa01723..9759f480dbd 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index f9e36e2d7d0..15b6ba5df98 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 030b4cbf57f..33d6464d5f4 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 -digest: sha256:6b96b3584fa1574e01e37d11686769482028c67bb9e032610a93473fb61f3602 -generated: "2023-08-30T21:26:08.620169815Z" + version: 0.48.5 +digest: sha256:5cffde2650218f52829dff0bcb34d9962c94b47f0eb0a34feea665ab1e5fdebc +generated: "2023-09-01T00:06:46.698637914Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 1cae5421001..0b030ee01c0 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.4 +version: 0.48.5 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 + version: 0.48.5 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.4 \ No newline at end of file + version: 0.48.5 \ No newline at end of file From 19b5afb786d9f31457596836ff3dabc1bc7fb6cd Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 1 Sep 2023 10:00:03 +0200 Subject: [PATCH 086/201] =?UTF-8?q?=F0=9F=AA=9F=F0=9F=94=A7=20Reduce=20re-?= =?UTF-8?q?renders=20on=20connectors=20page=20(#8607)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: lmossman --- .../pages/ConnectorsPage/DestinationsPage.tsx | 10 ++-- .../pages/ConnectorsPage/SourcesPage.tsx | 10 ++-- .../components/ConnectorCell.tsx | 46 +++++++++---------- .../components/ConnectorsView.tsx | 19 +++++--- .../components/VersionCell/VersionCell.tsx | 1 + 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx index eed312ba69c..f492d1fa298 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useMemo, useState } from "react"; +import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { DestinationDefinitionRead } from "core/request/AirbyteClient"; @@ -30,6 +30,8 @@ const DestinationsPage: React.FC = () => { }, new Map()), [destinationDefinitions] ); + const definitionMap = useRef(idToDestinationDefinition); + definitionMap.current = idToDestinationDefinition; const onUpdateVersion = useCallback( async ({ id, version }: { id: string; version: string }) => { @@ -44,7 +46,7 @@ const DestinationsPage: React.FC = () => { text: ( ), type: "success", @@ -55,7 +57,7 @@ const DestinationsPage: React.FC = () => { text: formatMessage( { id: "admin.upgradeConnector.error" }, - { name: idToDestinationDefinition.get(id)?.name, version } + { name: definitionMap.current.get(id)?.name, version } ) + (error.message ? `: ${error.message}` : ""), type: "error", }); @@ -63,7 +65,7 @@ const DestinationsPage: React.FC = () => { setUpdatingDefinitionId(undefined); } }, - [formatMessage, idToDestinationDefinition, registerNotification, updateDestinationDefinition] + [formatMessage, registerNotification, updateDestinationDefinition] ); const usedDestinationDefinitions: DestinationDefinitionRead[] = useMemo(() => { diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx index 7a670e69302..62e9f96561a 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useMemo, useState } from "react"; +import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useListBuilderProjects } from "core/api"; @@ -31,6 +31,8 @@ const SourcesPage: React.FC = () => { }, new Map()), [sourceDefinitions] ); + const definitionMap = useRef(idToSourceDefinition); + definitionMap.current = idToSourceDefinition; const onUpdateVersion = useCallback( async ({ id, version }: { id: string; version: string }) => { @@ -45,7 +47,7 @@ const SourcesPage: React.FC = () => { text: ( ), type: "success", @@ -56,7 +58,7 @@ const SourcesPage: React.FC = () => { text: formatMessage( { id: "admin.upgradeConnector.error" }, - { name: idToSourceDefinition.get(id)?.name, version } + { name: definitionMap.current.get(id)?.name, version } ) + (error.message ? `: ${error.message}` : ""), type: "error", }); @@ -64,7 +66,7 @@ const SourcesPage: React.FC = () => { setUpdatingDefinitionId(undefined); } }, - [formatMessage, idToSourceDefinition, registerNotification, updateSourceDefinition] + [formatMessage, registerNotification, updateSourceDefinition] ); const usedSourceDefinitions: SourceDefinitionRead[] = useMemo(() => { diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx index 08e75ad2be1..3df62015450 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx @@ -21,31 +21,27 @@ export interface ConnectorCellProps { id: string; } -const ConnectorCell: React.FC = ({ - connectorName, - img, - releaseStage, - type, - id, - currentVersion, -}) => { - const allowUpdateConnectors = useFeature(FeatureItem.AllowUpdateConnectors); +const ConnectorCell: React.FC = React.memo( + ({ connectorName, img, releaseStage, type, id, currentVersion }) => { + const allowUpdateConnectors = useFeature(FeatureItem.AllowUpdateConnectors); - return ( - - {allowUpdateConnectors && type === "sources" && ( - - )} - {allowUpdateConnectors && type === "destinations" && ( - - )} -
    - -
    -
    {connectorName}
    - -     - ); -}; + return ( + + {allowUpdateConnectors && type === "sources" && ( + + )} + {allowUpdateConnectors && type === "destinations" && ( + + )} +
    + +
    +
    {connectorName}
    + +     + ); + } +); +ConnectorCell.displayName = "ConnectorCell"; export default ConnectorCell; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx index 67bd2d272d9..b930ff68f5b 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx @@ -1,5 +1,6 @@ import { createColumnHelper } from "@tanstack/react-table"; import { useCallback, useMemo, useState } from "react"; +import React from "react"; import { FormattedMessage } from "react-intl"; import { HeadTitle } from "components/common/HeadTitle"; @@ -53,6 +54,8 @@ function filterByBuilderConnectors( const columnHelper = createColumnHelper(); +const MemoizedTable = React.memo(Table) as typeof Table; + const ConnectorsView: React.FC = ({ type, onUpdateVersion, @@ -163,13 +166,15 @@ const ConnectorsView: React.FC = ({ [updatingDefinitionId, updatingAllConnectors] ); + const showUpdateColumnForUsedDefinitions = showVersionUpdateColumn(usedConnectorsDefinitions); const usedDefinitionColumns = useMemo( - () => renderColumns(showVersionUpdateColumn(usedConnectorsDefinitions)), - [renderColumns, showVersionUpdateColumn, usedConnectorsDefinitions] + () => renderColumns(showUpdateColumnForUsedDefinitions), + [renderColumns, showUpdateColumnForUsedDefinitions] ); + const showUpdateColumnForDefinitions = showVersionUpdateColumn(connectorsDefinitions); const definitionColumns = useMemo( - () => renderColumns(showVersionUpdateColumn(connectorsDefinitions)), - [renderColumns, showVersionUpdateColumn, connectorsDefinitions] + () => renderColumns(showUpdateColumnForDefinitions), + [renderColumns, showUpdateColumnForDefinitions] ); const sections: Array<{ title: string; content: React.ReactNode }> = []; @@ -189,13 +194,15 @@ const ConnectorsView: React.FC = ({ if (usedConnectorsDefinitions.length > 0) { sections.push({ title: type === "sources" ? "admin.manageSource" : "admin.manageDestination", - content:
, + content: ( + + ), }); } sections.push({ title: type === "sources" ? "admin.availableSource" : "admin.availableDestinations", - content:
, + content: , }); return ( diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx index 831ddba6863..c9325febfa4 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell/VersionCell.tsx @@ -42,6 +42,7 @@ export const VersionCell: React.FC = ({ id: connectorDefinitionId, version: latestVersion || currentVersion, }} + reinitializeDefaultValues schema={versionCellFormSchema} onSubmit={onChange} > From 85ccdaee070ff1c0e2e9139b2afb3a310b9f00f3 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 1 Sep 2023 08:57:12 -0400 Subject: [PATCH 087/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20?= =?UTF-8?q?=F0=9F=8F=81=20(flagged)=20Allow=20OSS=20users=20to=20adjust=20?= =?UTF-8?q?permissions=20on=20RBAC=20pages=20(#8349)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Chandler Prall Co-authored-by: Joey Marshment-Howell --- airbyte-webapp/src/core/api/hooks/index.ts | 3 +- .../src/core/api/hooks/organizations.ts | 3 +- .../src/core/api/hooks/permissions.ts | 39 +++++ .../src/core/api/hooks/workspaces.tsx | 1 + airbyte-webapp/src/locales/en.json | 7 +- .../OrganizationAccessManagementPage.tsx | 8 +- .../WorkspaceAccessManagementPage.tsx | 4 +- .../components/AccessManagementCard.tsx | 20 ++- .../AccessManagementPageContent.module.scss | 5 + .../AccessManagementPageContent.tsx | 27 +++- .../components/AccessManagementTable.tsx | 43 ++++- .../RoleManagementControl.module.scss | 28 ++++ .../components/RoleManagementControl.tsx | 148 ++++++++++++++++++ 13 files changed, 314 insertions(+), 22 deletions(-) create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.module.scss create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.module.scss create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx diff --git a/airbyte-webapp/src/core/api/hooks/index.ts b/airbyte-webapp/src/core/api/hooks/index.ts index bff7582f306..6efbfde2bfa 100644 --- a/airbyte-webapp/src/core/api/hooks/index.ts +++ b/airbyte-webapp/src/core/api/hooks/index.ts @@ -4,13 +4,14 @@ export * from "./connectorBuilderApi"; export * from "./connectorBuilderProject"; export * from "./geographies"; export * from "./health"; +export * from "./instanceConfiguration"; export * from "./jobs"; export * from "./logs"; export * from "./notifications"; export * from "./operations"; export * from "./organizations"; +export * from "./permissions"; export * from "./security"; -export * from "./instanceConfiguration"; export * from "./streams"; export * from "./permissions"; export * from "./upgradeConnectorVersion"; diff --git a/airbyte-webapp/src/core/api/hooks/organizations.ts b/airbyte-webapp/src/core/api/hooks/organizations.ts index f41fe4ee314..b06af0b8438 100644 --- a/airbyte-webapp/src/core/api/hooks/organizations.ts +++ b/airbyte-webapp/src/core/api/hooks/organizations.ts @@ -10,6 +10,7 @@ import { useSuspenseQuery } from "../useSuspenseQuery"; export const organizationKeys = { all: [SCOPE_USER, "organizations"] as const, detail: (organizationId: string) => [...organizationKeys.all, "details", organizationId] as const, + allListUsers: [SCOPE_ORGANIZATION, "users", "list"] as const, listUsers: (organizationId: string) => [SCOPE_ORGANIZATION, "users", "list", organizationId] as const, workspaces: (organizationIds: string[]) => [...organizationKeys.all, "workspaces", organizationIds] as const, }; @@ -37,7 +38,7 @@ export const useUpdateOrganization = () => { export const useListUsersInOrganization = (organizationId: string) => { const requestOptions = useRequestOptions(); - const queryKey = [organizationKeys.listUsers(organizationId)]; + const queryKey = organizationKeys.listUsers(organizationId); return useSuspenseQuery(queryKey, () => listUsersInOrganization({ organizationId }, requestOptions)); }; diff --git a/airbyte-webapp/src/core/api/hooks/permissions.ts b/airbyte-webapp/src/core/api/hooks/permissions.ts index 953b8fbd6d0..ae4ed8cb454 100644 --- a/airbyte-webapp/src/core/api/hooks/permissions.ts +++ b/airbyte-webapp/src/core/api/hooks/permissions.ts @@ -1,6 +1,12 @@ +import { useMutation, useQueryClient } from "@tanstack/react-query"; + import { SCOPE_USER } from "services/Scope"; +import { organizationKeys } from "./organizations"; +import { workspaceKeys } from "./workspaces"; import { listPermissionsByUser } from "../generated/AirbyteClient"; +import { deletePermission, updatePermission } from "../generated/AirbyteClient"; +import { PermissionRead, PermissionUpdate } from "../generated/AirbyteClient.schemas"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; @@ -13,3 +19,36 @@ export const useListPermissions = (userId: string) => { const requestOptions = useRequestOptions(); return useSuspenseQuery(permissionKeys.listByUser(userId), () => listPermissionsByUser({ userId }, requestOptions)); }; + +export const useUpdatePermissions = () => { + const requestOptions = useRequestOptions(); + const queryClient = useQueryClient(); + + return useMutation( + (permission: PermissionUpdate): Promise => { + return updatePermission(permission, requestOptions); + }, + { + onSuccess: (data: PermissionRead) => { + if (data.organizationId) { + queryClient.invalidateQueries(organizationKeys.listUsers(data.organizationId)); + } + if (data.workspaceId) { + queryClient.invalidateQueries(workspaceKeys.listUsers(data.workspaceId)); + } + }, + } + ); +}; + +export const useDeletePermissions = () => { + const requestOptions = useRequestOptions(); + const queryClient = useQueryClient(); + + return useMutation(async (permissionId: string) => deletePermission({ permissionId }, requestOptions), { + onSuccess: () => { + queryClient.invalidateQueries(organizationKeys.allListUsers); + queryClient.invalidateQueries(workspaceKeys.allListUsers); + }, + }); +}; diff --git a/airbyte-webapp/src/core/api/hooks/workspaces.tsx b/airbyte-webapp/src/core/api/hooks/workspaces.tsx index 4f7b8dcf295..18d499bd190 100644 --- a/airbyte-webapp/src/core/api/hooks/workspaces.tsx +++ b/airbyte-webapp/src/core/api/hooks/workspaces.tsx @@ -22,6 +22,7 @@ export const workspaceKeys = { all: [SCOPE_USER, "workspaces"] as const, lists: () => [...workspaceKeys.all, "list"] as const, list: (filters: string) => [...workspaceKeys.lists(), { filters }] as const, + allListUsers: [SCOPE_WORKSPACE, "users", "list"] as const, listUsers: (workspaceId: string) => [SCOPE_WORKSPACE, "users", "list", workspaceId] as const, detail: (workspaceId: string) => [...workspaceKeys.all, "details", workspaceId] as const, state: (workspaceId: string) => [...workspaceKeys.all, "state", workspaceId] as const, diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 66063a08139..af9cad98341 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -777,8 +777,11 @@ "settings.accessManagement.table.column.email": "Email", "settings.accessManagement.table.column.role": "Role", "settings.accessManagement.table.column.action": "Action", - "settings.accessManagement.button.addNewUser": "New user", - "settings.accessManagement.user.remove": "Remove", + "settings.accessManagement.user.remove": "Remove user", + "settings.accessManagement.changeRole": "Change role", + "settings.accessManagement.removePermissions": "Are you sure you want to remove permissions for the user {user} from {resource}?", + "settings.accessManagement.removePermissions.additionalContent": "They may still have access to this resource from other permissions.", + "settings.accessManagement.confirmRemovePermissions": "Remove permission", "connector.connectorCount": "{count, plural, one {# connector} other {# connectors}}", "connector.noSearchResults": "No connectors match your search.", diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx index 915f8371a11..6e6fd8aa6d9 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/OrganizationAccessManagementPage.tsx @@ -13,5 +13,11 @@ export const OrganizationAccessManagementPage: React.FC = () => { const { organizationName } = useOrganization(organizationId); const organizationAccessUsers = useGetOrganizationAccessUsers(); - return ; + return ( + + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx index b3f78d563d1..fb1ccd460a2 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/WorkspaceAccessManagementPage.tsx @@ -11,5 +11,7 @@ export const WorkspaceAccessManagementPage: React.FC = () => { const { name } = useCurrentWorkspace(); const workspaceAccessUsers = useGetWorkspaceAccessUsers(); - return ; + return ( + + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx index 73e600e0f0c..82f67d00366 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx @@ -9,12 +9,24 @@ import { ResourceType, tableTitleDictionary } from "./useGetAccessManagementData interface AccessManagementCardProps { users: OrganizationUserRead[] | WorkspaceUserRead[]; - resourceType: ResourceType; + tableResourceType: ResourceType; + pageResourceType: ResourceType; + pageResourceName: string; } -export const AccessManagementCard: React.FC = ({ users, resourceType }) => { +export const AccessManagementCard: React.FC = ({ + users, + tableResourceType, + pageResourceType, + pageResourceName, +}) => { return ( - }> - + }> + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.module.scss new file mode 100644 index 00000000000..9d24a736b71 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.module.scss @@ -0,0 +1,5 @@ +@use "scss/variables"; + +.pageContent { + width: variables.$page-width - variables.$width-wide-menu; +} diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx index 69fba0350d9..67f9560fdc7 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementPageContent.tsx @@ -1,38 +1,51 @@ import { FormattedMessage } from "react-intl"; import { HeadTitle } from "components/common/HeadTitle"; -import { PageContainer } from "components/PageContainer"; import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; import { AccessManagementCard } from "./AccessManagementCard"; -import { AccessUsers } from "./useGetAccessManagementData"; +import styles from "./AccessManagementPageContent.module.scss"; +import { AccessUsers, ResourceType } from "./useGetAccessManagementData"; interface AccessManagementContentProps { resourceName: string; accessUsers: AccessUsers; + pageResourceType: ResourceType; } -export const AccessManagementPageContent: React.FC = ({ resourceName, accessUsers }) => { +export const AccessManagementPageContent: React.FC = ({ + resourceName, + accessUsers, + pageResourceType, +}) => { return ( - + <> - + {Object.keys(accessUsers).map((key) => { const resourceType = key as keyof typeof accessUsers; const users = accessUsers[resourceType]; return ( users && - users.length > 0 && + users.length > 0 && ( + + ) ); })} - + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx index 8e4a678f0df..8051d07cdd4 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx @@ -1,19 +1,23 @@ import { createColumnHelper } from "@tanstack/react-table"; -import { useMemo } from "react"; +import { useMemo, useState } from "react"; import { FormattedMessage } from "react-intl"; import { Table } from "components/ui/Table"; import { OrganizationUserRead, WorkspaceUserRead } from "core/request/AirbyteClient"; +import { RoleManagementControl } from "./RoleManagementControl"; import { RoleToolTip } from "./RoleToolTip"; -import { ResourceType, permissionStringDictionary } from "./useGetAccessManagementData"; +import { ResourceType } from "./useGetAccessManagementData"; export const AccessManagementTable: React.FC<{ users: WorkspaceUserRead[] | OrganizationUserRead[]; tableResourceType: ResourceType; -}> = ({ users, tableResourceType }) => { + pageResourceType: ResourceType; + pageResourceName: string; +}> = ({ users, tableResourceType, pageResourceType, pageResourceName }) => { const columnHelper = createColumnHelper(); + const [activeEditRow, setActiveEditRow] = useState(undefined); const columns = useMemo( () => [ @@ -21,11 +25,13 @@ export const AccessManagementTable: React.FC<{ header: () => , cell: (props) => props.cell.getValue(), sortingFn: "alphanumeric", + meta: { responsive: true }, }), columnHelper.accessor("email", { header: () => , cell: (props) => props.cell.getValue(), sortingFn: "alphanumeric", + meta: { responsive: true }, }), columnHelper.accessor("permissionType", { header: () => ( @@ -34,11 +40,38 @@ export const AccessManagementTable: React.FC<{ ), - cell: (props) => , + meta: { responsive: true }, + cell: (props) => { + let workspaceId; + let organizationId; + + const { userId, permissionType, permissionId, name } = props.row.original; + if ("organizationId" in props.row.original) { + organizationId = props.row.original.organizationId; + } + if ("workspaceId" in props.row.original) { + workspaceId = props.row.original.workspaceId; + } + return ( + + ); + }, sortingFn: "alphanumeric", }), ], - [columnHelper, tableResourceType] + [activeEditRow, columnHelper, pageResourceName, pageResourceType, tableResourceType] ); return
; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.module.scss new file mode 100644 index 00000000000..5a440cc5853 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.module.scss @@ -0,0 +1,28 @@ +@use "scss/variables"; +@use "scss/colors"; + +.roleManagementControl__button { + opacity: 0; + + &:focus, + &:hover { + opacity: 1; + } + + tr:hover &, + tr:focus-within & { + opacity: 1; + } +} + +.roleManagementControl__input { + padding-bottom: 0; + + button { + background-color: colors.$foreground; + } +} + +.roleManagementControl__roleLabel_view { + flex-basis: 89px; +} diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx new file mode 100644 index 00000000000..acbfc4435b8 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx @@ -0,0 +1,148 @@ +import { FormattedMessage, useIntl } from "react-intl"; +import * as yup from "yup"; + +import { Form, FormControl } from "components/forms"; +import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; +import { Box } from "components/ui/Box"; +import { Button } from "components/ui/Button"; +import { FlexContainer } from "components/ui/Flex"; +import { Text } from "components/ui/Text"; + +import { useDeletePermissions, useUpdatePermissions } from "core/api"; +import { PermissionType, PermissionUpdate } from "core/request/AirbyteClient"; +import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; + +import styles from "./RoleManagementControl.module.scss"; +import { ResourceType, permissionStringDictionary, permissionsByResourceType } from "./useGetAccessManagementData"; + +interface RoleManagementControlProps extends PermissionUpdate { + userName?: string; + resourceName: string; + pageResourceType: ResourceType; + tableResourceType: ResourceType; + activeEditRow?: string; + setActiveEditRow: (permissionId?: string) => void; +} + +const roleManagementFormSchema = yup.object().shape({ + userId: yup.string().required(), + permissionType: yup.mixed().oneOf(Object.values(PermissionType)).required(), + permissionId: yup.string().required(), + workspaceId: yup.string(), + organizationId: yup.string(), +}); + +export const RoleManagementControl: React.FC = ({ + userId, + permissionType, + workspaceId, + organizationId, + pageResourceType, + tableResourceType, + permissionId, + userName, + resourceName, + activeEditRow, + setActiveEditRow, +}) => { + const { mutateAsync: updatePermissions } = useUpdatePermissions(); + const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); + const { mutateAsync: deletePermissions } = useDeletePermissions(); + const { formatMessage } = useIntl(); + const isEditMode = activeEditRow === permissionId; + + if (!permissionType) { + return null; + } + + if (pageResourceType !== tableResourceType) { + return ( + + + + ); + } + + const onSubmitRoleChangeClick = async (values: PermissionUpdate) => { + await updatePermissions(values); + setActiveEditRow(undefined); + }; + + const onRemoveUserClick = () => { + openConfirmationModal({ + title: "Remove User", + text: formatMessage( + { + id: "settings.accessManagement.removePermissions", + }, + { user: userName, resource: resourceName } + ), + additionalContent: ( + + + + + + ), + onSubmit: async () => { + await deletePermissions(permissionId); + closeConfirmationModal(); + }, + submitButtonText: "settings.accessManagement.confirmRemovePermissions", + }); + }; + + return ( + <> + {isEditMode ? ( + + schema={roleManagementFormSchema} + defaultValues={{ userId, permissionType, workspaceId, organizationId, permissionId }} + onSubmit={onSubmitRoleChangeClick} + > + + + containerControlClassName={styles.roleManagementControl__input} + name="permissionType" + fieldType="dropdown" + options={permissionsByResourceType[tableResourceType].map((permission) => { + return { + value: permission, + label: ( + + + + ), + disabled: permission === permissionType, + }; + })} + /> + setActiveEditRow(undefined)} + allowNonDirtyCancel + /> + + + ) : ( + + + + + + + + + + )} + + ); +}; From b5281f0765ddadf85514fd63dafa43bf4df67d75 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Fri, 1 Sep 2023 15:04:54 +0200 Subject: [PATCH 088/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Connector?= =?UTF-8?q?=20OAuth=20APIs=20refactoring=20(#8620)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/core/api/hooks/connectorOAuth.ts | 58 +++++++++++++++++++ airbyte-webapp/src/core/api/hooks/index.ts | 1 + .../connector/DestinationAuthService.ts | 17 ------ .../domain/connector/SourceAuthService.ts | 23 -------- .../src/hooks/services/useConnectorAuth.tsx | 33 +++-------- .../services/useConnectorAuthRevocation.tsx | 29 +++------- .../ConnectorForm/ConnectorForm.test.tsx | 34 +++++------ .../components/Sections/auth/AuthSection.tsx | 5 +- .../components/Sections/auth/RevokeButton.tsx | 8 ++- .../auth/useOauthRevocationAdapter.tsx | 3 +- 10 files changed, 105 insertions(+), 106 deletions(-) create mode 100644 airbyte-webapp/src/core/api/hooks/connectorOAuth.ts delete mode 100644 airbyte-webapp/src/core/domain/connector/DestinationAuthService.ts delete mode 100644 airbyte-webapp/src/core/domain/connector/SourceAuthService.ts diff --git a/airbyte-webapp/src/core/api/hooks/connectorOAuth.ts b/airbyte-webapp/src/core/api/hooks/connectorOAuth.ts new file mode 100644 index 00000000000..1c0716aa983 --- /dev/null +++ b/airbyte-webapp/src/core/api/hooks/connectorOAuth.ts @@ -0,0 +1,58 @@ +import { useCallback, useMemo } from "react"; + +import { + completeDestinationOAuth, + completeSourceOAuth, + getDestinationOAuthConsent, + getSourceOAuthConsent, + revokeSourceOAuthTokens, +} from "../generated/AirbyteClient"; +import { + CompleteDestinationOAuthRequest, + CompleteSourceOauthRequest, + DestinationOauthConsentRequest, + RevokeSourceOauthTokensRequest, + SourceOauthConsentRequest, +} from "../types/AirbyteClient"; +import { useRequestOptions } from "../useRequestOptions"; + +export function useConsentUrls() { + const requestOptions = useRequestOptions(); + + return useMemo( + () => ({ + getSourceConsentUrl(request: SourceOauthConsentRequest) { + return getSourceOAuthConsent(request, requestOptions); + }, + getDestinationConsentUrl(request: DestinationOauthConsentRequest) { + return getDestinationOAuthConsent(request, requestOptions); + }, + }), + [requestOptions] + ); +} + +export function useCompleteOAuth() { + const requestOptions = useRequestOptions(); + return useMemo( + () => ({ + completeSourceOAuth(request: CompleteSourceOauthRequest) { + return completeSourceOAuth(request, requestOptions); + }, + completeDestinationOAuth(request: CompleteDestinationOAuthRequest) { + return completeDestinationOAuth(request, requestOptions); + }, + }), + [requestOptions] + ); +} + +export function useRevokeSourceOAuthToken() { + const requestOptions = useRequestOptions(); + return useCallback( + (request: RevokeSourceOauthTokensRequest) => { + return revokeSourceOAuthTokens(request, requestOptions); + }, + [requestOptions] + ); +} diff --git a/airbyte-webapp/src/core/api/hooks/index.ts b/airbyte-webapp/src/core/api/hooks/index.ts index 6efbfde2bfa..f98293856ae 100644 --- a/airbyte-webapp/src/core/api/hooks/index.ts +++ b/airbyte-webapp/src/core/api/hooks/index.ts @@ -2,6 +2,7 @@ export * from "./actorDefinitionVersions"; export * from "./connections"; export * from "./connectorBuilderApi"; export * from "./connectorBuilderProject"; +export * from "./connectorOAuth"; export * from "./geographies"; export * from "./health"; export * from "./instanceConfiguration"; diff --git a/airbyte-webapp/src/core/domain/connector/DestinationAuthService.ts b/airbyte-webapp/src/core/domain/connector/DestinationAuthService.ts deleted file mode 100644 index 0ea92861833..00000000000 --- a/airbyte-webapp/src/core/domain/connector/DestinationAuthService.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { - completeDestinationOAuth, - CompleteDestinationOAuthRequest, - DestinationOauthConsentRequest, - getDestinationOAuthConsent, -} from "../../request/AirbyteClient"; -import { AirbyteRequestService } from "../../request/AirbyteRequestService"; - -export class DestinationAuthService extends AirbyteRequestService { - public getConsentUrl(body: DestinationOauthConsentRequest) { - return getDestinationOAuthConsent(body, this.requestOptions); - } - - public completeOauth(body: CompleteDestinationOAuthRequest) { - return completeDestinationOAuth(body, this.requestOptions); - } -} diff --git a/airbyte-webapp/src/core/domain/connector/SourceAuthService.ts b/airbyte-webapp/src/core/domain/connector/SourceAuthService.ts deleted file mode 100644 index a27bc999be9..00000000000 --- a/airbyte-webapp/src/core/domain/connector/SourceAuthService.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { - completeSourceOAuth, - CompleteSourceOauthRequest, - getSourceOAuthConsent, - revokeSourceOAuthTokens, - RevokeSourceOauthTokensRequest, - SourceOauthConsentRequest, -} from "../../request/AirbyteClient"; -import { AirbyteRequestService } from "../../request/AirbyteRequestService"; - -export class SourceAuthService extends AirbyteRequestService { - public getConsentUrl(body: SourceOauthConsentRequest) { - return getSourceOAuthConsent(body, this.requestOptions); - } - - public completeOauth(body: CompleteSourceOauthRequest) { - return completeSourceOAuth(body, this.requestOptions); - } - - public revokeOauthTokens(body: RevokeSourceOauthTokensRequest) { - return revokeSourceOAuthTokens(body, this.requestOptions); - } -} diff --git a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx index fbc8e23d34d..e6e5bcdad35 100644 --- a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx +++ b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx @@ -1,12 +1,10 @@ -import { useCallback, useEffect, useMemo, useRef } from "react"; +import { useCallback, useEffect, useRef } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useAsyncFn, useEffectOnce, useEvent } from "react-use"; -import { useConfig } from "config"; +import { useCompleteOAuth, useConsentUrls } from "core/api"; import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; -import { DestinationAuthService } from "core/domain/connector/DestinationAuthService"; import { isSourceDefinitionSpecification } from "core/domain/connector/source"; -import { SourceAuthService } from "core/domain/connector/SourceAuthService"; import { CompleteOAuthResponse, CompleteOAuthResponseAuthPayload, @@ -20,7 +18,6 @@ import { useConnectorForm } from "views/Connector/ConnectorForm/connectorFormCon import { useAppMonitoringService } from "./AppMonitoringService"; import { useNotificationService } from "./Notification"; import { useCurrentWorkspace } from "./useWorkspace"; -import { useDefaultRequestMiddlewares } from "../../services/useDefaultRequestMiddlewares"; import { useQuery } from "../useQuery"; let windowObjectReference: Window | null = null; // global variable @@ -61,22 +58,11 @@ export function useConnectorAuth(): { const { formatMessage } = useIntl(); const { trackError } = useAppMonitoringService(); const { workspaceId } = useCurrentWorkspace(); - const { apiUrl } = useConfig(); + const { getDestinationConsentUrl, getSourceConsentUrl } = useConsentUrls(); + const { completeDestinationOAuth, completeSourceOAuth } = useCompleteOAuth(); const notificationService = useNotificationService(); const { connectorId } = useConnectorForm(); - // TODO: move to separate initFacade and use refs instead - const requestAuthMiddleware = useDefaultRequestMiddlewares(); - - const sourceAuthService = useMemo( - () => new SourceAuthService(apiUrl, requestAuthMiddleware), - [apiUrl, requestAuthMiddleware] - ); - const destinationAuthService = useMemo( - () => new DestinationAuthService(apiUrl, requestAuthMiddleware), - [apiUrl, requestAuthMiddleware] - ); - return { getConsentUrl: async ( connector: ConnectorDefinitionSpecification, @@ -94,7 +80,7 @@ export function useConnectorAuth(): { oAuthInputConfiguration, sourceId: connectorId, }; - const response = await sourceAuthService.getConsentUrl(payload); + const response = await getSourceConsentUrl(payload); return { consentUrl: response.consentUrl, payload }; } @@ -105,7 +91,7 @@ export function useConnectorAuth(): { oAuthInputConfiguration, destinationId: connectorId, }; - const response = await destinationAuthService.getConsentUrl(payload); + const response = await getDestinationConsentUrl(payload); return { consentUrl: response.consentUrl, payload }; } catch (e) { @@ -139,14 +125,11 @@ export function useConnectorAuth(): { params: SourceOauthConsentRequest | DestinationOauthConsentRequest, queryParams: Record ): Promise => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const payload: any = { + const payload = { ...params, queryParams, }; - return (payload as SourceOauthConsentRequest).sourceDefinitionId - ? sourceAuthService.completeOauth(payload) - : destinationAuthService.completeOauth(payload); + return "sourceDefinitionId" in payload ? completeSourceOAuth(payload) : completeDestinationOAuth(payload); }, }; } diff --git a/airbyte-webapp/src/hooks/services/useConnectorAuthRevocation.tsx b/airbyte-webapp/src/hooks/services/useConnectorAuthRevocation.tsx index b1ae874e416..d5bcf087463 100644 --- a/airbyte-webapp/src/hooks/services/useConnectorAuthRevocation.tsx +++ b/airbyte-webapp/src/hooks/services/useConnectorAuthRevocation.tsx @@ -1,38 +1,25 @@ import { useMutation } from "@tanstack/react-query"; -import { useMemo } from "react"; import { FormattedMessage } from "react-intl"; -import { useConfig } from "config"; -import { SourceAuthService } from "core/domain/connector/SourceAuthService"; -import { RevokeSourceOauthTokensRequest } from "core/request/AirbyteClient"; -import { useConnectorForm } from "views/Connector/ConnectorForm/connectorFormContext"; +import { useRevokeSourceOAuthToken } from "core/api"; import { useNotificationService } from "./Notification"; import { useCurrentWorkspace } from "./useWorkspace"; import { ConnectorDefinitionSpecification, ConnectorSpecification } from "../../core/domain/connector"; -import { useDefaultRequestMiddlewares } from "../../services/useDefaultRequestMiddlewares"; -export function useConnectorAuthRevocation(): { +export function useConnectorAuthRevocation(sourceId: string): { revokeAuthTokens: (connector: ConnectorDefinitionSpecification) => Promise; } { const { workspaceId } = useCurrentWorkspace(); - const { connectorId } = useConnectorForm(); - const { apiUrl } = useConfig(); - - const requestAuthMiddleware = useDefaultRequestMiddlewares(); - - const sourceAuthService = useMemo( - () => new SourceAuthService(apiUrl, requestAuthMiddleware), - [apiUrl, requestAuthMiddleware] - ); + const revokeSourceOAuthToken = useRevokeSourceOAuthToken(); return { revokeAuthTokens: async (connector: ConnectorDefinitionSpecification): Promise => { - return sourceAuthService.revokeOauthTokens({ + return revokeSourceOAuthToken({ sourceDefinitionId: ConnectorSpecification.id(connector), - sourceId: connectorId, + sourceId, workspaceId, - } as RevokeSourceOauthTokensRequest); + }); }, }; } @@ -40,9 +27,11 @@ export function useConnectorAuthRevocation(): { const OAUTH_REVOCATION_ERROR_ID = "connector.oauthRevocationError"; export function useRunOauthRevocation({ + sourceId, connector, onDone, }: { + sourceId: string; connector: ConnectorDefinitionSpecification; onDone?: () => void; }): { @@ -50,7 +39,7 @@ export function useRunOauthRevocation({ done?: boolean; run: () => void; } { - const { revokeAuthTokens } = useConnectorAuthRevocation(); + const { revokeAuthTokens } = useConnectorAuthRevocation(sourceId); const { registerNotification } = useNotificationService(); const { status, mutate } = useMutation(() => revokeAuthTokens(connector), { diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx index 05a5c26d4af..e2593e77793 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx @@ -6,8 +6,8 @@ import selectEvent from "react-select-event"; import { render, useMockIntersectionObserver } from "test-utils/testutils"; +import { useCompleteOAuth } from "core/api"; import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; -import { SourceAuthService } from "core/domain/connector/SourceAuthService"; import { AirbyteJSONSchema } from "core/jsonSchema/types"; import { DestinationDefinitionSpecificationRead } from "core/request/AirbyteClient"; import { FeatureItem } from "core/services/features"; @@ -23,16 +23,12 @@ jest.mock("../../../hooks/services/useDestinationHook", () => ({ useDestinationList: () => ({ destinations: [] }), })); -jest.mock("../../../core/domain/connector/SourceAuthService", () => ({ - SourceAuthService: class SourceAuthService { - public static mockedPayload: Record = {}; - getConsentUrl() { - return { consentUrl: "http://example.org" }; - } - completeOauth() { - return Promise.resolve(SourceAuthService.mockedPayload); - } - }, +jest.mock("core/api", () => ({ + useConsentUrls: () => ({ getSourceConsentUrl: () => "http://example.com" }), + useCompleteOAuth: jest.fn(() => ({ + completeSourceOAuth: () => Promise.resolve({}), + completeDestinationOAuth: () => Promise.resolve({}), + })), })); jest.mock("../ConnectorDocumentationLayout/DocumentationPanelContext", () => { @@ -1024,12 +1020,16 @@ describe("Connector form", () => { it("should insert values correctly and submit them", async () => { const container = await renderNewOAuthForm(); - (SourceAuthService as unknown as { mockedPayload: Record }).mockedPayload = { - request_succeeded: true, - auth_payload: { - access_token: "mytoken", - }, - }; + (useCompleteOAuth as jest.MockedFunction).mockReturnValue({ + completeDestinationOAuth: jest.fn(), + completeSourceOAuth: () => + Promise.resolve({ + request_succeeded: true, + auth_payload: { + access_token: "mytoken", + }, + }), + }); await executeOAuthFlow(container); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx index 4200e12001c..f1386acdfe4 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/AuthSection.tsx @@ -30,7 +30,10 @@ export const AuthSection: React.FC = () => { {supportsRevokingTokens && hasAuthFieldValues && connectorId && ( - + )} diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/RevokeButton.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/RevokeButton.tsx index 461bf96f3b6..ce2a851259f 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/RevokeButton.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/RevokeButton.tsx @@ -11,11 +11,15 @@ import { useFormOauthRevocationAdapter } from "./useOauthRevocationAdapter"; import { FlexContainer } from "../../../../../../components/ui/Flex"; import { useConnectorForm } from "../../../connectorFormContext"; -export const RevokeButton: React.FC<{ +interface RevokeButtonProps { + sourceId: string; selectedConnectorDefinitionSpecification: ConnectorDefinitionSpecification; -}> = ({ selectedConnectorDefinitionSpecification }) => { +} + +export const RevokeButton: React.FC = ({ sourceId, selectedConnectorDefinitionSpecification }) => { const { selectedConnectorDefinition } = useConnectorForm(); const { loading, run } = useFormOauthRevocationAdapter( + sourceId, selectedConnectorDefinitionSpecification, selectedConnectorDefinition ); diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx index 1bfac0c0a56..e04f1950e01 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx @@ -15,6 +15,7 @@ interface Credentials { } function useFormOauthRevocationAdapter( + sourceId: string, connector: ConnectorDefinitionSpecification, connectorDefinition?: ConnectorDefinition ): { @@ -44,7 +45,7 @@ function useFormOauthRevocationAdapter( }); }; - const { run, loading } = useRunOauthRevocation({ connector, onDone }); + const { run, loading } = useRunOauthRevocation({ sourceId, connector, onDone }); return { loading, From 646240341243a7b8a89e3070ad20818b06e55a2b Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 1 Sep 2023 10:32:29 -0400 Subject: [PATCH 089/201] =?UTF-8?q?Revert=20"Revert=20"=F0=9F=AA=9F=20Remo?= =?UTF-8?q?ve=20enrollment=20materials=20for=20Free=20Connector=20Program"?= =?UTF-8?q?"=20(#8647)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vladimir --- .../api/hooks/cloud/freeConnectorProgram.ts | 50 -- .../EnrollmentModal.module.scss | 53 -- .../EnrollmentModal/EnrollmentModal.tsx | 155 ---- .../EnrollmentModal/cards.svg | 24 - .../EnrollmentModal/connectorGrid.svg | 439 ------------ .../EnrollmentModal/free-alpha-beta-pills.svg | 34 - .../EnrollmentModal/free.svg | 13 - .../EnrollmentModal/index.ts | 1 - .../EnrollmentModal/mail.svg | 14 - .../useShowEnrollmentModal.tsx | 44 -- .../InlineEnrollmentCallout.module.scss | 18 - .../InlineEnrollmentCallout.tsx | 52 -- .../LargeEnrollmentCallout.module.scss | 30 - .../LargeEnrollmentCallout.tsx | 45 -- .../connectors-badges.svg | 22 - .../longtail-connectors.svg | 667 ------------------ .../src/packages/cloud/locales/en.json | 14 - .../views/billing/BillingPage/BillingPage.tsx | 4 - .../useBillingPageBanners.test.tsx | 110 +-- .../BillingPage/useBillingPageBanners.tsx | 13 +- 20 files changed, 38 insertions(+), 1764 deletions(-) delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg delete mode 100644 airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg diff --git a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts index f63848448a8..dd1c4ab90cd 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts @@ -1,70 +1,21 @@ import { useQuery } from "@tanstack/react-query"; -import { useState } from "react"; -import { useIntl } from "react-intl"; -import { useSearchParams } from "react-router-dom"; -import { useEffectOnce } from "react-use"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { FeatureItem, useFeature } from "core/services/features"; -import { pollUntil } from "core/utils/pollUntil"; -import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; -import { useNotificationService } from "hooks/services/Notification"; import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { webBackendGetFreeConnectorProgramInfoForWorkspace } from "../../generated/CloudApi"; -export const STRIPE_SUCCESS_QUERY = "fcpEnrollmentSuccess"; - export const useFreeConnectorProgram = () => { const workspaceId = useCurrentWorkspaceId(); const middlewares = useDefaultRequestMiddlewares(); const requestOptions = { middlewares }; const freeConnectorProgramEnabled = useFeature(FeatureItem.FreeConnectorProgram); - const [searchParams, setSearchParams] = useSearchParams(); - const [userDidEnroll, setUserDidEnroll] = useState(false); - const { formatMessage } = useIntl(); - const { registerNotification } = useNotificationService(); - const { trackError } = useAppMonitoringService(); - - const removeStripeSuccessQuery = () => { - const { [STRIPE_SUCCESS_QUERY]: _, ...unrelatedSearchParams } = Object.fromEntries(searchParams); - setSearchParams(unrelatedSearchParams, { replace: true }); - }; - - useEffectOnce(() => { - if (searchParams.has(STRIPE_SUCCESS_QUERY)) { - pollUntil( - () => webBackendGetFreeConnectorProgramInfoForWorkspace({ workspaceId }, requestOptions), - ({ hasPaymentAccountSaved }) => hasPaymentAccountSaved, - { intervalMs: 1000, maxTimeoutMs: 10000 } - ).then((maybeFcpInfo) => { - if (maybeFcpInfo) { - removeStripeSuccessQuery(); - setUserDidEnroll(true); - registerNotification({ - id: "fcp/enrollment-success", - text: formatMessage({ id: "freeConnectorProgram.enroll.success" }), - type: "success", - }); - } else { - trackError(new Error("Unable to confirm Free Connector Program enrollment before timeout"), { workspaceId }); - registerNotification({ - id: "fcp/enrollment-failure", - text: formatMessage({ id: "freeConnectorProgram.enroll.failure" }), - type: "error", - }); - } - }); - } - }); const programStatusQuery = useQuery(["freeConnectorProgramInfo", workspaceId], () => webBackendGetFreeConnectorProgramInfoForWorkspace({ workspaceId }, requestOptions).then( ({ hasPaymentAccountSaved, hasEligibleConnections, hasNonEligibleConnections }) => { - const userIsEligibleToEnroll = !hasPaymentAccountSaved; - return { - showEnrollmentUi: freeConnectorProgramEnabled && userIsEligibleToEnroll, isEnrolled: freeConnectorProgramEnabled && hasPaymentAccountSaved, hasEligibleConnections: freeConnectorProgramEnabled && hasEligibleConnections, hasNonEligibleConnections: freeConnectorProgramEnabled && hasNonEligibleConnections, @@ -75,6 +26,5 @@ export const useFreeConnectorProgram = () => { return { programStatusQuery, - userDidEnroll, }; }; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss deleted file mode 100644 index 62aba2c1d09..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.module.scss +++ /dev/null @@ -1,53 +0,0 @@ -@use "scss/colors"; -@use "scss/variables"; - -.header { - overflow: hidden; - position: relative; - background: colors.$blue-100; - height: 120px; - border-top-left-radius: variables.$border-radius-lg; - border-top-right-radius: variables.$border-radius-lg; -} - -.headerBackgroundImageContainer { - position: absolute; - left: 50%; - transform: translateX(-50%); -} - -.pill { - z-index: 2; - - // HACK: the hardcoded top margin is to vertically center the image, which was exported - // from Figma with an off-center vertical alignment - margin-top: 2em; -} - -.contentWrapper { - width: variables.$width-modal-sm; - padding: 0 variables.$spacing-xl variables.$spacing-2xl; -} - -.contentHeader { - margin: variables.$spacing-xl 0; -} - -.iconContainer { - flex: 0 0 82px; -} - -.resendEmailLink { - text-decoration: underline; - cursor: pointer; - padding: 0; - color: colors.$dark-blue; - border: none; - background-color: transparent; - font-size: inherit; - - &:hover, - &:active { - color: colors.$blue; - } -} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx deleted file mode 100644 index cdf1d5a96d8..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/EnrollmentModal.tsx +++ /dev/null @@ -1,155 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Heading } from "components/ui/Heading"; -import { Message } from "components/ui/Message"; -import { ModalFooter } from "components/ui/Modal/ModalFooter"; -import { Text } from "components/ui/Text"; - -import { STRIPE_SUCCESS_QUERY } from "core/api/cloud"; -import { StripeCheckoutSessionCreate, StripeCheckoutSessionRead } from "core/api/types/CloudApi"; - -import { ReactComponent as CardSVG } from "./cards.svg"; -import { ReactComponent as ConnectorGridSvg } from "./connectorGrid.svg"; -import styles from "./EnrollmentModal.module.scss"; -import { ReactComponent as FreeAlphaBetaPillsSVG } from "./free-alpha-beta-pills.svg"; -import { ReactComponent as FreeSVG } from "./free.svg"; -import { ReactComponent as MailSVG } from "./mail.svg"; - -interface EnrollmentModalContentProps { - closeModal: () => void; - createCheckout: (p: StripeCheckoutSessionCreate) => Promise; - workspaceId: string; - emailVerified: boolean; - sendEmailVerification: () => void; -} - -// we have to pass the email verification data and functions in as props, rather than -// directly using useAuthService(), because the modal renders outside of the -// AuthenticationProvider context. -export const EnrollmentModalContent: React.FC = ({ - closeModal, - createCheckout, - workspaceId, - emailVerified, - sendEmailVerification, -}) => { - const isMountedRef = useRef(false); - const [isLoading, setIsLoading] = useState(false); - - const startStripeCheckout = async () => { - setIsLoading(true); - // Use the current URL as a success URL but attach the STRIPE_SUCCESS_QUERY to it - const successUrl = new URL(window.location.href); - successUrl.searchParams.set(STRIPE_SUCCESS_QUERY, "true"); - const { stripeUrl } = await createCheckout({ - workspaceId, - successUrl: successUrl.href, - cancelUrl: window.location.href, - stripeMode: "setup", - }); - - // Forward to stripe as soon as we created a checkout session successfully - if (isMountedRef.current) { - window.location.assign(stripeUrl); - } - }; - - // If the user closes the modal while the request is processing, we don't want to redirect them - useEffect(() => { - isMountedRef.current = true; - return () => { - isMountedRef.current = false; - }; - }, []); - - const EnrollmentEmailVerificationWarning = () => { - const WarningContent = () => ( - - ( - - ), - }} - /> - - } - /> - ); - - return <>{!emailVerified && }; - }; - - return ( - <> - -
- -
- - -
- - - - - - - - - - {content}, - p2: (content: React.ReactNode) => {content}, - }} - /> - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - ); -}; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg deleted file mode 100644 index a86f40a6581..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/cards.svg +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg deleted file mode 100644 index da8c9978e4e..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/connectorGrid.svg +++ /dev/null @@ -1,439 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg deleted file mode 100644 index 6531acb626e..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free-alpha-beta-pills.svg +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg deleted file mode 100644 index 55c619b2c1d..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/free.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts deleted file mode 100644 index eb0c733b211..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./useShowEnrollmentModal"; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg deleted file mode 100644 index d3c1f86a68a..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/mail.svg +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx deleted file mode 100644 index c3642c65df5..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/EnrollmentModal/useShowEnrollmentModal.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import { useIntl } from "react-intl"; - -import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { useStripeCheckout } from "core/api/cloud"; -import { useAuthService } from "core/services/auth"; -import { useModalService } from "hooks/services/Modal"; -import { useNotificationService } from "hooks/services/Notification"; - -import { EnrollmentModalContent } from "./EnrollmentModal"; - -export const useShowEnrollmentModal = () => { - const { openModal, closeModal } = useModalService(); - const { mutateAsync: createCheckout } = useStripeCheckout(); - const workspaceId = useCurrentWorkspaceId(); - const { emailVerified, sendEmailVerification } = useAuthService(); - const { formatMessage } = useIntl(); - const { registerNotification } = useNotificationService(); - - const verifyEmail = () => - sendEmailVerification?.().then(() => { - registerNotification({ - id: "fcp/verify-email", - text: formatMessage({ id: "freeConnectorProgram.enrollmentModal.validationEmailConfirmation" }), - type: "info", - }); - }); - - return { - showEnrollmentModal: () => { - openModal({ - title: null, - content: () => ( - - ), - }); - }, - }; -}; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss deleted file mode 100644 index 98fb1eeeeb0..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.module.scss +++ /dev/null @@ -1,18 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.container { - width: 100%; -} - -.withMargin { - margin-top: variables.$spacing-lg; - margin-bottom: variables.$spacing-lg; -} - -.enrollLink { - all: unset; - text-decoration: underline; - cursor: pointer; - color: colors.$blue; -} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx deleted file mode 100644 index 5eb53293429..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/InlineEnrollmentCallout.tsx +++ /dev/null @@ -1,52 +0,0 @@ -import classNames from "classnames"; -import React, { PropsWithChildren } from "react"; -import { FormattedMessage } from "react-intl"; - -import { Message } from "components/ui/Message"; -import { Text } from "components/ui/Text"; - -import { useFreeConnectorProgram } from "core/api/cloud"; - -import { useShowEnrollmentModal } from "./EnrollmentModal"; -import styles from "./InlineEnrollmentCallout.module.scss"; - -export const EnrollLink: React.FC> = ({ children }) => { - const { showEnrollmentModal } = useShowEnrollmentModal(); - - return ( - - ); -}; - -interface InlineEnrollmentCalloutProps { - withMargin?: boolean; -} - -export const InlineEnrollmentCallout: React.FC = ({ withMargin }) => { - const { userDidEnroll, programStatusQuery } = useFreeConnectorProgram(); - const { showEnrollmentUi } = programStatusQuery.data || {}; - - if (userDidEnroll || !showEnrollmentUi) { - return null; - } - return ( - - {content}, - }} - /> - - } - className={classNames(styles.container, { [styles.withMargin]: withMargin })} - /> - ); -}; - -export default InlineEnrollmentCallout; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss deleted file mode 100644 index 3959159143b..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.module.scss +++ /dev/null @@ -1,30 +0,0 @@ -@use "scss/variables"; -@use "scss/colors"; - -.container { - padding: variables.$spacing-lg variables.$spacing-xl; - width: 100%; - border-radius: variables.$border-radius-md; - background-color: colors.$blue-200; - background-image: url("./longtail-connectors.svg"); - background-repeat: no-repeat; - background-position-y: center; - background-position-x: -50px; -} - -.title { - text-transform: uppercase; - color: colors.$white; -} - -.description { - color: colors.$white; -} - -.flexRow { - width: 100%; -} - -.enrollButton { - margin-left: auto; -} diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx deleted file mode 100644 index 161ae02dc4f..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import React from "react"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/ui/Button"; -import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { Heading } from "components/ui/Heading"; -import { Text } from "components/ui/Text"; - -import { useBillingPageBanners } from "packages/cloud/views/billing/BillingPage/useBillingPageBanners"; - -import { ReactComponent as ConnectorsBadges } from "./connectors-badges.svg"; -import { useShowEnrollmentModal } from "./EnrollmentModal"; -import styles from "./LargeEnrollmentCallout.module.scss"; - -export const LargeEnrollmentCallout: React.FC = () => { - const { showEnrollmentModal } = useShowEnrollmentModal(); - const { showFCPBanner } = useBillingPageBanners(); - - if (!showFCPBanner) { - return null; - } - - return ( -
- - - - - - - - - - - - - - -
- ); -}; - -export default LargeEnrollmentCallout; diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg deleted file mode 100644 index 71eb4f43730..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/connectors-badges.svg +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg b/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg deleted file mode 100644 index 92bfc42f8ad..00000000000 --- a/airbyte-webapp/src/packages/cloud/components/experiments/FreeConnectorProgram/longtail-connectors.svg +++ /dev/null @@ -1,667 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json index 0f13d79c6e9..6ec407b2e19 100644 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ b/airbyte-webapp/src/packages/cloud/locales/en.json @@ -195,21 +195,7 @@ "sidebar.billing": "Billing", "freeConnectorProgram.title": "Free Connector Program", - "freeConnectorProgram.enrollNow": "Enroll now!", - "freeConnectorProgram.enroll.description": "Enroll in the Free Connector Program to use Alpha and Beta connectors for free.", - "freeConnectorProgram.enroll.success": "Successfully enrolled in the Free Connector Program", - "freeConnectorProgram.enroll.failure": "Unable to verify that payment details were saved successfully. Please contact support for additional help.", - "freeConnectorProgram.enrollmentModal.title": "Free connector program", - "freeConnectorProgram.enrollmentModal.free": "Alpha and Beta Connectors are free while you're in the program.The whole Connection is free until both Connectors have moved into General Availability (GA)", - "freeConnectorProgram.enrollmentModal.emailNotification": "We will email you before your connection will start being charged.", - "freeConnectorProgram.enrollmentModal.cardOnFile": "When both Connectors are in GA, the Connection will no longer be free. You'll need to have a credit card on file to enroll so Airbyte can handle a Connection's transition to paid service.", - "freeConnectorProgram.enrollmentModal.unvalidatedEmailWarning": "You need to verify your email address before you can enroll in the Free Connector Program. Re-send verification email.", - "freeConnectorProgram.enrollmentModal.validationEmailConfirmation": "Verification email sent", - "freeConnectorProgram.enrollmentModal.cancelButtonText": "Cancel", - "freeConnectorProgram.enrollmentModal.enrollButtonText": "Enroll now!", - "freeConnectorProgram.enrollmentModal.unvalidatedEmailButtonText": "Resend email validation", "freeConnectorProgram.releaseStageBadge.free": "Free", - "freeConnectorProgram.joinTheFreeConnectorProgram": "Join the Free Connector Program to use Alpha and Beta connectors for free Enroll now", "experiment.speedyConnection": "Set up your first connection in the next and get 100 additional credits for your trial", diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx index 26f8fb51a41..3fd44c99cb2 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/BillingPage.tsx @@ -12,9 +12,7 @@ import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; -import { FeatureItem, useFeature } from "core/services/features"; import { links } from "core/utils/links"; -import LargeEnrollmentCallout from "packages/cloud/components/experiments/FreeConnectorProgram/LargeEnrollmentCallout"; import styles from "./BillingPage.module.scss"; import { CreditsUsage } from "./components/CreditsUsage"; @@ -44,7 +42,6 @@ const StripePortalLink: React.FC = () => { }; export const BillingPage: React.FC = () => { useTrackPage(PageTrackingCodes.CREDITS); - const fcpEnabled = useFeature(FeatureItem.FreeConnectorProgram); return ( { > - {fcpEnabled && } diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx index d2d93c0a581..aa0b9e0b9ab 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.test.tsx @@ -66,313 +66,275 @@ describe("useBillingPageBanners", () => { }; }); describe("enrolled in FCP", () => { - it("should show an info variant banner and no FCP materials", () => { + it("should show an info variant banner", () => { mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show FCP enrollment materials + an info variant banner", () => { + it("should an info variant banner", () => { mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("out of trial with 0 connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banners + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banners if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banners + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banners + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banners if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banners + no FCP enrollment materials if 0 credits", () => { + it("should show error banners if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); }); describe("out of trial with only eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info variant banner + no FCP banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error variant banner + no FCP banner if < 20 credits", () => { + it("should show error variant banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error variant banner + no FCP banner if user is in 0 credits state", () => { + it("should show error variant banner if user is in 0 credits state", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show FCP banner + info variant banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); it("should show FCP banner + warning variant banner if user has < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); it("should show FCP banner + error variant banner if user has 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("out of trial with only non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); }); describe("out of trial with eligible and non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + FCP enrollment materials if more than 20 credits", () => { + it("should show info banner if more than 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show warning banner + FCP enrollment materials if fewer than 20 credits", () => { + it("should show warning banner if fewer than 20 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show error banner + FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("credit purchased with only eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info variant banner + no FCP banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show info variant banner + no FCP banner if < 20 credits", () => { + it("should show info variant banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show info variant banner + no FCP banner if user is in 0 credits state", () => { + it("should show info variant banner if user is in 0 credits state", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show FCP banner + info variant banner if > 20 credits", () => { + it("should show info variant banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show FCP banner + info variant banner if user has < 20 credits", () => { + it("should show info variant banner if user has < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(true); }); - it("should show FCP banner + error variant banner if user has 0 credits", () => { + it("should show error variant banner if user has 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(true); }); }); }); describe("credit purchased with only non-eligible connections enabled", () => { describe("enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, true); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); describe("not enrolled in FCP", () => { - it("should show info banner + no FCP enrollment materials if > 20 credits", () => { + it("should show info banner if > 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("info"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show warning banner + no FCP enrollment materials if < 20 credits", () => { + it("should show warning banner if < 20 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("warning"); - expect(result.current.showFCPBanner).toEqual(false); }); - it("should show error banner + no FCP enrollment materials if 0 credits", () => { + it("should show error banner if 0 credits", () => { mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, false); const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); expect(result.current.bannerVariant).toEqual("error"); - expect(result.current.showFCPBanner).toEqual(false); }); }); }); diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx index 3c56c1a9dd6..2575e9abcd2 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx @@ -8,9 +8,8 @@ import { LOW_BALANCE_CREDIT_THRESHOLD } from "./components/LowCreditBalanceHint/ export const useBillingPageBanners = () => { const currentWorkspace = useCurrentWorkspace(); const cloudWorkspace = useGetCloudWorkspace(currentWorkspace.workspaceId); - const { programStatusQuery, userDidEnroll } = useFreeConnectorProgram(); - const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled, showEnrollmentUi } = - programStatusQuery.data || {}; + const { programStatusQuery } = useFreeConnectorProgram(); + const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled } = programStatusQuery.data || {}; const isNewTrialPolicyEnabled = useExperiment("billing.newTrialPolicy", false); const isPreTrial = isNewTrialPolicyEnabled @@ -40,15 +39,7 @@ export const useBillingPageBanners = () => { return "info"; }; - const calculateShowFcpBanner = () => { - if (!isEnrolled && !userDidEnroll && showEnrollmentUi && (hasEligibleConnections || isPreTrial)) { - return true; - } - return false; - }; - return { bannerVariant: calculateVariant(), - showFCPBanner: calculateShowFcpBanner(), }; }; From b49ca0096da39f5e4e81d14cb18203c27b662bef Mon Sep 17 00:00:00 2001 From: Chandler Prall Date: Fri, 1 Sep 2023 09:28:08 -0600 Subject: [PATCH 090/201] =?UTF-8?q?=F0=9F=AA=9F=C2=A0=F0=9F=8E=89=20histor?= =?UTF-8?q?ical=20status=20overview=20-=20data=20moved=20graph=20(#8637)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/package.json | 2 + airbyte-webapp/pnpm-lock.yaml | 15 ++++++ .../DataMovedGraph.module.scss | 15 ++++++ .../data-moved-graph/DataMovedGraph.tsx | 48 +++++++++++++++++++ .../components/data-moved-graph/index.ts | 1 + .../HistoricalOverview.tsx | 35 ++++++++++++++ .../components/historical-overview/index.ts | 1 + .../src/area/connection/components/index.ts | 2 + .../hooks/services/Experiment/experiments.ts | 1 + airbyte-webapp/src/locales/en.json | 3 ++ .../components/UsagePerDayGraph.tsx | 1 - .../StreamStatusPage/ConnectionStatusCard.tsx | 5 ++ 12 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.module.scss create mode 100644 airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.tsx create mode 100644 airbyte-webapp/src/area/connection/components/data-moved-graph/index.ts create mode 100644 airbyte-webapp/src/area/connection/components/historical-overview/HistoricalOverview.tsx create mode 100644 airbyte-webapp/src/area/connection/components/historical-overview/index.ts create mode 100644 airbyte-webapp/src/area/connection/components/index.ts diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 806cd154c0d..f78d56257b2 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -78,6 +78,7 @@ "@types/uuid": "^9.0.0", "anser": "^2.1.1", "any-base": "^1.1.0", + "byte-size": "^8.1.1", "classnames": "^2.3.1", "country-flag-icons": "^1.5.7", "date-fns": "^2.29.3", @@ -152,6 +153,7 @@ "@testing-library/jest-dom": "^6.0.0", "@testing-library/react": "^14.0.0", "@testing-library/user-event": "^14.4.3", + "@types/byte-size": "^8.1.0", "@types/flat": "^5.0.2", "@types/jest": "^29.5.3", "@types/js-yaml": "^4.0.5", diff --git a/airbyte-webapp/pnpm-lock.yaml b/airbyte-webapp/pnpm-lock.yaml index 59e6067c593..4ec763219de 100644 --- a/airbyte-webapp/pnpm-lock.yaml +++ b/airbyte-webapp/pnpm-lock.yaml @@ -96,6 +96,9 @@ dependencies: any-base: specifier: ^1.1.0 version: 1.1.0 + byte-size: + specifier: ^8.1.1 + version: 8.1.1 classnames: specifier: ^2.3.1 version: 2.3.2 @@ -314,6 +317,9 @@ devDependencies: '@testing-library/user-event': specifier: ^14.4.3 version: 14.4.3(@testing-library/dom@9.3.1) + '@types/byte-size': + specifier: ^8.1.0 + version: 8.1.0 '@types/flat': specifier: ^5.0.2 version: 5.0.2 @@ -7826,6 +7832,10 @@ packages: '@types/node': 18.16.3 dev: true + /@types/byte-size@8.1.0: + resolution: {integrity: sha512-LCIlZh8vyx+I2fgRycE1D34c33QDppYY6quBYYoaOpQ1nGhJ/avSP2VlrAefVotjJxgSk6WkKo0rTcCJwGG7vA==} + dev: true + /@types/connect@3.4.35: resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} dependencies: @@ -9438,6 +9448,11 @@ packages: resolution: {integrity: sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==} dev: true + /byte-size@8.1.1: + resolution: {integrity: sha512-tUkzZWK0M/qdoLEqikxBWe4kumyuwjl3HO6zHTr4yEI23EojPtLYXdG1+AQY7MN0cGyNDvEaJ8wiYQm6P2bPxg==} + engines: {node: '>=12.17'} + dev: false + /bytes@3.0.0: resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} engines: {node: '>= 0.8'} diff --git a/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.module.scss b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.module.scss new file mode 100644 index 00000000000..979694d3639 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.module.scss @@ -0,0 +1,15 @@ +@use "scss/colors"; + +.tooltipWrapper { + background-color: colors.$foreground !important; + color: colors.$dark-blue span { + color: colors.$dark-blue; + } +} + +:export { + graphColor: colors.$grey-100; + chartHoverFill: colors.$grey-50; + tooltipLabelColor: colors.$dark-blue; + tooltipItemColor: colors.$grey-700; +} diff --git a/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.tsx b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.tsx new file mode 100644 index 00000000000..6034f4852d7 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/data-moved-graph/DataMovedGraph.tsx @@ -0,0 +1,48 @@ +import byteSize from "byte-size"; +import { Bar, BarChart, ResponsiveContainer, Tooltip, XAxis } from "recharts"; + +import { Box } from "components/ui/Box"; + +import styles from "./DataMovedGraph.module.scss"; + +const sampleData: Array<{ date: number; bytes: number }> = []; +for (let i = 1; i <= 30; i++) { + sampleData.push({ date: Date.UTC(2023, 7, i), bytes: Math.round(Math.random() * 1000 + 200) }); +} + +export const DataMovedGraph: React.FC = () => { + return ( + + + new Date(x).toLocaleDateString(undefined, { month: "short", day: "numeric" })} + /> + {new Date(value).toLocaleDateString()}} + formatter={(value: number) => { + // The type cast is unfortunately necessary, due to broken typing in recharts. + // What we return is a [string, undefined], and the library accepts this as well, but the types + // require the first element to be of the same type as value, which isn't what the formatter + // is supposed to do: https://github.com/recharts/recharts/issues/3008 + const prettyvalues = byteSize(value); + return [ + <> + {prettyvalues.value} {prettyvalues.long} + , + undefined, + ] as unknown as [number, string]; + }} + /> + + + + ); +}; diff --git a/airbyte-webapp/src/area/connection/components/data-moved-graph/index.ts b/airbyte-webapp/src/area/connection/components/data-moved-graph/index.ts new file mode 100644 index 00000000000..4dae61d273e --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/data-moved-graph/index.ts @@ -0,0 +1 @@ +export * from "./DataMovedGraph"; diff --git a/airbyte-webapp/src/area/connection/components/historical-overview/HistoricalOverview.tsx b/airbyte-webapp/src/area/connection/components/historical-overview/HistoricalOverview.tsx new file mode 100644 index 00000000000..c89b9f26701 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/historical-overview/HistoricalOverview.tsx @@ -0,0 +1,35 @@ +import { useState } from "react"; +import { FormattedMessage } from "react-intl"; + +import { Box } from "components/ui/Box"; +import { Tabs } from "components/ui/Tabs"; +import { ButtonTab } from "components/ui/Tabs/ButtonTab"; + +import { DataMovedGraph } from "../data-moved-graph"; + +export const HistoricalOverview: React.FC = () => { + const [selectedTab, setSelectedTab] = useState<"uptimeStatus" | "dataMoved">("uptimeStatus"); + + return ( + + + } + isActive={selectedTab === "uptimeStatus"} + onSelect={() => setSelectedTab("uptimeStatus")} + /> + } + isActive={selectedTab === "dataMoved"} + onSelect={() => setSelectedTab("dataMoved")} + /> + + + {selectedTab === "uptimeStatus" &&
uptime
} + {selectedTab === "dataMoved" && } + + + ); +}; diff --git a/airbyte-webapp/src/area/connection/components/historical-overview/index.ts b/airbyte-webapp/src/area/connection/components/historical-overview/index.ts new file mode 100644 index 00000000000..77dbeb0ed2a --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/historical-overview/index.ts @@ -0,0 +1 @@ +export * from "./HistoricalOverview"; diff --git a/airbyte-webapp/src/area/connection/components/index.ts b/airbyte-webapp/src/area/connection/components/index.ts new file mode 100644 index 00000000000..c2536b3fb4c --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/index.ts @@ -0,0 +1,2 @@ +export * from "./data-moved-graph"; +export * from "./historical-overview"; diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts index d31dad825b1..18a4d53fde8 100644 --- a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts +++ b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts @@ -21,6 +21,7 @@ export interface Experiments { "connection.streamCentricUI.lateMultiplier": number; "connection.streamCentricUI.numberOfLogsToLoad": number; "connection.streamCentricUI.v2": boolean; + "connection.streamCentricUI.historicalOverview": boolean; "connection.syncCatalog.simplifiedCatalogRow": boolean; "connector.airbyteCloudIpAddresses": string; "connector.allowlistIpBanner": boolean; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index af9cad98341..16f85e99d53 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -581,6 +581,9 @@ "connection.stream.status.nextSync": "Next sync {sync}", "connection.stream.status.nextTry": "Next try {sync}", + "connection.overview.graph.uptimeStatus": "Uptime status", + "connection.overview.graph.dataMoved": "Data moved", + "form.frequency": "Replication frequency", "form.cronExpression": "Cron expression", "form.cronExpression.placeholder": "Cron expression", diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx index e45cf61de1f..1e293fe94e1 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx @@ -97,7 +97,6 @@ export const UsagePerDayGraph: React.FC = ({ chartData, m cursor={{ fill: chartHoverFill }} wrapperStyle={{ outline: "none" }} wrapperClassName={styles.tooltipWrapper} - labelClassName={styles.tooltipLabel} formatter={(value: number, payload) => { // The type cast is unfortunately necessary, due to broken typing in recharts. // What we return is a [string, string], and the library accepts this as well, but the types diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusCard.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusCard.tsx index ddac8e7d062..7965cd24d3a 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusCard.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusCard.tsx @@ -4,7 +4,9 @@ import { ConnectionSyncButtons } from "components/connection/ConnectionSync/Conn import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; +import { HistoricalOverview } from "area/connection/components"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; +import { useExperiment } from "hooks/services/Experiment"; import { ConnectionStatusOverview } from "./ConnectionStatusOverview"; import { ErrorMessage } from "./ErrorMessage"; @@ -16,6 +18,8 @@ export const ConnectionStatusCard: React.FC = () => { } = connection; const streamCount = streams.reduce((count, stream) => count + (stream.config?.selected ? 1 : 0), 0); + const showHistoricalOverview = useExperiment("connection.streamCentricUI.historicalOverview", false); + return ( { } > + {showHistoricalOverview && } ); }; From 1eecd6c9b811914c7d402f2022929887c3e734c8 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Fri, 1 Sep 2023 09:55:35 -0700 Subject: [PATCH 091/201] Improve error message of pod creation failure (#8144) --- .../java/io/airbyte/workers/process/KubeProcessFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java index b813dd9e1ef..4eba409a968 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java @@ -171,7 +171,7 @@ public Process create( internalToExternalPorts, args).toProcess(); } catch (final Exception e) { - throw new WorkerException(e.getMessage(), e); + throw new WorkerException("Failed to create pod for " + jobType + " step", e); } } From 69cd4dc94216da13dd41e2079951d3293675fbf9 Mon Sep 17 00:00:00 2001 From: Conor Date: Fri, 1 Sep 2023 12:25:25 -0500 Subject: [PATCH 092/201] Revert "Bump Airbyte version from 0.50.24 to 0.0.99" (#8662) --- .bumpversion.cfg | 2 +- .env | 2 +- airbyte-connector-builder-server/Dockerfile | 4 ++-- airbyte-container-orchestrator/Dockerfile | 2 +- airbyte-proxy/Dockerfile | 2 +- airbyte-server/Dockerfile | 2 +- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.yaml | 2 +- charts/airbyte/README.md | 20 +++++++++---------- 20 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0d91c47475d..7bc8e02fe00 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.99-test +current_version = 0.50.24 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index f9aa6a82fe7..de28947e9be 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.0.99 +VERSION=0.50.24 # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index 7e7c985c841..79e04ff511d 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -10,7 +10,7 @@ ENV PIP=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/pip COPY requirements.txt requirements.txt RUN ${PIP} install -r requirements.txt -ARG VERSION=0.0.99 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-connector-builder-server ENV VERSION ${VERSION} @@ -23,5 +23,5 @@ ADD airbyte-app.tar /app # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] -LABEL io.airbyte.version=0.0.99 +LABEL io.airbyte.version=0.50.24 LABEL io.airbyte.name=airbyte/connector-builder-server \ No newline at end of file diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index fa11a242d01..633d9fae785 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -10,7 +10,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.0.99 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-proxy/Dockerfile b/airbyte-proxy/Dockerfile index a6ed8826364..4542e765f64 100644 --- a/airbyte-proxy/Dockerfile +++ b/airbyte-proxy/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:latest -ARG VERSION=0.0.99 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-proxy ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 56310cbf809..4bdb76fb63e 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,7 +3,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 5005 -ARG VERSION=0.0.99 +ARG VERSION=0.50.24 ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index acd2e869cdf..671ff9be7e3 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 7b2763ffc95..8aa70821c28 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index e5ddfe5a591..2d8cc123fdc 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index f66b3e6c32e..f962620e5a5 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index e5091cd3a9e..b6deccfc7d6 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 238065e3b9d..175e105937a 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index e7f93f322c7..db84a557f79 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 7782ea7aa45..0a4967cbc40 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 420be52cd7a..4c75512614a 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index c4e7c570b9d..2e35cb9029f 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 9759f480dbd..6a5edc09e38 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 15b6ba5df98..974f323823a 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -22,7 +22,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 0b030ee01c0..335a0f13700 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -21,7 +21,7 @@ version: 0.48.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.0.99 +appVersion: 0.50.24 dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index cb3217ebeda..e87bc3d30de 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,6 @@ # airbyte -![Version: 0.0.99](https://img.shields.io/badge/Version-0.0.99-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.0.99](https://img.shields.io/badge/AppVersion-0.0.99-informational?style=flat-square) +![Version: 0.50.24](https://img.shields.io/badge/Version-0.50.24-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.50.24](https://img.shields.io/badge/AppVersion-0.50.24-informational?style=flat-square) Helm chart to deploy airbyte @@ -8,15 +8,15 @@ Helm chart to deploy airbyte | Repository | Name | Version | |------------|------|---------| -| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | cron | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | metrics | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | server | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | temporal | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | webapp | 0.0.99 | -| https://airbytehq.github.io/helm-charts/ | worker | 0.0.99 | +| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | cron | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | metrics | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | server | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | temporal | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | webapp | 0.50.24 | +| https://airbytehq.github.io/helm-charts/ | worker | 0.50.24 | | https://charts.bitnami.com/bitnami | common | 1.x.x | ## Values From 3b9f82592e189917c17061688383b7d1ddb0d0b6 Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Fri, 1 Sep 2023 11:01:58 -0700 Subject: [PATCH 093/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Fix=20fir?= =?UTF-8?q?st=20builder=20connector=20not=20appearing=20on=20Sources=20pag?= =?UTF-8?q?e=20after=20publishing=20(#8652)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/source/SelectConnector/SelectConnector.tsx | 7 ++----- .../src/core/api/hooks/connectorBuilderProject.ts | 2 ++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx index c06ff5ea104..80d5195fbb1 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx @@ -44,12 +44,9 @@ export const SelectConnector: React.FC = ({ const { openModal, closeModal } = useModalService(); const trackSelectConnector = useTrackSelectConnector(connectorType); const [searchTerm, setSearchTerm] = useState(""); - const defaultReleaseStages = DEFAULT_SELECTED_RELEASE_STAGES.filter((stage) => - connectorDefinitions.some((definition) => definition.releaseStage === stage) - ); const [releaseStagesInLocalStorage, setSelectedReleaseStages] = useLocalStorage( "airbyte_connector-grid-release-stage-filter", - defaultReleaseStages + [] ); const availableReleaseStages = RELEASE_STAGES.filter((stage) => connectorDefinitions.some((d) => d.releaseStage === stage) @@ -58,7 +55,7 @@ export const SelectConnector: React.FC = ({ availableReleaseStages.includes(releaseStage) ); if (selectedReleaseStages.length === 0) { - selectedReleaseStages.push(...defaultReleaseStages); + selectedReleaseStages.push(...DEFAULT_SELECTED_RELEASE_STAGES); } const updateSelectedReleaseStages = (updatedSelectedReleaseStages: ReleaseStage[]) => { diff --git a/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts b/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts index 8b4c4a9b50d..a60c1f87544 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts @@ -1,6 +1,7 @@ import { QueryClient, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { sourceDefinitionKeys } from "services/connector/SourceDefinitionService"; import { SCOPE_WORKSPACE } from "services/Scope"; import { useBuilderResolveManifestQuery } from "./connectorBuilderApi"; @@ -289,6 +290,7 @@ export const usePublishBuilderProject = () => { onSuccess(data, context) { queryClient.removeQueries(connectorBuilderProjectsKeys.versions(context.projectId)); updateProjectQueryCache(queryClient, context.projectId, data.sourceDefinitionId, FIRST_VERSION); + queryClient.invalidateQueries(sourceDefinitionKeys.lists()); }, } ); From b13785c77d22e08e58116721e8bfb2a67e1c9323 Mon Sep 17 00:00:00 2001 From: terencecho <3916587+terencecho@users.noreply.github.com> Date: Fri, 1 Sep 2023 11:34:25 -0700 Subject: [PATCH 094/201] Airbyte-API: Propagate read timeout and increase timeout limit (#8649) --- .../kotlin/io/airbyte/api/server/services/JobService.kt | 6 ++++++ airbyte-api-server/src/main/resources/application.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt index 30cec987772..ba1e616fcb4 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt @@ -84,6 +84,12 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi } catch (e: HttpClientResponseException) { log.error("Config api response error for job sync: ", e) e.response as HttpResponse + } catch (e: ReadTimeoutException) { + log.error("Config api response error for job sync: ", e) + throw UnexpectedProblem( + HttpStatus.REQUEST_TIMEOUT, + "Request timed out. Please check the latest job status to determine whether the sync started.", + ) } ConfigClientErrorHandler.handleError(response, connectionId.toString()) log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) diff --git a/airbyte-api-server/src/main/resources/application.yml b/airbyte-api-server/src/main/resources/application.yml index 8bee489ce57..ead9bac54a8 100644 --- a/airbyte-api-server/src/main/resources/application.yml +++ b/airbyte-api-server/src/main/resources/application.yml @@ -18,7 +18,7 @@ micronaut: idle-timeout: ${HTTP_IDLE_TIMEOUT:5m} http: client: - read-timeout: 20s + read-timeout: 150s max-content-length: 52428800 # 50MB airbyte: From 6fac09f22cd7a7aa2d4f9522e04d835b75ed51b7 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Fri, 1 Sep 2023 19:20:05 +0000 Subject: [PATCH 095/201] Bump helm chart version reference to 0.48.6 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 671ff9be7e3..979b7c9784d 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 8aa70821c28..428819a1a2c 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 2d8cc123fdc..67ffe9ef26f 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index f962620e5a5..c247333032d 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index b6deccfc7d6..b1891f74783 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 175e105937a..e8abb6f1cbf 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index db84a557f79..08566d418df 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 0a4967cbc40..f26dd1d5589 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 4c75512614a..ded18696c7e 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 2e35cb9029f..a4dc8fe4580 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 6a5edc09e38..506ce05e54b 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 974f323823a..5936e974af9 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 33d6464d5f4..869ba502171 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 -digest: sha256:5cffde2650218f52829dff0bcb34d9962c94b47f0eb0a34feea665ab1e5fdebc -generated: "2023-09-01T00:06:46.698637914Z" + version: 0.48.6 +digest: sha256:d7a814247ae1fc194fbdc2807ae114f06ae80ea68a626256fa25c2fdd5a9bfe7 +generated: "2023-09-01T19:19:52.389785043Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 335a0f13700..58e0c936cda 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.5 +version: 0.48.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 + version: 0.48.6 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.5 \ No newline at end of file + version: 0.48.6 \ No newline at end of file From 5c257c34906acb04b2f37acfa7e2c14d24907844 Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Fri, 1 Sep 2023 21:43:51 +0200 Subject: [PATCH 096/201] Skip non-normalization operations on resets (#8654) --- .../server/handlers/SchedulerHandler.java | 8 ++++++- .../server/handlers/SchedulerHandlerTest.java | 21 +++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index 798346964ce..36efa9c3dd5 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -636,7 +636,13 @@ public JobInfoRead createJob(final JobCreate jobCreate) throws JsonValidationExc final List standardSyncOperations = Lists.newArrayList(); for (final var operationId : standardSync.getOperationIds()) { final StandardSyncOperation standardSyncOperation = configRepository.getStandardSyncOperation(operationId); - standardSyncOperations.add(standardSyncOperation); + // NOTE: we must run normalization operations during resets, because we rely on them to clear the + // normalized tables. However, we don't want to run other operations (dbt, webhook) because those + // are meant to transform the data after the sync but there's no data to transform. Webhook + // operations particularly will fail because we don't populate some required config during resets. + if (StandardSyncOperation.OperatorType.NORMALIZATION.equals(standardSyncOperation.getOperatorType())) { + standardSyncOperations.add(standardSyncOperation); + } } final Optional jobIdOptional = diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java index 5bece60a777..44f915e6471 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SchedulerHandlerTest.java @@ -92,12 +92,15 @@ import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.config.JobTypeResourceLimit; import io.airbyte.config.JobTypeResourceLimit.JobType; +import io.airbyte.config.OperatorNormalization; +import io.airbyte.config.OperatorWebhook; import io.airbyte.config.ResourceRequirements; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.helpers.LogConfigs; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -222,6 +225,17 @@ class SchedulerHandlerTest { .withSupportedDestinationSyncModes(List.of(DestinationSyncMode.OVERWRITE, DestinationSyncMode.APPEND, DestinationSyncMode.APPEND_DEDUP)) .withDocumentationUrl(URI.create("unused"))); + private static final StandardSyncOperation NORMALIZATION_OPERATION = new StandardSyncOperation() + .withOperatorType(StandardSyncOperation.OperatorType.NORMALIZATION) + .withOperatorNormalization(new OperatorNormalization()); + + private static final UUID NORMALIZATION_OPERATION_ID = UUID.randomUUID(); + + public static final StandardSyncOperation WEBHOOK_OPERATION = new StandardSyncOperation() + .withOperatorType(StandardSyncOperation.OperatorType.WEBHOOK) + .withOperatorWebhook(new OperatorWebhook()); + private static final UUID WEBHOOK_OPERATION_ID = UUID.randomUUID(); + private SchedulerHandler schedulerHandler; private ConfigRepository configRepository; private SecretsRepositoryWriter secretsRepositoryWriter; @@ -322,7 +336,10 @@ void createJob() throws JsonValidationException, ConfigNotFoundException, IOExce @Test @DisplayName("Test reset job creation") void createResetJob() throws JsonValidationException, ConfigNotFoundException, IOException { - final StandardSync standardSync = new StandardSync().withDestinationId(DESTINATION_ID); + Mockito.when(configRepository.getStandardSyncOperation(NORMALIZATION_OPERATION_ID)).thenReturn(NORMALIZATION_OPERATION); + Mockito.when(configRepository.getStandardSyncOperation(WEBHOOK_OPERATION_ID)).thenReturn(WEBHOOK_OPERATION); + final StandardSync standardSync = + new StandardSync().withDestinationId(DESTINATION_ID).withOperationIds(List.of(NORMALIZATION_OPERATION_ID, WEBHOOK_OPERATION_ID)); Mockito.when(configRepository.getStandardSync(CONNECTION_ID)).thenReturn(standardSync); final DestinationConnection destination = new DestinationConnection() .withDestinationId(DESTINATION_ID) @@ -344,7 +361,7 @@ void createResetJob() throws JsonValidationException, ConfigNotFoundException, I Mockito .when(jobCreator.createResetConnectionJob(destination, standardSync, destinationDefinition, actorDefinitionVersion, DOCKER_IMAGE_NAME, destinationVersion, - false, List.of(), + false, List.of(NORMALIZATION_OPERATION), streamsToReset, WORKSPACE_ID)) .thenReturn(Optional.of(JOB_ID)); From 8a85fee4560e0c718eaee57074e0d2f0e11802e6 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 1 Sep 2023 15:54:19 -0400 Subject: [PATCH 097/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20(flagged)?= =?UTF-8?q?=20Restrict=20Permission=20Update=20UI=20to=20users=20with=20ad?= =?UTF-8?q?equate=20permissions=20(#8668)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/core/utils/rbac/intent.ts | 4 ++++ airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx | 7 +++++-- .../components/RoleManagementControl.tsx | 8 +++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/core/utils/rbac/intent.ts b/airbyte-webapp/src/core/utils/rbac/intent.ts index b75547acf38..550aa0acb45 100644 --- a/airbyte-webapp/src/core/utils/rbac/intent.ts +++ b/airbyte-webapp/src/core/utils/rbac/intent.ts @@ -6,14 +6,18 @@ export enum Intent { // organization "ListOrganizationMembers" = "ListOrganizationMembers", + "UpdateOrganizationPermissions" = "UpdateOrganizationPermissions", // workspace "ListWorkspaceMembers" = "ListWorkspaceMembers", + "UpdateWorkspacePermissions" = "UpdateWorkspacePermissions", } const intentToRbacQuery = { [Intent.ListOrganizationMembers]: { resourceType: "ORGANIZATION", role: "READER" }, + [Intent.UpdateOrganizationPermissions]: { resourceType: "ORGANIZATION", role: "ADMIN" }, + [Intent.UpdateWorkspacePermissions]: { resourceType: "WORKSPACE", role: "ADMIN" }, [Intent.ListWorkspaceMembers]: { resourceType: "WORKSPACE", role: "READER" }, } as const; diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx index 98b9bae8666..a93de0b4e88 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx @@ -11,6 +11,7 @@ import { PageHeader } from "components/ui/PageHeader"; import { SideMenu, CategoryItem, SideMenuItem } from "components/ui/SideMenu"; import { useCurrentWorkspace } from "core/api"; +import { Intent, useIntent } from "core/utils/rbac/intent"; import { useExperiment } from "hooks/services/Experiment"; import { useGetConnectorsOutOfDate } from "hooks/services/useConnector"; @@ -52,6 +53,8 @@ const SettingsPage: React.FC = ({ pageConfig }) => { const { countNewSourceVersion, countNewDestinationVersion } = useGetConnectorsOutOfDate(); const newWorkspacesUI = useExperiment("workspaces.newWorkspacesUI", false); const isAccessManagementEnabled = useExperiment("settings.accessManagement", false); + const canListWorkspaceUsers = useIntent(Intent.ListWorkspaceMembers); + const canListOrganizationUsers = useIntent(Intent.ListOrganizationMembers); const menuItems: CategoryItem[] = pageConfig?.menuConfig || [ { @@ -103,7 +106,7 @@ const SettingsPage: React.FC = ({ pageConfig }) => { name: , component: MetricsPage, }, - ...(isAccessManagementEnabled && !pageConfig + ...(isAccessManagementEnabled && !pageConfig && canListWorkspaceUsers ? [ { path: `${SettingsRoute.Workspace}/${SettingsRoute.AccessManagement}`, @@ -124,7 +127,7 @@ const SettingsPage: React.FC = ({ pageConfig }) => { name: , component: GeneralOrganizationSettingsPage, }, - ...(isAccessManagementEnabled && !pageConfig + ...(isAccessManagementEnabled && !pageConfig && canListOrganizationUsers ? [ { path: `${SettingsRoute.Organization}/${SettingsRoute.AccessManagement}`, diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx index acbfc4435b8..fc77a949876 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx @@ -10,6 +10,7 @@ import { Text } from "components/ui/Text"; import { useDeletePermissions, useUpdatePermissions } from "core/api"; import { PermissionType, PermissionUpdate } from "core/request/AirbyteClient"; +import { Intent, useIntent } from "core/utils/rbac/intent"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import styles from "./RoleManagementControl.module.scss"; @@ -51,11 +52,16 @@ export const RoleManagementControl: React.FC = ({ const { formatMessage } = useIntl(); const isEditMode = activeEditRow === permissionId; + const intentKey = + pageResourceType === "organization" ? Intent.UpdateOrganizationPermissions : Intent.UpdateWorkspacePermissions; + + const canUpdateUserPermissions = useIntent(intentKey); + if (!permissionType) { return null; } - if (pageResourceType !== tableResourceType) { + if (pageResourceType !== tableResourceType || !canUpdateUserPermissions) { return ( From 94e8350c7a13fb66a69f993e6e0959741084ddfc Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Fri, 1 Sep 2023 13:32:17 -0700 Subject: [PATCH 098/201] Add AttemptNumber to worker traces (#8641) --- .../general/BufferedReplicationWorker.java | 6 +++++- .../general/DefaultReplicationWorker.java | 3 +-- .../general/ReplicationWorkerHelper.java | 5 ++++- .../airbyte/workers/sync/LauncherWorker.java | 2 +- .../io/airbyte/metrics/lib/ApmTraceUtils.java | 6 +++++- .../metrics/lib/ApmTraceUtilsTest.java | 21 +++++++++++++++---- 6 files changed, 33 insertions(+), 10 deletions(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java index c9549bf2971..6687288c8d7 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/BufferedReplicationWorker.java @@ -4,6 +4,9 @@ package io.airbyte.workers.general; +import static io.airbyte.metrics.lib.ApmTraceConstants.WORKER_OPERATION_NAME; + +import datadog.trace.api.Trace; import io.airbyte.commons.concurrency.BoundedConcurrentLinkedQueue; import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.converters.ThreadedTimeTracker; @@ -136,6 +139,7 @@ public BufferedReplicationWorker(final String jobId, this.processFromDestStopwatch = new Stopwatch(); } + @Trace(operationName = WORKER_OPERATION_NAME) @Override public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRoot) throws WorkerException { final Map mdc = MDC.getCopyOfContextMap(); @@ -145,7 +149,7 @@ public ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRo try { final ReplicationContext replicationContext = getReplicationContext(syncInput); final ReplicationFeatureFlags flags = replicationFeatureFlagReader.readReplicationFeatureFlags(syncInput); - replicationWorkerHelper.initialize(replicationContext, flags); + replicationWorkerHelper.initialize(replicationContext, flags, jobRoot); // note: resources are closed in the opposite order in which they are declared. thus source will be // closed first (which is what we want). diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index 6091ffc8932..7b47ea85978 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -146,10 +146,9 @@ public final ReplicationOutput run(final StandardSyncInput syncInput, final Path new ReplicationContext(syncInput.getIsReset(), syncInput.getConnectionId(), syncInput.getSourceId(), syncInput.getDestinationId(), Long.parseLong(jobId), attempt, syncInput.getWorkspaceId()); - ApmTraceUtils.addTagsToTrace(replicationContext.connectionId(), jobId, jobRoot); final ReplicationFeatureFlags flags = replicationFeatureFlagReader.readReplicationFeatureFlags(syncInput); - replicationWorkerHelper.initialize(replicationContext, flags); + replicationWorkerHelper.initialize(replicationContext, flags, jobRoot); replicate(jobRoot, syncInput); diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java index 88bae872865..8bb14e5ef39 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerHelper.java @@ -21,6 +21,7 @@ import io.airbyte.config.SyncStats; import io.airbyte.config.WorkerDestinationConfig; import io.airbyte.config.WorkerSourceConfig; +import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricClientFactory; @@ -122,11 +123,13 @@ public void markFailed() { hasFailed.set(true); } - public void initialize(final ReplicationContext replicationContext, final ReplicationFeatureFlags replicationFeatureFlags) { + public void initialize(final ReplicationContext replicationContext, final ReplicationFeatureFlags replicationFeatureFlags, final Path jobRoot) { this.replicationContext = replicationContext; this.replicationFeatureFlags = replicationFeatureFlags; this.timeTracker.trackReplicationStartTime(); this.metricAttrs = toConnectionAttrs(replicationContext); + ApmTraceUtils.addTagsToTrace(replicationContext.connectionId(), replicationContext.attempt().longValue(), + replicationContext.jobId().toString(), jobRoot); } public void startDestination(final AirbyteDestination destination, final StandardSyncInput syncInput, final Path jobRoot) { diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java index 86bacc48fb8..ee7131c025f 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/LauncherWorker.java @@ -168,7 +168,7 @@ public OUTPUT run(final INPUT input, final Path jobRoot) throws WorkerException podName, mainContainerInfo); - ApmTraceUtils.addTagsToTrace(connectionId, jobRunConfig.getJobId(), jobRoot); + ApmTraceUtils.addTagsToTrace(connectionId, jobRunConfig.getAttemptId(), jobRunConfig.getJobId(), jobRoot); final String schedulerName = featureFlagClient.stringVariation(UseCustomK8sScheduler.INSTANCE, new Connection(connectionId)); diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java index 0bf2fccecb6..e8547961edb 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/ApmTraceUtils.java @@ -4,6 +4,7 @@ package io.airbyte.metrics.lib; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ROOT_KEY; @@ -77,12 +78,15 @@ public static void addTagsToTrace(final Span span, final Map tag * All tags added via this method will use the default {@link #TAG_PREFIX} namespace. Any null * values will be ignored. */ - public static void addTagsToTrace(final UUID connectionId, final String jobId, final Path jobRoot) { + public static void addTagsToTrace(final UUID connectionId, final Long attemptNumber, final String jobId, final Path jobRoot) { final Map tags = new HashMap<>(); if (connectionId != null) { tags.put(CONNECTION_ID_KEY, connectionId); } + if (attemptNumber != null) { + tags.put(ATTEMPT_NUMBER_KEY, attemptNumber); + } if (jobId != null) { tags.put(JOB_ID_KEY, jobId); } diff --git a/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java b/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java index 06c1ceef021..e927ab2dedf 100644 --- a/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java +++ b/airbyte-metrics/metrics-lib/src/test/java/io/airbyte/metrics/lib/ApmTraceUtilsTest.java @@ -4,6 +4,7 @@ package io.airbyte.metrics.lib; +import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.ATTEMPT_NUMBER_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.CONNECTION_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ROOT_KEY; @@ -134,27 +135,39 @@ void testAddingTagsWithNullChecks() { final UUID connectionID = UUID.randomUUID(); final String jobId = UUID.randomUUID().toString(); final Path jobRoot = Path.of("dev", "null"); + final Long attemptNumber = Long.valueOf(2L); - ApmTraceUtils.addTagsToTrace(connectionID, jobId, jobRoot); + ApmTraceUtils.addTagsToTrace(connectionID, attemptNumber, jobId, jobRoot); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); clearInvocations(span); - ApmTraceUtils.addTagsToTrace(null, jobId, jobRoot); + ApmTraceUtils.addTagsToTrace(null, null, jobId, jobRoot); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); clearInvocations(span); - ApmTraceUtils.addTagsToTrace(connectionID, jobId, null); + ApmTraceUtils.addTagsToTrace(connectionID, null, jobId, null); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); clearInvocations(span); - ApmTraceUtils.addTagsToTrace((UUID) null, null, null); + ApmTraceUtils.addTagsToTrace(null, attemptNumber, jobId, null); + verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); + verify(span, times(1)).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), jobRoot.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); + + clearInvocations(span); + ApmTraceUtils.addTagsToTrace((UUID) null, null, null, null); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, CONNECTION_ID_KEY), connectionID.toString()); + verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, ATTEMPT_NUMBER_KEY), attemptNumber.toString()); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ID_KEY), jobId); verify(span, never()).setTag(String.format(TAG_FORMAT, TAG_PREFIX, JOB_ROOT_KEY), jobRoot.toString()); } From 32eabb926a5c9497e1401ebf1e595b0f756d3117 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Fri, 1 Sep 2023 14:03:54 -0700 Subject: [PATCH 099/201] Add Workflow unreachable metrics (#8671) --- .../commons/temporal/ConnectionManagerUtils.java | 16 ++++++++++++++-- .../airbyte/commons/temporal/TemporalClient.java | 10 ++++++++++ .../airbyte/metrics/lib/OssMetricsRegistry.java | 3 +++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index 139b2360dca..4133b67e23a 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -9,6 +9,11 @@ import io.airbyte.commons.temporal.scheduling.ConnectionManagerWorkflow; import io.airbyte.commons.temporal.scheduling.ConnectionUpdaterInput; import io.airbyte.commons.temporal.scheduling.state.WorkflowState; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.temporal.api.common.v1.WorkflowExecution; import io.temporal.api.enums.v1.WorkflowExecutionStatus; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest; @@ -22,17 +27,22 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Function; -import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; /** * Utility functions for connection manager workflows. */ -@NoArgsConstructor @Singleton @Slf4j public class ConnectionManagerUtils { + private final MetricClient metricClient; + + public ConnectionManagerUtils() { + // TODO Inject it when MetricClient becomes injectable. + this.metricClient = MetricClientFactory.getMetricClient(); + } + /** * Send a cancellation to the workflow. It will swallow any exception and won't check if the * workflow is already deleted when being cancel. @@ -117,6 +127,8 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W } return connectionManagerWorkflow; } catch (final UnreachableWorkflowException e) { + metricClient.count(OssMetricsRegistry.WORFLOW_UNREACHABLE, 1, + new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString())); log.error( String.format( "Failed to retrieve ConnectionManagerWorkflow for connection %s. " diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index 0f91370625e..bb7d87f97fd 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -22,6 +22,11 @@ import io.airbyte.config.StandardCheckConnectionInput; import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; @@ -76,6 +81,7 @@ public class TemporalClient { private final ConnectionManagerUtils connectionManagerUtils; private final NotificationClient notificationClient; private final StreamResetRecordsHelper streamResetRecordsHelper; + private final MetricClient metricClient; public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, final WorkflowClient client, @@ -91,6 +97,8 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, this.connectionManagerUtils = connectionManagerUtils; this.notificationClient = notificationClient; this.streamResetRecordsHelper = streamResetRecordsHelper; + // TODO Inject it when MetricClient becomes injectable. + this.metricClient = MetricClientFactory.getMetricClient(); } private final Set workflowNames = new HashSet<>(); @@ -567,6 +575,8 @@ public void update(final UUID connectionId) { log.info("Connection {} is deleted, and therefore cannot be updated.", connectionId); return; } catch (final UnreachableWorkflowException e) { + metricClient.count(OssMetricsRegistry.WORFLOW_UNREACHABLE, 1, + new MetricAttribute(MetricTags.CONNECTION_ID, connectionId.toString())); log.error( String.format("Failed to retrieve ConnectionManagerWorkflow for connection %s. Repairing state by creating new workflow.", connectionId), e); diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java index 74d27ab29e2..5fc07cff077 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java @@ -317,6 +317,9 @@ public enum OssMetricsRegistry implements MetricsRegistry { WORKER_SOURCE_MESSAGE_READ(MetricEmittingApps.WORKER, "worker_source_message_read", "whenever a message is read from the source"), + WORFLOW_UNREACHABLE(MetricEmittingApps.WORKER, + "workflow_unreachable", + "whenever a workflow is unreachable"), WORKFLOWS_HEALED(MetricEmittingApps.CRON, "workflows_healed", "number of workflow the self healing cron healed"), From b73299e77ff8eba18a556c2c4e1971ed980cf003 Mon Sep 17 00:00:00 2001 From: Conor Date: Fri, 1 Sep 2023 17:38:14 -0500 Subject: [PATCH 100/201] New release flow (#8650) --- .bumpversion.cfg | 49 ++++++++++++++++--- .env | 2 +- airbyte-connector-builder-server/Dockerfile | 6 +-- airbyte-container-orchestrator/Dockerfile | 2 +- airbyte-proxy/Dockerfile | 2 +- airbyte-server/Dockerfile | 2 +- charts/airbyte-api-server/Chart.yaml | 4 +- charts/airbyte-bootloader/Chart.yaml | 4 +- .../Chart.yaml | 4 +- charts/airbyte-cron/Chart.yaml | 4 +- charts/airbyte-keycloak-setup/Chart.yaml | 6 +-- charts/airbyte-keycloak/Chart.yaml | 6 +-- charts/airbyte-metrics/Chart.yaml | 4 +- charts/airbyte-pod-sweeper/Chart.yaml | 4 +- charts/airbyte-server/Chart.yaml | 4 +- charts/airbyte-temporal/Chart.yaml | 4 +- charts/airbyte-webapp/Chart.yaml | 4 +- charts/airbyte-worker/Chart.yaml | 4 +- charts/airbyte/Chart.lock | 28 +++++------ charts/airbyte/Chart.yaml | 28 +++++------ charts/airbyte/README.md | 4 +- tools/bin/bump_version.sh | 19 +------ 22 files changed, 108 insertions(+), 86 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7bc8e02fe00..97d8099cf93 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,47 +1,82 @@ [bumpversion] -current_version = 0.50.24 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? serialize = {major}.{minor}.{patch} -[bumpversion:file:.bumpversion.cfg] - [bumpversion:file:.env] +search = VERSION=dev +replace = VERSION={new_version} [bumpversion:file:airbyte-container-orchestrator/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} [bumpversion:file:airbyte-proxy/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} [bumpversion:file:airbyte-server/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} + +[bumpversion:file:airbyte-connector-builder-server/Dockerfile] +search = ARG VERSION=dev +replace = ARG VERSION={new_version} [bumpversion:file:charts/airbyte-bootloader/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-cron/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-metrics/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-pod-sweeper/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-server/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-temporal/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-webapp/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-worker/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-connector-builder-server/Chart.yaml] - -[bumpversion:file:charts/airbyte/README.md] - -[bumpversion:file:airbyte-connector-builder-server/Dockerfile] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-keycloak/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-keycloak-setup/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} [bumpversion:file:charts/airbyte-api-server/Chart.yaml] +search = appVersion: dev +replace = appVersion: {new_version} + +[bumpversion:file:charts/airbyte/README.md] +search = ![appVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +replace = ![appVersion: {new_version}](https://img.shields.io/badge/AppVersion-{new_version}-informational?style=flat-square) diff --git a/.env b/.env index de28947e9be..980c57ad3b9 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.50.24 +VERSION=dev # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index 79e04ff511d..680bc610d75 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -10,7 +10,7 @@ ENV PIP=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/pip COPY requirements.txt requirements.txt RUN ${PIP} install -r requirements.txt -ARG VERSION=0.50.24 +ARG VERSION=dev ENV APPLICATION airbyte-connector-builder-server ENV VERSION ${VERSION} @@ -23,5 +23,5 @@ ADD airbyte-app.tar /app # wait for upstream dependencies to become available before starting server ENTRYPOINT ["/bin/bash", "-c", "airbyte-app/bin/${APPLICATION}"] -LABEL io.airbyte.version=0.50.24 -LABEL io.airbyte.name=airbyte/connector-builder-server \ No newline at end of file +LABEL io.airbyte.version=${VERSION} +LABEL io.airbyte.name=airbyte/connector-builder-server diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 633d9fae785..fdfe338daa6 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -10,7 +10,7 @@ RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/s && chmod +x kubectl && mv kubectl /usr/local/bin/ # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.50.24 +ARG VERSION=dev ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-proxy/Dockerfile b/airbyte-proxy/Dockerfile index 4542e765f64..77d7a247215 100644 --- a/airbyte-proxy/Dockerfile +++ b/airbyte-proxy/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:latest -ARG VERSION=0.50.24 +ARG VERSION=dev ENV APPLICATION airbyte-proxy ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index 4bdb76fb63e..1661a8a71c2 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -3,7 +3,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 5005 -ARG VERSION=0.50.24 +ARG VERSION=dev ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 979b7c9784d..4766c754b6e 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 428819a1a2c..83ff38bed1d 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 67ffe9ef26f..a1e921e43e5 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index c247333032d..12444fd33f6 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index b1891f74783..0c362e47de4 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,18 +15,18 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common repository: https://charts.bitnami.com/bitnami tags: - bitnami-common - version: 1.x.x \ No newline at end of file + version: 1.x.x diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index e8abb6f1cbf..edb543262f1 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,18 +15,18 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common repository: https://charts.bitnami.com/bitnami tags: - bitnami-common - version: 1.x.x \ No newline at end of file + version: 1.x.x diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 08566d418df..17abf288666 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index f26dd1d5589..96e94e9d77a 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index ded18696c7e..f1c8469b8c3 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index a4dc8fe4580..226a62164ca 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 506ce05e54b..c5a7f33b131 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 5936e974af9..69f4efc8955 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 869ba502171..9dbda671fb3 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 -digest: sha256:d7a814247ae1fc194fbdc2807ae114f06ae80ea68a626256fa25c2fdd5a9bfe7 -generated: "2023-09-01T19:19:52.389785043Z" + version: 0.48.8 +digest: sha256:6e736e04dd6da520859db8e5b450971cea7d80fc0d629c293cb38fb063447cc0 +generated: "2023-09-01T22:33:22.735182715Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 58e0c936cda..efa4ecae087 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.6 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.50.24 +appVersion: dev dependencies: - name: common @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 + version: 0.48.8 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.6 \ No newline at end of file + version: 0.48.8 diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index e87bc3d30de..f9df04ca652 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,8 @@ # airbyte -![Version: 0.50.24](https://img.shields.io/badge/Version-0.50.24-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.50.24](https://img.shields.io/badge/AppVersion-0.50.24-informational?style=flat-square) +![Version: 0.50.24](https://img.shields.io/badge/Version-0.50.24-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![appVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte diff --git a/tools/bin/bump_version.sh b/tools/bin/bump_version.sh index 56aeb95271d..511e9997f9e 100755 --- a/tools/bin/bump_version.sh +++ b/tools/bin/bump_version.sh @@ -1,33 +1,19 @@ #!/usr/bin/env bash set -eu -# If running in a tty (TeleTYpe AKA interactive) shell -# complain because this is supposed to be run in the sky without interactive shell -if ! test "$(tty)" == "not a tty"; then - echo "Ahoy There! This Script is meant to run as a GH action" - exit 1 -fi - set -o xtrace REPO=$(git ls-remote --get-url | xargs basename -s .git) echo $REPO -if [ "$REPO" == "airbyte" ]; then - PREV_VERSION=$(grep -w 'VERSION=[0-9]\+\(\.[0-9]\+\)\+' run-ab-platform.sh | cut -d"=" -f2) - echo "Bumping version for Airbyte" -else - PREV_VERSION=$(grep -w VERSION .env | cut -d"=" -f2) - echo "Bumping version for Airbyte Platform" -fi GIT_REVISION=$(git rev-parse HEAD) pip install bumpversion if [ -z "${OVERRIDE_VERSION:-}" ]; then # No override, so bump the version normally - bumpversion "$PART_TO_BUMP" + bumpversion --current-version "$PREV_VERSION" "$PART_TO_BUMP" else # We have an override version, so use it directly - bumpversion --current-version $PREV_VERSION --new-version $OVERRIDE_VERSION "$PART_TO_BUMP" + bumpversion --current-version "$PREV_VERSION" --new-version $OVERRIDE_VERSION "$PART_TO_BUMP" fi if [ "$REPO" == "airbyte" ]; then @@ -44,4 +30,3 @@ set +o xtrace echo "Bumped version from ${PREV_VERSION} to ${NEW_VERSION}" echo "PREV_VERSION=${PREV_VERSION}" >> $GITHUB_OUTPUT echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_OUTPUT -echo "GIT_REVISION=${GIT_REVISION}" >> $GITHUB_OUTPUT From 4c256703613cea0618bc0d0490939020f9fe24b5 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Fri, 1 Sep 2023 23:25:11 +0000 Subject: [PATCH 101/201] Bump helm chart version reference to 0.48.7 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 4766c754b6e..867f5bba008 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 83ff38bed1d..e5d0c956568 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index a1e921e43e5..684a97d0cf2 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 12444fd33f6..f4d79c4c1f8 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 0c362e47de4..f4bebddd2be 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index edb543262f1..e986f8eea73 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 17abf288666..ca609dbcd15 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 96e94e9d77a..fd72077b37a 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index f1c8469b8c3..b67ca2ef062 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 226a62164ca..1baf0b246f5 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index c5a7f33b131..2b3ce425792 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 69f4efc8955..a56704490dd 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 9dbda671fb3..bbffbb69b74 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 -digest: sha256:6e736e04dd6da520859db8e5b450971cea7d80fc0d629c293cb38fb063447cc0 -generated: "2023-09-01T22:33:22.735182715Z" + version: 0.48.7 +digest: sha256:fbefd6a15f7c63aa7f5d2c8cb6be9e1896ed35958398ec365b2ce79d660158ef +generated: "2023-09-01T23:25:00.955650887Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index efa4ecae087..8bee9aafb24 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.7 From 87d62ebcaf778ceac4dac07a0de798917710691c Mon Sep 17 00:00:00 2001 From: Jack Keller Date: Sat, 2 Sep 2023 01:33:41 -0400 Subject: [PATCH 102/201] Update local ci env var to fix local ci resources (#8656) --- airbyte-webapp/jest.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/jest.config.ts b/airbyte-webapp/jest.config.ts index 4063c727a08..9a07bbb9406 100644 --- a/airbyte-webapp/jest.config.ts +++ b/airbyte-webapp/jest.config.ts @@ -1,6 +1,6 @@ import type { Config } from "jest"; -const isInCI = process.env.CI; +const isInCI = process.env.DAGGER; const jestConfig: Config = { verbose: true, From 62ea09950b8516a4fb70d360741c89d3fe116c1d Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Sat, 2 Sep 2023 17:54:25 -0400 Subject: [PATCH 103/201] =?UTF-8?q?Revert=20"=F0=9F=AA=9F=20=F0=9F=8E=89?= =?UTF-8?q?=20(flagged)=20Restrict=20Permission=20Update=20UI=20to=20users?= =?UTF-8?q?=20with=20adequate=20permissions"=20(#8676)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/core/utils/rbac/intent.ts | 4 ---- airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx | 7 ++----- .../components/RoleManagementControl.tsx | 8 +------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/airbyte-webapp/src/core/utils/rbac/intent.ts b/airbyte-webapp/src/core/utils/rbac/intent.ts index 550aa0acb45..b75547acf38 100644 --- a/airbyte-webapp/src/core/utils/rbac/intent.ts +++ b/airbyte-webapp/src/core/utils/rbac/intent.ts @@ -6,18 +6,14 @@ export enum Intent { // organization "ListOrganizationMembers" = "ListOrganizationMembers", - "UpdateOrganizationPermissions" = "UpdateOrganizationPermissions", // workspace "ListWorkspaceMembers" = "ListWorkspaceMembers", - "UpdateWorkspacePermissions" = "UpdateWorkspacePermissions", } const intentToRbacQuery = { [Intent.ListOrganizationMembers]: { resourceType: "ORGANIZATION", role: "READER" }, - [Intent.UpdateOrganizationPermissions]: { resourceType: "ORGANIZATION", role: "ADMIN" }, - [Intent.UpdateWorkspacePermissions]: { resourceType: "WORKSPACE", role: "ADMIN" }, [Intent.ListWorkspaceMembers]: { resourceType: "WORKSPACE", role: "READER" }, } as const; diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx index a93de0b4e88..98b9bae8666 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx @@ -11,7 +11,6 @@ import { PageHeader } from "components/ui/PageHeader"; import { SideMenu, CategoryItem, SideMenuItem } from "components/ui/SideMenu"; import { useCurrentWorkspace } from "core/api"; -import { Intent, useIntent } from "core/utils/rbac/intent"; import { useExperiment } from "hooks/services/Experiment"; import { useGetConnectorsOutOfDate } from "hooks/services/useConnector"; @@ -53,8 +52,6 @@ const SettingsPage: React.FC = ({ pageConfig }) => { const { countNewSourceVersion, countNewDestinationVersion } = useGetConnectorsOutOfDate(); const newWorkspacesUI = useExperiment("workspaces.newWorkspacesUI", false); const isAccessManagementEnabled = useExperiment("settings.accessManagement", false); - const canListWorkspaceUsers = useIntent(Intent.ListWorkspaceMembers); - const canListOrganizationUsers = useIntent(Intent.ListOrganizationMembers); const menuItems: CategoryItem[] = pageConfig?.menuConfig || [ { @@ -106,7 +103,7 @@ const SettingsPage: React.FC = ({ pageConfig }) => { name: , component: MetricsPage, }, - ...(isAccessManagementEnabled && !pageConfig && canListWorkspaceUsers + ...(isAccessManagementEnabled && !pageConfig ? [ { path: `${SettingsRoute.Workspace}/${SettingsRoute.AccessManagement}`, @@ -127,7 +124,7 @@ const SettingsPage: React.FC = ({ pageConfig }) => { name: , component: GeneralOrganizationSettingsPage, }, - ...(isAccessManagementEnabled && !pageConfig && canListOrganizationUsers + ...(isAccessManagementEnabled && !pageConfig ? [ { path: `${SettingsRoute.Organization}/${SettingsRoute.AccessManagement}`, diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx index fc77a949876..acbfc4435b8 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx @@ -10,7 +10,6 @@ import { Text } from "components/ui/Text"; import { useDeletePermissions, useUpdatePermissions } from "core/api"; import { PermissionType, PermissionUpdate } from "core/request/AirbyteClient"; -import { Intent, useIntent } from "core/utils/rbac/intent"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import styles from "./RoleManagementControl.module.scss"; @@ -52,16 +51,11 @@ export const RoleManagementControl: React.FC = ({ const { formatMessage } = useIntl(); const isEditMode = activeEditRow === permissionId; - const intentKey = - pageResourceType === "organization" ? Intent.UpdateOrganizationPermissions : Intent.UpdateWorkspacePermissions; - - const canUpdateUserPermissions = useIntent(intentKey); - if (!permissionType) { return null; } - if (pageResourceType !== tableResourceType || !canUpdateUserPermissions) { + if (pageResourceType !== tableResourceType) { return ( From 82b8d1fc8dca6d60afede2d6bb2cf6c25339efd7 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 5 Sep 2023 13:03:24 +0200 Subject: [PATCH 104/201] Add milvus logo (#8655) --- .../init/src/main/resources/icons/milvus.svg | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/milvus.svg diff --git a/airbyte-config/init/src/main/resources/icons/milvus.svg b/airbyte-config/init/src/main/resources/icons/milvus.svg new file mode 100644 index 00000000000..b4e13796df2 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/milvus.svg @@ -0,0 +1,60 @@ + + + + + + + milvus-icon-color + + + + + + milvus-icon-color + + + + From b872bc4729d3ff3dc8f177b997ce7a17a755825a Mon Sep 17 00:00:00 2001 From: Vladimir Date: Tue, 5 Sep 2023 16:38:32 +0300 Subject: [PATCH 105/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20[Form=20M?= =?UTF-8?q?igration][CreateConnectionForm]=20add=20``,=20``=20and=20``=20fields=20(#8570)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Chandler Prall --- .../DataGeographyDropdown.tsx | 8 + .../HookFormFieldLayout.module.scss | 7 + .../ConnectionForm/HookFormFieldLayout.tsx | 14 ++ .../NormalizationHookFormField.tsx | 12 +- .../OperationsSectionHookForm.tsx | 11 +- .../ConnectionForm/hookFormConfig.tsx | 211 ++++++++++++++++++ .../CreateConnectionNameField.tsx | 5 + .../CreateConnectionForm/DataResidency.tsx | 4 + .../DataResidencyHookFormCard.tsx | 44 ++++ .../CreateConnectionForm/SchemaError.tsx | 14 +- .../ConnectionNameHookFormCard.tsx | 27 +++ .../CreateConnectionHookForm.module.scss | 6 + .../CreateConnectionHookForm.tsx | 161 +++++++++++++ .../forms/DataResidencyDropdown.tsx | 13 +- .../ConnectionHookFormService.tsx | 169 ++++++++++++++ .../hooks/services/Experiment/experiments.ts | 1 + .../ConfigureConnectionPage.tsx | 5 +- 17 files changed, 695 insertions(+), 17 deletions(-) create mode 100644 airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.module.scss create mode 100644 airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.tsx create mode 100644 airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx create mode 100644 airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidencyHookFormCard.tsx create mode 100644 airbyte-webapp/src/components/connection/CreateConnectionHookForm/ConnectionNameHookFormCard.tsx create mode 100644 airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.module.scss create mode 100644 airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx create mode 100644 airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionHookFormService.tsx diff --git a/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.tsx b/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.tsx index b32235b193f..e7e882e1608 100644 --- a/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.tsx +++ b/airbyte-webapp/src/components/common/DataGeographyDropdown/DataGeographyDropdown.tsx @@ -14,6 +14,14 @@ interface DataGeographyDropdownProps { value: Geography; } +/** + * @deprecated it's not form related component and will be removed in the future, use DataResidencyDropdown instead + * @param geographies + * @param isDisabled + * @param onChange + * @param value + * @constructor + */ export const DataGeographyDropdown: React.FC = ({ geographies, isDisabled = false, diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.module.scss b/airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.module.scss new file mode 100644 index 00000000000..55313bfea71 --- /dev/null +++ b/airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.module.scss @@ -0,0 +1,7 @@ +.container { + max-width: 1000px; + + & > :last-child { + padding-bottom: 0; // overwrite default FormControl style + } +} diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.tsx new file mode 100644 index 00000000000..40bf67f5c3f --- /dev/null +++ b/airbyte-webapp/src/components/connection/ConnectionForm/HookFormFieldLayout.tsx @@ -0,0 +1,14 @@ +import React, { PropsWithChildren } from "react"; + +import styles from "./HookFormFieldLayout.module.scss"; + +/** + * react-hook-form form control layout component + * will replace FormFieldLayout in future + * @see FormFieldLayout + * @param children + * @constructor + */ +export const HookFormFieldLayout: React.FC> = ({ children }) => { + return
{children}
; +}; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationHookFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationHookFormField.tsx index 618efbe2d7d..5b73fc37f16 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationHookFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/NormalizationHookFormField.tsx @@ -7,6 +7,8 @@ import { ExternalLink } from "components/ui/Link"; import { NormalizationType } from "area/connection/types"; import { links } from "core/utils/links"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useExperiment } from "hooks/services/Experiment"; import { LabeledRadioButtonFormControl } from "./LabeledRadioButtonFormControl"; @@ -19,7 +21,15 @@ import { LabeledRadioButtonFormControl } from "./LabeledRadioButtonFormControl"; */ export const NormalizationHookFormField: React.FC = () => { const { formatMessage } = useIntl(); - const { mode } = useConnectionFormService(); + /** + *TODO: remove after successful CreateConnectionForm migration + *https://github.com/airbytehq/airbyte-platform-internal/issues/8639 + */ + const doUseCreateConnectionHookForm = useExperiment("form.createConnectionHookForm", false); + const useConnectionFormContextProvider = doUseCreateConnectionHookForm + ? useConnectionHookFormService + : useConnectionFormService; + const { mode } = useConnectionFormContextProvider(); return ( diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx index 6d7048c3213..ad79c458b88 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/OperationsSectionHookForm.tsx @@ -6,23 +6,16 @@ import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { FeatureItem, useFeature } from "core/services/features"; -import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; import { NormalizationHookFormField } from "./NormalizationHookFormField"; import { TransformationFieldHookForm } from "./TransformationFieldHookForm"; -// not sure if we need to pass these callbacks, since we don't control modal via parent prop state -// but leaving them here just as note -// interface OperationsSectionHookFormProps { -// onStartEditTransformation?: () => void; -// onEndEditTransformation?: () => void; -// } - export const OperationsSectionHookForm: React.FC = () => { const { formatMessage } = useIntl(); const { destDefinitionVersion: { normalizationConfig, supportsDbt }, - } = useConnectionFormService(); + } = useConnectionHookFormService(); const supportsNormalization = normalizationConfig.supported; const supportsTransformations = useFeature(FeatureItem.AllowCustomDBT) && supportsDbt; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx new file mode 100644 index 00000000000..3338f2b48d5 --- /dev/null +++ b/airbyte-webapp/src/components/connection/ConnectionForm/hookFormConfig.tsx @@ -0,0 +1,211 @@ +import { useMemo } from "react"; +import * as yup from "yup"; + +import { validateCronExpression, validateCronFrequencyOneHourOrMore } from "area/connection/utils"; +import { + ConnectionScheduleData, + ConnectionScheduleType, + Geography, + NamespaceDefinitionType, + NonBreakingChangesPreference, +} from "core/request/AirbyteClient"; +import { FeatureItem, useFeature } from "core/services/features"; +import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; + +export type ConnectionHookFormMode = "create" | "edit" | "readonly"; + +/** + * react-hook-form form values type for the connection form. + * copied from + * @see FormikConnectionFormValues + */ +export interface HookFormConnectionFormValues { + name?: string; + scheduleType?: ConnectionScheduleType | null; + scheduleData?: ConnectionScheduleData | null; + nonBreakingChangesPreference?: NonBreakingChangesPreference | null; + prefix: string; + namespaceDefinition?: NamespaceDefinitionType; + namespaceFormat?: string; + geography?: Geography; + // syncCatalog: SyncSchema; + // syncCatalog: { + // streams: Array<{ + // id?: string; + // // how to override the type? + // stream?: Omit & { name?: string }; + // config?: AirbyteStreamConfiguration; + // }>; + // }; + // surprisingly but seems like we didn't handle this fields in the schema form, but why? + // normalization?: NormalizationType; + // transformations?: OperationRead[]; +} + +/** + * yup schema for the schedule data + * @param allowSubOneHourCronExpressions + */ +const getScheduleDataSchema = (allowSubOneHourCronExpressions: boolean) => + yup.mixed().when("scheduleType", (scheduleType) => { + if (scheduleType === ConnectionScheduleType.basic) { + return yup.object({ + basicSchedule: yup + .object({ + units: yup.number().required("form.empty.error"), + timeUnit: yup.string().required("form.empty.error"), + }) + .defined("form.empty.error"), + }); + } else if (scheduleType === ConnectionScheduleType.manual) { + return yup.mixed().notRequired(); + } + return yup.object({ + cron: yup + .object({ + cronExpression: yup + .string() + .trim() + .required("form.empty.error") + .test("validCron", (value, { createError, path }) => { + const validation = validateCronExpression(value); + return ( + validation.isValid || + createError({ + path, + message: validation.message ?? "form.cronExpression.invalid", + }) + ); + }) + .test( + "validCronFrequency", + "form.cronExpression.underOneHourNotAllowed", + (expression) => allowSubOneHourCronExpressions || validateCronFrequencyOneHourOrMore(expression) + ), + cronTimeZone: yup.string().required("form.empty.error"), + }) + .defined("form.empty.error"), + }); + }); + +// const streamAndConfigurationSchema = yup.object({ +// id: yup +// .string() +// // This is required to get rid of id fields we are using to detect stream for edition +// .when("$isRequest", (isRequest: boolean, schema: yup.StringSchema) => (isRequest ? schema.strip(true) : schema)), +// stream: yup +// .object({ +// name: yup.string().required("form.empty.error"), +// }) +// .optional(), +// config: yup +// .object({ +// selected: yup.boolean(), +// syncMode: yup.string(), +// destinationSyncMode: yup.string(), +// primaryKey: yup.array().of(yup.array().of(yup.string())), +// cursorField: yup.array().of(yup.string()), +// }) +// .test({ +// message: "form.empty.error", +// eslint-disable-next-line jest/no-commented-out-tests +// test(value) { +// if (!value.selected) { +// return true; +// } +// +// const errors: yup.ValidationError[] = []; +// const pathRoot = "syncCatalog"; +// +// // it's possible that primaryKey array is always present +// // however yup couldn't determine type correctly even with .required() call +// if (DestinationSyncMode.append_dedup === value.destinationSyncMode && value.primaryKey?.length === 0) { +// errors.push( +// this.createError({ +// message: "connectionForm.primaryKey.required", +// path: `${pathRoot}.streams[${this.parent.id}].config.primaryKey`, +// }) +// ); +// } +// +// // it's possible that cursorField array is always present +// // however yup couldn't determine type correctly even with .required() call +// if ( +// SyncMode.incremental === value.syncMode && +// !this.parent.stream.sourceDefinedCursor && +// value.cursorField?.length === 0 +// ) { +// errors.push( +// this.createError({ +// message: "connectionForm.cursorField.required", +// path: `${pathRoot}.streams[${this.parent.id}].config.cursorField`, +// }) +// ); +// } +// +// return errors.length > 0 ? new yup.ValidationError(errors) : true; +// }, +// }), +// }); + +/** + * yup schema for the sync catalog + */ +// const syncCatalogSchema = yup.object({ +// streams: yup +// .array() +// .of(streamAndConfigurationSchema) +// .test( +// "syncCatalog.streams.required", +// "connectionForm.streams.required", +// (streams) => streams?.some(({ config }) => !!config.selected) ?? false +// ), +// }); + +/** + * generate yup schema for the create connection form + * @param mode + * @param allowSubOneHourCronExpressions + * @param allowAutoDetectSchema + */ +const createConnectionValidationSchema = ( + mode: ConnectionHookFormMode, + allowSubOneHourCronExpressions: boolean, + allowAutoDetectSchema: boolean +) => + yup + .object({ + // The connection name during Editing is handled separately from the form + name: mode === "create" ? yup.string().required("form.empty.error") : yup.string().notRequired(), + geography: yup.mixed().oneOf(Object.values(Geography)), + scheduleType: yup.mixed().oneOf(Object.values(ConnectionScheduleType)), + scheduleData: getScheduleDataSchema(allowSubOneHourCronExpressions), + nonBreakingChangesPreference: allowAutoDetectSchema + ? yup.mixed().oneOf(Object.values(NonBreakingChangesPreference)).required("form.empty.error") + : yup.mixed().notRequired(), + namespaceDefinition: yup + .mixed() + .oneOf(Object.values(NamespaceDefinitionType)) + .required("form.empty.error"), + namespaceFormat: yup.string().when("namespaceDefinition", { + is: NamespaceDefinitionType.customformat, + then: yup.string().trim().required("form.empty.error"), + }), + prefix: yup.string().default(""), + // syncCatalog: syncCatalogSchema, + }) + .noUnknown(); + +/** + * useConnectionValidationSchema with additional arguments + */ +export const useConnectionHookFormValidationSchema = () => { + const allowSubOneHourCronExpressions = useFeature(FeatureItem.AllowSyncSubOneHourCronExpressions); + const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema); + const { mode } = useConnectionHookFormService(); + + return useMemo( + () => createConnectionValidationSchema(mode, allowSubOneHourCronExpressions, allowAutoDetectSchema), + [allowAutoDetectSchema, allowSubOneHourCronExpressions, mode] + ); +}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.tsx index be8c52ec4fe..515b6a08b61 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionNameField.tsx @@ -7,6 +7,11 @@ import { Input } from "components/ui/Input"; import { FormFieldLayout } from "../ConnectionForm/FormFieldLayout"; +/** + * @deprecated it's formik version of CreateConnectionNameField form control and will be removed in the future, use ConnectionNameHookFormCard instead + * @see ConnectionNameHookFormCard + * @constructor + */ export const CreateConnectionNameField = () => { const { formatMessage } = useIntl(); diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidency.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidency.tsx index a42e0b6061f..e9a63c76b40 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidency.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidency.tsx @@ -16,6 +16,10 @@ interface DataResidencyProps { name?: string; } +/** + * @deprecated it's formik version of DataResidency form control and will be removed in the future, use DataResidencyHookForm instead + * @see DataResidencyHookFormCard + */ export const DataResidency: React.FC = ({ name = "geography" }) => { const { formatMessage } = useIntl(); const { setFieldValue } = useFormikContext(); diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidencyHookFormCard.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidencyHookFormCard.tsx new file mode 100644 index 00000000000..c2fdd782b93 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/DataResidencyHookFormCard.tsx @@ -0,0 +1,44 @@ +import React from "react"; +import { FormattedMessage, useIntl } from "react-intl"; + +import { DataResidencyDropdown } from "components/forms/DataResidencyDropdown"; +import { CollapsibleCard } from "components/ui/CollapsibleCard"; +import { ExternalLink } from "components/ui/Link"; + +import { links } from "core/utils/links"; + +import { HookFormConnectionFormValues } from "../ConnectionForm/hookFormConfig"; +import { HookFormFieldLayout } from "../ConnectionForm/HookFormFieldLayout"; + +/** + * react-hook-form version of DataResidency + * @constructor + */ +export const DataResidencyHookFormCard = () => { + const { formatMessage } = useIntl(); + + return ( + + + + name="geography" + inline + labelId={formatMessage({ id: "connection.geographyTitle" })} + labelTooltip={ + ( + {node} + ), + docLink: (node: React.ReactNode) => ( + {node} + ), + }} + /> + } + /> + + + ); +}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx index 0bc5eac12d7..8e720927556 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx @@ -9,13 +9,25 @@ import { FlexContainer } from "components/ui/Flex"; import { LogsRequestError } from "core/request/LogsRequestError"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; +import { useConnectionHookFormService } from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useExperiment } from "hooks/services/Experiment"; import { SchemaError as SchemaErrorType } from "hooks/services/useSourceHook"; import styles from "./SchemaError.module.scss"; export const SchemaError = ({ schemaError }: { schemaError: Exclude }) => { const job = LogsRequestError.extractJobInfo(schemaError); - const { refreshSchema } = useConnectionFormService(); + + /** + *TODO: remove after successful CreateConnectionForm migration + *https://github.com/airbytehq/airbyte-platform-internal/issues/8639 + */ + const doUseCreateConnectionHookForm = useExperiment("form.createConnectionHookForm", false); + const useConnectionFormContextProvider = doUseCreateConnectionHookForm + ? useConnectionHookFormService + : useConnectionFormService; + const { refreshSchema } = useConnectionFormContextProvider(); + return ( diff --git a/airbyte-webapp/src/components/connection/CreateConnectionHookForm/ConnectionNameHookFormCard.tsx b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/ConnectionNameHookFormCard.tsx new file mode 100644 index 00000000000..bb30e0e90ee --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/ConnectionNameHookFormCard.tsx @@ -0,0 +1,27 @@ +import React from "react"; +import { FormattedMessage, useIntl } from "react-intl"; + +import { FormControl } from "components/forms"; +import { CollapsibleCard } from "components/ui/CollapsibleCard"; + +import { HookFormFieldLayout } from "../ConnectionForm/HookFormFieldLayout"; + +export const ConnectionNameHookFormCard: React.FC = () => { + const { formatMessage } = useIntl(); + + return ( + } collapsible={false}> + + + + + ); +}; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.module.scss b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.module.scss new file mode 100644 index 00000000000..a1204c4a4d7 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.module.scss @@ -0,0 +1,6 @@ +@use "scss/variables"; + +.formContainer { + width: 100%; + min-width: variables.$min-width-wide-table-container; +} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx new file mode 100644 index 00000000000..56053ef5a12 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionHookForm/CreateConnectionHookForm.tsx @@ -0,0 +1,161 @@ +import React, { Suspense, useCallback } from "react"; + +import { Form } from "components/forms"; +import LoadingSchema from "components/LoadingSchema"; +import { FlexContainer } from "components/ui/Flex"; + +import { FeatureItem, useFeature } from "core/services/features"; +import { useGetDestinationFromSearchParams } from "hooks/domain/connector/useGetDestinationFromParams"; +import { useGetSourceFromSearchParams } from "hooks/domain/connector/useGetSourceFromParams"; +import { + ConnectionHookFormServiceProvider, + useConnectionHookFormService, +} from "hooks/services/ConnectionForm/ConnectionHookFormService"; +import { useExperimentContext } from "hooks/services/Experiment"; +import { SchemaError as SchemaErrorType, useDiscoverSchema } from "hooks/services/useSourceHook"; + +import { ConnectionNameHookFormCard } from "./ConnectionNameHookFormCard"; +import styles from "./CreateConnectionHookForm.module.scss"; +import { HookFormConnectionFormValues, useConnectionHookFormValidationSchema } from "../ConnectionForm/hookFormConfig"; +import { OperationsSectionHookForm } from "../ConnectionForm/OperationsSectionHookForm"; +import { DataResidencyHookFormCard } from "../CreateConnectionForm/DataResidencyHookFormCard"; +import { SchemaError } from "../CreateConnectionForm/SchemaError"; + +interface CreateConnectionPropsInner { + schemaError: SchemaErrorType; +} + +const CreateConnectionFormInner: React.FC = ({ schemaError }) => { + // const navigate = useNavigate(); + const canEditDataGeographies = useFeature(FeatureItem.AllowChangeDataGeographies); + // const { mutateAsync: createConnection } = useCreateConnection(); + // const { clearFormChange } = useFormChangeTrackerService(); + + // const workspaceId = useCurrentWorkspaceId(); + + const { + connection, + initialValues, + // mode, + // formId + // , getErrorMessage, + // setSubmitError, + } = useConnectionHookFormService(); + + const validationSchema = useConnectionHookFormValidationSchema(); + useExperimentContext("source-definition", connection.source?.sourceDefinitionId); + + const onSubmit = useCallback(async (formValues: HookFormConnectionFormValues) => { + /** + *there is some magic , need to try get rid of tidyConnectionHookFormValues, or at least split it + */ + // const values = tidyConnectionHookFormValues(formValues, workspaceId, validationSchema); + + console.log(formValues); + // try { + // const createdConnection = await createConnection({ + // formValues, + // source: connection.source, + // destination: connection.destination, + // sourceDefinition: { + // sourceDefinitionId: connection.source?.sourceDefinitionId ?? "", + // }, + // destinationDefinition: { + // name: connection.destination?.name ?? "", + // destinationDefinitionId: connection.destination?.destinationDefinitionId ?? "", + // }, + // sourceCatalogId: connection.catalogId, + // }); + + // formikHelpers.resetForm(); + // We need to clear the form changes otherwise the dirty form intercept service will prevent navigation + // clearFormChange(formId); + + /** + * can't move to onSuccess since we get connectionId in the createdConnection response + */ + // navigate(`../../connections/${createdConnection.connectionId}`); + // } catch (e) { + // // setSubmitError(e); + // console.log(e); + // } + }, []); + + if (schemaError) { + return ; + } + + return ( + }> + defaultValues={initialValues} schema={validationSchema} onSubmit={onSubmit}> + + + {canEditDataGeographies && } + {/* */} + + {/* */} + + + + ); + + // return ( + // }> + //
+ // + // {({ isSubmitting, isValid, dirty, errors, validateForm }) => ( + //
+ // + // {canEditDataGeographies && } + // + // setEditingTransformation(true)} + // onEndEditTransformation={() => setEditingTransformation(false)} + // /> + // + // + // )} + // + //
+ // + // ); +}; + +/** + * react-hook-form version of the CreateConnectionForm + */ +export const CreateConnectionHookForm: React.FC = () => { + const source = useGetSourceFromSearchParams(); + const destination = useGetDestinationFromSearchParams(); + + const { schema, isLoading, schemaErrorStatus, catalogId, onDiscoverSchema } = useDiscoverSchema( + source.sourceId, + true + ); + + const partialConnection = { + syncCatalog: schema, + destination, + source, + catalogId, + }; + + return ( + + {isLoading ? : } + + ); +}; diff --git a/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx b/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx index bf6e2345217..f2e1abda89d 100644 --- a/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx +++ b/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx @@ -1,5 +1,5 @@ import * as Flags from "country-flag-icons/react/3x2"; -import React from "react"; +import React, { ReactNode } from "react"; import { Path } from "react-hook-form"; import { useIntl } from "react-intl"; @@ -13,7 +13,8 @@ import { FormControl } from "./FormControl"; interface DataResidencyFormControlProps { labelId: string; name: Path; - description: React.ReactNode; + description?: string | ReactNode; + labelTooltip?: React.ReactNode; inline?: boolean; disabled?: boolean; } @@ -22,6 +23,7 @@ export const DataResidencyDropdown = ({ labelId, name, description, + labelTooltip, inline, disabled = false, }: DataResidencyFormControlProps): JSX.Element => { @@ -43,12 +45,13 @@ export const DataResidencyDropdown = ({ return ( - label={formatMessage({ id: labelId })} - description={description} - fieldType="dropdown" name={name} + fieldType="dropdown" options={options} inline={inline} + label={formatMessage({ id: labelId })} + description={description} + labelTooltip={labelTooltip} disabled={disabled} /> ); diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionHookFormService.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionHookFormService.tsx new file mode 100644 index 00000000000..a954b1a44c2 --- /dev/null +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionHookFormService.tsx @@ -0,0 +1,169 @@ +/* eslint-disable check-file/filename-blocklist */ +// temporary disable eslint rule for this file during form migration +import React, { createContext, useContext } from "react"; + +import { FormikConnectionFormValues, useInitialValues } from "components/connection/ConnectionForm/formConfig"; + +import { useDestinationDefinitionVersion } from "core/api"; +import { + ActorDefinitionVersionRead, + DestinationDefinitionRead, + DestinationDefinitionSpecificationRead, + SourceDefinitionRead, + SourceDefinitionSpecificationRead, + WebBackendConnectionRead, +} from "core/request/AirbyteClient"; +import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; +import { useGetDestinationDefinitionSpecification } from "services/connector/DestinationDefinitionSpecificationService"; +import { useSourceDefinition } from "services/connector/SourceDefinitionService"; +import { useGetSourceDefinitionSpecification } from "services/connector/SourceDefinitionSpecificationService"; + +import { SchemaError } from "../useSourceHook"; + +export type ConnectionFormMode = "create" | "edit" | "readonly"; + +export type ConnectionOrPartialConnection = + | WebBackendConnectionRead + | (Partial & Pick); + +interface ConnectionServiceProps { + connection: ConnectionOrPartialConnection; + mode: ConnectionFormMode; + schemaError?: SchemaError | null; + refreshSchema: () => Promise; +} + +// not sure this one should be here, consider moving it to the another file or utils file +// export const tidyConnectionHookFormValues = ( +// values: HookFormConnectionFormValues, +// workspaceId: string, +// validationSchema: ConnectionValidationSchema, +// operations?: OperationRead[] +// ): ConnectionValues => { +// // TODO (https://github.com/airbytehq/airbyte/issues/17279): We should try to fix the types so we don't need the casting. +// const formValues: ConnectionFormValues = validationSchema.cast(values, { +// context: { isRequest: true }, +// }) as unknown as ConnectionFormValues; +// +// formValues.operations = mapFormPropsToOperation(values, operations, workspaceId); +// +// if (formValues.scheduleType === ConnectionScheduleType.manual) { +// // Have to set this to undefined to override the existing scheduleData +// formValues.scheduleData = undefined; +// } +// return formValues; +// }; + +interface ConnectionHookFormHook { + connection: ConnectionOrPartialConnection; + mode: ConnectionFormMode; + sourceDefinition: SourceDefinitionRead; + sourceDefinitionSpecification: SourceDefinitionSpecificationRead; + destDefinition: DestinationDefinitionRead; + destDefinitionVersion: ActorDefinitionVersionRead; + destDefinitionSpecification: DestinationDefinitionSpecificationRead; + initialValues: FormikConnectionFormValues; + schemaError?: SchemaError; + refreshSchema: () => Promise; + // formId: string; + // setSubmitError: (submitError: FormError | null) => void; + // getErrorMessage: ( + // formValid: boolean, + // errors?: FormikErrors + // ) => string | JSX.Element | null; +} + +const useConnectionHookForm = ({ + connection, + mode, + schemaError, + refreshSchema, +}: ConnectionServiceProps): ConnectionHookFormHook => { + const { + source: { sourceDefinitionId }, + destination: { destinationId, destinationDefinitionId }, + } = connection; + + const sourceDefinition = useSourceDefinition(sourceDefinitionId); + const sourceDefinitionSpecification = useGetSourceDefinitionSpecification(sourceDefinitionId, connection.sourceId); + + const destDefinition = useDestinationDefinition(destinationDefinitionId); + const destDefinitionVersion = useDestinationDefinitionVersion(destinationId); + const destDefinitionSpecification = useGetDestinationDefinitionSpecification( + destinationDefinitionId, + connection.destinationId + ); + + const initialValues = useInitialValues( + connection, + destDefinitionVersion, + destDefinitionSpecification, + mode !== "create" + ); + // const { formatMessage } = useIntl(); + // const [submitError, setSubmitError] = useState(null); + + /** + * not sure we need this in react-hook-form, since we have change tracker inside thr base form + */ + + // const formId = useUniqueFormId(); + + /** + * commented out because need to figure out how to manage errors with react-hook-form + */ + // const getErrorMessage = useCallback( + // (formValid, errors) => { + // if (submitError) { + // return generateMessageFromError(submitError); + // } + // + // // There is a case when some fields could be dropped in the database. We need to validate the form without property dirty + // const hasValidationError = !formValid; + // + // if (hasValidationError) { + // const hasNoStreamsSelectedError = errors?.syncCatalog?.streams === "connectionForm.streams.required"; + // return formatMessage({ + // id: hasNoStreamsSelectedError ? "connectionForm.streams.required" : "connectionForm.validation.error", + // }); + // } + // + // return null; + // }, + // [formatMessage, submitError] + // ); + + return { + connection, + mode, + sourceDefinition, + sourceDefinitionSpecification, + destDefinition, + destDefinitionVersion, + destDefinitionSpecification, + initialValues, + schemaError, + refreshSchema, + // formId, + // setSubmitError, + // getErrorMessage, + }; +}; + +const ConnectionHookFormContext = createContext(null); + +export const ConnectionHookFormServiceProvider: React.FC> = ({ + children, + ...props +}) => { + const data = useConnectionHookForm(props); + return {children}; +}; + +export const useConnectionHookFormService = () => { + const context = useContext(ConnectionHookFormContext); + if (context === null) { + throw new Error("useConnectionHookFormService must be used within a ConnectionHookFormProvider"); + } + return context; +}; diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts index 18a4d53fde8..bafabc0ce0a 100644 --- a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts +++ b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts @@ -35,4 +35,5 @@ export interface Experiments { "upcomingFeaturesPage.url": string; "workspaces.newWorkspacesUI": boolean; "settings.accessManagement": boolean; + "form.createConnectionHookForm": boolean; } diff --git a/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx b/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx index 08454d5dc6d..5f9f836a576 100644 --- a/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConfigureConnectionPage/ConfigureConnectionPage.tsx @@ -4,8 +4,10 @@ import { Navigate, useParams, useSearchParams } from "react-router-dom"; import { HeadTitle } from "components/common/HeadTitle"; import { MainPageWithScroll } from "components/common/MainPageWithScroll/MainPageWithScroll"; import { CreateConnectionForm } from "components/connection/CreateConnectionForm"; +import { CreateConnectionHookForm } from "components/connection/CreateConnectionHookForm/CreateConnectionHookForm"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; +import { useExperiment } from "hooks/services/Experiment"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; import { CreateConnectionTitleBlock } from "../CreateConnectionPage/CreateConnectionTitleBlock"; @@ -14,6 +16,7 @@ export const ConfigureConnectionPage = () => { const { formatMessage } = useIntl(); const { workspaceId } = useParams<{ workspaceId: string }>(); const [searchParams] = useSearchParams(); + const doUseCreateConnectionHookForm = useExperiment("form.createConnectionHookForm", false); const sourceId = searchParams.get("sourceId"); const destinationId = searchParams.get("destinationId"); @@ -46,7 +49,7 @@ export const ConfigureConnectionPage = () => { } > - + {doUseCreateConnectionHookForm ? : } ); }; From afca4a0a9861af2f5f35141fad183e65f911832d Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Tue, 5 Sep 2023 09:10:38 -0700 Subject: [PATCH 106/201] Fetch roles in KeycloakValidator (#8540) --- .../support/AuthenticationHeaderResolver.java | 120 ++++++++++ .../AuthenticationHeaderResolverTest.java | 209 ++++++++++++++++++ .../commons/auth/AuthRoleConstants.java | 4 + .../commons/auth/OrganizationAuthRole.java | 71 ++++++ .../persistence/PermissionPersistence.java | 53 +++++ .../airbyte/config/persistence/MockData.java | 31 ++- .../PermissionPersistenceTest.java | 16 ++ .../persistence/UserPersistenceTest.java | 3 +- .../persistence/job/WorkspaceHelper.java | 15 ++ .../server/pro/KeycloakTokenValidator.java | 79 ++++++- .../server/KeycloakTokenValidatorTest.java | 29 ++- 11 files changed, 616 insertions(+), 14 deletions(-) create mode 100644 airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java create mode 100644 airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java create mode 100644 airbyte-commons/src/main/java/io/airbyte/commons/auth/OrganizationAuthRole.java diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java new file mode 100644 index 00000000000..5a83c59672a --- /dev/null +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/support/AuthenticationHeaderResolver.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONFIG_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.persistence.job.WorkspaceHelper; +import io.airbyte.validation.json.JsonValidationException; +import jakarta.inject.Singleton; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; + +/** + * Resolves organization or workspace IDs from HTTP headers. + */ +@Slf4j +@Singleton +public class AuthenticationHeaderResolver { + + private final WorkspaceHelper workspaceHelper; + + public AuthenticationHeaderResolver(final WorkspaceHelper workspaceHelper) { + this.workspaceHelper = workspaceHelper; + } + + /** + * Resolve corresponding organization ID. Currently we support two ways to resolve organization ID: + * 1. If the organization ID is provided in the header, we will use it directly. 2. Otherwise, we + * infer the workspace ID from the header and use the workspace ID to find the organization Id. + */ + public List resolveOrganization(final Map properties) { + log.debug("properties: {}", properties); + + if (properties.containsKey(ORGANIZATION_ID_HEADER)) { + return List.of(UUID.fromString(properties.get(ORGANIZATION_ID_HEADER))); + } + // Else, determine the organization from workspace related fields. + + final List workspaceIds = resolveWorkspace(properties); + if (workspaceIds == null) { + return null; + } + return workspaceIds.stream().map(workspaceId -> workspaceHelper.getOrganizationForWorkspace(workspaceId)).collect(Collectors.toList()); + } + + /** + * Resolves workspaces from header. + */ + @SuppressWarnings("PMD.CyclomaticComplexity") // This is an indication that the workspace ID as a group for auth needs refactoring + public List resolveWorkspace(final Map properties) { + log.debug("properties: {}", properties); + try { + if (properties.containsKey(WORKSPACE_ID_HEADER)) { + final String workspaceId = properties.get(WORKSPACE_ID_HEADER); + return List.of(UUID.fromString(workspaceId)); + } else if (properties.containsKey(CONNECTION_ID_HEADER)) { + final String connectionId = properties.get(CONNECTION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForConnectionId(UUID.fromString(connectionId))); + } else if (properties.containsKey(SOURCE_ID_HEADER) && properties.containsKey(DESTINATION_ID_HEADER)) { + final String destinationId = properties.get(DESTINATION_ID_HEADER); + final String sourceId = properties.get(SOURCE_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForConnection(UUID.fromString(sourceId), UUID.fromString(destinationId))); + } else if (properties.containsKey(DESTINATION_ID_HEADER)) { + final String destinationId = properties.get(DESTINATION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForDestinationId(UUID.fromString(destinationId))); + } else if (properties.containsKey(JOB_ID_HEADER)) { + final String jobId = properties.get(JOB_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForJobId(Long.valueOf(jobId))); + } else if (properties.containsKey(SOURCE_ID_HEADER)) { + final String sourceId = properties.get(SOURCE_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForSourceId(UUID.fromString(sourceId))); + } else if (properties.containsKey(SOURCE_DEFINITION_ID_HEADER)) { + final String sourceDefinitionId = properties.get(SOURCE_DEFINITION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForSourceId(UUID.fromString(sourceDefinitionId))); + } else if (properties.containsKey(OPERATION_ID_HEADER)) { + final String operationId = properties.get(OPERATION_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForOperationId(UUID.fromString(operationId))); + } else if (properties.containsKey(CONFIG_ID_HEADER)) { + final String configId = properties.get(CONFIG_ID_HEADER); + return List.of(workspaceHelper.getWorkspaceForConnectionId(UUID.fromString(configId))); + } else if (properties.containsKey(WORKSPACE_IDS_HEADER)) { + return resolveWorkspaces(properties); + } else { + log.debug("Request does not contain any headers that resolve to a workspace ID."); + return null; + } + } catch (final JsonValidationException | ConfigNotFoundException e) { + log.debug("Unable to resolve workspace ID.", e); + return null; + } + } + + private List resolveWorkspaces(final Map properties) { + final String workspaceIds = properties.get(WORKSPACE_IDS_HEADER); + log.debug("workspaceIds from header: {}", workspaceIds); + if (workspaceIds != null) { + final List deserialized = Jsons.deserialize(workspaceIds, List.class); + return deserialized.stream().map(UUID::fromString).toList(); + } + log.debug("Request does not contain any headers that resolve to a list of workspace IDs."); + return null; + } + +} diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java new file mode 100644 index 00000000000..06c1ed7691f --- /dev/null +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/support/AuthenticationHeaderResolverTest.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.server.support; + +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.CONNECTION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.DESTINATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.JOB_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.OPERATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.ORGANIZATION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_DEFINITION_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.SOURCE_ID_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_IDS_HEADER; +import static io.airbyte.commons.server.support.AuthenticationHttpHeaders.WORKSPACE_ID_HEADER; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.persistence.job.WorkspaceHelper; +import io.airbyte.validation.json.JsonValidationException; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +class AuthenticationHeaderResolverTest { + + @Test + void testResolvingFromWorkspaceId() { + final UUID workspaceId = UUID.randomUUID(); + final Map properties = Map.of(WORKSPACE_ID_HEADER, workspaceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromConnectionId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID connectionId = UUID.randomUUID(); + final Map properties = Map.of(CONNECTION_ID_HEADER, connectionId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForConnectionId(connectionId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromSourceAndDestinationId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID destinationId = UUID.randomUUID(); + final UUID sourceId = UUID.randomUUID(); + final Map properties = Map.of(DESTINATION_ID_HEADER, destinationId.toString(), SOURCE_ID_HEADER, sourceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForConnection(sourceId, destinationId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromDestinationId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID destinationId = UUID.randomUUID(); + final Map properties = Map.of(DESTINATION_ID_HEADER, destinationId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForDestinationId(destinationId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromJobId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final Long jobId = System.currentTimeMillis(); + final Map properties = Map.of(JOB_ID_HEADER, String.valueOf(jobId)); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForJobId(jobId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromSourceId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID sourceId = UUID.randomUUID(); + final Map properties = Map.of(SOURCE_ID_HEADER, sourceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForSourceId(sourceId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromSourceDefinitionId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID sourceDefinitionId = UUID.randomUUID(); + final Map properties = Map.of(SOURCE_DEFINITION_ID_HEADER, sourceDefinitionId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForSourceId(sourceDefinitionId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromOperationId() throws JsonValidationException, ConfigNotFoundException { + final UUID workspaceId = UUID.randomUUID(); + final UUID operationId = UUID.randomUUID(); + final Map properties = Map.of(OPERATION_ID_HEADER, operationId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForOperationId(operationId)).thenReturn(workspaceId); + + final List result = workspaceResolver.resolveWorkspace(properties); + assertEquals(List.of(workspaceId), result); + } + + @Test + void testResolvingFromNoMatchingProperties() { + final Map properties = Map.of(); + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + final List workspaceId = workspaceResolver.resolveWorkspace(properties); + assertNull(workspaceId); + } + + @Test + void testResolvingWithException() throws JsonValidationException, ConfigNotFoundException { + final UUID connectionId = UUID.randomUUID(); + final Map properties = Map.of(CONNECTION_ID_HEADER, connectionId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + when(workspaceHelper.getWorkspaceForConnectionId(connectionId)).thenThrow(new JsonValidationException("test")); + final List workspaceId = workspaceResolver.resolveWorkspace(properties); + assertNull(workspaceId); + } + + @Test + void testResolvingMultiple() throws JsonValidationException, ConfigNotFoundException { + final List workspaceIds = List.of(UUID.randomUUID(), UUID.randomUUID()); + final Map properties = Map.of(WORKSPACE_IDS_HEADER, Jsons.serialize(workspaceIds)); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + final List resolvedWorkspaceIds = workspaceResolver.resolveWorkspace(properties); + assertEquals(workspaceIds, resolvedWorkspaceIds); + } + + @Test + void testResolvingOrganizationDirectlyFromHeader() throws ConfigNotFoundException { + final UUID organizationId = UUID.randomUUID(); + final Map properties = Map.of(ORGANIZATION_ID_HEADER, organizationId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + + final List result = workspaceResolver.resolveOrganization(properties); + assertEquals(List.of(organizationId), result); + } + + @Test + void testResolvingOrganizationFromWorkspaceHeader() throws ConfigNotFoundException { + final UUID organizationId = UUID.randomUUID(); + final UUID workspaceId = UUID.randomUUID(); + + final Map properties = Map.of(WORKSPACE_ID_HEADER, workspaceId.toString()); + + final WorkspaceHelper workspaceHelper = mock(WorkspaceHelper.class); + final AuthenticationHeaderResolver workspaceResolver = new AuthenticationHeaderResolver(workspaceHelper); + when(workspaceHelper.getOrganizationForWorkspace(workspaceId)).thenReturn(organizationId); + final List result = workspaceResolver.resolveOrganization(properties); + assertEquals(List.of(organizationId), result); + } + +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java index 01283fbee27..1c25cec4e03 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/AuthRoleConstants.java @@ -16,6 +16,10 @@ public final class AuthRoleConstants { public static final String NONE = "NONE"; public static final String READER = "READER"; + public static final String ORGANIZATION_ADMIN = "ORGANIZATION_ADMIN"; + public static final String ORGANIZATION_EDITOR = "ORGANIZATION_EDITOR"; + public static final String ORGANIZATION_READER = "ORGANIZATION_READER"; + private AuthRoleConstants() {} } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/auth/OrganizationAuthRole.java b/airbyte-commons/src/main/java/io/airbyte/commons/auth/OrganizationAuthRole.java new file mode 100644 index 00000000000..068ebdef8fd --- /dev/null +++ b/airbyte-commons/src/main/java/io/airbyte/commons/auth/OrganizationAuthRole.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.auth; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This enum describes the Organization auth levels for a given resource. A user will have + * organization leveled auth role and workspace leveled auth roles. See AuthRole.java for more + * information. + */ +public enum OrganizationAuthRole { + + ORGANIZATION_ADMIN(400, AuthRoleConstants.ORGANIZATION_ADMIN), + ORGANIZATION_EDITOR(300, AuthRoleConstants.ORGANIZATION_EDITOR), + ORGANIZATION_READER(200, AuthRoleConstants.ORGANIZATION_READER), + NONE(0, AuthRoleConstants.NONE); + + private final int authority; + private final String label; + + OrganizationAuthRole(final int authority, final String label) { + this.authority = authority; + this.label = label; + } + + public int getAuthority() { + return authority; + } + + public String getLabel() { + return label; + } + + /** + * Builds the set of roles based on the provided {@link OrganizationAuthRole} value. + *

+ * The generated set of auth roles contains the provided {@link OrganizationAuthRole} (if not + * {@code null}) and any other authentication roles with a lesser {@link #getAuthority()} value. + *

+ * + * @param authRole An {@link OrganizationAuthRole} (may be {@code null}). + * @return The set of {@link OrganizationAuthRole} labels based on the provided + * {@link OrganizationAuthRole}. + */ + public static Set buildOrganizationAuthRolesSet(final OrganizationAuthRole authRole) { + final Set authRoles = new HashSet<>(); + + if (authRole != null) { + authRoles.add(authRole); + authRoles.addAll(Stream.of(values()) + .filter(role -> !NONE.equals(role)) + .filter(role -> role.getAuthority() < authRole.getAuthority()) + .collect(Collectors.toSet())); + } + + // Sort final set by descending authority order + return authRoles.stream() + .sorted(Comparator.comparingInt(OrganizationAuthRole::getAuthority)) + .map(role -> role.getLabel()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + +} diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java index d8ef9a455ea..7be0fff4441 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java @@ -14,6 +14,7 @@ import io.airbyte.config.Permission; import io.airbyte.config.Permission.PermissionType; import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; import io.airbyte.config.UserPermission; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; @@ -201,6 +202,58 @@ public List listUsersInOrganization(final UUID organizationId) t } + public PermissionType findPermissionTypeForUserAndWorkspace(final UUID workspaceId, final String authUserId, final AuthProvider authProvider) + throws IOException { + return this.database.query(ctx -> findPermissionTypeForUserAndWorkspace(ctx, workspaceId, authUserId, authProvider)); + } + + private PermissionType findPermissionTypeForUserAndWorkspace(final DSLContext ctx, + final UUID workspaceId, + final String authUserId, + final AuthProvider authProvider) { + var record = ctx.select(PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.WORKSPACE_ID.eq(workspaceId)) + .and(USER.AUTH_USER_ID.eq(authUserId)) + .and(USER.AUTH_PROVIDER.eq(Enums.toEnum(authProvider.value(), io.airbyte.db.instance.configs.jooq.generated.enums.AuthProvider.class).get())) + .fetchOne(); + if (record == null) { + return null; + } + + final var jooqPermissionType = record.get(PERMISSION.PERMISSION_TYPE, io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.class); + + return Enums.toEnum(jooqPermissionType.getLiteral(), PermissionType.class).get(); + } + + public PermissionType findPermissionTypeForUserAndOrganization(final UUID organizationId, final String authUserId, final AuthProvider authProvider) + throws IOException { + return this.database.query(ctx -> findPermissionTypeForUserAndOrganization(ctx, organizationId, authUserId, authProvider)); + } + + private PermissionType findPermissionTypeForUserAndOrganization(final DSLContext ctx, + final UUID organizationId, + final String authUserId, + final AuthProvider authProvider) { + var record = ctx.select(PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.ORGANIZATION_ID.eq(organizationId)) + .and(USER.AUTH_USER_ID.eq(authUserId)) + .and(USER.AUTH_PROVIDER.eq(Enums.toEnum(authProvider.value(), io.airbyte.db.instance.configs.jooq.generated.enums.AuthProvider.class).get())) + .fetchOne(); + + if (record == null) { + return null; + } + + final var jooqPermissionType = record.get(PERMISSION.PERMISSION_TYPE, io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.class); + return Enums.toEnum(jooqPermissionType.getLiteral(), PermissionType.class).get(); + } + private List listPermissionsForWorkspace(final DSLContext ctx, final UUID workspaceId) { var records = ctx.select(USER.ID, USER.NAME, USER.EMAIL, USER.DEFAULT_WORKSPACE_ID, PERMISSION.ID, PERMISSION.PERMISSION_TYPE) .from(PERMISSION) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java index 65f577a1fb6..ce51a0c2e53 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -131,6 +131,8 @@ public class MockData { static final UUID CREATOR_USER_ID_2 = UUID.randomUUID(); static final UUID CREATOR_USER_ID_3 = UUID.randomUUID(); static final UUID CREATOR_USER_ID_4 = UUID.randomUUID(); + static final UUID CREATOR_USER_ID_5 = UUID.randomUUID(); + // Permission static final UUID PERMISSION_ID_1 = UUID.randomUUID(); static final UUID PERMISSION_ID_2 = UUID.randomUUID(); @@ -138,6 +140,8 @@ public class MockData { static final UUID PERMISSION_ID_4 = UUID.randomUUID(); static final UUID PERMISSION_ID_5 = UUID.randomUUID(); + static final UUID PERMISSION_ID_6 = UUID.randomUUID(); + static final UUID PERMISSION_ID_7 = UUID.randomUUID(); static final UUID ORGANIZATION_ID_1 = UUID.randomUUID(); static final UUID ORGANIZATION_ID_2 = UUID.randomUUID(); @@ -229,7 +233,18 @@ public static List users() { .withEmail("user-4@whatever.com") .withNews(true); - return Arrays.asList(user1, user2, user3, user4); + final User user5 = new User() + .withUserId(CREATOR_USER_ID_5) + .withName("user-5") + .withAuthUserId(CREATOR_USER_ID_5.toString()) + .withAuthProvider(AuthProvider.KEYCLOAK) + .withDefaultWorkspaceId(null) + .withStatus(User.Status.REGISTERED) + .withCompanyName("company-5") + .withEmail("user-5@whatever.com") + .withNews(true); + + return Arrays.asList(user1, user2, user3, user4, user5); } public static List permissions() { @@ -263,7 +278,19 @@ public static List permissions() { .withOrganizationId(ORGANIZATION_ID_1) .withPermissionType(PermissionType.ORGANIZATION_ADMIN); - return Arrays.asList(permission1, permission2, permission3, permission4, permission5); + final Permission permission6 = new Permission() + .withPermissionId(PERMISSION_ID_6) + .withUserId(CREATOR_USER_ID_5) + .withWorkspaceId(WORKSPACE_ID_2) + .withPermissionType(PermissionType.WORKSPACE_ADMIN); + + final Permission permission7 = new Permission() + .withPermissionId(PERMISSION_ID_7) + .withUserId(CREATOR_USER_ID_5) + .withOrganizationId(ORGANIZATION_ID_2) + .withPermissionType(PermissionType.ORGANIZATION_READER); + + return Arrays.asList(permission1, permission2, permission3, permission4, permission5, permission6, permission7); } public static List organizations() { diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java index 8725e533294..86ff2e03bcf 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java @@ -6,8 +6,10 @@ import io.airbyte.config.Organization; import io.airbyte.config.Permission; +import io.airbyte.config.Permission.PermissionType; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; import io.airbyte.config.UserPermission; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -102,4 +104,18 @@ void listUsersInWorkspaceTest() throws IOException { Assertions.assertEquals(2, userPermissions.size()); } + @Test + void findUsersInWorkspaceTest() throws Exception { + final PermissionType permissionType = permissionPersistence + .findPermissionTypeForUserAndWorkspace(MockData.WORKSPACE_ID_2, MockData.CREATOR_USER_ID_5.toString(), AuthProvider.KEYCLOAK); + Assertions.assertEquals(PermissionType.WORKSPACE_ADMIN, permissionType); + } + + @Test + void findUsersInOrganizationTest() throws Exception { + final PermissionType permissionType = permissionPersistence + .findPermissionTypeForUserAndOrganization(MockData.ORGANIZATION_ID_2, MockData.CREATOR_USER_ID_5.toString(), AuthProvider.KEYCLOAK); + Assertions.assertEquals(PermissionType.ORGANIZATION_READER, permissionType); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java index 9f4413811ac..82ac86fbbf0 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/UserPersistenceTest.java @@ -6,7 +6,6 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.User; -import io.airbyte.config.User.AuthProvider; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.Optional; @@ -48,7 +47,7 @@ void getUserByIdTest() throws IOException { @Test void getUserByAuthIdTest() throws IOException { for (final User user : MockData.users()) { - final Optional userFromDb = userPersistence.getUserByAuthId(user.getAuthUserId(), AuthProvider.GOOGLE_IDENTITY_PLATFORM); + final Optional userFromDb = userPersistence.getUserByAuthId(user.getAuthUserId(), user.getAuthProvider()); Assertions.assertEquals(user, userFromDb.get()); } } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java index 70679bec540..4974e9eaa65 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WorkspaceHelper.java @@ -44,6 +44,8 @@ public class WorkspaceHelper { private final LoadingCache operationToWorkspaceCache; private final LoadingCache jobToWorkspaceCache; + private final LoadingCache workspaceToOrganizationCache; + public WorkspaceHelper(final ConfigRepository configRepository, final JobPersistence jobPersistence) { this.sourceToWorkspaceCache = getExpiringCache(new CacheLoader<>() { @@ -104,6 +106,15 @@ public UUID load(@NonNull final Long jobId) throws ConfigNotFoundException, IOEx } }); + + this.workspaceToOrganizationCache = getExpiringCache(new CacheLoader<>() { + + @Override + public UUID load(UUID workspaceId) throws Exception { + return configRepository.getStandardWorkspaceNoSecrets(workspaceId, false).getOrganizationId(); + } + + }); } /** @@ -141,6 +152,10 @@ public UUID getWorkspaceForJobIdIgnoreExceptions(final Long jobId) { return swallowExecutionException(() -> jobToWorkspaceCache.get(jobId)); } + public UUID getOrganizationForWorkspace(final UUID workspaceId) { + return swallowExecutionException(() -> workspaceToOrganizationCache.get(workspaceId)); + } + // CONNECTION ID /** diff --git a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java index 384cfb28a02..2b7d5933dd6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java +++ b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java @@ -8,9 +8,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.commons.auth.AuthRole; +import io.airbyte.commons.auth.OrganizationAuthRole; import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.license.annotation.RequiresAirbyteProEnabled; +import io.airbyte.commons.server.support.AuthenticationHeaderResolver; +import io.airbyte.config.Permission.PermissionType; +import io.airbyte.config.User.AuthProvider; +import io.airbyte.config.persistence.PermissionPersistence; import io.micrometer.common.util.StringUtils; import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpRequest; @@ -22,9 +27,17 @@ import io.micronaut.security.authentication.AuthenticationException; import io.micronaut.security.token.validator.TokenValidator; import jakarta.inject.Singleton; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import lombok.extern.slf4j.Slf4j; import org.reactivestreams.Publisher; import reactor.core.publisher.Mono; @@ -41,11 +54,23 @@ public class KeycloakTokenValidator implements TokenValidator { private final HttpClient client; private final AirbyteKeycloakConfiguration keycloakConfiguration; + private final AuthenticationHeaderResolver headerResolver; + private final PermissionPersistence permissionPersistence; + + private static final Map WORKSPACE_PERMISSION_TYPE_TO_AUTH_ROLE = Map.of(PermissionType.WORKSPACE_ADMIN, AuthRole.ADMIN, + PermissionType.WORKSPACE_EDITOR, AuthRole.EDITOR, PermissionType.WORKSPACE_READER, AuthRole.READER); + private static final Map ORGANIZATION_PERMISSION_TYPE_TO_AUTH_ROLE = + Map.of(PermissionType.ORGANIZATION_ADMIN, OrganizationAuthRole.ORGANIZATION_ADMIN, PermissionType.ORGANIZATION_EDITOR, + OrganizationAuthRole.ORGANIZATION_EDITOR, PermissionType.ORGANIZATION_READER, OrganizationAuthRole.ORGANIZATION_READER); public KeycloakTokenValidator(final HttpClient httpClient, - final AirbyteKeycloakConfiguration keycloakConfiguration) { + final AirbyteKeycloakConfiguration keycloakConfiguration, + final AuthenticationHeaderResolver headerResolver, + final PermissionPersistence permissionPersistence) { this.client = httpClient; this.keycloakConfiguration = keycloakConfiguration; + this.headerResolver = headerResolver; + this.permissionPersistence = permissionPersistence; } @Override @@ -53,7 +78,7 @@ public Publisher validateToken(final String token, final HttpReq return validateTokenWithKeycloak(token) .flatMap(valid -> { if (valid) { - return Mono.just(getAuthentication(token)); + return Mono.just(getAuthentication(token, request)); } else { // pass to the next validator, if one exists log.warn("Token was not a valid Keycloak token: {}", token); @@ -62,7 +87,7 @@ public Publisher validateToken(final String token, final HttpReq }); } - private Authentication getAuthentication(final String token) { + private Authentication getAuthentication(final String token, final HttpRequest request) { final String[] tokenParts = token.split("\\."); final String payload = tokenParts[1]; @@ -76,7 +101,7 @@ private Authentication getAuthentication(final String token) { if (StringUtils.isNotBlank(userId)) { log.debug("Fetching roles for user '{}'...", userId); - final Collection roles = getRoles(userId); + final Collection roles = getRoles(userId, request); log.debug("Authenticating user '{}' with roles {}...", userId, roles); return Authentication.build(userId, roles); } else { @@ -92,9 +117,49 @@ private Authentication getAuthentication(final String token) { * For now, we are granting ADMIN to all authenticated users. This will change with the introduction * of RBAC. */ - private Collection getRoles(final String userId) { - log.debug("Granting ADMIN role to user {}", userId); - return AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + private Collection getRoles(final String userId, final HttpRequest request) { + Map headerMap = request.getHeaders().asMap(String.class, String.class); + + // We will check for permissions over organization and workspace + final List workspaceIds = headerResolver.resolveWorkspace(headerMap); + final List organizationIds = headerResolver.resolveOrganization(headerMap); + + Set roles = new HashSet<>(); + roles.add(AuthRole.AUTHENTICATED_USER.toString()); + + // Find the minimum permission for workspace + if (workspaceIds != null && !workspaceIds.isEmpty()) { + Optional minAuthRoleOptional = workspaceIds.stream() + .map(workspaceId -> { + try { + return permissionPersistence.findPermissionTypeForUserAndWorkspace(workspaceId, userId, AuthProvider.KEYCLOAK); + } catch (IOException ex) { + log.error("Failed to get permission for user {} and workspaces {}", userId, workspaceId, ex); + throw new RuntimeException(ex); + } + }) + .map(permissionType -> WORKSPACE_PERMISSION_TYPE_TO_AUTH_ROLE.get(permissionType)) + .min(Comparator.comparingInt(AuthRole::getAuthority)); + AuthRole authRole = minAuthRoleOptional.orElse(AuthRole.NONE); + roles.addAll(AuthRole.buildAuthRolesSet(authRole)); + } + if (organizationIds != null && !organizationIds.isEmpty()) { + Optional minAuthRoleOptional = organizationIds.stream() + .map(organizationId -> { + try { + return permissionPersistence.findPermissionTypeForUserAndOrganization(organizationId, userId, AuthProvider.KEYCLOAK); + } catch (IOException ex) { + log.error("Failed to get permission for user {} and organization {}", userId, organizationId, ex); + throw new RuntimeException(ex); + } + }) + .map(permissionType -> ORGANIZATION_PERMISSION_TYPE_TO_AUTH_ROLE.get(permissionType)) + .min(Comparator.comparingInt(OrganizationAuthRole::getAuthority)); + OrganizationAuthRole authRole = minAuthRoleOptional.orElse(OrganizationAuthRole.NONE); + roles.addAll(OrganizationAuthRole.buildOrganizationAuthRolesSet(authRole)); + } + + return roles; } private Mono validateTokenWithKeycloak(final String token) { diff --git a/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java b/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java index 723931057e2..995de69fe72 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java @@ -11,6 +11,10 @@ import io.airbyte.commons.auth.AuthRole; import io.airbyte.commons.auth.config.AirbyteKeycloakConfiguration; +import io.airbyte.commons.server.support.AuthenticationHeaderResolver; +import io.airbyte.config.Permission.PermissionType; +import io.airbyte.config.User.AuthProvider; +import io.airbyte.config.persistence.PermissionPersistence; import io.airbyte.server.pro.KeycloakTokenValidator; import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpRequest; @@ -21,6 +25,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,9 +42,14 @@ class KeycloakTokenValidatorTest { private static final String URI_PATH = "/some/path"; private static final Collection DEFAULT_ROLES = AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID ORGANIZATION_ID = UUID.randomUUID(); + private KeycloakTokenValidator keycloakTokenValidator; private HttpClient httpClient; private AirbyteKeycloakConfiguration keycloakConfiguration; + private AuthenticationHeaderResolver headerResolver; + private PermissionPersistence permissionPersistence; @BeforeEach void setUp() { @@ -45,8 +57,10 @@ void setUp() { keycloakConfiguration = mock(AirbyteKeycloakConfiguration.class); when(keycloakConfiguration.getKeycloakUserInfoEndpoint()).thenReturn(LOCALHOST + URI_PATH); + headerResolver = mock(AuthenticationHeaderResolver.class); + permissionPersistence = mock(PermissionPersistence.class); - keycloakTokenValidator = new KeycloakTokenValidator(httpClient, keycloakConfiguration); + keycloakTokenValidator = new KeycloakTokenValidator(httpClient, keycloakConfiguration, headerResolver, permissionPersistence); } @AfterEach @@ -55,7 +69,7 @@ void tearDown() { } @Test - void testValidateToken() throws URISyntaxException { + void testValidateToken() throws Exception { final URI uri = new URI(LOCALHOST + URI_PATH); final String accessToken = """ eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICIwM095c3pkWmNrZFd6Mk84d0ZFRkZVblJPLVJrN1lGLWZzRm1kWG1Q @@ -95,8 +109,17 @@ void testValidateToken() throws URISyntaxException { final Publisher responsePublisher = keycloakTokenValidator.validateToken(accessTokenWithoutNewline, httpRequest); + when(headerResolver.resolveWorkspace(any())).thenReturn(List.of(WORKSPACE_ID)); + when(headerResolver.resolveOrganization(any())).thenReturn(List.of(ORGANIZATION_ID)); + when(permissionPersistence.findPermissionTypeForUserAndWorkspace(WORKSPACE_ID, expectedUserId, AuthProvider.KEYCLOAK)) + .thenReturn(PermissionType.WORKSPACE_ADMIN); + when(permissionPersistence.findPermissionTypeForUserAndOrganization(ORGANIZATION_ID, expectedUserId, AuthProvider.KEYCLOAK)) + .thenReturn(PermissionType.ORGANIZATION_READER); + + Set expectedResult = Set.of("ORGANIZATION_READER", "ADMIN", "EDITOR", "READER"); + StepVerifier.create(responsePublisher) - .expectNextMatches(r -> matchSuccessfulResponse(r, expectedUserId, DEFAULT_ROLES)) + .expectNextMatches(r -> matchSuccessfulResponse(r, expectedUserId, expectedResult)) .verifyComplete(); } From fef39432c25713b109cb10c11dce4d014003c4b2 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Tue, 5 Sep 2023 15:18:10 -0400 Subject: [PATCH 107/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=A7=B9=20Rename=20?= =?UTF-8?q?"sync=20history"=20table=20to=20"job=20history"=20(#8679)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/locales/en.json | 3 +-- .../ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 16f85e99d53..f72ec9d2af4 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -349,7 +349,6 @@ "sources.schemaSelectAll": "select all", "sources.schemaUnselectAll": "unselect all", "sources.settings": "Settings", - "sources.syncHistory": "Sync History", "sources.source": "Source", "sources.noSync": "No sync yet", "sources.emptySchema": "Schema is empty", @@ -527,7 +526,7 @@ "connection.cancelSync": "Cancel Sync", "connection.cancelReset": "Cancel Reset", "connection.linkedJobNotFound": "Job not found", - "connection.returnToSyncHistory": "Return to Sync History", + "connection.returnToJobHistory": "Return to Job History", "connection.updateFailed": "Failed to update connection", "connection.progress.percentage": "{percent} percent progress", diff --git a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx index 360466059e0..8d5e127cc88 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/ConnectionJobHistoryPage.tsx @@ -67,7 +67,7 @@ export const ConnectionJobHistoryPage: React.FC = () => { - +
} />
@@ -81,7 +81,7 @@ export const ConnectionJobHistoryPage: React.FC = () => { text={} description={ - + } /> From 7603414acd185dfe16b2ea870e8f56009d68a136 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Tue, 5 Sep 2023 13:48:04 -0700 Subject: [PATCH 108/201] Make MetricClient injectable in orchestrator/server (#8686) --- .../temporal/ConnectionManagerUtils.java | 6 ++-- .../commons/temporal/TemporalClient.java | 7 ++-- .../commons/temporal/TemporalClientTest.java | 9 +++-- .../general/ReplicationWorkerFactory.java | 34 +++++++++---------- .../config/ContainerOrchestratorFactory.java | 9 +++++ .../server/config/ApplicationBeanFactory.java | 9 +++++ 6 files changed, 44 insertions(+), 30 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index 4133b67e23a..e5374ba02e9 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -11,7 +11,6 @@ import io.airbyte.commons.temporal.scheduling.state.WorkflowState; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.temporal.api.common.v1.WorkflowExecution; @@ -38,9 +37,8 @@ public class ConnectionManagerUtils { private final MetricClient metricClient; - public ConnectionManagerUtils() { - // TODO Inject it when MetricClient becomes injectable. - this.metricClient = MetricClientFactory.getMetricClient(); + public ConnectionManagerUtils(final MetricClient metricClient) { + this.metricClient = metricClient; } /** diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index bb7d87f97fd..e8dda943057 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -24,7 +24,6 @@ import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -89,7 +88,8 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, final StreamResetPersistence streamResetPersistence, final ConnectionManagerUtils connectionManagerUtils, final NotificationClient notificationClient, - final StreamResetRecordsHelper streamResetRecordsHelper) { + final StreamResetRecordsHelper streamResetRecordsHelper, + final MetricClient metricClient) { this.workspaceRoot = workspaceRoot; this.client = client; this.service = service; @@ -97,8 +97,7 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, this.connectionManagerUtils = connectionManagerUtils; this.notificationClient = notificationClient; this.streamResetRecordsHelper = streamResetRecordsHelper; - // TODO Inject it when MetricClient becomes injectable. - this.metricClient = MetricClientFactory.getMetricClient(); + this.metricClient = metricClient; } private final Set workflowNames = new HashSet<>(); diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 01425b6401a..30c2018ea9d 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -39,6 +39,7 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.metrics.lib.MetricClient; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; @@ -122,13 +123,12 @@ void setup() throws IOException { when(workflowServiceStubs.blockingStub()).thenReturn(workflowServiceBlockingStub); streamResetPersistence = mock(StreamResetPersistence.class); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); - connectionManagerUtils = spy(new ConnectionManagerUtils()); + connectionManagerUtils = spy(new ConnectionManagerUtils(mock(MetricClient.class))); notificationClient = spy(new NotificationClient(workflowClient)); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); temporalClient = spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, - notificationClient, - streamResetRecordsHelper)); + notificationClient, streamResetRecordsHelper, mock(MetricClient.class))); } @Nested @@ -144,8 +144,7 @@ void init() { temporalClient = spy( new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, - mNotificationClient, - streamResetRecordsHelper)); + mNotificationClient, streamResetRecordsHelper, mock(MetricClient.class))); } @Test diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index f1bf177b949..66c96c33607 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -29,8 +29,6 @@ import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; -import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -82,6 +80,7 @@ public class ReplicationWorkerFactory { private final AirbyteMessageDataExtractor airbyteMessageDataExtractor; private final FeatureFlagClient featureFlagClient; private final FeatureFlags featureFlags; + private final MetricClient metricClient; private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; public ReplicationWorkerFactory( @@ -93,7 +92,8 @@ public ReplicationWorkerFactory( final SyncPersistenceFactory syncPersistenceFactory, final FeatureFlagClient featureFlagClient, final FeatureFlags featureFlags, - final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper) { + final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, + final MetricClient metricClient) { this.airbyteIntegrationLauncherFactory = airbyteIntegrationLauncherFactory; this.sourceApi = sourceApi; this.sourceDefinitionApi = sourceDefinitionApi; @@ -104,6 +104,7 @@ public ReplicationWorkerFactory( this.featureFlagClient = featureFlagClient; this.featureFlags = featureFlags; + this.metricClient = metricClient; } /** @@ -121,7 +122,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, "get the source definition for feature flag checks"); final HeartbeatMonitor heartbeatMonitor = createHeartbeatMonitor(sourceDefinitionId, sourceDefinitionApi); final HeartbeatTimeoutChaperone heartbeatTimeoutChaperone = createHeartbeatTimeoutChaperone(heartbeatMonitor, - featureFlagClient, syncInput); + featureFlagClient, syncInput, metricClient); final RecordSchemaValidator recordSchemaValidator = createRecordSchemaValidator(syncInput); // Enable concurrent stream reads for testing purposes @@ -138,9 +139,6 @@ public ReplicationWorker create(final StandardSyncInput syncInput, final var airbyteDestination = airbyteIntegrationLauncherFactory.createAirbyteDestination(destinationLauncherConfig, syncInput.getSyncResourceRequirements(), syncInput.getCatalog()); - // TODO MetricClient should be injectable (please) - MetricClientFactory.initialize(MetricEmittingApps.WORKER); - final MetricClient metricClient = MetricClientFactory.getMetricClient(); final WorkerMetricReporter metricReporter = new WorkerMetricReporter(metricClient, sourceLauncherConfig.getDockerImage()); final FieldSelector fieldSelector = @@ -153,7 +151,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, return createReplicationWorker(airbyteSource, airbyteDestination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, heartbeatTimeoutChaperone, featureFlagClient, jobRunConfig, syncInput, airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning); + onReplicationRunning, metricClient); } /** @@ -212,13 +210,14 @@ private static HeartbeatMonitor createHeartbeatMonitor(final UUID sourceDefiniti */ private static HeartbeatTimeoutChaperone createHeartbeatTimeoutChaperone(final HeartbeatMonitor heartbeatMonitor, final FeatureFlagClient featureFlagClient, - final StandardSyncInput syncInput) { + final StandardSyncInput syncInput, + final MetricClient metricClient) { return new HeartbeatTimeoutChaperone(heartbeatMonitor, HeartbeatTimeoutChaperone.DEFAULT_TIMEOUT_CHECK_DURATION, featureFlagClient, syncInput.getWorkspaceId(), syncInput.getConnectionId(), - MetricClientFactory.getMetricClient()); + metricClient); } /** @@ -263,7 +262,8 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou final StandardSyncInput syncInput, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationEventPublishingHelper, - final VoidCallable onReplicationRunning) { + final VoidCallable onReplicationRunning, + final MetricClient metricClient) { final Context flagContext = getFeatureFlagContext(syncInput); final String workerImpl = featureFlagClient.stringVariation(ReplicationWorkerImpl.INSTANCE, flagContext); return buildReplicationWorkerInstance( @@ -284,7 +284,8 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationEventPublishingHelper, - onReplicationRunning); + onReplicationRunning, + metricClient); } private static Context getFeatureFlagContext(final StandardSyncInput syncInput) { @@ -323,16 +324,15 @@ private static ReplicationWorker buildReplicationWorkerInstance(final String wor final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final VoidCallable onReplicationRunning) { + final VoidCallable onReplicationRunning, + final MetricClient metricClient) { if ("buffered".equals(workerImpl)) { - MetricClientFactory.getMetricClient() - .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, messageEventPublishingHelper, onReplicationRunning); } else { - MetricClientFactory.getMetricClient() - .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, messageEventPublishingHelper, onReplicationRunning); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java index 7416ac78dce..ce3b98c47c8 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java @@ -16,6 +16,9 @@ import io.airbyte.container_orchestrator.orchestrator.NormalizationJobOrchestrator; import io.airbyte.container_orchestrator.orchestrator.ReplicationJobOrchestrator; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.general.ReplicationWorkerFactory; import io.airbyte.workers.internal.state_aggregator.StateAggregatorFactory; @@ -47,6 +50,12 @@ @Factory class ContainerOrchestratorFactory { + @Singleton + public MetricClient metricClient() { + MetricClientFactory.initialize(MetricEmittingApps.ORCHESTRATOR); + return MetricClientFactory.getMetricClient(); + } + @Singleton FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java index 9bd931f4e9b..63238e06e4c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java @@ -19,6 +19,9 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.DefaultJobCreator; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; @@ -127,6 +130,12 @@ public FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); } + @Singleton + public MetricClient metricClient() { + MetricClientFactory.initialize(MetricEmittingApps.SERVER); + return MetricClientFactory.getMetricClient(); + } + @Singleton @Named("workspaceRoot") public Path workspaceRoot(@Value("${airbyte.workspace.root}") final String workspaceRoot) { From ac56bb2a798667b4da035a37b8a48a19fff87f74 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Tue, 5 Sep 2023 16:29:17 -0500 Subject: [PATCH 109/201] remove unused YamlListToStandardDefinitions (#8622) --- .../YamlListToStandardDefinitions.java | 104 ------------- .../YamlListToStandardDefinitionsTest.java | 145 ------------------ 2 files changed, 249 deletions(-) delete mode 100644 airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/YamlListToStandardDefinitions.java delete mode 100644 airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/YamlListToStandardDefinitionsTest.java diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/YamlListToStandardDefinitions.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/YamlListToStandardDefinitions.java deleted file mode 100644 index b926f123705..00000000000 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/YamlListToStandardDefinitions.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.helpers; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.util.ClassUtil; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.yaml.Yamls; -import io.airbyte.config.StandardDestinationDefinition; -import io.airbyte.config.StandardSourceDefinition; -import java.util.AbstractMap.SimpleImmutableEntry; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * This is a convenience class for the conversion of a list of source/destination definitions from - * human-friendly yaml to processing friendly formats i.e. Java models or JSON. As this class - * performs validation, it is recommended to use this class to deal with plain lists. An example of - * such lists are Airbyte's master definition lists, which can be seen in the resources folder of - * the airbyte-config/seed module. - * - * In addition to usual deserialization validations, we check: 1) The given list contains no - * duplicate names. 2) The given list contains no duplicate ids. - * - * Methods in these class throw Runtime exceptions upon validation failure. - */ -@SuppressWarnings("PMD.ShortVariable") -public class YamlListToStandardDefinitions { - - private static final Map CLASS_NAME_TO_ID_NAME = Map.ofEntries( - new SimpleImmutableEntry<>(StandardDestinationDefinition.class.getCanonicalName(), "destinationDefinitionId"), - new SimpleImmutableEntry<>(StandardSourceDefinition.class.getCanonicalName(), "sourceDefinitionId")); - - public static List toStandardSourceDefinitions(final String yamlStr) { - return verifyAndConvertToModelList(StandardSourceDefinition.class, yamlStr); - } - - public static List toStandardDestinationDefinitions(final String yamlStr) { - return verifyAndConvertToModelList(StandardDestinationDefinition.class, yamlStr); - } - - static JsonNode verifyAndConvertToJsonNode(final String idName, final String yamlStr) { - final var jsonNode = Yamls.deserialize(yamlStr); - checkYamlIsPresentWithNoDuplicates(jsonNode, idName); - return jsonNode; - } - - @VisibleForTesting - static List verifyAndConvertToModelList(final Class klass, final String yamlStr) { - final var jsonNode = Yamls.deserialize(yamlStr); - final var idName = CLASS_NAME_TO_ID_NAME.get(klass.getCanonicalName()); - checkYamlIsPresentWithNoDuplicates(jsonNode, idName); - return toStandardXDefinitions(jsonNode.elements(), klass); - } - - private static void checkYamlIsPresentWithNoDuplicates(final JsonNode deserialize, final String idName) { - final var presentDestList = !deserialize.elements().equals(ClassUtil.emptyIterator()); - Preconditions.checkState(presentDestList, "Definition list is empty"); - checkNoDuplicateNames(deserialize.elements()); - checkNoDuplicateIds(deserialize.elements(), idName); - } - - private static void checkNoDuplicateNames(final Iterator iter) { - final var names = new HashSet(); - while (iter.hasNext()) { - final var element = Jsons.clone(iter.next()); - final var name = element.get("name").asText(); - if (names.contains(name)) { - throw new IllegalArgumentException("Multiple records have the name: " + name); - } - names.add(name); - } - } - - private static void checkNoDuplicateIds(final Iterator fileIterator, final String idName) { - final var ids = new HashSet(); - while (fileIterator.hasNext()) { - final var element = Jsons.clone(fileIterator.next()); - final var id = element.get(idName).asText(); - if (ids.contains(id)) { - throw new IllegalArgumentException("Multiple records have the id: " + id); - } - ids.add(id); - } - } - - private static List toStandardXDefinitions(final Iterator iter, final Class c) { - final Iterable iterable = () -> iter; - final var defList = new ArrayList(); - for (final JsonNode n : iterable) { - final var def = Jsons.object(n, c); - defList.add(def); - } - return defList; - } - -} diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/YamlListToStandardDefinitionsTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/YamlListToStandardDefinitionsTest.java deleted file mode 100644 index 56912b5774e..00000000000 --- a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/YamlListToStandardDefinitionsTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.config.helpers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.config.StandardDestinationDefinition; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -class YamlListToStandardDefinitionsTest { - - private static final String DESTINATION_DEFINITION_ID = "- destinationDefinitionId: a625d593-bba5-4a1c-a53d-2d246268a816\n"; - private static final String DESTINATION_NAME = " name: Local JSON\n"; - private static final String DOCKER_REPO = " dockerRepository: airbyte/destination-local-json\n"; - private static final String DOCKER_IMAGE_TAG = " dockerImageTag: 0.1.4\n"; - private static final String GOOD_DES_DEF_YAML = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json"; - private static final String DUPLICATE_ID = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json" - + DESTINATION_DEFINITION_ID - + " name: JSON 2\n" - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json"; - private static final String DUPLICATE_NAME = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + DOCKER_IMAGE_TAG - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-json\n" - + "- destinationDefinitionId: 8be1cf83-fde1-477f-a4ad-318d23c9f3c6\n" - + DESTINATION_NAME - + " dockerRepository: airbyte/destination-csv\n" - + " dockerImageTag: 0.1.8\n" - + " documentationUrl: https://docs.airbyte.io/integrations/destinations/local-csv"; - private static final String BAD_DATA = - DESTINATION_DEFINITION_ID - + DESTINATION_NAME - + DOCKER_REPO - + " dockerImageTag: 0.1.8\n" - + " documentationUrl"; - - @Nested - @DisplayName("vertifyAndConvertToJsonNode") - class VerifyAndConvertToJsonNode { - - private static final String ID_NAME = "destinationDefinitionId"; - - private final ObjectMapper mapper = MoreMappers.initMapper(); - - @Test - @DisplayName("should correctly read yaml file") - void correctlyReadTest() throws JsonProcessingException { - final var jsonDefs = YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, GOOD_DES_DEF_YAML); - final var defList = mapper.treeToValue(jsonDefs, StandardDestinationDefinition[].class); - assertEquals(1, defList.length); - assertEquals("Local JSON", defList[0].getName()); - } - - @Test - @DisplayName("should error out on duplicate id") - void duplicateIdTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, DUPLICATE_ID)); - } - - @Test - @DisplayName("should error out on duplicate name") - void duplicateNameTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, DUPLICATE_NAME)); - } - - @Test - @DisplayName("should error out on empty file") - void emptyFileTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, "")); - } - - @Test - @DisplayName("should error out on bad data") - void badDataTest() { - assertThrows(RuntimeException.class, () -> YamlListToStandardDefinitions.verifyAndConvertToJsonNode(ID_NAME, BAD_DATA)); - } - - } - - @Nested - @DisplayName("verifyAndConvertToModelList") - class VerifyAndConvertToModelList { - - @Test - @DisplayName("should correctly read yaml file") - void correctlyReadTest() { - final var defs = YamlListToStandardDefinitions - .verifyAndConvertToModelList(StandardDestinationDefinition.class, GOOD_DES_DEF_YAML); - assertEquals(1, defs.size()); - assertEquals("Local JSON", defs.get(0).getName()); - } - - @Test - @DisplayName("should error out on duplicate id") - void duplicateIdTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, DUPLICATE_ID)); - } - - @Test - @DisplayName("should error out on duplicate name") - void duplicateNameTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, DUPLICATE_NAME)); - } - - @Test - @DisplayName("should error out on empty file") - void emptyFileTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, "")); - } - - @Test - @DisplayName("should error out on bad data") - void badDataTest() { - assertThrows(RuntimeException.class, - () -> YamlListToStandardDefinitions.verifyAndConvertToModelList(StandardDestinationDefinition.class, BAD_DATA)); - } - - } - -} From b89bf6b6c52d761be11f32ef140684881dba2af1 Mon Sep 17 00:00:00 2001 From: Michael Siega <109092231+mfsiega-airbyte@users.noreply.github.com> Date: Wed, 6 Sep 2023 11:19:04 +0200 Subject: [PATCH 110/201] Revert "Make MetricClient injectable in orchestrator/server (#8686)" (#8701) --- .../temporal/ConnectionManagerUtils.java | 6 ++-- .../commons/temporal/TemporalClient.java | 7 ++-- .../commons/temporal/TemporalClientTest.java | 9 ++--- .../general/ReplicationWorkerFactory.java | 34 +++++++++---------- .../config/ContainerOrchestratorFactory.java | 9 ----- .../server/config/ApplicationBeanFactory.java | 9 ----- 6 files changed, 30 insertions(+), 44 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index e5374ba02e9..4133b67e23a 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -11,6 +11,7 @@ import io.airbyte.commons.temporal.scheduling.state.WorkflowState; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.temporal.api.common.v1.WorkflowExecution; @@ -37,8 +38,9 @@ public class ConnectionManagerUtils { private final MetricClient metricClient; - public ConnectionManagerUtils(final MetricClient metricClient) { - this.metricClient = metricClient; + public ConnectionManagerUtils() { + // TODO Inject it when MetricClient becomes injectable. + this.metricClient = MetricClientFactory.getMetricClient(); } /** diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index e8dda943057..bb7d87f97fd 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -24,6 +24,7 @@ import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -88,8 +89,7 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, final StreamResetPersistence streamResetPersistence, final ConnectionManagerUtils connectionManagerUtils, final NotificationClient notificationClient, - final StreamResetRecordsHelper streamResetRecordsHelper, - final MetricClient metricClient) { + final StreamResetRecordsHelper streamResetRecordsHelper) { this.workspaceRoot = workspaceRoot; this.client = client; this.service = service; @@ -97,7 +97,8 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, this.connectionManagerUtils = connectionManagerUtils; this.notificationClient = notificationClient; this.streamResetRecordsHelper = streamResetRecordsHelper; - this.metricClient = metricClient; + // TODO Inject it when MetricClient becomes injectable. + this.metricClient = MetricClientFactory.getMetricClient(); } private final Set workflowNames = new HashSet<>(); diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 30c2018ea9d..01425b6401a 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -39,7 +39,6 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.persistence.StreamResetPersistence; -import io.airbyte.metrics.lib.MetricClient; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; @@ -123,12 +122,13 @@ void setup() throws IOException { when(workflowServiceStubs.blockingStub()).thenReturn(workflowServiceBlockingStub); streamResetPersistence = mock(StreamResetPersistence.class); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); - connectionManagerUtils = spy(new ConnectionManagerUtils(mock(MetricClient.class))); + connectionManagerUtils = spy(new ConnectionManagerUtils()); notificationClient = spy(new NotificationClient(workflowClient)); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); temporalClient = spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, - notificationClient, streamResetRecordsHelper, mock(MetricClient.class))); + notificationClient, + streamResetRecordsHelper)); } @Nested @@ -144,7 +144,8 @@ void init() { temporalClient = spy( new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, - mNotificationClient, streamResetRecordsHelper, mock(MetricClient.class))); + mNotificationClient, + streamResetRecordsHelper)); } @Test diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index 66c96c33607..f1bf177b949 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -29,6 +29,8 @@ import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -80,7 +82,6 @@ public class ReplicationWorkerFactory { private final AirbyteMessageDataExtractor airbyteMessageDataExtractor; private final FeatureFlagClient featureFlagClient; private final FeatureFlags featureFlags; - private final MetricClient metricClient; private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; public ReplicationWorkerFactory( @@ -92,8 +93,7 @@ public ReplicationWorkerFactory( final SyncPersistenceFactory syncPersistenceFactory, final FeatureFlagClient featureFlagClient, final FeatureFlags featureFlags, - final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, - final MetricClient metricClient) { + final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper) { this.airbyteIntegrationLauncherFactory = airbyteIntegrationLauncherFactory; this.sourceApi = sourceApi; this.sourceDefinitionApi = sourceDefinitionApi; @@ -104,7 +104,6 @@ public ReplicationWorkerFactory( this.featureFlagClient = featureFlagClient; this.featureFlags = featureFlags; - this.metricClient = metricClient; } /** @@ -122,7 +121,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, "get the source definition for feature flag checks"); final HeartbeatMonitor heartbeatMonitor = createHeartbeatMonitor(sourceDefinitionId, sourceDefinitionApi); final HeartbeatTimeoutChaperone heartbeatTimeoutChaperone = createHeartbeatTimeoutChaperone(heartbeatMonitor, - featureFlagClient, syncInput, metricClient); + featureFlagClient, syncInput); final RecordSchemaValidator recordSchemaValidator = createRecordSchemaValidator(syncInput); // Enable concurrent stream reads for testing purposes @@ -139,6 +138,9 @@ public ReplicationWorker create(final StandardSyncInput syncInput, final var airbyteDestination = airbyteIntegrationLauncherFactory.createAirbyteDestination(destinationLauncherConfig, syncInput.getSyncResourceRequirements(), syncInput.getCatalog()); + // TODO MetricClient should be injectable (please) + MetricClientFactory.initialize(MetricEmittingApps.WORKER); + final MetricClient metricClient = MetricClientFactory.getMetricClient(); final WorkerMetricReporter metricReporter = new WorkerMetricReporter(metricClient, sourceLauncherConfig.getDockerImage()); final FieldSelector fieldSelector = @@ -151,7 +153,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, return createReplicationWorker(airbyteSource, airbyteDestination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, heartbeatTimeoutChaperone, featureFlagClient, jobRunConfig, syncInput, airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning, metricClient); + onReplicationRunning); } /** @@ -210,14 +212,13 @@ private static HeartbeatMonitor createHeartbeatMonitor(final UUID sourceDefiniti */ private static HeartbeatTimeoutChaperone createHeartbeatTimeoutChaperone(final HeartbeatMonitor heartbeatMonitor, final FeatureFlagClient featureFlagClient, - final StandardSyncInput syncInput, - final MetricClient metricClient) { + final StandardSyncInput syncInput) { return new HeartbeatTimeoutChaperone(heartbeatMonitor, HeartbeatTimeoutChaperone.DEFAULT_TIMEOUT_CHECK_DURATION, featureFlagClient, syncInput.getWorkspaceId(), syncInput.getConnectionId(), - metricClient); + MetricClientFactory.getMetricClient()); } /** @@ -262,8 +263,7 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou final StandardSyncInput syncInput, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationEventPublishingHelper, - final VoidCallable onReplicationRunning, - final MetricClient metricClient) { + final VoidCallable onReplicationRunning) { final Context flagContext = getFeatureFlagContext(syncInput); final String workerImpl = featureFlagClient.stringVariation(ReplicationWorkerImpl.INSTANCE, flagContext); return buildReplicationWorkerInstance( @@ -284,8 +284,7 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationEventPublishingHelper, - onReplicationRunning, - metricClient); + onReplicationRunning); } private static Context getFeatureFlagContext(final StandardSyncInput syncInput) { @@ -324,15 +323,16 @@ private static ReplicationWorker buildReplicationWorkerInstance(final String wor final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final VoidCallable onReplicationRunning, - final MetricClient metricClient) { + final VoidCallable onReplicationRunning) { if ("buffered".equals(workerImpl)) { - metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); + MetricClientFactory.getMetricClient() + .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, messageEventPublishingHelper, onReplicationRunning); } else { - metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); + MetricClientFactory.getMetricClient() + .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, messageEventPublishingHelper, onReplicationRunning); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java index ce3b98c47c8..7416ac78dce 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java @@ -16,9 +16,6 @@ import io.airbyte.container_orchestrator.orchestrator.NormalizationJobOrchestrator; import io.airbyte.container_orchestrator.orchestrator.ReplicationJobOrchestrator; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; -import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.general.ReplicationWorkerFactory; import io.airbyte.workers.internal.state_aggregator.StateAggregatorFactory; @@ -50,12 +47,6 @@ @Factory class ContainerOrchestratorFactory { - @Singleton - public MetricClient metricClient() { - MetricClientFactory.initialize(MetricEmittingApps.ORCHESTRATOR); - return MetricClientFactory.getMetricClient(); - } - @Singleton FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java index 63238e06e4c..9bd931f4e9b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java @@ -19,9 +19,6 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; -import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.DefaultJobCreator; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; @@ -130,12 +127,6 @@ public FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); } - @Singleton - public MetricClient metricClient() { - MetricClientFactory.initialize(MetricEmittingApps.SERVER); - return MetricClientFactory.getMetricClient(); - } - @Singleton @Named("workspaceRoot") public Path workspaceRoot(@Value("${airbyte.workspace.root}") final String workspaceRoot) { From 1fcd90165739dee28387a40c50d7217e32626586 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III <90398440+octavia-squidington-iii@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:09:20 -0600 Subject: [PATCH 111/201] Updating CDK version following release (#8452) Co-authored-by: maxi297 --- airbyte-connector-builder-resources/CDK_VERSION | 2 +- airbyte-connector-builder-server/Dockerfile | 2 +- airbyte-connector-builder-server/requirements.in | 2 +- airbyte-connector-builder-server/requirements.txt | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/airbyte-connector-builder-resources/CDK_VERSION b/airbyte-connector-builder-resources/CDK_VERSION index 969eb252497..5e29dbce896 100644 --- a/airbyte-connector-builder-resources/CDK_VERSION +++ b/airbyte-connector-builder-resources/CDK_VERSION @@ -1 +1 @@ -0.51.2 +0.51.11 diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index 680bc610d75..37b30012c91 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE=airbyte/airbyte-base-java-python-image:1.0 FROM ${BASE_IMAGE} AS connector-builder-server # Set up CDK requirements -ARG CDK_VERSION=0.51.2 +ARG CDK_VERSION=0.51.11 ENV CDK_PYTHON=${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin/python ENV CDK_ENTRYPOINT ${PYENV_ROOT}/versions/${PYTHON_VERSION}/lib/python3.9/site-packages/airbyte_cdk/connector_builder/main.py # Set up CDK diff --git a/airbyte-connector-builder-server/requirements.in b/airbyte-connector-builder-server/requirements.in index 90e5acfb336..8ae4bc3c04c 100644 --- a/airbyte-connector-builder-server/requirements.in +++ b/airbyte-connector-builder-server/requirements.in @@ -1 +1 @@ -airbyte-cdk==0.51.2 +airbyte-cdk==0.51.11 diff --git a/airbyte-connector-builder-server/requirements.txt b/airbyte-connector-builder-server/requirements.txt index 75fa38377d5..6e8762cf34b 100644 --- a/airbyte-connector-builder-server/requirements.txt +++ b/airbyte-connector-builder-server/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile # -airbyte-cdk==0.51.2 +airbyte-cdk==0.51.11 # via -r requirements.in airbyte-protocol-models==0.4.0 # via airbyte-cdk @@ -15,7 +15,7 @@ attrs==23.1.0 # requests-cache backoff==2.2.1 # via airbyte-cdk -bracex==2.3.post1 +bracex==2.4 # via wcmatch cachetools==5.3.1 # via airbyte-cdk From 2650ccc3bd016954f82fd1060c1ddd4fb6e32883 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 6 Sep 2023 11:30:12 -0400 Subject: [PATCH 112/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20allow=20f?= =?UTF-8?q?or=20missing=20organization=20id=20in=20useIntent=20queries=20(?= =?UTF-8?q?#8689)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/core/utils/rbac/rbac.test.ts | 12 ++++++++---- airbyte-webapp/src/core/utils/rbac/rbac.ts | 8 +++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts index 5444d652e55..2cd432660e6 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts +++ b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts @@ -162,14 +162,18 @@ describe("useRbac", () => { ).toThrowError("Invalid RBAC query: resource INSTANCE with resourceId some-workspace"); }); - it("throws an error when non-instance query is missing and cannot find a resourceId", () => { + it("does not throw an error when an organization query is missing and cannot find a resourceId", () => { mockUseListPermissions.mockImplementation(() => ({ permissions: [{ permissionType: "instance_admin" }], })); - expect(() => renderHook(() => useRbac({ resourceType: "ORGANIZATION", role: "ADMIN" }))).toThrowError( - "Invalid RBAC query: resource ORGANIZATION with resourceId undefined" - ); + renderHook(() => useRbac({ resourceType: "ORGANIZATION", role: "ADMIN" })); + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(2); + expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ + resourceType: "ORGANIZATION", + role: "ADMIN", + resourceId: undefined, + }); mockUseListPermissions.mockImplementation(() => ({ permissions: [{ permissionType: "instance_admin" }], diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.ts b/airbyte-webapp/src/core/utils/rbac/rbac.ts index c64e38d4597..b644b74baae 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.ts +++ b/airbyte-webapp/src/core/utils/rbac/rbac.ts @@ -27,7 +27,13 @@ export const useRbac = (query: RbacQuery | RbacQueryWithoutResourceId) => { // invariant check if ((!queryUsesResourceId && resourceId) || (queryUsesResourceId && !resourceId)) { - throw new Error(`Invalid RBAC query: resource ${resourceType} with resourceId ${resourceId}`); + // TODO: This is a patch to handle the fact that workspaces on cloud do not have an organization. + if (resourceType === "ORGANIZATION") { + console.log("Missing organization id"); + } else { + // this will throw if there is a missing workspace id OR a resourceId is passed to an instance query + throw new Error(`Invalid RBAC query: resource ${resourceType} with resourceId ${resourceId}`); + } } // above invariant guarantees resourceId is defined when needed From 12b21ebaf3fe6ef1a2c91bafd30d9e2e1efeb8a5 Mon Sep 17 00:00:00 2001 From: Flutra Osmani <129116758+flutra-osmani@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:37:20 -0700 Subject: [PATCH 113/201] identify early sync jobs and submit isFree to Orb (#8518) --- .../java/io/airbyte/config/EarlySyncJob.java | 31 +++++++++++++ .../config/persistence/ConfigRepository.java | 44 +++++++++++++++++++ .../ConfigRepositoryE2EReadWriteTest.java | 10 +++++ .../src/main/kotlin/FlagDefinitions.kt | 4 ++ 4 files changed, 89 insertions(+) create mode 100644 airbyte-config/config-models/src/main/java/io/airbyte/config/EarlySyncJob.java diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EarlySyncJob.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EarlySyncJob.java new file mode 100644 index 00000000000..e40b6f3ce85 --- /dev/null +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EarlySyncJob.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config; + +/** + * POJO for the EarlySyncJob domain model. We run the EARLY_SYNC_JOB_QUERY query to identify early + * syncs, by looking for sync jobs that are successful and have their associated connector created + * in the last 7 days. Query returns the id and is_free columns from the jobs table, which is then + * mapped to this POJO. + */ +public class EarlySyncJob { + + private final Long id; + private final Boolean isFree; + + public EarlySyncJob(final Long id, final Boolean isFree) { + this.id = id; + this.isFree = isFree; + } + + public Long getId() { + return id; + } + + public Boolean isFree() { + return isFree; + } + +} diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 1195e8c7690..c994c9862ec 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -54,6 +54,7 @@ import io.airbyte.config.DeclarativeManifest; import io.airbyte.config.DestinationConnection; import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.EarlySyncJob; import io.airbyte.config.Geography; import io.airbyte.config.OperatorDbt; import io.airbyte.config.OperatorNormalization; @@ -3786,4 +3787,47 @@ public List listBreakingChanges() throws IOExcept .collect(Collectors.toList())); } + /** + * This query retrieves the first successful sync jobs created in the past 7 days for each actor + * definition in a given workspace. It is used to mark these early syncs as free. + */ + private static final String EARLY_SYNC_JOB_QUERY = + "WITH EarliestActorRun AS (" + + " SELECT actor.actor_definition_id AS actor_definition_id, actor.workspace_id, MIN(jobs.created_at) AS earliest_job_created_at" + + " FROM jobs" + + " LEFT JOIN connection ON uuid(jobs.scope) = connection.id" + + " LEFT JOIN actor ON connection.source_id = actor.id" + + " WHERE jobs.status = 'succeeded' AND jobs.config_type = 'sync' AND jobs.created_at > now() - make_interval(days => ?)" + + " GROUP BY actor.actor_definition_id, actor.workspace_id" + + ")" + + " SELECT jobs.id," + + " CASE" + + " WHEN (jobs.created_at < earliest_job_created_at + make_interval(days => ?)) AND jobs.status = 'succeeded'" + + " THEN TRUE" + + " ELSE FALSE" + + " END AS IS_FREE" + + " FROM jobs" + + " LEFT JOIN connection ON uuid(jobs.scope) = connection.id" + + " LEFT JOIN actor ON connection.source_id = actor.id" + + " LEFT JOIN EarliestActorRun AS ear" + + " ON actor.workspace_id = ear.workspace_id" + + " AND actor.actor_definition_id = ear.actor_definition_id" + + " WHERE jobs.created_at > now() - make_interval(days => ?)"; + + public List listEarlySyncJobs(int jobsInterval, int earlySyncInterval) + throws IOException { + return database.query(ctx -> getEarlySyncJobsFromResult(ctx.fetch( + EARLY_SYNC_JOB_QUERY, jobsInterval, earlySyncInterval, jobsInterval))); + } + + private List getEarlySyncJobsFromResult(Result result) { + // Transform the result to a list of early sync jobs + List earlySyncJobs = new ArrayList<>(); + for (Record record : result) { + EarlySyncJob job = new EarlySyncJob((Long) record.get("id"), (Boolean) record.get("is_free")); + earlySyncJobs.add(job); + } + return earlySyncJobs; + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index 7e8ee2d3b63..a171a38cbb2 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -12,6 +12,7 @@ import static org.jooq.impl.DSL.select; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.spy; @@ -21,6 +22,7 @@ import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.DestinationConnection; import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.EarlySyncJob; import io.airbyte.config.Geography; import io.airbyte.config.ScopeType; import io.airbyte.config.SourceConnection; @@ -863,4 +865,12 @@ private static void writeActorCatalog(final List configs, final DS }); } + @Test + void testGetEarlySyncJobs() throws IOException { + // This test just verifies that the query can be run against configAPI DB. + // The query has been tested locally against prod DB to verify the outputs. + final List earlySyncJobs = configRepository.listEarlySyncJobs(30, 7); + assertNotNull(earlySyncJobs); + } + } diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index e11eddc5506..b3593dad642 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -43,6 +43,10 @@ object CanonicalCatalogSchema : Temporary(key = "canonical-catalog-sche object CatalogCanonicalJson : Temporary(key = "catalog-canonical-json", default = false) +object EarlySyncEnabled : Temporary(key = "billing.early-sync-enabled", default = false) + +object FetchEarlySyncJobs : Temporary(key = "billing.fetch-early-sync-jobs", default = false) + object ShouldRunOnGkeDataplane : Temporary(key = "should-run-on-gke-dataplane", default = false) object ShouldRunOnExpandedGkeDataplane : Temporary(key = "should-run-on-expanded-gke-dataplane", default = false) From e7ea0fd09e30fd4b37e09ac8a39f23879ccfaa4a Mon Sep 17 00:00:00 2001 From: Joey Marshment-Howell Date: Wed, 6 Sep 2023 18:51:03 +0200 Subject: [PATCH 114/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=89=20Add=20que?= =?UTF-8?q?ry=20params=20to=20keycloak=20redirect=20(#8677)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/core/services/auth/KeycloakAuthService.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/airbyte-webapp/src/core/services/auth/KeycloakAuthService.tsx b/airbyte-webapp/src/core/services/auth/KeycloakAuthService.tsx index a0b65820f8d..029ab7abf21 100644 --- a/airbyte-webapp/src/core/services/auth/KeycloakAuthService.tsx +++ b/airbyte-webapp/src/core/services/auth/KeycloakAuthService.tsx @@ -20,9 +20,17 @@ export const KeycloakAuthService: React.FC> = ({ chil const oidcConfig = { authority: `${webappUrl}/auth/realms/${auth.defaultRealm}`, client_id: auth.clientId, - redirect_uri: `${window.location.origin}/${window.location.pathname}`, + redirect_uri: window.location.href, onSigninCallback: () => { - window.history.replaceState({}, document.title, window.location.pathname); + // Remove OIDC params from URL, but don't remove other params that might be present + const searchParams = new URLSearchParams(window.location.search); + searchParams.delete("state"); + searchParams.delete("code"); + searchParams.delete("session_state"); + const newUrl = searchParams.toString().length + ? `${window.location.pathname}?${searchParams.toString()}` + : window.location.pathname; + window.history.replaceState({}, document.title, newUrl); }, }; From 118dd2aab2547c1c1884f0a90d04ef050fd79e6b Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Wed, 6 Sep 2023 10:08:18 -0700 Subject: [PATCH 115/201] Keycloak users default to instance_admin until we create an Airbyte User for them (#8705) --- .../server/pro/KeycloakTokenValidator.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java index 2b7d5933dd6..b9e67a8df32 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java +++ b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java @@ -49,7 +49,7 @@ @Slf4j @Singleton @RequiresAirbyteProEnabled -@SuppressWarnings({"PMD.PreserveStackTrace", "PMD.UseTryWithResources"}) +@SuppressWarnings({"PMD.PreserveStackTrace", "PMD.UseTryWithResources", "PMD.UnusedFormalParameter", "PMD.UnusedPrivateMethod"}) public class KeycloakTokenValidator implements TokenValidator { private final HttpClient client; @@ -101,7 +101,10 @@ private Authentication getAuthentication(final String token, final HttpRequest roles = getRoles(userId, request); + // For now, give all valid Keycloak users instance_admin, until we actually create an Airbyte User + // with permissions for such logins. + final Collection roles = getInstanceAdminRoles(); + // final Collection roles = getRoles(userId, request); log.debug("Authenticating user '{}' with roles {}...", userId, roles); return Authentication.build(userId, roles); } else { @@ -113,10 +116,10 @@ private Authentication getAuthentication(final String token, final HttpRequest getInstanceAdminRoles() { + return AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + } + private Collection getRoles(final String userId, final HttpRequest request) { Map headerMap = request.getHeaders().asMap(String.class, String.class); From 336ff2610be644cd9d608cb91eda05d77d0e7229 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Wed, 6 Sep 2023 10:57:19 -0700 Subject: [PATCH 116/201] Add connectionId to ShouldFailSyncIfHeartbeatFailure context (#8708) --- .../airbyte/workers/internal/HeartbeatTimeoutChaperone.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java index 065f38fd36b..ba00e9d974c 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/internal/HeartbeatTimeoutChaperone.java @@ -7,7 +7,9 @@ import static java.lang.Thread.sleep; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.featureflag.Connection; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.Multi; import io.airbyte.featureflag.ShouldFailSyncIfHeartbeatFailure; import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; @@ -15,6 +17,7 @@ import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import java.time.Duration; +import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -112,7 +115,8 @@ public void runWithHeartbeatThread(final CompletableFuture runnableFuture) LOGGER.info("thread status... heartbeat thread: {} , replication thread: {}", heartbeatFuture.isDone(), runnableFuture.isDone()); if (heartbeatFuture.isDone() && !runnableFuture.isDone()) { - if (featureFlagClient.boolVariation(ShouldFailSyncIfHeartbeatFailure.INSTANCE, new Workspace(workspaceId))) { + if (featureFlagClient.boolVariation(ShouldFailSyncIfHeartbeatFailure.INSTANCE, + new Multi(List.of(new Workspace(workspaceId), new Connection(connectionId))))) { runnableFuture.cancel(true); throw new HeartbeatTimeoutException( String.format("Heartbeat has stopped. Heartbeat freshness threshold: %s secs Actual heartbeat age: %s secs", From 9664e4926c58ffb520bb0182662dade0b467c6e3 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Wed, 6 Sep 2023 12:44:26 -0700 Subject: [PATCH 117/201] Include org related roles in admin; also fix unit test (#8711) --- .../java/io/airbyte/server/pro/KeycloakTokenValidator.java | 5 ++++- .../java/io/airbyte/server/KeycloakTokenValidatorTest.java | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java index b9e67a8df32..586bedddf3a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java +++ b/airbyte-server/src/main/java/io/airbyte/server/pro/KeycloakTokenValidator.java @@ -117,7 +117,10 @@ private Authentication getAuthentication(final String token, final HttpRequest getInstanceAdminRoles() { - return AuthRole.buildAuthRolesSet(AuthRole.ADMIN); + Set roles = new HashSet<>(); + roles.addAll(AuthRole.buildAuthRolesSet(AuthRole.ADMIN)); + roles.addAll(OrganizationAuthRole.buildOrganizationAuthRolesSet(OrganizationAuthRole.ORGANIZATION_ADMIN)); + return roles; } private Collection getRoles(final String userId, final HttpRequest request) { diff --git a/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java b/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java index 995de69fe72..54d544553e4 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/KeycloakTokenValidatorTest.java @@ -116,7 +116,9 @@ void testValidateToken() throws Exception { when(permissionPersistence.findPermissionTypeForUserAndOrganization(ORGANIZATION_ID, expectedUserId, AuthProvider.KEYCLOAK)) .thenReturn(PermissionType.ORGANIZATION_READER); - Set expectedResult = Set.of("ORGANIZATION_READER", "ADMIN", "EDITOR", "READER"); + // TODO: enable this once we are ready to enable getRoles function in KeycloakTokenValidator. + // Set expectedResult = Set.of("ORGANIZATION_READER", "ADMIN", "EDITOR", "READER"); + Set expectedResult = Set.of("ORGANIZATION_ADMIN", "ORGANIZATION_EDITOR", "ORGANIZATION_READER", "ADMIN", "EDITOR", "READER"); StepVerifier.create(responsePublisher) .expectNextMatches(r -> matchSuccessfulResponse(r, expectedUserId, expectedResult)) From 473e93b55a1694db9a927c5ee58a0886b480a59b Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 6 Sep 2023 16:17:33 -0400 Subject: [PATCH 118/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8F=81=20(flagged)?= =?UTF-8?q?=20Deprecate=20free=20connector=20program=20UI=20(#8612)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Vladimir --- .../ReleaseStageBadge/ReleaseStageBadge.tsx | 4 +- .../api/hooks/cloud/freeConnectorProgram.ts | 18 + .../hooks/services/Experiment/experiments.ts | 1 + .../src/packages/cloud/locales/en.json | 4 +- .../BillingPage/components/CreditsUsage.tsx | 4 +- .../components/CreditsUsageContext.tsx | 9 +- .../components/RemainingCredits.tsx | 10 +- .../components/UsagePerDayGraph.tsx | 11 +- .../useBillingPageBanners.test.tsx | 331 +++++------------- .../BillingPage/useBillingPageBanners.tsx | 13 +- .../useDeprecatedBillingPageBanners.tsx | 53 +++ 11 files changed, 184 insertions(+), 274 deletions(-) create mode 100644 airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useDeprecatedBillingPageBanners.tsx diff --git a/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx b/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx index 3b48c6f1ed6..3b7e89f2d90 100644 --- a/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx +++ b/airbyte-webapp/src/components/ReleaseStageBadge/ReleaseStageBadge.tsx @@ -4,8 +4,8 @@ import { FormattedMessage } from "react-intl"; import { GAIcon } from "components/icons/GAIcon"; import { Tooltip } from "components/ui/Tooltip"; +import { useIsFCPEnabled } from "core/api/cloud"; import { ReleaseStage } from "core/request/AirbyteClient"; -import { FeatureItem, useFeature } from "core/services/features"; import { FreeTag } from "packages/cloud/components/experiments/FreeConnectorProgram"; import styles from "./ReleaseStageBadge.module.scss"; @@ -20,7 +20,7 @@ interface ReleaseStageBadgeProps { } export const ReleaseStageBadge: React.FC = ({ stage, small, tooltip = true }) => { - const fcpEnabled = useFeature(FeatureItem.FreeConnectorProgram); + const fcpEnabled = useIsFCPEnabled(); if (!stage) { return null; diff --git a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts index dd1c4ab90cd..1ca7ea6bdd4 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/freeConnectorProgram.ts @@ -2,15 +2,27 @@ import { useQuery } from "@tanstack/react-query"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { FeatureItem, useFeature } from "core/services/features"; +import { useExperiment } from "hooks/services/Experiment"; import { useDefaultRequestMiddlewares } from "services/useDefaultRequestMiddlewares"; import { webBackendGetFreeConnectorProgramInfoForWorkspace } from "../../generated/CloudApi"; +/** + * @deprecated this hook will be removed after sunsetting the fcp + * + * do not use + */ + export const useFreeConnectorProgram = () => { const workspaceId = useCurrentWorkspaceId(); const middlewares = useDefaultRequestMiddlewares(); const requestOptions = { middlewares }; const freeConnectorProgramEnabled = useFeature(FeatureItem.FreeConnectorProgram); + const fcpSunsetPlatform = useExperiment("platform.sunset-fcp", false); + + if (fcpSunsetPlatform) { + throw new Error("FCP is sunset on platform"); + } const programStatusQuery = useQuery(["freeConnectorProgramInfo", workspaceId], () => webBackendGetFreeConnectorProgramInfoForWorkspace({ workspaceId }, requestOptions).then( @@ -28,3 +40,9 @@ export const useFreeConnectorProgram = () => { programStatusQuery, }; }; + +export const useIsFCPEnabled = () => { + const frontendFCPEnabled = useFeature(FeatureItem.FreeConnectorProgram); + const platformFCPSunset = useExperiment("platform.sunset-fcp", false); + return frontendFCPEnabled && !platformFCPSunset; +}; diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts index bafabc0ce0a..d8e68ae78da 100644 --- a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts +++ b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts @@ -35,5 +35,6 @@ export interface Experiments { "upcomingFeaturesPage.url": string; "workspaces.newWorkspacesUI": boolean; "settings.accessManagement": boolean; + "platform.sunset-fcp": boolean; "form.createConnectionHookForm": boolean; } diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json index 6ec407b2e19..4e34f2e219c 100644 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ b/airbyte-webapp/src/packages/cloud/locales/en.json @@ -156,7 +156,7 @@ "credits.source": "Source", "credits.destination": "Destination", "credits.schedule": "Schedule", - "credits.freeUsage": "Free Connector Program", + "credits.freeUsage": "Free", "credits.billedCost": "Billed", "credits.totalUsage": "Total credits usage", "credits.stripePortalLink": "Invoice history", @@ -193,8 +193,6 @@ "sidebar.credits": "Credits", "sidebar.billing": "Billing", - - "freeConnectorProgram.title": "Free Connector Program", "freeConnectorProgram.releaseStageBadge.free": "Free", "experiment.speedyConnection": "Set up your first connection in the next and get 100 additional credits for your trial", diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsage.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsage.tsx index ec596ecf63a..232a5449e0b 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsage.tsx @@ -13,7 +13,7 @@ import { UsagePerConnectionTable } from "./UsagePerConnectionTable"; import { UsagePerDayGraph } from "./UsagePerDayGraph"; export const CreditsUsage: React.FC = () => { - const { freeAndPaidUsageByTimeChunk } = useCreditsContext(); + const { freeAndPaidUsageByTimeChunk, hasFreeUsage } = useCreditsContext(); return ( @@ -28,7 +28,7 @@ export const CreditsUsage: React.FC = () => { - + diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx index b1581c632a2..544bdaabd58 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx @@ -44,6 +44,7 @@ interface CreditsUsageContext { setSelectedDestination: Dispatch>; selectedTimeWindow: ConsumptionTimeWindow; setSelectedTimeWindow: Dispatch>; + hasFreeUsage: boolean; } export const creditsUsageContext = createContext(null); @@ -51,13 +52,14 @@ export const creditsUsageContext = createContext(nul export const useCreditsContext = (): CreditsUsageContext => { const creditsUsageHelpers = useContext(creditsUsageContext); if (!creditsUsageHelpers) { - throw new Error("useConnectorForm should be used within ConnectorFormContextProvider"); + throw new Error("useCreditsContext should be used within CreditsUsageContextProvider"); } return creditsUsageHelpers; }; export const CreditsUsageContextProvider: React.FC> = ({ children }) => { const [selectedTimeWindow, setSelectedTimeWindow] = useState(ConsumptionTimeWindow.lastMonth); + const [hasFreeUsage, setHasFreeUsage] = useState(false); const { workspaceId } = useCurrentWorkspace(); const data = useGetCloudWorkspaceUsage(workspaceId, selectedTimeWindow); @@ -66,6 +68,10 @@ export const CreditsUsageContextProvider: React.FC { return consumptionPerConnectionPerTimeframe.map((consumption) => { + if (consumption.freeUsage > 0) { + setHasFreeUsage(true); + } + return { ...consumption, startTime: dayjs(consumption.startTime).format("YYYY-MM-DD"), @@ -133,6 +139,7 @@ export const CreditsUsageContextProvider: React.FC {children} diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx index 2088ddab083..c7ad0d522f8 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/RemainingCredits.tsx @@ -13,7 +13,7 @@ import { FlexContainer, FlexItem } from "components/ui/Flex"; import { ExternalLink } from "components/ui/Link"; import { Text } from "components/ui/Text"; -import { useStripeCheckout } from "core/api/cloud"; +import { useIsFCPEnabled, useStripeCheckout } from "core/api/cloud"; import { useGetCloudWorkspace, useInvalidateCloudWorkspace } from "core/api/cloud"; import { CloudWorkspaceRead } from "core/api/types/CloudApi"; import { Action, Namespace } from "core/services/analytics"; @@ -26,6 +26,7 @@ import { EmailVerificationHint } from "./EmailVerificationHint"; import { LowCreditBalanceHint } from "./LowCreditBalanceHint"; import styles from "./RemainingCredits.module.scss"; import { useBillingPageBanners } from "../useBillingPageBanners"; +import { useDeprecatedBillingPageBanners } from "../useDeprecatedBillingPageBanners"; const STRIPE_SUCCESS_QUERY = "stripeCheckoutSuccess"; @@ -47,7 +48,12 @@ export const RemainingCredits: React.FC = () => { const { isLoading, mutateAsync: createCheckout } = useStripeCheckout(); const analytics = useAnalyticsService(); const [isWaitingForCredits, setIsWaitingForCredits] = useState(false); - const { bannerVariant } = useBillingPageBanners(); + const billingPageBannerHook = useBillingPageBanners; + const deprecatedBillingPageBannerHook = useDeprecatedBillingPageBanners; + + const isFCPEnabled = useIsFCPEnabled(); + const { bannerVariant } = isFCPEnabled ? deprecatedBillingPageBannerHook() : billingPageBannerHook(); + const { emailVerified } = useAuthService(); useEffectOnce(() => { diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx index 1e293fe94e1..a53a31cbcd1 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx @@ -16,8 +16,6 @@ import { import { Text } from "components/ui/Text"; -import { useFreeConnectorProgram } from "core/api/cloud"; - import { UsagePerTimeChunk } from "./calculateUsageDataObjects"; import { FormattedCredits } from "./FormattedCredits"; import styles from "./UsagePerDayGraph.module.scss"; @@ -25,12 +23,9 @@ import styles from "./UsagePerDayGraph.module.scss"; interface UsagePerDayGraphProps { chartData: UsagePerTimeChunk; minimized?: boolean; + hasFreeUsage?: boolean; } -export const UsagePerDayGraph: React.FC = ({ chartData, minimized }) => { - const { - programStatusQuery: { data: freeConnectorEnrollment }, - } = useFreeConnectorProgram(); - const isEnrolledInFreeConnectorProgram = freeConnectorEnrollment?.isEnrolled; +export const UsagePerDayGraph: React.FC = ({ chartData, minimized, hasFreeUsage }) => { const { formatMessage } = useIntl(); const chartLinesColor = styles.grey100; const chartTicksColor = styles.grey; @@ -129,7 +124,7 @@ export const UsagePerDayGraph: React.FC = ({ chartData, m ); })} - {isEnrolledInFreeConnectorProgram && ( + {hasFreeUsage && ( >; -jest.mock("packages/cloud/components/experiments/FreeConnectorProgram"); -const mockUseFreeConnectorProgram = useFreeConnectorProgram as unknown as jest.Mock< - Partial ->; - jest.mock("core/api"); const mockGetCurrentWorkspace = useCurrentWorkspace as unknown as jest.Mock>; jest.mock("core/api/cloud"); const mockUseGetCloudWorkspace = useGetCloudWorkspace as unknown as jest.Mock>; -const mockHooks = ( - workspaceTrialStatus: WorkspaceTrialStatus, - remainingCredits: number, - hasEligibleConnections: boolean, - hasNonEligibleConnections: boolean, - isEnrolled: boolean -) => { +const mockHooks = (workspaceTrialStatus: WorkspaceTrialStatus, remainingCredits: number) => { mockGetCurrentWorkspace.mockImplementation(() => { return { workspaceId: "1234" }; }); @@ -40,15 +29,6 @@ const mockHooks = ( remainingCredits, }; }); - - mockUseFreeConnectorProgram.mockImplementation(() => { - return { - userDidEnroll: false, - programStatusQuery: { - data: { isEnrolled, showEnrollmentUi: true, hasEligibleConnections, hasNonEligibleConnections }, - }, - }; - }); }; describe("useBillingPageBanners", () => { @@ -65,277 +45,134 @@ describe("useBillingPageBanners", () => { ExperimentProvider: ({ children }: React.PropsWithChildren) => <>{children}, }; }); - describe("enrolled in FCP", () => { - it("should show an info variant banner", () => { - mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - }); - describe("not enrolled in FCP", () => { - it("should an info variant banner", () => { - mockHooks(WorkspaceTrialStatus.pre_trial, 0, false, false, false); + it("should show an info variant banner", () => { + mockHooks(WorkspaceTrialStatus.pre_trial, 0); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("info"); }); }); describe("out of trial with 0 connections enabled", () => { - describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banners if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, true); + it("should show info banner if > 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 100); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); - - it("should show error banner if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("info"); }); - describe("not enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, false, false); + it("should show warning banners if < 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 5); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banners if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 5, false, false, false); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("warning"); + }); - it("should show error banners if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, false, false); + it("should show error banner if 0 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 0); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("error"); }); }); describe("out of trial with only eligible connections enabled", () => { - describe("enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, true); + it("should show info variant banner if > 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 100); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show error variant banner if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show error variant banner if user is in 0 credits state", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("info"); }); - describe("not enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, false, false); + it("should show error variant banner if < 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 10); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show FCP banner + warning variant banner if user has < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, false, false); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show FCP banner + error variant banner if user has 0 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, false, false); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("warning"); + }); + it("should show error variant banner if user is in 0 credits state", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 0); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("error"); }); }); describe("out of trial with only non-eligible connections enabled", () => { - describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, true); + it("should show info banner if > 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 100); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banner if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); - it("should show error banner if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("info"); }); - describe("not enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, false, true, false); + it("should show warning banner if < 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 10); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banner if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 10, false, true, false); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); - it("should show error banner if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, false, true, false); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("warning"); + }); + it("should show error banner if 0 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 0); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("error"); }); }); describe("out of trial with eligible and non-eligible connections enabled", () => { - describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, true); + it("should show info banner if > 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 100); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banner if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); - it("should show error banner if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("info"); }); - describe("not enrolled in FCP", () => { - it("should show info banner if more than 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 100, true, true, false); + it("should show warning banner if < 20 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 10); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banner if fewer than 20 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 10, true, true, false); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); - it("should show error banner if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.out_of_trial, 0, true, true, false); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("warning"); + }); + it("should show error banner if 0 credits", () => { + mockHooks(WorkspaceTrialStatus.out_of_trial, 0); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("error"); }); }); describe("credit purchased with only eligible connections enabled", () => { - describe("enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, true); + it("should show info variant banner if > 20 credits", () => { + mockHooks(WorkspaceTrialStatus.credit_purchased, 100); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show info variant banner if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show info variant banner if user is in 0 credits state", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("info"); }); - describe("not enrolled in FCP", () => { - it("should show info variant banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 100, true, false, false); + it("should show info variant banner if < 20 credits", () => { + mockHooks(WorkspaceTrialStatus.credit_purchased, 10); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show info variant banner if user has < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 10, true, false, false); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show error variant banner if user has 0 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 0, true, false, false); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("warning"); + }); + it("should show info variant banner if user is in 0 credits state", () => { + mockHooks(WorkspaceTrialStatus.credit_purchased, 0); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("error"); }); }); describe("credit purchased with only non-eligible connections enabled", () => { - describe("enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, true); + it("should show info banner if > 20 credits", () => { + mockHooks(WorkspaceTrialStatus.credit_purchased, 100); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banner if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); - it("should show error banner if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, true); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("info"); }); - describe("not enrolled in FCP", () => { - it("should show info banner if > 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 100, false, true, false); + it("should show warning banner if < 20 credits", () => { + mockHooks(WorkspaceTrialStatus.credit_purchased, 10); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("info"); - }); - it("should show warning banner if < 20 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 10, false, true, false); - - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("warning"); - }); - it("should show error banner if 0 credits", () => { - mockHooks(WorkspaceTrialStatus.credit_purchased, 0, false, true, false); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("warning"); + }); + it("should show error banner if 0 credits", () => { + mockHooks(WorkspaceTrialStatus.credit_purchased, 0); - const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); - expect(result.current.bannerVariant).toEqual("error"); - }); + const { result } = renderHook(() => useBillingPageBanners(), { wrapper: TestWrapper }); + expect(result.current.bannerVariant).toEqual("error"); }); }); }); diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx index 2575e9abcd2..c9188eee735 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useBillingPageBanners.tsx @@ -1,5 +1,5 @@ import { useCurrentWorkspace } from "core/api"; -import { useFreeConnectorProgram, useGetCloudWorkspace } from "core/api/cloud"; +import { useGetCloudWorkspace } from "core/api/cloud"; import { CloudWorkspaceReadWorkspaceTrialStatus as WorkspaceTrialStatus } from "core/api/types/CloudApi"; import { useExperiment } from "hooks/services/Experiment"; @@ -8,8 +8,7 @@ import { LOW_BALANCE_CREDIT_THRESHOLD } from "./components/LowCreditBalanceHint/ export const useBillingPageBanners = () => { const currentWorkspace = useCurrentWorkspace(); const cloudWorkspace = useGetCloudWorkspace(currentWorkspace.workspaceId); - const { programStatusQuery } = useFreeConnectorProgram(); - const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled } = programStatusQuery.data || {}; + const isNewTrialPolicyEnabled = useExperiment("billing.newTrialPolicy", false); const isPreTrial = isNewTrialPolicyEnabled @@ -24,15 +23,11 @@ export const useBillingPageBanners = () => { : "positive"; const calculateVariant = (): "warning" | "error" | "info" => { - if (creditStatus === "low" && (hasNonEligibleConnections || !hasEligibleConnections)) { + if (creditStatus === "low") { return "warning"; } - if ( - creditStatus === "zero" && - !isPreTrial && - (hasNonEligibleConnections || !hasEligibleConnections || (hasEligibleConnections && !isEnrolled)) - ) { + if (creditStatus === "zero" && !isPreTrial) { return "error"; } diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useDeprecatedBillingPageBanners.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useDeprecatedBillingPageBanners.tsx new file mode 100644 index 00000000000..6d8844996e5 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/useDeprecatedBillingPageBanners.tsx @@ -0,0 +1,53 @@ +import { useCurrentWorkspace } from "core/api"; +import { useFreeConnectorProgram, useGetCloudWorkspace } from "core/api/cloud"; +import { CloudWorkspaceReadWorkspaceTrialStatus as WorkspaceTrialStatus } from "core/api/types/CloudApi"; +import { useExperiment } from "hooks/services/Experiment"; + +import { LOW_BALANCE_CREDIT_THRESHOLD } from "./components/LowCreditBalanceHint/LowCreditBalanceHint"; + +/** + * @deprecated this hook can and should be removed after the sunsetting of the free connector program + * + * it will cause the app to throw if called after the `platform.sunset-fcp` flag is toggled on sept 18 + * + * it is currently in use ONLY in RemainingCredits.tsx conditionally (if that ^ flag is false) + */ +export const useDeprecatedBillingPageBanners = () => { + const currentWorkspace = useCurrentWorkspace(); + const cloudWorkspace = useGetCloudWorkspace(currentWorkspace.workspaceId); + + const { programStatusQuery } = useFreeConnectorProgram(); + const { hasEligibleConnections, hasNonEligibleConnections, isEnrolled } = programStatusQuery.data || {}; + const isNewTrialPolicyEnabled = useExperiment("billing.newTrialPolicy", false); + + const isPreTrial = isNewTrialPolicyEnabled + ? cloudWorkspace.workspaceTrialStatus === WorkspaceTrialStatus.pre_trial + : false; + + const creditStatus = + (cloudWorkspace.remainingCredits ?? 0) < LOW_BALANCE_CREDIT_THRESHOLD + ? (cloudWorkspace.remainingCredits ?? 0) <= 0 + ? "zero" + : "low" + : "positive"; + + const calculateVariant = (): "warning" | "error" | "info" => { + if (creditStatus === "low" && (hasNonEligibleConnections || !hasEligibleConnections)) { + return "warning"; + } + + if ( + creditStatus === "zero" && + !isPreTrial && + (hasNonEligibleConnections || !hasEligibleConnections || (hasEligibleConnections && !isEnrolled)) + ) { + return "error"; + } + + return "info"; + }; + + return { + bannerVariant: calculateVariant(), + }; +}; From 20827b8da71303f4867ccbe1d30f707fd5e8010e Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Wed, 6 Sep 2023 13:40:28 -0700 Subject: [PATCH 119/201] =?UTF-8?q?Make=20MetricClient=20injectable=20in?= =?UTF-8?q?=20orchestrator/server=20=E2=80=93=20take2=20(#8706)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temporal/ConnectionManagerUtils.java | 6 ++-- .../commons/temporal/TemporalClient.java | 7 ++-- .../commons/temporal/TemporalClientTest.java | 9 +++-- .../general/ReplicationWorkerFactory.java | 34 +++++++++---------- .../config/ContainerOrchestratorFactory.java | 9 +++++ .../server/config/ApplicationBeanFactory.java | 9 +++++ 6 files changed, 44 insertions(+), 30 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index 4133b67e23a..e5374ba02e9 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -11,7 +11,6 @@ import io.airbyte.commons.temporal.scheduling.state.WorkflowState; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.temporal.api.common.v1.WorkflowExecution; @@ -38,9 +37,8 @@ public class ConnectionManagerUtils { private final MetricClient metricClient; - public ConnectionManagerUtils() { - // TODO Inject it when MetricClient becomes injectable. - this.metricClient = MetricClientFactory.getMetricClient(); + public ConnectionManagerUtils(final MetricClient metricClient) { + this.metricClient = metricClient; } /** diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index bb7d87f97fd..e8dda943057 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -24,7 +24,6 @@ import io.airbyte.config.persistence.StreamResetPersistence; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -89,7 +88,8 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, final StreamResetPersistence streamResetPersistence, final ConnectionManagerUtils connectionManagerUtils, final NotificationClient notificationClient, - final StreamResetRecordsHelper streamResetRecordsHelper) { + final StreamResetRecordsHelper streamResetRecordsHelper, + final MetricClient metricClient) { this.workspaceRoot = workspaceRoot; this.client = client; this.service = service; @@ -97,8 +97,7 @@ public TemporalClient(@Named("workspaceRootTemporal") final Path workspaceRoot, this.connectionManagerUtils = connectionManagerUtils; this.notificationClient = notificationClient; this.streamResetRecordsHelper = streamResetRecordsHelper; - // TODO Inject it when MetricClient becomes injectable. - this.metricClient = MetricClientFactory.getMetricClient(); + this.metricClient = metricClient; } private final Set workflowNames = new HashSet<>(); diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 01425b6401a..30c2018ea9d 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -39,6 +39,7 @@ import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.persistence.StreamResetPersistence; +import io.airbyte.metrics.lib.MetricClient; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.protocol.models.StreamDescriptor; @@ -122,13 +123,12 @@ void setup() throws IOException { when(workflowServiceStubs.blockingStub()).thenReturn(workflowServiceBlockingStub); streamResetPersistence = mock(StreamResetPersistence.class); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); - connectionManagerUtils = spy(new ConnectionManagerUtils()); + connectionManagerUtils = spy(new ConnectionManagerUtils(mock(MetricClient.class))); notificationClient = spy(new NotificationClient(workflowClient)); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); temporalClient = spy(new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, connectionManagerUtils, - notificationClient, - streamResetRecordsHelper)); + notificationClient, streamResetRecordsHelper, mock(MetricClient.class))); } @Nested @@ -144,8 +144,7 @@ void init() { temporalClient = spy( new TemporalClient(workspaceRoot, workflowClient, workflowServiceStubs, streamResetPersistence, mConnectionManagerUtils, - mNotificationClient, - streamResetRecordsHelper)); + mNotificationClient, streamResetRecordsHelper, mock(MetricClient.class))); } @Test diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java index f1bf177b949..66c96c33607 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/general/ReplicationWorkerFactory.java @@ -29,8 +29,6 @@ import io.airbyte.featureflag.Workspace; import io.airbyte.metrics.lib.MetricAttribute; import io.airbyte.metrics.lib.MetricClient; -import io.airbyte.metrics.lib.MetricClientFactory; -import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.metrics.lib.MetricTags; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; @@ -82,6 +80,7 @@ public class ReplicationWorkerFactory { private final AirbyteMessageDataExtractor airbyteMessageDataExtractor; private final FeatureFlagClient featureFlagClient; private final FeatureFlags featureFlags; + private final MetricClient metricClient; private final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper; public ReplicationWorkerFactory( @@ -93,7 +92,8 @@ public ReplicationWorkerFactory( final SyncPersistenceFactory syncPersistenceFactory, final FeatureFlagClient featureFlagClient, final FeatureFlags featureFlags, - final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper) { + final ReplicationAirbyteMessageEventPublishingHelper replicationAirbyteMessageEventPublishingHelper, + final MetricClient metricClient) { this.airbyteIntegrationLauncherFactory = airbyteIntegrationLauncherFactory; this.sourceApi = sourceApi; this.sourceDefinitionApi = sourceDefinitionApi; @@ -104,6 +104,7 @@ public ReplicationWorkerFactory( this.featureFlagClient = featureFlagClient; this.featureFlags = featureFlags; + this.metricClient = metricClient; } /** @@ -121,7 +122,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, "get the source definition for feature flag checks"); final HeartbeatMonitor heartbeatMonitor = createHeartbeatMonitor(sourceDefinitionId, sourceDefinitionApi); final HeartbeatTimeoutChaperone heartbeatTimeoutChaperone = createHeartbeatTimeoutChaperone(heartbeatMonitor, - featureFlagClient, syncInput); + featureFlagClient, syncInput, metricClient); final RecordSchemaValidator recordSchemaValidator = createRecordSchemaValidator(syncInput); // Enable concurrent stream reads for testing purposes @@ -138,9 +139,6 @@ public ReplicationWorker create(final StandardSyncInput syncInput, final var airbyteDestination = airbyteIntegrationLauncherFactory.createAirbyteDestination(destinationLauncherConfig, syncInput.getSyncResourceRequirements(), syncInput.getCatalog()); - // TODO MetricClient should be injectable (please) - MetricClientFactory.initialize(MetricEmittingApps.WORKER); - final MetricClient metricClient = MetricClientFactory.getMetricClient(); final WorkerMetricReporter metricReporter = new WorkerMetricReporter(metricClient, sourceLauncherConfig.getDockerImage()); final FieldSelector fieldSelector = @@ -153,7 +151,7 @@ public ReplicationWorker create(final StandardSyncInput syncInput, return createReplicationWorker(airbyteSource, airbyteDestination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, heartbeatTimeoutChaperone, featureFlagClient, jobRunConfig, syncInput, airbyteMessageDataExtractor, replicationAirbyteMessageEventPublishingHelper, - onReplicationRunning); + onReplicationRunning, metricClient); } /** @@ -212,13 +210,14 @@ private static HeartbeatMonitor createHeartbeatMonitor(final UUID sourceDefiniti */ private static HeartbeatTimeoutChaperone createHeartbeatTimeoutChaperone(final HeartbeatMonitor heartbeatMonitor, final FeatureFlagClient featureFlagClient, - final StandardSyncInput syncInput) { + final StandardSyncInput syncInput, + final MetricClient metricClient) { return new HeartbeatTimeoutChaperone(heartbeatMonitor, HeartbeatTimeoutChaperone.DEFAULT_TIMEOUT_CHECK_DURATION, featureFlagClient, syncInput.getWorkspaceId(), syncInput.getConnectionId(), - MetricClientFactory.getMetricClient()); + metricClient); } /** @@ -263,7 +262,8 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou final StandardSyncInput syncInput, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper replicationEventPublishingHelper, - final VoidCallable onReplicationRunning) { + final VoidCallable onReplicationRunning, + final MetricClient metricClient) { final Context flagContext = getFeatureFlagContext(syncInput); final String workerImpl = featureFlagClient.stringVariation(ReplicationWorkerImpl.INSTANCE, flagContext); return buildReplicationWorkerInstance( @@ -284,7 +284,8 @@ private static ReplicationWorker createReplicationWorker(final AirbyteSource sou new ReplicationFeatureFlagReader(), airbyteMessageDataExtractor, replicationEventPublishingHelper, - onReplicationRunning); + onReplicationRunning, + metricClient); } private static Context getFeatureFlagContext(final StandardSyncInput syncInput) { @@ -323,16 +324,15 @@ private static ReplicationWorker buildReplicationWorkerInstance(final String wor final ReplicationFeatureFlagReader replicationFeatureFlagReader, final AirbyteMessageDataExtractor airbyteMessageDataExtractor, final ReplicationAirbyteMessageEventPublishingHelper messageEventPublishingHelper, - final VoidCallable onReplicationRunning) { + final VoidCallable onReplicationRunning, + final MetricClient metricClient) { if ("buffered".equals(workerImpl)) { - MetricClientFactory.getMetricClient() - .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, workerImpl)); return new BufferedReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, messageEventPublishingHelper, onReplicationRunning); } else { - MetricClientFactory.getMetricClient() - .count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); + metricClient.count(OssMetricsRegistry.REPLICATION_WORKER_CREATED, 1, new MetricAttribute(MetricTags.IMPLEMENTATION, "default")); return new DefaultReplicationWorker(jobId, attempt, source, mapper, destination, messageTracker, syncPersistence, recordSchemaValidator, fieldSelector, srcHeartbeatTimeoutChaperone, replicationFeatureFlagReader, airbyteMessageDataExtractor, messageEventPublishingHelper, onReplicationRunning); diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java index 7416ac78dce..ce3b98c47c8 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/config/ContainerOrchestratorFactory.java @@ -16,6 +16,9 @@ import io.airbyte.container_orchestrator.orchestrator.NormalizationJobOrchestrator; import io.airbyte.container_orchestrator.orchestrator.ReplicationJobOrchestrator; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.models.JobRunConfig; import io.airbyte.workers.general.ReplicationWorkerFactory; import io.airbyte.workers.internal.state_aggregator.StateAggregatorFactory; @@ -47,6 +50,12 @@ @Factory class ContainerOrchestratorFactory { + @Singleton + public MetricClient metricClient() { + MetricClientFactory.initialize(MetricEmittingApps.ORCHESTRATOR); + return MetricClientFactory.getMetricClient(); + } + @Singleton FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java index 9bd931f4e9b..63238e06e4c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/ApplicationBeanFactory.java @@ -19,6 +19,9 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricClientFactory; +import io.airbyte.metrics.lib.MetricEmittingApps; import io.airbyte.persistence.job.DefaultJobCreator; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; @@ -127,6 +130,12 @@ public FeatureFlags featureFlags() { return new EnvVariableFeatureFlags(); } + @Singleton + public MetricClient metricClient() { + MetricClientFactory.initialize(MetricEmittingApps.SERVER); + return MetricClientFactory.getMetricClient(); + } + @Singleton @Named("workspaceRoot") public Path workspaceRoot(@Value("${airbyte.workspace.root}") final String workspaceRoot) { From 5ce8b9921013c89c7d747f5f43458596df36d14b Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Wed, 6 Sep 2023 22:43:01 +0200 Subject: [PATCH 120/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Fix=20pat?= =?UTF-8?q?h=20to=20local=20docs=20during=20development=20(#8687)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/packages/vite-plugins/doc-middleware.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/airbyte-webapp/packages/vite-plugins/doc-middleware.ts b/airbyte-webapp/packages/vite-plugins/doc-middleware.ts index 6d809c34b10..936323312c0 100644 --- a/airbyte-webapp/packages/vite-plugins/doc-middleware.ts +++ b/airbyte-webapp/packages/vite-plugins/doc-middleware.ts @@ -19,10 +19,7 @@ const localDocMiddleware = (docsPath: string): Plugin => { express.static(`${docsPath}/sources/google-ads.md`) as Connect.NextHandleFunction ); // Server assets that can be used during. Related gradle task: :airbyte-webapp:copyDocAssets - server.middlewares.use( - "/docs/.gitbook", - express.static(`${docsPath}/docs/.gitbook`) as Connect.NextHandleFunction - ); + server.middlewares.use("/docs/.gitbook", express.static(`${docsPath}/../.gitbook`) as Connect.NextHandleFunction); }, }; }; From 41a9774c1e77a43c9aba587a10d1ada76f6136fb Mon Sep 17 00:00:00 2001 From: Ben Church Date: Wed, 6 Sep 2023 13:47:50 -0700 Subject: [PATCH 121/201] Add Support Level to Actor Definition Version (#8316) Co-authored-by: Pedro S. Lopez Co-authored-by: Lake Mossman --- airbyte-api/src/main/openapi/config.yaml | 20 + .../io/airbyte/bootloader/BootloaderTest.java | 2 +- .../server/converters/ApiPojoConverters.java | 8 + .../ActorDefinitionVersionHandler.java | 2 + .../ConnectorBuilderProjectsHandler.java | 2 + .../DestinationDefinitionsHandler.java | 2 + .../handlers/SourceDefinitionsHandler.java | 2 + .../helpers/ActorDefinitionHandlerHelper.java | 2 + .../ActorDefinitionVersionHandlerTest.java | 6 + .../ConnectorBuilderProjectsHandlerTest.java | 2 + .../DestinationDefinitionsHandlerTest.java | 19 + .../SourceDefinitionsHandlerTest.java | 52 ++- .../ActorDefinitionHandlerHelperTest.java | 2 + .../helpers/ConnectorRegistryConverters.java | 3 + .../types/ActorDefinitionVersion.yaml | 5 + ...onnectorRegistryDestinationDefinition.yaml | 5 + .../ConnectorRegistrySourceDefinition.yaml | 5 + .../main/resources/types/SupportLevel.yaml | 10 + .../ConnectorRegistryConvertersTest.java | 57 ++- .../config/persistence/ConfigRepository.java | 6 + .../config/persistence/DbConverter.java | 3 + ...finitionBreakingChangePersistenceTest.java | 3 + .../ActorDefinitionPersistenceTest.java | 2 + ...ActorDefinitionVersionPersistenceTest.java | 144 ++++--- .../persistence/ConfigInjectionTest.java | 2 + ...onnectorBuilderProjectPersistenceTest.java | 2 + .../airbyte/config/persistence/MockData.java | 2 + .../StandardSyncPersistenceTest.java | 8 +- .../persistence/WorkspaceFilterTest.java | 8 +- .../persistence/WorkspacePersistenceTest.java | 2 + .../specs/RemoteDefinitionsProviderTest.java | 3 + .../src/test/resources/connector_catalog.json | 370 ++++++++++++------ ...dSupportLevelToActorDefinitionVersion.java | 84 ++++ ...llSupportLevelForActorDefitionVersion.java | 127 ++++++ .../configs_database/schema_dump.txt | 1 + ...portLevelToActorDefinitionVersionTest.java | 96 +++++ ...pportLevelForActorDefitionVersionTest.java | 164 ++++++++ .../reporter/MetricRepositoryTest.java | 8 +- 38 files changed, 1062 insertions(+), 179 deletions(-) create mode 100644 airbyte-config/config-models/src/main/resources/types/SupportLevel.yaml create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersion.java create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.java create mode 100644 airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest.java create mode 100644 airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest.java diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index ec108c806b3..aefcf336ff4 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -4201,6 +4201,12 @@ components: protocolVersion: description: The Airbyte Protocol version supported by the connector type: string + custom: + description: Whether the connector is custom or not + type: boolean + default: false + supportLevel: + $ref: "#/components/schemas/SupportLevel" releaseStage: $ref: "#/components/schemas/ReleaseStage" releaseDate: @@ -4678,6 +4684,12 @@ components: protocolVersion: description: The Airbyte Protocol version supported by the connector type: string + custom: + description: Whether the connector is custom or not + type: boolean + default: false + supportLevel: + $ref: "#/components/schemas/SupportLevel" releaseStage: $ref: "#/components/schemas/ReleaseStage" releaseDate: @@ -4938,6 +4950,8 @@ components: $ref: "#/components/schemas/NormalizationDestinationDefinitionConfig" isOverrideApplied: type: boolean + supportLevel: + $ref: "#/components/schemas/SupportLevel" supportState: $ref: "#/components/schemas/SupportState" breakingChanges: @@ -4987,6 +5001,12 @@ components: - beta - generally_available - custom + SupportLevel: + type: string + enum: + - community + - certified + - none # CONNECTION ConnectionId: type: string diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 300d558e952..7db72267bad 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -76,7 +76,7 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.23.002"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.23.004"; private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.001"; private static final String CDK_VERSION = "1.2.3"; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java index cb784d74f27..1b465164f62 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/ApiPojoConverters.java @@ -22,6 +22,7 @@ import io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig; import io.airbyte.api.model.generated.ReleaseStage; import io.airbyte.api.model.generated.ResourceRequirements; +import io.airbyte.api.model.generated.SupportLevel; import io.airbyte.api.model.generated.SupportState; import io.airbyte.commons.converters.StateConverter; import io.airbyte.commons.enums.Enums; @@ -234,6 +235,13 @@ public static ReleaseStage toApiReleaseStage(final io.airbyte.config.ReleaseStag return ReleaseStage.fromValue(releaseStage.value()); } + public static SupportLevel toApiSupportLevel(final io.airbyte.config.SupportLevel supportLevel) { + if (supportLevel == null) { + return SupportLevel.NONE; + } + return SupportLevel.fromValue(supportLevel.value()); + } + public static SupportState toApiSupportState(final io.airbyte.config.ActorDefinitionVersion.SupportState supportState) { if (supportState == null) { return null; diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java index a57bbd815d3..314b7722a91 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandler.java @@ -4,6 +4,7 @@ package io.airbyte.commons.server.handlers; +import static io.airbyte.commons.server.converters.ApiPojoConverters.toApiSupportLevel; import static io.airbyte.commons.server.converters.ApiPojoConverters.toApiSupportState; import com.google.common.annotations.VisibleForTesting; @@ -87,6 +88,7 @@ ActorDefinitionVersionRead createActorDefinitionVersionRead(final ActorDefinitio .supportsDbt(Objects.requireNonNullElse(actorDefinitionVersion.getSupportsDbt(), false)) .normalizationConfig(ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(actorDefinitionVersion.getNormalizationConfig())) .supportState(toApiSupportState(actorDefinitionVersion.getSupportState())) + .supportLevel(toApiSupportLevel(actorDefinitionVersion.getSupportLevel())) .isOverrideApplied(versionWithOverrideStatus.isOverrideApplied()); final List breakingChanges = configRepository.listBreakingChangesForActorDefinitionVersion(actorDefinitionVersion); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java index 910783ebf5f..bf5b2c5ea5d 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java @@ -28,6 +28,7 @@ import io.airbyte.config.ScopeType; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.init.CdkVersionProvider; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -220,6 +221,7 @@ private UUID createActorDefinition(final String name, final UUID workspaceId, fi .withSpec(connectorSpecification) .withProtocolVersion(DEFAULT_AIRBYTE_PROTOCOL_VERSION.serialize()) .withReleaseStage(ReleaseStage.CUSTOM) + .withSupportLevel(SupportLevel.NONE) .withDocumentationUrl(connectorSpecification.getDocumentationUrl().toString()); configRepository.writeCustomSourceDefinitionAndDefaultVersion(source, defaultVersion, workspaceId, ScopeType.WORKSPACE); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java index 57c24398710..53b3d15b339 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java @@ -104,8 +104,10 @@ static DestinationDefinitionRead buildDestinationDefinitionRead(final StandardDe .documentationUrl(new URI(destinationVersion.getDocumentationUrl())) .icon(loadIcon(standardDestinationDefinition.getIcon())) .protocolVersion(destinationVersion.getProtocolVersion()) + .supportLevel(ApiPojoConverters.toApiSupportLevel(destinationVersion.getSupportLevel())) .releaseStage(ApiPojoConverters.toApiReleaseStage(destinationVersion.getReleaseStage())) .releaseDate(ApiPojoConverters.toLocalDate(destinationVersion.getReleaseDate())) + .custom(standardDestinationDefinition.getCustom()) .supportsDbt(Objects.requireNonNullElse(destinationVersion.getSupportsDbt(), false)) .normalizationConfig( ApiPojoConverters.normalizationDestinationDefinitionConfigToApi(destinationVersion.getNormalizationConfig())) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java index aed724d27f5..037b57c945f 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java @@ -104,8 +104,10 @@ static SourceDefinitionRead buildSourceDefinitionRead(final StandardSourceDefini .documentationUrl(new URI(sourceVersion.getDocumentationUrl())) .icon(loadIcon(standardSourceDefinition.getIcon())) .protocolVersion(sourceVersion.getProtocolVersion()) + .supportLevel(ApiPojoConverters.toApiSupportLevel(sourceVersion.getSupportLevel())) .releaseStage(ApiPojoConverters.toApiReleaseStage(sourceVersion.getReleaseStage())) .releaseDate(ApiPojoConverters.toLocalDate(sourceVersion.getReleaseDate())) + .custom(standardSourceDefinition.getCustom()) .resourceRequirements(ApiPojoConverters.actorDefResourceReqsToApi(standardSourceDefinition.getResourceRequirements())) .maxSecondsBetweenMessages(standardSourceDefinition.getMaxSecondsBetweenMessages()); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java index 03305779cfa..4c3996a0692 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelper.java @@ -83,6 +83,7 @@ public ActorDefinitionVersion defaultDefinitionVersionFromCreate(final String do .withSpec(spec) .withDocumentationUrl(documentationUrl.toString()) .withProtocolVersion(protocolVersion) + .withSupportLevel(io.airbyte.config.SupportLevel.NONE) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM); } @@ -131,6 +132,7 @@ public ActorDefinitionVersion defaultDefinitionVersionFromUpdate(final ActorDefi .withProtocolVersion(protocolVersion) .withReleaseStage(currentVersion.getReleaseStage()) .withReleaseDate(currentVersion.getReleaseDate()) + .withSupportLevel(currentVersion.getSupportLevel()) .withNormalizationConfig(currentVersion.getNormalizationConfig()) .withSupportsDbt(currentVersion.getSupportsDbt()) .withAllowedHosts(currentVersion.getAllowedHosts()); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java index 2262f87b9e4..40c18a58923 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ActorDefinitionVersionHandlerTest.java @@ -26,6 +26,7 @@ import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.config.persistence.ActorDefinitionVersionHelper; import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; import io.airbyte.config.persistence.ConfigNotFoundException; @@ -65,6 +66,7 @@ private ActorDefinitionVersion createActorDefinitionVersion() { return new ActorDefinitionVersion() .withActorDefinitionId(ACTOR_DEFINITION_ID) .withVersionId(UUID.randomUUID()) + .withSupportLevel(SupportLevel.NONE) .withReleaseStage(ReleaseStage.BETA) .withSupportState(SupportState.SUPPORTED) .withDockerRepository("airbyte/source-faker") @@ -103,6 +105,7 @@ void testGetActorDefinitionVersionForSource(final boolean isOverrideApplied) actorDefinitionVersionHandler.getActorDefinitionVersionForSourceId(sourceIdRequestBody); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() .isOverrideApplied(isOverrideApplied) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) @@ -140,6 +143,7 @@ void testGetActorDefinitionVersionForDestination(final boolean isOverrideApplied actorDefinitionVersionHandler.getActorDefinitionVersionForDestinationId(destinationIdRequestBody); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() .isOverrideApplied(isOverrideApplied) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) @@ -177,6 +181,7 @@ void testGetActorDefinitionVersionForDestinationWithNormalization(final boolean actorDefinitionVersionHandler.getActorDefinitionVersionForDestinationId(destinationIdRequestBody); final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() .isOverrideApplied(isOverrideApplied) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) .supportState(io.airbyte.api.model.generated.SupportState.SUPPORTED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) @@ -218,6 +223,7 @@ void testCreateActorDefinitionVersionReadWithBreakingChange() throws IOException final ActorDefinitionVersionRead expectedRead = new ActorDefinitionVersionRead() .isOverrideApplied(false) + .supportLevel(io.airbyte.api.model.generated.SupportLevel.NONE) .supportState(io.airbyte.api.model.generated.SupportState.DEPRECATED) .dockerRepository(actorDefinitionVersion.getDockerRepository()) .dockerImageTag(actorDefinitionVersion.getDockerImageTag()) diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java index 9d554044d5c..6d6793164ef 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java @@ -40,6 +40,7 @@ import io.airbyte.config.ScopeType; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.init.CdkVersionProvider; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; @@ -391,6 +392,7 @@ void whenPublishConnectorBuilderProjectThenCreateActorDefinition() throws IOExce .withDockerRepository("airbyte/source-declarative-manifest") .withDockerImageTag(CDK_VERSION) .withSpec(adaptedConnectorSpecification) + .withSupportLevel(SupportLevel.NONE) .withReleaseStage(ReleaseStage.CUSTOM) .withDocumentationUrl(A_DOCUMENTATION_URL) .withProtocolVersion("0.2.0")), diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java index dfc2f9b0d19..1371d7a6de7 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java @@ -34,6 +34,7 @@ import io.airbyte.api.model.generated.PrivateDestinationDefinitionRead; import io.airbyte.api.model.generated.PrivateDestinationDefinitionReadList; import io.airbyte.api.model.generated.ReleaseStage; +import io.airbyte.api.model.generated.SupportLevel; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.errors.IdNotFoundKnownException; @@ -148,6 +149,7 @@ private ActorDefinitionVersion generateVersionFromDestinationDefinition(final St .withDocumentationUrl("https://hulu.com") .withSpec(spec) .withProtocolVersion("0.2.2") + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) @@ -168,6 +170,7 @@ private ActorDefinitionVersion generateCustomVersionFromDestinationDefinition(fi return generateVersionFromDestinationDefinition(destinationDefinition) .withProtocolVersion(DEFAULT_PROTOCOL_VERSION) .withReleaseDate(null) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM) .withAllowedHosts(null); } @@ -199,6 +202,7 @@ void testListDestinations() throws IOException, URISyntaxException { .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -216,6 +220,7 @@ void testListDestinations() throws IOException, URISyntaxException { .documentationUrl(new URI(destinationDefinitionVersionWithNormalization.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) .protocolVersion(destinationDefinitionVersionWithNormalization.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersionWithNormalization.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersionWithNormalization.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersionWithNormalization.getReleaseDate())) .supportsDbt(destinationDefinitionVersionWithNormalization.getSupportsDbt()) @@ -254,6 +259,7 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -271,6 +277,7 @@ void testListDestinationDefinitionsForWorkspace() throws IOException, URISyntaxE .documentationUrl(new URI(destinationDefinitionVersionWithNormalization.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) .protocolVersion(destinationDefinitionVersionWithNormalization.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersionWithNormalization.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersionWithNormalization.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersionWithNormalization.getReleaseDate())) .supportsDbt(destinationDefinitionVersionWithNormalization.getSupportsDbt()) @@ -339,6 +346,7 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -356,6 +364,7 @@ void testListPrivateDestinationDefinitions() throws IOException, URISyntaxExcept .documentationUrl(new URI(destinationDefinitionVersionWithNormalization.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinitionWithNormalization.getIcon())) .protocolVersion(destinationDefinitionVersionWithNormalization.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersionWithNormalization.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersionWithNormalization.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersionWithNormalization.getReleaseDate())) .supportsDbt(destinationDefinitionVersionWithNormalization.getSupportsDbt()) @@ -399,6 +408,7 @@ void testGetDestination() throws JsonValidationException, ConfigNotFoundExceptio .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -469,6 +479,7 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -508,6 +519,7 @@ void testGetDefinitionWithGrantForScope() throws JsonValidationException, Config .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -572,6 +584,8 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept .destinationDefinitionId(newDestinationDefinition.getDestinationDefinitionId()) .icon(DestinationDefinitionsHandler.loadIcon(newDestinationDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.CUSTOM) .supportsDbt(false) .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) @@ -635,6 +649,8 @@ void testCreateCustomDestinationDefinitionUsingScopes() throws URISyntaxExceptio .destinationDefinitionId(newDestinationDefinition.getDestinationDefinitionId()) .icon(DestinationDefinitionsHandler.loadIcon(newDestinationDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.CUSTOM) .supportsDbt(false) .normalizationConfig(new io.airbyte.api.model.generated.NormalizationDestinationDefinitionConfig().supported(false)) @@ -832,6 +848,7 @@ void testGrantDestinationDefinitionToWorkspace() throws JsonValidationException, .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -869,6 +886,7 @@ void testGrantDestinationDefinitionToOrganization() throws JsonValidationExcepti .documentationUrl(new URI(destinationDefinitionVersion.getDocumentationUrl())) .icon(DestinationDefinitionsHandler.loadIcon(destinationDefinition.getIcon())) .protocolVersion(destinationDefinitionVersion.getProtocolVersion()) + .supportLevel(SupportLevel.fromValue(destinationDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(destinationDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(destinationDefinitionVersion.getReleaseDate())) .supportsDbt(false) @@ -926,6 +944,7 @@ void testCorrect() { Jsons.jsonNode(ImmutableMap.of("key", "val")))) .withTombstone(false) .withProtocolVersion("0.2.2") + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java index dc2607815d3..76e5b9edbd7 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java @@ -35,6 +35,7 @@ import io.airbyte.api.model.generated.SourceDefinitionUpdate; import io.airbyte.api.model.generated.SourceRead; import io.airbyte.api.model.generated.SourceReadList; +import io.airbyte.api.model.generated.SupportLevel; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.server.errors.IdNotFoundKnownException; @@ -140,6 +141,7 @@ private ActorDefinitionVersion generateVersionFromSourceDefinition(final Standar .withDockerRepository("dockerstuff") .withDockerImageTag("12.3") .withSpec(spec) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) .withAllowedHosts(new AllowedHosts().withHosts(List.of("host1", "host2"))) @@ -160,6 +162,7 @@ private ActorDefinitionVersion generateCustomVersionFromSourceDefinition(final S return generateVersionFromSourceDefinition(sourceDefinition) .withProtocolVersion(DEFAULT_PROTOCOL_VERSION) .withReleaseDate(null) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM) .withAllowedHosts(null); } @@ -181,6 +184,7 @@ void testListSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -195,6 +199,7 @@ void testListSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion2.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion2.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion2.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion2.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion2.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -228,6 +233,7 @@ void testListSourceDefinitionsForWorkspace() throws IOException, URISyntaxExcept .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -242,6 +248,7 @@ void testListSourceDefinitionsForWorkspace() throws IOException, URISyntaxExcept .dockerImageTag(sourceDefinitionVersion2.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion2.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion2.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion2.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion2.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -303,6 +310,7 @@ void testListPrivateSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -317,6 +325,7 @@ void testListPrivateSourceDefinitions() throws IOException, URISyntaxException { .dockerImageTag(sourceDefinitionVersion2.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion2.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion2.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion2.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion2.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -353,6 +362,7 @@ void testGetSourceDefinition() throws JsonValidationException, ConfigNotFoundExc .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -415,6 +425,7 @@ void testGetDefinitionWithGrantForWorkspace() throws JsonValidationException, Co .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -448,6 +459,7 @@ void testGetDefinitionWithGrantForScope() throws JsonValidationException, Config .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -510,6 +522,8 @@ void testCreateCustomSourceDefinition() throws URISyntaxException, IOException { .sourceDefinitionId(newSourceDefinition.getSourceDefinitionId()) .icon(SourceDefinitionsHandler.loadIcon(newSourceDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.COMMUNITY) .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() @@ -571,6 +585,8 @@ void testCreateCustomSourceDefinitionUsingScopes() throws URISyntaxException, IO .sourceDefinitionId(newSourceDefinition.getSourceDefinitionId()) .icon(SourceDefinitionsHandler.loadIcon(newSourceDefinition.getIcon())) .protocolVersion(DEFAULT_PROTOCOL_VERSION) + .custom(true) + .supportLevel(SupportLevel.COMMUNITY) .releaseStage(ReleaseStage.CUSTOM) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() ._default(new io.airbyte.api.model.generated.ResourceRequirements() @@ -765,6 +781,7 @@ void testGrantSourceDefinitionToWorkspace() throws JsonValidationException, Conf .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -799,6 +816,7 @@ void testGrantSourceDefinitionToOrganization() throws JsonValidationException, C .dockerImageTag(sourceDefinitionVersion.getDockerImageTag()) .documentationUrl(new URI(sourceDefinitionVersion.getDocumentationUrl())) .icon(SourceDefinitionsHandler.loadIcon(sourceDefinition.getIcon())) + .supportLevel(SupportLevel.fromValue(sourceDefinitionVersion.getSupportLevel().value())) .releaseStage(ReleaseStage.fromValue(sourceDefinitionVersion.getReleaseStage().value())) .releaseDate(LocalDate.parse(sourceDefinitionVersion.getReleaseDate())) .resourceRequirements(new io.airbyte.api.model.generated.ActorDefinitionResourceRequirements() @@ -832,6 +850,32 @@ void testRevokeSourceDefinition() throws IOException { verify(configRepository).deleteActorDefinitionWorkspaceGrant(sourceDefinition.getSourceDefinitionId(), organizationId, ScopeType.ORGANIZATION); } + @Test + @DisplayName("should transform support level none to none") + void testNoneSupportLevel() { + final ConnectorRegistrySourceDefinition registrySourceDefinition = new ConnectorRegistrySourceDefinition() + .withSourceDefinitionId(UUID.randomUUID()) + .withName("some-source") + .withDocumentationUrl("https://airbyte.com") + .withDockerRepository("dockerrepo") + .withDockerImageTag("1.2.4") + .withIcon("source.svg") + .withSpec(new ConnectorSpecification().withConnectionSpecification( + Jsons.jsonNode(ImmutableMap.of("key", "val")))) + .withTombstone(false) + .withSupportLevel(io.airbyte.config.SupportLevel.NONE) + .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) + .withReleaseDate(TODAY_DATE_STRING) + .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); + when(remoteDefinitionsProvider.getSourceDefinitions()).thenReturn(Collections.singletonList(registrySourceDefinition)); + + final SourceDefinitionRead expectedRead = + SourceDefinitionsHandler.buildSourceDefinitionRead(ConnectorRegistryConverters.toStandardSourceDefinition(registrySourceDefinition), + ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDefinition)); + + assertEquals(expectedRead.getSupportLevel(), SupportLevel.NONE); + } + @SuppressWarnings("TypeName") @Nested @DisplayName("listLatest") @@ -850,6 +894,7 @@ void testCorrect() { .withSpec(new ConnectorSpecification().withConnectionSpecification( Jsons.jsonNode(ImmutableMap.of("key", "val")))) .withTombstone(false) + .withSupportLevel(io.airbyte.config.SupportLevel.COMMUNITY) .withReleaseStage(io.airbyte.config.ReleaseStage.ALPHA) .withReleaseDate(TODAY_DATE_STRING) .withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(new ResourceRequirements().withCpuRequest("2"))); @@ -859,10 +904,11 @@ void testCorrect() { assertEquals(1, sourceDefinitionReadList.size()); final var sourceDefinitionRead = sourceDefinitionReadList.get(0); - assertEquals( + final SourceDefinitionRead expectedRead = SourceDefinitionsHandler.buildSourceDefinitionRead(ConnectorRegistryConverters.toStandardSourceDefinition(registrySourceDefinition), - ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDefinition)), - sourceDefinitionRead); + ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDefinition)); + + assertEquals(expectedRead, sourceDefinitionRead); } @Test diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java index c27a48a135a..08acf424019 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/helpers/ActorDefinitionHandlerHelperTest.java @@ -30,6 +30,7 @@ import io.airbyte.config.ConnectorRegistrySourceDefinition; import io.airbyte.config.ConnectorReleases; import io.airbyte.config.JobConfig.ConfigType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.VersionBreakingChange; import io.airbyte.config.helpers.ConnectorRegistryConverters; import io.airbyte.config.persistence.ActorDefinitionVersionResolver; @@ -109,6 +110,7 @@ void testDefaultDefinitionVersionFromCreate() throws IOException { .withDockerRepository(DOCKER_REPOSITORY) .withSpec(new ConnectorSpecification().withProtocolVersion(VALID_PROTOCOL_VERSION)) .withDocumentationUrl(DOCUMENTATION_URL.toString()) + .withSupportLevel(SupportLevel.NONE) .withProtocolVersion(VALID_PROTOCOL_VERSION) .withReleaseStage(io.airbyte.config.ReleaseStage.CUSTOM); diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java index 229c149df61..a9583e6b932 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/ConnectorRegistryConverters.java @@ -13,6 +13,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSourceDefinition.SourceType; +import io.airbyte.config.SupportLevel; import io.airbyte.config.VersionBreakingChange; import io.airbyte.protocol.models.ConnectorSpecification; import java.util.Collections; @@ -84,6 +85,7 @@ public static ActorDefinitionVersion toActorDefinitionVersion(@Nullable final Co .withDocumentationUrl(def.getDocumentationUrl()) .withProtocolVersion(getProtocolVersion(def.getSpec())) .withReleaseDate(def.getReleaseDate()) + .withSupportLevel(def.getSupportLevel() == null ? SupportLevel.NONE : def.getSupportLevel()) .withReleaseStage(def.getReleaseStage()) .withSuggestedStreams(def.getSuggestedStreams()); } @@ -108,6 +110,7 @@ public static ActorDefinitionVersion toActorDefinitionVersion(@Nullable final Co .withProtocolVersion(getProtocolVersion(def.getSpec())) .withReleaseDate(def.getReleaseDate()) .withReleaseStage(def.getReleaseStage()) + .withSupportLevel(def.getSupportLevel() == null ? SupportLevel.NONE : def.getSupportLevel()) .withNormalizationConfig(def.getNormalizationConfig()) .withSupportsDbt(def.getSupportsDbt()); } diff --git a/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml b/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml index 933b83ecf04..22f55c4f2f3 100644 --- a/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml +++ b/airbyte-config/config-models/src/main/resources/types/ActorDefinitionVersion.yaml @@ -22,7 +22,12 @@ properties: type: string documentationUrl: type: string + supportLevel: + description: The level of support provided by Airbyte for this connector. + type: string + existingJavaType: io.airbyte.config.SupportLevel releaseStage: + description: Deprecated. Use supportLevel instead. type: string existingJavaType: io.airbyte.config.ReleaseStage releaseDate: diff --git a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml index b43dbf31cc5..26b1f788e7f 100644 --- a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistryDestinationDefinition.yaml @@ -42,7 +42,12 @@ properties: description: whether this is a custom connector definition type: boolean default: false + supportLevel: + description: The level of support provided by Airbyte for this connector. + type: string + existingJavaType: io.airbyte.config.SupportLevel releaseStage: + description: Deprecated. Use supportLevel instead. type: string existingJavaType: io.airbyte.config.ReleaseStage releaseDate: diff --git a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml index 9b27f8ab0bf..b44f6c45c77 100644 --- a/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/ConnectorRegistrySourceDefinition.yaml @@ -49,7 +49,12 @@ properties: description: whether this is a custom connector definition type: boolean default: false + supportLevel: + description: The level of support provided by Airbyte for this connector. + type: string + existingJavaType: io.airbyte.config.SupportLevel releaseStage: + description: Deprecated. Use supportLevel instead. type: string existingJavaType: io.airbyte.config.ReleaseStage releaseDate: diff --git a/airbyte-config/config-models/src/main/resources/types/SupportLevel.yaml b/airbyte-config/config-models/src/main/resources/types/SupportLevel.yaml new file mode 100644 index 00000000000..0afe1d03e83 --- /dev/null +++ b/airbyte-config/config-models/src/main/resources/types/SupportLevel.yaml @@ -0,0 +1,10 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/SupportLevel.yaml +title: SupportLevel +description: enum that describes a connector's support level +type: string +enum: + - community + - certified + - none diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java index 7f6627c7d3b..a63687a5189 100644 --- a/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java +++ b/airbyte-config/config-models/src/test/java/io/airbyte/config/helpers/ConnectorRegistryConvertersTest.java @@ -25,6 +25,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.SuggestedStreams; +import io.airbyte.config.SupportLevel; import io.airbyte.config.VersionBreakingChange; import io.airbyte.protocol.models.ConnectorSpecification; import java.util.Collections; @@ -72,6 +73,7 @@ void testConvertRegistrySourceToInternalTypes() { .withTombstone(false) .withPublic(true) .withCustom(false) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) .withProtocolVersion("doesnt matter") @@ -96,6 +98,7 @@ void testConvertRegistrySourceToInternalTypes() { .withDockerImageTag(DOCKER_TAG) .withSpec(SPEC) .withDocumentationUrl(DOCS_URL) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) .withProtocolVersion(PROTOCOL_VERSION) @@ -107,6 +110,32 @@ void testConvertRegistrySourceToInternalTypes() { assertEquals(expectedBreakingChanges, ConnectorRegistryConverters.toActorDefinitionBreakingChanges(registrySourceDef)); } + @Test + void testConvertRegistrySourceDefaults() { + final SuggestedStreams suggestedStreams = new SuggestedStreams().withStreams(List.of("stream1", "stream2")); + final ConnectorRegistrySourceDefinition registrySourceDef = new ConnectorRegistrySourceDefinition() + .withSourceDefinitionId(DEF_ID) + .withName(CONNECTOR_NAME) + .withDockerRepository(DOCKER_REPOSITORY) + .withDockerImageTag(DOCKER_TAG) + .withDocumentationUrl(DOCS_URL) + .withSpec(SPEC) + .withTombstone(false) + .withPublic(true) + .withCustom(false) + .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) + .withReleaseDate(RELEASE_DATE) + .withProtocolVersion("doesnt matter") + .withAllowedHosts(ALLOWED_HOSTS) + .withResourceRequirements(RESOURCE_REQUIREMENTS) + .withSuggestedStreams(suggestedStreams) + .withMaxSecondsBetweenMessages(10L) + .withReleases(new ConnectorReleases().withBreakingChanges(registryBreakingChanges)); + + final ActorDefinitionVersion convertedAdv = ConnectorRegistryConverters.toActorDefinitionVersion(registrySourceDef); + assertEquals(SupportLevel.NONE, convertedAdv.getSupportLevel()); + } + @Test void testConvertRegistryDestinationToInternalTypes() { final NormalizationDestinationDefinitionConfig normalizationConfig = new NormalizationDestinationDefinitionConfig() @@ -124,9 +153,10 @@ void testConvertRegistryDestinationToInternalTypes() { .withTombstone(false) .withPublic(true) .withCustom(false) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) - .withProtocolVersion("doesnt matter") + .withProtocolVersion(PROTOCOL_VERSION) .withAllowedHosts(ALLOWED_HOSTS) .withResourceRequirements(RESOURCE_REQUIREMENTS) .withNormalizationConfig(normalizationConfig) @@ -147,6 +177,7 @@ void testConvertRegistryDestinationToInternalTypes() { .withDockerImageTag(DOCKER_TAG) .withSpec(SPEC) .withDocumentationUrl(DOCS_URL) + .withSupportLevel(SupportLevel.CERTIFIED) .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) .withReleaseDate(RELEASE_DATE) .withProtocolVersion(PROTOCOL_VERSION) @@ -159,6 +190,30 @@ void testConvertRegistryDestinationToInternalTypes() { assertEquals(expectedBreakingChanges, ConnectorRegistryConverters.toActorDefinitionBreakingChanges(registryDestinationDef)); } + @Test + void testConvertRegistryDestinationDefaults() { + final ConnectorRegistryDestinationDefinition registryDestinationDef = new ConnectorRegistryDestinationDefinition() + .withDestinationDefinitionId(DEF_ID) + .withName(CONNECTOR_NAME) + .withDockerRepository(DOCKER_REPOSITORY) + .withDockerImageTag(DOCKER_TAG) + .withDocumentationUrl(DOCS_URL) + .withSpec(SPEC) + .withTombstone(false) + .withPublic(true) + .withCustom(false) + .withReleaseStage(ReleaseStage.GENERALLY_AVAILABLE) + .withReleaseDate(RELEASE_DATE) + .withProtocolVersion(PROTOCOL_VERSION) + .withAllowedHosts(ALLOWED_HOSTS) + .withResourceRequirements(RESOURCE_REQUIREMENTS) + .withSupportsDbt(true) + .withReleases(new ConnectorReleases().withBreakingChanges(registryBreakingChanges)); + + final ActorDefinitionVersion convertedAdv = ConnectorRegistryConverters.toActorDefinitionVersion(registryDestinationDef); + assertEquals(SupportLevel.NONE, convertedAdv.getSupportLevel()); + } + @Test void testParseSourceDefinitionWithNoBreakingChangesReturnsEmptyList() { ConnectorRegistrySourceDefinition registrySourceDef = new ConnectorRegistrySourceDefinition(); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index c994c9862ec..171c849a4fc 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -77,6 +77,7 @@ import io.airbyte.db.instance.configs.jooq.generated.enums.ReleaseStage; import io.airbyte.db.instance.configs.jooq.generated.enums.ScopeType; import io.airbyte.db.instance.configs.jooq.generated.enums.StatusType; +import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import io.airbyte.db.instance.configs.jooq.generated.enums.SupportState; import io.airbyte.db.instance.configs.jooq.generated.tables.records.ActorDefinitionWorkspaceGrantRecord; import io.airbyte.db.instance.configs.jooq.generated.tables.records.NotificationConfigurationRecord; @@ -3519,6 +3520,7 @@ public ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionV final OffsetDateTime timestamp = OffsetDateTime.now(); // Generate a new UUID if one is not provided. Passing an ID is useful for mocks. final UUID versionId = actorDefinitionVersion.getVersionId() != null ? actorDefinitionVersion.getVersionId() : UUID.randomUUID(); + ctx.insertInto(Tables.ACTOR_DEFINITION_VERSION) .set(Tables.ACTOR_DEFINITION_VERSION.ID, versionId) .set(ACTOR_DEFINITION_VERSION.CREATED_AT, timestamp) @@ -3529,6 +3531,9 @@ public ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionV .set(Tables.ACTOR_DEFINITION_VERSION.SPEC, JSONB.valueOf(Jsons.serialize(actorDefinitionVersion.getSpec()))) .set(Tables.ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL, actorDefinitionVersion.getDocumentationUrl()) .set(Tables.ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION, actorDefinitionVersion.getProtocolVersion()) + .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, actorDefinitionVersion.getSupportLevel() == null ? null + : Enums.toEnum(actorDefinitionVersion.getSupportLevel().value(), + SupportLevel.class).orElseThrow()) .set(Tables.ACTOR_DEFINITION_VERSION.RELEASE_STAGE, actorDefinitionVersion.getReleaseStage() == null ? null : Enums.toEnum(actorDefinitionVersion.getReleaseStage().value(), ReleaseStage.class).orElseThrow()) @@ -3555,6 +3560,7 @@ public ActorDefinitionVersion writeActorDefinitionVersion(final ActorDefinitionV .set(Tables.ACTOR_DEFINITION_VERSION.SUPPORT_STATE, Enums.toEnum(actorDefinitionVersion.getSupportState().value(), SupportState.class).orElseThrow()) .execute(); + return actorDefinitionVersion.withVersionId(versionId); } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java index 3fafe705124..5069935665e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/DbConverter.java @@ -61,6 +61,7 @@ import io.airbyte.config.StandardSync.Status; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.SuggestedStreams; +import io.airbyte.config.SupportLevel; import io.airbyte.config.WorkspaceServiceAccount; import io.airbyte.db.instance.configs.jooq.generated.enums.AutoPropagationStatus; import io.airbyte.db.instance.configs.jooq.generated.enums.NotificationType; @@ -472,6 +473,8 @@ public static ActorDefinitionVersion buildActorDefinitionVersion(final Record re .withDockerImageTag(record.get(ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG)) .withSpec(Jsons.deserialize(record.get(ACTOR_DEFINITION_VERSION.SPEC).data(), ConnectorSpecification.class)) .withDocumentationUrl(record.get(ACTOR_DEFINITION_VERSION.DOCUMENTATION_URL)) + .withSupportLevel(record.get(ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL) == null ? null + : Enums.toEnum(record.get(ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL, String.class), SupportLevel.class).orElseThrow()) .withProtocolVersion(AirbyteProtocolVersion.getWithDefault(record.get(ACTOR_DEFINITION_VERSION.PROTOCOL_VERSION)).serialize()) .withReleaseStage(record.get(ACTOR_DEFINITION_VERSION.RELEASE_STAGE) == null ? null : Enums.toEnum(record.get(ACTOR_DEFINITION_VERSION.RELEASE_STAGE, String.class), ReleaseStage.class).orElseThrow()) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java index 676d0e85d39..269012212ae 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java @@ -13,6 +13,7 @@ import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -71,6 +72,7 @@ final ActorDefinitionVersion createActorDefVersion(final UUID actorDefinitionId) .withActorDefinitionId(actorDefinitionId) .withDockerImageTag("1.0.0") .withDockerRepository("repo") + .withSupportLevel(SupportLevel.COMMUNITY) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); } @@ -177,6 +179,7 @@ ActorDefinitionVersion createActorDefinitionVersion(final String version) { return new ActorDefinitionVersion() .withActorDefinitionId(ACTOR_DEFINITION_ID_1) .withDockerRepository("some-repo") + .withSupportLevel(SupportLevel.COMMUNITY) .withDockerImageTag(version); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java index c155cefa73b..746daf42352 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java @@ -24,6 +24,7 @@ import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSync; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.sql.SQLException; @@ -567,6 +568,7 @@ private static ActorDefinitionVersion createBaseActorDefVersion(final UUID actor .withActorDefinitionId(actorDefId) .withDockerRepository("source-image-" + actorDefId) .withDockerImageTag(DOCKER_IMAGE_TAG) + .withSupportLevel(SupportLevel.COMMUNITY) .withProtocolVersion("0.2.0"); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java index 1b9c203bef3..5b966791856 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java @@ -21,6 +21,7 @@ import io.airbyte.config.ReleaseStage; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.SuggestedStreams; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -39,7 +40,6 @@ */ class ActorDefinitionVersionPersistenceTest extends BaseConfigDatabaseTest { - private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); private static final String SOURCE_NAME = "Test Source"; private static final String DOCKER_REPOSITORY = "airbyte/source-test"; private static final String DOCKER_IMAGE_TAG = "0.1.0"; @@ -51,24 +51,30 @@ class ActorDefinitionVersionPersistenceTest extends BaseConfigDatabaseTest { private static final ConnectorSpecification SPEC_2 = new ConnectorSpecification() .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value2"))).withProtocolVersion(PROTOCOL_VERSION); - private static final StandardSourceDefinition SOURCE_DEFINITION = new StandardSourceDefinition() - .withName(SOURCE_NAME) - .withSourceDefinitionId(ACTOR_DEFINITION_ID); + private static StandardSourceDefinition baseSourceDefinition(final UUID actorDefinitionId) { + return new StandardSourceDefinition() + .withName(SOURCE_NAME) + .withSourceDefinitionId(actorDefinitionId); + } - private static final ActorDefinitionVersion initialActorDefinitionVersion = new ActorDefinitionVersion() - .withActorDefinitionId(ACTOR_DEFINITION_ID) - .withDockerImageTag("0.0.0") - .withDockerRepository("overwrite me") - .withSpec(new ConnectorSpecification().withAdditionalProperty("overwrite", "me").withProtocolVersion("0.0.0")); + private static ActorDefinitionVersion initialActorDefinitionVersion(final UUID actorDefinitionId) { + return new ActorDefinitionVersion() + .withActorDefinitionId(actorDefinitionId) + .withDockerImageTag("0.0.0") + .withDockerRepository("overwrite me") + .withSupportLevel(SupportLevel.COMMUNITY) + .withSpec(new ConnectorSpecification().withAdditionalProperty("overwrite", "me").withProtocolVersion("0.0.0")); + } - private static ActorDefinitionVersion baseActorDefinitionVersion() { + private static ActorDefinitionVersion baseActorDefinitionVersion(final UUID actorDefinitionId) { return new ActorDefinitionVersion() - .withActorDefinitionId(ACTOR_DEFINITION_ID) + .withActorDefinitionId(actorDefinitionId) .withDockerRepository(DOCKER_REPOSITORY) .withDockerImageTag(DOCKER_IMAGE_TAG) .withSpec(SPEC) .withDocumentationUrl("https://airbyte.io/docs/") .withReleaseStage(ReleaseStage.BETA) + .withSupportLevel(SupportLevel.COMMUNITY) .withReleaseDate("2021-01-21") .withSuggestedStreams(new SuggestedStreams().withStreams(List.of("users"))) .withProtocolVersion("0.1.0") @@ -80,83 +86,102 @@ private static ActorDefinitionVersion baseActorDefinitionVersion() { .withNormalizationIntegrationType("bigquery")); } - private static final ActorDefinitionVersion ACTOR_DEFINITION_VERSION = baseActorDefinitionVersion(); - private ConfigRepository configRepository; + private StandardSourceDefinition sourceDefinition; @BeforeEach void beforeEach() throws Exception { truncateAllTables(); configRepository = new ConfigRepository(database, MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER); + final UUID defId = UUID.randomUUID(); + final ActorDefinitionVersion initialADV = initialActorDefinitionVersion(defId); + sourceDefinition = baseSourceDefinition(defId); + // Make sure that the source definition exists before we start writing actor definition versions - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, initialActorDefinitionVersion); + configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, initialADV); } @Test void testWriteActorDefinitionVersion() throws IOException { - final ActorDefinitionVersion writtenADV = configRepository.writeActorDefinitionVersion(ACTOR_DEFINITION_VERSION); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); + final ActorDefinitionVersion writtenADV = configRepository.writeActorDefinitionVersion(adv); + // All non-ID fields should match (the ID is randomly assigned) - assertEquals(ACTOR_DEFINITION_VERSION.withVersionId(writtenADV.getVersionId()), writtenADV); + final ActorDefinitionVersion expectedADV = adv.withVersionId(writtenADV.getVersionId()); + + assertEquals(expectedADV, writtenADV); } @Test void testGetActorDefinitionVersionByTag() throws IOException { - final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(ACTOR_DEFINITION_VERSION); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); + final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(adv); final UUID id = actorDefinitionVersion.getVersionId(); - final Optional optRetrievedADV = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG); + final Optional optRetrievedADV = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); assertTrue(optRetrievedADV.isPresent()); - assertEquals(ACTOR_DEFINITION_VERSION.withVersionId(id), optRetrievedADV.get()); + assertEquals(adv.withVersionId(id), optRetrievedADV.get()); } @Test void testGetForNonExistentTagReturnsEmptyOptional() throws IOException { - assertTrue(configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, UNPERSISTED_DOCKER_IMAGE_TAG).isEmpty()); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + assertTrue(configRepository.getActorDefinitionVersion(defId, UNPERSISTED_DOCKER_IMAGE_TAG).isEmpty()); } @Test void testGetActorDefinitionVersionById() throws IOException, ConfigNotFoundException { - final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(ACTOR_DEFINITION_VERSION); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); + final ActorDefinitionVersion actorDefinitionVersion = configRepository.writeActorDefinitionVersion(adv); final UUID id = actorDefinitionVersion.getVersionId(); final ActorDefinitionVersion retrievedADV = configRepository.getActorDefinitionVersion(id); assertNotNull(retrievedADV); - assertEquals(ACTOR_DEFINITION_VERSION.withVersionId(id), retrievedADV); + assertEquals(adv.withVersionId(id), retrievedADV); } @Test void testGetActorDefinitionVersionByIdNotExistentThrowsConfigNotFound() { - assertThrows(ConfigNotFoundException.class, () -> configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID)); + // Test using the definition id to catch any accidental assignment + final UUID defId = sourceDefinition.getSourceDefinitionId(); + + assertThrows(ConfigNotFoundException.class, () -> configRepository.getActorDefinitionVersion(defId)); } @Test void testWriteSourceDefinitionAndDefaultVersion() throws IOException, JsonValidationException, ConfigNotFoundException { // Write initial source definition and default version - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, ACTOR_DEFINITION_VERSION); + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); + final StandardSourceDefinition sourceDefinition = baseSourceDefinition(defId); + configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, adv); - final Optional optADVForTag = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG); + final Optional optADVForTag = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); assertTrue(optADVForTag.isPresent()); final ActorDefinitionVersion advForTag = optADVForTag.get(); final StandardSourceDefinition retrievedSourceDefinition = - configRepository.getStandardSourceDefinition(SOURCE_DEFINITION.getSourceDefinitionId()); + configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); assertEquals(retrievedSourceDefinition.getDefaultVersionId(), advForTag.getVersionId()); // Modify spec without changing docker image tag - final ActorDefinitionVersion modifiedADV = baseActorDefinitionVersion().withSpec(SPEC_2); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, modifiedADV); + final ActorDefinitionVersion modifiedADV = baseActorDefinitionVersion(defId).withSpec(SPEC_2); + configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, modifiedADV); - assertEquals(retrievedSourceDefinition, configRepository.getStandardSourceDefinition(SOURCE_DEFINITION.getSourceDefinitionId())); - final Optional optADVForTagAfterCall2 = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG); + assertEquals(retrievedSourceDefinition, configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())); + final Optional optADVForTagAfterCall2 = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); assertTrue(optADVForTagAfterCall2.isPresent()); // Versioned data does not get updated since the tag did not change - old spec is still returned assertEquals(advForTag, optADVForTagAfterCall2.get()); // Modifying docker image tag creates a new version (which can contain new versioned data) - final ActorDefinitionVersion newADV = baseActorDefinitionVersion().withDockerImageTag(DOCKER_IMAGE_TAG_2).withSpec(SPEC_2); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, newADV); + final ActorDefinitionVersion newADV = baseActorDefinitionVersion(defId).withDockerImageTag(DOCKER_IMAGE_TAG_2).withSpec(SPEC_2); + configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, newADV); - final Optional optADVForTag2 = configRepository.getActorDefinitionVersion(ACTOR_DEFINITION_ID, DOCKER_IMAGE_TAG_2); + final Optional optADVForTag2 = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG_2); assertTrue(optADVForTag2.isPresent()); final ActorDefinitionVersion advForTag2 = optADVForTag2.get(); @@ -166,17 +191,43 @@ void testWriteSourceDefinitionAndDefaultVersion() throws IOException, JsonValida assertNotEquals(advForTag2.getSpec(), advForTag.getSpec()); } + @Test + void testWriteSourceDefinitionSupportLevelNone() throws IOException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId).withActorDefinitionId(defId).withSupportLevel(SupportLevel.NONE); + + configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, adv); + + final Optional optADVForTag = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); + assertTrue(optADVForTag.isPresent()); + final ActorDefinitionVersion advForTag = optADVForTag.get(); + assertEquals(advForTag.getSupportLevel(), SupportLevel.NONE); + } + + @Test + void testWriteSourceDefinitionSupportLevelNonNullable() { + final UUID defId = sourceDefinition.getSourceDefinitionId(); + + final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId).withActorDefinitionId(defId).withSupportLevel(null); + + assertThrows( + RuntimeException.class, + () -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, adv)); + } + @Test void testAlwaysGetWithProtocolVersion() throws IOException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); + final List allActorDefVersions = List.of( - baseActorDefinitionVersion().withDockerImageTag("5.0.0").withProtocolVersion(null), - baseActorDefinitionVersion().withDockerImageTag("5.0.1") + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.0").withProtocolVersion(null), + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.1") .withProtocolVersion(null) .withSpec(new ConnectorSpecification().withProtocolVersion("0.3.1")), - baseActorDefinitionVersion().withDockerImageTag("5.0.2") + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.2") .withProtocolVersion("0.4.0") .withSpec(new ConnectorSpecification().withProtocolVersion("0.4.1")), - baseActorDefinitionVersion().withDockerImageTag("5.0.3") + baseActorDefinitionVersion(defId).withDockerImageTag("5.0.3") .withProtocolVersion("0.5.0") .withSpec(new ConnectorSpecification())); @@ -198,29 +249,31 @@ void testAlwaysGetWithProtocolVersion() throws IOException { @Test void testListActorDefinitionVersionsForDefinition() throws IOException, JsonValidationException, ConfigNotFoundException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); final StandardSourceDefinition otherSourceDef = new StandardSourceDefinition() .withName("Some other source") .withSourceDefinitionId(UUID.randomUUID()); - final ActorDefinitionVersion otherActorDefVersion = baseActorDefinitionVersion().withActorDefinitionId(otherSourceDef.getSourceDefinitionId()); + final ActorDefinitionVersion otherActorDefVersion = + baseActorDefinitionVersion(defId).withActorDefinitionId(otherSourceDef.getSourceDefinitionId()); configRepository.writeSourceDefinitionAndDefaultVersion(otherSourceDef, otherActorDefVersion); final UUID otherActorDefVersionId = configRepository.getStandardSourceDefinition(otherSourceDef.getSourceDefinitionId()).getDefaultVersionId(); final List actorDefinitionVersions = List.of( - baseActorDefinitionVersion().withDockerImageTag("1.0.0"), - baseActorDefinitionVersion().withDockerImageTag("2.0.0"), - baseActorDefinitionVersion().withDockerImageTag("3.0.0")); + baseActorDefinitionVersion(defId).withDockerImageTag("1.0.0"), + baseActorDefinitionVersion(defId).withDockerImageTag("2.0.0"), + baseActorDefinitionVersion(defId).withDockerImageTag("3.0.0")); final List expectedVersionIds = new ArrayList<>(); for (final ActorDefinitionVersion actorDefVersion : actorDefinitionVersions) { expectedVersionIds.add(configRepository.writeActorDefinitionVersion(actorDefVersion).getVersionId()); } - final UUID defaultVersionId = configRepository.getStandardSourceDefinition(ACTOR_DEFINITION_ID).getDefaultVersionId(); + final UUID defaultVersionId = configRepository.getStandardSourceDefinition(defId).getDefaultVersionId(); expectedVersionIds.add(defaultVersionId); final List actorDefinitionVersionsForDefinition = - configRepository.listActorDefinitionVersionsForDefinition(ACTOR_DEFINITION_ID); + configRepository.listActorDefinitionVersionsForDefinition(defId); assertThat(expectedVersionIds) .containsExactlyInAnyOrderElementsOf(actorDefinitionVersionsForDefinition.stream().map(ActorDefinitionVersion::getVersionId).toList()); assertFalse( @@ -237,12 +290,13 @@ void testListActorDefinitionVersionsForDefinition() throws IOException, JsonVali "UNSUPPORTED, DEPRECATED", }) void testSetActorDefinitionVersionSupportStates(final String initialSupportStateStr, final String targetSupportStateStr) throws IOException { + final UUID defId = sourceDefinition.getSourceDefinitionId(); final SupportState initialSupportState = SupportState.valueOf(initialSupportStateStr); final SupportState targetSupportState = SupportState.valueOf(targetSupportStateStr); final List actorDefinitionVersions = List.of( - baseActorDefinitionVersion().withDockerImageTag("1.0.0").withSupportState(initialSupportState), - baseActorDefinitionVersion().withDockerImageTag("2.0.0").withSupportState(initialSupportState)); + baseActorDefinitionVersion(defId).withDockerImageTag("1.0.0").withSupportState(initialSupportState), + baseActorDefinitionVersion(defId).withDockerImageTag("2.0.0").withSupportState(initialSupportState)); final List versionIds = new ArrayList<>(); for (final ActorDefinitionVersion actorDefVersion : actorDefinitionVersions) { diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java index 29dcc65ef8f..022e353bc99 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java @@ -15,6 +15,7 @@ import io.airbyte.config.ActorDefinitionConfigInjection; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import java.io.IOException; import java.util.Map; @@ -139,6 +140,7 @@ private static ActorDefinitionVersion createBaseActorDefVersion(final UUID actor .withActorDefinitionId(actorDefId) .withDockerRepository("source-image-" + actorDefId) .withDockerImageTag("1.0.0") + .withSupportLevel(SupportLevel.COMMUNITY) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java index 8412fc19e33..9316a1e3b69 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java @@ -19,6 +19,7 @@ import io.airbyte.config.DeclarativeManifest; import io.airbyte.config.ScopeType; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.Jsons; import java.io.IOException; @@ -321,6 +322,7 @@ private StandardSourceDefinition linkSourceDefinition(final UUID projectId) thro .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withDockerRepository("repo-" + id) .withDockerImageTag("0.0.1") + .withSupportLevel(SupportLevel.COMMUNITY) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java index ce51a0c2e53..7b86503e263 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/MockData.java @@ -49,6 +49,7 @@ import io.airbyte.config.StandardSyncState; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.State; +import io.airbyte.config.SupportLevel; import io.airbyte.config.User; import io.airbyte.config.User.AuthProvider; import io.airbyte.config.WebhookConfig; @@ -406,6 +407,7 @@ public static ActorDefinitionVersion actorDefinitionVersion() { .withDockerImageTag("0.0.1") .withDockerRepository("repository-4") .withSpec(connectorSpecification()) + .withSupportLevel(SupportLevel.COMMUNITY) .withProtocolVersion("0.2.0"); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java index a54bd41f281..b9b08f14837 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java @@ -32,6 +32,7 @@ import io.airbyte.config.StandardSync.Status; import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; import io.airbyte.db.instance.configs.jooq.generated.enums.AutoPropagationStatus; import io.airbyte.db.instance.configs.jooq.generated.enums.NotificationType; import io.airbyte.db.instance.configs.jooq.generated.tables.records.NotificationConfigurationRecord; @@ -355,7 +356,8 @@ private StandardSourceDefinition createStandardSourceDefinition(final String pro .withDocumentationUrl("documentation-url-1") .withSpec(new ConnectorSpecification()) .withProtocolVersion(protocolVersion) - .withReleaseStage(releaseStage); + .withReleaseStage(releaseStage) + .withSupportLevel(SupportLevel.COMMUNITY); configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef, sourceDefVersion); return sourceDef; } @@ -378,7 +380,9 @@ private StandardDestinationDefinition createStandardDestDefinition(final String .withDocumentationUrl("documentation-url-3") .withSpec(new ConnectorSpecification()) .withProtocolVersion(protocolVersion) - .withReleaseStage(releaseStage); + .withReleaseStage(releaseStage) + .withSupportLevel(SupportLevel.COMMUNITY); + configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, destDefVersion); return destDef; } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java index 2172d6cce88..0c1db316d07 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspaceFilterTest.java @@ -14,6 +14,7 @@ import io.airbyte.db.instance.configs.jooq.generated.enums.ActorType; import io.airbyte.db.instance.configs.jooq.generated.enums.NamespaceDefinitionType; +import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import java.io.IOException; import java.sql.SQLException; import java.time.OffsetDateTime; @@ -58,9 +59,10 @@ static void setUpAll() throws SQLException { .execute()); // create actor_definition_version database.transaction(ctx -> ctx.insertInto(ACTOR_DEFINITION_VERSION, ACTOR_DEFINITION_VERSION.ID, ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, - ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC) - .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}")) - .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}")) + ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC, + ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL) + .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}"), SupportLevel.community) + .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}"), SupportLevel.community) .execute()); // create workspace diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java index ee21d13d08f..51040bb58f8 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java @@ -24,6 +24,7 @@ import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardSync; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; @@ -125,6 +126,7 @@ private static ActorDefinitionVersion createActorDefinitionVersion(final UUID ac return new ActorDefinitionVersion() .withActorDefinitionId(actorDefinitionId) .withDockerRepository("dockerhub") + .withSupportLevel(SupportLevel.COMMUNITY) .withDockerImageTag("0.0.1") .withReleaseStage(releaseStage); } diff --git a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java index cac20d55d43..93d52723da5 100644 --- a/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java +++ b/airbyte-config/specs/src/test/java/io/airbyte/config/specs/RemoteDefinitionsProviderTest.java @@ -15,6 +15,7 @@ import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.ConnectorRegistryDestinationDefinition; import io.airbyte.config.ConnectorRegistrySourceDefinition; +import io.airbyte.config.SupportLevel; import io.airbyte.protocol.models.ConnectorSpecification; import java.io.IOException; import java.net.URI; @@ -79,6 +80,7 @@ void testGetSourceDefinition() throws Exception { assertEquals(URI.create("https://docs.airbyte.io/integrations/sources/stripe"), stripeSource.getSpec().getDocumentationUrl()); assertEquals(false, stripeSource.getTombstone()); assertEquals("0.2.1", stripeSource.getProtocolVersion()); + assertEquals(SupportLevel.COMMUNITY, stripeSource.getSupportLevel()); } @Test @@ -97,6 +99,7 @@ void testGetDestinationDefinition() throws Exception { assertEquals(URI.create("https://docs.airbyte.io/integrations/destinations/s3"), s3Destination.getSpec().getDocumentationUrl()); assertEquals(false, s3Destination.getTombstone()); assertEquals("0.2.2", s3Destination.getProtocolVersion()); + assertEquals(SupportLevel.COMMUNITY, s3Destination.getSupportLevel()); } @Test diff --git a/airbyte-config/specs/src/test/resources/connector_catalog.json b/airbyte-config/specs/src/test/resources/connector_catalog.json index f4e09b0d95c..1fd287a9779 100644 --- a/airbyte-config/specs/src/test/resources/connector_catalog.json +++ b/airbyte-config/specs/src/test/resources/connector_catalog.json @@ -104,7 +104,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "b4c5d105-31fd-4817-96b6-cb923bfc04cb", @@ -207,6 +208,7 @@ "public": true, "custom": false, "releaseStage": "alpha", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -425,6 +427,7 @@ "public": true, "custom": false, "releaseStage": "generally_available", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -634,6 +637,7 @@ "public": true, "custom": false, "releaseStage": "beta", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -720,7 +724,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "81740ce8-d764-4ea7-94df-16bb41de36ae", @@ -766,7 +771,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "ce0d828e-1dc4-496c-b122-2da42e637e48", @@ -952,7 +958,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "072d5540-f236-4294-ba7c-ade8fd918496", @@ -1141,7 +1148,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "8ccd8909-4e99-4141-b48d-4984b70b2d89", @@ -1234,7 +1242,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "68f351a7-2745-4bef-ad7f-996b8e51bb8c", @@ -1338,7 +1347,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "a7bcc9d8-13b3-4e49-b80d-d020b90045e3", @@ -1373,7 +1383,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "18081484-02a5-4662-8dba-b270b582f321", @@ -1492,7 +1503,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "ca8f6566-e555-4b40-943a-545bf123117a", @@ -1891,6 +1903,7 @@ "public": true, "custom": false, "releaseStage": "beta", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -1939,7 +1952,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "356668e2-7e34-47f3-a3b0-67a8a481b692", @@ -1982,7 +1996,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a", @@ -2055,7 +2070,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "d4353156-9217-4cad-8dd7-c108fd4f74cf", @@ -2294,7 +2310,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "294a4790-429b-40ae-9516-49826b9702e1", @@ -2470,7 +2487,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", @@ -2510,7 +2528,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "8b746512-8c2e-6ac1-4adc-b59faafd473c", @@ -2666,7 +2685,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "ca81ee7c-3163-4246-af40-094cc31e5e42", @@ -2848,7 +2868,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "3986776d-2319-4de9-8af8-db14c0996e72", @@ -3038,7 +3059,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", @@ -3351,7 +3373,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "2340cbba-358e-11ec-8d3d-0242ac130203", @@ -3499,7 +3522,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "e06ad785-ad6f-4647-b2e8-3027a5c59454", @@ -3559,7 +3583,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "d4d3fef9-e319-45c2-881a-bd02ce44cc9f", @@ -3623,7 +3648,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "f7a7d195-377f-cf5b-70a5-be6b819019dc", @@ -3851,6 +3877,7 @@ "public": true, "custom": false, "releaseStage": "beta", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -3912,7 +3939,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "4816b78f-1489-44c1-9060-4b19d5fa9362", @@ -4315,6 +4343,7 @@ "public": true, "custom": false, "releaseStage": "generally_available", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -4388,7 +4417,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "3dc6f384-cd6b-4be3-ad16-a41450899bf0", @@ -4456,7 +4486,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "destinationDefinitionId": "424892c4-daac-4491-b35d-c6688ba547ba", @@ -4958,6 +4989,7 @@ "public": true, "custom": false, "releaseStage": "generally_available", + "supportLevel": "community", "resourceRequirements": { "jobSpecific": [ { @@ -5027,7 +5059,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212", @@ -5076,7 +5109,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246", @@ -5211,7 +5245,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "983fd355-6bf3-4709-91b5-37afa391eeb6", @@ -5330,7 +5365,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e55879a8-0ef8-4557-abcf-ab34c53ec460", @@ -5581,7 +5617,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "fa9f58c6-2d03-4237-aaa4-07d75e0c1396", @@ -5627,7 +5664,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "47f17145-fe20-4ef5-a548-e29b048adf84", @@ -5664,7 +5702,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "798ae795-5189-42b6-b64e-3cb91db93338", @@ -5714,7 +5753,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "59c5501b-9f95-411e-9269-7143c939adbd", @@ -5759,7 +5799,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c", @@ -5803,7 +5844,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "47f25999-dd5e-4636-8c39-e7cea2453331", @@ -5937,7 +5979,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "63cea06f-1c75-458d-88fe-ad48c7cb27fd", @@ -6002,7 +6045,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "686473f1-76d9-4994-9cc7-9b13da46147c", @@ -6055,7 +6099,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "b6604cbd-1b12-4c08-8767-e140d0fb0877", @@ -6102,7 +6147,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "bad83517-5e54-4a3d-9b53-63e85fbd4d7c", @@ -6273,7 +6319,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "dfffecb7-9a13-43e9-acdc-b92af7997ca9", @@ -6312,7 +6359,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "cc88c43f-6f53-4e8a-8c4d-b284baaf9635", @@ -6354,7 +6402,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "0b5c867e-1b12-4d02-ab74-97b2184ff6d7", @@ -6399,7 +6448,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "72d405a3-56d8-499f-a571-667c03406e43", @@ -6432,7 +6482,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "50bd8338-7c4e-46f1-8c7f-3ef95de19fdd", @@ -6550,7 +6601,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e2b40e36-aa0e-4bed-b41b-bcea6fa348b1", @@ -6598,7 +6650,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", @@ -6957,7 +7010,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "dfd88b22-b603-4c3d-aad7-3701784586b1", @@ -7014,7 +7068,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "778daa7c-feaf-4db6-96f3-70fd645acc77", @@ -7261,7 +7316,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "6f2ac653-8623-43c4-8950-19218c7caf3d", @@ -7319,7 +7375,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "ec4b9503-13cb-48ab-a4ab-6ade4be46567", @@ -7372,7 +7429,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", @@ -7523,7 +7581,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "5e6175e5-68e1-4c17-bff9-56103bbb0d80", @@ -7577,7 +7636,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "253487c0-2246-43ba-a21f-5116b20a2c50", @@ -7730,7 +7790,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003", @@ -7861,7 +7922,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "d19ae824-e289-4b14-995a-0632eb46d246", @@ -7897,7 +7959,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "eb4c9e00-db83-4d63-a386-39cfa91012a8", @@ -8031,7 +8094,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "71607ba1-c0ac-4799-8049-7f4b90dd50f7", @@ -8141,7 +8205,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734", @@ -8183,7 +8248,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44", @@ -8215,7 +8281,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", @@ -8343,7 +8410,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "6acf6b55-4f1e-4fca-944e-1a3caef8aba8", @@ -8392,7 +8460,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", @@ -8440,7 +8509,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "2e875208-0c0b-4ee4-9e92-1cb3156ea799", @@ -8480,7 +8550,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "95e8cffd-b8c4-4039-968e-d32fb4a69bde", @@ -8520,7 +8591,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "cd06e646-31bf-4dc8-af48-cbc6530fcad3", @@ -8558,7 +8630,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "789f8e7a-2d28-11ec-8d3d-0242ac130003", @@ -8589,7 +8662,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "137ece28-5434-455c-8f34-69dc3782f451", @@ -8692,7 +8766,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "7b86879e-26c5-4ef6-a5ce-2be5c7b46d1e", @@ -8745,7 +8820,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c", @@ -8801,7 +8877,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b03a9f3e-22a5-11eb-adc1-0242ac120002", @@ -8925,7 +9002,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "9e0556f4-69df-4522-a3fb-03264d36b348", @@ -8987,7 +9065,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "c7cb421b-942e-4468-99ee-e369bcabaec5", @@ -9036,7 +9115,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", @@ -9314,7 +9394,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "eaf50f04-21dd-4620-913b-2a83f5635227", @@ -9480,7 +9561,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a", @@ -9566,7 +9648,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "80a54ea2-9959-4040-aac1-eee42423ec9b", @@ -9712,7 +9795,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e", @@ -9844,7 +9928,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "722ba4bf-06ec-45a4-8dd5-72e4a5cf3903", @@ -9899,7 +9984,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", @@ -10094,7 +10180,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "1d4fdb25-64fc-4569-92da-fcdca79a8372", @@ -10246,7 +10333,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b39a7370-74c3-45a6-ac3a-380d48520a83", @@ -10524,7 +10612,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "7f0455fb-4518-4ec0-b7a3-d808bf8081cc", @@ -10592,7 +10681,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a", @@ -10639,7 +10729,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "d913b0f2-cc51-4e55-a44c-8ba1697b9239", @@ -10697,7 +10788,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "3052c77e-8b91-47e2-97a0-a29a22794b4b", @@ -10729,7 +10821,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "6371b14b-bc68-4236-bfbd-468e8df8e968", @@ -10763,7 +10856,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "af6d50ee-dddf-4126-a8ee-7faee990774f", @@ -10809,7 +10903,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "decd338e-5647-4c0b-adf4-da0e75f5a750", @@ -11201,7 +11296,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "d60a46d4-709f-4092-a6b7-2457f7d455f5", @@ -11237,7 +11333,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b08e4776-d1de-4e80-ab5c-1e51dad934a2", @@ -11300,7 +11397,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "45d2e135-2ede-49e1-939f-3e3ec357a65e", @@ -11340,7 +11438,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "cd42861b-01fc-4658-a8ab-5d11d0510f01", @@ -11388,7 +11487,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e87ffa8e-a3b5-f69c-9076-6011339de1f6", @@ -11462,7 +11562,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "db04ecd1-42e7-4115-9cec-95812905c626", @@ -11591,7 +11692,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "69589781-7828-43c5-9f63-8925b1c1ccc2", @@ -11888,7 +11990,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "a827c52e-791c-4135-a245-e233c5255199", @@ -12008,7 +12111,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "b117307c-14b6-41aa-9422-947e34922962", @@ -12160,7 +12264,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "2470e835-feaf-4db6-96f3-70fd645acc77", @@ -12234,7 +12339,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87", @@ -12274,7 +12380,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "cdaf146a-9b75-49fd-9dd2-9d64a0bb4781", @@ -12323,7 +12430,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", @@ -12475,7 +12583,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "2fed2292-5586-480c-af92-9944e39fe12d", @@ -12519,7 +12628,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "374ebc65-6636-4ea0-925c-7d35999a8ffc", @@ -12603,7 +12713,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b", @@ -12675,7 +12786,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2", @@ -12876,7 +12988,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "77225a51-cd15-4a13-af02-65816bd0ecf4", @@ -13039,7 +13152,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de", @@ -13104,7 +13218,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "badc5925-0485-42be-8caa-b34096cb71b5", @@ -13164,7 +13279,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "d1aa448b-7c54-498e-ad95-263cbebcd2db", @@ -13197,7 +13313,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35", @@ -13366,7 +13483,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "b9dc6155-672e-42ea-b10d-9f1f1fb95ab1", @@ -13424,7 +13542,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "e7eff203-90bf-43e5-a240-19ea3056c474", @@ -13471,7 +13590,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "c4cfaeda-c757-489a-8aba-859fb08b6970", @@ -13522,7 +13642,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "ef580275-d9a9-48bb-af5e-db0f5855be04", @@ -13564,7 +13685,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "afa734e4-3571-11ec-991a-1e0031268139", @@ -13625,7 +13747,8 @@ }, "public": true, "custom": false, - "releaseStage": "beta" + "releaseStage": "beta", + "supportLevel": "community" }, { "sourceDefinitionId": "40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4", @@ -13778,7 +13901,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "325e0640-e7b3-4e24-b823-3361008f603f", @@ -13938,7 +14062,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "79c1aa37-dae3-42ae-b333-d1c105477715", @@ -14080,7 +14205,8 @@ }, "public": true, "custom": false, - "releaseStage": "generally_available" + "releaseStage": "generally_available", + "supportLevel": "community" }, { "sourceDefinitionId": "f1e4c7f6-db5c-4035-981f-d35ab4998794", @@ -14126,7 +14252,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" }, { "sourceDefinitionId": "3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5", @@ -14179,7 +14306,8 @@ }, "public": true, "custom": false, - "releaseStage": "alpha" + "releaseStage": "alpha", + "supportLevel": "community" } ] } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersion.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersion.java new file mode 100644 index 00000000000..8a1f4baa45d --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersion.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.Catalog; +import org.jooq.DSLContext; +import org.jooq.EnumType; +import org.jooq.Schema; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.impl.SchemaImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Inserts a support_level column to the actor_definition_version table. + */ +public class V0_50_23_003__AddSupportLevelToActorDefinitionVersion extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_23_003__AddSupportLevelToActorDefinitionVersion.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + final DSLContext ctx = DSL.using(context.getConnection()); + addSupportLevelToActorDefinitionVersion(ctx); + LOGGER.info("support_level column added to actor_definition_version table"); + } + + static void addSupportLevelToActorDefinitionVersion(final DSLContext ctx) { + createSupportLevelEnum(ctx); + addSupportLevelColumn(ctx); + } + + public static void createSupportLevelEnum(final DSLContext ctx) { + ctx.createType("support_level").asEnum("community", "certified", "none").execute(); + } + + static void addSupportLevelColumn(final DSLContext ctx) { + ctx.alterTable("actor_definition_version") + .addColumnIfNotExists(DSL.field("support_level", SQLDataType.VARCHAR.asEnumDataType( + V0_50_23_003__AddSupportLevelToActorDefinitionVersion.SupportLevel.class).nullable(false).defaultValue(SupportLevel.none))) + .execute(); + } + + enum SupportLevel implements EnumType { + + community("community"), + certified("certified"), + none("none"); + + private final String literal; + + SupportLevel(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "support_level"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.java new file mode 100644 index 00000000000..14e86ccaa0e --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.Catalog; +import org.jooq.DSLContext; +import org.jooq.EnumType; +import org.jooq.Schema; +import org.jooq.impl.DSL; +import org.jooq.impl.SchemaImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is a migration to naively populate all actor_definition_version records with a support_level + * relative to their release stage. alpha -> community beta -> community general_availability -> + * certified + */ +public class V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger( + V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.class); + + static void backfillSupportLevel(final DSLContext ctx) { + ctx.transaction(configuration -> { + final var transactionCtx = DSL.using(configuration); + + final var updateQuery = + "UPDATE actor_definition_version SET support_level = {0} WHERE release_stage = {1} AND support_level = 'none'::support_level"; + + // For all connections with invalid catalog, update to valid catalog + transactionCtx.execute(updateQuery, SupportLevel.community, ReleaseStage.alpha); + transactionCtx.execute(updateQuery, SupportLevel.community, ReleaseStage.beta); + transactionCtx.execute(updateQuery, SupportLevel.certified, ReleaseStage.generally_available); + transactionCtx.execute(updateQuery, SupportLevel.none, ReleaseStage.custom); + + // Drop the default Support Level + transactionCtx.alterTable("actor_definition_version").alterColumn("support_level").dropDefault().execute(); + }); + } + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + + backfillSupportLevel(ctx); + } + + enum SupportLevel implements EnumType { + + community("community"), + certified("certified"), + none("none"); + + private final String literal; + + SupportLevel(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "support_level"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + + enum ReleaseStage implements EnumType { + + alpha("alpha"), + beta("beta"), + generally_available("generally_available"), + custom("custom"); + + private final String literal; + + ReleaseStage(String literal) { + this.literal = literal; + } + + @Override + public Catalog getCatalog() { + return getSchema() == null ? null : getSchema().getCatalog(); + } + + @Override + public Schema getSchema() { + return new SchemaImpl(DSL.name("public"), null); + } + + @Override + public String getName() { + return "release_stage"; + } + + @Override + public String getLiteral() { + return literal; + } + + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index 88fd7e8a718..e7a4f6e30c6 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -100,6 +100,7 @@ create table "public"."actor_definition_version" ( "suggested_streams" jsonb, "release_stage" "public"."release_stage", "support_state" "public"."support_state" not null default cast('supported' as support_state), + "support_level" "public"."support_level" not null, constraint "actor_definition_version_pkey" primary key ("id") ); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest.java new file mode 100644 index 00000000000..1cc6bde0508 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_50_23_003__AddSupportLevelToActorDefinitionVersion.SupportLevel; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.io.IOException; +import java.sql.SQLException; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.JSONB; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest extends AbstractConfigsDatabaseTest { + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_6_002__AddDefaultVersionIdToActor(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + @Test + void test() throws SQLException, IOException { + final DSLContext context = getDslContext(); + + // ignore all foreign key constraints + context.execute("SET session_replication_role = replica;"); + + Assertions.assertFalse(supportLevelColumnExists(context)); + + V0_50_23_003__AddSupportLevelToActorDefinitionVersion.addSupportLevelToActorDefinitionVersion(context); + + Assertions.assertTrue(supportLevelColumnExists(context)); + + assertSupportLevelEnumWorks(context); + } + + private static boolean supportLevelColumnExists(final DSLContext ctx) { + return ctx.fetchExists(DSL.select() + .from("information_schema.columns") + .where(DSL.field("table_name").eq("actor_definition_version") + .and(DSL.field("column_name").eq("support_level")))); + } + + private static void assertSupportLevelEnumWorks(final DSLContext ctx) { + Assertions.assertDoesNotThrow(() -> { + insertWithSupportLevel(ctx, SupportLevel.community); + insertWithSupportLevel(ctx, SupportLevel.certified); + insertWithSupportLevel(ctx, SupportLevel.none); + }); + + Assertions.assertThrows(Exception.class, () -> { + insertWithSupportLevel(ctx, SupportLevel.valueOf("invalid")); + }); + + Assertions.assertThrows(Exception.class, () -> { + insertWithSupportLevel(ctx, SupportLevel.valueOf(null)); + }); + } + + private static void insertWithSupportLevel(final DSLContext ctx, final SupportLevel supportLevel) { + ctx.insertInto(DSL.table("actor_definition_version")) + .columns( + DSL.field("id"), + DSL.field("actor_definition_id"), + DSL.field("docker_repository"), + DSL.field("docker_image_tag"), + DSL.field("spec"), + DSL.field("support_level")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + "repo", + "1.0.0", + JSONB.valueOf("{}"), + supportLevel) + .execute(); + } + +} diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest.java new file mode 100644 index 00000000000..30f42f9f397 --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import io.airbyte.db.factory.FlywayFactory; +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import io.airbyte.db.instance.configs.ConfigsDatabaseMigrator; +import io.airbyte.db.instance.configs.migrations.V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.ReleaseStage; +import io.airbyte.db.instance.configs.migrations.V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.SupportLevel; +import io.airbyte.db.instance.development.DevDatabaseMigrator; +import java.util.List; +import java.util.UUID; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.jooq.DSLContext; +import org.jooq.JSONB; +import org.jooq.Record; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("PMD.AvoidDuplicateLiterals") +class V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersionTest extends AbstractConfigsDatabaseTest { + + @BeforeEach + void beforeEach() { + final Flyway flyway = + FlywayFactory.create(dataSource, "V0_50_23_003__AddSupportLevelToActorDefinitionVersionTest", ConfigsDatabaseMigrator.DB_IDENTIFIER, + ConfigsDatabaseMigrator.MIGRATION_FILE_LOCATION); + final ConfigsDatabaseMigrator configsDbMigrator = new ConfigsDatabaseMigrator(database, flyway); + + final BaseJavaMigration previousMigration = new V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion(); + final DevDatabaseMigrator devConfigsDbMigrator = new DevDatabaseMigrator(configsDbMigrator, previousMigration.getVersion()); + devConfigsDbMigrator.createBaseline(); + } + + private static void insertAdvWithReleaseStageAndSupportLevel( + final DSLContext ctx, + final ReleaseStage releaseStage, + final SupportLevel supportLevel) { + ctx.insertInto(DSL.table("actor_definition_version")) + .columns( + DSL.field("id"), + DSL.field("actor_definition_id"), + DSL.field("docker_repository"), + DSL.field("docker_image_tag"), + DSL.field("spec"), + DSL.field("support_level"), + DSL.field("release_stage")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + "repo", + "1.0.0", + JSONB.valueOf("{}"), + supportLevel, + releaseStage) + .execute(); + } + + private static void insertWithoutSupportLevel(final DSLContext ctx) { + ctx.insertInto(DSL.table("actor_definition_version")) + .columns( + DSL.field("id"), + DSL.field("actor_definition_id"), + DSL.field("docker_repository"), + DSL.field("docker_image_tag"), + DSL.field("spec"), + DSL.field("release_stage")) + .values( + UUID.randomUUID(), + UUID.randomUUID(), + "repo", + "1.0.0", + JSONB.valueOf("{}"), + ReleaseStage.alpha) + .execute(); + } + + @Test + void testBackfillSupportLevel() throws Exception { + final DSLContext ctx = getDslContext(); + + // ignore all foreign key constraints + ctx.execute("SET session_replication_role = replica;"); + + final int numberOfAdvs = 10; + + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.alpha, SupportLevel.certified); + + for (int i = 0; i < numberOfAdvs; i++) { + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.alpha, SupportLevel.none); + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.beta, SupportLevel.none); + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.generally_available, SupportLevel.none); + insertAdvWithReleaseStageAndSupportLevel(ctx, ReleaseStage.custom, SupportLevel.none); + } + + // assert that all advs have support level "none" + final List preAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("support_level").eq(SupportLevel.none)) + .fetch(); + + Assertions.assertEquals(numberOfAdvs * 4, preAdvs.size()); + + V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.backfillSupportLevel(ctx); + + // assert that all alpha advs have support level set to community + final List alphaAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.alpha).and(DSL.field("support_level").eq(SupportLevel.community))) + .fetch(); + Assertions.assertEquals(numberOfAdvs, alphaAdvs.size()); + + // assert that all beta advs have support level set to community + final List betaAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.beta).and(DSL.field("support_level").eq(SupportLevel.community))) + .fetch(); + + Assertions.assertEquals(numberOfAdvs, betaAdvs.size()); + + // assert that all generally_available advs have support level set to certified + final List gaAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.generally_available).and(DSL.field("support_level").eq(SupportLevel.certified))) + .fetch(); + + Assertions.assertEquals(numberOfAdvs, gaAdvs.size()); + + // assert that all custom advs have support level set to none + final List customAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.custom).and(DSL.field("support_level").eq(SupportLevel.none))) + .fetch(); + + Assertions.assertEquals(numberOfAdvs, customAdvs.size()); + + // assert that there is one adv with support level certified and release stage alpha (i.e. did not + // get overwritten) + final List certifiedAdvs = ctx.select() + .from(DSL.table("actor_definition_version")) + .where(DSL.field("release_stage").eq(ReleaseStage.alpha).and(DSL.field("support_level").eq(SupportLevel.certified))) + .fetch(); + + Assertions.assertEquals(1, certifiedAdvs.size()); + } + + @Test + void testNoDefaultSupportLevel() { + final DSLContext ctx = getDslContext(); + + // ignore all foreign key constraints + ctx.execute("SET session_replication_role = replica;"); + + V0_50_23_004__NaivelyBackfillSupportLevelForActorDefitionVersion.backfillSupportLevel(ctx); + + Assertions.assertThrows(RuntimeException.class, () -> insertWithoutSupportLevel(ctx)); + } + +} diff --git a/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java b/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java index 4d3128e3420..86d22a759f1 100644 --- a/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java +++ b/airbyte-metrics/reporter/src/test/java/io/airbyte/metrics/reporter/MetricRepositoryTest.java @@ -25,6 +25,7 @@ import io.airbyte.db.instance.configs.jooq.generated.enums.GeographyType; import io.airbyte.db.instance.configs.jooq.generated.enums.NamespaceDefinitionType; import io.airbyte.db.instance.configs.jooq.generated.enums.StatusType; +import io.airbyte.db.instance.configs.jooq.generated.enums.SupportLevel; import io.airbyte.db.instance.jobs.jooq.generated.enums.AttemptStatus; import io.airbyte.db.instance.jobs.jooq.generated.enums.JobConfigType; import io.airbyte.db.instance.jobs.jooq.generated.enums.JobStatus; @@ -84,9 +85,10 @@ public static void setUpAll() throws DatabaseInitializationException, IOExceptio .execute(); ctx.insertInto(ACTOR_DEFINITION_VERSION, ACTOR_DEFINITION_VERSION.ID, ACTOR_DEFINITION_VERSION.ACTOR_DEFINITION_ID, - ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC) - .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}")) - .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}")) + ACTOR_DEFINITION_VERSION.DOCKER_REPOSITORY, ACTOR_DEFINITION_VERSION.DOCKER_IMAGE_TAG, ACTOR_DEFINITION_VERSION.SPEC, + ACTOR_DEFINITION_VERSION.SUPPORT_LEVEL) + .values(SRC_DEF_VER_ID, SRC_DEF_ID, "airbyte/source", "tag", JSONB.valueOf("{}"), SupportLevel.community) + .values(DST_DEF_VER_ID, DST_DEF_ID, "airbyte/destination", "tag", JSONB.valueOf("{}"), SupportLevel.community) .execute(); // drop constraints to simplify test set up From 4f176fa027e894d14777fe9c5f02e6f92dce1712 Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Wed, 6 Sep 2023 13:54:58 -0700 Subject: [PATCH 122/201] Split the job persistence class into multiple elements (#8700) --- .../config/DatabaseBeanFactory.java | 7 + .../cron/config/DatabaseBeanFactory.java | 8 + .../job/DefaultMetadataPersistence.java | 161 ++++++++++++++++++ .../persistence/job/MetadataPersistence.java | 65 +++++++ .../server/config/DatabaseBeanFactory.java | 7 + .../workers/config/DatabaseBeanFactory.java | 8 + 6 files changed, 256 insertions(+) create mode 100644 airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultMetadataPersistence.java create mode 100644 airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/MetadataPersistence.java diff --git a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java index 57cfc10e9c8..bbe9f3c8aed 100644 --- a/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java +++ b/airbyte-bootloader/src/main/java/io/airbyte/bootloader/config/DatabaseBeanFactory.java @@ -19,7 +19,9 @@ import io.airbyte.db.instance.jobs.JobsDatabaseMigrator; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.DefaultJobPersistence; +import io.airbyte.persistence.job.DefaultMetadataPersistence; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.MetadataPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; import io.micronaut.flyway.FlywayConfigurationProperties; @@ -110,6 +112,11 @@ public JobPersistence jobPersistence(@Named("jobsDatabase") final Database jobDa return new DefaultJobPersistence(jobDatabase); } + @Singleton + public MetadataPersistence metadataPersistence(@Named("jobsDatabase") final Database jobDatabase) { + return new DefaultMetadataPersistence(jobDatabase); + } + @SuppressWarnings("LineLength") @Singleton @Named("configsDatabaseInitializer") diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java b/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java index ee624ceb13a..2dc53ab2e72 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java +++ b/airbyte-cron/src/main/java/io/airbyte/cron/config/DatabaseBeanFactory.java @@ -12,7 +12,9 @@ import io.airbyte.db.factory.DatabaseCheckFactory; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.DefaultJobPersistence; +import io.airbyte.persistence.job.DefaultMetadataPersistence; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.MetadataPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; @@ -113,4 +115,10 @@ public JobPersistence jobPersistence(@Named("jobsDatabase") final Database jobDa return new DefaultJobPersistence(jobDatabase); } + @Singleton + @Requires(env = WorkerMode.CONTROL_PLANE) + public MetadataPersistence metadataPersistence(@Named("jobsDatabase") final Database jobDatabase) { + return new DefaultMetadataPersistence(jobDatabase); + } + } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultMetadataPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultMetadataPersistence.java new file mode 100644 index 00000000000..949eabf4cc2 --- /dev/null +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultMetadataPersistence.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.persistence.job; + +import io.airbyte.commons.version.AirbyteProtocolVersion; +import io.airbyte.commons.version.AirbyteProtocolVersionRange; +import io.airbyte.commons.version.AirbyteVersion; +import io.airbyte.commons.version.Version; +import io.airbyte.db.Database; +import io.airbyte.db.ExceptionWrappingDatabase; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Encapsulates jobs db interactions for the metadata domain models. + */ +public class DefaultMetadataPersistence implements MetadataPersistence { + + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultMetadataPersistence.class); + + private static final String AIRBYTE_METADATA_TABLE = "airbyte_metadata"; + private static final String METADATA_KEY_COL = "key"; + private static final String METADATA_VAL_COL = "value"; + private static final String DEPLOYMENT_ID_KEY = "deployment_id"; + + private final ExceptionWrappingDatabase jobDatabase; + + public DefaultMetadataPersistence(final Database database) { + this.jobDatabase = new ExceptionWrappingDatabase(database); + } + + @Override + public Optional getVersion() throws IOException { + return getMetadata(AirbyteVersion.AIRBYTE_VERSION_KEY_NAME).findFirst(); + } + + @Override + public void setVersion(final String airbyteVersion) throws IOException { + // This is not using setMetadata due to the extra (s_init_db, airbyteVersion) that is + // added to the metadata table + jobDatabase.query(ctx -> ctx.execute(String.format( + "INSERT INTO %s(%s, %s) VALUES('%s', '%s'), ('%s_init_db', '%s') ON CONFLICT (%s) DO UPDATE SET %s = '%s'", + AIRBYTE_METADATA_TABLE, + METADATA_KEY_COL, + METADATA_VAL_COL, + AirbyteVersion.AIRBYTE_VERSION_KEY_NAME, + airbyteVersion, + ZonedDateTime.now().format(DateTimeFormatter.ISO_OFFSET_DATE_TIME), + airbyteVersion, + METADATA_KEY_COL, + METADATA_VAL_COL, + airbyteVersion))); + + } + + @Override + public Optional getAirbyteProtocolVersionMax() throws IOException { + return getMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MAX_KEY_NAME).findFirst().map(Version::new); + } + + @Override + public void setAirbyteProtocolVersionMax(final Version version) throws IOException { + setMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MAX_KEY_NAME, version.serialize()); + } + + @Override + public Optional getAirbyteProtocolVersionMin() throws IOException { + return getMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MIN_KEY_NAME).findFirst().map(Version::new); + } + + @Override + public void setAirbyteProtocolVersionMin(final Version version) throws IOException { + setMetadata(AirbyteProtocolVersion.AIRBYTE_PROTOCOL_VERSION_MIN_KEY_NAME, version.serialize()); + } + + @Override + public Optional getCurrentProtocolVersionRange() throws IOException { + final Optional min = getAirbyteProtocolVersionMin(); + final Optional max = getAirbyteProtocolVersionMax(); + + if (min.isPresent() != max.isPresent()) { + // Flagging this because this would be highly suspicious but not bad enough that we should fail + // hard. + // If the new config is fine, the system should self-heal. + LOGGER.warn("Inconsistent AirbyteProtocolVersion found, only one of min/max was found. (min:{}, max:{})", + min.map(Version::serialize).orElse(""), max.map(Version::serialize).orElse("")); + } + + if (min.isEmpty() && max.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(new AirbyteProtocolVersionRange(min.orElse(AirbyteProtocolVersion.DEFAULT_AIRBYTE_PROTOCOL_VERSION), + max.orElse(AirbyteProtocolVersion.DEFAULT_AIRBYTE_PROTOCOL_VERSION))); + } + + private Stream getMetadata(final String keyName) throws IOException { + return jobDatabase.query(ctx -> ctx.select() + .from(AIRBYTE_METADATA_TABLE) + .where(DSL.field(METADATA_KEY_COL).eq(keyName)) + .fetch()).stream().map(r -> r.getValue(METADATA_VAL_COL, String.class)); + } + + private void setMetadata(final String keyName, final String value) throws IOException { + jobDatabase.query(ctx -> ctx + .insertInto(DSL.table(AIRBYTE_METADATA_TABLE)) + .columns(DSL.field(METADATA_KEY_COL), DSL.field(METADATA_VAL_COL)) + .values(keyName, value) + .onConflict(DSL.field(METADATA_KEY_COL)) + .doUpdate() + .set(DSL.field(METADATA_VAL_COL), value) + .execute()); + } + + @Override + public Optional getDeployment() throws IOException { + final Result result = jobDatabase.query(ctx -> ctx.select() + .from(AIRBYTE_METADATA_TABLE) + .where(DSL.field(METADATA_KEY_COL).eq(DEPLOYMENT_ID_KEY)) + .fetch()); + return result.stream().findFirst().map(r -> UUID.fromString(r.getValue(METADATA_VAL_COL, String.class))); + } + + @Override + public void setDeployment(final UUID deployment) throws IOException { + // if an existing deployment id already exists, on conflict, return it so we can log it. + final UUID committedDeploymentId = jobDatabase.query(ctx -> ctx.fetch(String.format( + "INSERT INTO %s(%s, %s) VALUES('%s', '%s') ON CONFLICT (%s) DO NOTHING RETURNING (SELECT %s FROM %s WHERE %s='%s') as existing_deployment_id", + AIRBYTE_METADATA_TABLE, + METADATA_KEY_COL, + METADATA_VAL_COL, + DEPLOYMENT_ID_KEY, + deployment, + METADATA_KEY_COL, + METADATA_VAL_COL, + AIRBYTE_METADATA_TABLE, + METADATA_KEY_COL, + DEPLOYMENT_ID_KEY))) + .stream() + .filter(record -> record.get("existing_deployment_id", String.class) != null) + .map(record -> UUID.fromString(record.get("existing_deployment_id", String.class))) + .findFirst() + .orElse(deployment); // if no record was returned that means that the new deployment id was used. + + if (!deployment.equals(committedDeploymentId)) { + LOGGER.warn("Attempted to set a deployment id {}, but deployment id {} already set. Retained original value.", deployment, deployment); + } + } + +} diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/MetadataPersistence.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/MetadataPersistence.java new file mode 100644 index 00000000000..6ea59d23500 --- /dev/null +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/MetadataPersistence.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.persistence.job; + +import io.airbyte.commons.version.AirbyteProtocolVersionRange; +import io.airbyte.commons.version.Version; +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; + +/** + * General persistence interface for storing metadata. + */ +public interface MetadataPersistence { + + /** + * Returns the AirbyteVersion. + */ + Optional getVersion() throws IOException; + + /** + * Set the airbyte version. + */ + void setVersion(String airbyteVersion) throws IOException; + + /** + * Get the max supported Airbyte Protocol Version. + */ + Optional getAirbyteProtocolVersionMax() throws IOException; + + /** + * Set the max supported Airbyte Protocol Version. + */ + void setAirbyteProtocolVersionMax(Version version) throws IOException; + + /** + * Get the min supported Airbyte Protocol Version. + */ + Optional getAirbyteProtocolVersionMin() throws IOException; + + /** + * Set the min supported Airbyte Protocol Version. + */ + void setAirbyteProtocolVersionMin(Version version) throws IOException; + + /** + * Get the current Airbyte Protocol Version range if defined. + */ + Optional getCurrentProtocolVersionRange() throws IOException; + + /** + * Returns a deployment UUID. + */ + Optional getDeployment() throws IOException; + // a deployment references a setup of airbyte. it is created the first time the docker compose or + // K8s is ready. + + /** + * Set deployment id. If one is already set, the new value is ignored. + */ + void setDeployment(UUID uuid) throws IOException; + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java index 5d4b661779b..f9a9a986a6f 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java +++ b/airbyte-server/src/main/java/io/airbyte/server/config/DatabaseBeanFactory.java @@ -19,7 +19,9 @@ import io.airbyte.db.instance.DatabaseConstants; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.DefaultJobPersistence; +import io.airbyte.persistence.job.DefaultMetadataPersistence; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.MetadataPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Value; import io.micronaut.flyway.FlywayConfigurationProperties; @@ -94,6 +96,11 @@ public JobPersistence jobPersistence(@Named("configDatabase") final Database job return new DefaultJobPersistence(jobDatabase); } + @Singleton + public MetadataPersistence metadataPersistence(@Named("configDatabase") final Database jobDatabase) { + return new DefaultMetadataPersistence(jobDatabase); + } + @Singleton public PermissionPersistence permissionPersistence(@Named("configDatabase") final Database configDatabase) { return new PermissionPersistence(configDatabase); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/config/DatabaseBeanFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/config/DatabaseBeanFactory.java index 43c8c324df6..052d81c8e07 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/config/DatabaseBeanFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/config/DatabaseBeanFactory.java @@ -15,7 +15,9 @@ import io.airbyte.db.instance.DatabaseConstants; import io.airbyte.featureflag.FeatureFlagClient; import io.airbyte.persistence.job.DefaultJobPersistence; +import io.airbyte.persistence.job.DefaultMetadataPersistence; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.MetadataPersistence; import io.micronaut.context.annotation.Factory; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; @@ -98,6 +100,12 @@ public JobPersistence jobPersistence(@Named("jobsDatabase") final Database jobDa return new DefaultJobPersistence(jobDatabase); } + @Singleton + @Requires(env = WorkerMode.CONTROL_PLANE) + public MetadataPersistence metadataPersistence(@Named("jobsDatabase") final Database jobDatabase) { + return new DefaultMetadataPersistence(jobDatabase); + } + @Singleton @Requires(env = WorkerMode.CONTROL_PLANE) public StatePersistence statePersistence(@Named("configDatabase") final Database configDatabase) { From 3d0ecce1b1252486c7c0ad871039947f4c216268 Mon Sep 17 00:00:00 2001 From: Xiaohan Song Date: Wed, 6 Sep 2023 14:56:42 -0700 Subject: [PATCH 123/201] Add a new API list_instance_admin for listing all instance admin users (#8573) --- airbyte-api/src/main/openapi/config.yaml | 40 ++++++++++++++++++- .../commons/server/handlers/UserHandler.java | 21 +++++++++- .../persistence/PermissionPersistence.java | 14 +++++++ .../PermissionPersistenceTest.java | 8 ++++ .../server/apis/UserApiController.java | 16 ++++++-- .../server/apis/UserApiControllerTest.java | 11 +++++ 6 files changed, 104 insertions(+), 6 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index aefcf336ff4..6cc83dbab90 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3105,6 +3105,22 @@ paths: "404": $ref: "#/components/responses/NotFoundResponse" + /v1/users/list_instance_admin: + post: + summary: List all users with instance admin permissions. Only instance admin has permission to call this. + tags: + - user + operationId: listInstanceAdminUsers + responses: + "200": + description: List all instance admin users. + content: + application/json: + schema: + $ref: "#/components/schemas/UserWithPermissionInfoReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" + # PERMISSIONS /v1/permissions/create: post: @@ -7818,7 +7834,29 @@ components: $ref: "#/components/schemas/PermissionType" organizationId: $ref: "#/components/schemas/OrganizationId" - + UserWithPermissionInfoReadList: + type: object + required: + - users + properties: + users: + type: array + items: + $ref: "#/components/schemas/UserWithPermissionInfoRead" + UserWithPermissionInfoRead: + type: object + properties: + name: + description: Caption name for the user + type: string + userId: + $ref: "#/components/schemas/UserId" + email: + type: string + format: email + permissionId: + type: string + format: uuid InvalidInputProperty: type: object required: diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java index 0bbe88fe9c9..f401df1a61c 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/UserHandler.java @@ -15,6 +15,8 @@ import io.airbyte.api.model.generated.UserRead; import io.airbyte.api.model.generated.UserStatus; import io.airbyte.api.model.generated.UserUpdate; +import io.airbyte.api.model.generated.UserWithPermissionInfoRead; +import io.airbyte.api.model.generated.UserWithPermissionInfoReadList; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.api.model.generated.WorkspaceUserRead; import io.airbyte.api.model.generated.WorkspaceUserReadList; @@ -44,7 +46,7 @@ /** * UserHandler, provides basic CRUD operation access for users. Some are migrated from Cloud - * UserHandler {@link io.airbyte.cloud.server.handlers.UserHandler}. + * UserHandler. */ @SuppressWarnings({"MissingJavadocMethod"}) @Singleton @@ -253,6 +255,11 @@ public WorkspaceUserReadList listUsersInWorkspace(final WorkspaceIdRequestBody w return buildWorkspaceUserReadList(userPermissions, workspaceId); } + public UserWithPermissionInfoReadList listInstanceAdminUsers() throws IOException { + final List userPermissions = permissionPersistence.listInstanceAdminUsers(); + return buildUserWithPermissionInfoReadList(userPermissions); + } + private Map collectUserPermissionToMap(final List userPermissions) { return userPermissions.stream() .collect(Collectors.toMap( @@ -278,6 +285,18 @@ private WorkspaceUserReadList buildWorkspaceUserReadList(final List userPermissions) { + return new UserWithPermissionInfoReadList().users( + collectUserPermissionToMap(userPermissions) + .entrySet().stream() + .map((Entry entry) -> new UserWithPermissionInfoRead() + .userId(entry.getKey().getUserId()) + .email(entry.getKey().getEmail()) + .name(entry.getKey().getName()) + .permissionId(entry.getValue().getPermissionId())) + .collect(Collectors.toList())); + } + private OrganizationUserReadList buildOrganizationUserReadList(final List userPermissions, final UUID organizationId) { return new OrganizationUserReadList().users(collectUserPermissionToMap(userPermissions) .entrySet().stream() diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java index 7be0fff4441..0598bdbb0ee 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java @@ -197,9 +197,23 @@ public List listUsersInWorkspace(final UUID workspaceId) throws return this.database.query(ctx -> listPermissionsForWorkspace(ctx, workspaceId)); } + public List listInstanceAdminUsers() throws IOException { + return this.database.query(ctx -> listInstanceAdminPermissions(ctx)); + } + public List listUsersInOrganization(final UUID organizationId) throws IOException { return this.database.query(ctx -> listPermissionsForOrganization(ctx, organizationId)); + } + + private List listInstanceAdminPermissions(final DSLContext ctx) { + var records = ctx.select(USER.ID, USER.NAME, USER.EMAIL, USER.DEFAULT_WORKSPACE_ID, PERMISSION.ID, PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.PERMISSION_TYPE.eq(io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.instance_admin)) + .fetch(); + return records.stream().map(record -> buildUserPermissionFromRecord(record)).collect(Collectors.toList()); } public PermissionType findPermissionTypeForUserAndWorkspace(final UUID workspaceId, final String authUserId, final AuthProvider authProvider) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java index 86ff2e03bcf..cefe562bf11 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/PermissionPersistenceTest.java @@ -104,6 +104,14 @@ void listUsersInWorkspaceTest() throws IOException { Assertions.assertEquals(2, userPermissions.size()); } + @Test + void listInstanceUsersTest() throws IOException { + final List userPermissions = permissionPersistence.listInstanceAdminUsers(); + Assertions.assertEquals(1, userPermissions.size()); + UserPermission userPermission = userPermissions.get(0); + Assertions.assertEquals(MockData.CREATOR_USER_ID_1, userPermission.getUser().getUserId()); + } + @Test void findUsersInWorkspaceTest() throws Exception { final PermissionType permissionType = permissionPersistence diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java index 94d0ef194eb..58705529ba0 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/UserApiController.java @@ -15,6 +15,7 @@ import io.airbyte.api.model.generated.UserIdRequestBody; import io.airbyte.api.model.generated.UserRead; import io.airbyte.api.model.generated.UserUpdate; +import io.airbyte.api.model.generated.UserWithPermissionInfoReadList; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.api.model.generated.WorkspaceUserReadList; import io.airbyte.commons.auth.SecuredUser; @@ -28,10 +29,8 @@ import io.micronaut.security.rules.SecurityRule; /** - * This class is migrated from cloud-server UserApiController - * {@link io.airbyte.cloud.server.apis.UserApiController}. - * - * TODO: migrate all User endpoints (including some endpoints in WebBackend API) from Cloud to OSS. + * User related APIs. TODO: migrate all User endpoints (including some endpoints in WebBackend API) + * from Cloud to OSS. */ @SuppressWarnings("MissingJavadocType") @Controller("/api/v1/users") @@ -104,4 +103,13 @@ public WorkspaceUserReadList listUsersInWorkspace(@Body final WorkspaceIdRequest return ApiHelper.execute(() -> userHandler.listUsersInWorkspace(workspaceIdRequestBody)); } + // TODO: Update permission to instance admin once the permission PR is merged. + @Post("/list_instance_admins") + @Secured({READER}) + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public UserWithPermissionInfoReadList listInstanceAdminUsers() { + return ApiHelper.execute(() -> userHandler.listInstanceAdminUsers()); + } + } diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java index cd228d40a1b..52f1767170f 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/UserApiControllerTest.java @@ -11,6 +11,7 @@ import io.airbyte.api.model.generated.UserIdRequestBody; import io.airbyte.api.model.generated.UserRead; import io.airbyte.api.model.generated.UserUpdate; +import io.airbyte.api.model.generated.UserWithPermissionInfoReadList; import io.airbyte.api.model.generated.WorkspaceIdRequestBody; import io.airbyte.api.model.generated.WorkspaceUserReadList; import io.airbyte.commons.json.Jsons; @@ -99,4 +100,14 @@ void testListUsersInWorkspace() throws Exception { HttpStatus.OK); } + @Test + void testListInstanceAdminUsers() throws Exception { + Mockito.when(userHandler.listInstanceAdminUsers()) + .thenReturn(new UserWithPermissionInfoReadList()); + final String path = "/api/v1/users/list_instance_admin"; + testEndpointStatus( + HttpRequest.POST(path, Jsons.emptyObject()), + HttpStatus.OK); + } + } From 6016f82478d4dc119a12c7b19dcc91a3ab85ef79 Mon Sep 17 00:00:00 2001 From: Jon Tan Date: Wed, 6 Sep 2023 19:54:31 -0500 Subject: [PATCH 124/201] Fix OSS Airbyte API jobs controller endpoints (#8718) --- .../api/server/controllers/JobsController.kt | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt index 20d54f872f0..a059924db0a 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt @@ -13,6 +13,7 @@ import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.Ord import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.OrderByMethodEnum import io.airbyte.api.server.apiTracking.TrackingHelper import io.airbyte.api.server.constants.DELETE +import io.airbyte.api.server.constants.ENDPOINT_API_USER_INFO_HEADER import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.JOBS_PATH import io.airbyte.api.server.constants.JOBS_WITH_ID_PATH @@ -27,6 +28,11 @@ import io.airbyte.api.server.services.UserService import io.micronaut.http.annotation.Controller import java.time.OffsetDateTime import java.util.UUID +import javax.ws.rs.DELETE +import javax.ws.rs.GET +import javax.ws.rs.HeaderParam +import javax.ws.rs.Path +import javax.ws.rs.PathParam import javax.ws.rs.core.Response @Controller(JOBS_PATH) @@ -35,7 +41,13 @@ open class JobsController( private val userService: UserService, private val connectionService: ConnectionService, ) : JobsApi { - override fun cancelJob(jobId: Long, userInfo: String?): Response { + + @DELETE + @Path("/{jobId}") + override fun cancelJob( + @PathParam("jobId") jobId: Long, + @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, + ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) val jobResponse: Any? = TrackingHelper.callWithTracker( @@ -130,7 +142,12 @@ open class JobsController( } } - override fun getJob(jobId: Long, userInfo: String?): Response { + @GET + @Path("/{jobId}") + override fun getJob( + @PathParam("jobId") jobId: Long, + @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, + ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) val jobResponse: Any? = TrackingHelper.callWithTracker( From 44960022a4905789ec1b9dade913b3e379c872d5 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Wed, 6 Sep 2023 23:37:22 -0400 Subject: [PATCH 125/201] feat: disable syncs when version becomes unsupported (#8427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What close https://github.com/airbytehq/airbyte/issues/29803 With the breaking changes project, we want to pause syncs [in cloud] when the upgrade deadline is passed. The SupportStateUpdater is already setting ADV states to `unsupported` when the deadline passes. This extends the updater to also disable any syncs that have actors with unsupported versions. A follow up PR (https://github.com/airbytehq/airbyte-platform-internal/pull/8678) prevents these connections from being manually enabled again / prevents syncs from getting manually triggered. ## How - Add config repository methods to fetch source/destination actors using a specific version - Add new SupportStateUpdater methods to get syncs to disable by source def. - Checks that there isn't an override applied for the source in question - Internally this builds up a StandardSyncQuery per workspace to use the existing `configRepository.listWorkspaceStandardSyncs` method. I preferred this than introducing a new method because getting syncs is apparently kind of complex. - Add method to actually disable the syncs. It filters out anything that's already disabled and writes one by one to ensure we trigger anything we need to. Going one by one is ok in cron and is what billing cron does. - Add a feature flag to allow us to shut this off just in case. Should be safe to turn off since turning it back on will process any unsupported versions again. ## Can this PR be safely reverted / rolled back? *If you know that your PR is backwards-compatible and can be simply reverted or rolled back, check the YES box.* *Otherwise if your PR has a breaking change, like a database migration for example, check the NO box.* *If unsure, leave it blank.* - [x] YES 💚 - [ ] NO ❌ --- .../io/airbyte/bootloader/BootloaderTest.java | 10 +- ...NoOpDefinitionVersionOverrideProvider.java | 29 ++ .../config/persistence/ConfigRepository.java | 48 ++- .../persistence/SupportStateUpdater.java | 189 ++++++++++-- .../persistence/ActorPersistenceTest.java | 97 ++++++ .../persistence/SupportStateUpdaterTest.java | 289 +++++++++++++++++- airbyte-cron/build.gradle | 1 + .../cron/config/ApplicationBeanFactory.java | 10 + .../src/main/resources/application.yml | 3 + .../src/main/kotlin/FlagDefinitions.kt | 2 + 10 files changed, 647 insertions(+), 31 deletions(-) create mode 100644 airbyte-bootloader/src/test/java/io/airbyte/bootloader/NoOpDefinitionVersionOverrideProvider.java diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 7db72267bad..d9d50cac37b 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -15,10 +15,12 @@ import io.airbyte.commons.version.AirbyteProtocolVersionRange; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.commons.version.Version; +import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.init.ApplyDefinitionsHelper; import io.airbyte.config.init.CdkVersionProvider; import io.airbyte.config.init.DeclarativeSourceUpdater; import io.airbyte.config.init.PostLoadExecutor; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.SupportStateUpdater; import io.airbyte.config.specs.DefinitionsProvider; @@ -133,7 +135,9 @@ void testBootloaderAppBlankDb() throws Exception { val jobsDatabaseMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); val jobsPersistence = new DefaultJobPersistence(jobDatabase); val protocolVersionChecker = new ProtocolVersionChecker(jobsPersistence, airbyteProtocolRange, configRepository, definitionsProvider); - val supportStateUpdater = new SupportStateUpdater(configRepository); + val overrideProvider = new NoOpDefinitionVersionOverrideProvider(); + val actorDefinitionVersionHelper = new ActorDefinitionVersionHelper(configRepository, overrideProvider, featureFlagClient); + val supportStateUpdater = new SupportStateUpdater(configRepository, actorDefinitionVersionHelper, DeploymentMode.OSS, featureFlagClient); val applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient, supportStateUpdater); final CdkVersionProvider cdkVersionProvider = mock(CdkVersionProvider.class); @@ -188,7 +192,9 @@ void testRequiredVersionUpgradePredicate() throws Exception { jobsDatabaseInitializationTimeoutMs, MoreResources.readResource(DatabaseConstants.JOBS_INITIAL_SCHEMA_PATH)); val jobsDatabaseMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); val jobsPersistence = new DefaultJobPersistence(jobDatabase); - val supportStateUpdater = new SupportStateUpdater(configRepository); + val overrideProvider = new NoOpDefinitionVersionOverrideProvider(); + val actorDefinitionVersionHelper = new ActorDefinitionVersionHelper(configRepository, overrideProvider, featureFlagClient); + val supportStateUpdater = new SupportStateUpdater(configRepository, actorDefinitionVersionHelper, DeploymentMode.OSS, featureFlagClient); val protocolVersionChecker = new ProtocolVersionChecker(jobsPersistence, airbyteProtocolRange, configRepository, definitionsProvider); val applyDefinitionsHelper = new ApplyDefinitionsHelper(definitionsProvider, jobsPersistence, configRepository, featureFlagClient, supportStateUpdater); diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/NoOpDefinitionVersionOverrideProvider.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/NoOpDefinitionVersionOverrideProvider.java new file mode 100644 index 00000000000..867d77ce68f --- /dev/null +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/NoOpDefinitionVersionOverrideProvider.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.bootloader; + +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.ActorType; +import io.airbyte.config.persistence.version_overrides.DefinitionVersionOverrideProvider; +import java.util.Optional; +import java.util.UUID; +import org.jetbrains.annotations.Nullable; + +/** + * Implementation of {@link DefinitionVersionOverrideProvider} that does not override any versions. + * Used for testing. + */ +class NoOpDefinitionVersionOverrideProvider implements DefinitionVersionOverrideProvider { + + @Override + public Optional getOverride(final ActorType actorType, + final UUID actorDefinitionId, + final UUID workspaceId, + @Nullable final UUID actorId, + final ActorDefinitionVersion defaultVersion) { + return Optional.empty(); + } + +} diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 171c849a4fc..6c6c1038699 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -1602,6 +1602,38 @@ public List listSourcesForDefinition(final UUID definitionId) return result.stream().map(DbConverter::buildSourceConnection).collect(Collectors.toList()); } + /** + * Returns all active sources whose default_version_id is in a given list of version IDs. + * + * @param actorDefinitionVersionIds - list of actor definition version ids + * @return list of SourceConnections + * @throws IOException - you never know when you IO + */ + public List listSourcesWithVersionIds(final List actorDefinitionVersionIds) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(ACTOR) + .where(ACTOR.ACTOR_TYPE.eq(ActorType.source)) + .and(ACTOR.DEFAULT_VERSION_ID.in(actorDefinitionVersionIds)) + .andNot(ACTOR.TOMBSTONE).fetch()); + return result.stream().map(DbConverter::buildSourceConnection).toList(); + } + + /** + * Returns all active destinations whose default_version_id is in a given list of version IDs. + * + * @param actorDefinitionVersionIds - list of actor definition version ids + * @return list of DestinationConnections + * @throws IOException - you never know when you IO + */ + public List listDestinationsWithVersionIds(final List actorDefinitionVersionIds) throws IOException { + final Result result = database.query(ctx -> ctx.select(asterisk()) + .from(ACTOR) + .where(ACTOR.ACTOR_TYPE.eq(ActorType.destination)) + .and(ACTOR.DEFAULT_VERSION_ID.in(actorDefinitionVersionIds)) + .andNot(ACTOR.TOMBSTONE).fetch()); + return result.stream().map(DbConverter::buildDestinationConnection).toList(); + } + /** * Returns all active destinations using a definition. * @@ -2440,33 +2472,33 @@ private UUID getOrInsertCanonicalActorCatalog(final AirbyteCatalog airbyteCatalo return insertCatalog(airbyteCatalog, catalogHash, context, timestamp); } - private String generateCanonicalHash(AirbyteCatalog airbyteCatalog) { + private String generateCanonicalHash(final AirbyteCatalog airbyteCatalog) { final HashFunction hashFunction = Hashing.murmur3_32_fixed(); try { return hashFunction.hashBytes(Jsons.canonicalJsonSerialize(airbyteCatalog) .getBytes(Charsets.UTF_8)).toString(); - } catch (IOException e) { + } catch (final IOException e) { LOGGER.error("Failed to serialize AirbyteCatalog to canonical JSON", e); return null; } } - private String generateOldHash(AirbyteCatalog airbyteCatalog) { + private String generateOldHash(final AirbyteCatalog airbyteCatalog) { final HashFunction hashFunction = Hashing.murmur3_32_fixed(); return hashFunction.hashBytes(Jsons.serialize(airbyteCatalog).getBytes(Charsets.UTF_8)).toString(); } - private UUID lookupCatalogId(String catalogHash, AirbyteCatalog airbyteCatalog, DSLContext context) { + private UUID lookupCatalogId(final String catalogHash, final AirbyteCatalog airbyteCatalog, final DSLContext context) { if (catalogHash == null) { return null; } return findAndReturnCatalogId(catalogHash, airbyteCatalog, context); } - private UUID insertCatalog(AirbyteCatalog airbyteCatalog, - String catalogHash, - DSLContext context, - OffsetDateTime timestamp) { + private UUID insertCatalog(final AirbyteCatalog airbyteCatalog, + final String catalogHash, + final DSLContext context, + final OffsetDateTime timestamp) { final UUID catalogId = UUID.randomUUID(); context.insertInto(ACTOR_CATALOG) .set(ACTOR_CATALOG.ID, catalogId) diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java index a96a3880b96..ab973503849 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/SupportStateUpdater.java @@ -4,18 +4,34 @@ package io.airbyte.config.persistence; +import static io.airbyte.featureflag.ContextKt.ANONYMOUS; + import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.version.Version; import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ActorDefinitionVersion.SupportState; +import io.airbyte.config.Configs.DeploymentMode; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; +import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; +import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.PauseSyncsWithUnsupportedActors; +import io.airbyte.featureflag.Workspace; +import io.airbyte.validation.json.JsonValidationException; import jakarta.inject.Singleton; import java.io.IOException; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -23,8 +39,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Updates the support state of actor definition versions according to breaking changes. @@ -52,10 +66,28 @@ public static SupportStateUpdate merge(final SupportStateUpdate a, final Support } private final ConfigRepository configRepository; - private static final Logger LOGGER = LoggerFactory.getLogger(SupportStateUpdater.class); - - public SupportStateUpdater(final ConfigRepository configRepository) { + private final ActorDefinitionVersionHelper actorDefinitionVersionHelper; + private final DeploymentMode deploymentMode; + private final FeatureFlagClient featureFlagClient; + + public SupportStateUpdater(final ConfigRepository configRepository, + final ActorDefinitionVersionHelper actorDefinitionVersionHelper, + final DeploymentMode deploymentMode, + final FeatureFlagClient featureFlagClient) { this.configRepository = configRepository; + this.actorDefinitionVersionHelper = actorDefinitionVersionHelper; + this.deploymentMode = deploymentMode; + this.featureFlagClient = featureFlagClient; + } + + @VisibleForTesting + boolean shouldDisableSyncs() { + if (!featureFlagClient.boolVariation(PauseSyncsWithUnsupportedActors.INSTANCE, new Workspace(ANONYMOUS))) { + return false; + } + + // We only disable syncs on Cloud. OSS users can continue to run on unsupported versions. + return deploymentMode == DeploymentMode.CLOUD; } /** @@ -137,13 +169,15 @@ private SupportState calcVersionSupportState(final Version version, /** * Updates the version support states for a given source definition. */ - public void updateSupportStatesForSourceDefinition(final StandardSourceDefinition sourceDefinition) throws ConfigNotFoundException, IOException { + public void updateSupportStatesForSourceDefinition(final StandardSourceDefinition sourceDefinition) + throws ConfigNotFoundException, IOException { if (!sourceDefinition.getCustom()) { - LOGGER.info("Updating support states for source definition: {}", sourceDefinition.getName()); + log.info("Updating support states for source definition: {}", sourceDefinition.getName()); final ActorDefinitionVersion defaultActorDefinitionVersion = configRepository.getActorDefinitionVersion(sourceDefinition.getDefaultVersionId()); final Version currentDefaultVersion = new Version(defaultActorDefinitionVersion.getDockerImageTag()); updateSupportStatesForActorDefinition(sourceDefinition.getSourceDefinitionId(), currentDefaultVersion); - LOGGER.info("Finished updating support states for source definition: {}", sourceDefinition.getName()); + + log.info("Finished updating support states for source definition: {}", sourceDefinition.getName()); } } @@ -153,12 +187,13 @@ public void updateSupportStatesForSourceDefinition(final StandardSourceDefinitio public void updateSupportStatesForDestinationDefinition(final StandardDestinationDefinition destinationDefinition) throws ConfigNotFoundException, IOException { if (!destinationDefinition.getCustom()) { - LOGGER.info("Updating support states for destination definition: {}", destinationDefinition.getName()); + log.info("Updating support states for destination definition: {}", destinationDefinition.getName()); final ActorDefinitionVersion defaultActorDefinitionVersion = configRepository.getActorDefinitionVersion(destinationDefinition.getDefaultVersionId()); final Version currentDefaultVersion = new Version(defaultActorDefinitionVersion.getDockerImageTag()); updateSupportStatesForActorDefinition(destinationDefinition.getDestinationDefinitionId(), currentDefaultVersion); - LOGGER.info("Finished updating support states for destination definition: {}", destinationDefinition.getName()); + + log.info("Finished updating support states for destination definition: {}", destinationDefinition.getName()); } } @@ -178,20 +213,43 @@ private Version getVersionTag(final List actorDefinition .orElseThrow(); } + /** + * Calculates unsupported version IDs after the support state update would be applied. + *

+ * This is done by taking the set of all previously unsupported versions, adding newly unsupported + * versions, and removing any versions that would be supported or deprecated after the update. + * + * @param versionsBeforeUpdate - actor definition versions before applying the SupportStateUpdate + * @param supportStateUpdate - the SupportStateUpdate that would be applied + * @return - unsupported actor definition version IDs + */ + @VisibleForTesting + List getUnsupportedVersionIdsAfterUpdate(final List versionsBeforeUpdate, + final SupportStateUpdate supportStateUpdate) { + final List prevUnsupportedVersionIds = versionsBeforeUpdate.stream() + .filter(v -> v.getSupportState() == SupportState.UNSUPPORTED) + .map(ActorDefinitionVersion::getVersionId) + .filter(verId -> !supportStateUpdate.supportedVersionIds().contains(verId) && !supportStateUpdate.deprecatedVersionIds().contains(verId)) + .toList(); + return Stream.of(prevUnsupportedVersionIds, supportStateUpdate.unsupportedVersionIds()) + .flatMap(Collection::stream) + .toList(); + } + /** * Updates the version support states for all source and destination definitions. */ - public void updateSupportStates() throws IOException { + public void updateSupportStates() throws IOException, JsonValidationException, ConfigNotFoundException { updateSupportStates(LocalDate.now()); } /** * Updates the version support states for all source and destination definitions based on a - * reference date. + * reference date, and disables syncs with unsupported versions. */ @VisibleForTesting - void updateSupportStates(final LocalDate referenceDate) throws IOException { - LOGGER.info("Updating support states for all definitions"); + void updateSupportStates(final LocalDate referenceDate) throws IOException, JsonValidationException, ConfigNotFoundException { + log.info("Updating support states for all definitions"); final List sourceDefinitions = configRepository.listPublicSourceDefinitions(false); final List destinationDefinitions = configRepository.listPublicDestinationDefinitions(false); final List allBreakingChanges = configRepository.listBreakingChanges(); @@ -199,6 +257,7 @@ void updateSupportStates(final LocalDate referenceDate) throws IOException { .collect(Collectors.groupingBy(ActorDefinitionBreakingChange::getActorDefinitionId)); SupportStateUpdate comboSupportStateUpdate = new SupportStateUpdate(List.of(), List.of(), List.of()); + final List syncsToDisable = new ArrayList<>(); for (final StandardSourceDefinition sourceDefinition : sourceDefinitions) { final List actorDefinitionVersions = @@ -210,6 +269,10 @@ void updateSupportStates(final LocalDate referenceDate) throws IOException { final SupportStateUpdate supportStateUpdate = getSupportStateUpdate(currentDefaultVersion, referenceDate, breakingChangesForDef, actorDefinitionVersions); comboSupportStateUpdate = SupportStateUpdate.merge(comboSupportStateUpdate, supportStateUpdate); + + final List unsupportedVersionIds = getUnsupportedVersionIdsAfterUpdate(actorDefinitionVersions, supportStateUpdate); + final List syncsToDisableForSource = getSyncsToDisableForSource(sourceDefinition, unsupportedVersionIds); + syncsToDisable.addAll(syncsToDisableForSource); } for (final StandardDestinationDefinition destinationDefinition : destinationDefinitions) { @@ -222,10 +285,104 @@ void updateSupportStates(final LocalDate referenceDate) throws IOException { final SupportStateUpdate supportStateUpdate = getSupportStateUpdate(currentDefaultVersion, referenceDate, breakingChangesForDef, actorDefinitionVersions); comboSupportStateUpdate = SupportStateUpdate.merge(comboSupportStateUpdate, supportStateUpdate); + + final List unsupportedVersionIds = getUnsupportedVersionIdsAfterUpdate(actorDefinitionVersions, supportStateUpdate); + final List syncsToDisableForDestination = getSyncsToDisableForDestination(destinationDefinition, unsupportedVersionIds); + syncsToDisable.addAll(syncsToDisableForDestination); } executeSupportStateUpdate(comboSupportStateUpdate); - LOGGER.info("Finished updating support states for all definitions"); + disableSyncs(syncsToDisable); + log.info("Finished updating support states for all definitions"); + } + + @VisibleForTesting + List getSyncsToDisableForSource(final StandardSourceDefinition sourceDefinition, final List unsupportedVersionIds) + throws IOException, JsonValidationException, ConfigNotFoundException { + if (!shouldDisableSyncs() || unsupportedVersionIds.isEmpty()) { + return Collections.emptyList(); + } + + final List syncsToDisable = new ArrayList<>(); + final List sourceConnections = configRepository.listSourcesWithVersionIds(unsupportedVersionIds); + final Map> sourceConnectionsByWorkspace = new HashMap<>(); + + // verify that a version override has not been applied to the source, and collect by workspace + for (final SourceConnection source : sourceConnections) { + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, source.getWorkspaceId(), source.getSourceId()); + if (!versionWithOverrideStatus.isOverrideApplied()) { + sourceConnectionsByWorkspace + .computeIfAbsent(source.getWorkspaceId(), k -> new ArrayList<>()) + .add(source); + } + } + + // get affected syncs for each workspace and add them to the list + for (final Map.Entry> entry : sourceConnectionsByWorkspace.entrySet()) { + final UUID workspaceId = entry.getKey(); + final List sourcesForWorkspace = entry.getValue(); + final List sourceIds = sourcesForWorkspace.stream().map(SourceConnection::getSourceId).toList(); + final StandardSyncQuery syncQuery = new StandardSyncQuery(workspaceId, sourceIds, null, false); + final List standardSyncs = configRepository.listWorkspaceStandardSyncs(syncQuery); + syncsToDisable.addAll(standardSyncs); + } + + return syncsToDisable; + } + + @VisibleForTesting + List getSyncsToDisableForDestination(final StandardDestinationDefinition destinationDefinition, + final List unsupportedVersionIds) + throws IOException, JsonValidationException, ConfigNotFoundException { + if (!shouldDisableSyncs() || unsupportedVersionIds.isEmpty()) { + return Collections.emptyList(); + } + + final List syncsToDisable = new ArrayList<>(); + final List destinationConnections = configRepository.listDestinationsWithVersionIds(unsupportedVersionIds); + final Map> destinationConnectionsByWorkspace = new HashMap<>(); + + // verify that a version override has not been applied to the destination, and collect by workspace + for (final DestinationConnection destination : destinationConnections) { + final ActorDefinitionVersionWithOverrideStatus versionWithOverrideStatus = + actorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus( + destinationDefinition, + destination.getWorkspaceId(), + destination.getDestinationId()); + if (!versionWithOverrideStatus.isOverrideApplied()) { + destinationConnectionsByWorkspace + .computeIfAbsent(destination.getWorkspaceId(), k -> new ArrayList<>()) + .add(destination); + } + } + + // get affected syncs for each workspace and add them to the list + for (final Map.Entry> entry : destinationConnectionsByWorkspace.entrySet()) { + final UUID workspaceId = entry.getKey(); + final List destinationsForWorkspace = entry.getValue(); + final List destinationIds = destinationsForWorkspace.stream().map(DestinationConnection::getDestinationId).toList(); + final StandardSyncQuery syncQuery = new StandardSyncQuery(workspaceId, null, destinationIds, false); + final List standardSyncs = configRepository.listWorkspaceStandardSyncs(syncQuery); + syncsToDisable.addAll(standardSyncs); + } + + return syncsToDisable; + } + + @VisibleForTesting + void disableSyncs(final List syncsToDisable) throws IOException { + final List activeSyncs = syncsToDisable.stream() + .filter(s -> s.getStatus() == Status.ACTIVE) + .toList(); + + for (final StandardSync sync : activeSyncs) { + configRepository.writeStandardSync(sync.withStatus(Status.INACTIVE)); + } + + if (!activeSyncs.isEmpty()) { + log.info("Disabled {} syncs with unsupported versions", activeSyncs.size()); + } } /** @@ -234,8 +391,6 @@ void updateSupportStates(final LocalDate referenceDate) throws IOException { * @param supportStateUpdate - the SupportStateUpdate to process. */ private void executeSupportStateUpdate(final SupportStateUpdate supportStateUpdate) throws IOException { - // TODO(pedro): This is likely where we disable syncs for the now-unsupported versions. - if (!supportStateUpdate.unsupportedVersionIds.isEmpty()) { configRepository.setActorDefinitionVersionSupportStates(supportStateUpdate.unsupportedVersionIds, ActorDefinitionVersion.SupportState.UNSUPPORTED); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java index 5225fd2d646..1ca44d4a673 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java @@ -22,6 +22,7 @@ import java.sql.SQLException; import java.util.List; import java.util.UUID; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -263,4 +264,100 @@ void testSourceDefaultVersionIsNotModifiedOnBreakingUpgrade() assertEquals(initialSourceDefaultVersionId, sourceDefaultVersionIdAfterUpgrade); } + @Test + void testGetSourcesWithVersions() throws IOException { + final SourceConnection sourceConnection1 = new SourceConnection() + .withSourceId(UUID.randomUUID()) + .withSourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) + .withWorkspaceId(WORKSPACE_ID) + .withName(SOURCE_NAME); + configRepository.writeSourceConnectionNoSecrets(sourceConnection1); + + final SourceConnection sourceConnection2 = new SourceConnection() + .withSourceId(UUID.randomUUID()) + .withSourceDefinitionId(standardSourceDefinition.getSourceDefinitionId()) + .withWorkspaceId(WORKSPACE_ID) + .withName(SOURCE_NAME); + configRepository.writeSourceConnectionNoSecrets(sourceConnection2); + + final List sourceConnections = + configRepository.listSourcesWithVersionIds(List.of(standardSourceDefinition.getDefaultVersionId())); + assertEquals(2, sourceConnections.size()); + assertEquals( + Stream.of(sourceConnection1.getSourceId(), sourceConnection2.getSourceId()).sorted().toList(), + sourceConnections.stream().map(SourceConnection::getSourceId).sorted().toList()); + + final ActorDefinitionVersion newActorDefinitionVersion = configRepository.writeActorDefinitionVersion(MockData.actorDefinitionVersion() + .withActorDefinitionId(standardSourceDefinition.getSourceDefinitionId()) + .withVersionId(UUID.randomUUID()) + .withDockerImageTag(UPGRADE_IMAGE_TAG)); + + configRepository.setActorDefaultVersion(sourceConnection1.getSourceId(), newActorDefinitionVersion.getVersionId()); + + final List sourcesWithNewVersion = + configRepository.listSourcesWithVersionIds(List.of(newActorDefinitionVersion.getVersionId())); + assertEquals(1, sourcesWithNewVersion.size()); + assertEquals(sourceConnection1.getSourceId(), sourcesWithNewVersion.get(0).getSourceId()); + + final List sourcesWithOldVersion = + configRepository.listSourcesWithVersionIds(List.of(standardSourceDefinition.getDefaultVersionId())); + assertEquals(1, sourcesWithOldVersion.size()); + assertEquals(sourceConnection2.getSourceId(), sourcesWithOldVersion.get(0).getSourceId()); + + final List sourcesWithBothVersions = + configRepository.listSourcesWithVersionIds(List.of(standardSourceDefinition.getDefaultVersionId(), newActorDefinitionVersion.getVersionId())); + assertEquals(2, sourcesWithBothVersions.size()); + assertEquals( + Stream.of(sourceConnection1.getSourceId(), sourceConnection2.getSourceId()).sorted().toList(), + sourcesWithBothVersions.stream().map(SourceConnection::getSourceId).sorted().toList()); + } + + @Test + void testGetDestinationsWithVersions() throws IOException { + final DestinationConnection destinationConnection1 = new DestinationConnection() + .withDestinationId(UUID.randomUUID()) + .withDestinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) + .withWorkspaceId(WORKSPACE_ID) + .withName(DESTINATION_NAME); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection1); + + final DestinationConnection destinationConnection2 = new DestinationConnection() + .withDestinationId(UUID.randomUUID()) + .withDestinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) + .withWorkspaceId(WORKSPACE_ID) + .withName(DESTINATION_NAME); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection2); + + final List destinationConnections = + configRepository.listDestinationsWithVersionIds(List.of(standardDestinationDefinition.getDefaultVersionId())); + assertEquals(2, destinationConnections.size()); + assertEquals( + Stream.of(destinationConnection1.getDestinationId(), destinationConnection2.getDestinationId()).sorted().toList(), + destinationConnections.stream().map(DestinationConnection::getDestinationId).sorted().toList()); + + final ActorDefinitionVersion newActorDefinitionVersion = configRepository.writeActorDefinitionVersion(MockData.actorDefinitionVersion() + .withActorDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) + .withVersionId(UUID.randomUUID()) + .withDockerImageTag(UPGRADE_IMAGE_TAG)); + + configRepository.setActorDefaultVersion(destinationConnection1.getDestinationId(), newActorDefinitionVersion.getVersionId()); + + final List destinationsWithNewVersion = + configRepository.listDestinationsWithVersionIds(List.of(newActorDefinitionVersion.getVersionId())); + assertEquals(1, destinationsWithNewVersion.size()); + assertEquals(destinationConnection1.getDestinationId(), destinationsWithNewVersion.get(0).getDestinationId()); + + final List destinationsWithOldVersion = + configRepository.listDestinationsWithVersionIds(List.of(standardDestinationDefinition.getDefaultVersionId())); + assertEquals(1, destinationsWithOldVersion.size()); + assertEquals(destinationConnection2.getDestinationId(), destinationsWithOldVersion.get(0).getDestinationId()); + + final List destinationsWithBothVersions = configRepository + .listDestinationsWithVersionIds(List.of(standardDestinationDefinition.getDefaultVersionId(), newActorDefinitionVersion.getVersionId())); + assertEquals(2, destinationsWithBothVersions.size()); + assertEquals( + Stream.of(destinationConnection1.getDestinationId(), destinationConnection2.getDestinationId()).sorted().toList(), + destinationsWithBothVersions.stream().map(DestinationConnection::getDestinationId).sorted().toList()); + } + } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java index 8ee154d1031..3a607a6c21c 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/SupportStateUpdaterTest.java @@ -4,7 +4,10 @@ package io.airbyte.config.persistence; +import static io.airbyte.featureflag.ContextKt.ANONYMOUS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -15,32 +18,56 @@ import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.ActorDefinitionVersion.SupportState; +import io.airbyte.config.Configs.DeploymentMode; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardSync; +import io.airbyte.config.StandardSync.Status; +import io.airbyte.config.persistence.ActorDefinitionVersionHelper.ActorDefinitionVersionWithOverrideStatus; +import io.airbyte.config.persistence.ConfigRepository.StandardSyncQuery; import io.airbyte.config.persistence.SupportStateUpdater.SupportStateUpdate; +import io.airbyte.featureflag.FeatureFlagClient; +import io.airbyte.featureflag.PauseSyncsWithUnsupportedActors; +import io.airbyte.featureflag.TestClient; +import io.airbyte.featureflag.Workspace; +import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.time.LocalDate; import java.util.List; import java.util.UUID; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class SupportStateUpdaterTest { private static final UUID ACTOR_DEFINITION_ID = UUID.randomUUID(); + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + private static final UUID WORKSPACE_ID_2 = UUID.randomUUID(); private static final String V0_1_0 = "0.1.0"; private static final String V1_0_0 = "1.0.0"; private static final String V1_1_0 = "1.1.0"; private static final String V2_0_0 = "2.0.0"; + private static final String V3_0_0 = "3.0.0"; private ConfigRepository mConfigRepository; + private ActorDefinitionVersionHelper mActorDefinitionVersionHelper; + private FeatureFlagClient mFeatureFlagClient; + private SupportStateUpdater supportStateUpdater; @BeforeEach void setup() { mConfigRepository = mock(ConfigRepository.class); - supportStateUpdater = new SupportStateUpdater(mConfigRepository); + mActorDefinitionVersionHelper = mock(ActorDefinitionVersionHelper.class); + mFeatureFlagClient = mock(TestClient.class); + supportStateUpdater = new SupportStateUpdater(mConfigRepository, mActorDefinitionVersionHelper, DeploymentMode.CLOUD, mFeatureFlagClient); + + when(mFeatureFlagClient.boolVariation(PauseSyncsWithUnsupportedActors.INSTANCE, new Workspace(ANONYMOUS))) + .thenReturn(true); } ActorDefinitionBreakingChange createBreakingChange(final String version, final String upgradeDeadline) { @@ -61,6 +88,24 @@ ActorDefinitionVersion createActorDefinitionVersion(final String version) { .withSupportState(null); // clear support state to always need a SupportStateUpdate. } + @Test + void testShouldNotDisableSyncsInOSS() { + assertTrue(supportStateUpdater.shouldDisableSyncs()); + + supportStateUpdater = new SupportStateUpdater(mConfigRepository, mActorDefinitionVersionHelper, DeploymentMode.OSS, mFeatureFlagClient); + assertFalse(supportStateUpdater.shouldDisableSyncs()); + } + + @Test + void testShouldNotDisableSyncsWhenFFDisabled() { + assertTrue(supportStateUpdater.shouldDisableSyncs()); + + when(mFeatureFlagClient.boolVariation(PauseSyncsWithUnsupportedActors.INSTANCE, new Workspace(ANONYMOUS))) + .thenReturn(false); + + assertFalse(supportStateUpdater.shouldDisableSyncs()); + } + @Test void testUpdateSupportStatesForCustomDestinationDefinitionNoOp() throws ConfigNotFoundException, IOException { supportStateUpdater.updateSupportStatesForDestinationDefinition(new StandardDestinationDefinition().withCustom(true)); @@ -98,6 +143,7 @@ void testUpdateSupportStatesForDestinationDefinition() throws ConfigNotFoundExce verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_0_1_0.getVersionId()), SupportState.UNSUPPORTED); verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_1_0_0.getVersionId()), SupportState.SUPPORTED); verifyNoMoreInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); } @Test @@ -125,10 +171,11 @@ void testUpdateSupportStatesForSourceDefinition() throws ConfigNotFoundException verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_0_1_0.getVersionId()), SupportState.UNSUPPORTED); verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(ADV_1_0_0.getVersionId()), SupportState.SUPPORTED); verifyNoMoreInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); } @Test - void testUpdateSupportStates() throws IOException { + void testUpdateSupportStates() throws IOException, JsonValidationException, ConfigNotFoundException { final ActorDefinitionVersion SRC_V0_1_0 = createActorDefinitionVersion(V0_1_0); final ActorDefinitionVersion SRC_V1_0_0 = createActorDefinitionVersion(V1_0_0); final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() @@ -157,6 +204,7 @@ void testUpdateSupportStates() throws IOException { .thenReturn(List.of(SRC_V0_1_0, SRC_V1_0_0)); when(mConfigRepository.listActorDefinitionVersionsForDefinition(destinationDefinitionId)) .thenReturn(List.of(DEST_V0_1_0, DEST_V1_0_0)); + when(mConfigRepository.listSourcesWithVersionIds(List.of(SRC_V0_1_0.getVersionId()))).thenReturn(List.of()); supportStateUpdater.updateSupportStates(LocalDate.parse("2020-01-15")); @@ -165,11 +213,65 @@ void testUpdateSupportStates() throws IOException { verify(mConfigRepository).listBreakingChanges(); verify(mConfigRepository).listActorDefinitionVersionsForDefinition(ACTOR_DEFINITION_ID); verify(mConfigRepository).listActorDefinitionVersionsForDefinition(destinationDefinitionId); + verify(mConfigRepository).listSourcesWithVersionIds(List.of(SRC_V0_1_0.getVersionId())); verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V0_1_0.getVersionId()), SupportState.UNSUPPORTED); verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(DEST_V0_1_0.getVersionId()), SupportState.DEPRECATED); verify(mConfigRepository).setActorDefinitionVersionSupportStates(List.of(SRC_V1_0_0.getVersionId(), DEST_V1_0_0.getVersionId()), SupportState.SUPPORTED); verifyNoMoreInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testGetUnsupportedVersionIdsAfterUpdate() { + final ActorDefinitionVersion ADV_0_1_0 = createActorDefinitionVersion(V0_1_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0).withSupportState(SupportState.DEPRECATED); + final ActorDefinitionVersion ADV_1_1_0 = createActorDefinitionVersion(V1_1_0).withSupportState(SupportState.DEPRECATED); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0).withSupportState(SupportState.SUPPORTED); + final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion(V3_0_0).withSupportState(SupportState.SUPPORTED); + + final List versionsBeforeUpdate = List.of( + ADV_0_1_0, + ADV_1_0_0, + ADV_1_1_0, + ADV_2_0_0, + ADV_3_0_0); + final SupportStateUpdate supportStateUpdate = new SupportStateUpdate( + List.of(ADV_1_0_0.getVersionId(), ADV_1_1_0.getVersionId()), + List.of(ADV_2_0_0.getVersionId()), + List.of()); + + final List unsupportedVersionIds = supportStateUpdater.getUnsupportedVersionIdsAfterUpdate(versionsBeforeUpdate, supportStateUpdate); + + final List expectedUnsupportedVersionIds = List.of( + ADV_0_1_0.getVersionId(), + ADV_1_0_0.getVersionId(), + ADV_1_1_0.getVersionId()); + + assertEquals(expectedUnsupportedVersionIds, unsupportedVersionIds); + } + + @Test + void testGetUnsupportedVersionIdsAfterUpdateWithRollback() { + final ActorDefinitionVersion ADV_0_1_0 = createActorDefinitionVersion(V0_1_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_1_1_0 = createActorDefinitionVersion(V1_1_0).withSupportState(SupportState.UNSUPPORTED); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0).withSupportState(SupportState.DEPRECATED); + + final List versionsBeforeUpdate = List.of( + ADV_0_1_0, + ADV_1_0_0, + ADV_1_1_0, + ADV_2_0_0); + final SupportStateUpdate supportStateUpdate = new SupportStateUpdate( + List.of(), + List.of(ADV_1_0_0.getVersionId(), ADV_1_1_0.getVersionId()), + List.of(ADV_2_0_0.getVersionId())); + + final List unsupportedVersionIds = supportStateUpdater.getUnsupportedVersionIdsAfterUpdate(versionsBeforeUpdate, supportStateUpdate); + + final List expectedUnsupportedVersionIds = List.of(ADV_0_1_0.getVersionId()); + assertEquals(expectedUnsupportedVersionIds, unsupportedVersionIds); } @Test @@ -178,7 +280,7 @@ void testGetSupportStateUpdate() { final ActorDefinitionBreakingChange BC_1_0_0 = createBreakingChange(V1_0_0, "2021-02-01"); final ActorDefinitionBreakingChange BC_2_0_0 = createBreakingChange(V2_0_0, "2022-02-01"); - final ActorDefinitionBreakingChange BC_3_0_0 = createBreakingChange("3.0.0", "2024-01-01"); + final ActorDefinitionBreakingChange BC_3_0_0 = createBreakingChange(V3_0_0, "2024-01-01"); final ActorDefinitionBreakingChange BC_4_0_0 = createBreakingChange("4.0.0", "2025-01-01"); final ActorDefinitionBreakingChange BC_5_0_0 = createBreakingChange("5.0.0", "2026-01-01"); @@ -194,7 +296,7 @@ void testGetSupportStateUpdate() { final ActorDefinitionVersion ADV_1_1_0 = createActorDefinitionVersion(V1_1_0); final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0); final ActorDefinitionVersion ADV_2_1_0 = createActorDefinitionVersion("2.1.0").withSupportState(SupportState.DEPRECATED); - final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion("3.0.0"); + final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion(V3_0_0); final ActorDefinitionVersion ADV_3_1_0 = createActorDefinitionVersion("3.1.0"); final ActorDefinitionVersion ADV_4_0_0 = createActorDefinitionVersion("4.0.0"); final ActorDefinitionVersion ADV_4_1_0 = createActorDefinitionVersion("4.1.0").withSupportState(SupportState.SUPPORTED); @@ -226,6 +328,7 @@ void testGetSupportStateUpdate() { assertEquals(expectedSupportStateUpdate, supportStateUpdate); verifyNoInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); } @Test @@ -254,6 +357,184 @@ void testGetSupportStateUpdateNoBreakingChanges() { assertEquals(expectedSupportStateUpdate, supportStateUpdate); verifyNoInteractions(mConfigRepository); + verifyNoInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testGetSyncsToDisableForSource() throws JsonValidationException, ConfigNotFoundException, IOException { + final StandardSourceDefinition sourceDefinition = new StandardSourceDefinition() + .withSourceDefinitionId(ACTOR_DEFINITION_ID); + + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0); + final List unsupportedVersionIds = List.of(ADV_1_0_0.getVersionId(), ADV_2_0_0.getVersionId()); + + final SourceConnection sourceConnection = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID) + .withSourceId(UUID.randomUUID()); + final SourceConnection sourceConnection2 = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID) + .withSourceId(UUID.randomUUID()); + final SourceConnection sourceConnection3 = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID_2) + .withSourceId(UUID.randomUUID()); + + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, false)); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection2.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID_2, sourceConnection3.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + + final SourceConnection sourceWithOverride = new SourceConnection() + .withWorkspaceId(WORKSPACE_ID) + .withSourceId(UUID.randomUUID()); + when(mActorDefinitionVersionHelper.getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceWithOverride.getSourceId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, true)); + + when(mConfigRepository.listSourcesWithVersionIds(unsupportedVersionIds)) + .thenReturn(List.of(sourceConnection, sourceConnection2, sourceConnection3, sourceWithOverride)); + + final StandardSyncQuery workspaceQuery1 = new StandardSyncQuery( + WORKSPACE_ID, + List.of(sourceConnection.getSourceId(), sourceConnection2.getSourceId()), + null, false); + final List workspaceSyncs1 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery1)) + .thenReturn(workspaceSyncs1); + + final StandardSyncQuery workspaceQuery2 = new StandardSyncQuery( + WORKSPACE_ID_2, + List.of(sourceConnection3.getSourceId()), + null, false); + final List workspaceSyncs2 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery2)) + .thenReturn(workspaceSyncs2); + + final List expectedSyncIds = Stream.of( + workspaceSyncs1.get(0).getConnectionId(), + workspaceSyncs1.get(1).getConnectionId(), + workspaceSyncs1.get(2).getConnectionId(), + workspaceSyncs2.get(0).getConnectionId()).sorted().toList(); + + final List syncsToDisable = supportStateUpdater.getSyncsToDisableForSource(sourceDefinition, unsupportedVersionIds); + final List actualSyncIds = syncsToDisable.stream().map(StandardSync::getConnectionId).sorted().toList(); + assertEquals(expectedSyncIds, actualSyncIds); + + verify(mConfigRepository).listSourcesWithVersionIds(unsupportedVersionIds); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery1); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery2); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection.getSourceId()); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceConnection2.getSourceId()); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID, sourceWithOverride.getSourceId()); + verify(mActorDefinitionVersionHelper).getSourceVersionWithOverrideStatus(sourceDefinition, WORKSPACE_ID_2, sourceConnection3.getSourceId()); + verifyNoMoreInteractions(mConfigRepository); + verifyNoMoreInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testGetSyncsToDisableForDestination() throws JsonValidationException, ConfigNotFoundException, IOException { + final StandardDestinationDefinition destinationDefinition = new StandardDestinationDefinition() + .withDestinationDefinitionId(ACTOR_DEFINITION_ID); + + final ActorDefinitionVersion ADV_1_0_0 = createActorDefinitionVersion(V1_0_0); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion(V2_0_0); + final List unsupportedVersionIds = List.of(ADV_1_0_0.getVersionId(), ADV_2_0_0.getVersionId()); + + final DestinationConnection destinationConnection = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID) + .withDestinationId(UUID.randomUUID()); + final DestinationConnection destinationConnection2 = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID) + .withDestinationId(UUID.randomUUID()); + final DestinationConnection destinationConnection3 = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID_2) + .withDestinationId(UUID.randomUUID()); + + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, false)); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection2.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID_2, + destinationConnection3.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_2_0_0, false)); + + final DestinationConnection destinationWithOverride = new DestinationConnection() + .withWorkspaceId(WORKSPACE_ID) + .withDestinationId(UUID.randomUUID()); + when(mActorDefinitionVersionHelper.getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationWithOverride.getDestinationId())) + .thenReturn(new ActorDefinitionVersionWithOverrideStatus(ADV_1_0_0, true)); + + when(mConfigRepository.listDestinationsWithVersionIds(unsupportedVersionIds)) + .thenReturn(List.of(destinationConnection, destinationConnection2, destinationConnection3, destinationWithOverride)); + + final StandardSyncQuery workspaceQuery1 = new StandardSyncQuery( + WORKSPACE_ID, + null, + List.of(destinationConnection.getDestinationId(), destinationConnection2.getDestinationId()), + false); + final List workspaceSyncs1 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID()), + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery1)) + .thenReturn(workspaceSyncs1); + + final StandardSyncQuery workspaceQuery2 = new StandardSyncQuery( + WORKSPACE_ID_2, + null, + List.of(destinationConnection3.getDestinationId()), + false); + final List workspaceSyncs2 = List.of( + new StandardSync().withConnectionId(UUID.randomUUID())); + when(mConfigRepository.listWorkspaceStandardSyncs(workspaceQuery2)) + .thenReturn(workspaceSyncs2); + + final List expectedSyncIds = Stream.of( + workspaceSyncs1.get(0).getConnectionId(), + workspaceSyncs1.get(1).getConnectionId(), + workspaceSyncs1.get(2).getConnectionId(), + workspaceSyncs2.get(0).getConnectionId()).sorted().toList(); + + final List syncsToDisable = supportStateUpdater.getSyncsToDisableForDestination(destinationDefinition, unsupportedVersionIds); + final List actualSyncIds = syncsToDisable.stream().map(StandardSync::getConnectionId).sorted().toList(); + assertEquals(expectedSyncIds, actualSyncIds); + + verify(mConfigRepository).listDestinationsWithVersionIds(unsupportedVersionIds); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery1); + verify(mConfigRepository).listWorkspaceStandardSyncs(workspaceQuery2); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection.getDestinationId()); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationConnection2.getDestinationId()); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID, + destinationWithOverride.getDestinationId()); + verify(mActorDefinitionVersionHelper).getDestinationVersionWithOverrideStatus(destinationDefinition, WORKSPACE_ID_2, + destinationConnection3.getDestinationId()); + verifyNoMoreInteractions(mConfigRepository); + verifyNoMoreInteractions(mActorDefinitionVersionHelper); + } + + @Test + void testDisableSyncs() throws IOException { + final List syncsToDisable = List.of( + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.ACTIVE), + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.ACTIVE), + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.INACTIVE), + new StandardSync().withConnectionId(UUID.randomUUID()).withStatus(Status.DEPRECATED)); + + supportStateUpdater.disableSyncs(syncsToDisable); + + verify(mConfigRepository).writeStandardSync(syncsToDisable.get(0).withStatus(Status.INACTIVE)); + verify(mConfigRepository).writeStandardSync(syncsToDisable.get(1).withStatus(Status.INACTIVE)); + verifyNoMoreInteractions(mConfigRepository); } } diff --git a/airbyte-cron/build.gradle b/airbyte-cron/build.gradle index 68b06adcc76..9a39b0e1a7e 100644 --- a/airbyte-cron/build.gradle +++ b/airbyte-cron/build.gradle @@ -17,6 +17,7 @@ dependencies { implementation project(':airbyte-api') implementation project(':airbyte-analytics') + implementation project(':airbyte-commons') implementation project(':airbyte-commons-micronaut') implementation project(':airbyte-commons-temporal') implementation project(':airbyte-config:config-models') diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java b/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java index db0d5acad6f..9ecdc84d837 100644 --- a/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java +++ b/airbyte-cron/src/main/java/io/airbyte/cron/config/ApplicationBeanFactory.java @@ -4,11 +4,14 @@ package io.airbyte.cron.config; +import io.airbyte.commons.version.AirbyteProtocolVersionRange; +import io.airbyte.commons.version.Version; import io.airbyte.config.persistence.split_secrets.JsonSecretsProcessor; import io.airbyte.metrics.lib.MetricClient; import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.MetricEmittingApps; import io.micronaut.context.annotation.Factory; +import io.micronaut.context.annotation.Value; import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j; @@ -37,4 +40,11 @@ public MetricClient metricClient() { return io.airbyte.metrics.lib.MetricClientFactory.getMetricClient(); } + @Singleton + public AirbyteProtocolVersionRange airbyteProtocolVersionRange( + @Value("${airbyte.protocol.min-version}") final String minVersion, + @Value("${airbyte.protocol.max-version}") final String maxVersion) { + return new AirbyteProtocolVersionRange(new Version(minVersion), new Version(maxVersion)); + } + } diff --git a/airbyte-cron/src/main/resources/application.yml b/airbyte-cron/src/main/resources/application.yml index 865f75e42ac..50e3fae1929 100644 --- a/airbyte-cron/src/main/resources/application.yml +++ b/airbyte-cron/src/main/resources/application.yml @@ -28,6 +28,9 @@ airbyte: local: docker-mount: ${LOCAL_DOCKER_MOUNT:} root: ${LOCAL_ROOT} + protocol: + min-version: ${AIRBYTE_PROTOCOL_VERSION_MIN:0.0.0} + max-version: ${AIRBYTE_PROTOCOL_VERSION_MAX:0.3.0} role: ${AIRBYTE_ROLE:} temporal: worker: diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index b3593dad642..08f4f033557 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -101,6 +101,8 @@ object UseCustomK8sScheduler : Temporary(key = "platform.use-custom-k8s- object HideActorDefinitionFromList : Permanent(key = "connectors.hideActorDefinitionFromList", default = false) +object PauseSyncsWithUnsupportedActors : Temporary(key = "connectors.pauseSyncsWithUnsupportedActors", default = true) + // NOTE: this is deprecated in favor of FieldSelectionEnabled and will be removed once that flag is fully deployed. object FieldSelectionWorkspaces : EnvVar(envVar = "FIELD_SELECTION_WORKSPACES") { override fun enabled(ctx: Context): Boolean { From 3640bf154218c66922d208a098a2598cea5bbe40 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 7 Sep 2023 11:47:26 -0400 Subject: [PATCH 126/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Remove=20?= =?UTF-8?q?`useWorkspace()`=20from=20`useIntent()`=20(#8703)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Joey Marshment-Howell --- .../src/core/utils/rbac/rbac.docs.mdx | 8 ++- .../src/core/utils/rbac/rbac.docs.tsx | 8 ++- .../src/core/utils/rbac/rbac.test.ts | 68 ++++++------------- airbyte-webapp/src/core/utils/rbac/rbac.ts | 14 +--- 4 files changed, 34 insertions(+), 64 deletions(-) diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx b/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx index f6deac62dd4..72d548489c9 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx +++ b/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx @@ -24,12 +24,14 @@ Use intents to ask if a user can see something or perform an action. Intents bet The `useIntent` hook is provided to make inquiries. If an intent for your use case does not yet exist it's almost certainly the right decision to create it. ```typescript -const canReadWorkspace = useIntent(Intent.ReadWorkspace); +const canReadWorkspace = useIntent(Intent.ReadWorkspace, { workspaceId: "some-workspace" }); ``` #### Meta details -By default, `useIntent` will locate the necessary resource IDs from available React context. If those values are not available or must be overridden, an object with those extra values can be passed. +By default, `useIntent` will locate the user ID from the Auth Service. Resource ID's such as organization ID or workspace ID must be passed in via the `meta` object. + +If desired, the user ID can be overridden in the meta object as well. ```typescript const canThatUserReadThatWorkspace = useIntent(Intent.ReadWorkspace, { @@ -48,7 +50,7 @@ interface MetaDetails { ### Direct RBAC querying -If for some reason an intent does not make sense for your use case, `useRbac` is available to pass a specific query to. Similar to `useIntent`, this will use avaiable React context to fill in any un-provided query meta. +If for some reason an intent does not make sense for your use case, `useRbac` is available to pass a specific query to. `useRbac(permissions: RbacPermission[], query: RbacQuery | RbacQueryWithoutResourceId)` diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx b/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx index 302cbe3fb01..1384f88cb53 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx +++ b/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx @@ -37,7 +37,10 @@ const PermissionBuilder: React.FC<{ { label: "Organization", value: RbacResourceHierarchy[1] }, { label: "Workspace", value: RbacResourceHierarchy[2] }, ]} - onSelect={setResource} + onSelect={(value) => { + value === RbacResourceHierarchy[0] && setId(""); + setResource(value); + }} /> @@ -55,6 +58,7 @@ const PermissionBuilder: React.FC<{ { setId(e.target.value); }} @@ -180,7 +184,7 @@ const PermisisonTestViewInner = () => {
{permissions.map((permission, index) => ( -

+
{ diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts index 2cd432660e6..7fe0b58c4bd 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts +++ b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts @@ -1,10 +1,9 @@ import { renderHook } from "@testing-library/react"; import { mockUser } from "test-utils/mock-data/mockUser"; -import { mockWorkspace } from "test-utils/mock-data/mockWorkspace"; -import { useListPermissions, useCurrentWorkspace } from "core/api"; -import { PermissionRead, WorkspaceRead } from "core/request/AirbyteClient"; +import { useListPermissions } from "core/api"; +import { PermissionRead } from "core/request/AirbyteClient"; import { useRbac } from "./rbac"; import { RbacPermission, useRbacPermissionsQuery } from "./rbacPermissionsQuery"; @@ -23,19 +22,11 @@ jest.mock("core/api", () => { return { ...actual, useListPermissions: jest.fn(() => ({ permissions: [] })), - useCurrentWorkspace: jest.fn(() => ({ ...mockWorkspace, organizationId: "test-organization" })), }; }); const mockUseListPermissions = useListPermissions as unknown as jest.Mock<{ permissions: RbacPermission[] }>; -const mockUseCurrentWorkspace = useCurrentWorkspace as unknown as jest.Mock; -mockUseCurrentWorkspace.mockImplementation(() => ({ - ...mockWorkspace, - workspaceId: "test-workspace", - organizationId: "test-organization", -})); - describe("useRbac", () => { it("passes permissions", () => { mockUseRbacPermissionsQuery.mockClear(); @@ -78,21 +69,6 @@ describe("useRbac", () => { expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ resourceType: "INSTANCE", role: "ADMIN" }); }); - it("organizationId is found by the context", () => { - mockUseListPermissions.mockImplementation(() => ({ - permissions: [{ permissionType: "instance_admin" }], - })); - - mockUseRbacPermissionsQuery.mockClear(); - renderHook(() => useRbac({ resourceType: "ORGANIZATION", role: "ADMIN" })); - expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); - expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ - resourceType: "ORGANIZATION", - role: "ADMIN", - resourceId: "test-organization", - }); - }); - it("organizationId can be provided directly", () => { mockUseListPermissions.mockImplementation(() => ({ permissions: [{ permissionType: "instance_admin" }], @@ -108,21 +84,6 @@ describe("useRbac", () => { }); }); - it("workspaceId is found by the context", () => { - mockUseListPermissions.mockImplementation(() => ({ - permissions: [{ permissionType: "instance_admin" }], - })); - - mockUseRbacPermissionsQuery.mockClear(); - renderHook(() => useRbac({ resourceType: "WORKSPACE", role: "ADMIN" })); - expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); - expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ - resourceType: "WORKSPACE", - role: "ADMIN", - resourceId: "test-workspace", - }); - }); - it("workspaceId can be provided directly", () => { mockUseListPermissions.mockImplementation(() => ({ permissions: [{ permissionType: "instance_admin" }], @@ -140,15 +101,11 @@ describe("useRbac", () => { }); describe("degenerate cases", () => { - let existingWorkspaceMock: typeof mockUseCurrentWorkspace; // override any useCurrentWorkspaceId mock for this set of tests const consoleError = console.error; // testing for errors in these tests, so we need to silence them beforeAll(() => { - existingWorkspaceMock = mockUseCurrentWorkspace.getMockImplementation() as typeof mockUseCurrentWorkspace; - mockUseCurrentWorkspace.mockImplementation(() => mockWorkspace); console.error = () => void 0; }); afterAll(() => { - mockUseCurrentWorkspace.mockImplementation(existingWorkspaceMock); console.error = consoleError; }); @@ -160,15 +117,32 @@ describe("useRbac", () => { expect(() => renderHook(() => useRbac({ resourceType: "INSTANCE", role: "ADMIN", resourceId: "some-workspace" })) ).toThrowError("Invalid RBAC query: resource INSTANCE with resourceId some-workspace"); + + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); }); - it("does not throw an error when an organization query is missing and cannot find a resourceId", () => { + it("throws an error when an workspaceId is missing", () => { mockUseListPermissions.mockImplementation(() => ({ permissions: [{ permissionType: "instance_admin" }], })); + mockUseRbacPermissionsQuery.mockClear(); + expect(() => renderHook(() => useRbac({ resourceType: "WORKSPACE", role: "ADMIN" }))).toThrowError( + "Invalid RBAC query: resource WORKSPACE with resourceId undefined" + ); + }); + + it("does not throw an error when an organizationId is missing", () => { + mockUseListPermissions.mockImplementation(() => ({ + permissions: [{ permissionType: "instance_admin" }], + })); + + mockUseRbacPermissionsQuery.mockClear(); renderHook(() => useRbac({ resourceType: "ORGANIZATION", role: "ADMIN" })); - expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(2); + + expect(mockUseRbacPermissionsQuery).toHaveBeenCalledTimes(1); expect(mockUseRbacPermissionsQuery.mock.lastCall?.[1]).toEqual({ resourceType: "ORGANIZATION", role: "ADMIN", diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.ts b/airbyte-webapp/src/core/utils/rbac/rbac.ts index b644b74baae..c03f08edaab 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.ts +++ b/airbyte-webapp/src/core/utils/rbac/rbac.ts @@ -1,4 +1,4 @@ -import { useCurrentWorkspace, useListPermissions } from "core/api"; +import { useListPermissions } from "core/api"; import { useCurrentUser } from "core/services/auth"; import { RbacQuery, RbacQueryWithoutResourceId, useRbacPermissionsQuery } from "./rbacPermissionsQuery"; @@ -8,23 +8,13 @@ import { RbacQuery, RbacQueryWithoutResourceId, useRbacPermissionsQuery } from " */ export const useRbac = (query: RbacQuery | RbacQueryWithoutResourceId) => { const { resourceType, role } = query; - let resourceId = "resourceId" in query ? query.resourceId : undefined; + const resourceId = "resourceId" in query ? query.resourceId : undefined; const queryUsesResourceId = resourceType !== "INSTANCE"; const { userId } = useCurrentUser(); const { permissions } = useListPermissions(userId); - const contextWorkspace = useCurrentWorkspace(); - if (queryUsesResourceId && !resourceId) { - // attempt to locate this from context - if (resourceType === "WORKSPACE") { - resourceId = contextWorkspace.workspaceId; - } else if (resourceType === "ORGANIZATION") { - resourceId = contextWorkspace.organizationId; - } - } - // invariant check if ((!queryUsesResourceId && resourceId) || (queryUsesResourceId && !resourceId)) { // TODO: This is a patch to handle the fact that workspaces on cloud do not have an organization. From 4f53dd9f0f00aaa5b3f918fea672ad1296784e63 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 7 Sep 2023 11:58:36 -0400 Subject: [PATCH 127/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=A7=B9=20Cleaning?= =?UTF-8?q?=20up=20intl=20string=20types=20Phase=201=20(#8707)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Joey Marshment-Howell --- .../components/StreamStatusCell.tsx | 7 +- .../ConnectionForm/ScheduleField.tsx | 4 + .../StreamDetailsPanel/StreamPanelHeader.tsx | 4 +- .../src/core/api/hooks/cloud/dbtCloud.ts | 2 +- .../src/core/utils/numberHelper.tsx | 2 +- .../src/core/utils/rbac/rbac.test.ts | 1 + airbyte-webapp/src/locales/en.json | 202 +++++++++++++++++- airbyte-webapp/src/packages/cloud/App.tsx | 5 +- .../src/packages/cloud/locales/en.json | 202 ------------------ .../components/UsagePerDayGraph.tsx | 2 +- .../WorkspaceStatusBanner.test.tsx | 4 +- 11 files changed, 219 insertions(+), 216 deletions(-) delete mode 100644 airbyte-webapp/src/packages/cloud/locales/en.json diff --git a/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx b/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx index af301059360..ceb97c6ebd9 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx @@ -67,13 +67,16 @@ const StreamsPerStatus: React.FC<{ enabledStreams: AirbyteStreamAndConfigurationWithEnforcedStream[]; }> = ({ streamStatuses, enabledStreams }) => { const sortedStreamsMap = sortStreams(enabledStreams, streamStatuses); - const filteredAndSortedStreams = Object.entries(sortedStreamsMap).filter(([, streams]) => !!streams.length); + const filteredAndSortedStreams = Object.entries(sortedStreamsMap).filter(([, streams]) => !!streams.length) as Array< + [ConnectionStatusIndicatorStatus, StreamWithStatus[]] + >; + return ( <> {filteredAndSortedStreams.map(([statusType, streams]) => (
- + {streams.length}
diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.tsx index b97e55989b1..0e080e6acb4 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleField.tsx @@ -106,6 +106,10 @@ export const ScheduleField: React.FC = () => { return value.basicSchedule; } + if (!scheduleType) { + return null; + } + return formatMessage({ id: `frequency.${scheduleType}`, }).toLowerCase(); diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx index 36b00782a00..c1e4859609b 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx @@ -78,9 +78,9 @@ export const StreamPanelHeader: React.FC = ({ const syncMode = ( <> - + {config?.syncMode && } {` | `} - + {config?.destinationSyncMode && } ); diff --git a/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts b/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts index b4008ffa47f..b3a8fd70b3c 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts @@ -154,7 +154,7 @@ export const useDbtIntegration = (connection: WebBackendConnectionRead) => { // FIXME: remove this once we migrate to react-hook-form since it will handle onError and OnSuccess registerNotification({ id: "connection.updateFailed", - text: formatMessage({ id: "notification.connection.updateFailed" }), + text: formatMessage({ id: "connection.updateFailed" }), }); } }, diff --git a/airbyte-webapp/src/core/utils/numberHelper.tsx b/airbyte-webapp/src/core/utils/numberHelper.tsx index d44f6875e4c..9d2960ae07d 100644 --- a/airbyte-webapp/src/core/utils/numberHelper.tsx +++ b/airbyte-webapp/src/core/utils/numberHelper.tsx @@ -11,7 +11,7 @@ export const formatBytes = (bytes?: number) => { const k = 1024; const dm = 2; - const sizes = ["Bytes", "KB", "MB", "GB", "TB"]; + const sizes = ["Bytes", "KB", "MB", "GB", "TB"] as const; const i = Math.floor(Math.log(bytes) / Math.log(k)); const result = parseFloat((bytes / Math.pow(k, i)).toFixed(dm)); diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts index 7fe0b58c4bd..9c4db58eade 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts +++ b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts @@ -122,6 +122,7 @@ describe("useRbac", () => { permissions: [{ permissionType: "instance_admin" }], })); }); + // TODO: Update test to throw once cloud workspaces are migrated to organizations + rbac.ts invariant is adjusted to require organization id it("throws an error when an workspaceId is missing", () => { mockUseListPermissions.mockImplementation(() => ({ diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index f72ec9d2af4..1ebaecc82de 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -1131,5 +1131,205 @@ "modal.closeButtonLabel": "Close dialog", - "copyButton.title": "Copy" + "copyButton.title": "Copy", + + "login.haveAccount": "Already have an account?", + "login.noAccount": "Don’t have an account?", + "login.login": "Log in", + "login.signup": "Sign up", + "login.signup.submitButton": "Sign up", + "login.loginTitle": "Log in to Airbyte", + "login.resendEmail": "Didn’t receive the email? Send it again", + "login.yourEmail": "Your work email*", + "login.inviteEmail": "For security, re-enter your invite email*", + "login.yourEmail.placeholder": "name@company.com", + "login.yourEmail.notFound": "User not found", + "login.unknownError": "An unknown error has occurred", + "login.password": "Enter your password*", + "login.password.placeholder": "Enter a strong password", + "login.yourPassword": "Enter your password*", + "login.yourPassword.placeholder": "Your password", + "login.forgotPassword": "Forgot your password", + "login.backLogin": "Back to Log in", + "login.createPassword": "Create a password*", + "login.resetPassword": "Reset your password", + "login.resetPassword.emailSent": "A password reset email has been sent to you", + "login.activateAccess": "Activate your 14-day free trial", + "login.activateAccess.submitButton": "Sign up", + "login.fullName": "Full name*", + "login.fullName.placeholder": "Christopher Smith", + "login.companyName": "Company name*", + "login.companyName.placeholder": "Acme Inc.", + "login.disclaimer": "By signing up and continuing, you agree to our Terms of Service and Privacy Policy.", + "login.inviteTitle": "Invite access", + "login.rightSideFrameTitle": "More about Airbyte", + "login.quoteText": "Airbyte has cut months of employee hours off of our ELT pipeline development and delivered usable data to us in hours instead of weeks. We are excited for the future of Airbyte, enthusiastic about their approach, and optimistic about our future together.", + "login.quoteAuthor": "Micah Mangione", + "login.quoteAuthorJobTitle": "Director of Technology", + "login.oauth.or": "or", + "login.oauth.google": "Continue with Google", + "login.oauth.github": "Continue with GitHub", + "login.oauth.differentCredentialsError": "Use your email and password to sign in.", + "login.oauth.unknownError": "An unknown error happened during sign in: {error}", + "login.selfhosting": "Interested in self-hosting?", + "login.opensource": "Open-source", + "login.deployInfrastructure": "Deploy the open-source version on your own infrastructure", + "login.email.notfound": "Email not found", + "login.email.disabled": "Your account is disabled", + "login.password.invalid": "Invalid password", + "login.email.invalid": "The email is not in the correct format. Please make sure to enter a valid email address (e.g., name@company.com)", + "login.invite.email.mismatch": "This email does not match the email address sent to this invite.", + "login.invite.link.expired": "This invite link expired. A new invite link has been sent to your email.", + "login.invite.link.invalid": "This invite link is no longer valid.", + + "signup.details.noCreditCard": "No credit card required", + "signup.details.instantSetup": "Instant setup", + "signup.details.freeTrial": "14-day free trial", + "signup.title": "Create your Airbyte account", + "signup.method.email": "Sign up using email", + "signup.method.oauth": "Sign up using Google or GitHub", + "signup.password.minLength": "Password should be at least 12 characters", + "signup.email.duplicate": "Email already exists", + "signup.email.invalid": "The email is not in the correct format. Please make sure to enter a valid email address (e.g., name@company.com)", + "signup.password.weak": "Your password does not meet the minimum length", + + "confirmResetPassword.newPassword": "Enter a new password", + "confirmResetPassword.success": "Your password has been reset. Please log in with the new password.", + "confirmResetPassword.link.expired": "The password reset link has expired. Please request password reset again.", + "confirmResetPassword.link.invalid": "The password reset link is invalid. Please double check the reset email.", + "confirmResetPassword.password.weak": "Your password does not meet the minimum length", + + "connection.dbtCloudJobs.cardTitle": "Transformations", + "connection.dbtCloudJobs.addJob": "Add transformation", + "connection.dbtCloudJobs.dbtError": "There was an error communicating with dbt Cloud: {displayMessage}", + "connection.dbtCloudJobs.genericError": "There was an error communicating with dbt Cloud.", + "connection.dbtCloudJobs.explanation": "After an Airbyte sync job has completed, the following jobs will run", + "connection.dbtCloudJobs.noJobs": "No transformations", + "connection.dbtCloudJobs.noJobsFoundForAccount": "No jobs found for this dbt Cloud account", + "connection.dbtCloudJobs.updateSuccess": "dbt transformation settings were updated successfully!", + "connection.dbtCloudJobs.updateError": "There was an error updating your dbt transformations settings", + "connection.dbtCloudJobs.job.title": "dbt Cloud transform", + "connection.dbtCloudJobs.job.accountId": "Account ID", + "connection.dbtCloudJobs.job.jobId": "Job ID", + "connection.dbtCloudJobs.job.deleteButton": "Delete job", + "connection.dbtCloudJobs.noIntegration": "Go to your settings to connect a dbt Cloud account", + + "settings.accountSettings.firstName": "First name", + "settings.accountSettings.enterPassword": "Enter current password", + "settings.accountSettings.firstName.placeholder": "Christopher", + "settings.accountSettings.name": "Full name", + "settings.accountSettings.name.placeholder": "Christopher Smith", + "settings.accountSettings.name.empty.error": "Name cannot be empty", + "settings.accountSettings.lastName": "Last name", + "settings.accountSettings.lastName.placeholder": "Smith", + "settings.accountSettings.updateName": "Update Name", + "settings.accountSettings.email": "Email", + "settings.accountSettings.updateEmail": "Update Email", + "settings.accountSettings.password": "Password", + "settings.accountSettings.currentPassword": "Current password", + "settings.accountSettings.currentPassword.placeholder": "Current password", + "settings.accountSettings.newPassword": "New password", + "settings.accountSettings.newPasswordConfirmation": "Password confirmation", + "settings.accountSettings.error.newPasswordMismatch": "New password and confirmation do not match", + "settings.accountSettings.error.newPasswordSameAsCurrent": "New password is the same as the current one", + "settings.accountSettings.updatePasswordError": "Password update failed: ", + "settings.accountSettings.updatePasswordSuccess": "Your password has been updated.", + "settings.accountSettings.updatePassword": "Update password", + "settings.accountSettings.updateNameError": "Something went wrong while updating your name. Please try again.", + "settings.accountSettings.updateNameSuccess": "Your name has been updated!", + "settings.workspaceSettings.delete.confirmation.title": "Delete workspace", + "settings.workspaceSettings.delete.confirmation.text": "Deleting this workspace will remove it for all users and cancel all pending syncs. Do you want to proceed?", + "settings.workspaceSettings.delete.confirmation.submitButtonText": "Delete workspace", + + "settings.workspaceSettings.delete.permissionsError": "You do not have sufficient permissions to delete this workspace. Please consult with the workspace owner.", + "settings.integrationSettings": "Integration settings", + "settings.integrationSettings.dbtCloudSettings": "dbt Cloud Integration", + "settings.integrationSettings.dbtCloudSettings.actions.cancel": "Cancel", + "settings.integrationSettings.dbtCloudSettings.actions.delete": "Delete service token", + "settings.integrationSettings.dbtCloudSettings.actions.delete.confirm": "Confirm dbt Service Token deletion", + "settings.integrationSettings.dbtCloudSettings.action.delete.modal": "Are you sure you want to remove your dbt Service Token? \n \nThis will stop all dbt Cloud transformations from running and may cause sync failures. You will need to manually remove these transformations from your connections to keep syncing.", + "settings.integrationSettings.dbtCloudSettings.actions.delete.success": "Service token deleted successfully", + "settings.integrationSettings.dbtCloudSettings.actions.submit": "Save changes", + "settings.integrationSettings.dbtCloudSettings.actions.submit.success": "Service Token saved successfully", + "settings.integrationSettings.dbtCloudSettings.form.serviceTokenLabel": "Service Token", + "settings.integrationSettings.dbtCloudSettings.form.serviceTokenInputHidden": "Enter a service token", + "settings.integrationSettings.dbtCloudSettings.form.serviceTokenAlreadyExist": "This workspace has an existing dbt service token. To replace it, you must first remove the existing token.", + "settings.integrationSettings.dbtCloudSettings.form.description": "To use the dbt Cloud integration, enter your service token here. Learn more.", + + "settings.generalSettings.changeWorkspace": "Change Workspace", + "settings.accessManagementSettings": "Access Management", + "userSettings.table.title": "Current users", + "userSettings.table.column.fullname": "Full Name", + "userSettings.table.column.email": "Email", + "userSettings.table.column.role": "Role", + "userSettings.table.column.action": "Action", + "userSettings.button.addNewUser": "New user", + "userSettings.user.remove": "Remove", + "modals.addUser.title": "Add new users", + "modals.addUser.role.label": "Role", + "modals.addUser.role.placeholder": "Select a role", + "modals.addUser.button.addUser": "Add another user", + "modals.addUser.button.cancel": "Cancel", + "modals.addUser.button.submit": "Send invitation", + "modals.removeUser.text": "Are you sure you want to remove this user?", + "modals.removeUser.title": "Remove user", + "modals.removeUser.button.submit": "Remove", + "workspaces.viewAllWorkspaces": "View all workspaces", + "settings.accessManagement.roleViewers": "Viewers are in read-only and cannot edit or add connections.", + "settings.accessManagement.roleEditors": "Editors can edit connections", + "settings.accessManagement.roleAdmin": "Admin can also manage users", + + "inviteUsers.invitationsSentSuccess": "Invitations sent!", + "inviteUsers.invitationsSentError": "Something went wrong. Please try again.", + + "credits.date": "Date", + "credits.amount": "Credits", + "credits.billing": "Billing", + "credits.connection": "Connection", + "credits.source": "Source", + "credits.destination": "Destination", + "credits.schedule": "Schedule", + "credits.freeUsage": "Free", + "credits.billedCost": "Billed", + "credits.totalUsage": "Total credits usage", + "credits.stripePortalLink": "Invoice history", + "credits.deleted": "This connection has been deleted", + "credits.timePeriod": "Time period", + + "credits.loadingCreditsUsage": "Loading credits usage …", + "credits.totalCreditsUsage": "Total credits usage", + "credits.usagePerConnection": "Usage per connection", + "credits.whatAre": "What are credits?", + "credits.usage": "Usage", + "credits.noData": "You have no credits usage data for this period. Sync a connection to get started!", + "credits.creditsProblem": "You’re out of credits! To set up connections and run syncs, add credits.", + "credits.emailVerificationRequired": "You need to verify your email address before you can buy credits.", + "credits.emailVerification.resendConfirmation": "We sent you a new verification link.", + "credits.emailVerification.resend": "Send verification link again", + "credits.lowBalance": "Your credit balance is low. You need to add more credits to prevent your GA syncs from being stopped.", + "credits.zeroBalance": "All your GA syncs have been stopped because your credit balance is 0. Add more credits to enable your GA syncs again.", + + "firebase.auth.error.invalidPassword": "Incorrect password", + "firebase.auth.error.networkRequestFailed": "There appears to be a network issue. Please try again later.", + "firebase.auth.error.tooManyRequests": "Too many retries. Please wait 10 minutes before trying again.", + "firebase.auth.error.default": "Confirmation email cannot be sent. Please try again later.", + + "trial.alertMessage": "You are using a trial of Airbyte. Your trial ends in {remainingDays, plural, one {# day} other {# days}}. Purchase now", + + "trial.preTrialAlertMessage": "Your 14-day trial of Airbyte will start once your first sync has completed.", + "verifyEmail.notification": "You successfully verified your email. Thank you.", + + "inviteUsersHint.message": "Need help from a teammate to set up the {connector}?", + "inviteUsersHint.cta": "Invite users", + + "sidebar.credits": "Credits", + "sidebar.billing": "Billing", + + "freeConnectorProgram.title": "Free Connector Program", + "freeConnectorProgram.releaseStageBadge.free": "Free", + + "experiment.speedyConnection": "Set up your first connection in the next and get 100 additional credits for your trial", + + "workspace.adminWorkspaceWarning": "Admin", + "workspace.adminWorkspaceWarningTooltip": "You are not a member of this workspace. Be careful when making changes!" } diff --git a/airbyte-webapp/src/packages/cloud/App.tsx b/airbyte-webapp/src/packages/cloud/App.tsx index 14135769b64..e7c2bd9b901 100644 --- a/airbyte-webapp/src/packages/cloud/App.tsx +++ b/airbyte-webapp/src/packages/cloud/App.tsx @@ -20,7 +20,6 @@ import { NotificationService } from "hooks/services/Notification"; import { AirbyteThemeProvider } from "hooks/theme/useAirbyteTheme"; import en from "locales/en.json"; import { Routing } from "packages/cloud/cloudRoutes"; -import cloudLocales from "packages/cloud/locales/en.json"; import { AuthenticationProvider } from "packages/cloud/services/auth/AuthService"; import { theme } from "packages/cloud/theme"; import { ConnectorBuilderTestInputProvider } from "services/connectorBuilder/ConnectorBuilderTestInputService"; @@ -28,8 +27,6 @@ import { ConnectorBuilderTestInputProvider } from "services/connectorBuilder/Con import { AppServicesProvider } from "./services/AppServicesProvider"; import { ZendeskProvider } from "./services/thirdParty/zendesk"; -const messages = { ...en, ...cloudLocales }; - const StyleProvider: React.FC> = ({ children }) => ( {children} ); @@ -61,7 +58,7 @@ const App: React.FC = () => { - + }> diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json deleted file mode 100644 index 4e34f2e219c..00000000000 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ /dev/null @@ -1,202 +0,0 @@ -{ - "login.haveAccount": "Already have an account?", - "login.noAccount": "Don’t have an account?", - "login.login": "Log in", - "login.signup": "Sign up", - "login.signup.submitButton": "Sign up", - "login.loginTitle": "Log in to Airbyte", - "login.resendEmail": "Didn’t receive the email? Send it again", - "login.yourEmail": "Your work email*", - "login.inviteEmail": "For security, re-enter your invite email*", - "login.yourEmail.placeholder": "name@company.com", - "login.yourEmail.notFound": "User not found", - "login.unknownError": "An unknown error has occurred", - "login.password": "Enter your password*", - "login.password.placeholder": "Enter a strong password", - "login.yourPassword": "Enter your password*", - "login.yourPassword.placeholder": "Your password", - "login.forgotPassword": "Forgot your password", - "login.backLogin": "Back to Log in", - "login.createPassword": "Create a password*", - "login.resetPassword": "Reset your password", - "login.resetPassword.emailSent": "A password reset email has been sent to you", - "login.activateAccess": "Activate your 14-day free trial", - "login.activateAccess.submitButton": "Sign up", - "login.fullName": "Full name*", - "login.fullName.placeholder": "Christopher Smith", - "login.companyName": "Company name*", - "login.companyName.placeholder": "Acme Inc.", - "login.disclaimer": "By signing up and continuing, you agree to our Terms of Service and Privacy Policy.", - "login.inviteTitle": "Invite access", - "login.rightSideFrameTitle": "More about Airbyte", - "login.quoteText": "Airbyte has cut months of employee hours off of our ELT pipeline development and delivered usable data to us in hours instead of weeks. We are excited for the future of Airbyte, enthusiastic about their approach, and optimistic about our future together.", - "login.quoteAuthor": "Micah Mangione", - "login.quoteAuthorJobTitle": "Director of Technology", - "login.oauth.or": "or", - "login.oauth.google": "Continue with Google", - "login.oauth.github": "Continue with GitHub", - "login.oauth.differentCredentialsError": "Use your email and password to sign in.", - "login.oauth.unknownError": "An unknown error happened during sign in: {error}", - "login.selfhosting": "Interested in self-hosting?", - "login.opensource": "Open-source", - "login.deployInfrastructure": "Deploy the open-source version on your own infrastructure", - "login.email.notfound": "Email not found", - "login.email.disabled": "Your account is disabled", - "login.password.invalid": "Invalid password", - "login.email.invalid": "The email is not in the correct format. Please make sure to enter a valid email address (e.g., name@company.com)", - "login.invite.email.mismatch": "This email does not match the email address sent to this invite.", - "login.invite.link.expired": "This invite link expired. A new invite link has been sent to your email.", - "login.invite.link.invalid": "This invite link is no longer valid.", - - "signup.details.noCreditCard": "No credit card required", - "signup.details.instantSetup": "Instant setup", - "signup.details.freeTrial": "14-day free trial", - "signup.title": "Create your Airbyte account", - "signup.method.email": "Sign up using email", - "signup.method.oauth": "Sign up using Google or GitHub", - "signup.password.minLength": "Password should be at least 12 characters", - "signup.email.duplicate": "Email already exists", - "signup.email.invalid": "The email is not in the correct format. Please make sure to enter a valid email address (e.g., name@company.com)", - "signup.password.weak": "Your password does not meet the minimum length", - - "confirmResetPassword.newPassword": "Enter a new password", - "confirmResetPassword.success": "Your password has been reset. Please log in with the new password.", - "confirmResetPassword.link.expired": "The password reset link has expired. Please request password reset again.", - "confirmResetPassword.link.invalid": "The password reset link is invalid. Please double check the reset email.", - "confirmResetPassword.password.weak": "Your password does not meet the minimum length", - - "connection.dbtCloudJobs.cardTitle": "Transformations", - "connection.dbtCloudJobs.addJob": "Add transformation", - "connection.dbtCloudJobs.dbtError": "There was an error communicating with dbt Cloud: {displayMessage}", - "connection.dbtCloudJobs.genericError": "There was an error communicating with dbt Cloud.", - "connection.dbtCloudJobs.explanation": "After an Airbyte sync job has completed, the following jobs will run", - "connection.dbtCloudJobs.noJobs": "No transformations", - "connection.dbtCloudJobs.noJobsFoundForAccount": "No jobs found for this dbt Cloud account", - "connection.dbtCloudJobs.updateSuccess": "dbt transformation settings were updated successfully!", - "connection.dbtCloudJobs.updateError": "There was an error updating your dbt transformations settings", - "connection.dbtCloudJobs.job.title": "dbt Cloud transform", - "connection.dbtCloudJobs.job.accountId": "Account ID", - "connection.dbtCloudJobs.job.jobId": "Job ID", - "connection.dbtCloudJobs.job.deleteButton": "Delete job", - "connection.dbtCloudJobs.noIntegration": "Go to your settings to connect a dbt Cloud account", - - "settings.accountSettings.firstName": "First name", - "settings.accountSettings.enterPassword": "Enter current password", - "settings.accountSettings.firstName.placeholder": "Christopher", - "settings.accountSettings.name": "Full name", - "settings.accountSettings.name.placeholder": "Christopher Smith", - "settings.accountSettings.name.empty.error": "Name cannot be empty", - "settings.accountSettings.lastName": "Last name", - "settings.accountSettings.lastName.placeholder": "Smith", - "settings.accountSettings.updateName": "Update Name", - "settings.accountSettings.email": "Email", - "settings.accountSettings.updateEmail": "Update Email", - "settings.accountSettings.password": "Password", - "settings.accountSettings.currentPassword": "Current password", - "settings.accountSettings.currentPassword.placeholder": "Current password", - "settings.accountSettings.newPassword": "New password", - "settings.accountSettings.newPasswordConfirmation": "Password confirmation", - "settings.accountSettings.error.newPasswordMismatch": "New password and confirmation do not match", - "settings.accountSettings.error.newPasswordSameAsCurrent": "New password is the same as the current one", - "settings.accountSettings.updatePasswordError": "Password update failed: ", - "settings.accountSettings.updatePasswordSuccess": "Your password has been updated.", - "settings.accountSettings.updatePassword": "Update password", - "settings.accountSettings.updateNameError": "Something went wrong while updating your name. Please try again.", - "settings.accountSettings.updateNameSuccess": "Your name has been updated!", - "settings.workspaceSettings.delete.confirmation.title": "Delete workspace", - "settings.workspaceSettings.delete.confirmation.text": "Deleting this workspace will remove it for all users and cancel all pending syncs. Do you want to proceed?", - "settings.workspaceSettings.delete.confirmation.submitButtonText": "Delete workspace", - "settings.workspaceSettings.delete.success": "Workspace deleted successfully", - "settings.workspaceSettings.delete.error": "There was an error deleting this workspace.", - "settings.workspaceSettings.delete.permissionsError": "You do not have sufficient permissions to delete this workspace. Please consult with the workspace owner.", - "settings.integrationSettings": "Integration settings", - "settings.integrationSettings.dbtCloudSettings": "dbt Cloud Integration", - "settings.integrationSettings.dbtCloudSettings.actions.cancel": "Cancel", - "settings.integrationSettings.dbtCloudSettings.actions.delete": "Delete service token", - "settings.integrationSettings.dbtCloudSettings.actions.delete.confirm": "Confirm dbt Service Token deletion", - "settings.integrationSettings.dbtCloudSettings.action.delete.modal": "Are you sure you want to remove your dbt Service Token? \n \nThis will stop all dbt Cloud transformations from running and may cause sync failures. You will need to manually remove these transformations from your connections to keep syncing.", - "settings.integrationSettings.dbtCloudSettings.actions.delete.success": "Service token deleted successfully", - "settings.integrationSettings.dbtCloudSettings.actions.submit": "Save changes", - "settings.integrationSettings.dbtCloudSettings.actions.submit.success": "Service Token saved successfully", - "settings.integrationSettings.dbtCloudSettings.form.serviceTokenLabel": "Service Token", - "settings.integrationSettings.dbtCloudSettings.form.serviceTokenInputHidden": "Enter a service token", - "settings.integrationSettings.dbtCloudSettings.form.serviceTokenAlreadyExist": "This workspace has an existing dbt service token. To replace it, you must first remove the existing token.", - "settings.integrationSettings.dbtCloudSettings.form.description": "To use the dbt Cloud integration, enter your service token here. Learn more.", - - "settings.generalSettings.changeWorkspace": "Change Workspace", - "settings.accessManagementSettings": "Access Management", - "userSettings.table.title": "Current users", - "userSettings.table.column.fullname": "Full Name", - "userSettings.table.column.email": "Email", - "userSettings.table.column.role": "Role", - "userSettings.table.column.action": "Action", - "userSettings.button.addNewUser": "New user", - "userSettings.user.remove": "Remove", - "modals.addUser.title": "Add new users", - "modals.addUser.role.label": "Role", - "modals.addUser.role.placeholder": "Select a role", - "modals.addUser.button.addUser": "Add another user", - "modals.addUser.button.cancel": "Cancel", - "modals.addUser.button.submit": "Send invitation", - "modals.removeUser.text": "Are you sure you want to remove this user?", - "modals.removeUser.title": "Remove user", - "modals.removeUser.button.submit": "Remove", - "workspaces.viewAllWorkspaces": "View all workspaces", - "settings.accessManagement.roleViewers": "Viewers are in read-only and cannot edit or add connections.", - "settings.accessManagement.roleEditors": "Editors can edit connections", - "settings.accessManagement.roleAdmin": "Admin can also manage users", - - "inviteUsers.invitationsSentSuccess": "Invitations sent!", - "inviteUsers.invitationsSentError": "Something went wrong. Please try again.", - - "credits.date": "Date", - "credits.amount": "Credits", - "credits.billing": "Billing", - "credits.connection": "Connection", - "credits.source": "Source", - "credits.destination": "Destination", - "credits.schedule": "Schedule", - "credits.freeUsage": "Free", - "credits.billedCost": "Billed", - "credits.totalUsage": "Total credits usage", - "credits.stripePortalLink": "Invoice history", - "credits.deleted": "This connection has been deleted", - "credits.timePeriod": "Time period", - - "credits.loadingCreditsUsage": "Loading credits usage …", - "credits.totalCreditsUsage": "Total credits usage", - "credits.usagePerConnection": "Usage per connection", - "credits.whatAre": "What are credits?", - "credits.usage": "Usage", - "credits.noData": "You have no credits usage data for this period. Sync a connection to get started!", - "credits.creditsProblem": "You’re out of credits! To set up connections and run syncs, add credits.", - "credits.emailVerificationRequired": "You need to verify your email address before you can buy credits.", - "credits.emailVerification.resendConfirmation": "We sent you a new verification link.", - "credits.emailVerification.resend": "Send verification link again", - "credits.lowBalance": "Your credit balance is low. You need to add more credits to prevent your GA syncs from being stopped.", - "credits.zeroBalance": "All your GA syncs have been stopped because your credit balance is 0. Add more credits to enable your GA syncs again.", - - "firebase.auth.error.invalidPassword": "Incorrect password", - "firebase.auth.error.networkRequestFailed": "There appears to be a network issue. Please try again later.", - "firebase.auth.error.tooManyRequests": "Too many retries. Please wait 10 minutes before trying again.", - "firebase.auth.error.default": "Confirmation email cannot be sent. Please try again later.", - - "trial.alertMessage": "You are using a trial of Airbyte. Your trial ends in {remainingDays, plural, one {# day} other {# days}}. Purchase now", - - "trial.preTrialAlertMessage": "Your 14-day trial of Airbyte will start once your first sync has completed.", - "verifyEmail.notification": "You successfully verified your email. Thank you.", - - "webapp.cannotReachServer": "Cannot reach server.", - - "inviteUsersHint.message": "Need help from a teammate to set up the {connector}?", - "inviteUsersHint.cta": "Invite users", - - "sidebar.credits": "Credits", - "sidebar.billing": "Billing", - "freeConnectorProgram.releaseStageBadge.free": "Free", - - "experiment.speedyConnection": "Set up your first connection in the next and get 100 additional credits for your trial", - - "workspace.adminWorkspaceWarning": "Admin", - "workspace.adminWorkspaceWarningTooltip": "You are not a member of this workspace. Be careful when making changes!" -} diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx index a53a31cbcd1..61b58202975 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerDayGraph.tsx @@ -57,7 +57,7 @@ export const UsagePerDayGraph: React.FC = ({ chartData, m iconType="circle" height={40} wrapperStyle={{ color: `${styles.white}` }} - formatter={(value) => { + formatter={(value: "freeUsage" | "billedCost") => { return ( diff --git a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/WorkspaceStatusBanner.test.tsx b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/WorkspaceStatusBanner.test.tsx index 3e516be97bc..27dc297713b 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/WorkspaceStatusBanner.test.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/WorkspaceStatusBanner.test.tsx @@ -10,9 +10,9 @@ import { CloudWorkspaceReadWorkspaceTrialStatus as WorkspaceTrialStatus, } from "core/api/types/CloudApi"; import { I18nProvider } from "core/services/i18n"; +import messages from "locales/en.json"; import { WorkspaceStatusBanner } from "./WorkspaceStatusBanner"; -import cloudLocales from "../../../locales/en.json"; const defaultCloudWorkspace = { workspaceId: "123" }; @@ -20,7 +20,7 @@ const renderWorkspaceBanner = (cloudWorkspace: CloudWorkspaceRead) => { return render( - + From c3cfdd0424a9632c5398d2f19da7cc8583feed3f Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Thu, 7 Sep 2023 10:55:17 -0700 Subject: [PATCH 128/201] Introduce some error handling for network issues when calling temporal (#8556) --- airbyte-commons-temporal/build.gradle | 1 + .../temporal/ConnectionManagerUtils.java | 26 +++--- .../commons/temporal/TemporalClient.java | 12 +-- .../temporal/WorkflowClientWrapped.java | 89 +++++++++++++++++++ .../commons/temporal/TemporalClientTest.java | 2 +- .../temporal/WorkflowClientWrappedTest.java | 85 ++++++++++++++++++ .../metrics/lib/OssMetricsRegistry.java | 3 + 7 files changed, 198 insertions(+), 20 deletions(-) create mode 100644 airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java create mode 100644 airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java diff --git a/airbyte-commons-temporal/build.gradle b/airbyte-commons-temporal/build.gradle index bbfbed65fcf..874758fa01c 100644 --- a/airbyte-commons-temporal/build.gradle +++ b/airbyte-commons-temporal/build.gradle @@ -29,6 +29,7 @@ dependencies { compileOnly libs.lombok annotationProcessor libs.lombok implementation libs.bundles.apache + implementation libs.failsafe testImplementation libs.temporal.testing // Needed to be able to mock final class diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index e5374ba02e9..19bab81221d 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -35,10 +35,12 @@ @Slf4j public class ConnectionManagerUtils { + private final WorkflowClientWrapped workflowClientWrapped; private final MetricClient metricClient; - public ConnectionManagerUtils(final MetricClient metricClient) { + public ConnectionManagerUtils(final WorkflowClient workflowClient, final MetricClient metricClient) { this.metricClient = metricClient; + this.workflowClientWrapped = new WorkflowClientWrapped(workflowClient, this.metricClient); } /** @@ -113,7 +115,7 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W final Optional signalArgument) throws DeletedWorkflowException { try { - final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(connectionId); log.info("Retrieved existing connection manager workflow for connection {}. Executing signal.", connectionId); // retrieve the signal from the lambda final TemporalFunctionalInterfaceMarker signal = signalMethod.apply(connectionManagerWorkflow); @@ -208,16 +210,16 @@ public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowCl * @throws DeletedWorkflowException if the workflow was deleted, according to the workflow state * @throws UnreachableWorkflowException if the workflow is in an unreachable state */ - public ConnectionManagerWorkflow getConnectionManagerWorkflow(final WorkflowClient client, final UUID connectionId) + public ConnectionManagerWorkflow getConnectionManagerWorkflow(final UUID connectionId) throws DeletedWorkflowException, UnreachableWorkflowException { final ConnectionManagerWorkflow connectionManagerWorkflow; final WorkflowState workflowState; final WorkflowExecutionStatus workflowExecutionStatus; try { - connectionManagerWorkflow = client.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); + connectionManagerWorkflow = workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); workflowState = connectionManagerWorkflow.getState(); - workflowExecutionStatus = getConnectionManagerWorkflowStatus(client, connectionId); + workflowExecutionStatus = getConnectionManagerWorkflowStatus(connectionId); } catch (final Exception e) { throw new UnreachableWorkflowException( String.format("Failed to retrieve ConnectionManagerWorkflow for connection %s due to the following error:", connectionId), @@ -257,19 +259,18 @@ boolean isWorkflowStateRunning(final WorkflowClient client, final UUID connectio /** * Get status of a connection manager workflow. * - * @param workflowClient workflow client * @param connectionId connection id * @return workflow execution status */ - public WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final WorkflowClient workflowClient, final UUID connectionId) { + private WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final UUID connectionId) { final DescribeWorkflowExecutionRequest describeWorkflowExecutionRequest = DescribeWorkflowExecutionRequest.newBuilder() .setExecution(WorkflowExecution.newBuilder() .setWorkflowId(getConnectionManagerName(connectionId)) .build()) - .setNamespace(workflowClient.getOptions().getNamespace()).build(); + .setNamespace(workflowClientWrapped.getNamespace()).build(); - final DescribeWorkflowExecutionResponse describeWorkflowExecutionResponse = workflowClient.getWorkflowServiceStubs().blockingStub() - .describeWorkflowExecution(describeWorkflowExecutionRequest); + final DescribeWorkflowExecutionResponse describeWorkflowExecutionResponse = + workflowClientWrapped.blockingDescribeWorkflowExecution(describeWorkflowExecutionRequest); return describeWorkflowExecutionResponse.getWorkflowExecutionInfo().getStatus(); } @@ -277,13 +278,12 @@ public WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final Workflow /** * Get the job id for a connection is a workflow is running for it. Otherwise, throws. * - * @param client temporal workflow client * @param connectionId connection id * @return current job id */ - public long getCurrentJobId(final WorkflowClient client, final UUID connectionId) { + public long getCurrentJobId(final UUID connectionId) { try { - final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = getConnectionManagerWorkflow(connectionId); return connectionManagerWorkflow.getJobInformation().getJobId(); } catch (final Exception e) { return ConnectionManagerWorkflow.NON_RUNNING_JOB_ID; diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index e8dda943057..87dabb26ea6 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -250,7 +250,7 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { log.info("end of manual schedule"); - final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long jobId = connectionManagerUtils.getCurrentJobId(connectionId); return new ManualOperationResult( Optional.empty(), @@ -266,7 +266,7 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { public ManualOperationResult startNewCancellation(final UUID connectionId) { log.info("Manual cancellation request"); - final long jobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long jobId = connectionManagerUtils.getCurrentJobId(connectionId); try { connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob); @@ -319,7 +319,7 @@ public ManualOperationResult resetConnection(final UUID connectionId, } // get the job ID before the reset, defaulting to NON_RUNNING_JOB_ID if workflow is unreachable - final long oldJobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long oldJobId = connectionManagerUtils.getCurrentJobId(connectionId); try { if (syncImmediatelyAfter) { @@ -355,7 +355,7 @@ public ManualOperationResult resetConnection(final UUID connectionId, } private Optional getNewJobId(final UUID connectionId, final long oldJobId) { - final long currentJobId = connectionManagerUtils.getCurrentJobId(client, connectionId); + final long currentJobId = connectionManagerUtils.getCurrentJobId(connectionId); if (currentJobId == NON_RUNNING_JOB_ID || currentJobId == oldJobId) { return Optional.empty(); } else { @@ -569,7 +569,7 @@ public void sendSchemaChangeNotification(final UUID connectionId, public void update(final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow; try { - connectionManagerWorkflow = connectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); + connectionManagerWorkflow = connectionManagerUtils.getConnectionManagerWorkflow(connectionId); } catch (final DeletedWorkflowException e) { log.info("Connection {} is deleted, and therefore cannot be updated.", connectionId); return; @@ -599,7 +599,7 @@ private boolean getConnectorJobSucceeded(final ConnectorJobOutput output) { @VisibleForTesting boolean isWorkflowReachable(final UUID connectionId) { try { - connectionManagerUtils.getConnectionManagerWorkflow(client, connectionId); + connectionManagerUtils.getConnectionManagerWorkflow(connectionId); return true; } catch (final Exception e) { return false; diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java new file mode 100644 index 00000000000..a08e4579a7d --- /dev/null +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; +import dev.failsafe.function.CheckedSupplier; +import io.airbyte.metrics.lib.MetricAttribute; +import io.airbyte.metrics.lib.MetricClient; +import io.airbyte.metrics.lib.MetricTags; +import io.airbyte.metrics.lib.OssMetricsRegistry; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; +import io.temporal.client.WorkflowClient; +import java.time.Duration; + +/** + * Wrapper around a temporal.client.WorkflowClient. The interface is a subset of the methods that we + * used wrapped with specific handling for transient errors. The goal being to avoid spreading those + * error handling across our codebase. + */ +public class WorkflowClientWrapped { + + private static final int DEFAULT_MAX_ATTEMPT = 3; + private final WorkflowClient temporalWorkflowClient; + private final MetricClient metricClient; + private final int maxAttempt; + + public WorkflowClientWrapped(final WorkflowClient workflowClient, final MetricClient metricClient) { + this.temporalWorkflowClient = workflowClient; + this.metricClient = metricClient; + this.maxAttempt = DEFAULT_MAX_ATTEMPT; + } + + /** + * Return the namespace of the temporal client. + */ + public String getNamespace() { + return temporalWorkflowClient.getOptions().getNamespace(); + } + + /** + * Creates workflow client stub for a known execution. Use it to send signals or queries to a + * running workflow. + */ + public T newWorkflowStub(final Class workflowInterface, final String workflowId) { + return withRetries(() -> temporalWorkflowClient.newWorkflowStub(workflowInterface, workflowId), "newWorkflowStub"); + } + + /** + * Returns information about the specified workflow execution. + */ + public DescribeWorkflowExecutionResponse blockingDescribeWorkflowExecution(final DescribeWorkflowExecutionRequest request) { + return withRetries(() -> temporalWorkflowClient.getWorkflowServiceStubs().blockingStub().describeWorkflowExecution(request), + "describeWorkflowExecution"); + } + + /** + * Where the magic happens. + *

+ * We should only retry errors that are transient GRPC network errors. + *

+ * We should only retry idempotent calls. The caller should be responsible for retrying creates to + * avoid generating additional noise. + */ + private T withRetries(final CheckedSupplier call, final String name) { + final var retry = RetryPolicy.builder() + .handleIf(this::shouldRetry) + .withMaxAttempts(maxAttempt) + .withBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10)) + .onRetry((a) -> metricClient.count(OssMetricsRegistry.TEMPORAL_API_TRANSIENT_ERROR_RETRY, 1, + new MetricAttribute(MetricTags.ATTEMPT_NUMBER, String.valueOf(a.getAttemptCount())), + new MetricAttribute(MetricTags.FAILURE_ORIGIN, name), + new MetricAttribute(MetricTags.FAILURE_TYPE, a.getLastException().getClass().getName()))) + .build(); + return Failsafe.with(retry).get(call); + } + + private boolean shouldRetry(final Throwable t) { + // We are retrying Status.UNAVAILABLE because it is often sign of an unexpected connection + // termination. + return t instanceof StatusRuntimeException && Status.UNAVAILABLE.equals(((StatusRuntimeException) t).getStatus()); + } + +} diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 30c2018ea9d..559cfea0a93 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -123,7 +123,7 @@ void setup() throws IOException { when(workflowServiceStubs.blockingStub()).thenReturn(workflowServiceBlockingStub); streamResetPersistence = mock(StreamResetPersistence.class); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); - connectionManagerUtils = spy(new ConnectionManagerUtils(mock(MetricClient.class))); + connectionManagerUtils = spy(new ConnectionManagerUtils(workflowClient, mock(MetricClient.class))); notificationClient = spy(new NotificationClient(workflowClient)); streamResetRecordsHelper = mock(StreamResetRecordsHelper.class); temporalClient = diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java new file mode 100644 index 00000000000..dbf40afd726 --- /dev/null +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.temporal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.metrics.lib.MetricClient; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest; +import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; +import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc.WorkflowServiceBlockingStub; +import io.temporal.client.WorkflowClient; +import io.temporal.serviceclient.WorkflowServiceStubs; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class WorkflowClientWrappedTest { + + static class MyWorkflow {} + + private MetricClient metricClient; + private WorkflowServiceStubs temporalWorkflowServiceStubs; + private WorkflowServiceBlockingStub temporalWorkflowServiceBlockingStub; + private WorkflowClient temporalWorkflowClient; + private WorkflowClientWrapped workflowClient; + + @BeforeEach + void beforeEach() { + metricClient = mock(MetricClient.class); + temporalWorkflowServiceBlockingStub = mock(WorkflowServiceBlockingStub.class); + temporalWorkflowServiceStubs = mock(WorkflowServiceStubs.class); + when(temporalWorkflowServiceStubs.blockingStub()).thenReturn(temporalWorkflowServiceBlockingStub); + temporalWorkflowClient = mock(WorkflowClient.class); + when(temporalWorkflowClient.getWorkflowServiceStubs()).thenReturn(temporalWorkflowServiceStubs); + workflowClient = new WorkflowClientWrapped(temporalWorkflowClient, metricClient); + } + + @Test + void testRetryLogic() { + when(temporalWorkflowClient.newWorkflowStub(any(), anyString())) + .thenThrow(unavailable()); + + assertThrows(StatusRuntimeException.class, () -> workflowClient.newWorkflowStub(MyWorkflow.class, "fail")); + verify(temporalWorkflowClient, times(3)).newWorkflowStub(any(), anyString()); + verify(metricClient, times(2)).count(any(), anyLong(), any()); + } + + @Test + void testNewWorkflowStub() { + final MyWorkflow expected = new MyWorkflow(); + when(temporalWorkflowClient.newWorkflowStub(any(), anyString())) + .thenThrow(unavailable()) + .thenReturn(expected); + + final MyWorkflow actual = workflowClient.newWorkflowStub(MyWorkflow.class, "woot"); + assertEquals(expected, actual); + } + + @Test + void testBlockingDescribeWorkflowExecution() { + final DescribeWorkflowExecutionResponse expected = mock(DescribeWorkflowExecutionResponse.class); + when(temporalWorkflowServiceBlockingStub.describeWorkflowExecution(any())) + .thenThrow(unavailable()) + .thenReturn(expected); + + final DescribeWorkflowExecutionResponse actual = workflowClient.blockingDescribeWorkflowExecution(mock(DescribeWorkflowExecutionRequest.class)); + assertEquals(expected, actual); + } + + private static StatusRuntimeException unavailable() { + return new StatusRuntimeException(Status.UNAVAILABLE); + } + +} diff --git a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java index 5fc07cff077..f7d8c0261e1 100644 --- a/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java +++ b/airbyte-metrics/metrics-lib/src/main/java/io/airbyte/metrics/lib/OssMetricsRegistry.java @@ -286,6 +286,9 @@ public enum OssMetricsRegistry implements MetricsRegistry { STREAM_STATS_WRITE_NUM_QUERIES(MetricEmittingApps.WORKER, "stream_stats_write_num_queries", "number of separate queries to update the stream stats table"), + TEMPORAL_API_TRANSIENT_ERROR_RETRY(MetricEmittingApps.WORKER, + "temporal_api_transient_error_retry", + "whenever we retry a temporal api call for transient errors"), TEMPORAL_WORKFLOW_ATTEMPT(MetricEmittingApps.WORKER, "temporal_workflow_attempt", "count of the number of workflow attempts"), From 6706409fcd61e90c2521e0e9a6fbccec201f57ab Mon Sep 17 00:00:00 2001 From: terencecho <3916587+terencecho@users.noreply.github.com> Date: Thu, 7 Sep 2023 11:07:10 -0700 Subject: [PATCH 129/201] Allow airbtye-api timeout from config-api to be set by env var (#8699) --- airbyte-api-server/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-api-server/src/main/resources/application.yml b/airbyte-api-server/src/main/resources/application.yml index ead9bac54a8..cf82c373508 100644 --- a/airbyte-api-server/src/main/resources/application.yml +++ b/airbyte-api-server/src/main/resources/application.yml @@ -18,7 +18,7 @@ micronaut: idle-timeout: ${HTTP_IDLE_TIMEOUT:5m} http: client: - read-timeout: 150s + read-timeout: ${READ_TIMEOUT:150s} max-content-length: 52428800 # 50MB airbyte: From db8672603d6dabb90cf2f53ab40f707075b699a4 Mon Sep 17 00:00:00 2001 From: terencecho <3916587+terencecho@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:28:24 -0700 Subject: [PATCH 130/201] Airbyte-api: Fix cron validation error message (#8728) --- .../airbyte/api/server/helpers/AirbyteCatalogHelper.kt | 6 +++++- .../server/problems/ConnectionConfigurationProblem.kt | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt index 15335e8ee30..4d1244f3606 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/helpers/AirbyteCatalogHelper.kt @@ -135,10 +135,14 @@ object AirbyteCatalogHelper { // Ensure minutes value is not `*` Integer.valueOf(cronStrings[1]) } + } catch (e: NumberFormatException) { + log.debug("Invalid cron expression: " + connectionSchedule.cronExpression) + log.debug("NumberFormatException: $e") + throw ConnectionConfigurationProblem.invalidCronExpressionUnderOneHour(connectionSchedule.cronExpression) } catch (e: IllegalArgumentException) { log.debug("Invalid cron expression: " + connectionSchedule.cronExpression) log.debug("IllegalArgumentException: $e") - throw ConnectionConfigurationProblem.invalidCronExpression(connectionSchedule.cronExpression) + throw ConnectionConfigurationProblem.invalidCronExpression(connectionSchedule.cronExpression, e.message) } } } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt index 284822c62a3..3c35c0f2438 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/problems/ConnectionConfigurationProblem.kt @@ -85,13 +85,20 @@ class ConnectionConfigurationProblem private constructor(message: String) : Abst ) } - fun invalidCronExpression(cronExpression: String): ConnectionConfigurationProblem { + fun invalidCronExpressionUnderOneHour(cronExpression: String): ConnectionConfigurationProblem { return ConnectionConfigurationProblem( "The cron expression " + cronExpression + " is not valid or is less than the one hour minimum. The seconds and minutes values cannot be `*`.", ) } + fun invalidCronExpression(cronExpression: String, message: String?): ConnectionConfigurationProblem { + return ConnectionConfigurationProblem( + "The cron expression $cronExpression is not valid. Error: $message" + + ". Please check the cron expression format at https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html", + ) + } + fun missingCronExpression(): ConnectionConfigurationProblem { return ConnectionConfigurationProblem("Missing cron expression in the schedule.") } From 2cb25778e39a10723461113cfa5cad8f5b5e4e60 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 8 Sep 2023 09:46:34 +0200 Subject: [PATCH 131/201] Add weaviate logo (#8468) --- .../src/main/resources/icons/weaviate.svg | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/weaviate.svg diff --git a/airbyte-config/init/src/main/resources/icons/weaviate.svg b/airbyte-config/init/src/main/resources/icons/weaviate.svg new file mode 100644 index 00000000000..ff31522d53b --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/weaviate.svg @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1216e150718536dbb3bcb34c9c875a59141b7441 Mon Sep 17 00:00:00 2001 From: Chandler Prall Date: Fri, 8 Sep 2023 07:14:53 -0600 Subject: [PATCH 132/201] =?UTF-8?q?=F0=9F=AA=9F=C2=A0=F0=9F=A7=B9=20refact?= =?UTF-8?q?or=20intent=20to=20remove=20string=20duplication=20(#8670)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: josephkmh Co-authored-by: Teal Larson Co-authored-by: Joey Marshment-Howell --- .../src/core/utils/rbac/intent.test.ts | 24 +++++++++---------- airbyte-webapp/src/core/utils/rbac/intent.ts | 18 +++++--------- .../src/core/utils/rbac/rbac.docs.mdx | 4 ++-- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/airbyte-webapp/src/core/utils/rbac/intent.test.ts b/airbyte-webapp/src/core/utils/rbac/intent.test.ts index 6e288879bb5..5f238a46ab5 100644 --- a/airbyte-webapp/src/core/utils/rbac/intent.test.ts +++ b/airbyte-webapp/src/core/utils/rbac/intent.test.ts @@ -1,6 +1,6 @@ import { renderHook } from "@testing-library/react"; -import { Intent, useIntent } from "./intent"; +import { useIntent } from "./intent"; import { useRbac } from "./rbac"; jest.mock("./rbac", () => ({ @@ -11,7 +11,7 @@ const mockUseRbac = useRbac as unknown as jest.Mock; describe("useIntent", () => { it("maps intent to query", () => { mockUseRbac.mockClear(); - renderHook(() => useIntent(Intent.ListOrganizationMembers, undefined)); + renderHook(() => useIntent("ListOrganizationMembers", undefined)); expect(mockUseRbac).toHaveBeenCalledTimes(1); expect(mockUseRbac).toHaveBeenCalledWith({ resourceType: "ORGANIZATION", @@ -22,7 +22,7 @@ describe("useIntent", () => { describe("applies overriding details", () => { it("overrides the organizationId", () => { mockUseRbac.mockClear(); - renderHook(() => useIntent(Intent.ListOrganizationMembers, { organizationId: "some-other-org" })); + renderHook(() => useIntent("ListOrganizationMembers", { organizationId: "some-other-org" })); expect(mockUseRbac).toHaveBeenCalledTimes(1); expect(mockUseRbac).toHaveBeenCalledWith({ resourceType: "ORGANIZATION", @@ -33,7 +33,7 @@ describe("useIntent", () => { it("overrides the workspaceId", () => { mockUseRbac.mockClear(); - renderHook(() => useIntent(Intent.ListWorkspaceMembers, { workspaceId: "some-other-workspace" })); + renderHook(() => useIntent("ListWorkspaceMembers", { workspaceId: "some-other-workspace" })); expect(mockUseRbac).toHaveBeenCalledTimes(1); expect(mockUseRbac).toHaveBeenCalledWith({ resourceType: "WORKSPACE", @@ -46,7 +46,7 @@ describe("useIntent", () => { mockUseRbac.mockClear(); renderHook(() => // @ts-expect-error we're testing invalid object shapes - useIntent(Intent.ListOrganizationMembers, { workspaceId: "some-other-organization" }, mockUseRbac) + useIntent("ListOrganizationMembers", { workspaceId: "some-other-organization" }, mockUseRbac) ); expect(mockUseRbac).toHaveBeenCalledTimes(1); expect(mockUseRbac).toHaveBeenCalledWith({ @@ -56,7 +56,7 @@ describe("useIntent", () => { mockUseRbac.mockClear(); // @ts-expect-error we're testing invalid object shapes - renderHook(() => useIntent(Intent.ListWorkspaceMembers, { organizationId: "some-other-workspace" }, mockUseRbac)); + renderHook(() => useIntent("ListWorkspaceMembers", { organizationId: "some-other-workspace" }, mockUseRbac)); expect(mockUseRbac).toHaveBeenCalledTimes(1); expect(mockUseRbac).toHaveBeenCalledWith({ resourceType: "WORKSPACE", @@ -71,14 +71,14 @@ describe("useIntent", () => { // @TODO: if we have any instance-level intents, add checks here to exclude organizationId and workspaceId - processIntent(Intent.ListOrganizationMembers); - processIntent(Intent.ListOrganizationMembers, { organizationId: "org" }); + processIntent("ListOrganizationMembers"); + processIntent("ListOrganizationMembers", { organizationId: "org" }); // @ts-expect-error workspaceId is not valid for ListOrganizationMembers - processIntent(Intent.ListOrganizationMembers, { workspaceId: "workspace" }); + processIntent("ListOrganizationMembers", { workspaceId: "workspace" }); - processIntent(Intent.ListWorkspaceMembers); - processIntent(Intent.ListWorkspaceMembers, { workspaceId: "workspace" }); + processIntent("ListWorkspaceMembers"); + processIntent("ListWorkspaceMembers", { workspaceId: "workspace" }); // @ts-expect-error workspaceId is not valid for ListWorkspaceMembers - processIntent(Intent.ListWorkspaceMembers, { organizationId: "organizationId" }); + processIntent("ListWorkspaceMembers", { organizationId: "organizationId" }); }); }); diff --git a/airbyte-webapp/src/core/utils/rbac/intent.ts b/airbyte-webapp/src/core/utils/rbac/intent.ts index b75547acf38..979ca97382f 100644 --- a/airbyte-webapp/src/core/utils/rbac/intent.ts +++ b/airbyte-webapp/src/core/utils/rbac/intent.ts @@ -1,22 +1,16 @@ import { useRbac } from "./rbac"; import { RbacQuery, RbacQueryWithoutResourceId, RbacResource } from "./rbacPermissionsQuery"; -export enum Intent { - // instance - - // organization - "ListOrganizationMembers" = "ListOrganizationMembers", - - // workspace - "ListWorkspaceMembers" = "ListWorkspaceMembers", -} - const intentToRbacQuery = { - [Intent.ListOrganizationMembers]: { resourceType: "ORGANIZATION", role: "READER" }, + ListOrganizationMembers: { resourceType: "ORGANIZATION", role: "READER" }, + UpdateOrganizationPermissions: { resourceType: "ORGANIZATION", role: "ADMIN" }, - [Intent.ListWorkspaceMembers]: { resourceType: "WORKSPACE", role: "READER" }, + UpdateWorkspacePermissions: { resourceType: "WORKSPACE", role: "ADMIN" }, + ListWorkspaceMembers: { resourceType: "WORKSPACE", role: "READER" }, } as const; +export type Intent = keyof typeof intentToRbacQuery; + interface OrganizationIntentMeta { organizationId?: string; } diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx b/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx index 72d548489c9..af4789bad83 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx +++ b/airbyte-webapp/src/core/utils/rbac/rbac.docs.mdx @@ -24,7 +24,7 @@ Use intents to ask if a user can see something or perform an action. Intents bet The `useIntent` hook is provided to make inquiries. If an intent for your use case does not yet exist it's almost certainly the right decision to create it. ```typescript -const canReadWorkspace = useIntent(Intent.ReadWorkspace, { workspaceId: "some-workspace" }); +const canReadWorkspace = useIntent("ReadWorkspace", { workspaceId: "some-workspace" }); ``` #### Meta details @@ -34,7 +34,7 @@ By default, `useIntent` will locate the user ID from the Auth Service. Resource If desired, the user ID can be overridden in the meta object as well. ```typescript -const canThatUserReadThatWorkspace = useIntent(Intent.ReadWorkspace, { +const canThatUserReadThatWorkspace = useIntent("ReadWorkspace", { userId: "some-user", workspaceId: "some-workspace", }); From a3e4967a8fbbe6b845972d88438ed7186629914a Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Fri, 8 Sep 2023 09:05:49 -0700 Subject: [PATCH 133/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Fix=20cha?= =?UTF-8?q?nging=20connector=20names=20in=20builder=20(#8716)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/core/api/hooks/connectorBuilderProject.ts | 1 + .../connectorBuilder/ConnectorBuilderStateService.tsx | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts b/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts index a60c1f87544..443ca54c543 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts @@ -335,6 +335,7 @@ export const useReleaseNewBuilderProjectVersion = () => { if (context.useAsActiveVersion) { updateProjectQueryCache(queryClient, context.projectId, context.sourceDefinitionId, context.version); } + queryClient.invalidateQueries(sourceDefinitionKeys.lists()); }, } ); diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 7a7f8408095..414e71909f1 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -175,6 +175,13 @@ export const InternalConnectorBuilderFormStateProvider: React.FC< const mode = useBuilderWatch("mode"); const name = useBuilderWatch("name"); + useEffect(() => { + if (name !== currentProject.name) { + setPreviousManifestDraft(undefined); + setDisplayedVersion(undefined); + } + }, [currentProject.name, name]); + // use ref so that updateJsonManifest is not recreated on every change to jsonManifest const jsonManifestRef = useRef(jsonManifest); jsonManifestRef.current = jsonManifest; From aa26eb05c60e945ceb6dbe5cee17db7a06520bbb Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Fri, 8 Sep 2023 09:06:09 -0700 Subject: [PATCH 134/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Use=20def?= =?UTF-8?q?ault=20json=20manifest=20as=20resolved=20manifest=20in=20error?= =?UTF-8?q?=20case=20(#8710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/connectorBuilder/ConnectorBuilderStateService.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 414e71909f1..c188cb17be6 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -591,7 +591,8 @@ export const ConnectorBuilderTestReadProvider: React.FC Date: Fri, 8 Sep 2023 12:06:53 -0400 Subject: [PATCH 135/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Apply=20R?= =?UTF-8?q?BAC=20rules=20to=20existing=20workspace=20and=20organization=20?= =?UTF-8?q?update=20and=20delete=20operations=20(#8729)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/UpdateWorkspaceNameForm.tsx | 5 +- .../src/components/forms/Form.module.scss | 7 ++ airbyte-webapp/src/core/utils/rbac/intent.ts | 11 +- .../GeneralOrganizationSettingsPage.tsx | 5 +- .../GeneralWorkspaceSettingsPage.tsx | 7 +- .../src/pages/SettingsPage/SettingsPage.tsx | 103 ++++++++++-------- 6 files changed, 86 insertions(+), 52 deletions(-) diff --git a/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx b/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx index be5b1e438dc..ac6ceb6bc64 100644 --- a/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx +++ b/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx @@ -5,6 +5,7 @@ import { Form, FormControl } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; import { useCurrentWorkspace, useUpdateWorkspaceName } from "core/api"; +import { useIntent } from "core/utils/rbac/intent"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification"; @@ -22,6 +23,7 @@ export const UpdateWorkspaceNameForm = () => { const { mutateAsync: updateWorkspaceName } = useUpdateWorkspaceName(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); + const canUpdateWorkspace = useIntent("UpdateWorkspace", { workspaceId }); const onSubmit = async ({ name }: UpdateWorkspaceNameFormValues) => { await updateWorkspaceName({ @@ -55,13 +57,14 @@ export const UpdateWorkspaceNameForm = () => { onSubmit={onSubmit} onError={onError} onSuccess={onSuccess} + disabled={!canUpdateWorkspace} > name="name" fieldType="input" label={formatMessage({ id: "settings.workspaceSettings.updateWorkspaceNameForm.name.label" })} /> - + {canUpdateWorkspace && } ); }; diff --git a/airbyte-webapp/src/components/forms/Form.module.scss b/airbyte-webapp/src/components/forms/Form.module.scss index fbd60a64e39..4bc9ef1a1ee 100644 --- a/airbyte-webapp/src/components/forms/Form.module.scss +++ b/airbyte-webapp/src/components/forms/Form.module.scss @@ -1,4 +1,11 @@ .fieldset { all: unset; width: 100%; + + &[disabled] { + *:hover { + cursor: default; + border-color: transparent; + } + } } diff --git a/airbyte-webapp/src/core/utils/rbac/intent.ts b/airbyte-webapp/src/core/utils/rbac/intent.ts index 979ca97382f..1589fd42715 100644 --- a/airbyte-webapp/src/core/utils/rbac/intent.ts +++ b/airbyte-webapp/src/core/utils/rbac/intent.ts @@ -2,11 +2,20 @@ import { useRbac } from "./rbac"; import { RbacQuery, RbacQueryWithoutResourceId, RbacResource } from "./rbacPermissionsQuery"; const intentToRbacQuery = { + // instance + + // organization ListOrganizationMembers: { resourceType: "ORGANIZATION", role: "READER" }, + UpdateOrganization: { resourceType: "ORGANIZATION", role: "ADMIN" }, UpdateOrganizationPermissions: { resourceType: "ORGANIZATION", role: "ADMIN" }, + ViewOrganizationSettings: { resourceType: "ORGANIZATION", role: "READER" }, - UpdateWorkspacePermissions: { resourceType: "WORKSPACE", role: "ADMIN" }, + // workspace + DeleteWorkspace: { resourceType: "WORKSPACE", role: "ADMIN" }, ListWorkspaceMembers: { resourceType: "WORKSPACE", role: "READER" }, + UpdateWorkspace: { resourceType: "WORKSPACE", role: "ADMIN" }, + UpdateWorkspacePermissions: { resourceType: "WORKSPACE", role: "ADMIN" }, + ViewWorkspaceSettings: { resourceType: "WORKSPACE", role: "READER" }, } as const; export type Intent = keyof typeof intentToRbacQuery; diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx index dc8b76f8cc0..d8a7c134a1b 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx @@ -11,6 +11,7 @@ import { Card } from "components/ui/Card"; import { useCurrentWorkspace, useUpdateOrganization } from "core/api"; import { useOrganization } from "core/api"; import { OrganizationUpdateRequestBody } from "core/request/AirbyteClient"; +import { useIntent } from "core/utils/rbac/intent"; import { useNotificationService } from "hooks/services/Notification"; const ORGANIZATION_UPDATE_NOTIFICATION_ID = "organization-update-notification"; @@ -37,6 +38,7 @@ const OrganizationSettingsForm = ({ organizationId }: { organizationId: string } const { formatMessage } = useIntl(); const { registerNotification, unregisterNotificationById } = useNotificationService(); + const canUpdateOrganization = useIntent("UpdateOrganization", { organizationId }); const onSubmit = async (values: OrganizationFormValues) => { await updateOrganization({ @@ -71,13 +73,14 @@ const OrganizationSettingsForm = ({ organizationId }: { organizationId: string } }} schema={organizationValidationSchema} defaultValues={{ organizationName: organization.organizationName }} + disabled={!canUpdateOrganization} > label={formatMessage({ id: "settings.organizationSettings.organizationName" })} fieldType="input" name="organizationName" /> - + {canUpdateOrganization && } ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx index 35bb7bb9441..027ad76981f 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx @@ -5,10 +5,15 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { UpdateWorkspaceNameForm } from "area/workspace/components/UpdateWorkspaceNameForm"; +import { useCurrentWorkspace } from "core/api"; +import { useIntent } from "core/utils/rbac/intent"; import { DeleteWorkspace } from "./components/DeleteWorkspace"; export const GeneralWorkspaceSettingsPage = () => { + const { workspaceId } = useCurrentWorkspace(); + const canDeleteWorkspace = useIntent("DeleteWorkspace", { workspaceId }); + return ( }> @@ -16,7 +21,7 @@ export const GeneralWorkspaceSettingsPage = () => { - + {canDeleteWorkspace && } ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx index 98b9bae8666..3cfd9969f29 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx @@ -11,6 +11,7 @@ import { PageHeader } from "components/ui/PageHeader"; import { SideMenu, CategoryItem, SideMenuItem } from "components/ui/SideMenu"; import { useCurrentWorkspace } from "core/api"; +import { useIntent } from "core/utils/rbac/intent"; import { useExperiment } from "hooks/services/Experiment"; import { useGetConnectorsOutOfDate } from "hooks/services/useConnector"; @@ -47,11 +48,13 @@ export const SettingsRoute = { const SettingsPage: React.FC = ({ pageConfig }) => { const push = useNavigate(); - const { organizationId } = useCurrentWorkspace(); + const { organizationId, workspaceId } = useCurrentWorkspace(); const { pathname } = useLocation(); const { countNewSourceVersion, countNewDestinationVersion } = useGetConnectorsOutOfDate(); const newWorkspacesUI = useExperiment("workspaces.newWorkspacesUI", false); const isAccessManagementEnabled = useExperiment("settings.accessManagement", false); + const canViewWorkspaceSettings = useIntent("ViewWorkspaceSettings", { workspaceId }); + const canViewOrganizationSettings = useIntent("ViewOrganizationSettings", { organizationId }); const menuItems: CategoryItem[] = pageConfig?.menuConfig || [ { @@ -64,57 +67,61 @@ const SettingsPage: React.FC = ({ pageConfig }) => { }, ], }, - { - category: , - routes: [ - ...(newWorkspacesUI - ? [ + ...(canViewWorkspaceSettings + ? [ + { + category: , + routes: [ + ...(newWorkspacesUI + ? [ + { + path: `${SettingsRoute.Workspace}`, + name: , + component: GeneralWorkspaceSettingsPage, + }, + ] + : []), { - path: `${SettingsRoute.Workspace}`, - name: , - component: GeneralWorkspaceSettingsPage, + path: `${SettingsRoute.Source}`, + name: , + indicatorCount: countNewSourceVersion, + component: SourcesPage, }, - ] - : []), - { - path: `${SettingsRoute.Source}`, - name: , - indicatorCount: countNewSourceVersion, - component: SourcesPage, - }, - { - path: `${SettingsRoute.Destination}`, - name: , - indicatorCount: countNewDestinationVersion, - component: DestinationsPage, - }, - { - path: `${SettingsRoute.Configuration}`, - name: , - component: ConfigurationsPage, - }, - { - path: `${SettingsRoute.Notifications}`, - name: , - component: NotificationPage, - }, - { - path: `${SettingsRoute.Metrics}`, - name: , - component: MetricsPage, - }, - ...(isAccessManagementEnabled && !pageConfig - ? [ { - path: `${SettingsRoute.Workspace}/${SettingsRoute.AccessManagement}`, - name: , - component: WorkspaceAccessManagementPage, + path: `${SettingsRoute.Destination}`, + name: , + indicatorCount: countNewDestinationVersion, + component: DestinationsPage, }, - ] - : []), - ], - }, - ...(newWorkspacesUI && organizationId + { + path: `${SettingsRoute.Configuration}`, + name: , + component: ConfigurationsPage, + }, + { + path: `${SettingsRoute.Notifications}`, + name: , + component: NotificationPage, + }, + { + path: `${SettingsRoute.Metrics}`, + name: , + component: MetricsPage, + }, + ...(isAccessManagementEnabled && !pageConfig + ? [ + { + path: `${SettingsRoute.Workspace}/${SettingsRoute.AccessManagement}`, + name: , + component: WorkspaceAccessManagementPage, + }, + ] + : []), + ], + }, + ] + : []), + ...(newWorkspacesUI && organizationId && canViewOrganizationSettings ? [ { category: , From 185aa8b647a07005e274a77e27705e7ac86f27f0 Mon Sep 17 00:00:00 2001 From: Ben Church Date: Fri, 8 Sep 2023 09:26:43 -0700 Subject: [PATCH 136/201] Connector Support Level: Update Billing API (#8726) --- .../src/main/openapi/cloud-config.yaml | 25 +++++++++++++++++++ .../test-utils/mock-data/mockBillingData.ts | 4 +++ 2 files changed, 29 insertions(+) diff --git a/airbyte-api/src/main/openapi/cloud-config.yaml b/airbyte-api/src/main/openapi/cloud-config.yaml index 05e78e29874..64aacc4e132 100644 --- a/airbyte-api/src/main/openapi/cloud-config.yaml +++ b/airbyte-api/src/main/openapi/cloud-config.yaml @@ -1413,6 +1413,13 @@ components: - generally_available - custom + ActorSupportLevel: + type: string + enum: + - community + - certified + - none + ConnectionProto: type: object required: @@ -1425,12 +1432,16 @@ components: - sourceConnectionName - sourceIcon - sourceReleaseStage + - sourceSupportLevel + - sourceCustom - destinationId - destinationDefinitionId - destinationDefinitionName - destinationConnectionName - destinationIcon - destinationReleaseStage + - destinationSupportLevel + - destinationCustom properties: connectionId: type: string @@ -1462,6 +1473,13 @@ components: sourceIcon: description: Icon of the source actor. type: string + sourceCustom: + description: True if the source is custom + type: boolean + default: false + sourceSupportLevel: + description: Support Level of source actor. + $ref: "#/components/schemas/ActorSupportLevel" sourceReleaseStage: description: Release stage of source actor. $ref: "#/components/schemas/ActorReleaseStage" @@ -1480,6 +1498,13 @@ components: destinationIcon: description: Icon of the destination actor. type: string + destinationCustom: + description: True if the destination is custom + type: boolean + default: false + destinationSupportLevel: + description: Support Level of destination actor. + $ref: "#/components/schemas/ActorSupportLevel" destinationReleaseStage: description: Release stage of destination actor. $ref: "#/components/schemas/ActorReleaseStage" diff --git a/airbyte-webapp/src/test-utils/mock-data/mockBillingData.ts b/airbyte-webapp/src/test-utils/mock-data/mockBillingData.ts index 2d69c637a9f..b8d95bdd396 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockBillingData.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockBillingData.ts @@ -22,6 +22,10 @@ export const mockWorkspaceUsageConnection: ConsumptionRead["connection"] = { connectionScheduleUnits: undefined, destinationReleaseStage: "alpha", sourceReleaseStage: "alpha", + sourceSupportLevel: "community", + destinationSupportLevel: "community", + sourceCustom: false, + destinationCustom: false, }; export const secondMockWorkspaceConnection: ConsumptionRead["connection"] = { From bb708e3acae34fe0bbbe919223d92a4de1367670 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Fri, 8 Sep 2023 13:50:33 -0400 Subject: [PATCH 137/201] =?UTF-8?q?Revert=20"=F0=9F=AA=9F=20=F0=9F=94=A7?= =?UTF-8?q?=20Apply=20RBAC=20rules=20to=20existing=20workspace=20and=20org?= =?UTF-8?q?anization=20update=20and=20delete=20operations"=20(#8735)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/UpdateWorkspaceNameForm.tsx | 5 +- .../src/components/forms/Form.module.scss | 7 -- airbyte-webapp/src/core/utils/rbac/intent.ts | 11 +- .../GeneralOrganizationSettingsPage.tsx | 5 +- .../GeneralWorkspaceSettingsPage.tsx | 7 +- .../src/pages/SettingsPage/SettingsPage.tsx | 103 ++++++++---------- 6 files changed, 52 insertions(+), 86 deletions(-) diff --git a/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx b/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx index ac6ceb6bc64..be5b1e438dc 100644 --- a/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx +++ b/airbyte-webapp/src/area/workspace/components/UpdateWorkspaceNameForm.tsx @@ -5,7 +5,6 @@ import { Form, FormControl } from "components/forms"; import { FormSubmissionButtons } from "components/forms/FormSubmissionButtons"; import { useCurrentWorkspace, useUpdateWorkspaceName } from "core/api"; -import { useIntent } from "core/utils/rbac/intent"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification"; @@ -23,7 +22,6 @@ export const UpdateWorkspaceNameForm = () => { const { mutateAsync: updateWorkspaceName } = useUpdateWorkspaceName(); const { registerNotification } = useNotificationService(); const { trackError } = useAppMonitoringService(); - const canUpdateWorkspace = useIntent("UpdateWorkspace", { workspaceId }); const onSubmit = async ({ name }: UpdateWorkspaceNameFormValues) => { await updateWorkspaceName({ @@ -57,14 +55,13 @@ export const UpdateWorkspaceNameForm = () => { onSubmit={onSubmit} onError={onError} onSuccess={onSuccess} - disabled={!canUpdateWorkspace} > name="name" fieldType="input" label={formatMessage({ id: "settings.workspaceSettings.updateWorkspaceNameForm.name.label" })} /> - {canUpdateWorkspace && } + ); }; diff --git a/airbyte-webapp/src/components/forms/Form.module.scss b/airbyte-webapp/src/components/forms/Form.module.scss index 4bc9ef1a1ee..fbd60a64e39 100644 --- a/airbyte-webapp/src/components/forms/Form.module.scss +++ b/airbyte-webapp/src/components/forms/Form.module.scss @@ -1,11 +1,4 @@ .fieldset { all: unset; width: 100%; - - &[disabled] { - *:hover { - cursor: default; - border-color: transparent; - } - } } diff --git a/airbyte-webapp/src/core/utils/rbac/intent.ts b/airbyte-webapp/src/core/utils/rbac/intent.ts index 1589fd42715..979ca97382f 100644 --- a/airbyte-webapp/src/core/utils/rbac/intent.ts +++ b/airbyte-webapp/src/core/utils/rbac/intent.ts @@ -2,20 +2,11 @@ import { useRbac } from "./rbac"; import { RbacQuery, RbacQueryWithoutResourceId, RbacResource } from "./rbacPermissionsQuery"; const intentToRbacQuery = { - // instance - - // organization ListOrganizationMembers: { resourceType: "ORGANIZATION", role: "READER" }, - UpdateOrganization: { resourceType: "ORGANIZATION", role: "ADMIN" }, UpdateOrganizationPermissions: { resourceType: "ORGANIZATION", role: "ADMIN" }, - ViewOrganizationSettings: { resourceType: "ORGANIZATION", role: "READER" }, - // workspace - DeleteWorkspace: { resourceType: "WORKSPACE", role: "ADMIN" }, - ListWorkspaceMembers: { resourceType: "WORKSPACE", role: "READER" }, - UpdateWorkspace: { resourceType: "WORKSPACE", role: "ADMIN" }, UpdateWorkspacePermissions: { resourceType: "WORKSPACE", role: "ADMIN" }, - ViewWorkspaceSettings: { resourceType: "WORKSPACE", role: "READER" }, + ListWorkspaceMembers: { resourceType: "WORKSPACE", role: "READER" }, } as const; export type Intent = keyof typeof intentToRbacQuery; diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx index d8a7c134a1b..dc8b76f8cc0 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx @@ -11,7 +11,6 @@ import { Card } from "components/ui/Card"; import { useCurrentWorkspace, useUpdateOrganization } from "core/api"; import { useOrganization } from "core/api"; import { OrganizationUpdateRequestBody } from "core/request/AirbyteClient"; -import { useIntent } from "core/utils/rbac/intent"; import { useNotificationService } from "hooks/services/Notification"; const ORGANIZATION_UPDATE_NOTIFICATION_ID = "organization-update-notification"; @@ -38,7 +37,6 @@ const OrganizationSettingsForm = ({ organizationId }: { organizationId: string } const { formatMessage } = useIntl(); const { registerNotification, unregisterNotificationById } = useNotificationService(); - const canUpdateOrganization = useIntent("UpdateOrganization", { organizationId }); const onSubmit = async (values: OrganizationFormValues) => { await updateOrganization({ @@ -73,14 +71,13 @@ const OrganizationSettingsForm = ({ organizationId }: { organizationId: string } }} schema={organizationValidationSchema} defaultValues={{ organizationName: organization.organizationName }} - disabled={!canUpdateOrganization} > label={formatMessage({ id: "settings.organizationSettings.organizationName" })} fieldType="input" name="organizationName" /> - {canUpdateOrganization && } + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx index 027ad76981f..35bb7bb9441 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralWorkspaceSettingsPage.tsx @@ -5,15 +5,10 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { UpdateWorkspaceNameForm } from "area/workspace/components/UpdateWorkspaceNameForm"; -import { useCurrentWorkspace } from "core/api"; -import { useIntent } from "core/utils/rbac/intent"; import { DeleteWorkspace } from "./components/DeleteWorkspace"; export const GeneralWorkspaceSettingsPage = () => { - const { workspaceId } = useCurrentWorkspace(); - const canDeleteWorkspace = useIntent("DeleteWorkspace", { workspaceId }); - return ( }> @@ -21,7 +16,7 @@ export const GeneralWorkspaceSettingsPage = () => { - {canDeleteWorkspace && } + ); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx index 3cfd9969f29..98b9bae8666 100644 --- a/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/SettingsPage.tsx @@ -11,7 +11,6 @@ import { PageHeader } from "components/ui/PageHeader"; import { SideMenu, CategoryItem, SideMenuItem } from "components/ui/SideMenu"; import { useCurrentWorkspace } from "core/api"; -import { useIntent } from "core/utils/rbac/intent"; import { useExperiment } from "hooks/services/Experiment"; import { useGetConnectorsOutOfDate } from "hooks/services/useConnector"; @@ -48,13 +47,11 @@ export const SettingsRoute = { const SettingsPage: React.FC = ({ pageConfig }) => { const push = useNavigate(); - const { organizationId, workspaceId } = useCurrentWorkspace(); + const { organizationId } = useCurrentWorkspace(); const { pathname } = useLocation(); const { countNewSourceVersion, countNewDestinationVersion } = useGetConnectorsOutOfDate(); const newWorkspacesUI = useExperiment("workspaces.newWorkspacesUI", false); const isAccessManagementEnabled = useExperiment("settings.accessManagement", false); - const canViewWorkspaceSettings = useIntent("ViewWorkspaceSettings", { workspaceId }); - const canViewOrganizationSettings = useIntent("ViewOrganizationSettings", { organizationId }); const menuItems: CategoryItem[] = pageConfig?.menuConfig || [ { @@ -67,61 +64,57 @@ const SettingsPage: React.FC = ({ pageConfig }) => { }, ], }, - ...(canViewWorkspaceSettings - ? [ - { - category: , - routes: [ - ...(newWorkspacesUI - ? [ - { - path: `${SettingsRoute.Workspace}`, - name: , - component: GeneralWorkspaceSettingsPage, - }, - ] - : []), - { - path: `${SettingsRoute.Source}`, - name: , - indicatorCount: countNewSourceVersion, - component: SourcesPage, - }, - { - path: `${SettingsRoute.Destination}`, - name: , - indicatorCount: countNewDestinationVersion, - component: DestinationsPage, - }, - { - path: `${SettingsRoute.Configuration}`, - name: , - component: ConfigurationsPage, - }, + { + category: , + routes: [ + ...(newWorkspacesUI + ? [ { - path: `${SettingsRoute.Notifications}`, - name: , - component: NotificationPage, + path: `${SettingsRoute.Workspace}`, + name: , + component: GeneralWorkspaceSettingsPage, }, + ] + : []), + { + path: `${SettingsRoute.Source}`, + name: , + indicatorCount: countNewSourceVersion, + component: SourcesPage, + }, + { + path: `${SettingsRoute.Destination}`, + name: , + indicatorCount: countNewDestinationVersion, + component: DestinationsPage, + }, + { + path: `${SettingsRoute.Configuration}`, + name: , + component: ConfigurationsPage, + }, + { + path: `${SettingsRoute.Notifications}`, + name: , + component: NotificationPage, + }, + { + path: `${SettingsRoute.Metrics}`, + name: , + component: MetricsPage, + }, + ...(isAccessManagementEnabled && !pageConfig + ? [ { - path: `${SettingsRoute.Metrics}`, - name: , - component: MetricsPage, + path: `${SettingsRoute.Workspace}/${SettingsRoute.AccessManagement}`, + name: , + component: WorkspaceAccessManagementPage, }, - ...(isAccessManagementEnabled && !pageConfig - ? [ - { - path: `${SettingsRoute.Workspace}/${SettingsRoute.AccessManagement}`, - name: , - component: WorkspaceAccessManagementPage, - }, - ] - : []), - ], - }, - ] - : []), - ...(newWorkspacesUI && organizationId && canViewOrganizationSettings + ] + : []), + ], + }, + ...(newWorkspacesUI && organizationId ? [ { category: , From cb3a8f4f6c12502e18476f6d07482377ce68bc5a Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Fri, 8 Sep 2023 21:50:25 +0200 Subject: [PATCH 138/201] Allow enterprise startup without IDP config (#8736) --- .../micronaut/config/AirbytePropertySourceLoader.java | 8 +++++--- .../airbyte/keycloak/setup/IdentityProvidersCreator.java | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java index 3967d0bd96e..adb017c0172 100644 --- a/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java +++ b/airbyte-commons-micronaut/src/main/java/io/airbyte/micronaut/config/AirbytePropertySourceLoader.java @@ -13,9 +13,9 @@ import io.micronaut.core.io.file.DefaultFileSystemResourceLoader; import java.io.IOException; import java.io.InputStream; +import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; /** @@ -50,8 +50,10 @@ public Optional load(final String resourceName, final ResourceLo final var yaml = read(resourceName, stream); // Prefix all configuration from that file with airbyte. - final var prefixedProps = yaml.entrySet().stream().collect(Collectors.toMap(entry -> AIRBYTE_KEY + "." + entry.getKey(), Map.Entry::getValue)); - log.info("Loading properties as: {}", prefixedProps); + final var prefixedProps = new HashMap(); + for (var entry : yaml.entrySet()) { + prefixedProps.put(AIRBYTE_KEY + "." + entry.getKey(), entry.getValue()); + } return Optional.of(PropertySource.of(prefixedProps)); } catch (final IOException e) { throw new ConfigurationException("Could not load airbyte.yml configuration file.", e); diff --git a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java index 7c42cd9742f..7eb6495c107 100644 --- a/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java +++ b/airbyte-keycloak-setup/src/main/java/io/airbyte/keycloak/setup/IdentityProvidersCreator.java @@ -42,7 +42,8 @@ public IdentityProvidersCreator(final List identi public void createIdps(final RealmResource keycloakRealm) { // Create Identity Providers if (identityProviderConfigurations == null || identityProviderConfigurations.isEmpty()) { - throw new RuntimeException("No providers found in config"); + log.info("No identity providers configured. Skipping IDP setup."); + return; } for (final IdentityProviderConfiguration provider : identityProviderConfigurations) { From bab4603dc4fab2c5ff6b07890bce57730c779f34 Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Fri, 8 Sep 2023 18:54:49 -0500 Subject: [PATCH 139/201] write breaking changes and upgrade actors in the same transaction (#8632) --- .../ConnectorBuilderProjectsHandler.java | 2 +- .../DestinationDefinitionsHandler.java | 7 +- .../handlers/SourceDefinitionsHandler.java | 7 +- .../ConnectorBuilderProjectsHandlerTest.java | 2 +- .../DestinationDefinitionsHandlerTest.java | 13 +- .../SourceDefinitionsHandlerTest.java | 13 +- .../config/persistence/ConfigRepository.java | 193 ++++--- ...finitionBreakingChangePersistenceTest.java | 93 ++- .../ActorDefinitionPersistenceTest.java | 211 +------ ...ActorDefinitionVersionPersistenceTest.java | 51 +- .../persistence/ActorPersistenceTest.java | 145 +---- .../persistence/ConfigInjectionTest.java | 4 +- .../ConfigRepositoryE2EReadWriteTest.java | 10 +- ...onnectorBuilderProjectPersistenceTest.java | 6 +- .../ConnectorMetadataPersistenceTest.java | 529 ++++++++++++++++++ .../DeclarativeManifestPersistenceTest.java | 2 +- .../StandardSyncPersistenceTest.java | 4 +- .../persistence/StatePersistenceTest.java | 4 +- .../persistence/WorkspacePersistenceTest.java | 4 +- .../config/init/ApplyDefinitionsHelper.java | 16 +- .../init/ApplyDefinitionsHelperTest.java | 39 +- 21 files changed, 750 insertions(+), 605 deletions(-) create mode 100644 airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorMetadataPersistenceTest.java diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java index bf5b2c5ea5d..5b8b287b5c7 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandler.java @@ -224,7 +224,7 @@ private UUID createActorDefinition(final String name, final UUID workspaceId, fi .withSupportLevel(SupportLevel.NONE) .withDocumentationUrl(connectorSpecification.getDocumentationUrl().toString()); - configRepository.writeCustomSourceDefinitionAndDefaultVersion(source, defaultVersion, workspaceId, ScopeType.WORKSPACE); + configRepository.writeCustomConnectorMetadata(source, defaultVersion, workspaceId, ScopeType.WORKSPACE); configRepository.writeActorDefinitionConfigInjectionForPath(manifestInjector.createConfigInjection(source.getSourceDefinitionId(), manifest)); return source.getSourceDefinitionId(); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java index 53b3d15b339..300584d2148 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandler.java @@ -251,10 +251,10 @@ public DestinationDefinitionRead createCustomDestinationDefinition(final CustomD // legacy call; todo: remove once we drop workspace_id column if (customDestinationDefinitionCreate.getWorkspaceId() != null) { - configRepository.writeCustomDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(destinationDefinition, actorDefinitionVersion, customDestinationDefinitionCreate.getWorkspaceId(), ScopeType.WORKSPACE); } else { - configRepository.writeCustomDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(destinationDefinition, actorDefinitionVersion, customDestinationDefinitionCreate.getScopeId(), ScopeType.fromValue(customDestinationDefinitionCreate.getScopeType().toString())); } @@ -285,9 +285,8 @@ public DestinationDefinitionRead updateDestinationDefinition(final DestinationDe final List breakingChangesForDef = actorDefinitionHandlerHelper.getBreakingChanges(newVersion, ActorType.DESTINATION); - configRepository.writeDestinationDefinitionAndDefaultVersion(newDestination, newVersion, breakingChangesForDef); + configRepository.writeConnectorMetadata(newDestination, newVersion, breakingChangesForDef); - configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { final StandardDestinationDefinition updatedDestinationDefinition = configRepository .getStandardDestinationDefinition(destinationDefinitionUpdate.getDestinationDefinitionId()); diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java index 037b57c945f..1077e0f240b 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandler.java @@ -251,10 +251,10 @@ public SourceDefinitionRead createCustomSourceDefinition(final CustomSourceDefin // legacy call; todo: remove once we drop workspace_id column if (customSourceDefinitionCreate.getWorkspaceId() != null) { - configRepository.writeCustomSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(sourceDefinition, actorDefinitionVersion, customSourceDefinitionCreate.getWorkspaceId(), ScopeType.WORKSPACE); } else { - configRepository.writeCustomSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, + configRepository.writeCustomConnectorMetadata(sourceDefinition, actorDefinitionVersion, customSourceDefinitionCreate.getScopeId(), ScopeType.fromValue(customSourceDefinitionCreate.getScopeType().toString())); } @@ -285,8 +285,7 @@ public SourceDefinitionRead updateSourceDefinition(final SourceDefinitionUpdate currentVersion, ActorType.SOURCE, sourceDefinitionUpdate.getDockerImageTag(), currentSourceDefinition.getCustom()); final List breakingChangesForDef = actorDefinitionHandlerHelper.getBreakingChanges(newVersion, ActorType.SOURCE); - configRepository.writeSourceDefinitionAndDefaultVersion(newSource, newVersion, breakingChangesForDef); - configRepository.writeActorDefinitionBreakingChanges(breakingChangesForDef); + configRepository.writeConnectorMetadata(newSource, newVersion, breakingChangesForDef); if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { final StandardSourceDefinition updatedSourceDefinition = configRepository.getStandardSourceDefinition(newSource.getSourceDefinitionId()); diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java index 6d6793164ef..85c169e3c03 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectorBuilderProjectsHandlerTest.java @@ -380,7 +380,7 @@ void whenPublishConnectorBuilderProjectThenCreateActorDefinition() throws IOExce .initialDeclarativeManifest(anyInitialManifest().manifest(A_MANIFEST).spec(A_SPEC))); verify(manifestInjector, times(1)).addInjectedDeclarativeManifest(A_SPEC); - verify(configRepository, times(1)).writeCustomSourceDefinitionAndDefaultVersion(eq(new StandardSourceDefinition() + verify(configRepository, times(1)).writeCustomConnectorMetadata(eq(new StandardSourceDefinition() .withSourceDefinitionId(A_SOURCE_DEFINITION_ID) .withName(A_SOURCE_NAME) .withSourceType(SourceType.CUSTOM) diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java index 1371d7a6de7..4f1c13bd03f 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/DestinationDefinitionsHandlerTest.java @@ -600,7 +600,7 @@ void testCreateCustomDestinationDefinition() throws URISyntaxException, IOExcept verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository).writeCustomDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newDestinationDefinition .withCustom(true) .withDefaultVersionId(null), @@ -666,7 +666,7 @@ void testCreateCustomDestinationDefinitionUsingScopes() throws URISyntaxExceptio verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreateForWorkspace.getWorkspaceId()); - verify(configRepository).writeCustomDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newDestinationDefinition .withCustom(true) .withDefaultVersionId(null), @@ -691,7 +691,7 @@ void testCreateCustomDestinationDefinitionUsingScopes() throws URISyntaxExceptio verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), null); - verify(configRepository).writeCustomDestinationDefinitionAndDefaultVersion(newDestinationDefinition.withCustom(true).withDefaultVersionId(null), + verify(configRepository).writeCustomConnectorMetadata(newDestinationDefinition.withCustom(true).withDefaultVersionId(null), destinationDefinitionVersion, organizationId, ScopeType.ORGANIZATION); verifyNoMoreInteractions(actorDefinitionHandlerHelper); @@ -727,7 +727,7 @@ void testCreateCustomDestinationDefinitionShouldCheckProtocolVersion() throws UR verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository, never()).writeCustomDestinationDefinitionAndDefaultVersion(any(), any(), any(), any()); + verify(configRepository, never()).writeCustomConnectorMetadata(any(StandardDestinationDefinition.class), any(), any(), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } @@ -771,8 +771,7 @@ void testUpdateDestination(final boolean runSupportStateUpdaterFlagValue) throws verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(destinationDefinitionVersion, ActorType.DESTINATION, newDockerImageTag, destinationDefinition.getCustom()); verify(actorDefinitionHandlerHelper).getBreakingChanges(updatedDestinationDefVersion, ActorType.DESTINATION); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion(updatedDestination, updatedDestinationDefVersion, breakingChanges); - verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + verify(configRepository).writeConnectorMetadata(updatedDestination, updatedDestinationDefVersion, breakingChanges); if (runSupportStateUpdaterFlagValue) { verify(supportStateUpdater).updateSupportStatesForDestinationDefinition(persistedUpdatedDestination); } else { @@ -806,7 +805,7 @@ void testOutOfProtocolRangeUpdateDestination() throws ConfigNotFoundException, I verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(destinationDefinitionVersion, ActorType.DESTINATION, newDockerImageTag, destinationDefinition.getCustom()); - verify(configRepository, never()).writeDestinationDefinitionAndDefaultVersion(any(), any()); + verify(configRepository, never()).writeConnectorMetadata(any(StandardDestinationDefinition.class), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java index 76e5b9edbd7..824433cd921 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/SourceDefinitionsHandlerTest.java @@ -536,7 +536,7 @@ void testCreateCustomSourceDefinition() throws URISyntaxException, IOException { verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository).writeCustomSourceDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newSourceDefinition .withCustom(true) .withDefaultVersionId(null), @@ -600,7 +600,7 @@ void testCreateCustomSourceDefinitionUsingScopes() throws URISyntaxException, IO verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreateForWorkspace.getWorkspaceId()); - verify(configRepository).writeCustomSourceDefinitionAndDefaultVersion( + verify(configRepository).writeCustomConnectorMetadata( newSourceDefinition .withCustom(true) .withDefaultVersionId(null), @@ -625,7 +625,7 @@ void testCreateCustomSourceDefinitionUsingScopes() throws URISyntaxException, IO verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), null); - verify(configRepository).writeCustomSourceDefinitionAndDefaultVersion(newSourceDefinition.withCustom(true).withDefaultVersionId(null), + verify(configRepository).writeCustomConnectorMetadata(newSourceDefinition.withCustom(true).withDefaultVersionId(null), sourceDefinitionVersion, organizationId, ScopeType.ORGANIZATION); verifyNoMoreInteractions(actorDefinitionHandlerHelper); @@ -661,7 +661,7 @@ void testCreateCustomSourceDefinitionShouldCheckProtocolVersion() throws URISynt verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromCreate(create.getDockerRepository(), create.getDockerImageTag(), create.getDocumentationUrl(), customCreate.getWorkspaceId()); - verify(configRepository, never()).writeCustomSourceDefinitionAndDefaultVersion(any(), any(), any(), any()); + verify(configRepository, never()).writeCustomConnectorMetadata(any(StandardSourceDefinition.class), any(), any(), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } @@ -705,8 +705,7 @@ void testUpdateSource(final boolean runSupportStateUpdaterFlagValue) throws Conf verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(sourceDefinitionVersion, ActorType.SOURCE, newDockerImageTag, sourceDefinition.getCustom()); verify(actorDefinitionHandlerHelper).getBreakingChanges(updatedSourceDefVersion, ActorType.SOURCE); - verify(configRepository).writeSourceDefinitionAndDefaultVersion(updatedSource, updatedSourceDefVersion, breakingChanges); - verify(configRepository).writeActorDefinitionBreakingChanges(breakingChanges); + verify(configRepository).writeConnectorMetadata(updatedSource, updatedSourceDefVersion, breakingChanges); if (runSupportStateUpdaterFlagValue) { verify(supportStateUpdater).updateSupportStatesForSourceDefinition(persistedUpdatedSource); } else { @@ -740,7 +739,7 @@ void testOutOfProtocolRangeUpdateSource() throws ConfigNotFoundException, IOExce verify(actorDefinitionHandlerHelper).defaultDefinitionVersionFromUpdate(sourceDefinitionVersion, ActorType.SOURCE, newDockerImageTag, sourceDefinition.getCustom()); - verify(configRepository, never()).writeSourceDefinitionAndDefaultVersion(any(), any()); + verify(configRepository, never()).writeConnectorMetadata(any(StandardSourceDefinition.class), any()); verifyNoMoreInteractions(actorDefinitionHandlerHelper); } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 6c6c1038699..07ef4ccc62f 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -758,48 +758,6 @@ public void updateStandardSourceDefinition(final StandardSourceDefinition source }); } - /** - * Write a StandardSourceDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardSourceDefinition. - * - * @param sourceDefinition standard source definition - * @param actorDefinitionVersion actor definition version - * @param breakingChangesForDefinition - list of breaking changes for the definition - * @throws IOException - you never know when you IO - */ - public void writeSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition) - throws IOException { - database.transaction(ctx -> { - writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); - return null; - }); - } - - /** - * Write a StandardSourceDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardSourceDefinition. Assumes the - * definition has no breaking changes. - * - * @param sourceDefinition standard source definition - * @param actorDefinitionVersion actor definition version - * @throws IOException - you never know when you IO - */ - public void writeSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion actorDefinitionVersion) - throws IOException { - writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion, List.of()); - } - - private void writeSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { - ConfigWriter.writeStandardSourceDefinition(Collections.singletonList(sourceDefinition), ctx); - setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); - } - /** * Update the docker image tag for multiple actor definitions at once. * @@ -811,27 +769,6 @@ public int updateActorDefinitionsDockerImageTag(final List actorDefinition return database.transaction(ctx -> ConfigWriter.writeSourceDefinitionImageTag(actorDefinitionIds, targetImageTag, ctx)); } - /** - * Write custom source definition and its default version. - * - * @param sourceDefinition source definition - * @param defaultVersion default version - * @param scopeId scope id - * @param scopeType enum which defines if the scopeId is a workspace or organization id - * @throws IOException - you never know when you IO - */ - public void writeCustomSourceDefinitionAndDefaultVersion(final StandardSourceDefinition sourceDefinition, - final ActorDefinitionVersion defaultVersion, - final UUID scopeId, - final io.airbyte.config.ScopeType scopeType) - throws IOException { - database.transaction(ctx -> { - writeSourceDefinitionAndDefaultVersion(sourceDefinition, defaultVersion, List.of(), ctx); - writeActorDefinitionWorkspaceGrant(sourceDefinition.getSourceDefinitionId(), scopeId, ScopeType.valueOf(scopeType.toString()), ctx); - return null; - }); - } - private void updateDeclarativeActorDefinition(final ActorDefinitionConfigInjection configInjection, final ConnectorSpecification spec, final DSLContext ctx) { @@ -987,44 +924,96 @@ public void updateStandardDestinationDefinition(final StandardDestinationDefinit } /** - * Write a StandardDestinationDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardDestinationDefinition. + * Write metadata for a destination connector. Writes global metadata (destination definition) and + * versioned metadata (info for actor definition version to set as default). Sets the new version as + * the default version and updates actors accordingly, based on whether the upgrade will be breaking + * or not. * * @param destinationDefinition standard destination definition * @param actorDefinitionVersion actor definition version * @param breakingChangesForDefinition - list of breaking changes for the definition * @throws IOException - you never know when you IO */ - public void writeDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition) + public void writeConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition) throws IOException { database.transaction(ctx -> { - writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); + writeConnectorMetadata(destinationDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); return null; }); } /** - * Write a StandardDestinationDefinition and the ActorDefinitionVersion associated with it (if not - * pre-existing) to the DB, setting the default version on the StandardDestinationDefinition. - * Assumes the definition has no breaking changes. + * Write metadata for a destination connector. Writes global metadata (destination definition) and + * versioned metadata (info for actor definition version to set as default). Sets the new version as + * the default version and updates actors accordingly, based on whether the upgrade will be breaking + * or not. Usage of this version of the method assumes no new breaking changes need to be persisted + * for the definition. * * @param destinationDefinition standard destination definition * @param actorDefinitionVersion actor definition version * @throws IOException - you never know when you IO */ - public void writeDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion actorDefinitionVersion) + public void writeConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion actorDefinitionVersion) throws IOException { - writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion, List.of()); + writeConnectorMetadata(destinationDefinition, actorDefinitionVersion, List.of()); } - private void writeDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion actorDefinitionVersion, - final List breakingChangesForDefinition, - final DSLContext ctx) { + private void writeConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition, + final DSLContext ctx) { ConfigWriter.writeStandardDestinationDefinition(Collections.singletonList(destinationDefinition), ctx); + writeActorDefinitionBreakingChanges(breakingChangesForDefinition, ctx); + setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); + } + + /** + * Write metadata for a source connector. Writes global metadata (source definition, breaking + * changes) and versioned metadata (info for actor definition version to set as default). Sets the + * new version as the default version and updates actors accordingly, based on whether the upgrade + * will be breaking or not. + * + * @param sourceDefinition standard source definition + * @param actorDefinitionVersion actor definition version, containing tag to set as default + * @param breakingChangesForDefinition - list of breaking changes for the definition + * @throws IOException - you never know when you IO + */ + public void writeConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition) + throws IOException { + database.transaction(ctx -> { + writeConnectorMetadata(sourceDefinition, actorDefinitionVersion, breakingChangesForDefinition, ctx); + return null; + }); + } + + /** + * Write metadata for a source connector. Writes global metadata (source definition) and versioned + * metadata (info for actor definition version to set as default). Sets the new version as the + * default version and updates actors accordingly, based on whether the upgrade will be breaking or + * not. Usage of this version of the method assumes no new breaking changes need to be persisted for + * the definition. + * + * @param sourceDefinition standard source definition + * @param actorDefinitionVersion actor definition version + * @throws IOException - you never know when you IO + */ + public void writeConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion actorDefinitionVersion) + throws IOException { + writeConnectorMetadata(sourceDefinition, actorDefinitionVersion, List.of()); + } + + private void writeConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion actorDefinitionVersion, + final List breakingChangesForDefinition, + final DSLContext ctx) { + ConfigWriter.writeStandardSourceDefinition(Collections.singletonList(sourceDefinition), ctx); + writeActorDefinitionBreakingChanges(breakingChangesForDefinition, ctx); setActorDefinitionVersionForTagAsDefault(actorDefinitionVersion, breakingChangesForDefinition, ctx); } @@ -1087,25 +1076,49 @@ private void updateActorDefinitionDefaultVersionId(final UUID actorDefinitionId, } /** - * Write custom destination definition and its default version. + * Write metadata for a custom destination: global metadata (destination definition) and versioned + * metadata (actor definition version for the version to use). * * @param destinationDefinition destination definition + * @param defaultVersion default actor definition version * @param scopeId workspace or organization id * @param scopeType enum of workpsace or organization * @throws IOException - you never know when you IO */ - public void writeCustomDestinationDefinitionAndDefaultVersion(final StandardDestinationDefinition destinationDefinition, - final ActorDefinitionVersion defaultVersion, - final UUID scopeId, - final io.airbyte.config.ScopeType scopeType) + public void writeCustomConnectorMetadata(final StandardDestinationDefinition destinationDefinition, + final ActorDefinitionVersion defaultVersion, + final UUID scopeId, + final io.airbyte.config.ScopeType scopeType) throws IOException { database.transaction(ctx -> { - writeDestinationDefinitionAndDefaultVersion(destinationDefinition, defaultVersion, List.of(), ctx); + writeConnectorMetadata(destinationDefinition, defaultVersion, List.of(), ctx); writeActorDefinitionWorkspaceGrant(destinationDefinition.getDestinationDefinitionId(), scopeId, ScopeType.valueOf(scopeType.toString()), ctx); return null; }); } + /** + * Write metadata for a custom source: global metadata (source definition) and versioned metadata + * (actor definition version for the version to use). + * + * @param sourceDefinition source definition + * @param defaultVersion default actor definition version + * @param scopeId scope id + * @param scopeType enum which defines if the scopeId is a workspace or organization id + * @throws IOException - you never know when you IO + */ + public void writeCustomConnectorMetadata(final StandardSourceDefinition sourceDefinition, + final ActorDefinitionVersion defaultVersion, + final UUID scopeId, + final io.airbyte.config.ScopeType scopeType) + throws IOException { + database.transaction(ctx -> { + writeConnectorMetadata(sourceDefinition, defaultVersion, List.of(), ctx); + writeActorDefinitionWorkspaceGrant(sourceDefinition.getSourceDefinitionId(), scopeId, ScopeType.valueOf(scopeType.toString()), ctx); + return null; + }); + } + /** * Delete connection. * @@ -3684,13 +3697,15 @@ public List getActorDefinitionVersions(final List * already exist. * * @param breakingChanges - actor definition breaking changes to write + * @param ctx database context * @throws IOException - you never know when you io */ - public void writeActorDefinitionBreakingChanges(final List breakingChanges) throws IOException { + private void writeActorDefinitionBreakingChanges(final List breakingChanges, final DSLContext ctx) { final OffsetDateTime timestamp = OffsetDateTime.now(); - database.query(ctx -> ctx - .batch(breakingChanges.stream().map(breakingChange -> upsertBreakingChangeQuery(ctx, breakingChange, timestamp)).collect(Collectors.toList())) - .execute()); + final List upsertQueries = breakingChanges.stream() + .map(breakingChange -> upsertBreakingChangeQuery(ctx, breakingChange, timestamp)) + .collect(Collectors.toList()); + ctx.batch(upsertQueries).execute(); } /** diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java index 269012212ae..d724f15351f 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionBreakingChangePersistenceTest.java @@ -4,6 +4,7 @@ package io.airbyte.config.persistence; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -76,6 +77,15 @@ final ActorDefinitionVersion createActorDefVersion(final UUID actorDefinitionId) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); } + final ActorDefinitionVersion createActorDefVersion(final UUID actorDefinitionId, final String dockerImageTag) { + return new ActorDefinitionVersion() + .withActorDefinitionId(actorDefinitionId) + .withDockerImageTag(dockerImageTag) + .withDockerRepository("repo") + .withSupportLevel(SupportLevel.COMMUNITY) + .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); + } + private ConfigRepository configRepository; @BeforeEach @@ -84,32 +94,25 @@ void setup() throws SQLException, JsonValidationException, IOException { configRepository = spy(new ConfigRepository(database, mock(StandardSyncPersistence.class), MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, createActorDefVersion(SOURCE_DEFINITION.getSourceDefinitionId())); - configRepository.writeDestinationDefinitionAndDefaultVersion(DESTINATION_DEFINITION, - createActorDefVersion(DESTINATION_DEFINITION.getDestinationDefinitionId())); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, createActorDefVersion(SOURCE_DEFINITION.getSourceDefinitionId()), + List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4)); + configRepository.writeConnectorMetadata(DESTINATION_DEFINITION, + createActorDefVersion(DESTINATION_DEFINITION.getDestinationDefinitionId()), List.of(OTHER_CONNECTOR_BREAKING_CHANGE)); } @Test - void testReadAndWriteActorDefinitionBreakingChange() throws IOException { - final List prevBreakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(0, prevBreakingChanges.size()); - - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE)); - final List breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(1, breakingChanges.size()); - assertEquals(BREAKING_CHANGE, breakingChanges.get(0)); + void testGetBreakingChanges() throws IOException { + final List breakingChangesForDef1 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); + assertEquals(4, breakingChangesForDef1.size()); + assertEquals(BREAKING_CHANGE, breakingChangesForDef1.get(0)); + + final List breakingChangesForDef2 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_2); + assertEquals(1, breakingChangesForDef2.size()); + assertEquals(OTHER_CONNECTOR_BREAKING_CHANGE, breakingChangesForDef2.get(0)); } @Test void testUpdateActorDefinitionBreakingChange() throws IOException { - final List prevBreakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(0, prevBreakingChanges.size()); - - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE)); - List breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(1, breakingChanges.size()); - assertEquals(BREAKING_CHANGE, breakingChanges.get(0)); - // Update breaking change final ActorDefinitionBreakingChange updatedBreakingChange = new ActorDefinitionBreakingChange() .withActorDefinitionId(BREAKING_CHANGE.getActorDefinitionId()) @@ -117,70 +120,42 @@ void testUpdateActorDefinitionBreakingChange() throws IOException { .withMessage("Updated message") .withUpgradeDeadline("2025-01-01") .withMigrationDocumentationUrl(BREAKING_CHANGE.getMigrationDocumentationUrl()); - configRepository.writeActorDefinitionBreakingChanges(List.of(updatedBreakingChange)); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, createActorDefVersion(SOURCE_DEFINITION.getSourceDefinitionId()), + List.of(updatedBreakingChange, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4)); // Check updated breaking change - breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(1, breakingChanges.size()); + final List breakingChanges = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); + assertEquals(4, breakingChanges.size()); assertEquals(updatedBreakingChange, breakingChanges.get(0)); } - @Test - void testWriteMultipleActorDefinitionBreakingChanges() throws IOException { - assertEquals(0, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1).size()); - assertEquals(0, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_2).size()); - - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, OTHER_CONNECTOR_BREAKING_CHANGE)); - - final List breakingChangesForId1 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1); - assertEquals(2, breakingChangesForId1.size()); - assertEquals(BREAKING_CHANGE, breakingChangesForId1.get(0)); - assertEquals(BREAKING_CHANGE_2, breakingChangesForId1.get(1)); - - final List breakingChangesForId2 = configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_2); - assertEquals(1, breakingChangesForId2.size()); - assertEquals(OTHER_CONNECTOR_BREAKING_CHANGE, breakingChangesForId2.get(0)); - } - @Test void testListBreakingChanges() throws IOException { - final List breakingChanges = List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4); - assertEquals(0, configRepository.listBreakingChanges().size()); - configRepository.writeActorDefinitionBreakingChanges(breakingChanges); - assertEquals(breakingChanges, configRepository.listBreakingChanges()); + final List expectedAllBreakingChanges = + List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4, OTHER_CONNECTOR_BREAKING_CHANGE); + assertThat(expectedAllBreakingChanges).containsExactlyInAnyOrderElementsOf(configRepository.listBreakingChanges()); } @Test void testListBreakingChangesForVersion() throws IOException { - assertEquals(0, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1).size()); - configRepository.writeActorDefinitionBreakingChanges(List.of(BREAKING_CHANGE, BREAKING_CHANGE_2, BREAKING_CHANGE_3, BREAKING_CHANGE_4)); - - final ActorDefinitionVersion ADV_4_0_0 = createActorDefinitionVersion("4.0.0"); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, ADV_4_0_0); + final ActorDefinitionVersion ADV_4_0_0 = createActorDefVersion(ACTOR_DEFINITION_ID_1, "4.0.0"); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, ADV_4_0_0); // no breaking changes for latest default assertEquals(4, configRepository.listBreakingChangesForActorDefinition(ACTOR_DEFINITION_ID_1).size()); assertEquals(0, configRepository.listBreakingChangesForActorDefinitionVersion(ADV_4_0_0).size()); // should see future breaking changes for 2.0.0 - final ActorDefinitionVersion ADV_2_0_0 = createActorDefinitionVersion("2.0.0"); + final ActorDefinitionVersion ADV_2_0_0 = createActorDefVersion(ACTOR_DEFINITION_ID_1, "2.0.0"); assertEquals(2, configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0).size()); assertEquals(List.of(BREAKING_CHANGE_3, BREAKING_CHANGE_4), configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0)); // move back default version for Actor Definition to 3.0.0, should stop seeing "rolled back" // breaking changes - final ActorDefinitionVersion ADV_3_0_0 = createActorDefinitionVersion("3.0.0"); - configRepository.writeSourceDefinitionAndDefaultVersion(SOURCE_DEFINITION, ADV_3_0_0); + final ActorDefinitionVersion ADV_3_0_0 = createActorDefVersion(ACTOR_DEFINITION_ID_1, "3.0.0"); + configRepository.writeConnectorMetadata(SOURCE_DEFINITION, ADV_3_0_0); assertEquals(1, configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0).size()); assertEquals(List.of(BREAKING_CHANGE_3), configRepository.listBreakingChangesForActorDefinitionVersion(ADV_2_0_0)); } - ActorDefinitionVersion createActorDefinitionVersion(final String version) { - return new ActorDefinitionVersion() - .withActorDefinitionId(ACTOR_DEFINITION_ID_1) - .withDockerRepository("some-repo") - .withSupportLevel(SupportLevel.COMMUNITY) - .withDockerImageTag(version); - } - } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java index 746daf42352..554e25b9649 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionPersistenceTest.java @@ -4,18 +4,14 @@ package io.airbyte.config.persistence; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import io.airbyte.commons.version.Version; -import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.DestinationConnection; import io.airbyte.config.Geography; @@ -31,12 +27,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; class ActorDefinitionPersistenceTest extends BaseConfigDatabaseTest { @@ -44,8 +38,6 @@ class ActorDefinitionPersistenceTest extends BaseConfigDatabaseTest { private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final String DOCKER_IMAGE_TAG = "0.0.1"; - private static final String UPDATED_IMAGE_TAG = "0.0.2"; - private ConfigRepository configRepository; @BeforeEach @@ -82,7 +74,7 @@ void testSourceDefinitionMaxSeconds() throws JsonValidationException, ConfigNotF private void assertReturnsSrcDef(final StandardSourceDefinition srcDef) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); assertEquals(srcDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), configRepository.getStandardSourceDefinition(srcDef.getSourceDefinitionId())); } @@ -90,7 +82,7 @@ private void assertReturnsSrcDef(final StandardSourceDefinition srcDef) throws C private void assertReturnsSrcDefDefaultMaxSecondsBetweenMessages(final StandardSourceDefinition srcDef) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); assertEquals( srcDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()) .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES), @@ -98,13 +90,13 @@ private void assertReturnsSrcDefDefaultMaxSecondsBetweenMessages(final StandardS } @Test - void testSourceDefinitionFromSource() throws JsonValidationException, IOException { + void testGetSourceDefinitionFromSource() throws JsonValidationException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardSourceDefinition srcDef = createBaseSourceDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); final SourceConnection source = createSource(srcDef.getSourceDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); configRepository.writeSourceConnectionNoSecrets(source); assertEquals(srcDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), @@ -112,13 +104,13 @@ void testSourceDefinitionFromSource() throws JsonValidationException, IOExceptio } @Test - void testSourceDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetSourceDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardSourceDefinition srcDef = createBaseSourceDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(srcDef.getSourceDefinitionId()); final SourceConnection source = createSource(srcDef.getSourceDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion(srcDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(srcDef, actorDefinitionVersion); configRepository.writeSourceConnectionNoSecrets(source); final UUID connectionId = UUID.randomUUID(); @@ -148,7 +140,7 @@ void testListStandardSourceDefsHandlesTombstoneSourceDefs(final int numSrcDefs) if (!isTombstone) { notTombstoneSourceDefinitions.add(sourceDefinition); } - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); sourceDefinition.setDefaultVersionId(actorDefinitionVersion.getVersionId()); } @@ -176,19 +168,19 @@ void testDestinationDefinitionWithFalseTombstone() throws JsonValidationExceptio void assertReturnsDestDef(final StandardDestinationDefinition destDef) throws ConfigNotFoundException, IOException, JsonValidationException { final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion); assertEquals(destDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), configRepository.getStandardDestinationDefinition(destDef.getDestinationDefinitionId())); } @Test - void testDestinationDefinitionFromDestination() throws JsonValidationException, IOException { + void testGetDestinationDefinitionFromDestination() throws JsonValidationException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardDestinationDefinition destDef = createBaseDestDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); final DestinationConnection dest = createDest(destDef.getDestinationDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion); configRepository.writeDestinationConnectionNoSecrets(dest); assertEquals(destDef.withDefaultVersionId(actorDefinitionVersion.getVersionId()), @@ -196,13 +188,13 @@ void testDestinationDefinitionFromDestination() throws JsonValidationException, } @Test - void testDestinationDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetDestinationDefinitionsFromConnection() throws JsonValidationException, ConfigNotFoundException, IOException { final StandardWorkspace workspace = createBaseStandardWorkspace(); final StandardDestinationDefinition destDef = createBaseDestDef().withTombstone(false); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); final DestinationConnection dest = createDest(destDef.getDestinationDefinitionId(), workspace.getWorkspaceId()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion); configRepository.writeDestinationConnectionNoSecrets(dest); final UUID connectionId = UUID.randomUUID(); @@ -232,7 +224,7 @@ void testListStandardDestDefsHandlesTombstoneDestDefs(final int numDestinationDe if (!isTombstone) { notTombstoneDestinationDefinitions.add(destinationDefinition); } - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion); destinationDefinition.setDefaultVersionId(actorDefinitionVersion.getVersionId()); } @@ -256,9 +248,9 @@ void testUpdateAllImageTagsForDeclarativeSourceDefinition() throws JsonValidatio sourceVer2.setDockerImageTag(targetImageTag); final ActorDefinitionVersion sourceVer3 = createBaseActorDefVersion(sourceDef3.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef1, sourceVer1); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef2, sourceVer2); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef3, sourceVer3); + configRepository.writeConnectorMetadata(sourceDef1, sourceVer1); + configRepository.writeConnectorMetadata(sourceDef2, sourceVer2); + configRepository.writeConnectorMetadata(sourceDef3, sourceVer3); final int updatedDefinitions = configRepository .updateActorDefinitionsDockerImageTag(List.of(sourceDef1.getSourceDefinitionId(), sourceDef2.getSourceDefinitionId()), targetImageTag); @@ -282,23 +274,23 @@ void getActorDefinitionIdsInUse() throws IOException, JsonValidationException { final StandardSourceDefinition sourceDefInUse = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion3 = createBaseActorDefVersion(sourceDefInUse.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefInUse, actorDefinitionVersion3); + configRepository.writeConnectorMetadata(sourceDefInUse, actorDefinitionVersion3); final SourceConnection sourceConnection = createSource(sourceDefInUse.getSourceDefinitionId(), workspace.getWorkspaceId()); configRepository.writeSourceConnectionNoSecrets(sourceConnection); final StandardSourceDefinition sourceDefNotInUse = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion4 = createBaseActorDefVersion(sourceDefNotInUse.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefNotInUse, actorDefinitionVersion4); + configRepository.writeConnectorMetadata(sourceDefNotInUse, actorDefinitionVersion4); final StandardDestinationDefinition destDefInUse = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destDefInUse.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDefInUse, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destDefInUse, actorDefinitionVersion); final DestinationConnection destinationConnection = createDest(destDefInUse.getDestinationDefinitionId(), workspace.getWorkspaceId()); configRepository.writeDestinationConnectionNoSecrets(destinationConnection); final StandardDestinationDefinition destDefNotInUse = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion2 = createBaseActorDefVersion(destDefNotInUse.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDefNotInUse, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(destDefNotInUse, actorDefinitionVersion2); assertTrue(configRepository.getActorDefinitionIdsInUse().contains(sourceDefInUse.getSourceDefinitionId())); assertTrue(configRepository.getActorDefinitionIdsInUse().contains(destDefInUse.getDestinationDefinitionId())); @@ -310,11 +302,11 @@ void getActorDefinitionIdsInUse() throws IOException, JsonValidationException { void testGetActorDefinitionIdsToDefaultVersionsMap() throws IOException { final StandardSourceDefinition sourceDef = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(sourceDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDef, actorDefinitionVersion); final StandardDestinationDefinition destDef = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion2 = createBaseActorDefVersion(destDef.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(destDef, actorDefinitionVersion2); final Map actorDefIdToDefaultVersionId = configRepository.getActorDefinitionIdsToDefaultVersionsMap(); assertEquals(actorDefIdToDefaultVersionId.size(), 2); @@ -322,158 +314,12 @@ void testGetActorDefinitionIdsToDefaultVersionsMap() throws IOException { assertEquals(actorDefIdToDefaultVersionId.get(destDef.getDestinationDefinitionId()), actorDefinitionVersion2); } - @Test - void testWriteSourceDefinitionAndDefaultVersion() - throws JsonValidationException, IOException, ConfigNotFoundException { - // Initial insert - final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); - final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion1); - - StandardSourceDefinition sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); - final Optional actorDefinitionVersionFromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); - - assertTrue(actorDefinitionVersionFromDB.isPresent()); - final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); - - assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); - assertEquals(firstVersionId, sourceDefinitionFromDB.getDefaultVersionId()); - assertEquals(sourceDefinition.withDefaultVersionId(firstVersionId), sourceDefinitionFromDB); - - // Updating an existing source definition/version - final StandardSourceDefinition sourceDefinition2 = sourceDefinition.withName("updated name"); - final ActorDefinitionVersion actorDefinitionVersion2 = - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(UPDATED_IMAGE_TAG); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition2, actorDefinitionVersion2); - - sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); - final Optional actorDefinitionVersion2FromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); - - assertTrue(actorDefinitionVersion2FromDB.isPresent()); - final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); - - assertNotEquals(firstVersionId, newADVId); - assertEquals(newADVId, sourceDefinitionFromDB.getDefaultVersionId()); - assertEquals(sourceDefinition2.withDefaultVersionId(newADVId), sourceDefinitionFromDB); - } - - @Test - void testWriteDestinationDefinitionAndDefaultVersion() - throws JsonValidationException, IOException, ConfigNotFoundException { - // Initial insert - final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); - final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion1); - - StandardDestinationDefinition destinationDefinitionFromDB = - configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); - final Optional actorDefinitionVersionFromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); - - assertTrue(actorDefinitionVersionFromDB.isPresent()); - final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); - - assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); - assertEquals(firstVersionId, destinationDefinitionFromDB.getDefaultVersionId()); - assertEquals(destinationDefinition.withDefaultVersionId(firstVersionId), destinationDefinitionFromDB); - - // Updating an existing destination definition/version - final StandardDestinationDefinition destinationDefinition2 = destinationDefinition.withName("updated name"); - final ActorDefinitionVersion actorDefinitionVersion2 = - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(UPDATED_IMAGE_TAG); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition2, actorDefinitionVersion2); - - destinationDefinitionFromDB = configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); - final Optional actorDefinitionVersion2FromDB = - configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); - - assertTrue(actorDefinitionVersion2FromDB.isPresent()); - final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); - - assertNotEquals(firstVersionId, newADVId); - assertEquals(newADVId, destinationDefinitionFromDB.getDefaultVersionId()); - assertEquals(destinationDefinition2.withDefaultVersionId(newADVId), destinationDefinitionFromDB); - } - - @ParameterizedTest - @ValueSource(strings = {"1.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) - void testCustomImageTagsDoNotBreakCustomConnectorUpgrade(final String dockerImageTag) throws IOException { - // Initial insert - final StandardSourceDefinition customSourceDefinition = createBaseSourceDef().withCustom(true); - final StandardDestinationDefinition customDestinationDefinition = createBaseDestDef().withCustom(true); - final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()); - final ActorDefinitionVersion destinationActorDefinitionVersion = - createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(customSourceDefinition, sourceActorDefinitionVersion); - configRepository.writeDestinationDefinitionAndDefaultVersion(customDestinationDefinition, destinationActorDefinitionVersion); - - // Update - assertDoesNotThrow(() -> configRepository.writeSourceDefinitionAndDefaultVersion(customSourceDefinition, - createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - assertDoesNotThrow(() -> configRepository.writeDestinationDefinitionAndDefaultVersion(customDestinationDefinition, - createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - } - - @ParameterizedTest - @ValueSource(strings = {"1.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) - void testImageTagExpectationsNorNonCustomConnectorUpgradesWithoutBreakingChanges(final String dockerImageTag) throws IOException { - // Initial insert - final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); - final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); - final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, sourceActorDefinitionVersion); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, destinationActorDefinitionVersion); - - // Update - assertDoesNotThrow(() -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - assertDoesNotThrow(() -> configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); - } - - @ParameterizedTest - @CsvSource({"0.0.1, true", "dev, true", "test, false", "1.9.1-dev.33a53e6236, true", "97b69a76-1f06-4680-8905-8beda74311d0, false"}) - void testImageTagExpectationsNorNonCustomConnectorUpgradesWithBreakingChanges(final String dockerImageTag, final boolean upgradeShouldSucceed) - throws IOException { - // Initial insert - final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); - final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); - final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, sourceActorDefinitionVersion); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, destinationActorDefinitionVersion); - - final List sourceBreakingChanges = createBreakingChangesForDef(sourceDefinition.getSourceDefinitionId()); - final List destinationBreakingChanges = - createBreakingChangesForDef(destinationDefinition.getDestinationDefinitionId()); - - // Update - if (upgradeShouldSucceed) { - assertDoesNotThrow(() -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); - assertDoesNotThrow(() -> configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), - destinationBreakingChanges)); - } else { - assertThrows(IllegalArgumentException.class, () -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, - createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); - assertThrows(IllegalArgumentException.class, () -> configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, - createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), - destinationBreakingChanges)); - } - } - @Test void testUpdateStandardSourceDefinition() throws IOException, JsonValidationException, ConfigNotFoundException { final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); final StandardSourceDefinition sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); @@ -506,7 +352,7 @@ void testUpdateStandardDestinationDefinition() throws IOException, JsonValidatio final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion); final StandardDestinationDefinition destinationDefinitionFromDB = configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); @@ -572,15 +418,6 @@ private static ActorDefinitionVersion createBaseActorDefVersion(final UUID actor .withProtocolVersion("0.2.0"); } - private List createBreakingChangesForDef(final UUID actorDefId) { - return List.of(new ActorDefinitionBreakingChange() - .withActorDefinitionId(actorDefId) - .withVersion(new Version("1.0.0")) - .withMessage("This is a breaking change") - .withMigrationDocumentationUrl("https://docs.airbyte.com/migration#1.0.0") - .withUpgradeDeadline("2025-01-21")); - } - private static StandardSourceDefinition createBaseSourceDefWithoutMaxSecondsBetweenMessages() { final UUID id = UUID.randomUUID(); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java index 5b966791856..1cc3d9a4b3c 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorDefinitionVersionPersistenceTest.java @@ -7,7 +7,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -43,13 +42,10 @@ class ActorDefinitionVersionPersistenceTest extends BaseConfigDatabaseTest { private static final String SOURCE_NAME = "Test Source"; private static final String DOCKER_REPOSITORY = "airbyte/source-test"; private static final String DOCKER_IMAGE_TAG = "0.1.0"; - private static final String DOCKER_IMAGE_TAG_2 = "0.2.0"; private static final String UNPERSISTED_DOCKER_IMAGE_TAG = "0.1.1"; private static final String PROTOCOL_VERSION = "1.0.0"; private static final ConnectorSpecification SPEC = new ConnectorSpecification() .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value"))).withProtocolVersion(PROTOCOL_VERSION); - private static final ConnectorSpecification SPEC_2 = new ConnectorSpecification() - .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value2"))).withProtocolVersion(PROTOCOL_VERSION); private static StandardSourceDefinition baseSourceDefinition(final UUID actorDefinitionId) { return new StandardSourceDefinition() @@ -99,7 +95,7 @@ void beforeEach() throws Exception { sourceDefinition = baseSourceDefinition(defId); // Make sure that the source definition exists before we start writing actor definition versions - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, initialADV); + configRepository.writeConnectorMetadata(sourceDefinition, initialADV); } @Test @@ -152,51 +148,12 @@ void testGetActorDefinitionVersionByIdNotExistentThrowsConfigNotFound() { assertThrows(ConfigNotFoundException.class, () -> configRepository.getActorDefinitionVersion(defId)); } - @Test - void testWriteSourceDefinitionAndDefaultVersion() throws IOException, JsonValidationException, ConfigNotFoundException { - // Write initial source definition and default version - final UUID defId = sourceDefinition.getSourceDefinitionId(); - final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId); - final StandardSourceDefinition sourceDefinition = baseSourceDefinition(defId); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, adv); - - final Optional optADVForTag = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); - assertTrue(optADVForTag.isPresent()); - final ActorDefinitionVersion advForTag = optADVForTag.get(); - final StandardSourceDefinition retrievedSourceDefinition = - configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); - assertEquals(retrievedSourceDefinition.getDefaultVersionId(), advForTag.getVersionId()); - - // Modify spec without changing docker image tag - final ActorDefinitionVersion modifiedADV = baseActorDefinitionVersion(defId).withSpec(SPEC_2); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, modifiedADV); - - assertEquals(retrievedSourceDefinition, configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())); - final Optional optADVForTagAfterCall2 = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); - assertTrue(optADVForTagAfterCall2.isPresent()); - // Versioned data does not get updated since the tag did not change - old spec is still returned - assertEquals(advForTag, optADVForTagAfterCall2.get()); - - // Modifying docker image tag creates a new version (which can contain new versioned data) - final ActorDefinitionVersion newADV = baseActorDefinitionVersion(defId).withDockerImageTag(DOCKER_IMAGE_TAG_2).withSpec(SPEC_2); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, newADV); - - final Optional optADVForTag2 = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG_2); - assertTrue(optADVForTag2.isPresent()); - final ActorDefinitionVersion advForTag2 = optADVForTag2.get(); - - // Versioned data is updated as well as the version id - assertEquals(advForTag2, newADV.withVersionId(advForTag2.getVersionId())); - assertNotEquals(advForTag2.getVersionId(), advForTag.getVersionId()); - assertNotEquals(advForTag2.getSpec(), advForTag.getSpec()); - } - @Test void testWriteSourceDefinitionSupportLevelNone() throws IOException { final UUID defId = sourceDefinition.getSourceDefinitionId(); final ActorDefinitionVersion adv = baseActorDefinitionVersion(defId).withActorDefinitionId(defId).withSupportLevel(SupportLevel.NONE); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, adv); + configRepository.writeConnectorMetadata(sourceDefinition, adv); final Optional optADVForTag = configRepository.getActorDefinitionVersion(defId, DOCKER_IMAGE_TAG); assertTrue(optADVForTag.isPresent()); @@ -212,7 +169,7 @@ void testWriteSourceDefinitionSupportLevelNonNullable() { assertThrows( RuntimeException.class, - () -> configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, adv)); + () -> configRepository.writeConnectorMetadata(sourceDefinition, adv)); } @Test @@ -255,7 +212,7 @@ void testListActorDefinitionVersionsForDefinition() throws IOException, JsonVali .withSourceDefinitionId(UUID.randomUUID()); final ActorDefinitionVersion otherActorDefVersion = baseActorDefinitionVersion(defId).withActorDefinitionId(otherSourceDef.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(otherSourceDef, otherActorDefVersion); + configRepository.writeConnectorMetadata(otherSourceDef, otherActorDefVersion); final UUID otherActorDefVersionId = configRepository.getStandardSourceDefinition(otherSourceDef.getSourceDefinitionId()).getDefaultVersionId(); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java index 1ca44d4a673..6106dda28dc 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ActorPersistenceTest.java @@ -9,7 +9,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; -import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.DestinationConnection; import io.airbyte.config.Geography; @@ -43,10 +42,10 @@ void setup() throws SQLException, IOException, JsonValidationException { configRepository = spy(new ConfigRepository(database, mock(StandardSyncPersistence.class), MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); standardSourceDefinition = MockData.publicSourceDefinition(); standardDestinationDefinition = MockData.publicDestinationDefinition(); - configRepository.writeSourceDefinitionAndDefaultVersion(standardSourceDefinition, MockData.actorDefinitionVersion() + configRepository.writeConnectorMetadata(standardSourceDefinition, MockData.actorDefinitionVersion() .withActorDefinitionId(standardSourceDefinition.getSourceDefinitionId()) .withVersionId(standardSourceDefinition.getDefaultVersionId())); - configRepository.writeDestinationDefinitionAndDefaultVersion(standardDestinationDefinition, MockData.actorDefinitionVersion() + configRepository.writeConnectorMetadata(standardDestinationDefinition, MockData.actorDefinitionVersion() .withActorDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()) .withVersionId(standardDestinationDefinition.getDefaultVersionId())); configRepository.writeStandardWorkspaceNoSecrets(new StandardWorkspace() @@ -124,146 +123,6 @@ void testSetDestinationDefaultVersion() throws IOException, JsonValidationExcept assertEquals(newActorDefinitionVersion.getVersionId(), destinationConnectionFromDb.getDefaultVersionId()); } - @Test - void testSourceDefaultVersionIsUpgradedOnNonbreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID sourceDefId = standardSourceDefinition.getSourceDefinitionId(); - final SourceConnection sourceConnection = new SourceConnection() - .withSourceId(UUID.randomUUID()) - .withSourceDefinitionId(sourceDefId) - .withWorkspaceId(WORKSPACE_ID) - .withName(SOURCE_NAME); - configRepository.writeSourceConnectionNoSecrets(sourceConnection); - - final UUID initialSourceDefinitionDefaultVersionId = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID initialSourceDefaultVersionId = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); - assertNotNull(initialSourceDefinitionDefaultVersionId); - assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); - - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(sourceDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); - - configRepository.writeSourceDefinitionAndDefaultVersion(standardSourceDefinition, newVersion); - final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID sourceDefaultVersionIdAfterUpgrade = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); - - assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(newVersionId, sourceDefaultVersionIdAfterUpgrade); - } - - @Test - void testDestinationDefaultVersionIsUpgradedOnNonbreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID destinationDefId = standardDestinationDefinition.getDestinationDefinitionId(); - final DestinationConnection destinationConnection = new DestinationConnection() - .withDestinationId(UUID.randomUUID()) - .withDestinationDefinitionId(destinationDefId) - .withWorkspaceId(WORKSPACE_ID) - .withName(DESTINATION_NAME); - configRepository.writeDestinationConnectionNoSecrets(destinationConnection); - - final UUID initialDestinationDefinitionDefaultVersionId = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID initialDestinationDefaultVersionId = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - assertNotNull(initialDestinationDefinitionDefaultVersionId); - assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); - - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(destinationDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); - - configRepository.writeDestinationDefinitionAndDefaultVersion(standardDestinationDefinition, newVersion); - final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID destinationDefaultVersionIdAfterUpgrade = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - - assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(newVersionId, destinationDefaultVersionIdAfterUpgrade); - } - - @Test - void testDestinationDefaultVersionIsNotModifiedOnBreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID destinationDefId = standardDestinationDefinition.getDestinationDefinitionId(); - final DestinationConnection destinationConnection = new DestinationConnection() - .withDestinationId(UUID.randomUUID()) - .withDestinationDefinitionId(destinationDefId) - .withWorkspaceId(WORKSPACE_ID) - .withName(DESTINATION_NAME); - configRepository.writeDestinationConnectionNoSecrets(destinationConnection); - - final UUID initialDestinationDefinitionDefaultVersionId = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID initialDestinationDefaultVersionId = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - assertNotNull(initialDestinationDefinitionDefaultVersionId); - assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); - - // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking - final List breakingChangesForDef = - List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefId)); - - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(destinationDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); - - configRepository.writeDestinationDefinitionAndDefaultVersion(standardDestinationDefinition, newVersion, breakingChangesForDef); - final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = - configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); - final UUID destinationDefaultVersionIdAfterUpgrade = - configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); - - assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(initialDestinationDefaultVersionId, destinationDefaultVersionIdAfterUpgrade); - } - - @Test - void testSourceDefaultVersionIsNotModifiedOnBreakingUpgrade() - throws IOException, JsonValidationException, ConfigNotFoundException { - final UUID sourceDefId = standardSourceDefinition.getSourceDefinitionId(); - final SourceConnection sourceConnection = new SourceConnection() - .withSourceId(UUID.randomUUID()) - .withSourceDefinitionId(sourceDefId) - .withWorkspaceId(WORKSPACE_ID) - .withName(SOURCE_NAME); - configRepository.writeSourceConnectionNoSecrets(sourceConnection); - - final UUID initialSourceDefinitionDefaultVersionId = - configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID initialSourceDefaultVersionId = - configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); - assertNotNull(initialSourceDefinitionDefaultVersionId); - assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); - - // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking - final List breakingChangesForDef = - List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefId)); - - final UUID newVersionId = UUID.randomUUID(); - final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() - .withActorDefinitionId(sourceDefId) - .withVersionId(newVersionId) - .withDockerImageTag(UPGRADE_IMAGE_TAG); - - configRepository.writeSourceDefinitionAndDefaultVersion(standardSourceDefinition, newVersion, breakingChangesForDef); - final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = - configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); - final UUID sourceDefaultVersionIdAfterUpgrade = - configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); - - assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); - assertEquals(initialSourceDefaultVersionId, sourceDefaultVersionIdAfterUpgrade); - } - @Test void testGetSourcesWithVersions() throws IOException { final SourceConnection sourceConnection1 = new SourceConnection() diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java index 022e353bc99..ef94c6dea87 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigInjectionTest.java @@ -104,7 +104,7 @@ void testCreate() throws IOException { private void createBaseObjects() throws IOException { sourceDefinition = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); createInjection(sourceDefinition, "a"); createInjection(sourceDefinition, "b"); @@ -112,7 +112,7 @@ private void createBaseObjects() throws IOException { // unreachable injection, should not show up final StandardSourceDefinition otherSourceDefinition = createBaseSourceDef(); final ActorDefinitionVersion actorDefinitionVersion2 = createBaseActorDefVersion(otherSourceDefinition.getSourceDefinitionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(otherSourceDefinition, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(otherSourceDefinition, actorDefinitionVersion2); createInjection(otherSourceDefinition, "c"); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java index a171a38cbb2..51332c5a949 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConfigRepositoryE2EReadWriteTest.java @@ -85,13 +85,13 @@ void setup() throws IOException, JsonValidationException, SQLException { final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersionId(sourceDefinition.getDefaultVersionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); } for (final StandardDestinationDefinition destinationDefinition : MockData.standardDestinationDefinitions()) { final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(destinationDefinition.getDestinationDefinitionId()) .withVersionId(destinationDefinition.getDefaultVersionId()); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion); } for (final SourceConnection source : MockData.sourceConnections()) { configRepository.writeSourceConnectionNoSecrets(source); @@ -158,7 +158,7 @@ void testReadActorCatalog() throws IOException, JsonValidationException, SQLExce final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersionId(sourceDefinition.getDefaultVersionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); final SourceConnection source = new SourceConnection() .withSourceDefinitionId(sourceDefinition.getSourceDefinitionId()) @@ -216,7 +216,7 @@ void testWriteCanonicalActorCatalog() throws IOException, JsonValidationExceptio final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersionId(sourceDefinition.getDefaultVersionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); final SourceConnection source = new SourceConnection() .withSourceDefinitionId(sourceDefinition.getSourceDefinitionId()) @@ -270,7 +270,7 @@ void testSimpleInsertActorCatalog() throws IOException, JsonValidationException, final ActorDefinitionVersion actorDefinitionVersion = MockData.actorDefinitionVersion() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersionId(sourceDefinition.getDefaultVersionId()); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); final SourceConnection source = new SourceConnection() .withSourceDefinitionId(sourceDefinition.getSourceDefinitionId()) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java index 9316a1e3b69..3ed27eafca1 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorBuilderProjectPersistenceTest.java @@ -161,7 +161,7 @@ void testUpdate() throws IOException, ConfigNotFoundException { void whenUpdateBuilderProjectAndActorDefinitionThenUpdateConnectorBuilderAndActorDefinition() throws Exception { configRepository.writeBuilderProjectDraft(A_BUILDER_PROJECT_ID, A_WORKSPACE_ID, A_PROJECT_NAME, A_MANIFEST); configRepository.writeStandardWorkspaceNoSecrets(MockData.standardWorkspaces().get(0).withWorkspaceId(A_WORKSPACE_ID)); - configRepository.writeCustomSourceDefinitionAndDefaultVersion(MockData.customSourceDefinition() + configRepository.writeCustomConnectorMetadata(MockData.customSourceDefinition() .withSourceDefinitionId(A_SOURCE_DEFINITION_ID) .withName(A_PROJECT_NAME) .withPublic(false), @@ -180,7 +180,7 @@ void whenUpdateBuilderProjectAndActorDefinitionThenUpdateConnectorBuilderAndActo void givenSourceIsPublicWhenUpdateBuilderProjectAndActorDefinitionThenActorDefinitionNameIsNotUpdated() throws Exception { configRepository.writeBuilderProjectDraft(A_BUILDER_PROJECT_ID, A_WORKSPACE_ID, A_PROJECT_NAME, A_MANIFEST); configRepository.writeStandardWorkspaceNoSecrets(MockData.standardWorkspaces().get(0).withWorkspaceId(A_WORKSPACE_ID)); - configRepository.writeCustomSourceDefinitionAndDefaultVersion(MockData.customSourceDefinition() + configRepository.writeCustomConnectorMetadata(MockData.customSourceDefinition() .withSourceDefinitionId(A_SOURCE_DEFINITION_ID) .withName(A_PROJECT_NAME) .withPublic(true), @@ -325,7 +325,7 @@ private StandardSourceDefinition linkSourceDefinition(final UUID projectId) thro .withSupportLevel(SupportLevel.COMMUNITY) .withSpec(new ConnectorSpecification().withProtocolVersion("0.1.0")); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); configRepository.insertActiveDeclarativeManifest(new DeclarativeManifest() .withActorDefinitionId(sourceDefinition.getSourceDefinitionId()) .withVersion(MANIFEST_VERSION) diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorMetadataPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorMetadataPersistenceTest.java new file mode 100644 index 00000000000..a458b582943 --- /dev/null +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/ConnectorMetadataPersistenceTest.java @@ -0,0 +1,529 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.config.persistence; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ActorDefinitionBreakingChange; +import io.airbyte.config.ActorDefinitionVersion; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.Geography; +import io.airbyte.config.SourceConnection; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.SupportLevel; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * Tests for configRepository methods that write connector metadata together. Includes writing + * global metadata (source/destination definitions and breaking changes) and versioned metadata + * (actor definition versions) as well as the logic that determines whether actors should be + * upgraded upon changing the dockerImageTag. + */ +class ConnectorMetadataPersistenceTest extends BaseConfigDatabaseTest { + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + + private static final String DOCKER_IMAGE_TAG = "0.0.1"; + + private static final String UPGRADE_IMAGE_TAG = "0.0.2"; + private static final String PROTOCOL_VERSION = "1.0.0"; + + private ConfigRepository configRepository; + + @BeforeEach + void setup() throws SQLException, JsonValidationException, IOException { + truncateAllTables(); + + configRepository = spy(new ConfigRepository(database, mock(StandardSyncPersistence.class), MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); + configRepository.writeStandardWorkspaceNoSecrets(new StandardWorkspace() + .withWorkspaceId(WORKSPACE_ID) + .withName("default") + .withSlug("workspace-slug") + .withInitialSetupComplete(false) + .withTombstone(false) + .withDefaultGeography(Geography.US)); + } + + @Test + void testWriteConnectorMetadataForSource() + throws JsonValidationException, IOException, ConfigNotFoundException { + // Initial insert + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + StandardSourceDefinition sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); + final Optional actorDefinitionVersionFromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); + + assertTrue(actorDefinitionVersionFromDB.isPresent()); + final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); + + assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); + assertEquals(firstVersionId, sourceDefinitionFromDB.getDefaultVersionId()); + assertEquals(sourceDefinition.withDefaultVersionId(firstVersionId), sourceDefinitionFromDB); + + // Updating an existing source definition/version + final StandardSourceDefinition sourceDefinition2 = sourceDefinition.withName("updated name"); + final ActorDefinitionVersion actorDefinitionVersion2 = + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(UPGRADE_IMAGE_TAG); + final List breakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefinition2.getSourceDefinitionId())); + configRepository.writeConnectorMetadata(sourceDefinition2, actorDefinitionVersion2, breakingChanges); + + sourceDefinitionFromDB = configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); + final Optional actorDefinitionVersion2FromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); + final List breakingChangesForDefFromDb = + configRepository.listBreakingChangesForActorDefinition(sourceDefinition2.getSourceDefinitionId()); + + assertTrue(actorDefinitionVersion2FromDB.isPresent()); + final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); + + assertNotEquals(firstVersionId, newADVId); + assertEquals(newADVId, sourceDefinitionFromDB.getDefaultVersionId()); + assertEquals(sourceDefinition2.withDefaultVersionId(newADVId), sourceDefinitionFromDB); + assertThat(breakingChangesForDefFromDb).containsExactlyInAnyOrderElementsOf(breakingChanges); + } + + @Test + void testWriteConnectorMetadataForDestination() + throws JsonValidationException, IOException, ConfigNotFoundException { + // Initial insert + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion1); + + StandardDestinationDefinition destinationDefinitionFromDB = + configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); + final Optional actorDefinitionVersionFromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion1.getActorDefinitionId(), actorDefinitionVersion1.getDockerImageTag()); + + assertTrue(actorDefinitionVersionFromDB.isPresent()); + final UUID firstVersionId = actorDefinitionVersionFromDB.get().getVersionId(); + + assertEquals(actorDefinitionVersion1.withVersionId(firstVersionId), actorDefinitionVersionFromDB.get()); + assertEquals(firstVersionId, destinationDefinitionFromDB.getDefaultVersionId()); + assertEquals(destinationDefinition.withDefaultVersionId(firstVersionId), destinationDefinitionFromDB); + + // Updating an existing destination definition/version + final StandardDestinationDefinition destinationDefinition2 = destinationDefinition.withName("updated name"); + final ActorDefinitionVersion actorDefinitionVersion2 = + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(UPGRADE_IMAGE_TAG); + final List breakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefinition2.getDestinationDefinitionId())); + configRepository.writeConnectorMetadata(destinationDefinition2, actorDefinitionVersion2, breakingChanges); + + destinationDefinitionFromDB = configRepository.getStandardDestinationDefinition(destinationDefinition.getDestinationDefinitionId()); + final Optional actorDefinitionVersion2FromDB = + configRepository.getActorDefinitionVersion(actorDefinitionVersion2.getActorDefinitionId(), actorDefinitionVersion2.getDockerImageTag()); + final List breakingChangesForDefFromDb = + configRepository.listBreakingChangesForActorDefinition(destinationDefinition2.getDestinationDefinitionId()); + + assertTrue(actorDefinitionVersion2FromDB.isPresent()); + final UUID newADVId = actorDefinitionVersion2FromDB.get().getVersionId(); + + assertNotEquals(firstVersionId, newADVId); + assertEquals(newADVId, destinationDefinitionFromDB.getDefaultVersionId()); + assertEquals(destinationDefinition2.withDefaultVersionId(newADVId), destinationDefinitionFromDB); + assertThat(breakingChangesForDefFromDb).containsExactlyInAnyOrderElementsOf(breakingChanges); + } + + @Test + void testVersionedFieldsDoNotChangeWithoutVersionBump() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final UUID actorDefinitionId = sourceDefinition.getSourceDefinitionId(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(actorDefinitionId); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final Optional optADVForTag = configRepository.getActorDefinitionVersion(actorDefinitionId, DOCKER_IMAGE_TAG); + assertTrue(optADVForTag.isPresent()); + final ActorDefinitionVersion advForTag = optADVForTag.get(); + final StandardSourceDefinition retrievedSourceDefinition = + configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId()); + assertEquals(retrievedSourceDefinition.getDefaultVersionId(), advForTag.getVersionId()); + + final ConnectorSpecification updatedSpec = new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value2"))).withProtocolVersion(PROTOCOL_VERSION); + + // Modify spec without changing docker image tag + final ActorDefinitionVersion modifiedADV = createBaseActorDefVersion(actorDefinitionId).withSpec(updatedSpec); + configRepository.writeConnectorMetadata(sourceDefinition, modifiedADV); + + assertEquals(retrievedSourceDefinition, configRepository.getStandardSourceDefinition(sourceDefinition.getSourceDefinitionId())); + final Optional optADVForTagAfterCall2 = configRepository.getActorDefinitionVersion(actorDefinitionId, DOCKER_IMAGE_TAG); + assertTrue(optADVForTagAfterCall2.isPresent()); + // Versioned data does not get updated since the tag did not change - old spec is still returned + assertEquals(advForTag, optADVForTagAfterCall2.get()); + + // Modifying docker image tag creates a new version (which can contain new versioned data) + final ActorDefinitionVersion newADV = createBaseActorDefVersion(actorDefinitionId).withDockerImageTag(UPGRADE_IMAGE_TAG).withSpec(updatedSpec); + configRepository.writeConnectorMetadata(sourceDefinition, newADV); + + final Optional optADVForTag2 = configRepository.getActorDefinitionVersion(actorDefinitionId, UPGRADE_IMAGE_TAG); + assertTrue(optADVForTag2.isPresent()); + final ActorDefinitionVersion advForTag2 = optADVForTag2.get(); + + // Versioned data is updated as well as the version id + assertEquals(advForTag2, newADV.withVersionId(advForTag2.getVersionId())); + assertNotEquals(advForTag2.getVersionId(), advForTag.getVersionId()); + assertNotEquals(advForTag2.getSpec(), advForTag.getSpec()); + } + + @ParameterizedTest + @ValueSource(strings = {"2.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) + void testCustomImageTagsDoNotBreakCustomConnectorUpgrade(final String dockerImageTag) throws IOException { + // Initial insert + final StandardSourceDefinition customSourceDefinition = createBaseSourceDef().withCustom(true); + final StandardDestinationDefinition customDestinationDefinition = createBaseDestDef().withCustom(true); + final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()); + final ActorDefinitionVersion destinationActorDefinitionVersion = + createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()); + configRepository.writeConnectorMetadata(customSourceDefinition, sourceActorDefinitionVersion); + configRepository.writeConnectorMetadata(customDestinationDefinition, destinationActorDefinitionVersion); + + // Update + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(customSourceDefinition, + createBaseActorDefVersion(customSourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(customDestinationDefinition, + createBaseActorDefVersion(customDestinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + } + + @ParameterizedTest + @ValueSource(strings = {"2.0.0", "dev", "test", "1.9.1-dev.33a53e6236", "97b69a76-1f06-4680-8905-8beda74311d0"}) + void testImageTagExpectationsNorNonCustomConnectorUpgradesWithoutBreakingChanges(final String dockerImageTag) throws IOException { + // Initial insert + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + configRepository.writeConnectorMetadata(sourceDefinition, sourceActorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, destinationActorDefinitionVersion); + + // Update + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(sourceDefinition, + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(destinationDefinition, + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), List.of())); + } + + @ParameterizedTest + @CsvSource({"0.0.1, true", "dev, true", "test, false", "1.9.1-dev.33a53e6236, true", "97b69a76-1f06-4680-8905-8beda74311d0, false"}) + void testImageTagExpectationsNorNonCustomConnectorUpgradesWithBreakingChanges(final String dockerImageTag, final boolean upgradeShouldSucceed) + throws IOException { + // Initial insert + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion sourceActorDefinitionVersion = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + final ActorDefinitionVersion destinationActorDefinitionVersion = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + configRepository.writeConnectorMetadata(sourceDefinition, sourceActorDefinitionVersion); + configRepository.writeConnectorMetadata(destinationDefinition, destinationActorDefinitionVersion); + + final List sourceBreakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefinition.getSourceDefinitionId())); + final List destinationBreakingChanges = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefinition.getDestinationDefinitionId())); + + // Update + if (upgradeShouldSucceed) { + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(sourceDefinition, + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); + assertDoesNotThrow(() -> configRepository.writeConnectorMetadata(destinationDefinition, + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), + destinationBreakingChanges)); + } else { + assertThrows(IllegalArgumentException.class, () -> configRepository.writeConnectorMetadata(sourceDefinition, + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withDockerImageTag(dockerImageTag), sourceBreakingChanges)); + assertThrows(IllegalArgumentException.class, () -> configRepository.writeConnectorMetadata(destinationDefinition, + createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()).withDockerImageTag(dockerImageTag), + destinationBreakingChanges)); + } + } + + @Test + void testSourceDefaultVersionIsUpgradedOnNonbreakingUpgrade() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final UUID sourceDefId = sourceDefinition.getSourceDefinitionId(); + final SourceConnection sourceConnection = createBaseSourceActor(sourceDefId); + configRepository.writeSourceConnectionNoSecrets(sourceConnection); + + final UUID initialSourceDefinitionDefaultVersionId = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID initialSourceDefaultVersionId = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + assertNotNull(initialSourceDefinitionDefaultVersionId); + assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(sourceDefinition, newVersion); + final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID sourceDefaultVersionIdAfterUpgrade = configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + + assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(newVersionId, sourceDefaultVersionIdAfterUpgrade); + } + + @Test + void testDestinationDefaultVersionIsUpgradedOnNonbreakingUpgrade() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion1); + + final UUID destinationDefId = destinationDefinition.getDestinationDefinitionId(); + final DestinationConnection destinationConnection = createBaseDestinationActor(destinationDefId); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection); + + final UUID initialDestinationDefinitionDefaultVersionId = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID initialDestinationDefaultVersionId = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + assertNotNull(initialDestinationDefinitionDefaultVersionId); + assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(destinationDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(destinationDefinition, newVersion); + final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID destinationDefaultVersionIdAfterUpgrade = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + + assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(newVersionId, destinationDefaultVersionIdAfterUpgrade); + } + + @Test + void testDestinationDefaultVersionIsNotModifiedOnBreakingUpgrade() throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardDestinationDefinition destinationDefinition = createBaseDestDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(destinationDefinition.getDestinationDefinitionId()); + + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion1); + + final UUID destinationDefId = destinationDefinition.getDestinationDefinitionId(); + final DestinationConnection destinationConnection = createBaseDestinationActor(destinationDefId); + configRepository.writeDestinationConnectionNoSecrets(destinationConnection); + + final UUID initialDestinationDefinitionDefaultVersionId = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID initialDestinationDefaultVersionId = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + assertNotNull(initialDestinationDefinitionDefaultVersionId); + assertEquals(initialDestinationDefinitionDefaultVersionId, initialDestinationDefaultVersionId); + + // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking + final List breakingChangesForDef = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(destinationDefId)); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(destinationDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(destinationDefinition, newVersion, breakingChangesForDef); + final UUID destinationDefinitionDefaultVersionIdAfterUpgrade = + configRepository.getStandardDestinationDefinition(destinationDefId).getDefaultVersionId(); + final UUID destinationDefaultVersionIdAfterUpgrade = + configRepository.getDestinationConnection(destinationConnection.getDestinationId()).getDefaultVersionId(); + + assertEquals(newVersionId, destinationDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(initialDestinationDefaultVersionId, destinationDefaultVersionIdAfterUpgrade); + } + + @Test + void testSourceDefaultVersionIsNotModifiedOnBreakingUpgrade() + throws IOException, JsonValidationException, ConfigNotFoundException { + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final UUID sourceDefId = sourceDefinition.getSourceDefinitionId(); + final SourceConnection sourceConnection = createBaseSourceActor(sourceDefId); + configRepository.writeSourceConnectionNoSecrets(sourceConnection); + + final UUID initialSourceDefinitionDefaultVersionId = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID initialSourceDefaultVersionId = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + assertNotNull(initialSourceDefinitionDefaultVersionId); + assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); + + // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking + final List breakingChangesForDef = + List.of(MockData.actorDefinitionBreakingChange(UPGRADE_IMAGE_TAG).withActorDefinitionId(sourceDefId)); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefId) + .withVersionId(newVersionId) + .withDockerImageTag(UPGRADE_IMAGE_TAG); + + configRepository.writeConnectorMetadata(sourceDefinition, newVersion, breakingChangesForDef); + final UUID sourceDefinitionDefaultVersionIdAfterUpgrade = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID sourceDefaultVersionIdAfterUpgrade = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + + assertEquals(newVersionId, sourceDefinitionDefaultVersionIdAfterUpgrade); + assertEquals(initialSourceDefaultVersionId, sourceDefaultVersionIdAfterUpgrade); + } + + @Test + void testTransactionRollbackOnFailure() throws IOException, JsonValidationException, ConfigNotFoundException { + final UUID initialADVId = UUID.randomUUID(); + final StandardSourceDefinition sourceDefinition = createBaseSourceDef(); + final ActorDefinitionVersion actorDefinitionVersion1 = + createBaseActorDefVersion(sourceDefinition.getSourceDefinitionId()).withVersionId(initialADVId); + + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion1); + + final UUID sourceDefId = sourceDefinition.getSourceDefinitionId(); + final SourceConnection sourceConnection = createBaseSourceActor(sourceDefId); + configRepository.writeSourceConnectionNoSecrets(sourceConnection); + + final UUID initialSourceDefinitionDefaultVersionId = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID initialSourceDefaultVersionId = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + assertNotNull(initialSourceDefinitionDefaultVersionId); + assertEquals(initialSourceDefinitionDefaultVersionId, initialSourceDefaultVersionId); + + // Introduce a breaking change between 0.0.1 and UPGRADE_IMAGE_TAG to make the upgrade breaking, but + // with a tag that will + // fail validation. We want to check that the state is rolled back correctly. + final String invalidUpgradeTag = "1.0"; + final List breakingChangesForDef = + List.of(MockData.actorDefinitionBreakingChange("1.0.0").withActorDefinitionId(sourceDefId)); + + final UUID newVersionId = UUID.randomUUID(); + final ActorDefinitionVersion newVersion = MockData.actorDefinitionVersion() + .withActorDefinitionId(sourceDefId) + .withVersionId(newVersionId) + .withDockerImageTag(invalidUpgradeTag) + .withDocumentationUrl("https://www.something.new"); + + final StandardSourceDefinition updatedSourceDefinition = Jsons.clone(sourceDefinition).withName("updated name"); + + assertThrows(IllegalArgumentException.class, + () -> configRepository.writeConnectorMetadata(updatedSourceDefinition, newVersion, breakingChangesForDef)); + + final UUID sourceDefinitionDefaultVersionIdAfterFailedUpgrade = + configRepository.getStandardSourceDefinition(sourceDefId).getDefaultVersionId(); + final UUID sourceDefaultVersionIdAfterFailedUpgrade = + configRepository.getSourceConnection(sourceConnection.getSourceId()).getDefaultVersionId(); + final StandardSourceDefinition sourceDefinitionAfterFailedUpgrade = + configRepository.getStandardSourceDefinition(sourceDefId); + final Optional newActorDefinitionVersionAfterFailedUpgrade = + configRepository.getActorDefinitionVersion(sourceDefId, invalidUpgradeTag); + final ActorDefinitionVersion defaultActorDefinitionVersionAfterFailedUpgrade = + configRepository.getActorDefinitionVersion(sourceDefinitionDefaultVersionIdAfterFailedUpgrade); + + // New actor definition version was not persisted + assertFalse(newActorDefinitionVersionAfterFailedUpgrade.isPresent()); + // Valid breaking change was not persisted + assertEquals(0, configRepository.listBreakingChangesForActorDefinition(sourceDefId).size()); + + // Neither the default version nor the actors get upgraded, the actors are still on the default + // version + assertEquals(initialSourceDefaultVersionId, sourceDefinitionDefaultVersionIdAfterFailedUpgrade); + assertEquals(initialSourceDefaultVersionId, sourceDefaultVersionIdAfterFailedUpgrade); + + // Source definition metadata is the same as before + assertEquals(sourceDefinition.withDefaultVersionId(initialADVId), sourceDefinitionAfterFailedUpgrade); + // Actor definition metadata is the same as before + assertEquals(actorDefinitionVersion1, defaultActorDefinitionVersionAfterFailedUpgrade); + } + + private static StandardSourceDefinition createBaseSourceDef() { + final UUID id = UUID.randomUUID(); + + return new StandardSourceDefinition() + .withName("source-def-" + id) + .withSourceDefinitionId(id) + .withTombstone(false) + .withMaxSecondsBetweenMessages(MockData.DEFAULT_MAX_SECONDS_BETWEEN_MESSAGES); + } + + private static ActorDefinitionVersion createBaseActorDefVersion(final UUID actorDefId) { + return new ActorDefinitionVersion() + .withVersionId(UUID.randomUUID()) + .withActorDefinitionId(actorDefId) + .withDockerRepository("source-image-" + actorDefId) + .withDockerImageTag(DOCKER_IMAGE_TAG) + .withProtocolVersion(PROTOCOL_VERSION) + .withSupportLevel(SupportLevel.CERTIFIED) + .withSpec(new ConnectorSpecification() + .withConnectionSpecification(Jsons.jsonNode(Map.of("key", "value1"))).withProtocolVersion(PROTOCOL_VERSION)); + } + + private static StandardDestinationDefinition createBaseDestDef() { + final UUID id = UUID.randomUUID(); + + return new StandardDestinationDefinition() + .withName("source-def-" + id) + .withDestinationDefinitionId(id) + .withTombstone(false); + } + + private static SourceConnection createBaseSourceActor(final UUID actorDefinitionId) { + final UUID id = UUID.randomUUID(); + + return new SourceConnection() + .withSourceId(id) + .withSourceDefinitionId(actorDefinitionId) + .withWorkspaceId(WORKSPACE_ID) + .withName("source-" + id); + } + + private static DestinationConnection createBaseDestinationActor(final UUID actorDefinitionId) { + final UUID id = UUID.randomUUID(); + + return new DestinationConnection() + .withDestinationId(id) + .withDestinationDefinitionId(actorDefinitionId) + .withWorkspaceId(WORKSPACE_ID) + .withName("destination-" + id); + } + +} diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java index bd692ce260a..d0afcc88e06 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/DeclarativeManifestPersistenceTest.java @@ -248,7 +248,7 @@ void givenActiveDeclarativeManifestWithActorDefinitionId(final UUID actorDefinit void givenSourceDefinition(final UUID sourceDefinitionId) throws JsonValidationException, IOException { final UUID workspaceId = UUID.randomUUID(); configRepository.writeStandardWorkspaceNoSecrets(MockData.standardWorkspaces().get(0).withWorkspaceId(workspaceId)); - configRepository.writeCustomSourceDefinitionAndDefaultVersion( + configRepository.writeCustomConnectorMetadata( MockData.customSourceDefinition().withSourceDefinitionId(sourceDefinitionId), MockData.actorDefinitionVersion().withActorDefinitionId(sourceDefinitionId), workspaceId, diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java index b9b08f14837..aec78fe6c54 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StandardSyncPersistenceTest.java @@ -358,7 +358,7 @@ private StandardSourceDefinition createStandardSourceDefinition(final String pro .withProtocolVersion(protocolVersion) .withReleaseStage(releaseStage) .withSupportLevel(SupportLevel.COMMUNITY); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDef, sourceDefVersion); + configRepository.writeConnectorMetadata(sourceDef, sourceDefVersion); return sourceDef; } @@ -383,7 +383,7 @@ private StandardDestinationDefinition createStandardDestDefinition(final String .withReleaseStage(releaseStage) .withSupportLevel(SupportLevel.COMMUNITY); - configRepository.writeDestinationDefinitionAndDefaultVersion(destDef, destDefVersion); + configRepository.writeConnectorMetadata(destDef, destDefVersion); return destDef; } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java index d510de40044..e1966578ad9 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java @@ -74,9 +74,9 @@ private void setupTestData() throws JsonValidationException, IOException { final StandardSync sync = Jsons.clone(MockData.standardSyncs().get(0)).withOperationIds(Collections.emptyList()); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion(sourceDefinition, actorDefinitionVersion); + configRepository.writeConnectorMetadata(sourceDefinition, actorDefinitionVersion); configRepository.writeSourceConnectionNoSecrets(sourceConnection); - configRepository.writeDestinationDefinitionAndDefaultVersion(destinationDefinition, actorDefinitionVersion2); + configRepository.writeConnectorMetadata(destinationDefinition, actorDefinitionVersion2); configRepository.writeDestinationConnectionNoSecrets(destinationConnection); configRepository.writeStandardSync(sync); diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java index 51040bb58f8..b07e41937f5 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java @@ -205,10 +205,10 @@ void testWorkspaceHasAlphaOrBetaConnector(final ReleaseStage sourceReleaseStage, final StandardWorkspace workspace = createBaseStandardWorkspace(); configRepository.writeStandardWorkspaceNoSecrets(workspace); - configRepository.writeSourceDefinitionAndDefaultVersion( + configRepository.writeConnectorMetadata( createSourceDefinition(), createActorDefinitionVersion(SOURCE_DEFINITION_ID, sourceReleaseStage)); - configRepository.writeDestinationDefinitionAndDefaultVersion( + configRepository.writeConnectorMetadata( createDestinationDefinition(), createActorDefinitionVersion(DESTINATION_DEFINITION_ID, destinationReleaseStage)); diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java index 7d677d36973..f02d134d827 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/ApplyDefinitionsHelper.java @@ -28,7 +28,6 @@ import jakarta.inject.Named; import jakarta.inject.Singleton; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -55,7 +54,6 @@ public class ApplyDefinitionsHelper { private final SupportStateUpdater supportStateUpdater; private int newConnectorCount; private int changedConnectorCount; - private List allBreakingChanges; private static final Logger LOGGER = LoggerFactory.getLogger(ApplyDefinitionsHelper.class); public ApplyDefinitionsHelper(@Named("seedDefinitionsProvider") final DefinitionsProvider definitionsProvider, @@ -96,7 +94,6 @@ public void apply(final boolean updateAll) throws JsonValidationException, IOExc newConnectorCount = 0; changedConnectorCount = 0; - allBreakingChanges = new ArrayList<>(); for (final ConnectorRegistrySourceDefinition def : protocolCompatibleSourceDefinitions) { applySourceDefinition(actorDefinitionIdsToDefaultVersionsMap, def, actorDefinitionIdsInUse, updateAll); @@ -104,7 +101,6 @@ public void apply(final boolean updateAll) throws JsonValidationException, IOExc for (final ConnectorRegistryDestinationDefinition def : protocolCompatibleDestinationDefinitions) { applyDestinationDefinition(actorDefinitionIdsToDefaultVersionsMap, def, actorDefinitionIdsInUse, updateAll); } - configRepository.writeActorDefinitionBreakingChanges(allBreakingChanges); if (featureFlagClient.boolVariation(RunSupportStateUpdater.INSTANCE, new Workspace(ANONYMOUS))) { supportStateUpdater.updateSupportStates(); } @@ -132,13 +128,11 @@ private void applySourceDefinition(final Map actor return; } - allBreakingChanges.addAll(breakingChangesForDef); - final boolean connectorIsNew = !actorDefinitionIdsAndDefaultVersions.containsKey(newSourceDef.getSourceDefinitionId()); if (connectorIsNew) { LOGGER.info("Adding new connector {}:{}", newDef.getDockerRepository(), newDef.getDockerImageTag()); newConnectorCount++; - configRepository.writeSourceDefinitionAndDefaultVersion(newSourceDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newSourceDef, newADV, breakingChangesForDef); return; } @@ -151,7 +145,7 @@ private void applySourceDefinition(final Map actor currentDefaultADV.getDockerImageTag(), newADV.getDockerImageTag()); changedConnectorCount++; - configRepository.writeSourceDefinitionAndDefaultVersion(newSourceDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newSourceDef, newADV, breakingChangesForDef); } else { configRepository.updateStandardSourceDefinition(newSourceDef); } @@ -176,13 +170,11 @@ private void applyDestinationDefinition(final Map return; } - allBreakingChanges.addAll(breakingChangesForDef); - final boolean connectorIsNew = !actorDefinitionIdsAndDefaultVersions.containsKey(newDestinationDef.getDestinationDefinitionId()); if (connectorIsNew) { LOGGER.info("Adding new connector {}:{}", newDef.getDockerRepository(), newDef.getDockerImageTag()); newConnectorCount++; - configRepository.writeDestinationDefinitionAndDefaultVersion(newDestinationDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newDestinationDef, newADV, breakingChangesForDef); return; } @@ -195,7 +187,7 @@ private void applyDestinationDefinition(final Map currentDefaultADV.getDockerImageTag(), newADV.getDockerImageTag()); changedConnectorCount++; - configRepository.writeDestinationDefinitionAndDefaultVersion(newDestinationDef, newADV, breakingChangesForDef); + configRepository.writeConnectorMetadata(newDestinationDef, newADV, breakingChangesForDef); } else { configRepository.updateStandardDestinationDefinition(newDestinationDef); } diff --git a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java index 8f858baa85e..8b9b275b90e 100644 --- a/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java +++ b/airbyte-config/init/src/test/java/io/airbyte/config/init/ApplyDefinitionsHelperTest.java @@ -13,7 +13,6 @@ import io.airbyte.commons.version.AirbyteProtocolVersionRange; import io.airbyte.commons.version.Version; -import io.airbyte.config.ActorDefinitionBreakingChange; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.BreakingChanges; import io.airbyte.config.ConnectorRegistryDestinationDefinition; @@ -33,7 +32,6 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -137,15 +135,14 @@ void testNewConnectorIsWritten(final boolean updateAll) applyDefinitionsHelper.apply(updateAll); verifyConfigRepositoryGetInteractions(); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3)); - verify(configRepository).writeActorDefinitionBreakingChanges(List.of()); verify(supportStateUpdater).updateSupportStates(); verifyNoMoreInteractions(configRepository, supportStateUpdater); @@ -165,15 +162,14 @@ void testConnectorIsUpdatedIfItIsNotInUse(final boolean updateAll) applyDefinitionsHelper.apply(updateAll); verifyConfigRepositoryGetInteractions(); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); verify(supportStateUpdater).updateSupportStates(); verifyNoMoreInteractions(configRepository, supportStateUpdater); @@ -193,19 +189,17 @@ void testUpdateBehaviorIfConnectorIsInUse(final boolean updateAll) verifyConfigRepositoryGetInteractions(); if (updateAll) { - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); } else { verify(configRepository).updateStandardSourceDefinition(ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2)); verify(configRepository).updateStandardDestinationDefinition(ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); } verify(supportStateUpdater).updateSupportStates(); @@ -229,24 +223,23 @@ void testDefinitionsFiltering(final boolean updateAll) applyDefinitionsHelper.apply(updateAll); verifyConfigRepositoryGetInteractions(); - verify(configRepository, never()).writeSourceDefinitionAndDefaultVersion( + verify(configRepository, never()).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(postgresWithOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionVersion(s3withOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(postgresWithOldProtocolVersion)); - verify(configRepository, never()).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository, never()).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(s3withOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionVersion(postgresWithOldProtocolVersion), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(s3withOldProtocolVersion)); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); verify(supportStateUpdater).updateSupportStates(); verifyNoMoreInteractions(configRepository, supportStateUpdater); @@ -262,25 +255,17 @@ void testTurnOffRunSupportStateUpdaterFeatureFlag() throws JsonValidationExcepti applyDefinitionsHelper.apply(true); verifyConfigRepositoryGetInteractions(); - verify(configRepository).writeSourceDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardSourceDefinition(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionVersion(SOURCE_POSTGRES_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - verify(configRepository).writeDestinationDefinitionAndDefaultVersion( + verify(configRepository).writeConnectorMetadata( ConnectorRegistryConverters.toStandardDestinationDefinition(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionVersion(DESTINATION_S3_2), ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - verify(configRepository).writeActorDefinitionBreakingChanges(getExpectedBreakingChanges()); verify(supportStateUpdater, never()).updateSupportStates(); verifyNoMoreInteractions(configRepository, supportStateUpdater); } - private static List getExpectedBreakingChanges() { - final List breakingChanges = new ArrayList<>(); - breakingChanges.addAll(ConnectorRegistryConverters.toActorDefinitionBreakingChanges(SOURCE_POSTGRES_2)); - breakingChanges.addAll(ConnectorRegistryConverters.toActorDefinitionBreakingChanges(DESTINATION_S3_2)); - return breakingChanges; - } - } From f0bcf0ac095e06027d295dab0398dff93811d730 Mon Sep 17 00:00:00 2001 From: Conor Date: Mon, 11 Sep 2023 16:11:47 +0200 Subject: [PATCH 140/201] local and remote ci performance and usability improvements (#8722) Co-authored-by: Tim Roes --- airbyte-webapp/build.gradle | 32 +---------- airbyte-webapp/package.json | 1 + airbyte-webapp/scripts/validate-lock-files.js | 25 +++++++++ build.gradle | 35 ++++++++++-- ci/oss/ci.py | 8 ++- ci/oss/tasks.py | 55 +++++++++++-------- 6 files changed, 94 insertions(+), 62 deletions(-) create mode 100644 airbyte-webapp/scripts/validate-lock-files.js diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 1e010600a99..9c4423a5c85 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -28,23 +28,7 @@ node { distBaseUrl = "https://nodejs.org/dist" } -tasks.register("validateLockFiles") { - description "Validate only a pnpm-lock.yaml lock file exists" - inputs.files "pnpm-lock.yaml", "package-lock.json", "yarn.lock" - - // The validateLockFiles has no outputs, thus we always treat the outputs up to date - // as long as the inputs have not changed - outputs.upToDateWhen { true } - - doLast { - assert file("pnpm-lock.yaml").exists() - assert !file("package-lock.json").exists() - assert !file("yarn.lock").exists() - } -} - tasks.named("pnpmInstall") { - dependsOn tasks.named("validateLockFiles") // Add patches folder to inputs of pnpmInstall task, since it has pnpm-lock.yml as an output // thus wouldn't rerun in case a patch get changed inputs.dir "patches" @@ -125,19 +109,6 @@ tasks.register("cloudE2eTest", PnpmTask) { outputs.upToDateWhen { false } } -tasks.register("licenseCheck", PnpmTask) { - dependsOn tasks.named("pnpmInstall") - - args = ['run', 'license-check'] - - inputs.files nodeModules - inputs.file 'package.json' - inputs.file 'scripts/license-check.js' - - // The licenseCheck has no outputs, thus we always treat the outputs up to date - // as long as the inputs have not changed - outputs.upToDateWhen { true } -} // //tasks.register("validateLinks", PnpmTask) { // dependsOn tasks.named("pnpmInstall") @@ -189,10 +160,9 @@ tasks.register("copyNginx", Copy) { // Those tasks should be run as part of the "check" task tasks.named("check") { - dependsOn /*tasks.named("validateLinks"),*/ tasks.named("licenseCheck"), tasks.named("test") + dependsOn /*tasks.named("validateLinks"),*/ tasks.named("test") } - tasks.named("build") { dependsOn tasks.named("buildStorybook") } diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index f78d56257b2..2ea80be94e2 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -33,6 +33,7 @@ "license-check": "node ./scripts/license-check.js", "generate-client": "./scripts/load-declarative-schema.sh && orval", "validate-links": "ts-node --skip-project ./scripts/validate-links.ts", + "validate-lock": "node ./scripts/validate-lock-files.js", "preanalyze-lowcode": "TS_NODE_TRANSPILE_ONLY=true pnpm run generate-client", "analyze-lowcode": "ts-node --skip-project ./scripts/analyze-low-code-manifests.ts", "cypress:open": "cypress open --config-file cypress/cypress.config.ts", diff --git a/airbyte-webapp/scripts/validate-lock-files.js b/airbyte-webapp/scripts/validate-lock-files.js new file mode 100644 index 00000000000..756f9cb7358 --- /dev/null +++ b/airbyte-webapp/scripts/validate-lock-files.js @@ -0,0 +1,25 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +function assertFileExists(filepath) { + if (!fs.existsSync(filepath)) { + throw new Error(`File ${filepath} does not exist`); + } +} + +function assertFileNotExists(filepath) { + if (fs.existsSync(filepath)) { + throw new Error(`File ${filepath} exists but should not`); + } +} + +try { + assertFileExists(path.join(__dirname, "..", "pnpm-lock.yaml")); + assertFileNotExists(path.join(__dirname, "..", "package-lock.json")); + assertFileNotExists(path.join(__dirname, "..", "yarn.lock")); + + console.log("Lock file validation successful."); +} catch (error) { + console.error(`Lock file validation failed: ${error.message}`); + process.exit(1); +} diff --git a/build.gradle b/build.gradle index c12a91329c0..7efc5535dd5 100644 --- a/build.gradle +++ b/build.gradle @@ -24,11 +24,12 @@ buildscript { plugins { id "base" - id "io.airbyte.gradle.jvm" version "0.14.0" apply false - id "io.airbyte.gradle.jvm.app" version "0.14.0" apply false - id "io.airbyte.gradle.jvm.lib" version "0.14.0" apply false - id "io.airbyte.gradle.docker" version "0.14.0" apply false - id "io.airbyte.gradle.publish" version "0.14.0" apply false + id "com.dorongold.task-tree" version "2.1.1" + id "io.airbyte.gradle.jvm" version "0.15.0" apply false + id "io.airbyte.gradle.jvm.app" version "0.15.0" apply false + id "io.airbyte.gradle.jvm.lib" version "0.15.0" apply false + id "io.airbyte.gradle.docker" version "0.15.0" apply false + id "io.airbyte.gradle.publish" version "0.15.0" apply false } repositories { @@ -71,6 +72,30 @@ allprojects { version = rootProject.ext.version } +tasks.register('archiveReports', Tar) { + dependsOn subprojects.collect { it.getTasksByName('checkstyleMain', true) } + dependsOn subprojects.collect { it.getTasksByName('checkstyleTest', true) } + dependsOn subprojects.collect { it.getTasksByName('jacocoTestReport', true) } + dependsOn subprojects.collect { it.getTasksByName('pmdMain', true) } + dependsOn subprojects.collect { it.getTasksByName('pmdTest', true) } + dependsOn subprojects.collect { it.getTasksByName('spotbugsMain', true) } + dependsOn subprojects.collect { it.getTasksByName('spotbugsTest', true) } + dependsOn subprojects.collect { it.getTasksByName('test', true) } + dependsOn subprojects.collect { it.getTasksByName('checkstyleAcceptanceTests', true) } + dependsOn subprojects.collect { it.getTasksByName('pmdAcceptanceTests', true) } + dependsOn subprojects.collect { it.getTasksByName('spotbugsAcceptanceTests', true) } + + archiveFileName = "${project.name}-reports.tar" + destinationDirectory = layout.buildDirectory.dir('dist') + + // Collect reports from each subproject + subprojects.each { subproject -> + from("${subproject.buildDir}/reports") { + into("${subproject.name}/reports") + } + } +} + subprojects { sp -> // airbyte-webapp has not been converted to the gradle plugins if (sp.name != "airbyte-webapp") { diff --git a/ci/oss/ci.py b/ci/oss/ci.py index b5fa91ed291..cae383b0dc1 100644 --- a/ci/oss/ci.py +++ b/ci/oss/ci.py @@ -20,6 +20,7 @@ build_oss_backend_task, build_oss_frontend_task, build_storybook_oss_frontend_task, + check_oss_backend_task, test_oss_backend_task, test_oss_frontend_task, ) @@ -115,10 +116,11 @@ async def frontend_build(settings: OssSettings, ctx: PipelineContext, client: Op @pass_global_settings @pass_pipeline_context @flow(validate_parameters=False, name="OSS Test") -async def test(settings: OssSettings, ctx: PipelineContext, client: Optional[Client] = None, scan: bool = False) -> List[Container]: +async def test(settings: OssSettings, ctx: PipelineContext, build_results: Optional[List[Container]] = None, client: Optional[Client] = None, scan: bool = False) -> List[Container]: test_client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name) - build_results = await build(scan=scan, client=test_client) + build_results = await build(scan=scan, client=test_client) if build_results is None else build_results test_results = await test_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0][0], settings=settings, ctx=quote(ctx), scan=scan) + await check_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0][0], settings=settings, ctx=quote(ctx), scan=scan) # TODO: add cypress E2E tests here return [test_results] @@ -130,6 +132,8 @@ async def backend_test(settings: OssSettings, ctx: PipelineContext, client: Opti test_client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name) build_results = await backend_build(scan=scan, client=test_client) test_results = await test_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0], settings=settings, ctx=quote(ctx), scan=scan) + await check_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0], settings=settings, ctx=quote(ctx), scan=scan) + return test_results @oss_group.command(CICommand()) diff --git a/ci/oss/tasks.py b/ci/oss/tasks.py index e2ec8f2fbfc..ee5e2f393eb 100644 --- a/ci/oss/tasks.py +++ b/ci/oss/tasks.py @@ -8,11 +8,10 @@ with_gradle, with_node, with_pnpm, - with_typescript_gha, ) -from aircmd.actions.pipelines import get_repo_dir +from aircmd.actions.pipelines import get_repo_dir, sync_to_gradle_cache_from_homedir from aircmd.models.base import PipelineContext -from aircmd.models.settings import GithubActionsInputSettings, load_settings +from aircmd.models.settings import load_settings from dagger import CacheSharingMode, CacheVolume, Client, Container from prefect import task from prefect.artifacts import create_link_artifact @@ -49,7 +48,7 @@ async def build_oss_backend_task(settings: OssSettings, ctx: PipelineContext, cl .with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte") .with_exec(["./gradlew", ":airbyte-config:specs:downloadConnectorRegistry", "--rerun", "--build-cache", "--no-daemon"]) .with_exec(gradle_command + ["--scan"] if scan else gradle_command) - .with_exec(["rsync", "-az", "/root/.gradle/", "/root/gradle-cache"]) #TODO: Move this to a context manager + .with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) #TODO: Move this to a context manager ) await result.sync() if scan: @@ -75,6 +74,8 @@ async def build_oss_frontend_task(settings: OssSettings, ctx: PipelineContext, c .with_(load_settings(client, settings)) .with_mounted_cache("./build/airbyte-repository", airbyte_repo_cache, sharing=CacheSharingMode.LOCKED) .with_exec(["pnpm", "install"]) + .with_exec(["pnpm", "run", "license-check"]) + .with_exec(["pnpm", "run", "validate-lock"]) .with_exec(["pnpm", "build"])) await result.sync() return result @@ -120,6 +121,28 @@ async def build_storybook_oss_frontend_task(settings: OssSettings, ctx: Pipeline await result.sync() return result +@task +async def check_oss_backend_task(client: Client, oss_build_result: Container, settings: OssSettings, ctx: PipelineContext, scan: bool) -> Container: + gradle_command = ["./gradlew", "check", "-x", "test", "-x", ":airbyte-webapp:check","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"] + files_from_result = [ + "**/build.gradle", + "**/gradle.properties", + "**/settings.gradle", + "**/airbyte-*/**/*" + ] + result = ( + with_gradle(client, ctx, settings, directory=base_dir) + .with_directory("/root/.m2/repository", oss_build_result.directory("/root/.m2/repository")) # published jar files from mavenLocal + .with_directory("/airbyte/oss", oss_build_result.directory("/airbyte/oss"), include=files_from_result) + .with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte") + .with_(load_settings(client, settings)) + .with_env_variable("VERSION", "dev") + .with_env_variable("METRIC_CLIENT", "") # override 'datadog' value for metrics-lib test + .with_exec(gradle_command + ["--scan"] if scan else gradle_command) + .with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) + ) + await result.sync() + @task async def test_oss_backend_task(client: Client, oss_build_result: Container, settings: OssSettings, ctx: PipelineContext, scan: bool) -> Container: @@ -175,9 +198,9 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set .with_exec(["./run.sh"]) ) - gradle_command = ["./gradlew", "checK", "test", "-x", ":airbyte-webapp:test", "-x", "buildDockerImage", "--build-cache", "--no-daemon"] + gradle_command = ["./gradlew", "test", "-x", ":airbyte-webapp:test","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"] - result = ( + result = ( with_gradle(client, ctx, settings, directory=base_dir) .with_service_binding("airbyte-proxy-test-container", airbyte_proxy_service_auth) .with_service_binding("airbyte-proxy-test-container-newauth", airbyte_proxy_service_newauth) @@ -189,11 +212,8 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set .with_env_variable("VERSION", "dev") .with_env_variable("METRIC_CLIENT", "") # override 'datadog' value for metrics-lib test .with_exec(gradle_command + ["--scan"] if scan else gradle_command) - .with_exec(["rsync", "-az", "/root/.gradle/", "/root/gradle-cache"]) #TODO: Move this to a context manager) - .with_exec(["rsync", "-azm", "--include='*/'", "--include='jacocoTestReport.xml", "--exclude='*'", "/airbyte/oss", "/jacoco"]) # Collect jacoco reports - - - ) + .with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) + ) await result.sync() if scan: scan_file_contents: str = await result.file("/airbyte/oss/scan-journal.log").contents() @@ -203,17 +223,4 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set description="Gradle build scan for OSS Backend Tests", ) - - # TODO: We should move as much of these actions that interface with PR's to prefect Artifacts as we can - # instead of relying on CI only marketplace actions that assume that they are running in a CI environment - # This will allow us to run these actions locally as well - if settings.CI: - jacoco_inputs = GithubActionsInputSettings(settings, **{ - "paths": "/input/**/jacocoTestReport.xml", - "token": settings.GITHUB_TOKEN, - "debug-mode": "true" - }) - # TODO: Upload buildpulse here as well - await with_typescript_gha(client, result.directory("/jacoco"), "Madrapps/jacoco-report", "v1.6.1", jacoco_inputs) - return result From 1cab89e73ad1d51f226c09b9bc50327236f2454d Mon Sep 17 00:00:00 2001 From: Ella Rohm-Ensing Date: Mon, 11 Sep 2023 09:35:04 -0500 Subject: [PATCH 141/201] Add Breaking change notification settings (#8720) Co-authored-by: josephkmh --- airbyte-api/src/main/openapi/config.yaml | 6 ++++ .../NotificationSettingsConverter.java | 12 +++++++ .../server/handlers/WorkspacesHandler.java | 18 ++++++---- .../handlers/WorkspacesHandlerTest.java | 8 +++++ .../resources/types/NotificationSettings.yaml | 4 +++ .../airbyte/server/apis/WorkspaceApiTest.java | 8 ++--- .../NotificationItemField.module.scss | 3 ++ .../NotificationItemField.tsx | 35 +++++++++++++++---- .../NotificationSettingsForm.tsx | 24 +++++++++++-- .../formValuesToNotificationSettings.test.ts | 4 +++ .../notificationSettingsToFormValues.test.ts | 4 +++ .../hooks/services/Experiment/experiments.ts | 1 + .../src/hooks/services/useWorkspace.tsx | 6 ++++ airbyte-webapp/src/locales/en.json | 4 +++ .../src/test-utils/mock-data/mockWorkspace.ts | 6 ++++ 15 files changed, 124 insertions(+), 19 deletions(-) create mode 100644 airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.module.scss diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 6cc83dbab90..a674879f975 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -3746,6 +3746,10 @@ components: $ref: "#/components/schemas/NotificationItem" sendOnConnectionUpdateActionRequired: $ref: "#/components/schemas/NotificationItem" + sendOnBreakingChangeWarning: + $ref: "#/components/schemas/NotificationItem" + sendOnBreakingChangeSyncsDisabled: + $ref: "#/components/schemas/NotificationItem" Notification: type: object @@ -3792,6 +3796,8 @@ components: - sync_disabled_warning - connection_update - connection_update_action_required + - breaking_change_warning + - breaking_change_syncs_disabled NotificationRead: type: object required: diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java index d424646ab00..15a3bb79675 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/converters/NotificationSettingsConverter.java @@ -37,6 +37,12 @@ public static io.airbyte.config.NotificationSettings toConfig(final io.airbyte.a if (notification.getSendOnConnectionUpdateActionRequired() != null) { configNotificationSettings.setSendOnConnectionUpdateActionRequired(toConfig(notification.getSendOnConnectionUpdateActionRequired())); } + if (notification.getSendOnBreakingChangeWarning() != null) { + configNotificationSettings.setSendOnBreakingChangeWarning(toConfig(notification.getSendOnBreakingChangeWarning())); + } + if (notification.getSendOnBreakingChangeSyncsDisabled() != null) { + configNotificationSettings.setSendOnBreakingChangeSyncsDisabled(toConfig(notification.getSendOnBreakingChangeSyncsDisabled())); + } return configNotificationSettings; } @@ -90,6 +96,12 @@ public static io.airbyte.api.model.generated.NotificationSettings toApi(final io if (notificationSettings.getSendOnConnectionUpdateActionRequired() != null) { apiNotificationSetings.setSendOnConnectionUpdateActionRequired(toApi(notificationSettings.getSendOnConnectionUpdateActionRequired())); } + if (notificationSettings.getSendOnBreakingChangeWarning() != null) { + apiNotificationSetings.setSendOnBreakingChangeWarning(toApi(notificationSettings.getSendOnBreakingChangeWarning())); + } + if (notificationSettings.getSendOnBreakingChangeSyncsDisabled() != null) { + apiNotificationSetings.setSendOnBreakingChangeSyncsDisabled(toApi(notificationSettings.getSendOnBreakingChangeSyncsDisabled())); + } return apiNotificationSetings; } diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java index fdbc568fce0..6672b86047d 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java @@ -178,7 +178,9 @@ private NotificationSettings patchNotificationSettingsWithDefaultValue(final Wor .sendOnConnectionUpdate(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) .sendOnConnectionUpdateActionRequired(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) .sendOnSyncDisabled(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) - .sendOnSyncDisabledWarning(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)); + .sendOnSyncDisabledWarning(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeWarning(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeSyncsDisabled(new NotificationItem().addNotificationTypeItem(NotificationType.CUSTOMERIO)); if (workspaceCreate.getNotificationSettings() != null) { final NotificationSettings inputNotificationSettings = workspaceCreate.getNotificationSettings(); if (inputNotificationSettings.getSendOnSuccess() != null) { @@ -199,6 +201,12 @@ private NotificationSettings patchNotificationSettingsWithDefaultValue(final Wor if (inputNotificationSettings.getSendOnSyncDisabledWarning() != null) { notificationSettings.setSendOnSyncDisabledWarning(inputNotificationSettings.getSendOnSyncDisabledWarning()); } + if (inputNotificationSettings.getSendOnBreakingChangeWarning() != null) { + notificationSettings.setSendOnBreakingChangeWarning(inputNotificationSettings.getSendOnBreakingChangeWarning()); + } + if (inputNotificationSettings.getSendOnBreakingChangeSyncsDisabled() != null) { + notificationSettings.setSendOnBreakingChangeSyncsDisabled(inputNotificationSettings.getSendOnBreakingChangeSyncsDisabled()); + } } return notificationSettings; } @@ -248,8 +256,7 @@ public WorkspaceRead getWorkspace(final WorkspaceIdRequestBody workspaceIdReques } @SuppressWarnings("unused") - public WorkspaceRead getWorkspaceBySlug(final SlugRequestBody slugRequestBody) - throws JsonValidationException, IOException, ConfigNotFoundException { + public WorkspaceRead getWorkspaceBySlug(final SlugRequestBody slugRequestBody) throws IOException, ConfigNotFoundException { // for now we assume there is one workspace and it has a default uuid. final StandardWorkspace workspace = configRepository.getWorkspaceBySlug(slugRequestBody.getSlug(), false); return buildWorkspaceRead(workspace); @@ -260,8 +267,7 @@ public WorkspaceRead getWorkspaceByConnectionId(final ConnectionIdRequestBody co return buildWorkspaceRead(workspace); } - public WorkspaceReadList listWorkspacesInOrganization(final ListWorkspacesInOrganizationRequestBody request) - throws ConfigNotFoundException, IOException { + public WorkspaceReadList listWorkspacesInOrganization(final ListWorkspacesInOrganizationRequestBody request) throws IOException { Optional keyword = StringUtils.isBlank(request.getKeyword()) ? Optional.empty() : Optional.of(request.getKeyword()); final List standardWorkspaces = workspacePersistence .listWorkspacesByOrganizationId( @@ -333,7 +339,7 @@ private WorkspaceRead buildWorkspaceReadFromId(final UUID workspaceId) throws Co return buildWorkspaceRead(workspace); } - private String generateUniqueSlug(final String workspaceName) throws JsonValidationException, IOException { + private String generateUniqueSlug(final String workspaceName) throws IOException { final String proposedSlug = slugify.slugify(workspaceName); // todo (cgardens) - this is going to be too expensive once there are too many workspaces. needs to diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java index 43f02097cad..88550417eb6 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java @@ -184,6 +184,10 @@ private io.airbyte.api.model.generated.NotificationSettings generateApiNotificat .sendOnSyncDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) .sendOnSyncDisabledWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeSyncsDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)); } @@ -199,6 +203,10 @@ private io.airbyte.api.model.generated.NotificationSettings generateDefaultApiNo .sendOnSyncDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) .sendOnSyncDisabledWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeWarning(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( + io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)) + .sendOnBreakingChangeSyncsDisabled(new io.airbyte.api.model.generated.NotificationItem().addNotificationTypeItem( io.airbyte.api.model.generated.NotificationType.CUSTOMERIO)); } diff --git a/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml b/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml index 3def0a0783d..a4647f78b0e 100644 --- a/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml +++ b/airbyte-config/config-models/src/main/resources/types/NotificationSettings.yaml @@ -20,3 +20,7 @@ properties: $ref: NotificationItem.yaml sendOnConnectionUpdateActionRequired: $ref: NotificationItem.yaml + sendOnBreakingChangeWarning: + $ref: NotificationItem.yaml + sendOnBreakingChangeSyncsDisabled: + $ref: NotificationItem.yaml diff --git a/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java b/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java index b8b39ceaabf..be36110d313 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/apis/WorkspaceApiTest.java @@ -26,7 +26,7 @@ class WorkspaceApiTest extends BaseControllerTest { @Test - void testCreateWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + void testCreateWorkspace() throws JsonValidationException, IOException { Mockito.when(workspacesHandler.createWorkspace(Mockito.any())) .thenReturn(new WorkspaceRead()); final String path = "/api/v1/workspaces/create"; @@ -64,7 +64,7 @@ void testGetWorkspace() throws JsonValidationException, ConfigNotFoundException, } @Test - void testGetBySlugWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetBySlugWorkspace() throws ConfigNotFoundException, IOException { Mockito.when(workspacesHandler.getWorkspaceBySlug(Mockito.any())) .thenReturn(new WorkspaceRead()) .thenThrow(new ConfigNotFoundException("", "")); @@ -78,7 +78,7 @@ void testGetBySlugWorkspace() throws JsonValidationException, ConfigNotFoundExce } @Test - void testListWorkspace() throws JsonValidationException, ConfigNotFoundException, IOException { + void testListWorkspace() throws JsonValidationException, IOException { Mockito.when(workspacesHandler.listWorkspaces()) .thenReturn(new WorkspaceReadList()); final String path = "/api/v1/workspaces/list"; @@ -130,7 +130,7 @@ void testUpdateWorkspaceName() throws JsonValidationException, ConfigNotFoundExc } @Test - void testGetWorkspaceByConnectionId() throws JsonValidationException, ConfigNotFoundException, IOException { + void testGetWorkspaceByConnectionId() throws ConfigNotFoundException { Mockito.when(workspacesHandler.getWorkspaceByConnectionId(Mockito.any())) .thenReturn(new WorkspaceRead()) .thenThrow(new ConfigNotFoundException("", "")); diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.module.scss b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.module.scss new file mode 100644 index 00000000000..38093bc6a93 --- /dev/null +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.module.scss @@ -0,0 +1,3 @@ +.notificationItemField { + grid-column: 1 / 1; +} diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx index 2a0796a3868..dabf90db27c 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationItemField.tsx @@ -8,17 +8,24 @@ import { Tooltip } from "components/ui/Tooltip"; import { FeatureItem, useFeature } from "core/services/features"; +import styles from "./NotificationItemField.module.scss"; import { NotificationSettingsFormValues } from "./NotificationSettingsForm"; import { SlackNotificationUrlInput } from "./SlackNotificationUrlInput"; import { TestWebhookButton } from "./TestWebhookButton"; interface NotificationItemFieldProps { emailNotificationRequired?: boolean; + slackNotificationUnsupported?: boolean; name: keyof NotificationSettingsFormValues; } -export const NotificationItemField: React.FC = ({ emailNotificationRequired, name }) => { +export const NotificationItemField: React.FC = ({ + emailNotificationRequired, + slackNotificationUnsupported, + name, +}) => { const emailNotificationsFeature = useFeature(FeatureItem.EmailNotifications); + const atLeastOneNotificationTypeEnableable = emailNotificationsFeature || !slackNotificationUnsupported; const { setValue, trigger } = useFormContext(); const field = useWatch({ name }); @@ -36,9 +43,13 @@ export const NotificationItemField: React.FC = ({ em } }; + if (!atLeastOneNotificationTypeEnableable) { + return null; + } + return ( <> -

+
@@ -48,6 +59,7 @@ export const NotificationItemField: React.FC = ({ em
+ {emailNotificationsFeature && ( {!emailNotificationRequired && ( @@ -60,11 +72,20 @@ export const NotificationItemField: React.FC = ({ em )} )} - - - - - + + {!slackNotificationUnsupported && ( + <> + + + + + + + )} ); }; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx index f6220c636f8..aec4d2bf40d 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx @@ -27,6 +27,7 @@ import { formValuesToNotificationSettings } from "./formValuesToNotificationSett import { NotificationItemField } from "./NotificationItemField"; import styles from "./NotificationSettingsForm.module.scss"; import { notificationSettingsToFormValues } from "./notificationSettingsToFormValues"; +import { useExperiment } from "../../hooks/services/Experiment"; interface NotificationSettingsFormProps { updateNotificationSettings: (notificationSettings: NotificationSettings) => Promise; @@ -34,6 +35,7 @@ interface NotificationSettingsFormProps { export const NotificationSettingsForm: React.FC = ({ updateNotificationSettings }) => { const emailNotificationsFeatureEnabled = useFeature(FeatureItem.EmailNotifications); + const breakingChangeNotificationsExperimentEnabled = useExperiment("settings.breakingChangeNotifications", false); const { notificationSettings, email } = useCurrentWorkspace(); const defaultValues = notificationSettingsToFormValues(notificationSettings); const testWebhook = useTryNotificationWebhook(); @@ -54,7 +56,7 @@ export const NotificationSettingsForm: React.FC = notificationKeys.map(async (key) => { const notification = values[key]; - // If slack is not set as a notification type, or if the webhook has not changed, we can skip the validation + // If Slack is not set as a notification type, or if the webhook has not changed, we can skip the validation if ( !notification.slack || (!methods.formState.dirtyFields[key]?.slack && !methods.formState.dirtyFields[key]?.slackWebhookLink) @@ -109,7 +111,7 @@ export const NotificationSettingsForm: React.FC = trackError(e, { name: "notification_settings_update_error", formValues: values, - requestPayloas: newNotificationSettings, + requestPayload: newNotificationSettings, }); registerNotification({ id: "notification_settings_update", @@ -164,6 +166,16 @@ export const NotificationSettingsForm: React.FC = + {breakingChangeNotificationsExperimentEnabled && ( + <> + + + + )}
@@ -188,6 +200,8 @@ export interface NotificationSettingsFormValues { sendOnConnectionUpdateActionRequired: NotificationItemFieldValue; sendOnSyncDisabled: NotificationItemFieldValue; sendOnSyncDisabledWarning: NotificationItemFieldValue; + sendOnBreakingChangeWarning: NotificationItemFieldValue; + sendOnBreakingChangeSyncsDisabled: NotificationItemFieldValue; } const notificationItemSchema: SchemaOf = yup.object({ @@ -206,6 +220,8 @@ const validationSchema: SchemaOf = yup.object({ sendOnConnectionUpdateActionRequired: notificationItemSchema, sendOnSyncDisabled: notificationItemSchema, sendOnSyncDisabledWarning: notificationItemSchema, + sendOnBreakingChangeWarning: notificationItemSchema, + sendOnBreakingChangeSyncsDisabled: notificationItemSchema, }); export const notificationKeys: Array = [ @@ -215,6 +231,8 @@ export const notificationKeys: Array = [ "sendOnConnectionUpdateActionRequired", "sendOnSyncDisabled", "sendOnSyncDisabledWarning", + "sendOnBreakingChangeWarning", + "sendOnBreakingChangeSyncsDisabled", ]; export const notificationTriggerMap: Record = { @@ -224,4 +242,6 @@ export const notificationTriggerMap: Record { diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts index 0bd6eea36d3..678500667b3 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts +++ b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts @@ -20,6 +20,8 @@ const mockEmptyFormValues: NotificationSettingsFormValues = { sendOnConnectionUpdateActionRequired: mockNotificationItemFieldValue, sendOnSyncDisabled: mockNotificationItemFieldValue, sendOnSyncDisabledWarning: mockNotificationItemFieldValue, + sendOnBreakingChangeWarning: mockNotificationItemFieldValue, + sendOnBreakingChangeSyncsDisabled: mockNotificationItemFieldValue, }; const mockEmptyNotificationSettings: NotificationSettings = { sendOnFailure: mockNotificationItem, @@ -28,6 +30,8 @@ const mockEmptyNotificationSettings: NotificationSettings = { sendOnConnectionUpdateActionRequired: mockNotificationItem, sendOnSyncDisabled: mockNotificationItem, sendOnSyncDisabledWarning: mockNotificationItem, + sendOnBreakingChangeWarning: mockNotificationItem, + sendOnBreakingChangeSyncsDisabled: mockNotificationItem, }; describe("notificationSettingsToFormValues", () => { diff --git a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts index d8e68ae78da..7ba4d304be6 100644 --- a/airbyte-webapp/src/hooks/services/Experiment/experiments.ts +++ b/airbyte-webapp/src/hooks/services/Experiment/experiments.ts @@ -32,6 +32,7 @@ export interface Experiments { "connector.suggestedDestinationConnectors": string; "onboarding.speedyConnection": boolean; "settings.emailNotifications": boolean; + "settings.breakingChangeNotifications": boolean; "upcomingFeaturesPage.url": string; "workspaces.newWorkspacesUI": boolean; "settings.accessManagement": boolean; diff --git a/airbyte-webapp/src/hooks/services/useWorkspace.tsx b/airbyte-webapp/src/hooks/services/useWorkspace.tsx index c6ff271cff1..26e95482c21 100644 --- a/airbyte-webapp/src/hooks/services/useWorkspace.tsx +++ b/airbyte-webapp/src/hooks/services/useWorkspace.tsx @@ -81,6 +81,12 @@ const useWorkspace = () => { sendOnConnectionUpdateActionRequired: { notificationType: ["customerio"], }, + sendOnBreakingChangeWarning: { + notificationType: ["customerio"], + }, + sendOnBreakingChangeSyncsDisabled: { + notificationType: ["customerio"], + }, }; return await updateWorkspace({ diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 1ebaecc82de..e0d11cbdf1d 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -740,6 +740,10 @@ "settings.notifications.sendOnSyncDisabled.description": "A connection was automatically disabled due to repeated failures", "settings.notifications.sendOnSyncDisabledWarning": "Sync Disabled Warning", "settings.notifications.sendOnSyncDisabledWarning.description": "A connection will be disabled soon due to repeated failures", + "settings.notifications.sendOnBreakingChangeWarning": "Breaking Change Warning", + "settings.notifications.sendOnBreakingChangeWarning.description": "A breaking change upgrade is available for a source or destination", + "settings.notifications.sendOnBreakingChangeSyncsDisabled": "Syncs Disabled - Upgrade Required", + "settings.notifications.sendOnBreakingChangeSyncsDisabled.description": "One or more connections were automatically disabled due to a connector upgrade deadline passing", "settings.notifications.testWebhookButtonLabel": "Test", "settings.notifications.requiredNotificationTooltip": "This notification cannot be disabled", "settings.notifications.successMessage": "Notification settings updated", diff --git a/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts b/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts index 03a61978545..ba9407f0a62 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts @@ -31,5 +31,11 @@ export const mockWorkspace: WorkspaceRead = { sendOnSyncDisabledWarning: { notificationType: [], }, + sendOnBreakingChangeWarning: { + notificationType: [], + }, + sendOnBreakingChangeSyncsDisabled: { + notificationType: [], + }, }, }; From 3024099f3c7d97fe1df853350ab42f56cc494d4c Mon Sep 17 00:00:00 2001 From: keyihuang Date: Mon, 11 Sep 2023 10:30:59 -0700 Subject: [PATCH 142/201] Add OSS Workspace endpoint: workspaces/list_by_user_id (#8572) --- airbyte-api/src/main/openapi/config.yaml | 40 +++++- .../server/handlers/WorkspacesHandler.java | 78 ++++++++++-- .../handlers/WorkspacesHandlerTest.java | 8 +- .../config/persistence/ConfigRepository.java | 14 +++ .../persistence/PermissionPersistence.java | 26 ++++ .../persistence/WorkspacePersistence.java | 119 +++++++++++++++++- .../persistence/WorkspacePersistenceTest.java | 97 +++++++++++++- .../server/apis/WorkspaceApiController.java | 9 ++ 8 files changed, 371 insertions(+), 20 deletions(-) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index a674879f975..e520df071c9 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -137,7 +137,7 @@ paths: post: tags: - workspace - summary: List all workspaces registered in the current Airbyte deployment, paginated. + summary: List all workspaces registered in the current Airbyte deployment. This function also supports searching by keyword and pagination. operationId: listAllWorkspacesPaginated requestBody: content: @@ -156,7 +156,7 @@ paths: post: tags: - workspace - summary: List all workspaces registered in the current Airbyte deployment, paginated + summary: List workspaces by given workspace IDs registered in the current Airbyte deployment. This function also supports pagination. operationId: listWorkspacesPaginated requestBody: content: @@ -173,7 +173,7 @@ paths: $ref: "#/components/schemas/WorkspaceReadList" /v1/workspaces/list_by_organization_id: post: - summary: List workspaces under the given org id + summary: List workspaces under the given org id. This function also supports searching by keyword and pagination. tags: - workspace operationId: listWorkspacesInOrganization @@ -191,7 +191,26 @@ paths: $ref: "#/components/schemas/WorkspaceReadList" "404": $ref: "#/components/responses/NotFoundResponse" - + /v1/workspaces/list_by_user_id: + post: + summary: List workspaces by a given user id. The function also supports searching by keyword and pagination. + tags: + - workspace + operationId: listWorkspacesByUser + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ListWorkspacesByUserRequestBody" + responses: + "200": + description: Successfully retrieved workspaces by given user id. + content: + application/json: + schema: + $ref: "#/components/schemas/WorkspaceReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" /v1/workspaces/get: post: tags: @@ -3839,6 +3858,7 @@ components: type: object required: - workspaceIds + - pagination properties: workspaceIds: type: array @@ -7758,6 +7778,18 @@ components: $ref: "#/components/schemas/Pagination" keyword: type: string + ListWorkspacesByUserRequestBody: + type: object + required: + - userId + properties: + userId: + type: string + format: uuid + pagination: + $ref: "#/components/schemas/Pagination" + keyword: + type: string OrganizationRead: type: object required: diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java index 6672b86047d..2ea424d05a1 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/WorkspacesHandler.java @@ -14,6 +14,7 @@ import io.airbyte.api.model.generated.DestinationRead; import io.airbyte.api.model.generated.Geography; import io.airbyte.api.model.generated.ListResourcesForWorkspacesRequestBody; +import io.airbyte.api.model.generated.ListWorkspacesByUserRequestBody; import io.airbyte.api.model.generated.ListWorkspacesInOrganizationRequestBody; import io.airbyte.api.model.generated.NotificationItem; import io.airbyte.api.model.generated.NotificationSettings; @@ -35,9 +36,11 @@ import io.airbyte.commons.server.errors.InternalServerKnownException; import io.airbyte.commons.server.errors.ValueConflictKnownException; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.UserPermission; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByUserQueryPaginated; import io.airbyte.config.persistence.ConfigRepository.ResourcesQueryPaginated; import io.airbyte.config.persistence.PermissionPersistence; import io.airbyte.config.persistence.SecretsRepositoryWriter; @@ -269,14 +272,73 @@ public WorkspaceRead getWorkspaceByConnectionId(final ConnectionIdRequestBody co public WorkspaceReadList listWorkspacesInOrganization(final ListWorkspacesInOrganizationRequestBody request) throws IOException { Optional keyword = StringUtils.isBlank(request.getKeyword()) ? Optional.empty() : Optional.of(request.getKeyword()); - final List standardWorkspaces = workspacePersistence - .listWorkspacesByOrganizationId( - new ResourcesByOrganizationQueryPaginated(request.getOrganizationId(), - false, request.getPagination().getPageSize(), request.getPagination().getRowOffset()), - keyword) - .stream() - .map(WorkspacesHandler::buildWorkspaceRead) - .collect(Collectors.toList()); + List standardWorkspaces; + if (request.getPagination() != null) { + standardWorkspaces = workspacePersistence + .listWorkspacesByOrganizationIdPaginated( + new ResourcesByOrganizationQueryPaginated(request.getOrganizationId(), + false, request.getPagination().getPageSize(), request.getPagination().getRowOffset()), + keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } else { + standardWorkspaces = workspacePersistence + .listWorkspacesByOrganizationId(request.getOrganizationId(), false, keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } + return new WorkspaceReadList().workspaces(standardWorkspaces); + } + + private WorkspaceReadList listWorkspacesByInstanceAdminUser(final ListWorkspacesByUserRequestBody request) throws IOException { + Optional keyword = StringUtils.isBlank(request.getKeyword()) ? Optional.empty() : Optional.of(request.getKeyword()); + List standardWorkspaces; + if (request.getPagination() != null) { + standardWorkspaces = workspacePersistence + .listWorkspacesByInstanceAdminUserPaginated( + false, request.getPagination().getPageSize(), request.getPagination().getRowOffset(), + keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } else { + standardWorkspaces = workspacePersistence + .listWorkspacesByInstanceAdminUser(false, keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } + return new WorkspaceReadList().workspaces(standardWorkspaces); + } + + public WorkspaceReadList listWorkspacesByUser(final ListWorkspacesByUserRequestBody request) + throws IOException { + // If user has instance_admin permission, list all workspaces. + final UserPermission userInstanceAdminPermission = permissionPersistence.getUserInstanceAdminPermission(request.getUserId()); + if (userInstanceAdminPermission != null) { + return listWorkspacesByInstanceAdminUser(request); + } + // User has no instance_admin permission. + Optional keyword = StringUtils.isBlank(request.getKeyword()) ? Optional.empty() : Optional.of(request.getKeyword()); + List standardWorkspaces; + if (request.getPagination() != null) { + standardWorkspaces = workspacePersistence + .listWorkspacesByUserIdPaginated( + new ResourcesByUserQueryPaginated(request.getUserId(), + false, request.getPagination().getPageSize(), request.getPagination().getRowOffset()), + keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } else { + standardWorkspaces = workspacePersistence + .listWorkspacesByUserId(request.getUserId(), false, keyword) + .stream() + .map(WorkspacesHandler::buildWorkspaceRead) + .collect(Collectors.toList()); + } return new WorkspaceReadList().workspaces(standardWorkspaces); } diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java index 88550417eb6..eba9fda6c96 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/WorkspacesHandlerTest.java @@ -740,11 +740,11 @@ void testListWorkspacesInOrgNoKeyword() throws Exception { new ListWorkspacesInOrganizationRequestBody().organizationId(ORGANIZAION_ID).pagination(new Pagination().pageSize(100).rowOffset(0)); List expectedWorkspaces = List.of(generateWorkspace(), generateWorkspace()); - when(workspacePersistence.listWorkspacesByOrganizationId(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), + when(workspacePersistence.listWorkspacesByOrganizationIdPaginated(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), Optional.empty())) .thenReturn(expectedWorkspaces); WorkspaceReadList result = workspacesHandler.listWorkspacesInOrganization(request); - assertEquals(result.getWorkspaces().size(), 2); + assertEquals(2, result.getWorkspaces().size()); } @Test @@ -753,11 +753,11 @@ void testListWorkspacesInOrgWithKeyword() throws Exception { .keyword("keyword").pagination(new Pagination().pageSize(100).rowOffset(0)); List expectedWorkspaces = List.of(generateWorkspace(), generateWorkspace()); - when(workspacePersistence.listWorkspacesByOrganizationId(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), + when(workspacePersistence.listWorkspacesByOrganizationIdPaginated(new ResourcesByOrganizationQueryPaginated(ORGANIZAION_ID, false, 100, 0), Optional.of("keyword"))) .thenReturn(expectedWorkspaces); WorkspaceReadList result = workspacesHandler.listWorkspacesInOrganization(request); - assertEquals(result.getWorkspaces().size(), 2); + assertEquals(2, result.getWorkspaces().size()); } } diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 07ef4ccc62f..41930d41e2a 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -203,6 +203,20 @@ public record ResourcesByOrganizationQueryPaginated( } + /** + * Query object for paginated querying of resource for a user. + * + * @param userId user to fetch resources for + * @param includeDeleted include tombstoned resources + * @param pageSize limit + * @param rowOffset offset + */ + public record ResourcesByUserQueryPaginated( + @Nonnull UUID userId, + boolean includeDeleted, + int pageSize, + int rowOffset) {} + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigRepository.class); private static final String OPERATION_IDS_AGG_FIELD = "operation_ids_agg"; private static final String OPERATION_IDS_AGG_DELIMITER = ","; diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java index 0598bdbb0ee..9aa27cb2ffa 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/PermissionPersistence.java @@ -216,6 +216,32 @@ private List listInstanceAdminPermissions(final DSLContext ctx) return records.stream().map(record -> buildUserPermissionFromRecord(record)).collect(Collectors.toList()); } + private UserPermission getUserInstanceAdminPermission(final DSLContext ctx, final UUID userId) { + var record = ctx.select(USER.ID, USER.NAME, USER.EMAIL, USER.DEFAULT_WORKSPACE_ID, PERMISSION.ID, PERMISSION.PERMISSION_TYPE) + .from(PERMISSION) + .join(USER) + .on(PERMISSION.USER_ID.eq(USER.ID)) + .where(PERMISSION.PERMISSION_TYPE.eq(io.airbyte.db.instance.configs.jooq.generated.enums.PermissionType.instance_admin)) + .and(PERMISSION.USER_ID.eq(userId)) + .fetchOne(); + if (record == null) { + return null; + } + return buildUserPermissionFromRecord(record); + } + + /** + * Check and get instance_admin permission for a user. + * + * @param userId user id + * @return UserPermission User details with instance_admin permission, null if user does not have + * instance_admin role. + * @throws IOException if there is an issue while interacting with the db. + */ + public UserPermission getUserInstanceAdminPermission(final UUID userId) throws IOException { + return this.database.query(ctx -> getUserInstanceAdminPermission(ctx, userId)); + } + public PermissionType findPermissionTypeForUserAndWorkspace(final UUID workspaceId, final String authUserId, final AuthProvider authProvider) throws IOException { return this.database.query(ctx -> findPermissionTypeForUserAndWorkspace(ctx, workspaceId, authUserId, authProvider)); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java index 352fd312da6..ba945f12c1e 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/WorkspacePersistence.java @@ -9,6 +9,7 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByUserQueryPaginated; import io.airbyte.db.Database; import io.airbyte.db.ExceptionWrappingDatabase; import java.io.IOException; @@ -29,16 +30,56 @@ public WorkspacePersistence(final Database database) { this.database = new ExceptionWrappingDatabase(database); } + /** + * List all workspaces as user has instance_admin role. Returning result ordered by workspace name. + * Supports pagination and keyword search. + */ + public List listWorkspacesByInstanceAdminUserPaginated(final boolean includeDeleted, + final int pageSize, + final int rowOffset, + Optional keyword) + throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(includeDeleted ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.NAME.asc()) + .limit(pageSize) + .offset(rowOffset) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * List all workspaces as user has instance_admin role. Returning result ordered by workspace name. + * Supports keyword search. + */ + public List listWorkspacesByInstanceAdminUser(final boolean includeDeleted, Optional keyword) + throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(includeDeleted ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.NAME.asc()) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + /** * List all workspaces owned by org id, returning result ordered by workspace name. Supports * pagination and keyword search. */ - public List listWorkspacesByOrganizationId(final ResourcesByOrganizationQueryPaginated query, Optional keyword) + public List listWorkspacesByOrganizationIdPaginated(final ResourcesByOrganizationQueryPaginated query, Optional keyword) throws IOException { return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) .from(WORKSPACE) .where(WORKSPACE.ORGANIZATION_ID.eq(query.organizationId())) .and(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(query.includeDeleted() ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) .orderBy(WORKSPACE.NAME.asc()) .limit(query.pageSize()) .offset(query.rowOffset()) @@ -48,6 +89,82 @@ public List listWorkspacesByOrganizationId(final ResourcesByO .toList(); } + /** + * List all workspaces owned by org id, returning result ordered by workspace name. Supports keyword + * search. + */ + public List listWorkspacesByOrganizationId(UUID organizationId, boolean includeDeleted, Optional keyword) + throws IOException { + return database.query(ctx -> ctx.select(WORKSPACE.asterisk()) + .from(WORKSPACE) + .where(WORKSPACE.ORGANIZATION_ID.eq(organizationId)) + .and(keyword.isPresent() ? WORKSPACE.NAME.containsIgnoreCase(keyword.get()) : noCondition()) + .and(includeDeleted ? noCondition() : WORKSPACE.TOMBSTONE.notEqual(true)) + .orderBy(WORKSPACE.NAME.asc()) + .fetch()) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * This query is to list all workspaces that a user has read permissions. + */ + private final String listWorkspacesByUserIdBasicQuery = + "WITH userOrgs AS (SELECT organization_id FROM permission WHERE user_id = {0})," + + " userWorkspaces AS (" + + " SELECT workspace.id AS workspace_id FROM userOrgs JOIN workspace" + + " ON workspace.organization_id = userOrgs.organization_id" + + " UNION" + + " SELECT workspace_id FROM permission WHERE user_id = {1}" + + " )" + + " SELECT * from workspace" + + " WHERE workspace.id IN (SELECT workspace_id from userWorkspaces)" + + " AND name ILIKE {2}" + + " AND tombstone = false" + + " ORDER BY name ASC"; + + /** + * Get search keyword with flexible matching. + */ + private String getSearchKeyword(Optional keyword) { + if (keyword.isPresent()) { + return "%" + keyword.get().toLowerCase() + "%"; + } else { + return "%%"; + } + } + + /** + * List all workspaces owned by user id, returning result ordered by workspace name. Supports + * keyword search. + */ + public List listWorkspacesByUserId(UUID userId, boolean includeDeleted, Optional keyword) + throws IOException { + final String searchKeyword = getSearchKeyword(keyword); + return database.query(ctx -> ctx.fetch(listWorkspacesByUserIdBasicQuery, userId, userId, searchKeyword)) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + + /** + * List all workspaces owned by user id, returning result ordered by workspace name. Supports + * pagination and keyword search. + */ + public List listWorkspacesByUserIdPaginated(final ResourcesByUserQueryPaginated query, Optional keyword) + throws IOException { + final String searchKeyword = getSearchKeyword(keyword); + final String workspaceQuery = listWorkspacesByUserIdBasicQuery + + " LIMIT {3}" + + " OFFSET {4}"; + + return database.query(ctx -> ctx.fetch(workspaceQuery, query.userId(), query.userId(), searchKeyword, query.pageSize(), query.rowOffset())) + .stream() + .map(DbConverter::buildStandardWorkspace) + .toList(); + } + /** * Fetch the oldest, non-tombstoned Workspace that belongs to the given Organization. */ diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java index b07e41937f5..57394a517b5 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/WorkspacePersistenceTest.java @@ -18,6 +18,8 @@ import io.airbyte.config.DestinationConnection; import io.airbyte.config.Geography; import io.airbyte.config.Organization; +import io.airbyte.config.Permission; +import io.airbyte.config.Permission.PermissionType; import io.airbyte.config.ReleaseStage; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; @@ -25,7 +27,10 @@ import io.airbyte.config.StandardSync; import io.airbyte.config.StandardWorkspace; import io.airbyte.config.SupportLevel; +import io.airbyte.config.User; +import io.airbyte.config.User.AuthProvider; import io.airbyte.config.persistence.ConfigRepository.ResourcesByOrganizationQueryPaginated; +import io.airbyte.config.persistence.ConfigRepository.ResourcesByUserQueryPaginated; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.List; @@ -50,11 +55,15 @@ class WorkspacePersistenceTest extends BaseConfigDatabaseTest { private ConfigRepository configRepository; private WorkspacePersistence workspacePersistence; + private PermissionPersistence permissionPersistence; + private UserPersistence userPersistence; @BeforeEach void setup() throws Exception { configRepository = spy(new ConfigRepository(database, null, MockData.MAX_SECONDS_BETWEEN_MESSAGE_SUPPLIER)); workspacePersistence = new WorkspacePersistence(database); + permissionPersistence = new PermissionPersistence(database); + userPersistence = new UserPersistence(database); final OrganizationPersistence organizationPersistence = new OrganizationPersistence(database); truncateAllTables(); @@ -229,7 +238,7 @@ void testListWorkspacesInOrgNoKeyword() throws Exception { configRepository.writeStandardWorkspaceNoSecrets(workspace); configRepository.writeStandardWorkspaceNoSecrets(otherWorkspace); - final List workspaces = workspacePersistence.listWorkspacesByOrganizationId( + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 10, 0), Optional.empty()); assertReturnsWorkspace(createBaseStandardWorkspace().withTombstone(false)); @@ -249,7 +258,7 @@ void testListWorkspacesInOrgWithPagination() throws Exception { configRepository.writeStandardWorkspaceNoSecrets(workspace); configRepository.writeStandardWorkspaceNoSecrets(otherWorkspace); - final List workspaces = workspacePersistence.listWorkspacesByOrganizationId( + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 1, 0), Optional.empty()); assertEquals(1, workspaces.size()); @@ -268,7 +277,7 @@ void testListWorkspacesInOrgWithKeyword() throws Exception { configRepository.writeStandardWorkspaceNoSecrets(workspace); configRepository.writeStandardWorkspaceNoSecrets(otherWorkspace); - final List workspaces = workspacePersistence.listWorkspacesByOrganizationId( + final List workspaces = workspacePersistence.listWorkspacesByOrganizationIdPaginated( new ResourcesByOrganizationQueryPaginated(MockData.ORGANIZATION_ID_1, false, 10, 0), Optional.of("keyword")); assertEquals(1, workspaces.size()); @@ -304,4 +313,86 @@ void testGetDefaultWorkspaceForOrganization() throws JsonValidationException, IO assertEquals(expectedWorkspace, actualWorkspace); } + @Test + void testListWorkspacesByUserIdWithKeywordWithPagination() throws Exception { + final UUID workspaceId = UUID.randomUUID(); + // create a user + final UUID userId = UUID.randomUUID(); + userPersistence.writeUser(new User() + .withUserId(userId) + .withName("user") + .withAuthUserId("auth_id") + .withEmail("email") + .withAuthProvider(AuthProvider.AIRBYTE)); + // create a workspace in org_1, name contains search "keyword" + final StandardWorkspace orgWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspace_with_keyword_1"); + configRepository.writeStandardWorkspaceNoSecrets(orgWorkspace); + // create a workspace in org_2, name contains search "Keyword" + final StandardWorkspace userWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(workspaceId).withOrganizationId(MockData.ORGANIZATION_ID_2) + .withName("workspace_with_Keyword_2"); + configRepository.writeStandardWorkspaceNoSecrets(userWorkspace); + // create a workspace permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withWorkspaceId(workspaceId) + .withUserId(userId) + .withPermissionType(PermissionType.WORKSPACE_READER)); + // create an org permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withUserId(userId) + .withPermissionType(PermissionType.ORGANIZATION_ADMIN)); + + final List workspaces = workspacePersistence.listWorkspacesByUserIdPaginated( + new ResourcesByUserQueryPaginated(userId, false, 10, 0), Optional.of("keyWord")); + + assertEquals(2, workspaces.size()); + } + + @Test + void testListWorkspacesByUserIdWithoutKeywordWithoutPagination() throws Exception { + final UUID workspaceId = UUID.randomUUID(); + // create a user + final UUID userId = UUID.randomUUID(); + userPersistence.writeUser(new User() + .withUserId(userId) + .withName("user") + .withAuthUserId("auth_id") + .withEmail("email") + .withAuthProvider(AuthProvider.AIRBYTE)); + // create a workspace in org_1 + final StandardWorkspace orgWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withName("workspace1"); + configRepository.writeStandardWorkspaceNoSecrets(orgWorkspace); + // create a workspace in org_2 + final StandardWorkspace userWorkspace = createBaseStandardWorkspace() + .withWorkspaceId(workspaceId).withOrganizationId(MockData.ORGANIZATION_ID_2) + .withName("workspace2"); + configRepository.writeStandardWorkspaceNoSecrets(userWorkspace); + // create a workspace permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withWorkspaceId(workspaceId) + .withUserId(userId) + .withPermissionType(PermissionType.WORKSPACE_READER)); + // create an org permission + permissionPersistence.writePermission(new Permission() + .withPermissionId(UUID.randomUUID()) + .withOrganizationId(MockData.ORGANIZATION_ID_1) + .withUserId(userId) + .withPermissionType(PermissionType.ORGANIZATION_ADMIN)); + + final List workspaces = workspacePersistence.listWorkspacesByUserId( + userId, false, Optional.empty()); + + assertEquals(2, workspaces.size()); + } + } diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java index 9800165f874..e1a11df154a 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/WorkspaceApiController.java @@ -11,6 +11,7 @@ import io.airbyte.api.generated.WorkspaceApi; import io.airbyte.api.model.generated.ConnectionIdRequestBody; import io.airbyte.api.model.generated.ListResourcesForWorkspacesRequestBody; +import io.airbyte.api.model.generated.ListWorkspacesByUserRequestBody; import io.airbyte.api.model.generated.ListWorkspacesInOrganizationRequestBody; import io.airbyte.api.model.generated.SlugRequestBody; import io.airbyte.api.model.generated.WorkspaceCreate; @@ -149,4 +150,12 @@ public WorkspaceReadList listWorkspacesInOrganization(@Body final ListWorkspaces return ApiHelper.execute(() -> workspacesHandler.listWorkspacesInOrganization(request)); } + @Post("/list_by_user_id") + @Secured({READER}) + @ExecuteOn(AirbyteTaskExecutors.IO) + @Override + public WorkspaceReadList listWorkspacesByUser(@Body final ListWorkspacesByUserRequestBody request) { + return ApiHelper.execute(() -> workspacesHandler.listWorkspacesByUser(request)); + } + } From 89e616a0c8e6285347306a81add3bca926996a34 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Mon, 11 Sep 2023 20:54:11 +0200 Subject: [PATCH 143/201] Increase helm chart timeouts (#8760) --- charts/airbyte-keycloak/values.yaml | 4 ++-- charts/airbyte-webapp/templates/deployment.yaml | 2 +- charts/airbyte/values.yaml | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/airbyte-keycloak/values.yaml b/charts/airbyte-keycloak/values.yaml index 3247fdbf5cd..1bdb6a79d75 100644 --- a/charts/airbyte-keycloak/values.yaml +++ b/charts/airbyte-keycloak/values.yaml @@ -52,7 +52,7 @@ livenessProbe: enabled: true initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 @@ -67,7 +67,7 @@ readinessProbe: enabled: true initialDelaySeconds: 10 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 diff --git a/charts/airbyte-webapp/templates/deployment.yaml b/charts/airbyte-webapp/templates/deployment.yaml index 1db50651ba3..881796426a9 100644 --- a/charts/airbyte-webapp/templates/deployment.yaml +++ b/charts/airbyte-webapp/templates/deployment.yaml @@ -131,7 +131,7 @@ spec: {{- if .Values.readinessProbe.enabled }} readinessProbe: httpGet: - path: /api/v1/health + path: /index.html port: http initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 36949ba4488..2a919223db5 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -566,7 +566,7 @@ server: enabled: true initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 @@ -581,7 +581,7 @@ server: enabled: true initialDelaySeconds: 10 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 @@ -1520,7 +1520,7 @@ connector-builder-server: enabled: true initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 @@ -1535,7 +1535,7 @@ connector-builder-server: enabled: true initialDelaySeconds: 10 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 @@ -1593,7 +1593,7 @@ airbyte-api-server: enabled: true initialDelaySeconds: 30 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 @@ -1608,7 +1608,7 @@ airbyte-api-server: enabled: true initialDelaySeconds: 10 periodSeconds: 10 - timeoutSeconds: 1 + timeoutSeconds: 10 failureThreshold: 3 successThreshold: 1 From 0fbd4ac2b9a62c7a12a5d6799f98299cd7a9e849 Mon Sep 17 00:00:00 2001 From: timroes Date: Mon, 11 Sep 2023 19:02:47 +0000 Subject: [PATCH 144/201] Bump helm chart version reference to 0.48.8 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 867f5bba008..4766c754b6e 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index e5d0c956568..83ff38bed1d 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 684a97d0cf2..a1e921e43e5 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index f4d79c4c1f8..12444fd33f6 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index f4bebddd2be..0c362e47de4 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index e986f8eea73..edb543262f1 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index ca609dbcd15..17abf288666 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index fd72077b37a..96e94e9d77a 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index b67ca2ef062..f1c8469b8c3 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 1baf0b246f5..226a62164ca 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 2b3ce425792..c5a7f33b131 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index a56704490dd..69f4efc8955 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index bbffbb69b74..d466a471914 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 -digest: sha256:fbefd6a15f7c63aa7f5d2c8cb6be9e1896ed35958398ec365b2ce79d660158ef -generated: "2023-09-01T23:25:00.955650887Z" + version: 0.48.8 +digest: sha256:6e736e04dd6da520859db8e5b450971cea7d80fc0d629c293cb38fb063447cc0 +generated: "2023-09-11T19:02:30.966383665Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 8bee9aafb24..efa4ecae087 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.7 +version: 0.48.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.7 + version: 0.48.8 From 65f2b20a1bf4bfe5803df35e10916de08d44857d Mon Sep 17 00:00:00 2001 From: timroes Date: Mon, 11 Sep 2023 20:11:49 +0000 Subject: [PATCH 145/201] Bump helm chart version reference to 0.48.9 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 4766c754b6e..7bbef5e9a7c 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 83ff38bed1d..54023b03f22 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index a1e921e43e5..fb8cd0e6995 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 12444fd33f6..0d2e89a4e05 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 0c362e47de4..95da4185eb8 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index edb543262f1..946a2ea6dd3 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 17abf288666..bbf4a568f08 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 96e94e9d77a..ff39e925276 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index f1c8469b8c3..d6a94714df6 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 226a62164ca..954eeb96f95 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index c5a7f33b131..707219c455d 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 69f4efc8955..2ebba0bb57e 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index d466a471914..8ffd7ce0879 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 -digest: sha256:6e736e04dd6da520859db8e5b450971cea7d80fc0d629c293cb38fb063447cc0 -generated: "2023-09-11T19:02:30.966383665Z" + version: 0.48.9 +digest: sha256:fdf2dc5084a7fde5a769eecb3b90918c13cf864165db4d984f0c4c84237f0613 +generated: "2023-09-11T20:11:35.790563651Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index efa4ecae087..78a13dd5eb7 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.8 +version: 0.48.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.8 + version: 0.48.9 From 0941da37c35d4da16b0844a345e48029ff7cd31c Mon Sep 17 00:00:00 2001 From: Conor Date: Tue, 12 Sep 2023 15:53:45 +0200 Subject: [PATCH 146/201] Revert "local and remote ci performance and usability improvements" (#8775) --- airbyte-webapp/build.gradle | 32 ++++++++++- airbyte-webapp/package.json | 1 - airbyte-webapp/scripts/validate-lock-files.js | 25 --------- build.gradle | 35 ++---------- ci/oss/ci.py | 8 +-- ci/oss/tasks.py | 55 ++++++++----------- 6 files changed, 62 insertions(+), 94 deletions(-) delete mode 100644 airbyte-webapp/scripts/validate-lock-files.js diff --git a/airbyte-webapp/build.gradle b/airbyte-webapp/build.gradle index 9c4423a5c85..1e010600a99 100644 --- a/airbyte-webapp/build.gradle +++ b/airbyte-webapp/build.gradle @@ -28,7 +28,23 @@ node { distBaseUrl = "https://nodejs.org/dist" } +tasks.register("validateLockFiles") { + description "Validate only a pnpm-lock.yaml lock file exists" + inputs.files "pnpm-lock.yaml", "package-lock.json", "yarn.lock" + + // The validateLockFiles has no outputs, thus we always treat the outputs up to date + // as long as the inputs have not changed + outputs.upToDateWhen { true } + + doLast { + assert file("pnpm-lock.yaml").exists() + assert !file("package-lock.json").exists() + assert !file("yarn.lock").exists() + } +} + tasks.named("pnpmInstall") { + dependsOn tasks.named("validateLockFiles") // Add patches folder to inputs of pnpmInstall task, since it has pnpm-lock.yml as an output // thus wouldn't rerun in case a patch get changed inputs.dir "patches" @@ -109,6 +125,19 @@ tasks.register("cloudE2eTest", PnpmTask) { outputs.upToDateWhen { false } } +tasks.register("licenseCheck", PnpmTask) { + dependsOn tasks.named("pnpmInstall") + + args = ['run', 'license-check'] + + inputs.files nodeModules + inputs.file 'package.json' + inputs.file 'scripts/license-check.js' + + // The licenseCheck has no outputs, thus we always treat the outputs up to date + // as long as the inputs have not changed + outputs.upToDateWhen { true } +} // //tasks.register("validateLinks", PnpmTask) { // dependsOn tasks.named("pnpmInstall") @@ -160,9 +189,10 @@ tasks.register("copyNginx", Copy) { // Those tasks should be run as part of the "check" task tasks.named("check") { - dependsOn /*tasks.named("validateLinks"),*/ tasks.named("test") + dependsOn /*tasks.named("validateLinks"),*/ tasks.named("licenseCheck"), tasks.named("test") } + tasks.named("build") { dependsOn tasks.named("buildStorybook") } diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 2ea80be94e2..f78d56257b2 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -33,7 +33,6 @@ "license-check": "node ./scripts/license-check.js", "generate-client": "./scripts/load-declarative-schema.sh && orval", "validate-links": "ts-node --skip-project ./scripts/validate-links.ts", - "validate-lock": "node ./scripts/validate-lock-files.js", "preanalyze-lowcode": "TS_NODE_TRANSPILE_ONLY=true pnpm run generate-client", "analyze-lowcode": "ts-node --skip-project ./scripts/analyze-low-code-manifests.ts", "cypress:open": "cypress open --config-file cypress/cypress.config.ts", diff --git a/airbyte-webapp/scripts/validate-lock-files.js b/airbyte-webapp/scripts/validate-lock-files.js deleted file mode 100644 index 756f9cb7358..00000000000 --- a/airbyte-webapp/scripts/validate-lock-files.js +++ /dev/null @@ -1,25 +0,0 @@ -const fs = require("node:fs"); -const path = require("node:path"); - -function assertFileExists(filepath) { - if (!fs.existsSync(filepath)) { - throw new Error(`File ${filepath} does not exist`); - } -} - -function assertFileNotExists(filepath) { - if (fs.existsSync(filepath)) { - throw new Error(`File ${filepath} exists but should not`); - } -} - -try { - assertFileExists(path.join(__dirname, "..", "pnpm-lock.yaml")); - assertFileNotExists(path.join(__dirname, "..", "package-lock.json")); - assertFileNotExists(path.join(__dirname, "..", "yarn.lock")); - - console.log("Lock file validation successful."); -} catch (error) { - console.error(`Lock file validation failed: ${error.message}`); - process.exit(1); -} diff --git a/build.gradle b/build.gradle index 7efc5535dd5..c12a91329c0 100644 --- a/build.gradle +++ b/build.gradle @@ -24,12 +24,11 @@ buildscript { plugins { id "base" - id "com.dorongold.task-tree" version "2.1.1" - id "io.airbyte.gradle.jvm" version "0.15.0" apply false - id "io.airbyte.gradle.jvm.app" version "0.15.0" apply false - id "io.airbyte.gradle.jvm.lib" version "0.15.0" apply false - id "io.airbyte.gradle.docker" version "0.15.0" apply false - id "io.airbyte.gradle.publish" version "0.15.0" apply false + id "io.airbyte.gradle.jvm" version "0.14.0" apply false + id "io.airbyte.gradle.jvm.app" version "0.14.0" apply false + id "io.airbyte.gradle.jvm.lib" version "0.14.0" apply false + id "io.airbyte.gradle.docker" version "0.14.0" apply false + id "io.airbyte.gradle.publish" version "0.14.0" apply false } repositories { @@ -72,30 +71,6 @@ allprojects { version = rootProject.ext.version } -tasks.register('archiveReports', Tar) { - dependsOn subprojects.collect { it.getTasksByName('checkstyleMain', true) } - dependsOn subprojects.collect { it.getTasksByName('checkstyleTest', true) } - dependsOn subprojects.collect { it.getTasksByName('jacocoTestReport', true) } - dependsOn subprojects.collect { it.getTasksByName('pmdMain', true) } - dependsOn subprojects.collect { it.getTasksByName('pmdTest', true) } - dependsOn subprojects.collect { it.getTasksByName('spotbugsMain', true) } - dependsOn subprojects.collect { it.getTasksByName('spotbugsTest', true) } - dependsOn subprojects.collect { it.getTasksByName('test', true) } - dependsOn subprojects.collect { it.getTasksByName('checkstyleAcceptanceTests', true) } - dependsOn subprojects.collect { it.getTasksByName('pmdAcceptanceTests', true) } - dependsOn subprojects.collect { it.getTasksByName('spotbugsAcceptanceTests', true) } - - archiveFileName = "${project.name}-reports.tar" - destinationDirectory = layout.buildDirectory.dir('dist') - - // Collect reports from each subproject - subprojects.each { subproject -> - from("${subproject.buildDir}/reports") { - into("${subproject.name}/reports") - } - } -} - subprojects { sp -> // airbyte-webapp has not been converted to the gradle plugins if (sp.name != "airbyte-webapp") { diff --git a/ci/oss/ci.py b/ci/oss/ci.py index cae383b0dc1..b5fa91ed291 100644 --- a/ci/oss/ci.py +++ b/ci/oss/ci.py @@ -20,7 +20,6 @@ build_oss_backend_task, build_oss_frontend_task, build_storybook_oss_frontend_task, - check_oss_backend_task, test_oss_backend_task, test_oss_frontend_task, ) @@ -116,11 +115,10 @@ async def frontend_build(settings: OssSettings, ctx: PipelineContext, client: Op @pass_global_settings @pass_pipeline_context @flow(validate_parameters=False, name="OSS Test") -async def test(settings: OssSettings, ctx: PipelineContext, build_results: Optional[List[Container]] = None, client: Optional[Client] = None, scan: bool = False) -> List[Container]: +async def test(settings: OssSettings, ctx: PipelineContext, client: Optional[Client] = None, scan: bool = False) -> List[Container]: test_client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name) - build_results = await build(scan=scan, client=test_client) if build_results is None else build_results + build_results = await build(scan=scan, client=test_client) test_results = await test_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0][0], settings=settings, ctx=quote(ctx), scan=scan) - await check_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0][0], settings=settings, ctx=quote(ctx), scan=scan) # TODO: add cypress E2E tests here return [test_results] @@ -132,8 +130,6 @@ async def backend_test(settings: OssSettings, ctx: PipelineContext, client: Opti test_client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name) build_results = await backend_build(scan=scan, client=test_client) test_results = await test_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0], settings=settings, ctx=quote(ctx), scan=scan) - await check_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0], settings=settings, ctx=quote(ctx), scan=scan) - return test_results @oss_group.command(CICommand()) diff --git a/ci/oss/tasks.py b/ci/oss/tasks.py index ee5e2f393eb..e2ec8f2fbfc 100644 --- a/ci/oss/tasks.py +++ b/ci/oss/tasks.py @@ -8,10 +8,11 @@ with_gradle, with_node, with_pnpm, + with_typescript_gha, ) -from aircmd.actions.pipelines import get_repo_dir, sync_to_gradle_cache_from_homedir +from aircmd.actions.pipelines import get_repo_dir from aircmd.models.base import PipelineContext -from aircmd.models.settings import load_settings +from aircmd.models.settings import GithubActionsInputSettings, load_settings from dagger import CacheSharingMode, CacheVolume, Client, Container from prefect import task from prefect.artifacts import create_link_artifact @@ -48,7 +49,7 @@ async def build_oss_backend_task(settings: OssSettings, ctx: PipelineContext, cl .with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte") .with_exec(["./gradlew", ":airbyte-config:specs:downloadConnectorRegistry", "--rerun", "--build-cache", "--no-daemon"]) .with_exec(gradle_command + ["--scan"] if scan else gradle_command) - .with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) #TODO: Move this to a context manager + .with_exec(["rsync", "-az", "/root/.gradle/", "/root/gradle-cache"]) #TODO: Move this to a context manager ) await result.sync() if scan: @@ -74,8 +75,6 @@ async def build_oss_frontend_task(settings: OssSettings, ctx: PipelineContext, c .with_(load_settings(client, settings)) .with_mounted_cache("./build/airbyte-repository", airbyte_repo_cache, sharing=CacheSharingMode.LOCKED) .with_exec(["pnpm", "install"]) - .with_exec(["pnpm", "run", "license-check"]) - .with_exec(["pnpm", "run", "validate-lock"]) .with_exec(["pnpm", "build"])) await result.sync() return result @@ -121,28 +120,6 @@ async def build_storybook_oss_frontend_task(settings: OssSettings, ctx: Pipeline await result.sync() return result -@task -async def check_oss_backend_task(client: Client, oss_build_result: Container, settings: OssSettings, ctx: PipelineContext, scan: bool) -> Container: - gradle_command = ["./gradlew", "check", "-x", "test", "-x", ":airbyte-webapp:check","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"] - files_from_result = [ - "**/build.gradle", - "**/gradle.properties", - "**/settings.gradle", - "**/airbyte-*/**/*" - ] - result = ( - with_gradle(client, ctx, settings, directory=base_dir) - .with_directory("/root/.m2/repository", oss_build_result.directory("/root/.m2/repository")) # published jar files from mavenLocal - .with_directory("/airbyte/oss", oss_build_result.directory("/airbyte/oss"), include=files_from_result) - .with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte") - .with_(load_settings(client, settings)) - .with_env_variable("VERSION", "dev") - .with_env_variable("METRIC_CLIENT", "") # override 'datadog' value for metrics-lib test - .with_exec(gradle_command + ["--scan"] if scan else gradle_command) - .with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) - ) - await result.sync() - @task async def test_oss_backend_task(client: Client, oss_build_result: Container, settings: OssSettings, ctx: PipelineContext, scan: bool) -> Container: @@ -198,9 +175,9 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set .with_exec(["./run.sh"]) ) - gradle_command = ["./gradlew", "test", "-x", ":airbyte-webapp:test","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"] + gradle_command = ["./gradlew", "checK", "test", "-x", ":airbyte-webapp:test", "-x", "buildDockerImage", "--build-cache", "--no-daemon"] - result = ( + result = ( with_gradle(client, ctx, settings, directory=base_dir) .with_service_binding("airbyte-proxy-test-container", airbyte_proxy_service_auth) .with_service_binding("airbyte-proxy-test-container-newauth", airbyte_proxy_service_newauth) @@ -212,8 +189,11 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set .with_env_variable("VERSION", "dev") .with_env_variable("METRIC_CLIENT", "") # override 'datadog' value for metrics-lib test .with_exec(gradle_command + ["--scan"] if scan else gradle_command) - .with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) - ) + .with_exec(["rsync", "-az", "/root/.gradle/", "/root/gradle-cache"]) #TODO: Move this to a context manager) + .with_exec(["rsync", "-azm", "--include='*/'", "--include='jacocoTestReport.xml", "--exclude='*'", "/airbyte/oss", "/jacoco"]) # Collect jacoco reports + + + ) await result.sync() if scan: scan_file_contents: str = await result.file("/airbyte/oss/scan-journal.log").contents() @@ -223,4 +203,17 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set description="Gradle build scan for OSS Backend Tests", ) + + # TODO: We should move as much of these actions that interface with PR's to prefect Artifacts as we can + # instead of relying on CI only marketplace actions that assume that they are running in a CI environment + # This will allow us to run these actions locally as well + if settings.CI: + jacoco_inputs = GithubActionsInputSettings(settings, **{ + "paths": "/input/**/jacocoTestReport.xml", + "token": settings.GITHUB_TOKEN, + "debug-mode": "true" + }) + # TODO: Upload buildpulse here as well + await with_typescript_gha(client, result.directory("/jacoco"), "Madrapps/jacoco-report", "v1.6.1", jacoco_inputs) + return result From 55c74c13becc106c5b5f5f5b5f519027d5d43004 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 12 Sep 2023 17:21:55 +0200 Subject: [PATCH 147/201] Move airbyte.yml to configs directory (#8777) --- .gitignore | 5 +---- charts/airbyte/templates/airbyte-yml-configmap.yaml | 2 +- airbyte.sample.yml => configs/airbyte.sample.yml | 0 tools/bin/install_airbyte_pro_on_helm.sh | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) rename airbyte.sample.yml => configs/airbyte.sample.yml (100%) diff --git a/.gitignore b/.gitignore index 62657420c01..fa6dc52c190 100644 --- a/.gitignore +++ b/.gitignore @@ -63,9 +63,6 @@ crash.log resources/examples/airflow/logs/* !resources/examples/airflow/logs/.gitkeep -# Cloud Demo -!airbyte-webapp/src/packages/cloud/data - # Ignore docs folder, since we're using it to temporarily copy files into on CI /docs @@ -87,4 +84,4 @@ dd-java-agent.jar # Ignore airbyte.yml since this file contains user-provided information that is sensitive and should not be committed # See airbyte.sample.yml for an example of the expected file structure. -airbyte.yml +/configs/airbyte.yml diff --git a/charts/airbyte/templates/airbyte-yml-configmap.yaml b/charts/airbyte/templates/airbyte-yml-configmap.yaml index f42e8d03d5c..20a210371b2 100644 --- a/charts/airbyte/templates/airbyte-yml-configmap.yaml +++ b/charts/airbyte/templates/airbyte-yml-configmap.yaml @@ -11,7 +11,7 @@ metadata: labels: {{- include "airbyte.labels" . | nindent 4 }} data: - # helm install with `--set-file airbyteYml=../../airbyte.yml` to populate .Values.airbyteYml with the root-level airbyte.yml file + # helm install with `--set-file airbyteYml=../../configs/airbyte.yml` to populate .Values.airbyteYml with the airbyte.yml config file fileContents: |- {{ .Values.airbyteYml | nindent 4 }} {{- end }} \ No newline at end of file diff --git a/airbyte.sample.yml b/configs/airbyte.sample.yml similarity index 100% rename from airbyte.sample.yml rename to configs/airbyte.sample.yml diff --git a/tools/bin/install_airbyte_pro_on_helm.sh b/tools/bin/install_airbyte_pro_on_helm.sh index 1341d414d5a..1a0bd11466f 100755 --- a/tools/bin/install_airbyte_pro_on_helm.sh +++ b/tools/bin/install_airbyte_pro_on_helm.sh @@ -4,7 +4,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" # Now use this to get the relative path of the airbyte.yml file and airbyte-pro-values.yml file -airbyte_yml_file_path="$script_dir/../../airbyte.yml" +airbyte_yml_file_path="$script_dir/../../configs/airbyte.yml" airbyte_pro_values_yml_file_path="$script_dir/../../charts/airbyte/airbyte-pro-values.yaml" # Define the helm release name for this installation of Airbyte Pro. From 55710156aca6268f02d84cec6748f4168cf10573 Mon Sep 17 00:00:00 2001 From: timroes Date: Tue, 12 Sep 2023 15:29:49 +0000 Subject: [PATCH 148/201] Bump helm chart version reference to 0.48.10 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 28 +++++++++---------- charts/airbyte/Chart.yaml | 26 ++++++++--------- 14 files changed, 39 insertions(+), 39 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 7bbef5e9a7c..420113932d6 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 54023b03f22..512801e95a4 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index fb8cd0e6995..078ec94b5df 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 0d2e89a4e05..1013492c002 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 95da4185eb8..41e6c14016f 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 946a2ea6dd3..ffdd1d81b53 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index bbf4a568f08..af5e16530bd 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index ff39e925276..de1207ab75c 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index d6a94714df6..6285a9edd2a 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 954eeb96f95..43d7d9e2981 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 707219c455d..ed0cbce5b29 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 2ebba0bb57e..7ba642dc61b 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 8ffd7ce0879..f8cd00680c2 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,39 +4,39 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 -digest: sha256:fdf2dc5084a7fde5a769eecb3b90918c13cf864165db4d984f0c4c84237f0613 -generated: "2023-09-11T20:11:35.790563651Z" + version: 0.48.10 +digest: sha256:750055e63590c45e5644fce3716e4ae35b718e6ba16610fca70be700ae67be46 +generated: "2023-09-12T15:29:37.593008122Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 78a13dd5eb7..9d0e05aafbc 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.48.9 +version: 0.48.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,48 +32,48 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.48.9 + version: 0.48.10 From 6559f33bb16cd7c3a656cc9b9c27912e1df7255d Mon Sep 17 00:00:00 2001 From: Jonathan Pearlin Date: Tue, 12 Sep 2023 11:45:34 -0400 Subject: [PATCH 149/201] Allow override of orchestrator/source resources via feature flag (#8768) --- .../src/main/kotlin/FlagDefinitions.kt | 8 +- .../persistence/job/DefaultJobCreator.java | 67 ++++++-- .../job/DefaultJobCreatorTest.java | 149 +++++++++++++++++- 3 files changed, 202 insertions(+), 22 deletions(-) diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index 08f4f033557..1cfb7a43c07 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -103,6 +103,12 @@ object HideActorDefinitionFromList : Permanent(key = "connectors.hideAc object PauseSyncsWithUnsupportedActors : Temporary(key = "connectors.pauseSyncsWithUnsupportedActors", default = true) +object DestResourceOverrides : Temporary(key = "dest-resource-overrides", default = "") + +object OrchestratorResourceOverrides : Temporary(key = "orchestrator-resource-overrides", default = "") + +object SourceResourceOverrides : Temporary(key = "source-resource-overrides", default = "") + // NOTE: this is deprecated in favor of FieldSelectionEnabled and will be removed once that flag is fully deployed. object FieldSelectionWorkspaces : EnvVar(envVar = "FIELD_SELECTION_WORKSPACES") { override fun enabled(ctx: Context): Boolean { @@ -128,6 +134,4 @@ object FieldSelectionWorkspaces : EnvVar(envVar = "FIELD_SELECTION_WORKSPACES") object ConnectorOAuthConsentDisabled : Permanent(key = "connectors.oauth.disableOAuthConsent", default = false) object AddSchedulingJitter : Temporary(key = "platform.add-scheduling-jitter", default = false) - - object DestResourceOverrides : Temporary(key = "dest-resource-overrides", default = "") } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java index 104b2e9a6c9..5c36f3b50ac 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/DefaultJobCreator.java @@ -27,13 +27,15 @@ import io.airbyte.config.provider.ResourceRequirementsProvider; import io.airbyte.featureflag.Connection; import io.airbyte.featureflag.Context; +import io.airbyte.featureflag.DestResourceOverrides; import io.airbyte.featureflag.Destination; import io.airbyte.featureflag.DestinationDefinition; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.FieldSelectionWorkspaces.DestResourceOverrides; import io.airbyte.featureflag.Multi; +import io.airbyte.featureflag.OrchestratorResourceOverrides; import io.airbyte.featureflag.Source; import io.airbyte.featureflag.SourceDefinition; +import io.airbyte.featureflag.SourceResourceOverrides; import io.airbyte.featureflag.UseResourceRequirementsVariant; import io.airbyte.featureflag.Workspace; import io.airbyte.protocol.models.CatalogHelpers; @@ -183,7 +185,7 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac // destination based on the source to avoid oversizing orchestrator and destination when the source // is slow. final Optional sourceType = getSourceType(sourceDefinition); - final ResourceRequirements mergedOrchestratorResourceReq = getOrchestratorResourceRequirements(standardSync, sourceType, variant); + final ResourceRequirements mergedOrchestratorResourceReq = getOrchestratorResourceRequirements(standardSync, sourceType, variant, ffContext); final ResourceRequirements mergedDstResourceReq = getDestinationResourceRequirements(standardSync, destinationDefinition, sourceType, variant, ffContext); @@ -197,7 +199,7 @@ private SyncResourceRequirements getSyncResourceRequirements(final UUID workspac .withOrchestrator(mergedOrchestratorResourceReq); if (!isReset) { - final ResourceRequirements mergedSrcResourceReq = getSourceResourceRequirements(standardSync, sourceDefinition, variant); + final ResourceRequirements mergedSrcResourceReq = getSourceResourceRequirements(standardSync, sourceDefinition, variant, ffContext); syncResourceRequirements .withSource(mergedSrcResourceReq) .withSourceStdErr(resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE_STDERR, sourceType, variant)) @@ -230,35 +232,36 @@ private static void addIfNotNull(final List contextList, final UUID uui private ResourceRequirements getOrchestratorResourceRequirements(final StandardSync standardSync, final Optional sourceType, - final String variant) { + final String variant, + final Context ffContext) { final ResourceRequirements defaultOrchestratorRssReqs = resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.ORCHESTRATOR, sourceType, variant); - return ResourceRequirementsUtils.getResourceRequirements( + + final var mergedRrsReqs = ResourceRequirementsUtils.getResourceRequirements( standardSync.getResourceRequirements(), defaultOrchestratorRssReqs); + + final var overrides = getOrchestratorResourceOverrides(ffContext); + + return ResourceRequirementsUtils.getResourceRequirements(overrides, mergedRrsReqs); } private ResourceRequirements getSourceResourceRequirements(final StandardSync standardSync, final StandardSourceDefinition sourceDefinition, - final String variant) { + final String variant, + final Context ffContext) { final ResourceRequirements defaultSrcRssReqs = resourceRequirementsProvider.getResourceRequirements(ResourceRequirementsType.SOURCE, getSourceType(sourceDefinition), variant); - return ResourceRequirementsUtils.getResourceRequirements( + + final var mergedRssReqs = ResourceRequirementsUtils.getResourceRequirements( standardSync.getResourceRequirements(), sourceDefinition != null ? sourceDefinition.getResourceRequirements() : null, defaultSrcRssReqs, JobType.SYNC); - } - private ResourceRequirements getDestinationResourceOverrides(final Context ffCtx) { - final String destOverrides = featureFlagClient.stringVariation(DestResourceOverrides.INSTANCE, ffCtx); - try { - return ResourceRequirementsUtils.parse(destOverrides); - } catch (final Exception e) { - log.warn("Could not parse DESTINATION resource overrides from feature flag string: '{}'", destOverrides); - log.warn("Error parsing DESTINATION resource overrides: {}", e.getMessage()); - return null; - } + final var overrides = getSourceResourceOverrides(ffContext); + + return ResourceRequirementsUtils.getResourceRequirements(overrides, mergedRssReqs); } private ResourceRequirements getDestinationResourceRequirements(final StandardSync standardSync, @@ -280,6 +283,36 @@ private ResourceRequirements getDestinationResourceRequirements(final StandardSy return ResourceRequirementsUtils.getResourceRequirements(overrides, mergedRssReqs); } + private ResourceRequirements getDestinationResourceOverrides(final Context ffCtx) { + final String destOverrides = featureFlagClient.stringVariation(DestResourceOverrides.INSTANCE, ffCtx); + try { + return ResourceRequirementsUtils.parse(destOverrides); + } catch (final Exception e) { + log.warn("Could not parse DESTINATION resource overrides '{}' from feature flag string: {}", destOverrides, e.getMessage()); + return null; + } + } + + private ResourceRequirements getOrchestratorResourceOverrides(final Context ffCtx) { + final String orchestratorOverrides = featureFlagClient.stringVariation(OrchestratorResourceOverrides.INSTANCE, ffCtx); + try { + return ResourceRequirementsUtils.parse(orchestratorOverrides); + } catch (final Exception e) { + log.warn("Could not parse ORCHESTRATOR resource overrides '{}' from feature flag string: {}", orchestratorOverrides, e.getMessage()); + return null; + } + } + + private ResourceRequirements getSourceResourceOverrides(final Context ffCtx) { + final String sourceOverrides = featureFlagClient.stringVariation(SourceResourceOverrides.INSTANCE, ffCtx); + try { + return ResourceRequirementsUtils.parse(sourceOverrides); + } catch (final Exception e) { + log.warn("Could not parse SOURCE resource overrides '{}' from feature flag string: {}", sourceOverrides, e.getMessage()); + return null; + } + } + private Optional getSourceType(final StandardSourceDefinition sourceDefinition) { if (sourceDefinition == null) { return Optional.empty(); diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java index a4f40a369b5..745f88f6244 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/DefaultJobCreatorTest.java @@ -41,7 +41,9 @@ import io.airbyte.config.SyncResourceRequirements; import io.airbyte.config.SyncResourceRequirementsKey; import io.airbyte.config.provider.ResourceRequirementsProvider; -import io.airbyte.featureflag.FieldSelectionWorkspaces.DestResourceOverrides; +import io.airbyte.featureflag.DestResourceOverrides; +import io.airbyte.featureflag.OrchestratorResourceOverrides; +import io.airbyte.featureflag.SourceResourceOverrides; import io.airbyte.featureflag.TestClient; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; @@ -92,6 +94,7 @@ class DefaultJobCreatorTest { private static final StandardDestinationDefinition STANDARD_DESTINATION_DEFINITION; private static final ActorDefinitionVersion SOURCE_DEFINITION_VERSION; private static final ActorDefinitionVersion DESTINATION_DEFINITION_VERSION; + private static final ConfiguredAirbyteCatalog CONFIGURED_AIRBYTE_CATALOG; private static final long JOB_ID = 12L; private static final UUID WORKSPACE_ID = UUID.randomUUID(); @@ -151,7 +154,7 @@ class DefaultJobCreatorTest { .withStream(CatalogHelpers.createAirbyteStream(STREAM3_NAME, NAMESPACE, Field.of(FIELD_NAME, JsonSchemaType.STRING))) .withSyncMode(SyncMode.FULL_REFRESH) .withDestinationSyncMode(DestinationSyncMode.OVERWRITE); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(stream1, stream2, stream3)); + CONFIGURED_AIRBYTE_CATALOG = new ConfiguredAirbyteCatalog().withStreams(List.of(stream1, stream2, stream3)); STANDARD_SYNC = new StandardSync() .withConnectionId(connectionId) @@ -160,7 +163,7 @@ class DefaultJobCreatorTest { .withNamespaceFormat(null) .withPrefix("presto_to_hudi") .withStatus(StandardSync.Status.ACTIVE) - .withCatalog(catalog) + .withCatalog(CONFIGURED_AIRBYTE_CATALOG) .withSourceId(sourceId) .withDestinationId(destinationId) .withOperationIds(List.of(operationId)); @@ -581,6 +584,146 @@ void testDestinationResourceReqsOverrides(final String cpuReqOverride, assertEquals(expectedMemLimit, destConfigValues.getMemoryLimit()); } + @ParameterizedTest + @MethodSource("resourceOverrideMatrix") + void testOrchestratorResourceReqsOverrides(final String cpuReqOverride, + final String cpuLimitOverride, + final String memReqOverride, + final String memLimitOverride) + throws IOException { + final var overrides = new HashMap<>(); + if (cpuReqOverride != null) { + overrides.put("cpu_request", cpuReqOverride); + } + if (cpuLimitOverride != null) { + overrides.put("cpu_limit", cpuLimitOverride); + } + if (memReqOverride != null) { + overrides.put("memory_request", memReqOverride); + } + if (memLimitOverride != null) { + overrides.put("memory_limit", memLimitOverride); + } + + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(OrchestratorResourceOverrides.INSTANCE.getKey(), Jsons.serialize(overrides)))); + + final var standardSync = new StandardSync() + .withConnectionId(UUID.randomUUID()) + .withName("presto to hudi") + .withNamespaceDefinition(NamespaceDefinitionType.SOURCE) + .withNamespaceFormat(null) + .withPrefix("presto_to_hudi") + .withStatus(StandardSync.Status.ACTIVE) + .withCatalog(CONFIGURED_AIRBYTE_CATALOG) + .withSourceId(UUID.randomUUID()) + .withDestinationId(UUID.randomUUID()) + .withOperationIds(List.of(UUID.randomUUID())) + .withResourceRequirements(originalReqs); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + standardSync, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(sourceResourceRequirements)), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(destResourceRequirements)), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var orchestratorConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getOrchestrator(); + + final var expectedCpuReq = StringUtils.isNotBlank(cpuReqOverride) ? cpuReqOverride : originalReqs.getCpuRequest(); + assertEquals(expectedCpuReq, orchestratorConfigValues.getCpuRequest()); + + final var expectedCpuLimit = StringUtils.isNotBlank(cpuLimitOverride) ? cpuLimitOverride : originalReqs.getCpuLimit(); + assertEquals(expectedCpuLimit, orchestratorConfigValues.getCpuLimit()); + + final var expectedMemReq = StringUtils.isNotBlank(memReqOverride) ? memReqOverride : originalReqs.getMemoryRequest(); + assertEquals(expectedMemReq, orchestratorConfigValues.getMemoryRequest()); + + final var expectedMemLimit = StringUtils.isNotBlank(memLimitOverride) ? memLimitOverride : originalReqs.getMemoryLimit(); + assertEquals(expectedMemLimit, orchestratorConfigValues.getMemoryLimit()); + } + + @ParameterizedTest + @MethodSource("resourceOverrideMatrix") + void testSourceResourceReqsOverrides(final String cpuReqOverride, + final String cpuLimitOverride, + final String memReqOverride, + final String memLimitOverride) + throws IOException { + final var overrides = new HashMap<>(); + if (cpuReqOverride != null) { + overrides.put("cpu_request", cpuReqOverride); + } + if (cpuLimitOverride != null) { + overrides.put("cpu_limit", cpuLimitOverride); + } + if (memReqOverride != null) { + overrides.put("memory_request", memReqOverride); + } + if (memLimitOverride != null) { + overrides.put("memory_limit", memLimitOverride); + } + + final ResourceRequirements originalReqs = new ResourceRequirements() + .withCpuLimit("0.8") + .withCpuRequest("0.8") + .withMemoryLimit("800Mi") + .withMemoryRequest("800Mi"); + + final var jobCreator = new DefaultJobCreator(jobPersistence, resourceRequirementsProvider, + new TestClient(Map.of(SourceResourceOverrides.INSTANCE.getKey(), Jsons.serialize(overrides)))); + + jobCreator.createSyncJob( + SOURCE_CONNECTION, + DESTINATION_CONNECTION, + STANDARD_SYNC, + SOURCE_IMAGE_NAME, + SOURCE_PROTOCOL_VERSION, + DESTINATION_IMAGE_NAME, + DESTINATION_PROTOCOL_VERSION, + List.of(STANDARD_SYNC_OPERATION), + null, + new StandardSourceDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withJobSpecific(List.of( + new JobTypeResourceLimit().withJobType(JobType.SYNC).withResourceRequirements(originalReqs)))), + new StandardDestinationDefinition().withResourceRequirements(new ActorDefinitionResourceRequirements().withDefault(destResourceRequirements)), + SOURCE_DEFINITION_VERSION, + DESTINATION_DEFINITION_VERSION, + WORKSPACE_ID); + + final ArgumentCaptor configCaptor = ArgumentCaptor.forClass(JobConfig.class); + verify(jobPersistence, times(1)).enqueueJob(any(), configCaptor.capture()); + final var sourceConfigValues = configCaptor.getValue().getSync().getSyncResourceRequirements().getSource(); + + final var expectedCpuReq = StringUtils.isNotBlank(cpuReqOverride) ? cpuReqOverride : originalReqs.getCpuRequest(); + assertEquals(expectedCpuReq, sourceConfigValues.getCpuRequest()); + + final var expectedCpuLimit = StringUtils.isNotBlank(cpuLimitOverride) ? cpuLimitOverride : originalReqs.getCpuLimit(); + assertEquals(expectedCpuLimit, sourceConfigValues.getCpuLimit()); + + final var expectedMemReq = StringUtils.isNotBlank(memReqOverride) ? memReqOverride : originalReqs.getMemoryRequest(); + assertEquals(expectedMemReq, sourceConfigValues.getMemoryRequest()); + + final var expectedMemLimit = StringUtils.isNotBlank(memLimitOverride) ? memLimitOverride : originalReqs.getMemoryLimit(); + assertEquals(expectedMemLimit, sourceConfigValues.getMemoryLimit()); + } + private static Stream resourceOverrideMatrix() { return Stream.of( Arguments.of("0.7", "0.4", "1000Mi", "2000Mi"), From 99162c46a91b90f9b66a13080cf49999a9e1732d Mon Sep 17 00:00:00 2001 From: keyihuang Date: Tue, 12 Sep 2023 09:55:35 -0700 Subject: [PATCH 150/201] Sunset FCP (Phase 1) (#8580) --- airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt index 1cfb7a43c07..bd46be4f19b 100644 --- a/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt +++ b/airbyte-featureflag/src/main/kotlin/FlagDefinitions.kt @@ -103,6 +103,8 @@ object HideActorDefinitionFromList : Permanent(key = "connectors.hideAc object PauseSyncsWithUnsupportedActors : Temporary(key = "connectors.pauseSyncsWithUnsupportedActors", default = true) +object SunsetFCP : Temporary(key = "platform.sunset-fcp", default = false) + object DestResourceOverrides : Temporary(key = "dest-resource-overrides", default = "") object OrchestratorResourceOverrides : Temporary(key = "orchestrator-resource-overrides", default = "") From 18420b5325755b33472366193c0e6a812d04a030 Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Tue, 12 Sep 2023 10:08:07 -0700 Subject: [PATCH 151/201] Connection manager utils temporal error handling (#8772) --- .../temporal/ConnectionManagerUtils.java | 54 +++++++--------- .../commons/temporal/TemporalClient.java | 26 ++++---- .../temporal/WorkflowClientWrapped.java | 61 ++++++++++++++++++- .../commons/temporal/TemporalClientTest.java | 7 +-- .../temporal/WorkflowClientWrappedTest.java | 40 +++++++++++- 5 files changed, 137 insertions(+), 51 deletions(-) diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java index 19bab81221d..b1cc60a6f03 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/ConnectionManagerUtils.java @@ -47,11 +47,10 @@ public ConnectionManagerUtils(final WorkflowClient workflowClient, final MetricC * Send a cancellation to the workflow. It will swallow any exception and won't check if the * workflow is already deleted when being cancel. */ - public void deleteWorkflowIfItExist(final WorkflowClient client, - final UUID connectionId) { + public void deleteWorkflowIfItExist(final UUID connectionId) { try { final ConnectionManagerWorkflow connectionManagerWorkflow = - client.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); + workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); connectionManagerWorkflow.deleteConnection(); } catch (final Exception e) { log.warn("The workflow is not reachable when trying to cancel it", e); @@ -66,18 +65,16 @@ public void deleteWorkflowIfItExist(final WorkflowClient client, * batched request. Batching is used to avoid race conditions between starting the workflow and * executing the signal. * - * @param client the WorkflowClient for interacting with temporal * @param connectionId the connection ID to execute this operation for * @param signalMethod a function that takes in a connection manager workflow and executes a signal * method on it, with no arguments * @return the healthy connection manager workflow that was signaled * @throws DeletedWorkflowException if the connection manager workflow was deleted */ - public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, + public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final UUID connectionId, final Function signalMethod) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.empty()); + return signalWorkflowAndRepairIfNecessary(connectionId, signalMethod, Optional.empty()); } /** @@ -87,7 +84,6 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Workfl * batched request. Batching is used to avoid race conditions between starting the workflow and * executing the signal. * - * @param client the WorkflowClient for interacting with temporal * @param connectionId the connection ID to execute this operation for * @param signalMethod a function that takes in a connection manager workflow and executes a signal * method on it, with 1 argument @@ -95,12 +91,11 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Workfl * @return the healthy connection manager workflow that was signaled * @throws DeletedWorkflowException if the connection manager workflow was deleted */ - public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, + public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final UUID connectionId, final Function> signalMethod, final T signalArgument) throws DeletedWorkflowException { - return signalWorkflowAndRepairIfNecessary(client, connectionId, signalMethod, Optional.of(signalArgument)); + return signalWorkflowAndRepairIfNecessary(connectionId, signalMethod, Optional.of(signalArgument)); } // This method unifies the logic of the above two, by using the optional signalArgument parameter to @@ -109,8 +104,7 @@ public ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final Wo // type enforcement for external calls, and means this method can assume consistent type // implementations for both cases. @SuppressWarnings({"MissingJavadocMethod", "LineLength"}) - private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final WorkflowClient client, - final UUID connectionId, + private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final UUID connectionId, final Function signalMethod, final Optional signalArgument) throws DeletedWorkflowException { @@ -138,12 +132,12 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W // in case there is an existing workflow in a bad state, attempt to terminate it first before // starting a new workflow - safeTerminateWorkflow(client, connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); + safeTerminateWorkflow(connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); - final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); + final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(connectionId); final ConnectionUpdaterInput startWorkflowInput = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); - final BatchRequest batchRequest = client.newSignalWithStartRequest(); + final BatchRequest batchRequest = workflowClientWrapped.newSignalWithStartRequest(); batchRequest.add(connectionManagerWorkflow::run, startWorkflowInput); // retrieve the signal from the lambda @@ -155,17 +149,17 @@ private ConnectionManagerWorkflow signalWorkflowAndRepairIfNecessary(final W batchRequest.add((Proc) signal); } - client.signalWithStart(batchRequest); + workflowClientWrapped.signalWithStart(batchRequest); log.info("Connection manager workflow for connection {} has been started and signaled.", connectionId); return connectionManagerWorkflow; } } - void safeTerminateWorkflow(final WorkflowClient client, final String workflowId, final String reason) { + void safeTerminateWorkflow(final String workflowId, final String reason) { log.info("Attempting to terminate existing workflow for workflowId {}.", workflowId); try { - client.newUntypedWorkflowStub(workflowId).terminate(reason); + workflowClientWrapped.terminateWorkflow(workflowId, reason); } catch (final Exception e) { log.warn( "Could not terminate temporal workflow due to the following error; " @@ -177,25 +171,23 @@ void safeTerminateWorkflow(final WorkflowClient client, final String workflowId, /** * Terminate a temporal workflow and throw a useful exception. * - * @param client temporal workflow client * @param connectionId connection id * @param reason reason for terminating the workflow */ // todo (cgardens) - what makes this safe - public void safeTerminateWorkflow(final WorkflowClient client, final UUID connectionId, final String reason) { - safeTerminateWorkflow(client, getConnectionManagerName(connectionId), reason); + public void safeTerminateWorkflow(final UUID connectionId, final String reason) { + safeTerminateWorkflow(getConnectionManagerName(connectionId), reason); } /** * Start a connection manager workflow for a connection. * - * @param client temporal workflow client * @param connectionId connection id * @return new connection manager workflow */ // todo (cgardens) - what does no signal mean in this context? - public ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, final UUID connectionId) { - final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); + public ConnectionManagerWorkflow startConnectionManagerNoSignal(final UUID connectionId) { + final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(connectionId); final ConnectionUpdaterInput input = TemporalWorkflowUtils.buildStartWorkflowInput(connectionId); WorkflowClient.start(connectionManagerWorkflow::run, input); @@ -241,9 +233,9 @@ public ConnectionManagerWorkflow getConnectionManagerWorkflow(final UUID connect return connectionManagerWorkflow; } - Optional getWorkflowState(final WorkflowClient client, final UUID connectionId) { + Optional getWorkflowState(final UUID connectionId) { try { - final ConnectionManagerWorkflow connectionManagerWorkflow = client.newWorkflowStub(ConnectionManagerWorkflow.class, + final ConnectionManagerWorkflow connectionManagerWorkflow = workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, getConnectionManagerName(connectionId)); return Optional.of(connectionManagerWorkflow.getState()); } catch (final Exception e) { @@ -252,8 +244,8 @@ Optional getWorkflowState(final WorkflowClient client, final UUID } } - boolean isWorkflowStateRunning(final WorkflowClient client, final UUID connectionId) { - return getWorkflowState(client, connectionId).map(WorkflowState::isRunning).orElse(false); + boolean isWorkflowStateRunning(final UUID connectionId) { + return getWorkflowState(connectionId).map(WorkflowState::isRunning).orElse(false); } /** @@ -290,8 +282,8 @@ public long getCurrentJobId(final UUID connectionId) { } } - public ConnectionManagerWorkflow newConnectionManagerWorkflowStub(final WorkflowClient client, final UUID connectionId) { - return client.newWorkflowStub(ConnectionManagerWorkflow.class, + private ConnectionManagerWorkflow newConnectionManagerWorkflowStub(final UUID connectionId) { + return workflowClientWrapped.newWorkflowStub(ConnectionManagerWorkflow.class, TemporalWorkflowUtils.buildWorkflowOptions(TemporalJobType.CONNECTION_UPDATER, getConnectionManagerName(connectionId))); } diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java index 87dabb26ea6..9fcb4cf8908 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/TemporalClient.java @@ -113,9 +113,9 @@ public int restartClosedWorkflowByStatus(final WorkflowExecutionStatus execution final Set nonRunningWorkflow = filterOutRunningWorkspaceId(workflowExecutionInfos); nonRunningWorkflow.forEach(connectionId -> { - connectionManagerUtils.safeTerminateWorkflow(client, connectionId, + connectionManagerUtils.safeTerminateWorkflow(connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + connectionManagerUtils.startConnectionManagerNoSignal(connectionId); }); return nonRunningWorkflow.size(); @@ -210,7 +210,7 @@ public static class ManualOperationResult { } public Optional getWorkflowState(final UUID connectionId) { - return connectionManagerUtils.getWorkflowState(client, connectionId); + return connectionManagerUtils.getWorkflowState(connectionId); } /** @@ -222,7 +222,7 @@ public Optional getWorkflowState(final UUID connectionId) { public ManualOperationResult startNewManualSync(final UUID connectionId) { log.info("Manual sync request"); - if (connectionManagerUtils.isWorkflowStateRunning(client, connectionId)) { + if (connectionManagerUtils.isWorkflowStateRunning(connectionId)) { // TODO Bmoric: Error is running return new ManualOperationResult( Optional.of("A sync is already running for: " + connectionId), @@ -230,7 +230,7 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { } try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::submitManualSync); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::submitManualSync); } catch (final DeletedWorkflowException e) { log.error("Can't sync a deleted connection.", e); return new ManualOperationResult( @@ -246,7 +246,7 @@ public ManualOperationResult startNewManualSync(final UUID connectionId) { Optional.of("Didn't managed to start a sync for: " + connectionId), Optional.empty(), Optional.of(ErrorCode.UNKNOWN)); } - } while (!connectionManagerUtils.isWorkflowStateRunning(client, connectionId)); + } while (!connectionManagerUtils.isWorkflowStateRunning(connectionId)); log.info("end of manual schedule"); @@ -269,7 +269,7 @@ public ManualOperationResult startNewCancellation(final UUID connectionId) { final long jobId = connectionManagerUtils.getCurrentJobId(connectionId); try { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::cancelJob); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::cancelJob); } catch (final DeletedWorkflowException e) { log.error("Can't cancel a deleted workflow", e); return new ManualOperationResult( @@ -285,7 +285,7 @@ public ManualOperationResult startNewCancellation(final UUID connectionId) { Optional.of("Didn't manage to cancel a sync for: " + connectionId), Optional.empty(), Optional.of(ErrorCode.UNKNOWN)); } - } while (connectionManagerUtils.isWorkflowStateRunning(client, connectionId)); + } while (connectionManagerUtils.isWorkflowStateRunning(connectionId)); streamResetRecordsHelper.deleteStreamResetRecordsForJob(jobId, connectionId); @@ -323,9 +323,9 @@ public ManualOperationResult resetConnection(final UUID connectionId, try { if (syncImmediatelyAfter) { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::resetConnectionAndSkipNextScheduling); } else { - connectionManagerUtils.signalWorkflowAndRepairIfNecessary(client, connectionId, workflow -> workflow::resetConnection); + connectionManagerUtils.signalWorkflowAndRepairIfNecessary(connectionId, workflow -> workflow::resetConnection); } } catch (final DeletedWorkflowException e) { log.error("Can't reset a deleted workflow", e); @@ -522,7 +522,7 @@ private T getWorkflowStubWithTaskQueue(final Class workflowClass, final S public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connectionId) { log.info("Starting the scheduler temporal wf"); final ConnectionManagerWorkflow connectionManagerWorkflow = - connectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); + connectionManagerUtils.startConnectionManagerNoSignal(connectionId); try { CompletableFuture.supplyAsync(() -> { try { @@ -549,7 +549,7 @@ public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connect * @param connectionId - connectionId to cancel */ public void forceDeleteWorkflow(final UUID connectionId) { - connectionManagerUtils.deleteWorkflowIfItExist(client, connectionId); + connectionManagerUtils.deleteWorkflowIfItExist(connectionId); } public void sendSchemaChangeNotification(final UUID connectionId, @@ -579,7 +579,7 @@ public void update(final UUID connectionId) { log.error( String.format("Failed to retrieve ConnectionManagerWorkflow for connection %s. Repairing state by creating new workflow.", connectionId), e); - connectionManagerUtils.safeTerminateWorkflow(client, connectionId, + connectionManagerUtils.safeTerminateWorkflow(connectionId, "Terminating workflow in unreachable state before starting a new workflow for this connection"); submitConnectionUpdaterAsync(connectionId); return; diff --git a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java index a08e4579a7d..7a1ee177de2 100644 --- a/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java +++ b/airbyte-commons-temporal/src/main/java/io/airbyte/commons/temporal/WorkflowClientWrapped.java @@ -4,6 +4,7 @@ package io.airbyte.commons.temporal; +import com.google.common.annotations.VisibleForTesting; import dev.failsafe.Failsafe; import dev.failsafe.RetryPolicy; import dev.failsafe.function.CheckedSupplier; @@ -13,9 +14,12 @@ import io.airbyte.metrics.lib.OssMetricsRegistry; import io.grpc.Status; import io.grpc.StatusRuntimeException; +import io.temporal.api.common.v1.WorkflowExecution; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest; import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; +import io.temporal.client.BatchRequest; import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; import java.time.Duration; /** @@ -26,14 +30,29 @@ public class WorkflowClientWrapped { private static final int DEFAULT_MAX_ATTEMPT = 3; + private static final int DEFAULT_BACKOFF_DELAY_IN_MILLIS = 1000; + private static final int DEFAULT_BACKOFF_MAX_DELAY_IN_MILLIS = 10000; private final WorkflowClient temporalWorkflowClient; private final MetricClient metricClient; private final int maxAttempt; + private final int backoffDelayInMillis; + private final int backoffMaxDelayInMillis; public WorkflowClientWrapped(final WorkflowClient workflowClient, final MetricClient metricClient) { + this(workflowClient, metricClient, DEFAULT_MAX_ATTEMPT, DEFAULT_BACKOFF_DELAY_IN_MILLIS, DEFAULT_BACKOFF_MAX_DELAY_IN_MILLIS); + } + + @VisibleForTesting + WorkflowClientWrapped(final WorkflowClient workflowClient, + final MetricClient metricClient, + int maxAttempt, + int backoffDelayInMillis, + int backoffMaxDelayInMillis) { this.temporalWorkflowClient = workflowClient; this.metricClient = metricClient; - this.maxAttempt = DEFAULT_MAX_ATTEMPT; + this.maxAttempt = maxAttempt; + this.backoffDelayInMillis = backoffDelayInMillis; + this.backoffMaxDelayInMillis = backoffMaxDelayInMillis; } /** @@ -51,6 +70,14 @@ public T newWorkflowStub(final Class workflowInterface, final String work return withRetries(() -> temporalWorkflowClient.newWorkflowStub(workflowInterface, workflowId), "newWorkflowStub"); } + /** + * Creates workflow client stub for a known execution. Use it to send signals or queries to a + * running workflow. + */ + public T newWorkflowStub(final Class workflowInterface, final WorkflowOptions workflowOptions) { + return withRetries(() -> temporalWorkflowClient.newWorkflowStub(workflowInterface, workflowOptions), "newWorkflowStub"); + } + /** * Returns information about the specified workflow execution. */ @@ -59,6 +86,36 @@ public DescribeWorkflowExecutionResponse blockingDescribeWorkflowExecution(final "describeWorkflowExecution"); } + /** + * Terminates a workflow execution. Termination is a hard stop of a workflow execution which doesn't + * give workflow code any chance to perform cleanup. + */ + public void terminateWorkflow(final String workflowId, final String reason) { + withRetries(() -> { + temporalWorkflowClient.newUntypedWorkflowStub(workflowId).terminate(reason); + return null; + }, + "terminate"); + } + + /** + * Creates BatchRequest that can be used to signal an existing workflow or start a new one if not + * running. The batch before invocation must contain exactly two operations. One annotated + * with @WorkflowMethod and another with @SignalMethod. + */ + public BatchRequest newSignalWithStartRequest() { + // NOTE we do not retry here because signals are not idempotent + return temporalWorkflowClient.newSignalWithStartRequest(); + } + + /** + * Invoke SignalWithStart operation. + */ + public WorkflowExecution signalWithStart(final BatchRequest batchRequest) { + // NOTE we do not retry here because signals are not idempotent + return temporalWorkflowClient.signalWithStart(batchRequest); + } + /** * Where the magic happens. *

@@ -71,7 +128,7 @@ private T withRetries(final CheckedSupplier call, final String name) { final var retry = RetryPolicy.builder() .handleIf(this::shouldRetry) .withMaxAttempts(maxAttempt) - .withBackoff(Duration.ofSeconds(1), Duration.ofSeconds(10)) + .withBackoff(Duration.ofMillis(backoffDelayInMillis), Duration.ofMillis(backoffMaxDelayInMillis)) .onRetry((a) -> metricClient.count(OssMetricsRegistry.TEMPORAL_API_TRANSIENT_ERROR_RETRY, 1, new MetricAttribute(MetricTags.ATTEMPT_NUMBER, String.valueOf(a.getAttemptCount())), new MetricAttribute(MetricTags.FAILURE_ORIGIN, name), diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java index 559cfea0a93..8c694b01dba 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/TemporalClientTest.java @@ -161,9 +161,8 @@ void testRestartFailed() { .when(temporalClient).filterOutRunningWorkspaceId(workflowIds); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); temporalClient.restartClosedWorkflowByStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_FAILED); - verify(mConnectionManagerUtils).safeTerminateWorkflow(eq(workflowClient), eq(connectionId), - anyString()); - verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(workflowClient), eq(connectionId)); + verify(mConnectionManagerUtils).safeTerminateWorkflow(eq(connectionId), anyString()); + verify(mConnectionManagerUtils).startConnectionManagerNoSignal(eq(connectionId)); } } @@ -309,7 +308,7 @@ class ForceCancelConnection { void testForceCancelConnection() { temporalClient.forceDeleteWorkflow(CONNECTION_ID); - verify(connectionManagerUtils).deleteWorkflowIfItExist(workflowClient, CONNECTION_ID); + verify(connectionManagerUtils).deleteWorkflowIfItExist(CONNECTION_ID); } } diff --git a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java index dbf40afd726..400c679a1d3 100644 --- a/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java +++ b/airbyte-commons-temporal/src/test/java/io/airbyte/commons/temporal/WorkflowClientWrappedTest.java @@ -21,6 +21,8 @@ import io.temporal.api.workflowservice.v1.DescribeWorkflowExecutionResponse; import io.temporal.api.workflowservice.v1.WorkflowServiceGrpc.WorkflowServiceBlockingStub; import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowOptions; +import io.temporal.client.WorkflowStub; import io.temporal.serviceclient.WorkflowServiceStubs; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -29,6 +31,9 @@ class WorkflowClientWrappedTest { static class MyWorkflow {} + private static final int maxAttempt = 3; + private static final int backoffDelayInMillis = 1; + private static final int backoffMaxDelayInMillis = 10; private MetricClient metricClient; private WorkflowServiceStubs temporalWorkflowServiceStubs; private WorkflowServiceBlockingStub temporalWorkflowServiceBlockingStub; @@ -43,7 +48,7 @@ void beforeEach() { when(temporalWorkflowServiceStubs.blockingStub()).thenReturn(temporalWorkflowServiceBlockingStub); temporalWorkflowClient = mock(WorkflowClient.class); when(temporalWorkflowClient.getWorkflowServiceStubs()).thenReturn(temporalWorkflowServiceStubs); - workflowClient = new WorkflowClientWrapped(temporalWorkflowClient, metricClient); + workflowClient = new WorkflowClientWrapped(temporalWorkflowClient, metricClient, maxAttempt, backoffDelayInMillis, backoffMaxDelayInMillis); } @Test @@ -67,6 +72,29 @@ void testNewWorkflowStub() { assertEquals(expected, actual); } + @Test + void testNewWorkflowStubWithOptions() { + final MyWorkflow expected = new MyWorkflow(); + when(temporalWorkflowClient.newWorkflowStub(any(), (WorkflowOptions) any())) + .thenThrow(unavailable()) + .thenReturn(expected); + + final MyWorkflow actual = workflowClient.newWorkflowStub(MyWorkflow.class, WorkflowOptions.getDefaultInstance()); + assertEquals(expected, actual); + } + + @Test + void testTerminateWorkflow() { + final var workflowStub = mock(WorkflowStub.class); + when(temporalWorkflowClient.newUntypedWorkflowStub(anyString())) + .thenThrow(unavailable()) + .thenReturn(workflowStub); + + workflowClient.terminateWorkflow("workflow", "test terminate"); + verify(temporalWorkflowClient, times(2)).newUntypedWorkflowStub("workflow"); + verify(workflowStub).terminate("test terminate"); + } + @Test void testBlockingDescribeWorkflowExecution() { final DescribeWorkflowExecutionResponse expected = mock(DescribeWorkflowExecutionResponse.class); @@ -78,6 +106,16 @@ void testBlockingDescribeWorkflowExecution() { assertEquals(expected, actual); } + @Test + void testSignalsAreNotRetried() { + when(temporalWorkflowClient.signalWithStart(any())).thenThrow(unavailable()); + assertThrows(StatusRuntimeException.class, () -> { + final var request = workflowClient.newSignalWithStartRequest(); + workflowClient.signalWithStart(request); + }); + verify(temporalWorkflowClient, times(1)).signalWithStart(any()); + } + private static StatusRuntimeException unavailable() { return new StatusRuntimeException(Status.UNAVAILABLE); } From b12a5f7b259d9d15ab5e6d0a01485b11b8dd7895 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Tue, 12 Sep 2023 15:18:20 -0400 Subject: [PATCH 152/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8F=81=20(flagged)?= =?UTF-8?q?=20Adjust=20wording=20of=20low=20credits=20hints=20after=20FCP?= =?UTF-8?q?=20sunset=20(#8776)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/locales/en.json | 6 ++++-- .../LowCreditBalanceHint/LowCreditBalanceHint.tsx | 13 +++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index e0d11cbdf1d..80ecc64f673 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -1310,8 +1310,10 @@ "credits.emailVerificationRequired": "You need to verify your email address before you can buy credits.", "credits.emailVerification.resendConfirmation": "We sent you a new verification link.", "credits.emailVerification.resend": "Send verification link again", - "credits.lowBalance": "Your credit balance is low. You need to add more credits to prevent your GA syncs from being stopped.", - "credits.zeroBalance": "All your GA syncs have been stopped because your credit balance is 0. Add more credits to enable your GA syncs again.", + "credits.lowBalance.fcp": "Your credit balance is low. You need to add more credits to prevent your GA syncs from being stopped.", + "credits.zeroBalance.fcp": "All your GA syncs have been stopped because your credit balance is 0. Add more credits to enable your GA syncs again.", + "credits.lowBalance": "Your credit balance is low. You need to add more credits to prevent your syncs from being stopped.", + "credits.zeroBalance": "All your syncs have been stopped because your credit balance is 0. Add more credits to allow enabling your syncs again.", "firebase.auth.error.invalidPassword": "Incorrect password", "firebase.auth.error.networkRequestFailed": "There appears to be a network issue. Please try again later.", diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/LowCreditBalanceHint/LowCreditBalanceHint.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/LowCreditBalanceHint/LowCreditBalanceHint.tsx index a061326a30c..116a5d4c41b 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/LowCreditBalanceHint/LowCreditBalanceHint.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/LowCreditBalanceHint/LowCreditBalanceHint.tsx @@ -2,6 +2,8 @@ import { FormattedMessage } from "react-intl"; import { Message } from "components/ui/Message"; +import { useIsFCPEnabled } from "core/api/cloud"; + interface LowCreditBalanceHintProps { variant: "info" | "warning" | "error"; } @@ -9,14 +11,13 @@ interface LowCreditBalanceHintProps { export const LOW_BALANCE_CREDIT_THRESHOLD = 20; export const LowCreditBalanceHint: React.FC> = ({ variant }) => { + const isFCPEnabled = useIsFCPEnabled(); if (variant === "info") { return null; } - return ( - } - type={variant} - /> - ); + const zeroBalanceId = isFCPEnabled ? "credits.zeroBalance.fcp" : "credits.zeroBalance"; + const lowBalanceId = isFCPEnabled ? "credits.lowBalance.fcp" : "credits.lowBalance"; + + return } type={variant} />; }; From 52a80f44d1519d9541bda6807e7f74b1189ec7db Mon Sep 17 00:00:00 2001 From: Cole Snodgrass Date: Tue, 12 Sep 2023 13:49:39 -0700 Subject: [PATCH 153/201] add inNamespace to log call (#8804) --- .../workers/process/AsyncOrchestratorPodProcess.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java index 89b8148df84..d09af1922ee 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/process/AsyncOrchestratorPodProcess.java @@ -202,7 +202,12 @@ private boolean isOOM(final Pod pod) { return false; } try { - return kubernetesClient.pods().withName(pod.getFullResourceName()).tailingLines(5).getLog().contains(JAVA_OOM_EXCEPTION_STRING); + return kubernetesClient.pods() + .inNamespace(pod.getMetadata().getNamespace()) + .withName(pod.getFullResourceName()) + .tailingLines(5) + .getLog() + .contains(JAVA_OOM_EXCEPTION_STRING); } catch (final Exception e) { // We are trying to detect if the pod OOMed, if we fail to check the logs, we don't want to add more // exception noise since this is an extra inspection attempt for more precise error logging. From 2d1806f2a90d3fb9e68fd83c12b201d6fd6c0daa Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Tue, 12 Sep 2023 15:33:51 -0700 Subject: [PATCH 154/201] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8F=81=20Switch=20?= =?UTF-8?q?to=20new=20connector=20certification=20badges,=20behind=20featu?= =?UTF-8?q?re=20flag=20(#8695)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ben Church --- .../cypress/pages/createConnectorPage.ts | 2 +- .../ConnectorCard/ConnectorCard.module.scss | 55 ------ .../ConnectorCard/ConnectorCard.tsx | 40 ----- .../src/components/ConnectorCard/index.tsx | 1 - .../ConnectorIcon/ConnectorIcon.stories.tsx | 9 - .../ConnectionInfoCard.module.scss | 55 ------ .../ConnectionInfoCard.test.tsx | 103 ------------ .../ConnectionInfoCard/ConnectionInfoCard.tsx | 79 --------- .../SchemaChangesDetected.module.scss | 21 --- .../SchemaChangesDetected.test.tsx | 105 ------------ .../SchemaChangesDetected.tsx | 52 ------ .../connection/ConnectionInfoCard/index.ts | 1 - .../CreateConnectionForm.test.tsx | 7 +- .../EnabledControl.module.scss | 0 .../EnabledControl.tsx | 0 .../connection/EnabledControl/index.ts | 1 + .../connector/ConnectorTitleBlock.tsx | 25 +-- .../DestinationForm/DestinationForm.tsx | 9 +- airbyte-webapp/src/components/index.tsx | 1 - .../ConnectorButton.module.scss | 2 +- .../SelectConnector/ConnectorButton.tsx | 10 +- .../FilterSupportLevel.module.scss | 8 + .../SelectConnector/FilterSupportLevel.tsx | 72 ++++++++ .../SelectConnector/SelectConnector.tsx | 157 +++++++++++++++++- .../ConnectorDefinitionBranding.tsx | 14 +- .../src/components/ui/Icon/Icon.tsx | 4 +- .../icons/{gAIcon.svg => certifiedIcon.svg} | 2 +- .../src/components/ui/Icon/types.ts | 2 +- .../SupportLevelBadge.module.scss | 27 +++ .../SupportLevelBadge.stories.tsx | 25 +++ .../SupportLevelBadge/SupportLevelBadge.tsx | 85 ++++++++++ .../components/ui/SupportLevelBadge/index.ts | 1 + .../core/api/hooks/actorDefinitionVersions.ts | 17 +- .../src/core/domain/Documentation.ts | 6 +- .../src/core/utils/useLocalStorage.ts | 3 +- .../ConnectionEditService.test.tsx | 7 +- .../ConnectionFormService.test.tsx | 7 +- .../ConnectionForm/ConnectionFormService.tsx | 8 +- .../src/hooks/services/useDocumentation.ts | 6 +- airbyte-webapp/src/locales/en.json | 6 + .../components/ConnectorOptionLabel.tsx | 9 +- .../components/CreditsUsageContext.tsx | 6 +- .../components/UsagePerConnectionTable.tsx | 16 +- ...lculateAvailableSourcesAndDestinations.tsx | 4 + .../components/ConnectorCell.tsx | 24 ++- .../components/ConnectorsView.tsx | 14 +- .../components/SourceUpdateIndicator.tsx | 9 +- .../components/VersionCell/VersionCell.tsx | 14 +- .../ConnectionPage/ConnectionTitleBlock.tsx | 33 +++- .../ConnectionReplicationPage.test.tsx | 7 +- .../CreateConnectionTitleBlock.tsx | 36 ++-- .../DestinationSettingsPage.tsx | 4 +- .../source/CreateSourcePage/SourceForm.tsx | 9 +- .../SourceSettingsPage/SourceSettingsPage.tsx | 4 +- .../test-utils/mock-data/mockDestination.ts | 3 + .../mockFrequentlyUsedDestinations.ts | 67 -------- .../src/test-utils/mock-data/mockSource.ts | 21 ++- .../mock-data/mockStartWithDestination.ts | 10 -- .../Connector/ConnectorCard/ConnectorCard.tsx | 10 +- .../DocumentationPanel.tsx | 4 +- .../components/WarningMessage.tsx | 20 +-- .../views/Connector/ConnectorForm/types.ts | 13 -- 62 files changed, 636 insertions(+), 736 deletions(-) delete mode 100644 airbyte-webapp/src/components/ConnectorCard/ConnectorCard.module.scss delete mode 100644 airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx delete mode 100644 airbyte-webapp/src/components/ConnectorCard/index.tsx delete mode 100644 airbyte-webapp/src/components/connection/ConnectionInfoCard/ConnectionInfoCard.module.scss delete mode 100644 airbyte-webapp/src/components/connection/ConnectionInfoCard/ConnectionInfoCard.test.tsx delete mode 100644 airbyte-webapp/src/components/connection/ConnectionInfoCard/ConnectionInfoCard.tsx delete mode 100644 airbyte-webapp/src/components/connection/ConnectionInfoCard/SchemaChangesDetected.module.scss delete mode 100644 airbyte-webapp/src/components/connection/ConnectionInfoCard/SchemaChangesDetected.test.tsx delete mode 100644 airbyte-webapp/src/components/connection/ConnectionInfoCard/SchemaChangesDetected.tsx delete mode 100644 airbyte-webapp/src/components/connection/ConnectionInfoCard/index.ts rename airbyte-webapp/src/components/connection/{ConnectionInfoCard => EnabledControl}/EnabledControl.module.scss (100%) rename airbyte-webapp/src/components/connection/{ConnectionInfoCard => EnabledControl}/EnabledControl.tsx (100%) create mode 100644 airbyte-webapp/src/components/connection/EnabledControl/index.ts create mode 100644 airbyte-webapp/src/components/source/SelectConnector/FilterSupportLevel.module.scss create mode 100644 airbyte-webapp/src/components/source/SelectConnector/FilterSupportLevel.tsx rename airbyte-webapp/src/components/ui/Icon/icons/{gAIcon.svg => certifiedIcon.svg} (96%) create mode 100644 airbyte-webapp/src/components/ui/SupportLevelBadge/SupportLevelBadge.module.scss create mode 100644 airbyte-webapp/src/components/ui/SupportLevelBadge/SupportLevelBadge.stories.tsx create mode 100644 airbyte-webapp/src/components/ui/SupportLevelBadge/SupportLevelBadge.tsx create mode 100644 airbyte-webapp/src/components/ui/SupportLevelBadge/index.ts delete mode 100644 airbyte-webapp/src/test-utils/mock-data/mockFrequentlyUsedDestinations.ts delete mode 100644 airbyte-webapp/src/test-utils/mock-data/mockStartWithDestination.ts diff --git a/airbyte-webapp/cypress/pages/createConnectorPage.ts b/airbyte-webapp/cypress/pages/createConnectorPage.ts index cf0a6840ce7..1c153ea7d93 100644 --- a/airbyte-webapp/cypress/pages/createConnectorPage.ts +++ b/airbyte-webapp/cypress/pages/createConnectorPage.ts @@ -13,7 +13,7 @@ const xminOption = "label[data-testid='radio-option.1']"; export const selectServiceType = (type: string) => { // Make sure alpha connectors are visible in the grid, since they are hidden by default - cy.get("#filter-release-stage-alpha").check({ force: true }); + cy.get("#filter-support-level-community").check({ force: true }); cy.contains("button", type).click(); }; diff --git a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.module.scss b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.module.scss deleted file mode 100644 index bcad38ce6b0..00000000000 --- a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.module.scss +++ /dev/null @@ -1,55 +0,0 @@ -@use "../../scss/colors"; -@use "../../scss/variables"; - -.container { - display: flex; - padding: variables.$spacing-md; - width: 220px; - align-items: center; -} - -.details { - width: 160px; - margin-left: variables.$spacing-md; - display: flex; - flex-direction: column; - font-weight: normal; -} - -.entityIcon { - height: 30px; - width: 30px; -} - -.connectionName { - font-size: variables.$font-size-lg; - color: colors.$dark-blue-900; - text-align: left; - margin-right: variables.$spacing-md; -} - -.connectorDetails { - display: flex; - justify-content: flex-start; - align-items: center; -} - -.connectorName { - font-size: 11px; - margin-top: 1px; - color: colors.$grey-300; - text-align: left; - word-wrap: break-word; -} - -.fullWidth { - width: 100%; - - .details { - width: 100%; - - .connectorDetails { - justify-content: space-between; - } - } -} diff --git a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx deleted file mode 100644 index 49cc04ad683..00000000000 --- a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import classnames from "classnames"; -import React from "react"; - -import { ReleaseStageBadge } from "components/ReleaseStageBadge"; - -import { SvgIcon } from "area/connector/utils"; -import { ReleaseStage } from "core/request/AirbyteClient"; - -import styles from "./ConnectorCard.module.scss"; - -export interface ConnectorCardProps { - connectionName: string; - icon?: string; - connectorName?: string; - releaseStage?: ReleaseStage; - fullWidth?: boolean; -} - -export const ConnectorCard: React.FC = ({ - connectionName, - connectorName, - icon, - releaseStage, - fullWidth, -}) => ( -

- {icon && ( -
- -
- )} -
-
-
{connectionName}
- {releaseStage && } -
- {connectorName &&
{connectorName}
} -
-
-); diff --git a/airbyte-webapp/src/components/ConnectorCard/index.tsx b/airbyte-webapp/src/components/ConnectorCard/index.tsx deleted file mode 100644 index 77f1bb6271b..00000000000 --- a/airbyte-webapp/src/components/ConnectorCard/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from "./ConnectorCard"; diff --git a/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx b/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx index 70790970f45..a14cc483bd2 100644 --- a/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx +++ b/airbyte-webapp/src/components/common/ConnectorIcon/ConnectorIcon.stories.tsx @@ -1,8 +1,6 @@ import { ComponentStory, ComponentMeta } from "@storybook/react"; import classNames from "classnames"; -import { ConnectorCard } from "components/ConnectorCard"; - import { ConnectorIcon } from "./ConnectorIcon"; import styles from "./ConnectorIcon.story.module.scss"; @@ -27,13 +25,6 @@ export const ValidateIcons = ({ icon }: { icon: string }) => (