diff --git a/airbyte-webapp/src/area/connector/utils/destinations.json b/airbyte-webapp/src/area/connector/utils/destinations.json index a3189be25cc..f06846603a4 100644 --- a/airbyte-webapp/src/area/connector/utils/destinations.json +++ b/airbyte-webapp/src/area/connector/utils/destinations.json @@ -13,6 +13,7 @@ "DatabricksLakehouse": "072d5540-f236-4294-ba7c-ade8fd918496", "DynamoDb": "8ccd8909-4e99-4141-b48d-4984b70b2d89", "E2ETesting": "2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537", + "EndToEndTesting": "a7bcc9d8-13b3-4e49-b80d-d020b90045e3", "ElasticSearch": "68f351a7-2745-4bef-ad7f-996b8e51bb8c", "Firebolt": "18081484-02a5-4662-8dba-b270b582f321", "GoogleCloudStorageGcs": "ca8f6566-e555-4b40-943a-545bf123117a", @@ -22,12 +23,14 @@ "Kinesis": "6d1d66d4-26ab-4602-8d32-f85894b04955", "LocalCsv": "8be1cf83-fde1-477f-a4ad-318d23c9f3c6", "LocalJson": "a625d593-bba5-4a1c-a53d-2d246268a816", + "Milvus": "65de8962-48c9-11ee-be56-0242ac120002", "Mqtt": "f3802bc4-5406-4752-9e8d-01e504ca8194", "MsSqlServer": "d4353156-9217-4cad-8dd7-c108fd4f74cf", "MeiliSearch": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", "MongoDb": "8b746512-8c2e-6ac1-4adc-b59faafd473c", "MySql": "ca81ee7c-3163-4246-af40-094cc31e5e42", "Oracle": "3986776d-2319-4de9-8af8-db14c0996e72", + "Pinecone": "3d2b6f84-7f0d-4e3f-a5e5-7c7d4b50eabd", "Postgres": "25c5221d-dce2-4163-ade9-739ef790f503", "Pulsar": "2340cbba-358e-11ec-8d3d-0242ac130203", "RabbitMq": "e06ad785-ad6f-4647-b2e8-3027a5c59454", @@ -46,5 +49,6 @@ "LocalSqLite": "b76be0a6-27dc-4560-95f6-2623da0bd7b6", "TiDb": "06ec60c7-7468-45c0-91ac-174f6e1a788b", "Typesense": "36be8dc6-9851-49af-b776-9d4c30e4ab6a", - "YugabyteDb": "2300fdcf-a532-419f-9f24-a014336e7966" + "YugabyteDb": "2300fdcf-a532-419f-9f24-a014336e7966", + "Weaviate": "7b7d7a0d-954c-45a0-bcfc-39a634b97736" } diff --git a/airbyte-webapp/src/area/connector/utils/sources.json b/airbyte-webapp/src/area/connector/utils/sources.json index 6a692ca5525..21d88eaa5b4 100644 --- a/airbyte-webapp/src/area/connector/utils/sources.json +++ b/airbyte-webapp/src/area/connector/utils/sources.json @@ -54,6 +54,7 @@ "Dv360": "1356e1d9-977f-4057-ad4b-65f25329cf61", "DynamoDb": "50401137-8871-4c5a-abb7-1f5fda35545a", "E2ETesting": "d53f9084-fa6b-4a5a-976c-5b8392f4ad8a", + "EndToEndTesting": "50bd8338-7c4e-46f1-8c7f-3ef95de19fdd", "EmailOctopus": "46b25e70-c980-4590-a811-8deaf50ee09f", "ExchangeRatesApi": "e2b40e36-aa0e-4bed-b41b-bcea6fa348b1", "FacebookMarketing": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c", diff --git a/airbyte-webapp/src/components/connection/CreateConnection/RadioButtonTiles.tsx b/airbyte-webapp/src/components/connection/CreateConnection/RadioButtonTiles.tsx index e2e3bb2fcf0..605a2840027 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/RadioButtonTiles.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/RadioButtonTiles.tsx @@ -14,6 +14,7 @@ interface RadioButtonTilesOption { label: string; labelValues?: ComponentProps["values"]; description: string; + descriptionValues?: ComponentProps["values"]; extra?: React.ReactNode; disabled?: boolean; } @@ -65,7 +66,7 @@ export const RadioButtonTiles = ({ - {formatMessage({ id: option.description })} + {formatMessage({ id: option.description }, option.descriptionValues)} {option.extra && {option.extra}} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts new file mode 100644 index 00000000000..6f08f94637c --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts @@ -0,0 +1,19 @@ +import { ConnectorIds } from "area/connector/utils"; + +export const SourceNamespaceConfiguration = { + [ConnectorIds.Sources.E2ETesting]: { supportsNamespaces: false }, + [ConnectorIds.Sources.EndToEndTesting]: { supportsNamespaces: false }, +} as const; + +export const DestinationNamespaceConfiguration = { + [ConnectorIds.Destinations.BigQuery]: { supportsNamespaces: true, defaultNamespacePath: "dataset_id" }, + [ConnectorIds.Destinations.E2ETesting]: { supportsNamespaces: false }, + [ConnectorIds.Destinations.EndToEndTesting]: { supportsNamespaces: false }, + [ConnectorIds.Destinations.Milvus]: { supportsNamespaces: true }, + [ConnectorIds.Destinations.Pinecone]: { supportsNamespaces: true }, + [ConnectorIds.Destinations.Postgres]: { supportsNamespaces: true, defaultNamespacePath: "schema" }, + [ConnectorIds.Destinations.Redshift]: { supportsNamespaces: true, defaultNamespacePath: "schema" }, + [ConnectorIds.Destinations.S3]: { supportsNamespaces: true }, + [ConnectorIds.Destinations.Snowflake]: { supportsNamespaces: true, defaultNamespacePath: "schema" }, + [ConnectorIds.Destinations.Weaviate]: { supportsNamespaces: false }, +} as const; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx index b1ba1186d47..27a174a0926 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx @@ -82,6 +82,7 @@ const SimplifiedConnectionCreationConfigureConnection: React.FC = () => { const { trackFormChange } = useFormChangeTrackerService(); const source = useGetSourceFromSearchParams(); + const destination = useGetDestinationFromSearchParams(); // if the user is navigating from the first step the form may be dirty useMount(() => { @@ -91,7 +92,8 @@ const SimplifiedConnectionCreationConfigureConnection: React.FC = () => { return ( ); @@ -99,8 +101,8 @@ const SimplifiedConnectionCreationConfigureConnection: React.FC = () => { const FirstNav: React.FC = () => { const createLink = useCurrentWorkspaceLink(); - const source = useGetSourceFromSearchParams(); const destination = useGetDestinationFromSearchParams(); + const source = useGetSourceFromSearchParams(); const { isValid, errors } = useFormState(); const { trigger } = useFormContext(); diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx index 98519b6fbfe..08d41fbb492 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx @@ -8,6 +8,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; +import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { useExperiment } from "hooks/services/Experiment"; @@ -24,14 +25,16 @@ import { SimplifiedSchemaChangeNotificationFormField } from "./SimplifiedSchemaC interface SimplifiedConnectionsSettingsCardProps { title: string; isCreating: boolean; - sourceName: string; + source: SourceRead; + destination: DestinationRead; isDeprecated?: boolean; } export const SimplifiedConnectionsSettingsCard: React.FC = ({ title, isCreating, - sourceName, + source, + destination, isDeprecated = false, }) => { const [isAdvancedOpen, setIsAdvancedOpen] = useState(false); @@ -43,7 +46,9 @@ export const SimplifiedConnectionsSettingsCard: React.FC - {isCreating && } + {isCreating && ( + + )} {isCreating && } @@ -79,7 +84,8 @@ export const SimplifiedConnectionsSettingsCard: React.FC )} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss new file mode 100644 index 00000000000..a9a255ac590 --- /dev/null +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss @@ -0,0 +1,14 @@ +@use "scss/variables"; + +.originalCasing, +%originalCasing { + text-transform: unset; +} + +.sourceNamespace { + @extend %originalCasing; + + &:not(:first-child) { + margin-left: variables.$spacing-xs; + } +} diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx index 499d5abb33b..fdd5f6e7582 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx @@ -1,3 +1,4 @@ +import get from "lodash/get"; import { ComponentProps, useEffect } from "react"; import { Controller, useFormContext, useFormState, useWatch } from "react-hook-form"; import { FormattedMessage, useIntl } from "react-intl"; @@ -5,53 +6,134 @@ import { FormattedMessage, useIntl } from "react-intl"; import { FormConnectionFormValues } from "components/connection/ConnectionForm/formConfig"; import { FormFieldLayout } from "components/connection/ConnectionForm/FormFieldLayout"; import { RadioButtonTiles } from "components/connection/CreateConnection/RadioButtonTiles"; -import { FormControl } from "components/forms"; import { ControlLabels } from "components/LabeledControl"; +import { Badge } from "components/ui/Badge"; import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; +import { Input } from "components/ui/Input"; import { ExternalLink } from "components/ui/Link"; import { ListBox } from "components/ui/ListBox"; import { Text } from "components/ui/Text"; -import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; +import { DestinationRead, NamespaceDefinitionType, SourceRead } from "core/api/types/AirbyteClient"; import { links } from "core/utils/links"; +import { naturalComparator } from "core/utils/objects"; +import { SourceNamespaceConfiguration, DestinationNamespaceConfiguration } from "./ConnectorNamespaceConfiguration"; import { InputContainer } from "./InputContainer"; +import styles from "./SimplifiedDestinationNamespaceFormField.module.scss"; + +// eslint-disable-next-line no-template-curly-in-string +const SOURCE_NAMESPACE_REPLACEMENT_STRING = "${SOURCE_NAMESPACE}"; export const SimplifiedDestinationNamespaceFormField: React.FC<{ isCreating: boolean; - sourceName: string; + source: SourceRead; + destination: DestinationRead; disabled?: boolean; -}> = ({ isCreating, sourceName, disabled }) => { +}> = ({ isCreating, source, destination, disabled }) => { const { trigger, setValue, control, watch } = useFormContext(); const { defaultValues } = useFormState(); const namespaceDefinition = useWatch({ name: "namespaceDefinition", control }); + const streams = useWatch({ name: "syncCatalog.streams", control }); + const namespaceFormat = useWatch({ name: "namespaceFormat", control }); const { formatMessage } = useIntl(); const watchedNamespaceDefinition = watch("namespaceDefinition"); useEffect(() => { - if (watchedNamespaceDefinition === NamespaceDefinitionType.customformat) { - setValue("namespaceFormat", defaultValues?.namespaceFormat, { shouldDirty: true }); - } - trigger("namespaceFormat", { shouldFocus: true }); }, [trigger, setValue, defaultValues?.namespaceFormat, watchedNamespaceDefinition]); - const watchedNamespaceFormat = watch("namespaceFormat"); - useEffect(() => { - trigger("namespaceFormat"); - }, [trigger, watchedNamespaceFormat]); + const sourceNamespaceAbilities = SourceNamespaceConfiguration[source.sourceDefinitionId] ?? { + supportsNamespaces: true, + }; + const destinationNamespaceAbilities = DestinationNamespaceConfiguration[destination.destinationDefinitionId] ?? { + supportsNamespaces: true, + }; + + if (!destinationNamespaceAbilities.supportsNamespaces) { + return null; + } + + const destinationDefinedNamespace = + (destinationNamespaceAbilities.defaultNamespacePath && + get(destination.connectionConfiguration, destinationNamespaceAbilities.defaultNamespacePath)) ?? + "no_value_provided"; + + const destinationDefinedDescriptionValues = { + destinationDefinedNamespace, + badge: (children: React.ReactNode[]) => ( + + {children} + + ), + }; + + const enabledStreamNamespaces = Array.from( + streams.reduce((acc, stream) => { + if (stream.config?.selected && stream.stream?.namespace) { + acc.add(stream.stream.namespace); + } + return acc; + }, new Set()) + ).sort(naturalComparator); + + const sourceDefinedDescriptionValues = { + sourceDefinedNamespaces: enabledStreamNamespaces, + badges: (children: React.ReactNode[]) => { + if (children.length === 0) { + return null; + } + + return ( + + {children.map((child) => ( + + {child} + + ))} + + ); + }, + }; const customFormatField = namespaceDefinition === NamespaceDefinitionType.customformat ? ( - + <> + ( + <> + + + + {fieldState.error && ( + + + + + + )} + + )} + /> + {namespaceFormat?.includes(SOURCE_NAMESPACE_REPLACEMENT_STRING) && + enabledStreamNamespaces.map((namespace) => { + return ( + + {namespaceFormat.replace(SOURCE_NAMESPACE_REPLACEMENT_STRING, namespace)} + + ); + })} + ) : null; const destinationNamespaceOptions: ComponentProps>["options"] = [ @@ -67,12 +149,20 @@ export const SimplifiedDestinationNamespaceFormField: React.FC<{ ? "connectionForm.destinationFormatNext" : formatMessage({ id: "connectionForm.destinationFormatNext" }), description: "connectionForm.destinationFormatDescriptionNext", + descriptionValues: destinationDefinedDescriptionValues, }, - { - value: NamespaceDefinitionType.source, - label: isCreating ? "connectionForm.sourceFormatNext" : formatMessage({ id: "connectionForm.sourceFormatNext" }), - description: "connectionForm.sourceFormatDescriptionNext", - }, + ...(sourceNamespaceAbilities.supportsNamespaces + ? [ + { + value: NamespaceDefinitionType.source, + label: isCreating + ? "connectionForm.sourceFormatNext" + : formatMessage({ id: "connectionForm.sourceFormatNext" }), + description: "connectionForm.sourceFormatDescriptionNext", + descriptionValues: sourceDefinedDescriptionValues, + }, + ] + : []), ]; return ( @@ -119,14 +209,20 @@ export const SimplifiedDestinationNamespaceFormField: React.FC<{ {field.value === NamespaceDefinitionType.destination && ( - + )} {field.value === NamespaceDefinitionType.source && ( - + )} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 0fb43c01f3b..022600405cf 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -231,11 +231,11 @@ "connectionForm.sourceFormat": "Mirror source structure", "connectionForm.sourceFormatNext": "Source-defined", "connectionForm.sourceFormatDescription": "Match the schema the source is in", - "connectionForm.sourceFormatDescriptionNext": "Use the schema(s) defined by the source.", + "connectionForm.sourceFormatDescriptionNext": "Use the schema(s) defined by the source.{sourceDefinedNamespaces}", "connectionForm.destinationFormat": "Destination default", "connectionForm.destinationFormatNext": "Destination-defined", "connectionForm.destinationFormatDescription": "Sync all streams to the default schema defined in the destination", - "connectionForm.destinationFormatDescriptionNext": "Sync all streams to the schema defined in the destination's settings.", + "connectionForm.destinationFormatDescriptionNext": "Sync all streams to {destinationDefinedNamespace, select, no_value_provided {} other {{destinationDefinedNamespace},}} the schema defined in the destination's settings.", "connectionForm.customFormat": "Custom format", "connectionForm.customFormatDescription": "Sync all streams to a unique new schema", "connectionForm.customFormatDescriptionNext": "Sync all streams to a unique new schema. Useful when syncing from multiple sources.", diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx index eb69cc93fdb..3217e1e3fa2 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx @@ -257,7 +257,8 @@ const SimplifiedConnectionSettingsPage = () => { >