🪟 🎉 provide source/destination namespace information to users during …

…simplified connection flow (#11753)
airbytehq · Mar 22, 2024 · a393233 · a393233
1 parent 7f2ed67
commit a393233
Show file tree

Hide file tree

Showing 10 changed files with 182 additions and 38 deletions.
diff --git a/airbyte-webapp/src/area/connector/utils/destinations.json b/airbyte-webapp/src/area/connector/utils/destinations.json
@@ -13,6 +13,7 @@
   "DatabricksLakehouse": "072d5540-f236-4294-ba7c-ade8fd918496",
   "DynamoDb": "8ccd8909-4e99-4141-b48d-4984b70b2d89",
   "E2ETesting": "2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537",
+  "EndToEndTesting": "a7bcc9d8-13b3-4e49-b80d-d020b90045e3",
   "ElasticSearch": "68f351a7-2745-4bef-ad7f-996b8e51bb8c",
   "Firebolt": "18081484-02a5-4662-8dba-b270b582f321",
   "GoogleCloudStorageGcs": "ca8f6566-e555-4b40-943a-545bf123117a",
@@ -22,12 +23,14 @@
   "Kinesis": "6d1d66d4-26ab-4602-8d32-f85894b04955",
   "LocalCsv": "8be1cf83-fde1-477f-a4ad-318d23c9f3c6",
   "LocalJson": "a625d593-bba5-4a1c-a53d-2d246268a816",
+  "Milvus": "65de8962-48c9-11ee-be56-0242ac120002",
   "Mqtt": "f3802bc4-5406-4752-9e8d-01e504ca8194",
   "MsSqlServer": "d4353156-9217-4cad-8dd7-c108fd4f74cf",
   "MeiliSearch": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e",
   "MongoDb": "8b746512-8c2e-6ac1-4adc-b59faafd473c",
   "MySql": "ca81ee7c-3163-4246-af40-094cc31e5e42",
   "Oracle": "3986776d-2319-4de9-8af8-db14c0996e72",
+  "Pinecone": "3d2b6f84-7f0d-4e3f-a5e5-7c7d4b50eabd",
   "Postgres": "25c5221d-dce2-4163-ade9-739ef790f503",
   "Pulsar": "2340cbba-358e-11ec-8d3d-0242ac130203",
   "RabbitMq": "e06ad785-ad6f-4647-b2e8-3027a5c59454",
@@ -46,5 +49,6 @@
   "LocalSqLite": "b76be0a6-27dc-4560-95f6-2623da0bd7b6",
   "TiDb": "06ec60c7-7468-45c0-91ac-174f6e1a788b",
   "Typesense": "36be8dc6-9851-49af-b776-9d4c30e4ab6a",
-  "YugabyteDb": "2300fdcf-a532-419f-9f24-a014336e7966"
+  "YugabyteDb": "2300fdcf-a532-419f-9f24-a014336e7966",
+  "Weaviate": "7b7d7a0d-954c-45a0-bcfc-39a634b97736"
 }
diff --git a/airbyte-webapp/src/area/connector/utils/sources.json b/airbyte-webapp/src/area/connector/utils/sources.json
@@ -54,6 +54,7 @@
   "Dv360": "1356e1d9-977f-4057-ad4b-65f25329cf61",
   "DynamoDb": "50401137-8871-4c5a-abb7-1f5fda35545a",
   "E2ETesting": "d53f9084-fa6b-4a5a-976c-5b8392f4ad8a",
+  "EndToEndTesting": "50bd8338-7c4e-46f1-8c7f-3ef95de19fdd",
   "EmailOctopus": "46b25e70-c980-4590-a811-8deaf50ee09f",
   "ExchangeRatesApi": "e2b40e36-aa0e-4bed-b41b-bcea6fa348b1",
   "FacebookMarketing": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c",

diff --git a/airbyte-webapp/src/components/connection/CreateConnection/RadioButtonTiles.tsx b/airbyte-webapp/src/components/connection/CreateConnection/RadioButtonTiles.tsx
@@ -14,6 +14,7 @@ interface RadioButtonTilesOption<T> {
   label: string;
   labelValues?: ComponentProps<typeof FormattedMessage>["values"];
   description: string;
+  descriptionValues?: ComponentProps<typeof FormattedMessage>["values"];
   extra?: React.ReactNode;
   disabled?: boolean;
 }
@@ -65,7 +66,7 @@ export const RadioButtonTiles = <T extends string>({
                 </Text>
               </Box>
               <Text size="sm" color={option.disabled ? "grey" : "darkBlue"}>
-                {formatMessage({ id: option.description })}
+                {formatMessage({ id: option.description }, option.descriptionValues)}
               </Text>
               {option.extra && <Box mt="sm">{option.extra}</Box>}
             </div>

diff --git a/...tion/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts b/...tion/CreateConnectionForm/SimplifiedConnectionCreation/ConnectorNamespaceConfiguration.ts
@@ -0,0 +1,19 @@
+import { ConnectorIds } from "area/connector/utils";
+
+export const SourceNamespaceConfiguration = {
+  [ConnectorIds.Sources.E2ETesting]: { supportsNamespaces: false },
+  [ConnectorIds.Sources.EndToEndTesting]: { supportsNamespaces: false },
+} as const;
+
+export const DestinationNamespaceConfiguration = {
+  [ConnectorIds.Destinations.BigQuery]: { supportsNamespaces: true, defaultNamespacePath: "dataset_id" },
+  [ConnectorIds.Destinations.E2ETesting]: { supportsNamespaces: false },
+  [ConnectorIds.Destinations.EndToEndTesting]: { supportsNamespaces: false },
+  [ConnectorIds.Destinations.Milvus]: { supportsNamespaces: true },
+  [ConnectorIds.Destinations.Pinecone]: { supportsNamespaces: true },
+  [ConnectorIds.Destinations.Postgres]: { supportsNamespaces: true, defaultNamespacePath: "schema" },
+  [ConnectorIds.Destinations.Redshift]: { supportsNamespaces: true, defaultNamespacePath: "schema" },
+  [ConnectorIds.Destinations.S3]: { supportsNamespaces: true },
+  [ConnectorIds.Destinations.Snowflake]: { supportsNamespaces: true, defaultNamespacePath: "schema" },
+  [ConnectorIds.Destinations.Weaviate]: { supportsNamespaces: false },
+} as const;
diff --git a/...n/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx b/...n/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionConfiguration.tsx
@@ -82,6 +82,7 @@ const SimplifiedConnectionCreationConfigureConnection: React.FC = () => {
   const { trackFormChange } = useFormChangeTrackerService();
 
   const source = useGetSourceFromSearchParams();
+  const destination = useGetDestinationFromSearchParams();
 
   // if the user is navigating from the first step the form may be dirty
   useMount(() => {
@@ -91,16 +92,17 @@ const SimplifiedConnectionCreationConfigureConnection: React.FC = () => {
   return (
     <SimplifiedConnectionsSettingsCard
       title={formatMessage({ id: "connectionForm.configureConnection" })}
-      sourceName={source.name}
+      source={source}
+      destination={destination}
       isCreating
     />
   );
 };
 
 const FirstNav: React.FC = () => {
   const createLink = useCurrentWorkspaceLink();
-  const source = useGetSourceFromSearchParams();
   const destination = useGetDestinationFromSearchParams();
+  const source = useGetSourceFromSearchParams();
 
   const { isValid, errors } = useFormState<FormConnectionFormValues>();
   const { trigger } = useFormContext<FormConnectionFormValues>();

diff --git a/...on/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx b/...on/CreateConnectionForm/SimplifiedConnectionCreation/SimplifiedConnectionSettingsCard.tsx
@@ -8,6 +8,7 @@ import { FlexContainer } from "components/ui/Flex";
 import { Icon } from "components/ui/Icon";
 import { Text } from "components/ui/Text";
 
+import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient";
 import { FeatureItem, useFeature } from "core/services/features";
 import { useExperiment } from "hooks/services/Experiment";
 
@@ -24,14 +25,16 @@ import { SimplifiedSchemaChangeNotificationFormField } from "./SimplifiedSchemaC
 interface SimplifiedConnectionsSettingsCardProps {
   title: string;
   isCreating: boolean;
-  sourceName: string;
+  source: SourceRead;
+  destination: DestinationRead;
   isDeprecated?: boolean;
 }
 
 export const SimplifiedConnectionsSettingsCard: React.FC<SimplifiedConnectionsSettingsCardProps> = ({
   title,
   isCreating,
-  sourceName,
+  source,
+  destination,
   isDeprecated = false,
 }) => {
   const [isAdvancedOpen, setIsAdvancedOpen] = useState(false);
@@ -43,7 +46,9 @@ export const SimplifiedConnectionsSettingsCard: React.FC<SimplifiedConnectionsSe
       <FlexContainer direction="column" gap="xl">
         <SimplifiedConnectionNameFormField />
         <SimplifiedConnectionScheduleFormField disabled={isDeprecated} />
-        {isCreating && <SimplifiedDestinationNamespaceFormField isCreating={isCreating} sourceName={sourceName} />}
+        {isCreating && (
+          <SimplifiedDestinationNamespaceFormField isCreating={isCreating} source={source} destination={destination} />
+        )}
         {isCreating && <SimplifiedDestinationStreamPrefixNameFormField />}
       </FlexContainer>
 
@@ -79,7 +84,8 @@ export const SimplifiedConnectionsSettingsCard: React.FC<SimplifiedConnectionsSe
           {!isCreating && (
             <SimplifiedDestinationNamespaceFormField
               isCreating={isCreating}
-              sourceName={sourceName}
+              source={source}
+              destination={destination}
               disabled={isDeprecated}
             />
           )}

diff --git a/...tionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss b/...tionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.module.scss
@@ -0,0 +1,14 @@
+@use "scss/variables";
+
+.originalCasing,
+%originalCasing {
+  text-transform: unset;
+}
+
+.sourceNamespace {
+  @extend %originalCasing;
+
+  &:not(:first-child) {
+    margin-left: variables.$spacing-xs;
+  }
+}
diff --git a/...teConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx b/...teConnectionForm/SimplifiedConnectionCreation/SimplifiedDestinationNamespaceFormField.tsx
@@ -1,57 +1,139 @@
+import get from "lodash/get";
 import { ComponentProps, useEffect } from "react";
 import { Controller, useFormContext, useFormState, useWatch } from "react-hook-form";
 import { FormattedMessage, useIntl } from "react-intl";
 
 import { FormConnectionFormValues } from "components/connection/ConnectionForm/formConfig";
 import { FormFieldLayout } from "components/connection/ConnectionForm/FormFieldLayout";
 import { RadioButtonTiles } from "components/connection/CreateConnection/RadioButtonTiles";
-import { FormControl } from "components/forms";
 import { ControlLabels } from "components/LabeledControl";
+import { Badge } from "components/ui/Badge";
 import { Box } from "components/ui/Box";
 import { FlexContainer } from "components/ui/Flex";
+import { Input } from "components/ui/Input";
 import { ExternalLink } from "components/ui/Link";
 import { ListBox } from "components/ui/ListBox";
 import { Text } from "components/ui/Text";
 
-import { NamespaceDefinitionType } from "core/api/types/AirbyteClient";
+import { DestinationRead, NamespaceDefinitionType, SourceRead } from "core/api/types/AirbyteClient";
 import { links } from "core/utils/links";
+import { naturalComparator } from "core/utils/objects";
 
+import { SourceNamespaceConfiguration, DestinationNamespaceConfiguration } from "./ConnectorNamespaceConfiguration";
 import { InputContainer } from "./InputContainer";
+import styles from "./SimplifiedDestinationNamespaceFormField.module.scss";
+
+// eslint-disable-next-line no-template-curly-in-string
+const SOURCE_NAMESPACE_REPLACEMENT_STRING = "${SOURCE_NAMESPACE}";
 
 export const SimplifiedDestinationNamespaceFormField: React.FC<{
   isCreating: boolean;
-  sourceName: string;
+  source: SourceRead;
+  destination: DestinationRead;
   disabled?: boolean;
-}> = ({ isCreating, sourceName, disabled }) => {
+}> = ({ isCreating, source, destination, disabled }) => {
   const { trigger, setValue, control, watch } = useFormContext<FormConnectionFormValues>();
   const { defaultValues } = useFormState<FormConnectionFormValues>();
   const namespaceDefinition = useWatch({ name: "namespaceDefinition", control });
+  const streams = useWatch({ name: "syncCatalog.streams", control });
+  const namespaceFormat = useWatch({ name: "namespaceFormat", control });
   const { formatMessage } = useIntl();
 
   const watchedNamespaceDefinition = watch("namespaceDefinition");
   useEffect(() => {
-    if (watchedNamespaceDefinition === NamespaceDefinitionType.customformat) {
-      setValue("namespaceFormat", defaultValues?.namespaceFormat, { shouldDirty: true });
-    }
-
     trigger("namespaceFormat", { shouldFocus: true });
   }, [trigger, setValue, defaultValues?.namespaceFormat, watchedNamespaceDefinition]);
 
-  const watchedNamespaceFormat = watch("namespaceFormat");
-  useEffect(() => {
-    trigger("namespaceFormat");
-  }, [trigger, watchedNamespaceFormat]);
+  const sourceNamespaceAbilities = SourceNamespaceConfiguration[source.sourceDefinitionId] ?? {
+    supportsNamespaces: true,
+  };
+  const destinationNamespaceAbilities = DestinationNamespaceConfiguration[destination.destinationDefinitionId] ?? {
+    supportsNamespaces: true,
+  };
+
+  if (!destinationNamespaceAbilities.supportsNamespaces) {
+    return null;
+  }
+
+  const destinationDefinedNamespace =
+    (destinationNamespaceAbilities.defaultNamespacePath &&
+      get(destination.connectionConfiguration, destinationNamespaceAbilities.defaultNamespacePath)) ??
+    "no_value_provided";
+
+  const destinationDefinedDescriptionValues = {
+    destinationDefinedNamespace,
+    badge: (children: React.ReactNode[]) => (
+      <Badge variant="grey" className={styles.originalCasing}>
+        {children}
+      </Badge>
+    ),
+  };
+
+  const enabledStreamNamespaces = Array.from(
+    streams.reduce((acc, stream) => {
+      if (stream.config?.selected && stream.stream?.namespace) {
+        acc.add(stream.stream.namespace);
+      }
+      return acc;
+    }, new Set<string>())
+  ).sort(naturalComparator);
+
+  const sourceDefinedDescriptionValues = {
+    sourceDefinedNamespaces: enabledStreamNamespaces,
+    badges: (children: React.ReactNode[]) => {
+      if (children.length === 0) {
+        return null;
+      }
+
+      return (
+        <Box mt="sm">
+          {children.map((child) => (
+            <Badge variant="grey" className={styles.sourceNamespace}>
+              {child}
+            </Badge>
+          ))}
+        </Box>
+      );
+    },
+  };
 
   const customFormatField =
     namespaceDefinition === NamespaceDefinitionType.customformat ? (
-      <FormControl
-        name="namespaceFormat"
-        fieldType="input"
-        type="text"
-        placeholder={sourceName.toLowerCase().replace(/[^a-z0-9]+/g, "_")}
-        data-testid="namespace-definition-custom-format-input"
-        disabled={disabled}
-      />
+      <>
+        <Controller
+          name="namespaceFormat"
+          control={control}
+          render={({ field, fieldState }) => (
+            <>
+              <InputContainer>
+                <Input
+                  name="namespaceFormat"
+                  placeholder={source.name.toLowerCase().replace(/[^a-z0-9]+/g, "_")}
+                  inline={false}
+                  value={field.value}
+                  onChange={field.onChange}
+                  disabled={disabled}
+                />
+              </InputContainer>
+              {fieldState.error && (
+                <Box mt="sm">
+                  <Text color="red">
+                    <FormattedMessage id={fieldState.error.message} />
+                  </Text>
+                </Box>
+              )}
+            </>
+          )}
+        />
+        {namespaceFormat?.includes(SOURCE_NAMESPACE_REPLACEMENT_STRING) &&
+          enabledStreamNamespaces.map((namespace) => {
+            return (
+              <Badge variant="grey" className={styles.sourceNamespace}>
+                {namespaceFormat.replace(SOURCE_NAMESPACE_REPLACEMENT_STRING, namespace)}
+              </Badge>
+            );
+          })}
+      </>
     ) : null;
 
   const destinationNamespaceOptions: ComponentProps<typeof RadioButtonTiles<NamespaceDefinitionType>>["options"] = [
@@ -67,12 +149,20 @@ export const SimplifiedDestinationNamespaceFormField: React.FC<{
         ? "connectionForm.destinationFormatNext"
         : formatMessage({ id: "connectionForm.destinationFormatNext" }),
       description: "connectionForm.destinationFormatDescriptionNext",
+      descriptionValues: destinationDefinedDescriptionValues,
     },
-    {
-      value: NamespaceDefinitionType.source,
-      label: isCreating ? "connectionForm.sourceFormatNext" : formatMessage({ id: "connectionForm.sourceFormatNext" }),
-      description: "connectionForm.sourceFormatDescriptionNext",
-    },
+    ...(sourceNamespaceAbilities.supportsNamespaces
+      ? [
+          {
+            value: NamespaceDefinitionType.source,
+            label: isCreating
+              ? "connectionForm.sourceFormatNext"
+              : formatMessage({ id: "connectionForm.sourceFormatNext" }),
+            description: "connectionForm.sourceFormatDescriptionNext",
+            descriptionValues: sourceDefinedDescriptionValues,
+          },
+        ]
+      : []),
   ];
 
   return (
@@ -119,14 +209,20 @@ export const SimplifiedDestinationNamespaceFormField: React.FC<{
               {field.value === NamespaceDefinitionType.destination && (
                 <Box mt="sm">
                   <Text size="sm">
-                    <FormattedMessage id="connectionForm.destinationFormatDescriptionNext" />
+                    <FormattedMessage
+                      id="connectionForm.destinationFormatDescriptionNext"
+                      values={destinationDefinedDescriptionValues}
+                    />
                   </Text>
                 </Box>
               )}
               {field.value === NamespaceDefinitionType.source && (
                 <Box mt="sm">
                   <Text size="sm">
-                    <FormattedMessage id="connectionForm.sourceFormatDescriptionNext" />
+                    <FormattedMessage
+                      id="connectionForm.sourceFormatDescriptionNext"
+                      values={sourceDefinedDescriptionValues}
+                    />
                   </Text>
                 </Box>
               )}

diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json
@@ -231,11 +231,11 @@
   "connectionForm.sourceFormat": "Mirror source structure",
   "connectionForm.sourceFormatNext": "Source-defined",
   "connectionForm.sourceFormatDescription": "Match the schema the source is in",
-  "connectionForm.sourceFormatDescriptionNext": "Use the schema(s) defined by the source.",
+  "connectionForm.sourceFormatDescriptionNext": "Use the schema(s) defined by the source.<badges>{sourceDefinedNamespaces}</badges>",
   "connectionForm.destinationFormat": "Destination default",
   "connectionForm.destinationFormatNext": "Destination-defined",
   "connectionForm.destinationFormatDescription": "Sync all streams to the default schema defined in the destination",
-  "connectionForm.destinationFormatDescriptionNext": "Sync all streams to the schema defined in the destination's settings.",
+  "connectionForm.destinationFormatDescriptionNext": "Sync all streams to {destinationDefinedNamespace, select, no_value_provided {} other {<badge>{destinationDefinedNamespace}</badge>,}} the schema defined in the destination's settings.",
   "connectionForm.customFormat": "Custom format",
   "connectionForm.customFormatDescription": "Sync all streams to a unique new schema",
   "connectionForm.customFormatDescriptionNext": "Sync all streams to a unique new schema. Useful when syncing from multiple sources.",

diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx
@@ -257,7 +257,8 @@ const SimplifiedConnectionSettingsPage = () => {
       >
         <SimplifiedConnectionsSettingsCard
           title={formatMessage({ id: "sources.settings" })}
-          sourceName={connection.source.name}
+          source={connection.source}
+          destination={connection.destination}
           isCreating={false}
           isDeprecated={isDeprecated}
         />