Make path optional (#1143)

* Make path optional * fix: make path optional for doc editor tests data * fix: for lineage * Only take models with path --------- Co-authored-by: saravmajestic <[email protected]> Co-authored-by: Aditya <[email protected]> Co-authored-by: anandgupta42 <[email protected]>
AltimateAI · May 20, 2024 · bb0e54e · bb0e54e
1 parent 4191b42
commit bb0e54e
Show file tree

Hide file tree

Showing 17 changed files with 51 additions and 44 deletions.
diff --git a/new_lineage_panel/src/App.tsx b/new_lineage_panel/src/App.tsx
@@ -50,7 +50,7 @@ import { handleResponse, init, columnLineage } from "./service_utils";
 import { ActionWidget } from "./ActionWidget";
 import { DEFAULT_MIN_ZOOM, createTableNode } from "./utils";
 import { Settings } from "./Settings";
-import { getLineageSettings } from "./service";
+import { Table, getLineageSettings } from "./service";
 
 export let aiEnabled = false;
 export let isDarkMode = false;
@@ -160,19 +160,7 @@ function App() {
   const [minRange, setMinRange] = useState<[number, number]>([0, 0]);
 
   useEffect(() => {
-    const render = async (args: {
-      node: {
-        table: string;
-        label: string;
-        url: string;
-        nodeType: string;
-        materialization?: string;
-        downstreamCount: number;
-        upstreamCount: number;
-        tests: { key: string; path: string }[];
-      };
-      aiEnabled: boolean;
-    }) => {
+    const render = async (args: { node: Table; aiEnabled: boolean }) => {
       setIsOpen(false);
       setSidebarScreen("");
       if (!args) return;

diff --git a/new_lineage_panel/src/CustomNodes.tsx b/new_lineage_panel/src/CustomNodes.tsx
@@ -99,6 +99,7 @@ export const TableNode: FunctionComponent<NodeProps> = ({ data }) => {
     rightExpansion,
     selectCheck,
     nonSelectCheck,
+    setSelectedTable,
   } = useContext(LineageContext);
 
   const _columnLen = Object.keys(collectColumns[table] || {}).length;
@@ -207,7 +208,10 @@ export const TableNode: FunctionComponent<NodeProps> = ({ data }) => {
             )
           );
           highlightTable();
-          openFile(url);
+          setSelectedTable(table);
+          if (url) {
+            openFile(url);
+          }
         }}
       >
         <div

diff --git a/new_lineage_panel/src/service.ts b/new_lineage_panel/src/service.ts
@@ -3,7 +3,7 @@ import { requestExecutor } from "./service_utils";
 export type Table = {
   table: string;
   label: string;
-  url: string;
+  url: string | undefined;
   nodeType: string;
   materialization?: string;
   downstreamCount: number;

diff --git a/src/commands/tests/missingSchemaTest.ts b/src/commands/tests/missingSchemaTest.ts
@@ -27,7 +27,7 @@ export class MissingSchemaTest implements AltimateScanStep {
       ) {
         continue;
       }
-      if (!value.patch_path) {
+      if (!value.patch_path && value.path) {
         const errMessage = `Documentation missing for model: ${value.name}`;
         let projDiagnostic = projectDiagnostics[value.path];
         if (projDiagnostic === undefined) {

diff --git a/src/commands/tests/undocumentedModelColumnTest.ts b/src/commands/tests/undocumentedModelColumnTest.ts
@@ -55,6 +55,9 @@ export class UndocumentedModelColumnTest implements AltimateScanStep {
               getColumnNameByCase(column.column_name, project.getAdapterType()),
             )
           ) {
+            if (!value.path) {
+              continue;
+            }
             const errMessage = `Column ${column.column_name} is undocumented in model: ${value.name}`;
 
             let modelDiagnostics = projectDiagnostics[value.path];

diff --git a/src/commands/tests/unmaterializedModelTest.ts b/src/commands/tests/unmaterializedModelTest.ts
@@ -36,7 +36,8 @@ export class UnmaterializedModelTest implements AltimateScanStep {
       if (
         !Object.keys(altimateCatalog[projectName + projectRootUri]).includes(
           modelKey,
-        )
+        ) &&
+        value.path
       ) {
         // When the model is not in model dict, we could not find the table or view in
         // information schema. meaning it was not materialized.

diff --git a/src/definition_provider/macroDefinitionProvider.ts b/src/definition_provider/macroDefinitionProvider.ts
@@ -102,7 +102,7 @@ export class MacroDefinitionProvider implements DefinitionProvider, Disposable {
       return;
     }
     const location = macroMap.get(macroName);
-    if (location) {
+    if (location && location.path) {
       return new Location(
         Uri.file(location.path),
         new Position(location.line, location.character),

diff --git a/src/definition_provider/modelDefinitionProvider.ts b/src/definition_provider/modelDefinitionProvider.ts
@@ -126,7 +126,7 @@ export class ModelDefinitionProvider implements DefinitionProvider, Disposable {
       return;
     }
     const location = nodeMap.get(modelName);
-    if (location) {
+    if (location && location.path) {
       return new Location(Uri.file(location.path), new Range(0, 0, 999, 999));
     }
     return undefined;

diff --git a/src/definition_provider/sourceDefinitionProvider.ts b/src/definition_provider/sourceDefinitionProvider.ts
@@ -120,7 +120,7 @@ export class SourceDefinitionProvider
     const location = sourceMap
       .get(sourceName)
       ?.tables.find((table) => table.name === tableName);
-    if (location) {
+    if (location && location.path) {
       const sourceFile: string = readFileSync(location.path).toString("utf8");
       const sourceFileLines = sourceFile.split("\n");
       for (let index = 0; index < sourceFileLines.length; index++) {

diff --git a/src/domain.ts b/src/domain.ts
@@ -11,7 +11,7 @@ export type NodeMetaType = NodeMetaData;
 export type SourceMetaType = SourceTable;
 
 interface MacroMetaData {
-  path: string;
+  path: string | undefined; // in dbt cloud, packages are not downloaded locally
   line: number;
   character: number;
   uniqueId: string;
@@ -23,7 +23,7 @@ interface MetricMetaData {
 
 export interface NodeMetaData {
   uniqueId: string;
-  path: string;
+  path: string | undefined; // in dbt cloud, packages are not downloaded locally
   database: string;
   schema: string;
   alias: string;
@@ -57,7 +57,7 @@ export interface SourceMetaData {
 export interface SourceTable {
   name: string;
   identifier: string;
-  path: string;
+  path: string | undefined; // in dbt cloud, packages are not downloaded locally
   description: string;
   columns: { [columnName: string]: ColumnMetaData };
 }
@@ -91,7 +91,7 @@ interface DependsOn {
 }
 
 export interface TestMetaData {
-  path: string;
+  path: string | undefined; // in dbt cloud, packages are not downloaded locally
   database: string;
   schema: string;
   alias: string;
@@ -118,7 +118,7 @@ export interface ExposureMetaData {
   url?: string;
   type: string;
   config: { enabled: boolean };
-  path: string;
+  path: string | undefined; // in dbt cloud, packages are not downloaded locally
   unique_id: string;
   sources?: [string];
   metrics?: unknown[];

diff --git a/src/manifest/parsers/nodeParser.ts b/src/manifest/parsers/nodeParser.ts
@@ -57,9 +57,6 @@ export class NodeParser {
           packagePath,
           original_file_path,
         );
-        if (!fullPath) {
-          return;
-        }
         modelMetaMap.set(name, {
           path: fullPath,
           database,

diff --git a/src/services/dbtTestService.ts b/src/services/dbtTestService.ts
@@ -103,10 +103,18 @@ export class DbtTestService {
       return;
     }
 
-    const patchPath = node?.patch_path.includes("://")
+    const patchPath = node?.patch_path?.includes("://")
       ? path.join(project.projectRoot.fsPath, node.patch_path.split("://")[1])
       : node.patch_path;
 
+    if (!patchPath) {
+      this.dbtTerminal.debug(
+        "getDbtTestCode",
+        "unable to find patch path",
+        patchPath,
+      );
+      return null;
+    }
     this.dbtTerminal.debug(
       "getDbtTestCode",
       "finding test from yaml",
@@ -160,7 +168,7 @@ export class DbtTestService {
   }
 
   // Find the file path of test macro
-  public getMacroFilePath = (
+  private getMacroFilePath = (
     macros: [string],
     projectName: string,
     macroMetaMap: MacroMetaMap,
@@ -178,7 +186,7 @@ export class DbtTestService {
     if (macro) {
       // return the file path if it ends with sql
       const macroData = macroMetaMap.get(`test_${testName}`);
-      return macroData?.path.endsWith(".sql") ? macroData?.path : undefined;
+      return macroData?.path?.endsWith(".sql") ? macroData?.path : undefined;
     }
   };
 

diff --git a/src/webview_provider/newDocsGenPanel.ts b/src/webview_provider/newDocsGenPanel.ts
@@ -106,7 +106,7 @@ export class NewDocsGenPanel
     );
 
     return {
-      sql: testPath.endsWith(".sql")
+      sql: testPath?.endsWith(".sql")
         ? readFileSync(testPath, { encoding: "utf-8" })
         : undefined,
       config: this.dbtTestService.getConfigByTest(test, modelName, column_name),

diff --git a/webview_panels/src/modules/documentationEditor/components/tests/DbtTestCode.tsx b/webview_panels/src/modules/documentationEditor/components/tests/DbtTestCode.tsx
@@ -13,18 +13,24 @@ const DbtTestCode = ({ test }: { test: DBTModelTest }): JSX.Element | null => {
   const {
     state: { currentDocsData },
   } = useDocumentationContext();
-  const [testCode, setTestCode] = useState<GetTestCodeResponse | null>(null);
+  const [testCode, setTestCode] = useState<
+    (GetTestCodeResponse & { error?: string }) | null
+  >(null);
 
   panelLogger.info(testCode);
   const loadTestCode = async () => {
     if (!currentDocsData?.name) {
       return;
     }
-    const result = (await executeRequestInSync("getTestCode", {
-      test,
-      model: currentDocsData.name,
-    })) as GetTestCodeResponse;
-    setTestCode(result);
+    try {
+      const result = (await executeRequestInSync("getTestCode", {
+        test,
+        model: currentDocsData.name,
+      })) as GetTestCodeResponse;
+      setTestCode(result);
+    } catch (err) {
+      setTestCode({ error: (err as Error).message });
+    }
   };
 
   useEffect(() => {
@@ -49,6 +55,9 @@ const DbtTestCode = ({ test }: { test: DBTModelTest }): JSX.Element | null => {
       {testCode.sql ? (
         <CodeBlock code={testCode.sql} language="sql" fileName="Source" />
       ) : null}
+      {testCode.error ? (
+        <CodeBlock code={testCode.error} language="yaml" fileName="Source" />
+      ) : null}
     </>
   );
 };

diff --git a/webview_panels/src/modules/documentationEditor/components/tests/hooks/useTestFormSave.ts b/webview_panels/src/modules/documentationEditor/components/tests/hooks/useTestFormSave.ts
@@ -76,7 +76,6 @@ const useTestFormSave = (): {
           database: "",
           schema: "",
           key: `${key}_${params.model}`,
-          path: `${key}_${params.model}`,
           test_metadata: {
             // @ts-expect-error test
             kwargs: {
@@ -105,7 +104,6 @@ const useTestFormSave = (): {
             schema: "",
             column_name: column.name,
             key: `${key}_${column.name}`,
-            path: `${key}_${column.name}`,
             test_metadata: {
               kwargs: {
                 column_name: column.name,
@@ -186,7 +184,6 @@ const useTestFormSave = (): {
         schema: "",
         column_name: column,
         key: `${data.test}_${column}`,
-        path: `${data.test}_${column}`,
         test_metadata: {
           name: data.test!,
           kwargs: {

diff --git a/webview_panels/src/modules/documentationEditor/components/tests/utils.ts b/webview_panels/src/modules/documentationEditor/components/tests/utils.ts
@@ -13,7 +13,7 @@ import {
  */
 export const findDbtTestType = (test: DBTModelTest): DbtTestTypes => {
   if (!test.test_metadata?.name) {
-    if (test.path.endsWith(".sql")) {
+    if (test.path?.endsWith(".sql")) {
       return DbtTestTypes.SINGULAR;
     }
     return DbtTestTypes.UNKNOWN;

diff --git a/webview_panels/src/modules/documentationEditor/state/types.ts b/webview_panels/src/modules/documentationEditor/state/types.ts
@@ -89,7 +89,7 @@ export interface DBTModelTest {
   column_name?: string;
   database: string;
   key: string;
-  path: string;
+  path?: string;
   schema: string;
   test_metadata?: {
     kwargs: TestMetadataAcceptedValuesKwArgs | TestMetadataRelationshipsKwArgs;