From bb0e54eb1b470a9e82a8e69377ef1f0ab41206dd Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Mon, 20 May 2024 23:51:40 +0800 Subject: [PATCH] Make path optional (#1143) * Make path optional * fix: make path optional for doc editor tests data * fix: for lineage * Only take models with path --------- Co-authored-by: saravmajestic Co-authored-by: Aditya Co-authored-by: anandgupta42 <93243293+anandgupta42@users.noreply.github.com> --- new_lineage_panel/src/App.tsx | 16 ++------------ new_lineage_panel/src/CustomNodes.tsx | 6 +++++- new_lineage_panel/src/service.ts | 2 +- src/commands/tests/missingSchemaTest.ts | 2 +- .../tests/undocumentedModelColumnTest.ts | 3 +++ src/commands/tests/unmaterializedModelTest.ts | 3 ++- .../macroDefinitionProvider.ts | 2 +- .../modelDefinitionProvider.ts | 2 +- .../sourceDefinitionProvider.ts | 2 +- src/domain.ts | 10 ++++----- src/manifest/parsers/nodeParser.ts | 3 --- src/services/dbtTestService.ts | 14 ++++++++++--- src/webview_provider/newDocsGenPanel.ts | 2 +- .../components/tests/DbtTestCode.tsx | 21 +++++++++++++------ .../components/tests/hooks/useTestFormSave.ts | 3 --- .../components/tests/utils.ts | 2 +- .../documentationEditor/state/types.ts | 2 +- 17 files changed, 51 insertions(+), 44 deletions(-) diff --git a/new_lineage_panel/src/App.tsx b/new_lineage_panel/src/App.tsx index 06aa1feb8..d4c2990c5 100644 --- a/new_lineage_panel/src/App.tsx +++ b/new_lineage_panel/src/App.tsx @@ -50,7 +50,7 @@ import { handleResponse, init, columnLineage } from "./service_utils"; import { ActionWidget } from "./ActionWidget"; import { DEFAULT_MIN_ZOOM, createTableNode } from "./utils"; import { Settings } from "./Settings"; -import { getLineageSettings } from "./service"; +import { Table, getLineageSettings } from "./service"; export let aiEnabled = false; export let isDarkMode = false; @@ -160,19 +160,7 @@ function App() { const [minRange, setMinRange] = useState<[number, number]>([0, 0]); useEffect(() => { - const render = async (args: { - node: { - table: string; - label: string; - url: string; - nodeType: string; - materialization?: string; - downstreamCount: number; - upstreamCount: number; - tests: { key: string; path: string }[]; - }; - aiEnabled: boolean; - }) => { + const render = async (args: { node: Table; aiEnabled: boolean }) => { setIsOpen(false); setSidebarScreen(""); if (!args) return; diff --git a/new_lineage_panel/src/CustomNodes.tsx b/new_lineage_panel/src/CustomNodes.tsx index 25e77fdb4..d37abf529 100644 --- a/new_lineage_panel/src/CustomNodes.tsx +++ b/new_lineage_panel/src/CustomNodes.tsx @@ -99,6 +99,7 @@ export const TableNode: FunctionComponent = ({ data }) => { rightExpansion, selectCheck, nonSelectCheck, + setSelectedTable, } = useContext(LineageContext); const _columnLen = Object.keys(collectColumns[table] || {}).length; @@ -207,7 +208,10 @@ export const TableNode: FunctionComponent = ({ data }) => { ) ); highlightTable(); - openFile(url); + setSelectedTable(table); + if (url) { + openFile(url); + } }} >
table.name === tableName); - if (location) { + if (location && location.path) { const sourceFile: string = readFileSync(location.path).toString("utf8"); const sourceFileLines = sourceFile.split("\n"); for (let index = 0; index < sourceFileLines.length; index++) { diff --git a/src/domain.ts b/src/domain.ts index 52425f534..3be04dc9f 100755 --- a/src/domain.ts +++ b/src/domain.ts @@ -11,7 +11,7 @@ export type NodeMetaType = NodeMetaData; export type SourceMetaType = SourceTable; interface MacroMetaData { - path: string; + path: string | undefined; // in dbt cloud, packages are not downloaded locally line: number; character: number; uniqueId: string; @@ -23,7 +23,7 @@ interface MetricMetaData { export interface NodeMetaData { uniqueId: string; - path: string; + path: string | undefined; // in dbt cloud, packages are not downloaded locally database: string; schema: string; alias: string; @@ -57,7 +57,7 @@ export interface SourceMetaData { export interface SourceTable { name: string; identifier: string; - path: string; + path: string | undefined; // in dbt cloud, packages are not downloaded locally description: string; columns: { [columnName: string]: ColumnMetaData }; } @@ -91,7 +91,7 @@ interface DependsOn { } export interface TestMetaData { - path: string; + path: string | undefined; // in dbt cloud, packages are not downloaded locally database: string; schema: string; alias: string; @@ -118,7 +118,7 @@ export interface ExposureMetaData { url?: string; type: string; config: { enabled: boolean }; - path: string; + path: string | undefined; // in dbt cloud, packages are not downloaded locally unique_id: string; sources?: [string]; metrics?: unknown[]; diff --git a/src/manifest/parsers/nodeParser.ts b/src/manifest/parsers/nodeParser.ts index 7c63a0155..9690df86a 100755 --- a/src/manifest/parsers/nodeParser.ts +++ b/src/manifest/parsers/nodeParser.ts @@ -57,9 +57,6 @@ export class NodeParser { packagePath, original_file_path, ); - if (!fullPath) { - return; - } modelMetaMap.set(name, { path: fullPath, database, diff --git a/src/services/dbtTestService.ts b/src/services/dbtTestService.ts index ff75a0588..77adf05c8 100644 --- a/src/services/dbtTestService.ts +++ b/src/services/dbtTestService.ts @@ -103,10 +103,18 @@ export class DbtTestService { return; } - const patchPath = node?.patch_path.includes("://") + const patchPath = node?.patch_path?.includes("://") ? path.join(project.projectRoot.fsPath, node.patch_path.split("://")[1]) : node.patch_path; + if (!patchPath) { + this.dbtTerminal.debug( + "getDbtTestCode", + "unable to find patch path", + patchPath, + ); + return null; + } this.dbtTerminal.debug( "getDbtTestCode", "finding test from yaml", @@ -160,7 +168,7 @@ export class DbtTestService { } // Find the file path of test macro - public getMacroFilePath = ( + private getMacroFilePath = ( macros: [string], projectName: string, macroMetaMap: MacroMetaMap, @@ -178,7 +186,7 @@ export class DbtTestService { if (macro) { // return the file path if it ends with sql const macroData = macroMetaMap.get(`test_${testName}`); - return macroData?.path.endsWith(".sql") ? macroData?.path : undefined; + return macroData?.path?.endsWith(".sql") ? macroData?.path : undefined; } }; diff --git a/src/webview_provider/newDocsGenPanel.ts b/src/webview_provider/newDocsGenPanel.ts index fee045272..58930c10f 100644 --- a/src/webview_provider/newDocsGenPanel.ts +++ b/src/webview_provider/newDocsGenPanel.ts @@ -106,7 +106,7 @@ export class NewDocsGenPanel ); return { - sql: testPath.endsWith(".sql") + sql: testPath?.endsWith(".sql") ? readFileSync(testPath, { encoding: "utf-8" }) : undefined, config: this.dbtTestService.getConfigByTest(test, modelName, column_name), diff --git a/webview_panels/src/modules/documentationEditor/components/tests/DbtTestCode.tsx b/webview_panels/src/modules/documentationEditor/components/tests/DbtTestCode.tsx index b7779d2e0..abbedb69d 100644 --- a/webview_panels/src/modules/documentationEditor/components/tests/DbtTestCode.tsx +++ b/webview_panels/src/modules/documentationEditor/components/tests/DbtTestCode.tsx @@ -13,18 +13,24 @@ const DbtTestCode = ({ test }: { test: DBTModelTest }): JSX.Element | null => { const { state: { currentDocsData }, } = useDocumentationContext(); - const [testCode, setTestCode] = useState(null); + const [testCode, setTestCode] = useState< + (GetTestCodeResponse & { error?: string }) | null + >(null); panelLogger.info(testCode); const loadTestCode = async () => { if (!currentDocsData?.name) { return; } - const result = (await executeRequestInSync("getTestCode", { - test, - model: currentDocsData.name, - })) as GetTestCodeResponse; - setTestCode(result); + try { + const result = (await executeRequestInSync("getTestCode", { + test, + model: currentDocsData.name, + })) as GetTestCodeResponse; + setTestCode(result); + } catch (err) { + setTestCode({ error: (err as Error).message }); + } }; useEffect(() => { @@ -49,6 +55,9 @@ const DbtTestCode = ({ test }: { test: DBTModelTest }): JSX.Element | null => { {testCode.sql ? ( ) : null} + {testCode.error ? ( + + ) : null} ); }; diff --git a/webview_panels/src/modules/documentationEditor/components/tests/hooks/useTestFormSave.ts b/webview_panels/src/modules/documentationEditor/components/tests/hooks/useTestFormSave.ts index 048810047..74dac966b 100644 --- a/webview_panels/src/modules/documentationEditor/components/tests/hooks/useTestFormSave.ts +++ b/webview_panels/src/modules/documentationEditor/components/tests/hooks/useTestFormSave.ts @@ -76,7 +76,6 @@ const useTestFormSave = (): { database: "", schema: "", key: `${key}_${params.model}`, - path: `${key}_${params.model}`, test_metadata: { // @ts-expect-error test kwargs: { @@ -105,7 +104,6 @@ const useTestFormSave = (): { schema: "", column_name: column.name, key: `${key}_${column.name}`, - path: `${key}_${column.name}`, test_metadata: { kwargs: { column_name: column.name, @@ -186,7 +184,6 @@ const useTestFormSave = (): { schema: "", column_name: column, key: `${data.test}_${column}`, - path: `${data.test}_${column}`, test_metadata: { name: data.test!, kwargs: { diff --git a/webview_panels/src/modules/documentationEditor/components/tests/utils.ts b/webview_panels/src/modules/documentationEditor/components/tests/utils.ts index b0c11dbc1..d16a74311 100644 --- a/webview_panels/src/modules/documentationEditor/components/tests/utils.ts +++ b/webview_panels/src/modules/documentationEditor/components/tests/utils.ts @@ -13,7 +13,7 @@ import { */ export const findDbtTestType = (test: DBTModelTest): DbtTestTypes => { if (!test.test_metadata?.name) { - if (test.path.endsWith(".sql")) { + if (test.path?.endsWith(".sql")) { return DbtTestTypes.SINGULAR; } return DbtTestTypes.UNKNOWN; diff --git a/webview_panels/src/modules/documentationEditor/state/types.ts b/webview_panels/src/modules/documentationEditor/state/types.ts index 106b5e3ab..168d6a644 100644 --- a/webview_panels/src/modules/documentationEditor/state/types.ts +++ b/webview_panels/src/modules/documentationEditor/state/types.ts @@ -89,7 +89,7 @@ export interface DBTModelTest { column_name?: string; database: string; key: string; - path: string; + path?: string; schema: string; test_metadata?: { kwargs: TestMetadataAcceptedValuesKwArgs | TestMetadataRelationshipsKwArgs;