From 2ad254033c65ed00c2650b2770450883f6bd2f59 Mon Sep 17 00:00:00 2001 From: Zaid Jan <98650818+zaidjan-devrev@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:14:30 +0530 Subject: [PATCH] refactor: unifying node and browser code (#115) * unifiying node and browser code * bumped package versions * formatting fix * added test case for get-filter-params-sql * added test cases * reverted uneeded changes --- meerkat-browser/package.json | 2 +- .../browser-cube-to-sql.ts | 93 +++--------- meerkat-core/package.json | 2 +- .../get-filter-params-sql.spec.ts | 132 ++++++++++++++++++ .../get-filter-params-sql.ts | 37 +++++ .../get-final-base-sql.spec.ts | 69 +++++++++ .../get-final-base-sql/get-final-base-sql.ts | 33 +++++ meerkat-core/src/index.ts | 5 +- meerkat-node/package.json | 2 +- meerkat-node/src/cube-to-sql/cube-to-sql.ts | 77 ++-------- 10 files changed, 307 insertions(+), 145 deletions(-) create mode 100644 meerkat-core/src/get-filter-params-sql/get-filter-params-sql.spec.ts create mode 100644 meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts create mode 100644 meerkat-core/src/get-final-base-sql/get-final-base-sql.spec.ts create mode 100644 meerkat-core/src/get-final-base-sql/get-final-base-sql.ts diff --git a/meerkat-browser/package.json b/meerkat-browser/package.json index 33ecf465..4c674116 100644 --- a/meerkat-browser/package.json +++ b/meerkat-browser/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-browser", - "version": "0.0.83", + "version": "0.0.84", "dependencies": { "@swc/helpers": "~0.5.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts index 20bfaba7..14fa2463 100644 --- a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts +++ b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts @@ -1,7 +1,6 @@ import { BASE_TABLE_NAME, ContextParams, - FilterType, Query, TableSchema, applyFilterParamsToBaseSQL, @@ -11,95 +10,39 @@ import { deserializeQuery, detectApplyContextParamsToBaseSQL, getCombinedTableSchema, - getFilterParamsAST, - getWrappedBaseQueryWithProjections, + getFilterParamsSQL, + getFinalBaseSQL } from '@devrev/meerkat-core'; import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; -const getFilterParamsSQL = async ({ - query, - tableSchema, - filterType, - connection, -}: { - query: Query; - tableSchema: TableSchema; - filterType?: FilterType; - connection: AsyncDuckDBConnection; -}) => { - const filterParamsAST = getFilterParamsAST( - query, - tableSchema, - filterType - ); - const filterParamsSQL = []; - - for (const filterParamAST of filterParamsAST) { - if (!filterParamAST.ast) { - continue; - } - const queryOutput = await connection.query( - astDeserializerQuery(filterParamAST.ast) - ); - const parsedOutputQuery = queryOutput.toArray().map((row) => row.toJSON()); +const getQueryOutput = async (query: string, connection: AsyncDuckDBConnection) => { + const queryOutput = await connection.query(query); + const parsedOutputQuery = queryOutput.toArray().map((row) => row.toJSON()); + return parsedOutputQuery; +} - const sql = deserializeQuery(parsedOutputQuery); - filterParamsSQL.push({ - memberKey: filterParamAST.memberKey, - sql: sql, - matchKey: filterParamAST.matchKey, - }); - } - return filterParamsSQL; -}; - -const getFinalBaseSQL = async ( +interface CubeQueryToSQLParams { + connection: AsyncDuckDBConnection, query: Query, - tableSchema: TableSchema, - connection: AsyncDuckDBConnection -) => { - /** - * Apply transformation to the supplied base query. - * This involves updating the filter placeholder with the actual filter values. - */ - const baseFilterParamsSQL = await getFilterParamsSQL({ - query: query, - tableSchema, - filterType: 'BASE_FILTER', - connection, - }); - const baseSQL = applyFilterParamsToBaseSQL( - tableSchema.sql, - baseFilterParamsSQL - ); - const baseSQLWithFilterProjection = getWrappedBaseQueryWithProjections({ - baseQuery: baseSQL, - tableSchema, - query: query, - }); - return baseSQLWithFilterProjection; -}; + tableSchemas: TableSchema[], + contextParams?: ContextParams, +} export const cubeQueryToSQL = async ({ connection, query, tableSchemas, contextParams -}: { - connection: AsyncDuckDBConnection, - query: Query, - tableSchemas: TableSchema[], - contextParams?: ContextParams -}) => { +}: CubeQueryToSQLParams) => { const updatedTableSchemas: TableSchema[] = await Promise.all( tableSchemas.map(async (schema: TableSchema) => { - const baseFilterParamsSQL = await getFinalBaseSQL( + const baseFilterParamsSQL = await getFinalBaseSQL({ query, - schema, - connection - ); + tableSchema: schema, + getQueryOutput: (query) => getQueryOutput(query, connection) + }); return { ...schema, sql: baseFilterParamsSQL, @@ -124,7 +67,7 @@ export const cubeQueryToSQL = async ({ const preBaseQuery = deserializeQuery(parsedOutputQuery); const filterParamsSQL = await getFilterParamsSQL({ - connection, + getQueryOutput: (query) => getQueryOutput(query, connection), query, tableSchema: updatedTableSchema, filterType: 'BASE_FILTER', diff --git a/meerkat-core/package.json b/meerkat-core/package.json index d243b20a..6c6586a6 100644 --- a/meerkat-core/package.json +++ b/meerkat-core/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-core", - "version": "0.0.83", + "version": "0.0.84", "dependencies": { "@swc/helpers": "~0.5.0" }, diff --git a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.spec.ts b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.spec.ts new file mode 100644 index 00000000..4a214d11 --- /dev/null +++ b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.spec.ts @@ -0,0 +1,132 @@ +import { Database } from 'duckdb'; +import { TableSchema } from '../types/cube-types'; +import { getFilterParamsSQL } from './get-filter-params-sql'; + +const getQueryOutput = async (sql: string) => { + const db = new Database(':memory:') + return new Promise((resolve, reject) => { + db.all(sql, (err, res) => { + if (err) { + reject(err); + } + resolve(res); + }); + }); +} + +const TABLE_SCHEMA: TableSchema = { + name: 'orders', + sql: "select * from orders WHERE ${FILTER_PARAMS.orders.status.filter('status')}", + measures: [], + dimensions: [ + { name: 'id', sql: 'id', type: 'number' }, + { name: 'date', sql: 'date', type: 'string' }, + { name: 'status', sql: 'status', type: 'string' }, + { name: 'amount', sql: 'amount', type: 'number' } + ] +} +describe('getFilterParamsSQL', () => { + it('should find filter params when there are filters of base filter type', async () => { + const result = await getFilterParamsSQL({ + filterType: 'BASE_FILTER', + query: { + measures: [ '*' ], + filters: [ + { + and: [ + { member: 'orders.amount', operator: 'notSet' }, + { member: 'orders.status', operator: 'set' } + ] + } + ], + dimensions: [] + }, + tableSchema: TABLE_SCHEMA, + getQueryOutput, + }); + expect(result).toEqual([ + { + "matchKey": "${FILTER_PARAMS.orders.status.filter('status')}", + "memberKey": "orders.status", + "sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders.status IS NOT NULL))", + }, + ]); + }); + it('should not find filter params when there are no filters of base filter type', async () => { + const result = await getFilterParamsSQL({ + filterType: 'BASE_FILTER', + query: { + measures: [ '*' ], + filters: [], + dimensions: [] + }, + tableSchema: TABLE_SCHEMA, + getQueryOutput, + }); + expect(result).toEqual([]); + }); + it('should find filter params when there are filters of projection filter type', async () => { + const result = await getFilterParamsSQL({ + filterType: 'PROJECTION_FILTER', + query: { + measures: [ '*' ], + filters: [ + { + and: [ + { member: 'orders.amount', operator: 'notSet' }, + { member: 'orders.status', operator: 'set' } + ] + } + ], + dimensions: [] + }, + tableSchema: TABLE_SCHEMA, + getQueryOutput, + }); + expect(result).toEqual([ + { + "matchKey": "${FILTER_PARAMS.orders.status.filter('status')}", + "memberKey": "orders.status", + "sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders__status IS NOT NULL))", + }, + ]); + }); + it('should not find filter params when there are no filters', async () => { + const result = await getFilterParamsSQL({ + filterType: 'PROJECTION_FILTER', + query: { + measures: [ '*' ], + filters: [], + dimensions: [] + }, + tableSchema: TABLE_SCHEMA, + getQueryOutput, + }); + expect(result).toEqual([]); + }); + it('should find filter params when there are filters of no defined type', async () => { + const result = await getFilterParamsSQL({ + query: { + measures: [ '*' ], + filters: [ + { + and: [ + { member: 'orders.amount', operator: 'notSet' }, + { member: 'orders.status', operator: 'set' } + ] + } + ], + dimensions: [] + }, + tableSchema: TABLE_SCHEMA, + getQueryOutput, + }); + expect(result).toEqual([ + { + "matchKey": "${FILTER_PARAMS.orders.status.filter('status')}", + "memberKey": "orders.status", + "sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders__status IS NOT NULL))", + }, + ]); + }); +}); diff --git a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts new file mode 100644 index 00000000..8e9d9b23 --- /dev/null +++ b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts @@ -0,0 +1,37 @@ +import { astDeserializerQuery, deserializeQuery } from "../ast-deserializer/ast-deserializer"; +import { getFilterParamsAST } from "../filter-params/filter-params-ast"; +import { FilterType, Query, TableSchema } from "../types/cube-types"; + +export const getFilterParamsSQL = async ({ + query, + tableSchema, + filterType, + getQueryOutput +}: { + query: Query; + tableSchema: TableSchema; + filterType?: FilterType; + getQueryOutput: (query: string) => Promise; +}) => { + const filterParamsAST = getFilterParamsAST( + query, + tableSchema, + filterType + ); + const filterParamsSQL = []; + for (const filterParamAST of filterParamsAST) { + if (!filterParamAST.ast) { + continue; + } + + const queryOutput = await getQueryOutput(astDeserializerQuery(filterParamAST.ast)) + const sql = deserializeQuery(queryOutput); + + filterParamsSQL.push({ + memberKey: filterParamAST.memberKey, + sql: sql, + matchKey: filterParamAST.matchKey, + }); + } + return filterParamsSQL; +}; diff --git a/meerkat-core/src/get-final-base-sql/get-final-base-sql.spec.ts b/meerkat-core/src/get-final-base-sql/get-final-base-sql.spec.ts new file mode 100644 index 00000000..b7f531a2 --- /dev/null +++ b/meerkat-core/src/get-final-base-sql/get-final-base-sql.spec.ts @@ -0,0 +1,69 @@ +import { Database } from 'duckdb'; +import { TableSchema } from '../types/cube-types'; +import { getFinalBaseSQL } from './get-final-base-sql'; + +const getQueryOutput = async (sql: string) => { + const db = new Database(':memory:') + return new Promise((resolve, reject) => { + db.all(sql, (err, res) => { + if (err) { + reject(err); + } + resolve(res); + }); + }); +} + +const TABLE_SCHEMA: TableSchema = { + name: 'orders', + sql: "select * from orders WHERE ${FILTER_PARAMS.orders.status.filter('status')}", + measures: [ + { name: 'count', sql: 'COUNT(*)', type: 'number' } + ], + dimensions: [ + { name: 'id', sql: 'id', type: 'number' }, + { name: 'date', sql: 'date', type: 'string' }, + { name: 'status', sql: 'status', type: 'string' }, + { name: 'amount', sql: 'amount', type: 'number' } + ] + } + describe('get final base sql', () => { + it('should not return measures in the projected base sql when filter param passed', async () => { + const result = await getFinalBaseSQL({ + query: { + measures: ['orders.count'], + filters: [ + { + and: [ + { member: 'orders.amount', operator: 'notSet' }, + { member: 'orders.status', operator: 'set' } + ] + } + ], + dimensions: ['orders.status'] + }, + tableSchema: TABLE_SCHEMA, + getQueryOutput, + }); + expect(result).toEqual('SELECT *, amount AS orders__amount, status AS orders__status FROM (select * from orders WHERE ((orders.status IS NOT NULL))) AS orders'); + }); + it('should not return measures in the projected base sql when filter param not passed', async () => { + const result = await getFinalBaseSQL({ + query: { + measures: ['orders.count'], + filters: [ + { + and: [ + { member: 'orders.amount', operator: 'notSet' }, + ] + } + ], + dimensions: ['orders.status'] + }, + tableSchema: TABLE_SCHEMA, + getQueryOutput, + }); + expect(result).toEqual('SELECT *, amount AS orders__amount, status AS orders__status FROM (select * from orders WHERE TRUE) AS orders'); + }); + }); + \ No newline at end of file diff --git a/meerkat-core/src/get-final-base-sql/get-final-base-sql.ts b/meerkat-core/src/get-final-base-sql/get-final-base-sql.ts new file mode 100644 index 00000000..ae0031f3 --- /dev/null +++ b/meerkat-core/src/get-final-base-sql/get-final-base-sql.ts @@ -0,0 +1,33 @@ +import { applyFilterParamsToBaseSQL } from "../filter-params/filter-params-ast"; +import { getFilterParamsSQL } from "../get-filter-params-sql/get-filter-params-sql"; +import { getWrappedBaseQueryWithProjections } from "../get-wrapped-base-query-with-projections/get-wrapped-base-query-with-projections"; +import { Query } from "../types/cube-types/query"; +import { TableSchema } from "../types/cube-types/table"; + +export const getFinalBaseSQL = async ({ + query, + getQueryOutput, + tableSchema, +}: { query: Query, tableSchema: TableSchema, getQueryOutput: (query: string) => Promise }) => { + /** + * Apply transformation to the supplied base query. + * This involves updating the filter placeholder with the actual filter values. + */ + const baseFilterParamsSQL = await getFilterParamsSQL({ + query: query, + tableSchema, + filterType: 'BASE_FILTER', + getQueryOutput, + }); + const baseSQL = applyFilterParamsToBaseSQL( + tableSchema.sql, + baseFilterParamsSQL + ); + const baseSQLWithFilterProjection = getWrappedBaseQueryWithProjections({ + baseQuery: baseSQL, + tableSchema, + query: query, + }); + return baseSQLWithFilterProjection; +}; + diff --git a/meerkat-core/src/index.ts b/meerkat-core/src/index.ts index 7e8dffc2..9d5df04c 100644 --- a/meerkat-core/src/index.ts +++ b/meerkat-core/src/index.ts @@ -6,8 +6,10 @@ export * from './cube-to-duckdb/cube-filter-to-duckdb'; export { applyFilterParamsToBaseSQL, detectAllFilterParamsFromSQL, - getFilterParamsAST, + getFilterParamsAST } from './filter-params/filter-params-ast'; +export { getFilterParamsSQL } from './get-filter-params-sql/get-filter-params-sql'; +export { getFinalBaseSQL } from './get-final-base-sql/get-final-base-sql'; export { getWrappedBaseQueryWithProjections } from './get-wrapped-base-query-with-projections/get-wrapped-base-query-with-projections'; export * from './joins/joins'; export { FilterType } from './types/cube-types'; @@ -18,3 +20,4 @@ export * from './utils/cube-to-table-schema'; export * from './utils/get-possible-nodes'; export { meerkatPlaceholderReplacer } from './utils/meerkat-placeholder-replacer'; export { memberKeyToSafeKey } from './utils/member-key-to-safe-key'; + diff --git a/meerkat-node/package.json b/meerkat-node/package.json index ba902610..a2eff235 100644 --- a/meerkat-node/package.json +++ b/meerkat-node/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-node", - "version": "0.0.83", + "version": "0.0.84", "dependencies": { "@swc/helpers": "~0.5.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-node/src/cube-to-sql/cube-to-sql.ts b/meerkat-node/src/cube-to-sql/cube-to-sql.ts index 4bafa288..eb6b56ca 100644 --- a/meerkat-node/src/cube-to-sql/cube-to-sql.ts +++ b/meerkat-node/src/cube-to-sql/cube-to-sql.ts @@ -1,7 +1,6 @@ import { BASE_TABLE_NAME, ContextParams, - FilterType, Query, TableSchema, applyFilterParamsToBaseSQL, @@ -11,82 +10,27 @@ import { deserializeQuery, detectApplyContextParamsToBaseSQL, getCombinedTableSchema, - getFilterParamsAST, - getWrappedBaseQueryWithProjections, + getFilterParamsSQL, + getFinalBaseSQL } from '@devrev/meerkat-core'; import { duckdbExec } from '../duckdb-exec'; -const getFilterParamsSQL = async ({ - query, - tableSchema, - filterType, -}: { - query: Query; - tableSchema: TableSchema; - filterType?: FilterType; -}) => { - const filterParamsAST = getFilterParamsAST( - query, - tableSchema, - filterType - ); - const filterParamsSQL = []; - for (const filterParamAST of filterParamsAST) { - if (!filterParamAST.ast) { - continue; - } - - const queryOutput = await duckdbExec< - { - [key: string]: string; - }[] - >(astDeserializerQuery(filterParamAST.ast)); - - const sql = deserializeQuery(queryOutput); - filterParamsSQL.push({ - memberKey: filterParamAST.memberKey, - sql: sql, - matchKey: filterParamAST.matchKey, - }); - } - return filterParamsSQL; -}; -const getFinalBaseSQL = async (query: Query, tableSchema: TableSchema) => { - /** - * Apply transformation to the supplied base query. - * This involves updating the filter placeholder with the actual filter values. - */ - const baseFilterParamsSQL = await getFilterParamsSQL({ - query: query, - tableSchema, - filterType: 'BASE_FILTER', - }); - const baseSQL = applyFilterParamsToBaseSQL( - tableSchema.sql, - baseFilterParamsSQL - ); - const baseSQLWithFilterProjection = getWrappedBaseQueryWithProjections({ - baseQuery: baseSQL, - tableSchema, - query: query, - }); - return baseSQLWithFilterProjection; -}; +interface CubeQueryToSQLParams { + query: Query, + tableSchemas: TableSchema[], + contextParams?: ContextParams +} export const cubeQueryToSQL = async ({ query, tableSchemas, - contextParams -}: { - query: Query, - tableSchemas: TableSchema[], - contextParams?: ContextParams -}) => { + contextParams, +}: CubeQueryToSQLParams) => { const updatedTableSchemas: TableSchema[] = await Promise.all( tableSchemas.map(async (schema: TableSchema) => { - const baseFilterParamsSQL = await getFinalBaseSQL(query, schema); + const baseFilterParamsSQL = await getFinalBaseSQL({query, tableSchema: schema, getQueryOutput: duckdbExec }); return { ...schema, sql: baseFilterParamsSQL, @@ -116,6 +60,7 @@ export const cubeQueryToSQL = async ({ const filterParamsSQL = await getFilterParamsSQL({ query, tableSchema: updatedTableSchema, + getQueryOutput: duckdbExec, }); const filterParamQuery = applyFilterParamsToBaseSQL(