refactor: unifying node and browser code (#115)

* unifiying node and browser code * bumped package versions * formatting fix * added test case for get-filter-params-sql * added test cases * reverted uneeded changes
devrev · Nov 7, 2024 · 2ad2540 · 2ad2540
1 parent 61708d0
commit 2ad2540
Show file tree

Hide file tree

Showing 10 changed files with 307 additions and 145 deletions.
diff --git a/meerkat-browser/package.json b/meerkat-browser/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-browser",
-  "version": "0.0.83",
+  "version": "0.0.84",
   "dependencies": {
     "@swc/helpers": "~0.5.0",
     "@devrev/meerkat-core": "*",

diff --git a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts
@@ -1,7 +1,6 @@
 import {
   BASE_TABLE_NAME,
   ContextParams,
-  FilterType,
   Query,
   TableSchema,
   applyFilterParamsToBaseSQL,
@@ -11,95 +10,39 @@ import {
   deserializeQuery,
   detectApplyContextParamsToBaseSQL,
   getCombinedTableSchema,
-  getFilterParamsAST,
-  getWrappedBaseQueryWithProjections,
+  getFilterParamsSQL,
+  getFinalBaseSQL
 } from '@devrev/meerkat-core';
 import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
 
-const getFilterParamsSQL = async ({
-  query,
-  tableSchema,
-  filterType,
-  connection,
-}: {
-  query: Query;
-  tableSchema: TableSchema;
-  filterType?: FilterType;
-  connection: AsyncDuckDBConnection;
-}) => {
-  const filterParamsAST = getFilterParamsAST(
-    query,
-    tableSchema,
-    filterType
-  );
-  const filterParamsSQL = [];
-
-  for (const filterParamAST of filterParamsAST) {
-    if (!filterParamAST.ast) {
-      continue;
-    }
 
-    const queryOutput = await connection.query(
-      astDeserializerQuery(filterParamAST.ast)
-    );
-    const parsedOutputQuery = queryOutput.toArray().map((row) => row.toJSON());
+const getQueryOutput = async (query: string, connection: AsyncDuckDBConnection) => {
+  const queryOutput = await connection.query(query);
+  const parsedOutputQuery = queryOutput.toArray().map((row) => row.toJSON());
+  return parsedOutputQuery;
+}
 
-    const sql = deserializeQuery(parsedOutputQuery);
 
-    filterParamsSQL.push({
-      memberKey: filterParamAST.memberKey,
-      sql: sql,
-      matchKey: filterParamAST.matchKey,
-    });
-  }
-  return filterParamsSQL;
-};
-
-const getFinalBaseSQL = async (
+interface CubeQueryToSQLParams {
+  connection: AsyncDuckDBConnection,
   query: Query,
-  tableSchema: TableSchema,
-  connection: AsyncDuckDBConnection
-) => {
-  /**
-   * Apply transformation to the supplied base query.
-   * This involves updating the filter placeholder with the actual filter values.
-   */
-  const baseFilterParamsSQL = await getFilterParamsSQL({
-    query: query,
-    tableSchema,
-    filterType: 'BASE_FILTER',
-    connection,
-  });
-  const baseSQL = applyFilterParamsToBaseSQL(
-    tableSchema.sql,
-    baseFilterParamsSQL
-  );
-  const baseSQLWithFilterProjection = getWrappedBaseQueryWithProjections({
-    baseQuery: baseSQL,
-    tableSchema,
-    query: query,
-  });
-  return baseSQLWithFilterProjection;
-};
+  tableSchemas: TableSchema[],
+  contextParams?: ContextParams,
+}
 
 export const cubeQueryToSQL = async ({
   connection,
   query,
   tableSchemas,
   contextParams
-}: {
-  connection: AsyncDuckDBConnection,
-  query: Query,
-  tableSchemas: TableSchema[],
-  contextParams?: ContextParams
-}) => {
+}: CubeQueryToSQLParams) => {
   const updatedTableSchemas: TableSchema[] = await Promise.all(
     tableSchemas.map(async (schema: TableSchema) => {
-      const baseFilterParamsSQL = await getFinalBaseSQL(
+      const baseFilterParamsSQL = await getFinalBaseSQL({
         query,
-        schema,
-        connection
-      );
+        tableSchema: schema,
+        getQueryOutput: (query) => getQueryOutput(query, connection)
+      });
       return {
         ...schema,
         sql: baseFilterParamsSQL,
@@ -124,7 +67,7 @@ export const cubeQueryToSQL = async ({
 
   const preBaseQuery = deserializeQuery(parsedOutputQuery);
   const filterParamsSQL = await getFilterParamsSQL({
-    connection,
+    getQueryOutput: (query) => getQueryOutput(query, connection),
     query,
     tableSchema: updatedTableSchema,
     filterType: 'BASE_FILTER',

diff --git a/meerkat-core/package.json b/meerkat-core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-core",
-  "version": "0.0.83",
+  "version": "0.0.84",
   "dependencies": {
     "@swc/helpers": "~0.5.0"
   },

diff --git a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.spec.ts b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.spec.ts
@@ -0,0 +1,132 @@
+import { Database } from 'duckdb';
+import { TableSchema } from '../types/cube-types';
+import { getFilterParamsSQL } from './get-filter-params-sql';
+
+const getQueryOutput = async (sql: string) => {
+  const db = new Database(':memory:')
+  return new Promise((resolve, reject) => {
+    db.all(sql, (err, res) => {
+      if (err) {
+        reject(err);
+      }
+      resolve(res);
+    });
+  });
+}
+
+const TABLE_SCHEMA: TableSchema = {
+  name: 'orders',
+  sql: "select * from orders WHERE ${FILTER_PARAMS.orders.status.filter('status')}",
+  measures: [],
+  dimensions: [
+    { name: 'id', sql: 'id', type: 'number' },
+    { name: 'date', sql: 'date', type: 'string' },
+    { name: 'status', sql: 'status', type: 'string' },
+    { name: 'amount', sql: 'amount', type: 'number' }
+  ]
+}
+describe('getFilterParamsSQL', () => {
+  it('should find filter params when there are filters of base filter type', async () => {
+    const result = await getFilterParamsSQL({
+      filterType: 'BASE_FILTER',
+      query:  {
+        measures: [ '*' ],
+        filters: [
+          {
+            and: [
+              { member: 'orders.amount', operator: 'notSet' },
+              { member: 'orders.status', operator: 'set' }
+            ]
+          }
+        ],
+        dimensions: []
+      },
+      tableSchema: TABLE_SCHEMA,
+      getQueryOutput,
+    });
+    expect(result).toEqual([
+        {
+          "matchKey": "${FILTER_PARAMS.orders.status.filter('status')}",
+          "memberKey": "orders.status",
+          "sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders.status IS NOT NULL))",
+        },
+      ]);
+  });
+  it('should not find filter params when there are no filters of base filter type', async () => {
+    const result = await getFilterParamsSQL({
+      filterType: 'BASE_FILTER',
+      query:  {
+        measures: [ '*' ],
+        filters: [],
+        dimensions: []
+      },
+      tableSchema: TABLE_SCHEMA,
+      getQueryOutput,
+    });
+    expect(result).toEqual([]);
+  });
+  it('should find filter params when there are filters of projection filter type', async () => {
+    const result = await getFilterParamsSQL({
+      filterType: 'PROJECTION_FILTER',
+      query:  {
+        measures: [ '*' ],
+        filters: [
+          {
+            and: [
+              { member: 'orders.amount', operator: 'notSet' },
+              { member: 'orders.status', operator: 'set' }
+            ]
+          }
+        ],
+        dimensions: []
+      },
+      tableSchema: TABLE_SCHEMA,
+      getQueryOutput,
+    });
+    expect(result).toEqual([
+        {
+          "matchKey": "${FILTER_PARAMS.orders.status.filter('status')}",
+          "memberKey": "orders.status",
+          "sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders__status IS NOT NULL))",
+        },
+      ]);
+  });
+  it('should not find filter params when there are no filters', async () => {
+    const result = await getFilterParamsSQL({
+      filterType: 'PROJECTION_FILTER',
+      query:  {
+        measures: [ '*' ],
+        filters: [],
+        dimensions: []
+      },
+      tableSchema: TABLE_SCHEMA,
+      getQueryOutput,
+    });
+    expect(result).toEqual([]);
+  });
+  it('should find filter params when there are filters of no defined type', async () => {
+    const result = await getFilterParamsSQL({
+      query:  {
+        measures: [ '*' ],
+        filters: [
+          {
+            and: [
+              { member: 'orders.amount', operator: 'notSet' },
+              { member: 'orders.status', operator: 'set' }
+            ]
+          }
+        ],
+        dimensions: []
+      },
+      tableSchema: TABLE_SCHEMA,
+      getQueryOutput,
+    });
+    expect(result).toEqual([
+        {
+          "matchKey": "${FILTER_PARAMS.orders.status.filter('status')}",
+          "memberKey": "orders.status",
+          "sql": "SELECT * FROM REPLACE_BASE_TABLE WHERE ((orders__status IS NOT NULL))",
+        },
+      ]);
+  });
+});
diff --git a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts
@@ -0,0 +1,37 @@
+import { astDeserializerQuery, deserializeQuery } from "../ast-deserializer/ast-deserializer";
+import { getFilterParamsAST } from "../filter-params/filter-params-ast";
+import { FilterType, Query, TableSchema } from "../types/cube-types";
+
+export const getFilterParamsSQL = async ({
+  query,
+  tableSchema,
+  filterType,
+  getQueryOutput
+}: {
+  query: Query;
+  tableSchema: TableSchema;
+  filterType?: FilterType;
+  getQueryOutput: (query: string) => Promise<any>;
+}) => {
+  const filterParamsAST = getFilterParamsAST(
+    query,
+    tableSchema,
+    filterType
+  );
+  const filterParamsSQL = [];
+  for (const filterParamAST of filterParamsAST) {
+    if (!filterParamAST.ast) {
+      continue;
+    }
+
+    const queryOutput = await getQueryOutput(astDeserializerQuery(filterParamAST.ast))
+    const sql = deserializeQuery(queryOutput);
+
+    filterParamsSQL.push({
+      memberKey: filterParamAST.memberKey,
+      sql: sql,
+      matchKey: filterParamAST.matchKey,
+    });
+  }
+  return filterParamsSQL;
+};
diff --git a/meerkat-core/src/get-final-base-sql/get-final-base-sql.spec.ts b/meerkat-core/src/get-final-base-sql/get-final-base-sql.spec.ts
@@ -0,0 +1,69 @@
+import { Database } from 'duckdb';
+import { TableSchema } from '../types/cube-types';
+import { getFinalBaseSQL } from './get-final-base-sql';
+
+const getQueryOutput = async (sql: string) => {
+  const db = new Database(':memory:')
+  return new Promise((resolve, reject) => {
+    db.all(sql, (err, res) => {
+      if (err) {
+        reject(err);
+      }
+      resolve(res);
+    });
+  });
+}
+
+const TABLE_SCHEMA: TableSchema = {
+    name: 'orders',
+    sql: "select * from orders WHERE ${FILTER_PARAMS.orders.status.filter('status')}",
+    measures: [
+      { name: 'count', sql: 'COUNT(*)', type: 'number' }
+    ],
+    dimensions: [
+      { name: 'id', sql: 'id', type: 'number' },
+      { name: 'date', sql: 'date', type: 'string' },
+      { name: 'status', sql: 'status', type: 'string' },
+      { name: 'amount', sql: 'amount', type: 'number' }
+    ]
+  }
+  describe('get final base sql', () => {
+    it('should not return measures in the projected base sql when filter param passed', async () => {
+      const result = await getFinalBaseSQL({
+        query:  {
+          measures: ['orders.count'],
+          filters: [
+            {
+              and: [
+                { member: 'orders.amount', operator: 'notSet' },
+                { member: 'orders.status', operator: 'set' }
+              ]
+            }
+          ],
+          dimensions: ['orders.status']
+        },
+        tableSchema: TABLE_SCHEMA,
+        getQueryOutput,
+      });
+      expect(result).toEqual('SELECT *, amount AS orders__amount, status AS orders__status FROM (select * from orders WHERE  ((orders.status IS NOT NULL))) AS orders');
+    });
+    it('should not return measures in the projected base sql when filter param not passed', async () => {
+        const result = await getFinalBaseSQL({
+          query:  {
+            measures: ['orders.count'],
+            filters: [
+              {
+                and: [
+                  { member: 'orders.amount', operator: 'notSet' },
+                ]
+              }
+            ],
+            dimensions: ['orders.status']
+          },
+          tableSchema: TABLE_SCHEMA,
+          getQueryOutput,
+        });
+        expect(result).toEqual('SELECT *, amount AS orders__amount, status AS orders__status FROM (select * from orders WHERE TRUE) AS orders');
+      });
+  });
+