Skip to content

Commit

Permalink
feat: Adding support for having clause (#56)
Browse files Browse the repository at this point in the history
* having support added

* reverted test case changes

* reusing traverse and filter

* removed unneeded function

* code refactor

* bumped up versions

* fixed test cases

* fixed test cases

* self review

* added test case for having/where filter
  • Loading branch information
zaidjan-devrev authored Jan 25, 2024
1 parent 1dd17d4 commit db41f2b
Show file tree
Hide file tree
Showing 13 changed files with 406 additions and 135 deletions.
2 changes: 1 addition & 1 deletion meerkat-core/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@devrev/meerkat-core",
"version": "0.0.62",
"version": "0.0.63",
"dependencies": {
"@swc/helpers": "~0.5.0"
},
Expand Down
62 changes: 46 additions & 16 deletions meerkat-core/src/ast-builder/ast-builder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,44 @@ import { cubeFilterToDuckdbAST } from '../cube-filter-transformer/factory';
import { cubeDimensionToGroupByAST } from '../cube-group-by-transformer/cube-group-by-transformer';
import { cubeLimitOffsetToAST } from '../cube-limit-offset-transformer/cube-limit-offset-transformer';
import { cubeOrderByToAST } from '../cube-order-by-transformer/cube-order-by-transformer';
import { QueryFiltersWithInfo } from '../cube-to-duckdb/cube-filter-to-duckdb';
import { FilterType, Query } from '../types/cube-types/query';
import { QueryFiltersWithInfo, QueryFiltersWithInfoSingular } from '../cube-to-duckdb/cube-filter-to-duckdb';
import { traverseAndFilter } from '../filter-params/filter-params-ast';
import { FilterType, MeerkatQueryFilter, Query } from '../types/cube-types/query';
import { TableSchema } from '../types/cube-types/table';
import { SelectStatement } from '../types/duckdb-serialization-types';
import { SelectNode } from '../types/duckdb-serialization-types/serialization/QueryNode';
import { getBaseAST } from '../utils/base-ast';
import { cubeFiltersEnrichment } from '../utils/cube-filter-enrichment';
import { modifyLeafMeerkatFilter } from '../utils/modify-meerkat-filter';


const formatFilters = (queryFiltersWithInfo: QueryFiltersWithInfo, filterType?: FilterType) => {
/*
* If the type of filter is set to base filter where
*/
return filterType === 'BASE_FILTER' ? queryFiltersWithInfo : modifyLeafMeerkatFilter(queryFiltersWithInfo, (item) => {
return {
...item,
member: item.member.split('.').join('__')
};
}) as QueryFiltersWithInfo;
}


const getFormattedFilters = ({ queryFiltersWithInfo, filterType, mapperFn, baseAST }: {
queryFiltersWithInfo: QueryFiltersWithInfo,
filterType?: FilterType,
baseAST: SelectStatement,
mapperFn: (val: QueryFiltersWithInfoSingular) => MeerkatQueryFilter | null
}) => {
const filters = queryFiltersWithInfo.map(item => mapperFn(item)).filter(Boolean) as QueryFiltersWithInfoSingular[];
const formattedFilters = formatFilters(filters, filterType);
return cubeFilterToDuckdbAST(
formattedFilters,
baseAST
);
}

export const cubeToDuckdbAST = (query: Query, tableSchema: TableSchema, options?: { filterType: FilterType }
) => {
/**
Expand All @@ -35,23 +64,24 @@ export const cubeToDuckdbAST = (query: Query, tableSchema: TableSchema, options?
return null;
}

/*
* If the type of filter is set to base filter where
*/
const finalFilters = options?.filterType === 'BASE_FILTER' ? queryFiltersWithInfo : modifyLeafMeerkatFilter(queryFiltersWithInfo, (item) => {
return {
...item,
member: item.member.split('.').join('__')
};
}) as QueryFiltersWithInfo;
const whereClause = getFormattedFilters({
baseAST,
mapperFn: (item) => traverseAndFilter(item, (value) => !query.measures.includes(value.member)),
queryFiltersWithInfo,
filterType: options?.filterType
})

const havingClause = getFormattedFilters({
baseAST,
mapperFn: (item) => traverseAndFilter(item, (value) => query.measures.includes(value.member)),
queryFiltersWithInfo,
filterType: options?.filterType
})

const duckdbWhereClause = cubeFilterToDuckdbAST(
finalFilters,
baseAST
);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
//@ts-ignore
node.where_clause = duckdbWhereClause;
node.where_clause = whereClause;
node.having = havingClause
}

if (query.dimensions && query.dimensions?.length > 0) {
Expand Down
12 changes: 6 additions & 6 deletions meerkat-core/src/cube-to-duckdb/cube-filter-to-duckdb.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Dimension, Measure } from '../types/cube-types/table';
import { QueryFilter } from '../types/cube-types/query';
import { Dimension, Measure } from '../types/cube-types/table';

export type QueryOperatorsWithInfo = QueryFilter & {
memberInfo: Measure | Dimension;
Expand Down Expand Up @@ -28,8 +28,8 @@ export type QueryFilterWithInfo =
| LogicalOrFilterWithInfo
)[];

export type QueryFiltersWithInfo = (
| QueryOperatorsWithInfo
| LogicalAndFilterWithInfo
| LogicalOrFilterWithInfo
)[];
export type QueryFiltersWithInfoSingular = QueryOperatorsWithInfo
| LogicalAndFilterWithInfo
| LogicalOrFilterWithInfo;

export type QueryFiltersWithInfo = QueryFiltersWithInfoSingular[];
14 changes: 8 additions & 6 deletions meerkat-core/src/filter-params/filter-params-ast.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,26 @@ import {
LogicalOrFilter,
MeerkatQueryFilter,
Query,
QueryFilter,
TableSchema
} from '../types/cube-types';
import { SelectStatement } from '../types/duckdb-serialization-types/serialization/Statement';

/**
* Get the query filter with only where filterKey matches
*/
const traverseAndFilter = (

export const traverseAndFilter = (
filter: MeerkatQueryFilter,
memberKey: string
callback: (value: QueryFilter) => boolean
): MeerkatQueryFilter | null => {
if ('member' in filter) {
return filter.member === memberKey ? filter : null;
return callback(filter) ? filter : null;
}

if ('and' in filter) {
const filteredAndFilters = filter.and
.map((subFilter) => traverseAndFilter(subFilter, memberKey))
.map((subFilter) => traverseAndFilter(subFilter, callback))
.filter(Boolean) as MeerkatQueryFilter[];
const obj =
filteredAndFilters.length > 0 ? { and: filteredAndFilters } : null;
Expand All @@ -31,7 +33,7 @@ const traverseAndFilter = (

if ('or' in filter) {
const filteredOrFilters = filter.or
.map((subFilter) => traverseAndFilter(subFilter, memberKey))
.map((subFilter) => traverseAndFilter(subFilter, callback))
.filter(Boolean);
const obj = filteredOrFilters.length > 0 ? { or: filteredOrFilters } : null;
return obj as LogicalOrFilter;
Expand All @@ -47,7 +49,7 @@ export const getFilterByMemberKey = (
): MeerkatQueryFilter[] => {
if (!filters) return [];
return filters
.map((filter) => traverseAndFilter(filter, memberKey))
.map((filter) => traverseAndFilter(filter, (value) => value.member === memberKey))
.filter(Boolean) as MeerkatQueryFilter[];
};

Expand Down
Original file line number Diff line number Diff line change
@@ -1,22 +1,20 @@
import { get } from "http";
import { TableSchema } from "../types/cube-types";
import { getMemberProjection, getProjectionClause } from './get-projection-clause';
import { getDimensionProjection, getFilterMeasureProjection, getProjectionClause } from './get-projection-clause';


const TABLE_SCHEMA: TableSchema = {
dimensions: [{ name: 'a', sql: 'others', type: 'number' }, { name: 'c', sql: 'any', type: 'string' }],
measures: [],
measures: [{ name: 'x', sql: 'x', type: 'number' }, { name: 'y', sql: 'y', type: 'number' }, { name: 'z', sql: 'z', type: 'number' }],
name: 'test',
sql: 'SELECT * from test'
// Define your table schema here
};
describe("get-projection-clause", () => {
describe("getMemberProjection", () => {
describe("getDimensionProjection", () => {
it("should return the member projection when the key exists in the table schema", () => {
const key = "test.a";


const result = getMemberProjection({ key, tableSchema: TABLE_SCHEMA });
const result = getDimensionProjection({ key, tableSchema: TABLE_SCHEMA });
expect(result).toEqual({ aliasKey: "test__a", foundMember: {"name": "a", "sql": "others", "type": "number"}, sql: "others AS test__a"});
});

Expand All @@ -27,25 +25,48 @@ describe("get-projection-clause", () => {
dimensions: [{ name: 'b', sql: 'others', type: 'number' }],
};

const result = getMemberProjection({ key, tableSchema });
const result = getDimensionProjection({ key, tableSchema });
expect(result).toEqual({ aliasKey: undefined, foundMember: undefined, sql: undefined });
});
})

describe("getFilterMeasureProjection", () => {
it("should return the member projection when the key exists in the table schema", () => {
const key = "test.x";
const result = getFilterMeasureProjection({ key, tableSchema: TABLE_SCHEMA, measures: ['test.a']});
expect(result).toEqual({ aliasKey: "test__x", foundMember: {"name": "x", "sql": "x", "type": "number"}, sql: "test.x AS test__x"});
});

it("should not create alias when item in measure list", () => {
const key = "test.x";
const result = getFilterMeasureProjection({ key, tableSchema: TABLE_SCHEMA, measures: ['test.x']});
expect(result).toEqual({ aliasKey: undefined, foundMember: undefined, sql: undefined});
});

it("should return the object with undefined values when the key doesn't exist in the table schema", () => {
const key = "test.a";
const tableSchema: TableSchema = {
...TABLE_SCHEMA,
measures: [{ name: 'b', sql: 'others', type: 'number' }],
};

const result = getFilterMeasureProjection({ key, tableSchema, measures: ['test.b'] });
expect(result).toEqual({ aliasKey: undefined, foundMember: undefined, sql: undefined });
});
})

describe("getProjectionClause", () => {
it('should return the projection clause when the members are present in the table schema', () => {
const members = ['test.a', 'test.c'];
const tableSchema = TABLE_SCHEMA;
const aliasedColumnSet = new Set<string>();
const result = getProjectionClause(members, tableSchema, aliasedColumnSet);
expect(result).toEqual(', others AS test__a, any AS test__c');
const result = getProjectionClause([], members, TABLE_SCHEMA, aliasedColumnSet);
expect(result).toEqual('others AS test__a, any AS test__c');
})
it('should skip aliased items present in already seen', () => {
const members = ['test.a', 'test.c'];
const tableSchema = TABLE_SCHEMA;
const aliasedColumnSet = new Set<string>(['test.c']);
const result = getProjectionClause(members, tableSchema, aliasedColumnSet);
expect(result).toEqual(', others AS test__a');
const result = getProjectionClause([], members, TABLE_SCHEMA, aliasedColumnSet);
expect(result).toEqual('others AS test__a, ');
})
})

Expand Down
101 changes: 75 additions & 26 deletions meerkat-core/src/get-projection-clause/get-projection-clause.ts
Original file line number Diff line number Diff line change
@@ -1,36 +1,85 @@
import { TableSchema } from "../types/cube-types";
import { findInSchema } from "../utils/find-in-table-schema";
import { findInDimensionSchema, findInMeasureSchema } from "../utils/find-in-table-schema";
import { memberKeyToSafeKey } from "../utils/member-key-to-safe-key";

export const getMemberProjection = ({ key, tableSchema }: {
key: string;
tableSchema: TableSchema;
}) => {
// Find the table access key
const measureWithoutTable = key.split('.')[1];

const foundMember = findInSchema(measureWithoutTable, tableSchema)
if (!foundMember) {
// If the selected member is not found in the table schema or if it is already selected, continue.
return {
sql: undefined,
foundMember: undefined,
aliasKey: undefined
}

export const getFilterMeasureProjection = ({ key, tableSchema, measures }: {
key: string;
tableSchema: TableSchema;
measures: string[]
}) => {
const measureWithoutTable = key.split('.')[1];
const foundMember = findInMeasureSchema(measureWithoutTable, tableSchema);
const isMeasure = measures.includes(key);
if (!foundMember || isMeasure) {
// If the selected member is not found in the table schema or if it is already selected, continue.
// If the selected member is a measure, don't create an alias. Since measure computation is done in the outermost level of the query
return {
sql: undefined,
foundMember: undefined,
aliasKey: undefined
}
const aliasKey = memberKeyToSafeKey(key);
// Add the alias key to the set. So we have a reference to all the previously selected members.
return { sql: `${foundMember.sql} AS ${aliasKey}` , foundMember, aliasKey }
}
const aliasKey = memberKeyToSafeKey(key);
return { sql: `${key} AS ${aliasKey}` , foundMember, aliasKey }
}

export const getDimensionProjection = ({ key, tableSchema }: {
key: string;
tableSchema: TableSchema;
}) => {
// Find the table access key
const measureWithoutTable = key.split('.')[1];

export const getProjectionClause = (members: string[], tableSchema: TableSchema, aliasedColumnSet: Set<string>) => {
return members.reduce((acc, member) => {
const { sql: memberSql } = getMemberProjection({ key: member, tableSchema })
if (aliasedColumnSet.has(member)) {
return acc
const foundMember = findInDimensionSchema(measureWithoutTable, tableSchema)
if (!foundMember) {
// If the selected member is not found in the table schema or if it is already selected, continue.
return {
sql: undefined,
foundMember: undefined,
aliasKey: undefined
}
acc += `, ${memberSql}`
}
const aliasKey = memberKeyToSafeKey(key);
// Add the alias key to the set. So we have a reference to all the previously selected members.
return { sql: `${foundMember.sql} AS ${aliasKey}` , foundMember, aliasKey }
}

const aggregator = ({
member,
aliasedColumnSet,
acc,
currentIndex,
members,
sql
}: {
member: string;
aliasedColumnSet: Set<string>;
acc: string;
sql?: string;
currentIndex: number,
members: string[]
}) => {
if (aliasedColumnSet.has(member) || !sql) {
return acc
}
aliasedColumnSet.add(member)
acc += sql
if (currentIndex !== members.length - 1) {
acc += `, `
}
return acc
}


export const getProjectionClause = (measures: string[], dimensions: string[], tableSchema: TableSchema, aliasedColumnSet: Set<string>) => {
const dimensionsProjections = dimensions.reduce((acc, member, currentIndex, members) => {
const { sql: memberSql } = getDimensionProjection({ key: member, tableSchema })
return aggregator({ member, aliasedColumnSet, acc, currentIndex, members, sql: memberSql })
}, '')
}
const measureProjections = measures.reduce((acc, member, currentIndex, members) => {
const { sql: memberSql } = getFilterMeasureProjection({ key: member, tableSchema, measures })
return aggregator({ member, aliasedColumnSet, acc, currentIndex, members, sql: memberSql })
}, '')
return dimensionsProjections + (dimensionsProjections.length && measureProjections.length ? ', ' : '') + measureProjections
}
Loading

0 comments on commit db41f2b

Please sign in to comment.