From eaccab59274ffb607b80244c243e65bafd7dc228 Mon Sep 17 00:00:00 2001 From: Rohit Kadhe <113367036+rohit-kadhe@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:27:17 -0600 Subject: [PATCH] feature compelete & cleanup (#1) * feature compelete & cleanup * fix * fix --- .github/workflows/prerequisites.yml | 5 ++++- core/clickhouse_schema.ts | 8 +++++--- core/infer_schema_type.ts | 29 +++++------------------------ data_types/ch_array.ts | 11 ++++++----- data_types/ch_boolean.ts | 1 + data_types/ch_date.ts | 7 +++++++ data_types/ch_enum.ts | 7 ++++--- data_types/ch_float.ts | 3 +++ data_types/ch_integer.ts | 8 ++++++++ data_types/ch_json.ts | 5 ++++- data_types/ch_nullable.ts | 16 ++++++++++++++++ data_types/ch_string.ts | 2 ++ data_types/ch_uuid.ts | 1 + data_types/index.ts | 12 +++++++++++- examples/server.ts | 2 +- tests/unit/data_types.test.ts | 5 +++++ utils/type_inference.ts | 7 ------- utils/util.ts | 28 ++++++++++++++++++++++++++++ 18 files changed, 111 insertions(+), 46 deletions(-) create mode 100644 data_types/ch_nullable.ts delete mode 100644 utils/type_inference.ts create mode 100644 utils/util.ts diff --git a/.github/workflows/prerequisites.yml b/.github/workflows/prerequisites.yml index ba905ee..1740ec7 100644 --- a/.github/workflows/prerequisites.yml +++ b/.github/workflows/prerequisites.yml @@ -4,13 +4,16 @@ on: pull_request: types: [synchronize, opened, edited, ready_for_review, reopened, unlocked] branches: [main, development] + push: + branches: [main, development] permissions: checks: write contents: write + pull-requests: write jobs: - test: + Run-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/core/clickhouse_schema.ts b/core/clickhouse_schema.ts index 7f1ec52..4840f84 100644 --- a/core/clickhouse_schema.ts +++ b/core/clickhouse_schema.ts @@ -28,12 +28,14 @@ interface IClickhouseSchema { GetCreateTableQueryAsList: () => string[] } +export type ChSchemaDefinition = Record + /* This class is used to represent a clickhouse table schema */ -export class ClickhouseSchema implements IClickhouseSchema { - readonly schema: ChSchema +export class ClickhouseSchema implements IClickhouseSchema { + readonly schema: SchemaDefinition private readonly options: ChSchemaOptions - constructor (schema: ChSchema, options: ChSchemaOptions) { + constructor (schema: SchemaDefinition, options: ChSchemaOptions) { this.schema = schema this.options = options } diff --git a/core/infer_schema_type.ts b/core/infer_schema_type.ts index 1544fa6..cd7abcc 100644 --- a/core/infer_schema_type.ts +++ b/core/infer_schema_type.ts @@ -1,28 +1,9 @@ -import { type ClickhouseSchema } from '@clickhouse-schema-core/clickhouse_schema' -import { type ChDataType, type MapChSchemaTypes } from '@clickhouse-schema-data-types/index' -import { type ChArray } from '@clickhouse-schema-data-types/ch_array' -import { type ChJSON } from '@clickhouse-schema-data-types/ch_json' -import { type ExtractInnerType, type ExtractOuterType } from '@clickhouse-schema-utils/type_inference' -import { type ChEnum } from '@clickhouse-schema-data-types/ch_enum' +import { type ClickhouseSchema, type ChSchemaDefinition } from '@clickhouse-schema-core/clickhouse_schema' +import { type ChDataType } from '@clickhouse-schema-data-types/index' +import { type InferArray, type InferEnum, type InferJSON, type InferNullable, type InferTypeFromMap } from '@clickhouse-schema-utils/util' /** InferSchemaClickhouseSchemaType is a type that takes a ClickhouseSchema and returns the typescript that it represents */ -export type InferClickhouseSchemaType> = { [K in keyof T['schema']]: InferType } - -/** InferTypeFromMap is a type that takes a string and returns the typescript that it represents */ -type InferTypeFromMap = ExtractInnerType extends keyof MapChSchemaTypes - ? MapChSchemaTypes[ExtractInnerType] - : ExtractOuterType extends keyof MapChSchemaTypes ? - MapChSchemaTypes[ExtractOuterType] - : unknown +export type InferClickhouseSchemaType> = { [K in keyof T['schema']]: InferType } /** InferType is a type that takes a ChDataType and returns the typescript that it represents */ -type InferType = -T extends ChArray - ? Array> - : T extends ChEnum - ? keyof EnumType - : T extends ChJSON - ? { [K in keyof T['dataType']]: InferType } - : T extends ChDataType - ? InferTypeFromMap - : unknown +type InferType = T['dataTypeMarker'] extends 'Array' ? InferArray : T['dataTypeMarker'] extends 'Enum' ? InferEnum : T['dataTypeMarker'] extends 'JSON' ? InferJSON : T['dataTypeMarker'] extends 'Nullable' ? InferNullable : InferTypeFromMap diff --git a/data_types/ch_array.ts b/data_types/ch_array.ts index b6c3778..eb671ad 100644 --- a/data_types/ch_array.ts +++ b/data_types/ch_array.ts @@ -1,20 +1,21 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChArray> implements ChDataType { - readonly dataType: T + readonly innerType: T readonly typeStr: string + readonly dataTypeMarker = 'Array' as const constructor (t: T) { if (t instanceof ChArray) { - this.dataType = new ChArray(t.dataType) as T + this.innerType = new ChArray(t.innerType) as T } else { - this.dataType = t + this.innerType = t } this.typeStr = this.toString() } toString (): string { - if (this.dataType instanceof ChArray) return `Array(${this.dataType.toString()})` - return `Array(${this.dataType})` + if (this.innerType instanceof ChArray) return `Array(${this.innerType.toString()})` + return `Array(${this.innerType})` } } diff --git a/data_types/ch_boolean.ts b/data_types/ch_boolean.ts index 827f72f..e2b2b78 100644 --- a/data_types/ch_boolean.ts +++ b/data_types/ch_boolean.ts @@ -2,6 +2,7 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChBoolean implements ChDataType { readonly typeStr: 'Boolean' = 'Boolean' as const + readonly dataTypeMarker = 'Boolean' as const toString (): string { return this.typeStr diff --git a/data_types/ch_date.ts b/data_types/ch_date.ts index a13431a..00393fc 100644 --- a/data_types/ch_date.ts +++ b/data_types/ch_date.ts @@ -2,6 +2,8 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChDate implements ChDataType { readonly typeStr: 'Date' = 'Date' as const + readonly dataTypeMarker = 'Date' as const + toString (): string { return this.typeStr } @@ -9,6 +11,7 @@ export class ChDate implements ChDataType { export class ChDate32 implements ChDataType { readonly typeStr: 'Date32' = 'Date32' as const + readonly dataTypeMarker = 'Date32' as const toString (): string { return this.typeStr @@ -17,6 +20,8 @@ export class ChDate32 implements ChDataType { export class ChDateTime implements ChDataType { readonly typeStr: `DateTime(${T})` + readonly dataTypeMarker = 'DateTime' as const + constructor (readonly timezone: T) { this.typeStr = `DateTime(${timezone})` } @@ -28,6 +33,8 @@ export class ChDateTime implements ChDataType { export class ChDateTime64 implements ChDataType { readonly typeStr: `DateTime64(${T}, ${V})` + readonly dataTypeMarker = 'DateTime64' as const + constructor (readonly precision: T, readonly timezone: V) { this.typeStr = `DateTime64(${precision}, ${timezone})` } diff --git a/data_types/ch_enum.ts b/data_types/ch_enum.ts index 930954e..637fe8e 100644 --- a/data_types/ch_enum.ts +++ b/data_types/ch_enum.ts @@ -2,14 +2,15 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChEnum> implements ChDataType { readonly typeStr: string - readonly dataType: T + readonly innerType: T + readonly dataTypeMarker = 'Enum' as const constructor (enumObj: T) { - this.dataType = enumObj + this.innerType = enumObj this.typeStr = this.toString() } toString (): string { - return `Enum(${Object.keys(this.dataType).map((key) => `'${key}' = ${this.dataType[key]}`).join(',')})` + return `Enum(${Object.keys(this.innerType).map((key) => `'${key}' = ${this.innerType[key]}`).join(',')})` } } diff --git a/data_types/ch_float.ts b/data_types/ch_float.ts index 1c690cb..b558031 100644 --- a/data_types/ch_float.ts +++ b/data_types/ch_float.ts @@ -2,6 +2,7 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChFloat32 implements ChDataType { readonly typeStr: 'Float32' = 'Float32' as const + readonly dataTypeMarker = 'Float32' as const toString (): string { return this.typeStr @@ -10,6 +11,8 @@ export class ChFloat32 implements ChDataType { export class ChFloat64 implements ChDataType { readonly typeStr: 'Float64' = 'Float64' as const + readonly dataTypeMarker = 'Float64' as const + toString (): string { return this.typeStr } diff --git a/data_types/ch_integer.ts b/data_types/ch_integer.ts index cb437a4..458b1f4 100644 --- a/data_types/ch_integer.ts +++ b/data_types/ch_integer.ts @@ -2,6 +2,7 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChUInt8 implements ChDataType { readonly typeStr: 'UInt8' = 'UInt8' as const + readonly dataTypeMarker = 'UInt8' as const constructor () { this.typeStr = 'UInt8' } @@ -13,6 +14,7 @@ export class ChUInt8 implements ChDataType { export class ChUInt16 implements ChDataType { readonly typeStr: 'UInt16' = 'UInt16' as const + readonly dataTypeMarker = 'UInt16' as const constructor () { this.typeStr = 'UInt16' } @@ -24,6 +26,8 @@ export class ChUInt16 implements ChDataType { export class ChUInt32 implements ChDataType { readonly typeStr: 'UInt32' = 'UInt32' as const + readonly dataTypeMarker = 'UInt32' as const + toString (): string { return this.typeStr } @@ -31,6 +35,8 @@ export class ChUInt32 implements ChDataType { export class ChUInt64 implements ChDataType { readonly typeStr: 'UInt64' = 'UInt64' as const + readonly dataTypeMarker = 'UInt64' as const + toString (): string { return this.typeStr } @@ -38,6 +44,7 @@ export class ChUInt64 implements ChDataType { export class ChInt8 implements ChDataType { readonly typeStr: 'Int8' = 'Int8' as const + readonly dataTypeMarker = 'Int8' as const toString (): string { return this.typeStr @@ -46,6 +53,7 @@ export class ChInt8 implements ChDataType { export class ChInt16 implements ChDataType { readonly typeStr: 'Int16' = 'Int16' as const + readonly dataTypeMarker = 'Int16' as const toString (): string { return this.typeStr diff --git a/data_types/ch_json.ts b/data_types/ch_json.ts index fa4a75c..b0c340f 100644 --- a/data_types/ch_json.ts +++ b/data_types/ch_json.ts @@ -3,8 +3,11 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChJSON> implements ChDataType { readonly typeStr: 'JSON' - constructor (readonly dataType: T) { + readonly innerType: T + readonly dataTypeMarker = 'JSON' as const + constructor (innerType: T) { this.typeStr = 'JSON' + this.innerType = innerType } toString (): string { diff --git a/data_types/ch_nullable.ts b/data_types/ch_nullable.ts new file mode 100644 index 0000000..369d9d3 --- /dev/null +++ b/data_types/ch_nullable.ts @@ -0,0 +1,16 @@ +import { type ChPrimitiveType, type ChDataType } from '@clickhouse-schema-data-types/index' + +export class ChNullable implements ChDataType { + readonly typeStr + readonly innerType: T + readonly dataTypeMarker = 'Nullable' as const + + constructor (t: T) { + this.innerType = t + this.typeStr = `Nullable(${this.innerType})` + } + + toString (): string { + return this.typeStr + } +} diff --git a/data_types/ch_string.ts b/data_types/ch_string.ts index b9221ff..45ec40f 100644 --- a/data_types/ch_string.ts +++ b/data_types/ch_string.ts @@ -2,6 +2,7 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChString implements ChDataType { readonly typeStr: 'String' = 'String' as const + readonly dataTypeMarker = 'String' as const toString (): string { return this.typeStr } @@ -9,6 +10,7 @@ export class ChString implements ChDataType { export class ChFixedString implements ChDataType { readonly typeStr: `FixedString(${string})` + readonly dataTypeMarker = 'FixedString' as const constructor (readonly length: T) { this.typeStr = `FixedString(${length})` } diff --git a/data_types/ch_uuid.ts b/data_types/ch_uuid.ts index 8d4b578..5c92735 100644 --- a/data_types/ch_uuid.ts +++ b/data_types/ch_uuid.ts @@ -2,6 +2,7 @@ import { type ChDataType } from '@clickhouse-schema-data-types/index' export class ChUUID implements ChDataType { readonly typeStr: 'UUID' = 'UUID' as const + readonly dataTypeMarker = 'UUID' as const toString (): string { return this.typeStr } diff --git a/data_types/index.ts b/data_types/index.ts index 2d00b0a..f0581bd 100644 --- a/data_types/index.ts +++ b/data_types/index.ts @@ -8,9 +8,11 @@ import { ChUInt8, ChUInt16, ChUInt32, ChUInt64, ChInt8, ChInt16 } from '@clickho import { ChJSON } from '@clickhouse-schema-data-types/ch_json' import { ChString, ChFixedString } from '@clickhouse-schema-data-types/ch_string' import { ChUUID } from '@clickhouse-schema-data-types/ch_uuid' +import { ChNullable } from '@clickhouse-schema-data-types/ch_nullable' export interface ChDataType { typeStr: string + dataTypeMarker: string toString: () => string } @@ -33,7 +35,8 @@ export const ClickhouseTypes = { FixedString: (length: T) => new ChFixedString(length), JSON: >(schema: T) => new ChJSON(schema), Array: (t: T) => new ChArray(t), - Enum: >(enumObj: T) => new ChEnum(enumObj) + Enum: >(enumObj: T) => new ChEnum(enumObj), + Nullable: (type: T) => new ChNullable(type) } export interface MapChSchemaTypes { @@ -56,3 +59,10 @@ export interface MapChSchemaTypes { DateTime64: Date FixedString: string } + +export type ChPrimitiveType = +ChBoolean | ChUInt8 | ChUInt16 | ChUInt32 | ChUInt64 | ChInt8 | ChInt16 | +ChFloat32 | ChFloat64 | ChString | ChUUID | ChDate | ChDate32 | ChDateTime | +ChDateTime64 | ChFixedString + +export type ChCompositeType = ChArray | ChDataType> | ChEnum> | ChNullable diff --git a/examples/server.ts b/examples/server.ts index 8bb915f..9e1b163 100644 --- a/examples/server.ts +++ b/examples/server.ts @@ -11,7 +11,7 @@ const requestFieldsTable = new ClickhouseSchema({ status_code: { type: ClickhouseTypes.UInt32 }, url: { type: ClickhouseTypes.String }, request_method: { type: ClickhouseTypes.Enum({ GET: 0, POST: 1, PUT: 2, DELETE: 3 }) }, - request_time: { type: ClickhouseTypes.DateTime('UTC') } + request_time: { type: ClickhouseTypes.Nullable(ClickhouseTypes.DateTime('UTC')) } }, { table_name: 'request_data', primary_key: 'id' }) type RequestFields = InferClickhouseSchemaType diff --git a/tests/unit/data_types.test.ts b/tests/unit/data_types.test.ts index cb77bdb..dda7cad 100644 --- a/tests/unit/data_types.test.ts +++ b/tests/unit/data_types.test.ts @@ -90,4 +90,9 @@ describe('Data Types Tests', () => { const enumType = new ChEnum({ POST: 1, PUT: 2, DELETE: 3, GET: 4 }) expect(enumType.toString()).toEqual('Enum(\'POST\' = 1,\'PUT\' = 2,\'DELETE\' = 3,\'GET\' = 4)') }) + + it('should correctly create a nullable data types with the correct typeStr', () => { + const nullableString = ClickhouseTypes.Nullable(ClickhouseTypes.String) + expect(nullableString.toString()).toEqual('Nullable(String)') + }) }) diff --git a/utils/type_inference.ts b/utils/type_inference.ts deleted file mode 100644 index 38afd42..0000000 --- a/utils/type_inference.ts +++ /dev/null @@ -1,7 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ -export type ExtractInnerType = T extends `${infer _BeforeBracket}(${infer Rest})` - ? ExtractInnerType - : T -export type ExtractOuterType = T extends `${infer BeforeBracket}(${infer _Rest})` - ? BeforeBracket - : T diff --git a/utils/util.ts b/utils/util.ts new file mode 100644 index 0000000..76ddb98 --- /dev/null +++ b/utils/util.ts @@ -0,0 +1,28 @@ +import { type ChArray } from '@clickhouse-schema-data-types/ch_array' +import { type ChEnum } from '@clickhouse-schema-data-types/ch_enum' +import { type ChJSON } from '@clickhouse-schema-data-types/ch_json' +import { type ChNullable } from '@clickhouse-schema-data-types/ch_nullable' +import { type MapChSchemaTypes, type ChDataType } from '@clickhouse-schema-data-types/index' + +/* eslint-disable @typescript-eslint/no-unused-vars */ +export type ExtractInnerType = T extends `${infer _BeforeBracket}(${infer Rest})` + ? ExtractInnerType + : T +export type ExtractOuterType = T extends `${infer BeforeBracket}(${infer _Rest})` + ? BeforeBracket + : T + +/** InferTypeFromMap is a type that takes a string and returns the typescript that it represents */ +export type InferTypeFromMap = ExtractInnerType extends keyof MapChSchemaTypes + ? MapChSchemaTypes[ExtractInnerType] + : ExtractOuterType extends keyof MapChSchemaTypes ? + MapChSchemaTypes[ExtractOuterType] + : unknown + +export type InferArray = T extends ChArray ? Array> : InferTypeFromMap + +export type InferEnum = T extends ChEnum ? keyof EnumType : InferTypeFromMap + +export type InferJSON = T extends ChJSON ? { [K in keyof T['innerType']]: InferJSON } : InferTypeFromMap + +export type InferNullable = T extends ChNullable ? InferTypeFromMap | null : InferTypeFromMap