diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index 9bb7ba27..c7d5541d 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -40,7 +40,6 @@ import { DEFAULT_TOPK, HybridSearchReq, promisify, - findKeyValue, sleep, parseToKeyValue, checkCollectionName, @@ -157,7 +156,7 @@ export class Data extends Collection { name: v.name, type: v.data_type, // milvus return string here elementType: v.element_type, - dim: Number(findKeyValue(v.type_params, 'dim')), + dim: Number(v.dim), data: [], // values container nullable: v.nullable, default_value: v.default_value, diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts index 2b410d07..65f9512a 100644 --- a/milvus/types/Collection.ts +++ b/milvus/types/Collection.ts @@ -16,27 +16,6 @@ import { FunctionType, } from '../'; -// returned from milvus -export interface FieldSchema { - type_params: KeyValuePair[]; - index_params: KeyValuePair[]; - fieldID: string | number; - name: string; - is_primary_key: boolean; - description: string; - data_type: keyof typeof DataType; - autoID: boolean; - state: string; - element_type?: keyof typeof DataType; - default_value?: number | string; - dataType: DataType; - is_partition_key?: boolean; - is_dynamic?: boolean; - is_clustering_key?: boolean; - nullable?: boolean; - is_function_output: boolean; -} - export interface CollectionData { name: string; id: string; @@ -59,11 +38,38 @@ export interface ReplicaInfo { node_ids: string[]; } -export type TypeParam = string | number | Record; -export type TypeParamKey = 'dim' | 'max_length' | 'max_capacity'; +export type TypeParam = string | number | boolean | Record; +export type TypeParamKey = + | 'dim' + | 'max_length' + | 'max_capacity' + | 'analyzer_params' + | 'enable_analyzer' + | 'enable_match'; + +// returned from milvus +export type FieldSchema = { + type_params: KeyValuePair[]; + index_params: KeyValuePair[]; + fieldID: string | number; + name: string; + is_primary_key: boolean; + description: string; + data_type: keyof typeof DataType; + autoID: boolean; + state: string; + element_type?: keyof typeof DataType; + default_value?: number | string; + dataType: DataType; + is_partition_key?: boolean; + is_dynamic?: boolean; + is_clustering_key?: boolean; + nullable?: boolean; + is_function_output: boolean; +} & Partial>; // create collection -export interface FieldType { +export type FieldType = { name: string; description?: string; data_type: DataType | keyof typeof DataTypeMap; @@ -72,19 +78,11 @@ export interface FieldType { is_partition_key?: boolean; is_function_output?: boolean; is_clustering_key?: boolean; - type_params?: { - [key: string]: TypeParam; - }; + type_params?: Partial>; autoID?: boolean; - dim?: TypeParam; - max_capacity?: TypeParam; - max_length?: TypeParam; default_value?: number | string; nullable?: boolean; - enable_match?: boolean; - tokenizer_params?: Record; - enable_analyzer?: boolean; -} +} & Partial>; export interface ShowCollectionsReq extends GrpcTimeOut { type?: ShowCollectionsType; diff --git a/milvus/types/Common.ts b/milvus/types/Common.ts index 08b839c8..0841b631 100644 --- a/milvus/types/Common.ts +++ b/milvus/types/Common.ts @@ -11,9 +11,9 @@ export interface MsgBase { }; } -export interface KeyValuePair { - key: string; - value: string | number; +export interface KeyValuePair { + key: T; + value: U; } interface NumberArray { diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index 6e614029..2a8e63ef 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -41,10 +41,11 @@ import { f32ArrayToF16Bytes, bf16BytesToF32Array, f16BytesToF32Array, - TypeParam, SearchDataType, FieldSchema, SearchMultipleDataType, + TypeParamKey, + TypeParam, } from '../'; /** @@ -198,7 +199,7 @@ export const formatAddress = (address: string) => { */ export const assignTypeParams = ( field: FieldType, - typeParamKeys: string[] = [ + typeParamKeys: TypeParamKey[] = [ 'dim', 'max_length', 'max_capacity', @@ -210,7 +211,7 @@ export const assignTypeParams = ( const newField = cloneObj(field); // Initialize `type_params` if undefined - newField.type_params ??= {}; + newField.type_params ??= {} as Record; typeParamKeys.forEach(key => { if (key in newField) { @@ -413,6 +414,17 @@ export const formatDescribedCol = ( // add a dataType property which indicate datatype number newData.schema?.fields?.forEach(f => { f.dataType = DataTypeMap[f.data_type]; + // if default_value is set, parse it to the correct format + if (f.default_value) { + const defaultValue = f.default_value as any; + f.default_value = defaultValue[defaultValue.data]; + } + // extract type params(key value pair = {key: 'xxx', value: any}), and assign it to the field object(key) + if (f.type_params && f.type_params.length > 0) { + f.type_params.forEach(keyValuePair => { + f[keyValuePair.key] = keyValuePair.value; + }); + } }); return newData; diff --git a/test/grpc/Collection.spec.ts b/test/grpc/Collection.spec.ts index 8f79d6bb..5fe785b5 100644 --- a/test/grpc/Collection.spec.ts +++ b/test/grpc/Collection.spec.ts @@ -79,7 +79,9 @@ describe(`Collection API`, () => { expect( Boolean( - formatKeyValueData(describe.properties, ['mmap.enabled'])['mmap.enabled'] + formatKeyValueData(describe.properties, ['mmap.enabled'])[ + 'mmap.enabled' + ] ) ).toEqual(true); diff --git a/test/grpc/DynamicSchema.spec.ts b/test/grpc/DynamicSchema.spec.ts index 530a08f9..94949f72 100644 --- a/test/grpc/DynamicSchema.spec.ts +++ b/test/grpc/DynamicSchema.spec.ts @@ -53,6 +53,20 @@ describe(`Dynamic schema API`, () => { const describe = await milvusClient.describeCollection({ collection_name: COLLECTION, }); + + // find varchar field + const varCharField = describe.schema.fields.find( + v => v.name === 'varChar' + )!; + // find int64 field + const int64Field = describe.schema.fields.find( + v => v.name === 'default_value' + )!; + + // test default value + expect(varCharField.default_value).toEqual(DEFAULT_STRING_VALUE); + expect(int64Field.default_value).toEqual(DEFAULT_NUM_VALUE); + expect(describe.schema.enable_dynamic_field).toEqual(true); }); @@ -115,6 +129,7 @@ describe(`Dynamic schema API`, () => { expect(query.status.error_code).toEqual(ErrorCode.SUCCESS); expect(query.data.length).toEqual(10); + // get test values const varChars = query.data.map(v => v.varChar); const defaultValues = query.data.map(v => v.default_value); @@ -123,13 +138,11 @@ describe(`Dynamic schema API`, () => { const bools = query.data.map(v => v.bool); const floats = query.data.map(v => v.float); // some of floats should be equal to DEFAULT_NUM_VALUE - expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true); + expect(floats.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true); // some of varChar should be equal to DEFAULT_STRING_VALUE expect(varChars.some(v => v === DEFAULT_STRING_VALUE)).toEqual(true); // some of default_value should be equal to DEFAULT_NUM_VALUE - expect(defaultValues.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual( - true - ); + expect(defaultValues.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true); // some of json should be null expect(jsons.some(v => v === null)).toEqual(true); // some of bools should be null @@ -173,13 +186,11 @@ describe(`Dynamic schema API`, () => { const bools = search.results.map(v => v.bool); const floats = search.results.map(v => v.float); // some of floats should be equal to DEFAULT_NUM_VALUE - expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true); + expect(floats.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true); // some of varChar should be equal to DEFAULT_STRING_VALUE expect(varChars.some(v => v === DEFAULT_STRING_VALUE)).toEqual(true); // some of default_value should be equal to DEFAULT_NUM_VALUE - expect(defaultValues.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual( - true - ); + expect(defaultValues.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true); // some of json should be null expect(jsons.some(v => v === null)).toEqual(true); // some of bools should be null diff --git a/test/tools/collection.ts b/test/tools/collection.ts index a77d733a..22bb4ef4 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -122,8 +122,8 @@ export const genCollectionParams = (data: { name: 'default_value', nullable: true, default_value: DEFAULT_NUM_VALUE, - description: 'int64 field', - data_type: 'Int64', // + description: 'int32 field', + data_type: 'Int32', // }, { name: 'varChar', diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index 756529da..a350a2ec 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -440,6 +440,7 @@ describe('utils/format', () => { is_clustering_key: false, is_function_output: false, nullable: false, + default_value: { long_data: 123, data: 'long_data' } as any, is_partition_key: false, }, { @@ -453,12 +454,37 @@ describe('utils/format', () => { dataType: 5, autoID: true, state: 'created', + default_value: { string_data: 'd', data: 'string_data' } as any, is_dynamic: false, is_clustering_key: false, is_function_output: false, nullable: false, is_partition_key: false, }, + { + fieldID: '3', + type_params: [ + { key: 'enable_match', value: 'true' }, + { key: 'enable_analyzer', value: 'true' }, + { key: 'dim', value: '3' }, + { key: 'analyzer_params', value: '{}' }, + { key: 'max_capacity', value: '23' }, + ], + index_params: [], + name: 'vector', + is_primary_key: false, + description: 'vector field', + data_type: 'FloatVector', + dataType: 101, + autoID: false, + state: 'created', + is_dynamic: false, + is_clustering_key: false, + is_function_output: false, + nullable: false, + default_value: { bool_data: false, data: 'bool_data' } as any, + is_partition_key: false, + }, ], name: 'collection_v8mt0v7x', description: '', @@ -481,7 +507,16 @@ describe('utils/format', () => { const formatted = formatDescribedCol(response); expect(formatted.schema.fields[0].dataType).toBe(101); + expect(formatted.schema.fields[0].dim).toBe('128'); + expect(formatted.schema.fields[0].default_value).toBe(123); expect(formatted.schema.fields[1].dataType).toBe(5); + expect(formatted.schema.fields[1].default_value).toBe('d'); + expect(formatted.schema.fields[2].default_value).toBe(false); + expect(formatted.schema.fields[2].enable_match).toBe('true'); + expect(formatted.schema.fields[2].enable_analyzer).toBe('true'); + expect(formatted.schema.fields[2].dim).toBe('3'); + expect(formatted.schema.fields[2].analyzer_params).toBe('{}'); + expect(formatted.schema.fields[2].max_capacity).toBe('23'); }); it('should return an empty object when data is empty', () => {