format describe collection response (#381)

* format describe collection response Signed-off-by: ryjiang <[email protected]> * fix Signed-off-by: ryjiang <[email protected]> * Fix test Signed-off-by: ryjiang <[email protected]> --------- Signed-off-by: ryjiang <[email protected]>
milvus-io · Dec 3, 2024 · c4f47d5 · c4f47d5
1 parent edde8d3
commit c4f47d5
Show file tree

Hide file tree

Showing 8 changed files with 110 additions and 53 deletions.
diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts
@@ -40,7 +40,6 @@ import {
   DEFAULT_TOPK,
   HybridSearchReq,
   promisify,
-  findKeyValue,
   sleep,
   parseToKeyValue,
   checkCollectionName,
@@ -157,7 +156,7 @@ export class Data extends Collection {
               name: v.name,
               type: v.data_type, // milvus return string here
               elementType: v.element_type,
-              dim: Number(findKeyValue(v.type_params, 'dim')),
+              dim: Number(v.dim),
               data: [], // values container
               nullable: v.nullable,
               default_value: v.default_value,

diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts
@@ -16,27 +16,6 @@ import {
   FunctionType,
 } from '../';
 
-// returned from milvus
-export interface FieldSchema {
-  type_params: KeyValuePair[];
-  index_params: KeyValuePair[];
-  fieldID: string | number;
-  name: string;
-  is_primary_key: boolean;
-  description: string;
-  data_type: keyof typeof DataType;
-  autoID: boolean;
-  state: string;
-  element_type?: keyof typeof DataType;
-  default_value?: number | string;
-  dataType: DataType;
-  is_partition_key?: boolean;
-  is_dynamic?: boolean;
-  is_clustering_key?: boolean;
-  nullable?: boolean;
-  is_function_output: boolean;
-}
-
 export interface CollectionData {
   name: string;
   id: string;
@@ -59,11 +38,38 @@ export interface ReplicaInfo {
   node_ids: string[];
 }
 
-export type TypeParam = string | number | Record<string, any>;
-export type TypeParamKey = 'dim' | 'max_length' | 'max_capacity';
+export type TypeParam = string | number | boolean | Record<string, any>;
+export type TypeParamKey =
+  | 'dim'
+  | 'max_length'
+  | 'max_capacity'
+  | 'analyzer_params'
+  | 'enable_analyzer'
+  | 'enable_match';
+
+// returned from milvus
+export type FieldSchema = {
+  type_params: KeyValuePair<TypeParamKey, TypeParam>[];
+  index_params: KeyValuePair[];
+  fieldID: string | number;
+  name: string;
+  is_primary_key: boolean;
+  description: string;
+  data_type: keyof typeof DataType;
+  autoID: boolean;
+  state: string;
+  element_type?: keyof typeof DataType;
+  default_value?: number | string;
+  dataType: DataType;
+  is_partition_key?: boolean;
+  is_dynamic?: boolean;
+  is_clustering_key?: boolean;
+  nullable?: boolean;
+  is_function_output: boolean;
+} & Partial<Record<TypeParamKey, TypeParam>>;
 
 // create collection
-export interface FieldType {
+export type FieldType = {
   name: string;
   description?: string;
   data_type: DataType | keyof typeof DataTypeMap;
@@ -72,19 +78,11 @@ export interface FieldType {
   is_partition_key?: boolean;
   is_function_output?: boolean;
   is_clustering_key?: boolean;
-  type_params?: {
-    [key: string]: TypeParam;
-  };
+  type_params?: Partial<Record<TypeParamKey, TypeParam>>;
   autoID?: boolean;
-  dim?: TypeParam;
-  max_capacity?: TypeParam;
-  max_length?: TypeParam;
   default_value?: number | string;
   nullable?: boolean;
-  enable_match?: boolean;
-  tokenizer_params?: Record<string, any>;
-  enable_analyzer?: boolean;
-}
+} & Partial<Record<TypeParamKey, TypeParam>>;
 
 export interface ShowCollectionsReq extends GrpcTimeOut {
   type?: ShowCollectionsType;

diff --git a/milvus/types/Common.ts b/milvus/types/Common.ts
@@ -11,9 +11,9 @@ export interface MsgBase {
   };
 }
 
-export interface KeyValuePair {
-  key: string;
-  value: string | number;
+export interface KeyValuePair<T = string, U = string | number> {
+  key: T;
+  value: U;
 }
 
 interface NumberArray {

diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts
@@ -41,10 +41,11 @@ import {
   f32ArrayToF16Bytes,
   bf16BytesToF32Array,
   f16BytesToF32Array,
-  TypeParam,
   SearchDataType,
   FieldSchema,
   SearchMultipleDataType,
+  TypeParamKey,
+  TypeParam,
 } from '../';
 
 /**
@@ -198,7 +199,7 @@ export const formatAddress = (address: string) => {
  */
 export const assignTypeParams = (
   field: FieldType,
-  typeParamKeys: string[] = [
+  typeParamKeys: TypeParamKey[] = [
     'dim',
     'max_length',
     'max_capacity',
@@ -210,7 +211,7 @@ export const assignTypeParams = (
   const newField = cloneObj<FieldType>(field);
 
   // Initialize `type_params` if undefined
-  newField.type_params ??= {};
+  newField.type_params ??= {} as Record<TypeParamKey, TypeParam>;
 
   typeParamKeys.forEach(key => {
     if (key in newField) {
@@ -413,6 +414,17 @@ export const formatDescribedCol = (
   // add a dataType property which indicate datatype number
   newData.schema?.fields?.forEach(f => {
     f.dataType = DataTypeMap[f.data_type];
+    // if default_value is set, parse it to the correct format
+    if (f.default_value) {
+      const defaultValue = f.default_value as any;
+      f.default_value = defaultValue[defaultValue.data];
+    }
+    // extract type params(key value pair = {key: 'xxx', value: any}), and assign it to the field object(key)
+    if (f.type_params && f.type_params.length > 0) {
+      f.type_params.forEach(keyValuePair => {
+        f[keyValuePair.key] = keyValuePair.value;
+      });
+    }
   });
 
   return newData;

diff --git a/test/grpc/Collection.spec.ts b/test/grpc/Collection.spec.ts
@@ -79,7 +79,9 @@ describe(`Collection API`, () => {
 
     expect(
       Boolean(
-        formatKeyValueData(describe.properties, ['mmap.enabled'])['mmap.enabled']
+        formatKeyValueData(describe.properties, ['mmap.enabled'])[
+          'mmap.enabled'
+        ]
       )
     ).toEqual(true);
 

diff --git a/test/grpc/DynamicSchema.spec.ts b/test/grpc/DynamicSchema.spec.ts
@@ -53,6 +53,20 @@ describe(`Dynamic schema API`, () => {
     const describe = await milvusClient.describeCollection({
       collection_name: COLLECTION,
     });
+
+    // find varchar field
+    const varCharField = describe.schema.fields.find(
+      v => v.name === 'varChar'
+    )!;
+    // find int64 field
+    const int64Field = describe.schema.fields.find(
+      v => v.name === 'default_value'
+    )!;
+
+    // test default value
+    expect(varCharField.default_value).toEqual(DEFAULT_STRING_VALUE);
+    expect(int64Field.default_value).toEqual(DEFAULT_NUM_VALUE);
+
     expect(describe.schema.enable_dynamic_field).toEqual(true);
   });
 
@@ -115,6 +129,7 @@ describe(`Dynamic schema API`, () => {
 
     expect(query.status.error_code).toEqual(ErrorCode.SUCCESS);
     expect(query.data.length).toEqual(10);
+
     // get test values
     const varChars = query.data.map(v => v.varChar);
     const defaultValues = query.data.map(v => v.default_value);
@@ -123,13 +138,11 @@ describe(`Dynamic schema API`, () => {
     const bools = query.data.map(v => v.bool);
     const floats = query.data.map(v => v.float);
     //  some of floats should be equal to DEFAULT_NUM_VALUE
-    expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true);
+    expect(floats.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true);
     // some of varChar should be equal to DEFAULT_STRING_VALUE
     expect(varChars.some(v => v === DEFAULT_STRING_VALUE)).toEqual(true);
     // some of default_value should be equal to DEFAULT_NUM_VALUE
-    expect(defaultValues.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(
-      true
-    );
+    expect(defaultValues.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true);
     // some of json should be null
     expect(jsons.some(v => v === null)).toEqual(true);
     // some of bools should be null
@@ -173,13 +186,11 @@ describe(`Dynamic schema API`, () => {
     const bools = search.results.map(v => v.bool);
     const floats = search.results.map(v => v.float);
     //  some of floats should be equal to DEFAULT_NUM_VALUE
-    expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true);
+    expect(floats.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true);
     // some of varChar should be equal to DEFAULT_STRING_VALUE
     expect(varChars.some(v => v === DEFAULT_STRING_VALUE)).toEqual(true);
     // some of default_value should be equal to DEFAULT_NUM_VALUE
-    expect(defaultValues.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(
-      true
-    );
+    expect(defaultValues.some(v => v === DEFAULT_NUM_VALUE)).toEqual(true);
     // some of json should be null
     expect(jsons.some(v => v === null)).toEqual(true);
     // some of bools should be null

diff --git a/test/tools/collection.ts b/test/tools/collection.ts
@@ -122,8 +122,8 @@ export const genCollectionParams = (data: {
         name: 'default_value',
         nullable: true,
         default_value: DEFAULT_NUM_VALUE,
-        description: 'int64 field',
-        data_type: 'Int64', //
+        description: 'int32 field',
+        data_type: 'Int32', //
       },
       {
         name: 'varChar',

diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts
@@ -440,6 +440,7 @@ describe('utils/format', () => {
             is_clustering_key: false,
             is_function_output: false,
             nullable: false,
+            default_value: { long_data: 123, data: 'long_data' } as any,
             is_partition_key: false,
           },
           {
@@ -453,12 +454,37 @@ describe('utils/format', () => {
             dataType: 5,
             autoID: true,
             state: 'created',
+            default_value: { string_data: 'd', data: 'string_data' } as any,
             is_dynamic: false,
             is_clustering_key: false,
             is_function_output: false,
             nullable: false,
             is_partition_key: false,
           },
+          {
+            fieldID: '3',
+            type_params: [
+              { key: 'enable_match', value: 'true' },
+              { key: 'enable_analyzer', value: 'true' },
+              { key: 'dim', value: '3' },
+              { key: 'analyzer_params', value: '{}' },
+              { key: 'max_capacity', value: '23' },
+            ],
+            index_params: [],
+            name: 'vector',
+            is_primary_key: false,
+            description: 'vector field',
+            data_type: 'FloatVector',
+            dataType: 101,
+            autoID: false,
+            state: 'created',
+            is_dynamic: false,
+            is_clustering_key: false,
+            is_function_output: false,
+            nullable: false,
+            default_value: { bool_data: false, data: 'bool_data' } as any,
+            is_partition_key: false,
+          },
         ],
         name: 'collection_v8mt0v7x',
         description: '',
@@ -481,7 +507,16 @@ describe('utils/format', () => {
     const formatted = formatDescribedCol(response);
 
     expect(formatted.schema.fields[0].dataType).toBe(101);
+    expect(formatted.schema.fields[0].dim).toBe('128');
+    expect(formatted.schema.fields[0].default_value).toBe(123);
     expect(formatted.schema.fields[1].dataType).toBe(5);
+    expect(formatted.schema.fields[1].default_value).toBe('d');
+    expect(formatted.schema.fields[2].default_value).toBe(false);
+    expect(formatted.schema.fields[2].enable_match).toBe('true');
+    expect(formatted.schema.fields[2].enable_analyzer).toBe('true');
+    expect(formatted.schema.fields[2].dim).toBe('3');
+    expect(formatted.schema.fields[2].analyzer_params).toBe('{}');
+    expect(formatted.schema.fields[2].max_capacity).toBe('23');
   });
 
   it('should return an empty object when data is empty', () => {