fix: function output field should not be inserted as dynamic field

Signed-off-by: ryjiang <[email protected]>
milvus-io · Nov 26, 2024 · a796b05 · a796b05
1 parent 1415a5b
commit a796b05
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 39 deletions.
diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts
@@ -145,22 +145,27 @@ export class Data extends Collection {
     // Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`.
     // If primarykey is set `autoid = true`, you cannot insert the data.
     // and if function field is set, you need to ignore the field value in the data.
+    const functionOutputFields: string[] = [];
     const fieldMap = new Map<string, _Field>(
-      collectionInfo.schema.fields
-        .filter(v => !v.is_primary_key || !v.autoID)
-        .filter(v => !v.is_function_output)
-        .map(v => [
-          v.name,
-          {
-            name: v.name,
-            type: v.data_type, // milvus return string here
-            elementType: v.element_type,
-            dim: Number(findKeyValue(v.type_params, 'dim')),
-            data: [], // values container
-            nullable: v.nullable,
-            default_value: v.default_value,
-          },
-        ])
+      collectionInfo.schema.fields.reduce((acc, v) => {
+        if (v.is_function_output) {
+          functionOutputFields.push(v.name); // ignore function field
+        } else if (!v.is_primary_key || !v.autoID) {
+          acc.push([
+            v.name,
+            {
+              name: v.name,
+              type: v.data_type, // milvus return string here
+              elementType: v.element_type,
+              dim: Number(findKeyValue(v.type_params, 'dim')),
+              data: [], // values container
+              nullable: v.nullable,
+              default_value: v.default_value,
+            },
+          ]);
+        }
+        return acc;
+      }, [] as [string, _Field][])
     );
 
     // dynamic field is enabled, create $meta field
@@ -179,7 +184,12 @@ export class Data extends Collection {
     data.fields_data.forEach((rowData, rowIndex) => {
       // if support dynamic field, all field not in the schema would be grouped to a dynamic field
       rowData = isDynamic
-        ? buildDynamicRow(rowData, fieldMap, DEFAULT_DYNAMIC_FIELD)
+        ? buildDynamicRow(
+            rowData,
+            fieldMap,
+            DEFAULT_DYNAMIC_FIELD,
+            functionOutputFields
+          )
         : rowData;
 
       // get each fieldname from the row object
@@ -230,10 +240,18 @@ export class Data extends Collection {
       const dataKey = getDataKey(type);
       const elementType = DataTypeMap[field.elementType!];
       const elementTypeKey = getDataKey(elementType);
-      const valid_data = getValidDataArray(
-        field.data,
-        data.fields_data?.length!
-      );
+
+      // check if need valid data
+      const needValidData =
+        key !== 'vectors' &&
+        (field.nullable === true ||
+          (typeof field.default_value !== 'undefined' &&
+            field.default_value !== null));
+
+      // get valid data
+      const valid_data = needValidData
+        ? getValidDataArray(field.data, data.fields_data?.length!)
+        : [];
 
       // build key value
       let keyValue;
@@ -295,18 +313,12 @@ export class Data extends Collection {
           break;
       }
 
-      const needValidData =
-        key !== 'vectors' &&
-        (field.nullable === true ||
-          (typeof field.default_value !== 'undefined' &&
-            field.default_value !== null));
-
       return {
         type,
         field_name: field.name,
         is_dynamic: field.name === DEFAULT_DYNAMIC_FIELD,
         [key]: keyValue,
-        valid_data: needValidData ? valid_data : [],
+        valid_data: valid_data,
       };
     });
 
@@ -478,7 +490,9 @@ export class Data extends Collection {
       describeCollectionRequest.db_name = data.db_name;
     }
 
-    const collectionInfo = await this.describeCollection(describeCollectionRequest);
+    const collectionInfo = await this.describeCollection(
+      describeCollectionRequest
+    );
 
     // build search params
     const { request, nq, round_decimal, isHybridSearch } = buildSearchRequest(

diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts
@@ -429,7 +429,8 @@ export const formatDescribedCol = (
 export const buildDynamicRow = (
   rowData: RowData,
   fieldMap: Map<string, _Field>,
-  dynamicFieldName: string
+  dynamicFieldName: string,
+  functionOutputFields: string[]
 ) => {
   const originRow = cloneObj(rowData);
 
@@ -443,9 +444,11 @@ export const buildDynamicRow = (
       // if the key is in the fieldMap, add it to the non-dynamic fields
       row[key] = originRow[key];
     } else {
-      const obj: JSON = row[dynamicFieldName] as JSON;
-      // otherwise, add it to the dynamic field
-      obj[key] = originRow[key];
+      if (!functionOutputFields.includes(key)) {
+        const obj: JSON = row[dynamicFieldName] as JSON;
+        // otherwise, add it to the dynamic field
+        obj[key] = originRow[key];
+      }
     }
   }
 

diff --git a/test/grpc/FullTextSearch.spec.ts b/test/grpc/FullTextSearch.spec.ts
@@ -14,7 +14,7 @@ import {
   dynamicFields,
 } from '../tools';
 
-const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' });
+const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' });
 const COLLECTION = GENERATE_NAME();
 const dbParam = {
   db_name: 'FullTextSearch',

diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts
@@ -488,15 +488,21 @@ describe('utils/format', () => {
     const data = {};
     const fieldsDataMap = new Map();
     const dynamicField = 'dynamic';
-    const result = buildDynamicRow(data, fieldsDataMap, dynamicField);
+    const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []);
     expect(result).toEqual({});
   });
 
   it('should return an object with dynamicField key when all data contains keys not in fieldsDataMap', () => {
-    const data = { key: 'value' };
+    const data = { key: 'value', key2: 'value2' };
     const fieldsDataMap = new Map();
     const dynamicField = 'dynamic';
-    const result = buildDynamicRow(data, fieldsDataMap, dynamicField);
+    const ignoreFields = ['key2'];
+    const result = buildDynamicRow(
+      data,
+      fieldsDataMap,
+      dynamicField,
+      ignoreFields
+    );
     expect(result).toEqual({ [dynamicField]: { key: 'value' } });
   });
 
@@ -513,7 +519,7 @@ describe('utils/format', () => {
       ],
     ]);
     const dynamicField = 'dynamic';
-    const result = buildDynamicRow(data, fieldsDataMap, dynamicField);
+    const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []);
     expect(result).toEqual({
       key1: 'value1',
       [dynamicField]: { key2: 'value2' },
@@ -541,7 +547,7 @@ describe('utils/format', () => {
       ],
     ]);
     const dynamicField = 'dynamic';
-    const result = buildDynamicRow(data, fieldsDataMap, dynamicField);
+    const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []);
     expect(result).toEqual({
       [dynamicField]: {},
       key1: 'value1',
@@ -562,7 +568,7 @@ describe('utils/format', () => {
       ],
     ]);
     const dynamicField = 'dynamic';
-    const result = buildDynamicRow(data, fieldsDataMap, dynamicField);
+    const result = buildDynamicRow(data, fieldsDataMap, dynamicField, []);
     expect(result).toEqual({
       key1: 'value1',
       [dynamicField]: { key2: 'value2' },