Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[2.5] support default value & null #366

Closed
wants to merge 12 commits into from
13 changes: 9 additions & 4 deletions milvus/grpc/Collection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ import {
CreateCollectionWithFieldsReq,
CreateCollectionWithSchemaReq,
FieldSchema,
isVectorType,
} from '../';

/**
Expand Down Expand Up @@ -1054,9 +1055,12 @@ export class Collection extends Database {
* });
* ```
*/
async getPkFieldName(data: DescribeCollectionReq): Promise<string> {
async getPkFieldName(
data: DescribeCollectionReq,
desc?: DescribeCollectionResponse
): Promise<string> {
// get collection info
const collectionInfo = await this.describeCollection(data);
const collectionInfo = desc ? desc : await this.describeCollection(data);

// pk field
let pkField = '';
Expand Down Expand Up @@ -1094,10 +1098,11 @@ export class Collection extends Database {
* ```
*/
async getPkFieldType(
data: DescribeCollectionReq
data: DescribeCollectionReq,
desc?: DescribeCollectionResponse
): Promise<keyof typeof DataType> {
// get collection info
const collectionInfo = await this.describeCollection(data);
const collectionInfo = desc ? desc : await this.describeCollection(data);

// pk field type
let pkFieldType: keyof typeof DataType = 'Int64';
Expand Down
40 changes: 29 additions & 11 deletions milvus/grpc/Data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ import {
buildDynamicRow,
buildFieldDataMap,
getDataKey,
Field,
_Field,
buildFieldData,
BinaryVector,
RowData,
Expand All @@ -65,6 +65,7 @@ import {
sparseRowsToBytes,
getSparseDim,
f32ArrayToBinaryBytes,
getValidDataArray,
NO_LIMIT,
} from '../';
import { Collection } from './Collection';
Expand Down Expand Up @@ -143,7 +144,7 @@ export class Data extends Collection {
// Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`.
// If primarykey is set `autoid = true`, you cannot insert the data.
// and if function field is set, you need to ignore the field value in the data.
const fieldMap = new Map<string, Field>(
const fieldMap = new Map<string, _Field>(
collectionInfo.schema.fields
.filter(v => !v.is_primary_key || !v.autoID)
.filter(v => !v.is_function_output)
Expand All @@ -155,6 +156,8 @@ export class Data extends Collection {
elementType: v.element_type,
dim: Number(findKeyValue(v.type_params, 'dim')),
data: [], // values container
nullable: v.nullable,
default_value: v.default_value,
},
])
);
Expand All @@ -167,6 +170,7 @@ export class Data extends Collection {
type: 'JSON',
elementType: 'None',
data: [], // value container
nullable: false,
});
}

Expand Down Expand Up @@ -225,6 +229,10 @@ export class Data extends Collection {
const dataKey = getDataKey(type);
const elementType = DataTypeMap[field.elementType!];
const elementTypeKey = getDataKey(elementType);
const valid_data = getValidDataArray(
field.data,
data.fields_data?.length!
);

// build key value
let keyValue;
Expand Down Expand Up @@ -263,32 +271,41 @@ export class Data extends Collection {
case DataType.Array:
keyValue = {
[dataKey]: {
data: field.data.map(d => {
return {
[elementTypeKey]: {
type: elementType,
data: d,
},
};
}),
data: field.data
.filter(v => v !== undefined)
.map(d => {
return {
[elementTypeKey]: {
type: elementType,
data: d,
},
};
}),
element_type: elementType,
},
};
break;
default:
keyValue = {
[dataKey]: {
data: field.data,
data: field.data.filter(v => v !== undefined),
},
};
break;
}

const needValidData =
key !== 'vectors' &&
(field.nullable === true ||
(typeof field.default_value !== 'undefined' &&
field.default_value !== null));

return {
type,
field_name: field.name,
is_dynamic: field.name === DEFAULT_DYNAMIC_FIELD,
[key]: keyValue,
valid_data: needValidData ? valid_data : [],
};
});

Expand Down Expand Up @@ -931,6 +948,7 @@ export class Data extends Collection {
// always get output_fields from fields_data
const output_fields = promise.fields_data.map(f => f.field_name);

// build field data map
const fieldsDataMap = buildFieldDataMap(
promise.fields_data,
data.transformers
Expand Down
9 changes: 5 additions & 4 deletions milvus/types/Collection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ export interface FieldSchema {
element_type?: keyof typeof DataType;
default_value?: number | string;
dataType: DataType;
is_partition_key: boolean;
is_dynamic: boolean;
is_clustering_key: boolean;
is_partition_key?: boolean;
is_dynamic?: boolean;
is_clustering_key?: boolean;
nullable?: boolean;
is_function_output: boolean;
nullable: boolean;
}

export interface CollectionData {
Expand Down Expand Up @@ -80,6 +80,7 @@ export interface FieldType {
max_capacity?: TypeParam;
max_length?: TypeParam;
default_value?: number | string;
nullable?: boolean;
enable_match?: boolean;
tokenizer_params?: Record<string, any>;
enable_tokenizer?: boolean;
Expand Down
9 changes: 7 additions & 2 deletions milvus/types/Data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,19 +72,22 @@ export type FieldData =
| VarChar
| JSON
| Array
| VectorTypes;
| VectorTypes
| null;

// Represents a row of data in Milvus.
export interface RowData {
[x: string]: FieldData;
}

export interface Field {
export interface _Field {
name: string;
type: keyof typeof DataType;
elementType?: keyof typeof DataType;
data: FieldData[];
dim?: number;
nullable?: boolean;
default_value?: FieldData;
}

export interface FlushReq extends GrpcTimeOut {
Expand Down Expand Up @@ -455,6 +458,8 @@ export interface QueryRes extends resStatusResponse {
[x: string]: any;
data: string;
};
is_dynamic: boolean;
valid_data: boolean[];
}[];
output_fields: string[];
collection_name: string;
Expand Down
26 changes: 19 additions & 7 deletions milvus/utils/Format.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
DescribeCollectionResponse,
getDataKey,
RowData,
Field,
_Field,
JSON,
FieldData,
CreateCollectionWithFieldsReq,
Expand Down Expand Up @@ -421,7 +421,7 @@ export const formatDescribedCol = (
*/
export const buildDynamicRow = (
rowData: RowData,
fieldMap: Map<string, Field>,
fieldMap: Map<string, _Field>,
dynamicFieldName: string
) => {
const originRow = cloneObj(rowData);
Expand Down Expand Up @@ -537,17 +537,27 @@ export const buildFieldDataMap = (
});
}

// decode json
switch (dataKey) {
// decode json
case 'json_data':
field_data.forEach((buffer: any, i: number) => {
// console.log(JSON.parse(buffer.toString()));
field_data[i] = JSON.parse(buffer.toString());
field_data[i] = buffer.length
? JSON.parse(buffer.toString())
: null;
});
break;
default:
break;
}

// set the field data with null if item.valid_data is not empty array, it the item in valid_data is false, set the field data with null
if (item.valid_data && item.valid_data.length) {
item.valid_data.forEach((v: any, i: number) => {
if (!v) {
field_data[i] = null;
}
});
}
}

// Add the parsed data to the fieldsDataMap
Expand Down Expand Up @@ -591,7 +601,7 @@ export const getAuthString = (data: {
*/
export const buildFieldData = (
rowData: RowData,
field: Field,
field: _Field,
transformers?: InsertTransformers
): FieldData => {
const { type, elementType, name } = field;
Expand All @@ -614,7 +624,9 @@ export const buildFieldData = (
? f16Transformer(rowData[name] as Float16Vector)
: rowData[name];
case DataType.JSON:
return Buffer.from(JSON.stringify(rowData[name] || {}));
return rowData[name]
? Buffer.from(JSON.stringify(rowData[name] || {}))
: Buffer.alloc(0);
case DataType.Array:
const elementField = { ...field, type: elementType! };
return buildFieldData(rowData, elementField, transformers);
Expand Down
9 changes: 9 additions & 0 deletions milvus/utils/Function.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
DEFAULT_MIN_INT64,
SearchResultData,
SparseFloatVector,
FieldData,
} from '../';
import { Pool } from 'generic-pool';

Expand Down Expand Up @@ -236,3 +237,11 @@ export const getSparseDim = (data: SparseFloatVector[]) => {
}
return dim;
};

// get valid data
// create a length array with valid data, if the data is undefined or null, return false, otherwise return true
export const getValidDataArray = (data: FieldData[], length: number) => {
return Array.from({ length }).map((_, i) => {
return data[i] !== undefined && data[i] !== null;
});
};
1 change: 1 addition & 0 deletions test/grpc/Data.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { timeoutTest } from '../tools';

const milvusClient = new MilvusClient({
address: IP,
logLevel: 'debug',
});
const COLLECTION_NAME = GENERATE_NAME();
const VARCHAR_ID_COLLECTION_NAME = GENERATE_NAME();
Expand Down
Loading
Loading