Merge branch 'main' of github.com:milvus-io/milvus-sdk-node into 2.5

Signed-off-by: ryjiang <[email protected]>

examples/milvus/package.json

@@ -8,11 +8,11 @@
     "benchp": "autocannon http://localhost:4000 -c 1 -p 10"
   },
   "dependencies": {
-    "@zilliz/milvus2-sdk-node": "^2.4.3",
-    "express": "^4.19.2"
+    "@zilliz/milvus2-sdk-node": "^2.4.9",
+    "express": "^4.21.0"
   },
   "devDependencies": {
     "@types/express": "^4.17.17",
-    "autocannon": "^7.11.0"
+    "autocannon": "^7.15.0"
   }
 }

examples/nextjs/app/page.tsx

@@ -1,8 +1,8 @@
 import { MilvusClient } from '@zilliz/milvus2-sdk-node';
 
-async function getData() {
+async function getData(address: string) {
   const milvusClient = new MilvusClient({
-    address: 'localhost:19530',
+    address,
   });
 
   let res: any = await milvusClient.getMetric({
@@ -17,7 +17,24 @@ async function getData() {
 }
 
 export default async function Home() {
-  const data = await getData();
+  const address = `10.102.5.223:19530`;
 
-  return <>{JSON.stringify(data)}</>;
+  const data = await getData(address);
+
+  return (
+    <div className="min-h-screen flex items-center justify-center bg-gray-100">
+      <div className="max-w-4xl w-full p-6 bg-white rounded-lg shadow-md">
+        <h1 className="text-3xl font-bold text-center mb-4">
+          Welcome to Milvus
+        </h1>
+        <h2 className="text-xl font-bold text-center mb-4">{address}</h2>
+
+        <div className="bg-gray-50 p-4 rounded-lg">
+          <pre className="whitespace-pre-wrap text-sm text-gray-800">
+            {JSON.stringify(data, null, 2)}
+          </pre>
+        </div>
+      </div>
+    </div>
+  );
 }

examples/nextjs/package.json

@@ -9,10 +9,12 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@zilliz/milvus2-sdk-node": "^2.4.3",
-    "next": "^14.2.4",
-    "react": "^18",
-    "react-dom": "^18"
+    "@uiw/react-codemirror": "^4.23.3",
+    "@zilliz/milvus2-sdk-node": "^2.4.9",
+    "codemirror": "^6.0.1",
+    "next": "^14.2.13",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1"
   },
   "devDependencies": {
     "@types/node": "^20",

milvus/const/limits.ts

@@ -1,3 +1,5 @@
 // limits
 export const MAX_PARTITION_KEY_FIELD_COUNT = 1;
-export const MAX_PARTITIONS_NUMBER = 4096;
+export const MAX_PARTITIONS_NUMBER = 4096;
+
+export const NO_LIMIT = -1;

milvus/grpc/Data.ts

@@ -65,6 +65,7 @@ import {
   sparseRowsToBytes,
   getSparseDim,
   f32ArrayToBinaryBytes,
+  NO_LIMIT,
 } from '../';
 import { Collection } from './Collection';
 
@@ -706,6 +707,10 @@ export class Data extends Collection {
       collection_name: data.collection_name,
       expr: userExpr,
     });
+    // if limit not set, set it to count
+    if (!data.limit || data.limit === NO_LIMIT) {
+      data.limit = count.data;
+    }
     // total should be the minimum of total and count
     const total = data.limit > count.data ? count.data : data.limit;
     const batchSize =

milvus/types/Collection.ts

@@ -71,6 +71,7 @@ export interface FieldType {
   is_primary_key?: boolean;
   is_partition_key?: boolean;
   is_function_output?: boolean;
+  is_clustering_key?: boolean;
   type_params?: {
     [key: string]: TypeParam;
   };
@@ -114,6 +115,7 @@ export interface BaseCreateCollectionReq extends GrpcTimeOut {
     | 'Customized'; // optional,consistency level, default is 'Bounded'
   num_partitions?: number; // optional, partitions number, default is 1
   partition_key_field?: string; // optional, partition key field
+  clustring_key_field?: string; // optional, clustring key field
   enable_dynamic_field?: boolean; // optional, enable dynamic field, default is false
   enableDynamicField?: boolean; // optional, alias of enable_dynamic_field
   properties?: Properties; // optional, collection properties

milvus/types/Data.ts

@@ -423,7 +423,7 @@ export interface QueryReq extends collectionNameReq {
 
 export interface QueryIteratorReq
   extends Omit<QueryReq, 'ids' | 'offset' | 'limit'> {
-  limit: number;
+  limit?: number;
   batchSize: number;
 }
 

milvus/utils/Format.ts

@@ -319,6 +319,7 @@ export const formatCollectionSchema = (
     enableDynamicField,
     partition_key_field,
     functions,
+    clustring_key_field,
   } = data;
 
   let fields = (data as CreateCollectionWithFieldsReq).fields;
@@ -352,6 +353,8 @@ export const formatCollectionSchema = (
         isPartitionKey:
           !!is_partition_key || field.name === partition_key_field,
         isFunctionOutput: !!is_function_output,
+        isClusteringKey:
+          !!field.is_clustering_key || field.name === clustring_key_field,
       };
 
       // if element type exist and

test/grpc/Basic.spec.ts

@@ -1,7 +1,7 @@
 import { MilvusClient, ErrorCode, DataType } from '../../milvus';
 import { IP, GENERATE_NAME, generateInsertData } from '../tools';
 
-const milvusClient = new MilvusClient({ address: IP });
+const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' });
 const COLLECTION_NAME = GENERATE_NAME();
 const schema = [
   {
@@ -49,6 +49,14 @@ describe(`Basic API without database`, () => {
       collection_name: COLLECTION_NAME,
     });
     expect(desc.schema.fields.length).toEqual(schema.length);
+    expect(desc.schema.fields[0].name).toEqual('vector');
+    expect(desc.schema.fields[1].name).toEqual('id');
+    expect(desc.schema.fields[2].name).toEqual('varChar');
+    expect(desc.schema.fields[3].name).toEqual('array');
+    // test primary key
+    expect(desc.schema.fields[1].is_primary_key).toEqual(true);
+    // test partition key
+    expect(desc.schema.fields[2].is_partition_key).toEqual(false);
   });
 
   it(`Create index should be successful`, async () => {

test/grpc/ClusteringKey.spec.ts

@@ -0,0 +1,111 @@
+import { MilvusClient, DataType, ErrorCode } from '../../milvus';
+import { IP, GENERATE_NAME, genCollectionParams } from '../tools';
+
+const milvusClient = new MilvusClient({ address: IP });
+const COLLECTION_NAME = GENERATE_NAME();
+const COLLECTION_NAME2 = GENERATE_NAME();
+const dbParam = {
+  db_name: 'ClusteringKey',
+};
+
+describe(`Clustering key API`, () => {
+  beforeAll(async () => {
+    await milvusClient.createDatabase(dbParam);
+    await milvusClient.use(dbParam);
+  });
+
+  afterAll(async () => {
+    await milvusClient.dropCollection({
+      collection_name: COLLECTION_NAME,
+    });
+    await milvusClient.dropCollection({
+      collection_name: COLLECTION_NAME2,
+    });
+    await milvusClient.dropDatabase(dbParam);
+  });
+
+  it(`Create Collection with schema should successfully`, async () => {
+    const res = await milvusClient.createCollection(
+      genCollectionParams({
+        collectionName: COLLECTION_NAME,
+        dim: [4],
+        vectorType: [DataType.FloatVector],
+        autoID: true,
+        clusterKeyEnabled: true,
+      })
+    );
+    expect(res.error_code).toEqual(ErrorCode.SUCCESS);
+
+    // describe
+    const desc = await milvusClient.describeCollection({
+      collection_name: COLLECTION_NAME,
+    });
+
+    // test clustering key
+    let found = 0;
+    desc.schema.fields.forEach(field => {
+      if (field.is_clustering_key) {
+        found++;
+      }
+    });
+
+    expect(found).toEqual(1);
+  });
+
+  it(`Create Collection should be successful with clusteringkey`, async () => {
+    const schema = [
+      {
+        name: 'vector',
+        description: 'Vector field',
+        data_type: DataType.FloatVector,
+        dim: Number(4),
+      },
+      {
+        name: 'id',
+        description: 'ID field',
+        data_type: DataType.Int64,
+        is_primary_key: true,
+        autoID: true,
+      },
+      {
+        name: 'varChar',
+        description: 'VarChar field',
+        data_type: DataType.VarChar,
+        max_length: 128,
+        is_partition_key: false,
+        is_clustering_key: false,
+      },
+      {
+        name: 'array',
+        description: 'array field',
+        data_type: DataType.Array,
+        element_type: DataType.VarChar,
+        max_capacity: 128,
+        max_length: 128,
+        is_partition_key: false,
+      },
+    ];
+    const res = await milvusClient.createCollection({
+      collection_name: COLLECTION_NAME2,
+      fields: schema,
+      clustring_key_field: 'varChar',
+    });
+    expect(res.error_code).toEqual(ErrorCode.SUCCESS);
+
+    // describe
+    const desc = await milvusClient.describeCollection({
+      collection_name: COLLECTION_NAME2,
+    });
+
+    // test clustering key
+    let found = 0;
+    desc.schema.fields.forEach(field => {
+      if (field.is_clustering_key) {
+        found++;
+        expect(field.name).toEqual('varChar');
+      }
+    });
+
+    expect(found).toEqual(1);
+  });
+});

test/grpc/Iterator.spec.ts

@@ -1,4 +1,4 @@
-import { MilvusClient, DataType } from '../../milvus';
+import { MilvusClient, DataType, NO_LIMIT } from '../../milvus';
 import {
   IP,
   genCollectionParams,
@@ -299,6 +299,79 @@ describe(`Iterator API`, () => {
     });
   });
 
+  it(`query iterator with limit unset should success`, async () => {
+    // page size
+    const batchSize = 5000;
+    const iterator = await milvusClient.queryIterator({
+      collection_name: COLLECTION,
+      batchSize: batchSize,
+      expr: 'id > 0',
+      output_fields: ['id'],
+    });
+
+    const results: any = [];
+    let page = 0;
+    for await (const value of iterator) {
+      results.push(...value);
+      page += 1;
+    }
+
+    // page size should equal to page
+    expect(page).toEqual(Math.ceil(data.length / batchSize));
+    // results length should equal to data length
+    expect(results.length).toEqual(data.length);
+
+    // results id should be unique
+    const idSet = new Set();
+    results.forEach((result: any) => {
+      idSet.add(result.id);
+    });
+    expect(idSet.size).toEqual(data.length);
+
+    // every id in query result should be founded in the original data
+    results.forEach((result: any) => {
+      const item = dataMap.get(result.id.toString());
+      expect(typeof item !== 'undefined').toEqual(true);
+    });
+  });
+
+  it(`query iterator with limit = -1 should success`, async () => {
+    // page size
+    const batchSize = 5000;
+    const iterator = await milvusClient.queryIterator({
+      collection_name: COLLECTION,
+      batchSize: batchSize,
+      expr: 'id > 0',
+      output_fields: ['id'],
+      limit: NO_LIMIT,
+    });
+
+    const results: any = [];
+    let page = 0;
+    for await (const value of iterator) {
+      results.push(...value);
+      page += 1;
+    }
+
+    // page size should equal to page
+    expect(page).toEqual(Math.ceil(data.length / batchSize));
+    // results length should equal to data length
+    expect(results.length).toEqual(data.length);
+
+    // results id should be unique
+    const idSet = new Set();
+    results.forEach((result: any) => {
+      idSet.add(result.id);
+    });
+    expect(idSet.size).toEqual(data.length);
+
+    // every id in query result should be founded in the original data
+    results.forEach((result: any) => {
+      const item = dataMap.get(result.id.toString());
+      expect(typeof item !== 'undefined').toEqual(true);
+    });
+  });
+
   // it('search iterator with batch size = total should success', async () => {
   //   const batchSize = 100;
   //   const total = 100;

test/tools/collection.ts

@@ -42,6 +42,7 @@ export const genCollectionParams = (data: {
   autoID?: boolean;
   fields?: any[];
   partitionKeyEnabled?: boolean;
+  clusterKeyEnabled?: boolean;
   numPartitions?: number;
   enableDynamic?: boolean;
   maxCapacity?: number;
@@ -60,6 +61,7 @@ export const genCollectionParams = (data: {
     maxCapacity,
     idType = DataType.Int64,
     functions,
+    clusterKeyEnabled = false,
   } = data;
 
   const vectorFields = vectorType.map((type, i) => {
@@ -163,6 +165,9 @@ export const genCollectionParams = (data: {
   if (functions && functions?.length > 0) {
     params.functions = functions;
   }
+  if (clusterKeyEnabled) {
+    params.clustring_key_field = 'int64';
+  }
 
   return params;
 };