add typescript http client (#253)

* initial http client Signed-off-by: ryjiang <[email protected]> * finish type def Signed-off-by: ryjiang <[email protected]> * finish type system Signed-off-by: ruiyi.jiang <[email protected]> * update some comments Signed-off-by: ruiyi.jiang <[email protected]> * finish client test Signed-off-by: ruiyi.jiang <[email protected]> * update more client Signed-off-by: ruiyi.jiang <[email protected]> * optimize tests Signed-off-by: ryjiang <[email protected]> * finish collections api Signed-off-by: ryjiang <[email protected]> * fix build; init vectors Signed-off-by: ryjiang <[email protected]> * vector part1 Signed-off-by: ryjiang <[email protected]> * finish query and search Signed-off-by: ryjiang <[email protected]> * add get and delete api without test Signed-off-by: ryjiang <[email protected]> * fix typo Signed-off-by: ruiyi.jiang <[email protected]> * optimize coe Signed-off-by: ruiyi.jiang <[email protected]> * finish search Signed-off-by: ryjiang <[email protected]> * update get/delete type Signed-off-by: ryjiang <[email protected]> * adjust timeout and address => endpoint Signed-off-by: ruiyi.jiang <[email protected]> * recover ip Signed-off-by: ruiyi.jiang <[email protected]> * add cloud test Signed-off-by: ruiyi.jiang <[email protected]> * update test config Signed-off-by: ruiyi.jiang <[email protected]> * update test config Signed-off-by: ruiyi.jiang <[email protected]> * recover test ip Signed-off-by: ruiyi.jiang <[email protected]> * fix build Signed-off-by: ruiyi.jiang <[email protected]> * fix build Signed-off-by: ruiyi.jiang <[email protected]> * update comment Signed-off-by: ryjiang <[email protected]> * listColleciton -> listCollections Signed-off-by: ryjiang <[email protected]> --------- Signed-off-by: ryjiang <[email protected]> Signed-off-by: ruiyi.jiang <[email protected]>
milvus-io · Oct 26, 2023 · 9c59ff2 · 9c59ff2
1 parent 98bb237
commit 9c59ff2
Show file tree

Hide file tree

Showing 34 changed files with 891 additions and 44 deletions.
diff --git a/jest.config.js b/jest.config.js
@@ -6,6 +6,10 @@ module.exports = {
   testTimeout: 60000,
   // because user will cause other test fail, but we still have user spec
   coveragePathIgnorePatterns: ['/milvus/User.ts'],
+  testPathIgnorePatterns: [
+    'cloud.spec.ts',
+    'serverless.spec.ts',
+  ], // add this line
   testEnvironmentOptions: {
     NODE_ENV: 'production',
   },

diff --git a/milvus/HttpClient.ts b/milvus/HttpClient.ts
@@ -0,0 +1,97 @@
+import axios, { AxiosInstance } from 'axios';
+import { HttpClientConfig } from './types';
+import { Collection, Vector } from './http';
+import {
+  DEFAULT_DB,
+  DEFAULT_HTTP_TIMEOUT,
+  DEFAULT_HTTP_ENDPOINT_VERSION,
+} from '../milvus/const';
+
+// base class
+export class HttpBaseClient {
+  // The client configuration.
+  public config: HttpClientConfig;
+
+  // axios
+  public client: AxiosInstance;
+
+  constructor(config: HttpClientConfig) {
+    // Assign the configuration object.
+    this.config = config;
+
+    // setup axios client
+    this.client = axios.create({
+      baseURL: this.baseURL,
+      timeout: this.timeout,
+      timeoutErrorMessage: '',
+      withCredentials: true,
+      headers: {
+        Authorization: this.authorization,
+        Accept: 'application/json',
+        ContentType: 'application/json',
+      },
+    });
+
+    // interceptors
+    this.client.interceptors.request.use(request => {
+      // if dbName is not set, using default database
+      // GET
+      if (request.params) {
+        request.params.dbName = request.params.dbName || this.database;
+      }
+      // POST
+      if (request.data) {
+        request.data.dbName = request.data.dbName || this.database;
+        request.data = JSON.stringify(request.data);
+      }
+
+      // console.log('request: ', request.data);
+      return request;
+    });
+    this.client.interceptors.response.use(response => {
+      return response.data;
+    });
+  }
+
+  // baseURL
+  get baseURL() {
+    return (
+      this.config.baseURL ||
+      `${this.config.endpoint}/${
+        this.config.version || DEFAULT_HTTP_ENDPOINT_VERSION
+      }`
+    );
+  }
+
+  // authorization
+  get authorization() {
+    let token = this.config.token || '';
+
+    if (!token && this.config.username && this.config.password) {
+      token = this.config.username + ':' + this.config.password;
+    }
+
+    return `Bearer ${token}`;
+  }
+
+  // database
+  get database() {
+    return this.config.database || DEFAULT_DB;
+  }
+
+  // timeout
+  get timeout() {
+    return this.config.timeout || DEFAULT_HTTP_TIMEOUT;
+  }
+
+  get POST() {
+    return this.client.post;
+  }
+
+  get GET() {
+    return this.client.get;
+  }
+}
+
+// mixin APIs
+export class HttpClient extends Collection(Vector(HttpBaseClient)) {}
diff --git a/milvus/const/defaults.ts b/milvus/const/defaults.ts
@@ -12,3 +12,5 @@ export const DEFAULT_RESOURCE_GROUP = '__default_resource_group';
 export const DEFAULT_DB = 'default';
 export const DEFAULT_DYNAMIC_FIELD = '$meta';
 export const DEFAULT_COUNT_QUERY_STRING = 'count(*)';
+export const DEFAULT_HTTP_TIMEOUT = 60000; // 60s
+export const DEFAULT_HTTP_ENDPOINT_VERSION = 'v1'; // api version, default v1
diff --git a/milvus/http/Collection.ts b/milvus/http/Collection.ts
@@ -0,0 +1,46 @@
+import { HttpBaseClient } from '../HttpClient';
+import { Constructor } from '../types/index';
+import {
+  HttpCollectionCreateReq,
+  HttpCollectionListReq,
+  HttpCollectionListResponse,
+  HttpCollectionDescribeResponse,
+  HttpBaseResponse,
+  HttpBaseReq,
+} from '../types';
+
+export function Collection<T extends Constructor<HttpBaseClient>>(Base: T) {
+  return class extends Base {
+    // POST create collection
+    async createCollection(
+      data: HttpCollectionCreateReq
+    ): Promise<HttpBaseResponse> {
+      const url = `/vector/collections/create`;
+      return await this.POST(url, data);
+    }
+
+    // GET describe collection
+    async describeCollection(
+      params: HttpBaseReq
+    ): Promise<HttpCollectionDescribeResponse> {
+      const url = `/vector/collections/describe`;
+      return await this.GET(url, { params });
+    }
+
+    // POST drop collection
+    async dropCollection(data: HttpBaseReq): Promise<HttpBaseResponse> {
+      const url = `/vector/collections/drop`;
+
+      return await this.POST(url, data);
+    }
+
+    // GET list collections
+    async listCollections(
+      params: HttpCollectionListReq = {}
+    ): Promise<HttpCollectionListResponse> {
+      const url = `/vector/collections`;
+
+      return await this.GET(url, { params });
+    }
+  };
+}
diff --git a/milvus/http/Vector.ts b/milvus/http/Vector.ts
@@ -0,0 +1,47 @@
+import { HttpBaseClient } from '../HttpClient';
+import {
+  Constructor,
+  HttpVectorGetReq,
+  HttpVectorInsertReq,
+  HttpVectorInsertResponse,
+  HttpVectorQueryReq,
+  HttpVectorQueryResponse,
+  HttpVectorSearchReq,
+  HttpVectorDeleteReq,
+  HttpVectorSearchResponse,
+  HttpBaseResponse,
+} from '../types';
+
+export function Vector<T extends Constructor<HttpBaseClient>>(Base: T) {
+  return class extends Base {
+    // GET get data
+    async get(params: HttpVectorGetReq): Promise<HttpBaseResponse> {
+      const url = `/vector/get`;
+      return await this.GET(url, { params });
+    }
+
+    // POST insert data
+    async insert(data: HttpVectorInsertReq): Promise<HttpVectorInsertResponse> {
+      const url = `/vector/insert`;
+      return await this.POST(url, data);
+    }
+
+    // POST query data
+    async query(data: HttpVectorQueryReq): Promise<HttpVectorQueryResponse> {
+      const url = `/vector/query`;
+      return await this.client.post(url, data);
+    }
+
+    // POST search data
+    async search(data: HttpVectorSearchReq): Promise<HttpVectorSearchResponse> {
+      const url = `/vector/search`;
+      return await this.POST(url, data);
+    }
+
+    // POST delete collection
+    async delete(data: HttpVectorDeleteReq): Promise<HttpBaseResponse> {
+      const url = `/vector/delete`;
+      return await this.POST(url, data);
+    }
+  };
+}
diff --git a/milvus/http/index.ts b/milvus/http/index.ts
@@ -0,0 +1,2 @@
+export * from './Collection';
+export * from './Vector';
diff --git a/milvus/index.ts b/milvus/index.ts
@@ -8,3 +8,4 @@ export * from './types';
 export * from './grpc/GrpcClient';
 export * from './MilvusClient';
 export * from './OrmClient';
+export * from './HttpClient';
diff --git a/milvus/types.ts b/milvus/types.ts
@@ -10,3 +10,4 @@ export * from './types/User';
 export * from './types/Resource';
 export * from './types/Client';
 export * from './types/HighLevel';
+export * from './types/Http';
diff --git a/milvus/types/Http.ts b/milvus/types/Http.ts
@@ -0,0 +1,136 @@
+import { FloatVectors } from '..';
+// Class types
+export type Constructor<T = {}> = new (...args: any[]) => T;
+
+type HttpClientConfigBase = {
+  // database name
+  database?: string;
+  // version
+  version?: string;
+  // token
+  token?: string;
+  // The username to use for authentication.
+  username?: string;
+  // The password to use for authentication.
+  password?: string;
+  // request timeout, number in milliseconds.
+  timeout?: number;
+};
+
+type HttpClientConfigAddress = HttpClientConfigBase & {
+  // The address in the format of ${MILVUS_HOST}:${MILVUS_PORT}, for example: 127.0.0.1:19530. It is used to build the baseURL for the HTTP client.
+  endpoint: string;
+  baseURL?: string;
+};
+
+type HttpClientConfigBaseURL = HttpClientConfigBase & {
+  endpoint?: string;
+  // The baseURL is the endpoint's base URL. It is in the format https://${MILVUS_HOST}:${MILVUS_PORT}/v1/. If baseURL is set, it will override the address property.
+  baseURL: string;
+};
+
+export type HttpClientConfig =
+  | HttpClientConfigAddress
+  | HttpClientConfigBaseURL;
+
+// http base request
+export interface HttpBaseReq {
+  dbName?: string;
+  collectionName: string;
+}
+// http base response
+export interface HttpBaseResponse<T = {}> {
+  code: number;
+  data: T;
+  message?: string;
+}
+
+// collection operations
+export interface HttpCollectionCreateReq extends HttpBaseReq {
+  dimension: number;
+  metricType: string;
+  primaryField: string;
+  vectorField: string;
+  description?: string;
+}
+// list collection request
+export interface HttpCollectionListReq
+  extends Omit<HttpBaseReq, 'collectionName'> {}
+
+type Field = {
+  autoId?: boolean;
+  description: string;
+  primaryKey?: boolean;
+  type: string;
+};
+
+type Index = {
+  fieldName: string;
+  indexName: string;
+  metricType: string;
+};
+
+// describe collection response
+export interface HttpCollectionDescribeResponse
+  extends HttpBaseResponse<{
+    collectionName: string;
+    description: string;
+    fields: Field[];
+    indexes: Index[];
+    load: string;
+    shardsNum: number;
+    enableDynamic: boolean;
+  }> {}
+
+// list collections response
+export interface HttpCollectionListResponse
+  extends HttpBaseResponse<string[]> {}
+
+// vector operations
+// insert data request
+export interface HttpVectorInsertReq extends HttpBaseReq {
+  data: Record<string, any>[];
+}
+
+// insert data response
+export interface HttpVectorInsertResponse
+  extends HttpBaseResponse<{
+    insertCount: number;
+    insertIds: number | string[];
+  }> {}
+
+// get vector request
+export interface HttpVectorGetReq extends HttpBaseReq {
+  id: number | number[] | string | string[];
+  outputFields: string[];
+}
+
+// delete vector request
+export interface HttpVectorDeleteReq
+  extends Omit<HttpVectorGetReq, 'outputFields'> {}
+
+// query data request
+export interface HttpVectorQueryReq extends HttpBaseReq {
+  outputFields: string[];
+  filter?: string;
+  limit?: number;
+  offset?: number;
+  params?: Record<string, string | number>;
+}
+
+type QueryResult = Record<string, any>[];
+
+// query response
+export interface HttpVectorQueryResponse
+  extends HttpBaseResponse<QueryResult> {}
+
+// search request
+export interface HttpVectorSearchReq
+  extends Omit<HttpVectorQueryReq, 'filter'> {
+  vector: FloatVectors;
+  filter?: string;
+}
+
+export interface HttpVectorSearchResponse extends HttpVectorQueryResponse {
+  data: QueryResult & { distance: number | string };
+}
diff --git a/milvus/types/index.ts b/milvus/types/index.ts
@@ -11,3 +11,4 @@ export * from './Resource';
 export * from './Client';
 export * from './MilvusIndex';
 export * from './HighLevel';
+export * from './Http';
diff --git a/package.json b/package.json
@@ -10,18 +10,18 @@
   "scripts": {
     "pre": "git submodule update --remote && rm -rf proto/proto/google && mkdir -p proto/proto/google/protobuf && wget https://raw.githubusercontent.com/protocolbuffers/protobuf/main/src/google/protobuf/descriptor.proto -O proto/proto/google/protobuf/descriptor.proto",
     "build": "rm -rf dist && tsc --declaration && node build.js",
-    "testAll": "jest --testPathIgnorePatterns=/test/build/",
     "test": "NODE_ENV=dev jest",
     "bench": "ts-node test/tools/bench.ts",
-    "coverage": "NODE_ENV=dev jest --coverage=true --config jest.config.js --no-cache --testPathIgnorePatterns=/test/build/",
-    "build-test": " yarn build && jest test/build/Collection.spec.ts",
+    "coverage": "NODE_ENV=dev jest --coverage=true --config jest.config.js --no-cache",
+    "build-test": " yarn build && NODE_ENV=dev jest test/build/Collection.spec.ts --testPathIgnorePatterns=none",
     "example": "npx ts-node",
     "doc": "rm -rf docs && npx typedoc",
     "doc-json": "npx typedoc milvus --json"
   },
   "dependencies": {
     "@grpc/grpc-js": "1.8.17",
     "@grpc/proto-loader": "0.7.7",
+    "axios": "^1.5.1",
     "dayjs": "^1.11.7",
     "lru-cache": "^9.1.2",
     "protobufjs": "7.2.4",

diff --git a/test/Alias.spec.ts → test/grpc/Alias.spec.ts b/test/Alias.spec.ts → test/grpc/Alias.spec.ts
@@ -1,5 +1,5 @@
-import { MilvusClient, ERROR_REASONS, ErrorCode } from '../milvus';
-import { IP, genCollectionParams, GENERATE_NAME } from './tools';
+import { MilvusClient, ERROR_REASONS, ErrorCode } from '../../milvus';
+import { IP, genCollectionParams, GENERATE_NAME } from '../tools';
 
 let milvusClient = new MilvusClient({ address: IP });
 const COLLECTION_NAME = GENERATE_NAME();
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		export * from './Collection';
		export * from './Vector';