Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add typescript http client #253

Merged
merged 26 commits into from
Oct 26, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ module.exports = {
testTimeout: 60000,
// because user will cause other test fail, but we still have user spec
coveragePathIgnorePatterns: ['/milvus/User.ts'],
testPathIgnorePatterns: [
'cloud.spec.ts',
'serverless.spec.ts',
], // add this line
testEnvironmentOptions: {
NODE_ENV: 'production',
},
Expand Down
97 changes: 97 additions & 0 deletions milvus/HttpClient.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import axios, { AxiosInstance } from 'axios';
import { HttpClientConfig } from './types';
import { Collection, Vector } from './http';
import {
DEFAULT_DB,
DEFAULT_HTTP_TIMEOUT,
DEFAULT_HTTP_ENDPOINT_VERSION,
} from '../milvus/const';

// base class
export class HttpBaseClient {
// The client configuration.
public config: HttpClientConfig;

// axios
public client: AxiosInstance;

constructor(config: HttpClientConfig) {
// Assign the configuration object.
this.config = config;

// setup axios client
this.client = axios.create({
baseURL: this.baseURL,
timeout: this.timeout,
timeoutErrorMessage: '',
withCredentials: true,
headers: {
Authorization: this.authorization,
Accept: 'application/json',
ContentType: 'application/json',
},
});

// interceptors
this.client.interceptors.request.use(request => {
// if dbName is not set, using default database
// GET
if (request.params) {
request.params.dbName = request.params.dbName || this.database;
}
// POST
if (request.data) {
request.data.dbName = request.data.dbName || this.database;
request.data = JSON.stringify(request.data);
}

// console.log('request: ', request.data);
return request;
});
this.client.interceptors.response.use(response => {
return response.data;
});
}

// baseURL
get baseURL() {
return (
this.config.baseURL ||
`${this.config.endpoint}/${
this.config.version || DEFAULT_HTTP_ENDPOINT_VERSION
}`
);
}

// authorization
get authorization() {
let token = this.config.token || '';

if (!token && this.config.username && this.config.password) {
token = this.config.username + ':' + this.config.password;
}

return `Bearer ${token}`;
}

// database
get database() {
return this.config.database || DEFAULT_DB;
}

// timeout
get timeout() {
return this.config.timeout || DEFAULT_HTTP_TIMEOUT;
}

get POST() {
return this.client.post;
}

get GET() {
return this.client.get;
}
}

// mixin APIs
export class HttpClient extends Collection(Vector(HttpBaseClient)) {}
2 changes: 2 additions & 0 deletions milvus/const/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,5 @@ export const DEFAULT_RESOURCE_GROUP = '__default_resource_group';
export const DEFAULT_DB = 'default';
export const DEFAULT_DYNAMIC_FIELD = '$meta';
export const DEFAULT_COUNT_QUERY_STRING = 'count(*)';
export const DEFAULT_HTTP_TIMEOUT = 60000; // 60s
export const DEFAULT_HTTP_ENDPOINT_VERSION = 'v1'; // api version, default v1
46 changes: 46 additions & 0 deletions milvus/http/Collection.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { HttpBaseClient } from '../HttpClient';
import { Constructor } from '../types/index';
import {
HttpCollectionCreateReq,
HttpCollectionListReq,
HttpCollectionListResponse,
HttpCollectionDescribeResponse,
HttpBaseResponse,
HttpBaseReq,
} from '../types';

export function Collection<T extends Constructor<HttpBaseClient>>(Base: T) {
return class extends Base {
// POST create collection
async createCollection(
data: HttpCollectionCreateReq
): Promise<HttpBaseResponse> {
const url = `/vector/collections/create`;
return await this.POST(url, data);
}

// GET describe collection
async describeCollection(
params: HttpBaseReq
): Promise<HttpCollectionDescribeResponse> {
const url = `/vector/collections/describe`;
return await this.GET(url, { params });
}

// POST drop collection
async dropCollection(data: HttpBaseReq): Promise<HttpBaseResponse> {
const url = `/vector/collections/drop`;

return await this.POST(url, data);
}

// GET list collections
async listCollections(
params: HttpCollectionListReq = {}
): Promise<HttpCollectionListResponse> {
const url = `/vector/collections`;

return await this.GET(url, { params });
}
};
}
47 changes: 47 additions & 0 deletions milvus/http/Vector.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { HttpBaseClient } from '../HttpClient';
import {
Constructor,
HttpVectorGetReq,
HttpVectorInsertReq,
HttpVectorInsertResponse,
HttpVectorQueryReq,
HttpVectorQueryResponse,
HttpVectorSearchReq,
HttpVectorDeleteReq,
HttpVectorSearchResponse,
HttpBaseResponse,
} from '../types';

export function Vector<T extends Constructor<HttpBaseClient>>(Base: T) {
return class extends Base {
// GET get data
async get(params: HttpVectorGetReq): Promise<HttpBaseResponse> {
const url = `/vector/get`;
return await this.GET(url, { params });
}

// POST insert data
async insert(data: HttpVectorInsertReq): Promise<HttpVectorInsertResponse> {
const url = `/vector/insert`;
return await this.POST(url, data);
}

// POST query data
async query(data: HttpVectorQueryReq): Promise<HttpVectorQueryResponse> {
const url = `/vector/query`;
return await this.client.post(url, data);
}

// POST search data
async search(data: HttpVectorSearchReq): Promise<HttpVectorSearchResponse> {
const url = `/vector/search`;
return await this.POST(url, data);
}

// POST delete collection
async delete(data: HttpVectorDeleteReq): Promise<HttpBaseResponse> {
const url = `/vector/delete`;
return await this.POST(url, data);
}
};
}
2 changes: 2 additions & 0 deletions milvus/http/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
export * from './Collection';
export * from './Vector';
1 change: 1 addition & 0 deletions milvus/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ export * from './types';
export * from './grpc/GrpcClient';
export * from './MilvusClient';
export * from './OrmClient';
export * from './HttpClient';
1 change: 1 addition & 0 deletions milvus/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ export * from './types/User';
export * from './types/Resource';
export * from './types/Client';
export * from './types/HighLevel';
export * from './types/Http';
136 changes: 136 additions & 0 deletions milvus/types/Http.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
import { FloatVectors } from '..';
// Class types
export type Constructor<T = {}> = new (...args: any[]) => T;

type HttpClientConfigBase = {
// database name
database?: string;
// version
version?: string;
// token
token?: string;
// The username to use for authentication.
username?: string;
// The password to use for authentication.
password?: string;
// request timeout, number in milliseconds.
timeout?: number;
};

type HttpClientConfigAddress = HttpClientConfigBase & {
// The address in the format of ${MILVUS_HOST}:${MILVUS_PORT}, for example: 127.0.0.1:19530. It is used to build the baseURL for the HTTP client.
endpoint: string;
baseURL?: string;
};

type HttpClientConfigBaseURL = HttpClientConfigBase & {
endpoint?: string;
// The baseURL is the endpoint's base URL. It is in the format https://${MILVUS_HOST}:${MILVUS_PORT}/v1/. If baseURL is set, it will override the address property.
baseURL: string;
};

export type HttpClientConfig =
| HttpClientConfigAddress
| HttpClientConfigBaseURL;

// http base request
export interface HttpBaseReq {
dbName?: string;
collectionName: string;
}
// http base response
export interface HttpBaseResponse<T = {}> {
code: number;
data: T;
message?: string;
}

// collection operations
export interface HttpCollectionCreateReq extends HttpBaseReq {
dimension: number;
metricType: string;
primaryField: string;
vectorField: string;
description?: string;
}
// list collection request
export interface HttpCollectionListReq
extends Omit<HttpBaseReq, 'collectionName'> {}

type Field = {
autoId?: boolean;
description: string;
primaryKey?: boolean;
type: string;
};

type Index = {
fieldName: string;
indexName: string;
metricType: string;
};

// describe collection response
export interface HttpCollectionDescribeResponse
extends HttpBaseResponse<{
collectionName: string;
description: string;
fields: Field[];
indexes: Index[];
load: string;
shardsNum: number;
enableDynamic: boolean;
}> {}

// list collections response
export interface HttpCollectionListResponse
extends HttpBaseResponse<string[]> {}

// vector operations
// insert data request
export interface HttpVectorInsertReq extends HttpBaseReq {
data: Record<string, any>[];
}

// insert data response
export interface HttpVectorInsertResponse
extends HttpBaseResponse<{
insertCount: number;
insertIds: number | string[];
}> {}

// get vector request
export interface HttpVectorGetReq extends HttpBaseReq {
id: number | number[] | string | string[];
outputFields: string[];
}

// delete vector request
export interface HttpVectorDeleteReq
extends Omit<HttpVectorGetReq, 'outputFields'> {}

// query data request
export interface HttpVectorQueryReq extends HttpBaseReq {
outputFields: string[];
filter?: string;
limit?: number;
offset?: number;
params?: Record<string, string | number>;
}

type QueryResult = Record<string, any>[];

// query response
export interface HttpVectorQueryResponse
extends HttpBaseResponse<QueryResult> {}

// search request
export interface HttpVectorSearchReq
extends Omit<HttpVectorQueryReq, 'filter'> {
vector: FloatVectors;
filter?: string;
}

export interface HttpVectorSearchResponse extends HttpVectorQueryResponse {
data: QueryResult & { distance: number | string };
}
1 change: 1 addition & 0 deletions milvus/types/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,3 +11,4 @@ export * from './Resource';
export * from './Client';
export * from './MilvusIndex';
export * from './HighLevel';
export * from './Http';
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,18 @@
"scripts": {
"pre": "git submodule update --remote && rm -rf proto/proto/google && mkdir -p proto/proto/google/protobuf && wget https://raw.githubusercontent.com/protocolbuffers/protobuf/main/src/google/protobuf/descriptor.proto -O proto/proto/google/protobuf/descriptor.proto",
"build": "rm -rf dist && tsc --declaration && node build.js",
"testAll": "jest --testPathIgnorePatterns=/test/build/",
"test": "NODE_ENV=dev jest",
"bench": "ts-node test/tools/bench.ts",
"coverage": "NODE_ENV=dev jest --coverage=true --config jest.config.js --no-cache --testPathIgnorePatterns=/test/build/",
"build-test": " yarn build && jest test/build/Collection.spec.ts",
"coverage": "NODE_ENV=dev jest --coverage=true --config jest.config.js --no-cache",
"build-test": " yarn build && NODE_ENV=dev jest test/build/Collection.spec.ts --testPathIgnorePatterns=none",
"example": "npx ts-node",
"doc": "rm -rf docs && npx typedoc",
"doc-json": "npx typedoc milvus --json"
},
"dependencies": {
"@grpc/grpc-js": "1.8.17",
"@grpc/proto-loader": "0.7.7",
"axios": "^1.5.1",
"dayjs": "^1.11.7",
"lru-cache": "^9.1.2",
"protobufjs": "7.2.4",
Expand Down
4 changes: 2 additions & 2 deletions test/Alias.spec.ts → test/grpc/Alias.spec.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { MilvusClient, ERROR_REASONS, ErrorCode } from '../milvus';
import { IP, genCollectionParams, GENERATE_NAME } from './tools';
import { MilvusClient, ERROR_REASONS, ErrorCode } from '../../milvus';
import { IP, genCollectionParams, GENERATE_NAME } from '../tools';

let milvusClient = new MilvusClient({ address: IP });
const COLLECTION_NAME = GENERATE_NAME();
Expand Down
Loading
Loading