Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

replace axios with node-fetch #254

Merged
merged 9 commits into from
Nov 1, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ The following table shows the recommended `@zilliz/milvus2-sdk-node` versions fo

- [Milvus](https://milvus.io/)
- [Zilliz Cloud](https://cloud.zilliz.com/signup)
- Node: v12+
- Node: v18+

## Installation

Expand Down
5 changes: 1 addition & 4 deletions jest.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,7 @@ module.exports = {
testTimeout: 60000,
// because user will cause other test fail, but we still have user spec
coveragePathIgnorePatterns: ['/milvus/User.ts'],
testPathIgnorePatterns: [
'cloud.spec.ts',
'serverless.spec.ts',
], // add this line
testPathIgnorePatterns: ['cloud.spec.ts', 'serverless.spec.ts'], // add this line
testEnvironmentOptions: {
NODE_ENV: 'production',
},
Expand Down
160 changes: 115 additions & 45 deletions milvus/HttpClient.ts
Original file line number Diff line number Diff line change
@@ -1,56 +1,47 @@
import axios, { AxiosInstance } from 'axios';
import { HttpClientConfig } from './types';
import { HttpClientConfig, FetchOptions } from './types';
import { Collection, Vector } from './http';
import {
DEFAULT_DB,
DEFAULT_HTTP_TIMEOUT,
DEFAULT_HTTP_ENDPOINT_VERSION,
} from '../milvus/const';

// base class
/**
* HttpBaseClient is a base class for making HTTP requests to a Milvus server.
* It provides basic functionality for making GET and POST requests, and handles
* configuration, headers, and timeouts.
*
* The HttpClientConfig object should contain the following properties:
* - endpoint: The URL of the Milvus server.
* - username: (Optional) The username for authentication.
* - password: (Optional) The password for authentication.
* - token: (Optional) The token for authentication.
* - fetch: (Optional) An alternative fetch API implementation, e.g., node-fetch for Node.js environments.
* - baseURL: (Optional) The base URL for the API endpoints.
* - version: (Optional) The version of the API endpoints.
* - database: (Optional) The default database to use for requests.
* - timeout: (Optional) The timeout for requests in milliseconds.
*
* Note: This is a base class and does not provide specific methods for interacting
* with Milvus entities like collections or vectors. For that, use the HttpClient class
* which extends this class and mixes in the Collection and Vector APIs.
*/
export class HttpBaseClient {
// The client configuration.
public config: HttpClientConfig;

// axios
public client: AxiosInstance;

constructor(config: HttpClientConfig) {
shanghaikid marked this conversation as resolved.
Show resolved Hide resolved
// Assign the configuration object.
this.config = config;

// setup axios client
this.client = axios.create({
baseURL: this.baseURL,
timeout: this.timeout,
timeoutErrorMessage: '',
withCredentials: true,
headers: {
Authorization: this.authorization,
Accept: 'application/json',
ContentType: 'application/json',
},
});

// interceptors
this.client.interceptors.request.use(request => {
// if dbName is not set, using default database
// GET
if (request.params) {
request.params.dbName = request.params.dbName || this.database;
}
// POST
if (request.data) {
request.data.dbName = request.data.dbName || this.database;
request.data = JSON.stringify(request.data);
}

// console.log('request: ', request.data);
return request;
});
this.client.interceptors.response.use(response => {
return response.data;
});
// The fetch method used for requests can be customized by providing a fetch property in the configuration.
// If no fetch method is provided, the global fetch method will be used if available.
// If no global fetch method is available, an error will be thrown.
if (!this.config.fetch && typeof fetch === 'undefined') {
throw new Error(
'The Fetch API is not supported in this environment. Please provide an alternative, for example, node-fetch.'
);
}
}

// baseURL
Expand All @@ -76,22 +67,101 @@ export class HttpBaseClient {

// database
get database() {
return this.config.database || DEFAULT_DB;
return this.config.database ?? DEFAULT_DB;
}

// timeout
get timeout() {
return this.config.timeout || DEFAULT_HTTP_TIMEOUT;
return this.config.timeout ?? DEFAULT_HTTP_TIMEOUT;
}

// headers
get headers() {
return {
Authorization: this.authorization,
Accept: 'application/json',
ContentType: 'application/json',
};
}

get POST() {
return this.client.post;
// fetch
get fetch() {
return this.config.fetch ?? fetch;
}

get GET() {
return this.client.get;
// POST API
async POST<T>(
url: string,
data: Record<string, any> = {},
options?: FetchOptions
): Promise<T> {
try {
// timeout controller
const timeout = options?.timeout ?? this.timeout;
const abortController = options?.abortController ?? new AbortController();
const id = setTimeout(() => abortController.abort(), timeout);

// assign database
if (data) {
data.dbName = data.dbName ?? this.database;
}

const response = await this.fetch(`${this.baseURL}${url}`, {
method: 'post',
headers: this.headers,
body: JSON.stringify(data),
signal: abortController.signal,
});

clearTimeout(id);
return response.json() as T;
shanghaikid marked this conversation as resolved.
Show resolved Hide resolved
} catch (error) {
if (error.name === 'AbortError') {
console.warn(`post ${url} request was timeout`);
}
return Promise.reject(error);
}
}

// GET API
async GET<T>(
url: string,
params: Record<string, any> = {},
options?: FetchOptions
): Promise<T> {
try {
// timeout controller
const timeout = options?.timeout ?? this.timeout;
const abortController = options?.abortController ?? new AbortController();
const id = setTimeout(() => abortController.abort(), timeout);

// assign database
if (params) {
params.dbName = params.dbName ?? this.database;
}

const queryParams = new URLSearchParams(params);

const response = await this.fetch(
`${this.baseURL}${url}?${queryParams}`,
{
method: 'get',
headers: this.headers,
signal: abortController.signal,
}
);

clearTimeout(id);

return response.json() as T;
} catch (error) {
if (error.name === 'AbortError') {
console.warn(`milvus http client: request was timeout`);
}
return Promise.reject(error);
}
}
}

// mixin APIs
// The HttpClient class extends the functionality of the HttpBaseClient class by mixing in the Collection and Vector APIs.
export class HttpClient extends Collection(Vector(HttpBaseClient)) {}
9 changes: 6 additions & 3 deletions milvus/MilvusClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ import {
CreateCollectionReq,
ERROR_REASONS,
checkCreateCollectionCompatibility,
DEFAULT_PRIMARY_KEY_FIELD,
DEFAULT_METRIC_TYPE,
DEFAULT_VECTOR_FIELD,
} from '.';
import sdkInfo from '../sdk.json';

Expand Down Expand Up @@ -91,10 +94,10 @@ export class MilvusClient extends GRPCClient {
const {
collection_name,
dimension,
primary_field_name = 'id',
primary_field_name = DEFAULT_PRIMARY_KEY_FIELD,
id_type = DataType.Int64,
metric_type = 'IP',
vector_field_name = 'vector',
metric_type = DEFAULT_METRIC_TYPE,
vector_field_name = DEFAULT_VECTOR_FIELD,
enableDynamicField = true,
enable_dynamic_field = true,
auto_id = false,
Expand Down
4 changes: 3 additions & 1 deletion milvus/const/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ export const DEFAULT_DEBUG = false;
export const DEFAULT_MILVUS_PORT = 19530; // default milvus port
export const DEFAULT_CONNECT_TIMEOUT = 15 * 1000; // 15s
export const DEFAULT_TOPK = 100; // default topk
export const DEFAULT_METRIC_TYPE = 'L2';
export const DEFAULT_METRIC_TYPE = 'IP';
export const DEFAULT_VECTOR_FIELD = 'vector';
export const DEFAULT_PRIMARY_KEY_FIELD = 'id';
export const DEFAULT_MAX_RETRIES = 3; // max retry time
export const DEFAULT_RETRY_DELAY = 30; // retry delay, 30ms
export const DEFAULT_PARTITIONS_NUMBER = 64;
Expand Down
50 changes: 42 additions & 8 deletions milvus/http/Collection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,40 +7,74 @@ import {
HttpCollectionDescribeResponse,
HttpBaseResponse,
HttpBaseReq,
FetchOptions,
} from '../types';
import {
DEFAULT_PRIMARY_KEY_FIELD,
DEFAULT_METRIC_TYPE,
DEFAULT_VECTOR_FIELD,
} from '../const';

/**
* Collection is a mixin function that extends the functionality of a base class.
* It provides methods to interact with collections in a Milvus cluster.
*
* @param {Constructor<HttpBaseClient>} Base - The base class to be extended.
* @returns {class} - The extended class with additional methods for collection management.
*
* @method createCollection - Creates a new collection in Milvus.
* @method describeCollection - Retrieves the description of a specific collection.
* @method dropCollection - Deletes a specific collection from Milvus.
* @method listCollections - Lists all collections in the Milvus cluster.
*/
export function Collection<T extends Constructor<HttpBaseClient>>(Base: T) {
return class extends Base {
// POST create collection
async createCollection(
data: HttpCollectionCreateReq
data: HttpCollectionCreateReq,
options?: FetchOptions
): Promise<HttpBaseResponse> {
const url = `/vector/collections/create`;
return await this.POST(url, data);

// if some keys not provided, using default value
data.metricType = data.metricType || DEFAULT_METRIC_TYPE;
data.primaryField = data.primaryField || DEFAULT_PRIMARY_KEY_FIELD;
data.vectorField = data.vectorField || DEFAULT_VECTOR_FIELD;

return await this.POST<HttpBaseResponse>(url, data, options);
}

// GET describe collection
async describeCollection(
params: HttpBaseReq
params: HttpBaseReq,
options?: FetchOptions
): Promise<HttpCollectionDescribeResponse> {
const url = `/vector/collections/describe`;
return await this.GET(url, { params });
return await this.GET<HttpCollectionDescribeResponse>(
url,
params,
options
);
}

// POST drop collection
async dropCollection(data: HttpBaseReq): Promise<HttpBaseResponse> {
async dropCollection(
data: HttpBaseReq,
options?: FetchOptions
): Promise<HttpBaseResponse> {
const url = `/vector/collections/drop`;

return await this.POST(url, data);
return await this.POST<HttpBaseResponse>(url, data, options);
}

// GET list collections
async listCollections(
params: HttpCollectionListReq = {}
params: HttpCollectionListReq = {},
options?: FetchOptions
): Promise<HttpCollectionListResponse> {
const url = `/vector/collections`;

return await this.GET(url, { params });
return await this.GET<HttpCollectionListResponse>(url, params, options);
}
};
}
Loading