enhance: streaming node client implementation

- add streaming node grpc client wrapper - add unittest for streaming node grpc client side - fix binary unsafe bug for message Signed-off-by: chyezh <[email protected]>
milvus-io · Jul 16, 2024 · aa50e62 · aa50e62
1 parent 3306bc2
commit aa50e62
Show file tree

Hide file tree

Showing 35 changed files with 4,481 additions and 42 deletions.
diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -572,6 +572,17 @@ dataNode:
   clusteringCompaction:
     memoryBufferRatio: 0.1 # The ratio of memory buffer of clustering compaction. Data larger than threshold will be spilled to storage.
 
+streamingNode:
+  # can specify ip for example
+  # ip: 127.0.0.1
+  ip: # if not specify address, will use the first unicastable address as local ip
+  port: 19532
+  grpc:
+    serverMaxSendSize: 536870912
+    serverMaxRecvSize: 536870912
+    clientMaxSendSize: 268435456
+    clientMaxRecvSize: 268435456
+
 # Configures the system log output.
 log:
   level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.

diff --git a/internal/.mockery.yaml b/internal/.mockery.yaml
@@ -11,6 +11,15 @@ packages:
   github.com/milvus-io/milvus/internal/streamingnode/client/manager:
     interfaces:
       ManagerClient:
+  github.com/milvus-io/milvus/internal/streamingnode/client/handler/assignment:
+    interfaces:
+      Watcher:
+  github.com/milvus-io/milvus/internal/streamingnode/client/handler/producer:
+    interfaces:
+      Producer:
+  github.com/milvus-io/milvus/internal/streamingnode/client/handler/consumer:
+    interfaces:
+      Consumer:
   github.com/milvus-io/milvus/internal/streamingnode/server/wal:
     interfaces:
       OpenerBuilder:
@@ -30,6 +39,10 @@ packages:
       StreamingNodeHandlerService_ConsumeServer:
       StreamingNodeHandlerService_ProduceServer:
       StreamingCoordAssignmentService_AssignmentDiscoverServer:
+      StreamingNodeManagerServiceClient:
+      StreamingNodeHandlerServiceClient:
+      StreamingNodeHandlerService_ConsumeClient:
+      StreamingNodeHandlerService_ProduceClient:
   github.com/milvus-io/milvus/internal/streamingnode/server/walmanager:
     interfaces:
       Manager:
@@ -40,9 +53,13 @@ packages:
     interfaces:
       Discoverer:
       AssignmentDiscoverWatcher:
+  github.com/milvus-io/milvus/internal/util/streamingutil/service/lazygrpc:
+    interfaces:
+      Service:
   github.com/milvus-io/milvus/internal/util/streamingutil/service/resolver:
     interfaces:
       Resolver:
+      Builder:
   google.golang.org/grpc/resolver:
     interfaces:
       ClientConn:

diff --git a/internal/mocks/proto/mock_streamingpb/mock_StreamingNodeHandlerServiceClient.go b/internal/mocks/proto/mock_streamingpb/mock_StreamingNodeHandlerServiceClient.go