Dynamically generate PROTOS list and add the generated files.

Makefile

@@ -42,20 +42,8 @@ BUF_VERSION_URL := https://raw.githubusercontent.com/vdaas/vald/main/versions/BU
 SHADOW_ROOT       = vald
 SHADOW_PROTO_ROOT = $(SHADOW_ROOT)/$(SHADOW_ROOT)
 
-PROTOS = \
-	v1/agent/core/agent.proto \
-	v1/filter/egress/egress_filter.proto \
-	v1/filter/ingress/ingress_filter.proto \
-	v1/vald/filter.proto \
-	v1/vald/insert.proto \
-	v1/vald/object.proto \
-	v1/vald/index.proto \
-	v1/vald/remove.proto \
-	v1/vald/search.proto \
-	v1/vald/update.proto \
-	v1/vald/upsert.proto \
-	v1/payload/payload.proto
-PROTOS := $(PROTOS:%=$(PROTO_ROOT)/%)
+# Use lazy evaluation for PROTOS to ensure it is evaluated after cloning
+PROTOS = $(shell find $(PROTO_ROOT)/v1 -name '*.proto')
 SHADOWS = $(PROTOS:$(PROTO_ROOT)/%.proto=$(SHADOW_PROTO_ROOT)/%.proto)
 PB2PYS  = $(PROTOS:$(PROTO_ROOT)/%.proto=$(PB2DIR_ROOT)/$(SHADOW_ROOT)/%_pb2.py)
 
@@ -210,4 +198,4 @@ sync/k3d/mk: Makefile.d
 	rm -rf $(K3D_MAKEFILE)
 	curl -fsSLo $(K3D_MAKEFILE) $(K3D_MAKEFILE_URL)
 
-include $(K3D_MAKEFILE)
+include $(K3D_MAKEFILE)

README.md

@@ -24,6 +24,7 @@ from vald.v1.vald import insert_pb2_grpc
 from vald.v1.vald import search_pb2_grpc
 from vald.v1.vald import update_pb2_grpc
 from vald.v1.vald import remove_pb2_grpc
+from vald.v1.vald import flush_pb2_grpc
 from vald.v1.payload import payload_pb2
 
 ## create a channel by passing "{host}:{port}"
@@ -34,6 +35,7 @@ istub = insert_pb2_grpc.InsertStub(channel)
 sstub = search_pb2_grpc.SearchStub(channel)
 ustub = update_pb2_grpc.UpdateStub(channel)
 rstub = remove_pb2_grpc.RemoveStub(channel)
+fstub = flush_pb2_grpc.FlushStub(channel)
 
 ## call RPCs: Insert
 vec = payload_pb2.Object.Vector(id='vector_id_1', vector=[0.1, 0.2, 0.3])
@@ -59,6 +61,9 @@ rcfg = payload_pb2.Remove.Config(skip_strict_exist_check=True)
 rid = payload_pb2.Object.ID(id='vector_id_1')
 rstub.Remove(payload_pb2.Remove.Request(id=rid, config=rcfg))
 
+## call RPCs: Flush
+fstub.Flush(payload_pb2.Flush.Request())
+
 ## close channel
 channel.close()
 ```

src/tests/test_e2e.py

@@ -9,6 +9,7 @@
 from vald.v1.vald import update_pb2_grpc
 from vald.v1.vald import upsert_pb2_grpc
 from vald.v1.vald import remove_pb2_grpc
+from vald.v1.vald import flush_pb2_grpc
 from vald.v1.vald import object_pb2_grpc
 from vald.v1.vald import index_pb2_grpc
 from vald.v1.payload import payload_pb2
@@ -286,3 +287,9 @@ def test_stream_remove(self):
         for result in results:
             self.assertIsInstance(result, payload_pb2.Object.StreamLocation)
             self.assertEqual(result.status.code, 0)
+
+    def test_flush(self):
+        stub = flush_pb2_grpc.FlushStub(self.channel)
+        request = payload_pb2.Flush.Request()
+        results = stub.Flush(request)
+        self.assertIsInstance(results, payload_pb2.Info.Index.Count)

src/vald/v1/agent/sidecar/sidecar_pb2_grpc.py

@@ -0,0 +1,36 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+
+
+class SidecarStub(object):
+    """Represent the agent sidecar service.
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+
+
+class SidecarServicer(object):
+    """Represent the agent sidecar service.
+    """
+
+
+def add_SidecarServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'sidecar.v1.Sidecar', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+    server.add_registered_method_handlers('sidecar.v1.Sidecar', rpc_method_handlers)
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Sidecar(object):
+    """Represent the agent sidecar service.
+    """

src/vald/v1/discoverer/discoverer_pb2_grpc.py

@@ -0,0 +1,169 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from vald.v1.payload import payload_pb2 as vald_dot_v1_dot_payload_dot_payload__pb2
+
+
+class DiscovererStub(object):
+    """Represent the discoverer service.
+    """
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.Pods = channel.unary_unary(
+                '/discoverer.v1.Discoverer/Pods',
+                request_serializer=vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.SerializeToString,
+                response_deserializer=vald_dot_v1_dot_payload_dot_payload__pb2.Info.Pods.FromString,
+                _registered_method=True)
+        self.Nodes = channel.unary_unary(
+                '/discoverer.v1.Discoverer/Nodes',
+                request_serializer=vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.SerializeToString,
+                response_deserializer=vald_dot_v1_dot_payload_dot_payload__pb2.Info.Nodes.FromString,
+                _registered_method=True)
+        self.Services = channel.unary_unary(
+                '/discoverer.v1.Discoverer/Services',
+                request_serializer=vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.SerializeToString,
+                response_deserializer=vald_dot_v1_dot_payload_dot_payload__pb2.Info.Services.FromString,
+                _registered_method=True)
+
+
+class DiscovererServicer(object):
+    """Represent the discoverer service.
+    """
+
+    def Pods(self, request, context):
+        """Represent the RPC to get the agent pods information.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Nodes(self, request, context):
+        """Represent the RPC to get the node information.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def Services(self, request, context):
+        """Represent the RPC to get the readreplica svc information.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_DiscovererServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'Pods': grpc.unary_unary_rpc_method_handler(
+                    servicer.Pods,
+                    request_deserializer=vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.FromString,
+                    response_serializer=vald_dot_v1_dot_payload_dot_payload__pb2.Info.Pods.SerializeToString,
+            ),
+            'Nodes': grpc.unary_unary_rpc_method_handler(
+                    servicer.Nodes,
+                    request_deserializer=vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.FromString,
+                    response_serializer=vald_dot_v1_dot_payload_dot_payload__pb2.Info.Nodes.SerializeToString,
+            ),
+            'Services': grpc.unary_unary_rpc_method_handler(
+                    servicer.Services,
+                    request_deserializer=vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.FromString,
+                    response_serializer=vald_dot_v1_dot_payload_dot_payload__pb2.Info.Services.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'discoverer.v1.Discoverer', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+    server.add_registered_method_handlers('discoverer.v1.Discoverer', rpc_method_handlers)
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Discoverer(object):
+    """Represent the discoverer service.
+    """
+
+    @staticmethod
+    def Pods(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/discoverer.v1.Discoverer/Pods',
+            vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.SerializeToString,
+            vald_dot_v1_dot_payload_dot_payload__pb2.Info.Pods.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
+    @staticmethod
+    def Nodes(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/discoverer.v1.Discoverer/Nodes',
+            vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.SerializeToString,
+            vald_dot_v1_dot_payload_dot_payload__pb2.Info.Nodes.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
+    @staticmethod
+    def Services(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/discoverer.v1.Discoverer/Services',
+            vald_dot_v1_dot_payload_dot_payload__pb2.Discoverer.Request.SerializeToString,
+            vald_dot_v1_dot_payload_dot_payload__pb2.Info.Services.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)