enhance: Update the template expression proto to improve transmission…

… efficiency (milvus-io#37484) issue: milvus-io#36672 Signed-off-by: Cai Zhang <[email protected]>
bigsheeper · Nov 7, 2024 · de62764 · de62764
1 parent 4dc6841
commit de62764
Show file tree

Hide file tree

Showing 6 changed files with 347 additions and 316 deletions.
diff --git a/go.mod b/go.mod
@@ -23,7 +23,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/klauspost/compress v1.17.9
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
-	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7
+	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241106083218-5de5d0cfb1c1
 	github.com/minio/minio-go/v7 v7.0.73
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
 	github.com/prometheus/client_golang v1.14.0

diff --git a/go.sum b/go.sum
@@ -627,8 +627,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
 github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7 h1:HwAitQk+V59QdYUwwVVYHTujd4QZrebg2Cc2hmcjhAg=
-github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241106083218-5de5d0cfb1c1 h1:GFS5AxKPcEstcfJgMGxRH+l/mKA0kK1sHDOxnOqMnoA=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241106083218-5de5d0cfb1c1/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
 github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
 github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

diff --git a/internal/parser/planparserv2/convert_field_data_to_generic_value.go b/internal/parser/planparserv2/convert_field_data_to_generic_value.go
@@ -1,60 +1,140 @@
 package planparserv2
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/internal/proto/planpb"
 )
 
+func convertArrayValue(templateName string, templateValue *schemapb.TemplateArrayValue) (*planpb.GenericValue, error) {
+	var arrayValues []*planpb.GenericValue
+	var elementType schemapb.DataType
+	switch templateValue.GetData().(type) {
+	case *schemapb.TemplateArrayValue_BoolData:
+		elements := templateValue.GetBoolData().GetData()
+		arrayValues = make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			arrayValues[i] = &planpb.GenericValue{
+				Val: &planpb.GenericValue_BoolVal{
+					BoolVal: element,
+				},
+			}
+		}
+		elementType = schemapb.DataType_Bool
+	case *schemapb.TemplateArrayValue_LongData:
+		elements := templateValue.GetLongData().GetData()
+		arrayValues = make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			arrayValues[i] = &planpb.GenericValue{
+				Val: &planpb.GenericValue_Int64Val{
+					Int64Val: element,
+				},
+			}
+		}
+		elementType = schemapb.DataType_Int64
+	case *schemapb.TemplateArrayValue_DoubleData:
+		elements := templateValue.GetDoubleData().GetData()
+		arrayValues = make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			arrayValues[i] = &planpb.GenericValue{
+				Val: &planpb.GenericValue_FloatVal{
+					FloatVal: element,
+				},
+			}
+		}
+		elementType = schemapb.DataType_Double
+	case *schemapb.TemplateArrayValue_StringData:
+		elements := templateValue.GetStringData().GetData()
+		arrayValues = make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			arrayValues[i] = &planpb.GenericValue{
+				Val: &planpb.GenericValue_StringVal{
+					StringVal: element,
+				},
+			}
+		}
+		elementType = schemapb.DataType_VarChar
+	case *schemapb.TemplateArrayValue_ArrayData:
+		elements := templateValue.GetArrayData().GetData()
+		arrayValues = make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			targetValue, err := convertArrayValue(templateName, element)
+			if err != nil {
+				return nil, err
+			}
+			arrayValues[i] = targetValue
+		}
+		elementType = schemapb.DataType_Array
+	case *schemapb.TemplateArrayValue_JsonData:
+		elements := templateValue.GetJsonData().GetData()
+		arrayValues = make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			var jsonElement interface{}
+			err := json.Unmarshal(element, &jsonElement)
+			if err != nil {
+				return nil, err
+			}
+			decoder := json.NewDecoder(bytes.NewBuffer(element))
+			decoder.UseNumber()
+			var value interface{}
+			if err = decoder.Decode(&value); err != nil {
+				return nil, err
+			}
+			parsedValue, _, err := parseJSONValue(value)
+			if err != nil {
+				return nil, err
+			}
+			arrayValues[i] = parsedValue
+		}
+		elementType = schemapb.DataType_JSON
+	default:
+		return nil, fmt.Errorf("unknown template variable value type: %v", templateValue.GetData())
+	}
+	return &planpb.GenericValue{
+		Val: &planpb.GenericValue_ArrayVal{
+			ArrayVal: &planpb.Array{
+				Array:       arrayValues,
+				SameType:    elementType != schemapb.DataType_JSON,
+				ElementType: elementType,
+			},
+		},
+	}, nil
+}
+
 func ConvertToGenericValue(templateName string, templateValue *schemapb.TemplateValue) (*planpb.GenericValue, error) {
 	if templateValue == nil {
 		return nil, fmt.Errorf("expression template variable value is nil, template name: {%s}", templateName)
 	}
-	switch templateValue.GetType() {
-	case schemapb.DataType_Bool:
+	switch templateValue.GetVal().(type) {
+	case *schemapb.TemplateValue_BoolVal:
 		return &planpb.GenericValue{
 			Val: &planpb.GenericValue_BoolVal{
 				BoolVal: templateValue.GetBoolVal(),
 			},
 		}, nil
-	case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32, schemapb.DataType_Int64:
+	case *schemapb.TemplateValue_Int64Val:
 		return &planpb.GenericValue{
 			Val: &planpb.GenericValue_Int64Val{
 				Int64Val: templateValue.GetInt64Val(),
 			},
 		}, nil
-	case schemapb.DataType_Float, schemapb.DataType_Double:
+	case *schemapb.TemplateValue_FloatVal:
 		return &planpb.GenericValue{
 			Val: &planpb.GenericValue_FloatVal{
 				FloatVal: templateValue.GetFloatVal(),
 			},
 		}, nil
-	case schemapb.DataType_String, schemapb.DataType_VarChar:
+	case *schemapb.TemplateValue_StringVal:
 		return &planpb.GenericValue{
 			Val: &planpb.GenericValue_StringVal{
 				StringVal: templateValue.GetStringVal(),
 			},
 		}, nil
-	case schemapb.DataType_Array:
-		elements := templateValue.GetArrayVal().GetArray()
-		arrayValues := make([]*planpb.GenericValue, len(elements))
-		for i, element := range elements {
-			arrayElement, err := ConvertToGenericValue(templateName, element)
-			if err != nil {
-				return nil, err
-			}
-			arrayValues[i] = arrayElement
-		}
-		return &planpb.GenericValue{
-			Val: &planpb.GenericValue_ArrayVal{
-				ArrayVal: &planpb.Array{
-					Array:       arrayValues,
-					SameType:    templateValue.GetArrayVal().GetSameType(),
-					ElementType: templateValue.GetArrayVal().GetElementType(),
-				},
-			},
-		}, nil
+	case *schemapb.TemplateValue_ArrayVal:
+		return convertArrayValue(templateName, templateValue.GetArrayVal())
 	default:
 		return nil, fmt.Errorf("expression elements can only be scalars")
 	}