feat: [cherry-pick]The expression supports filling elements through templates

go.mod

@@ -26,7 +26,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/klauspost/compress v1.17.9
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
-	github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13
+	github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b
 	github.com/minio/minio-go/v7 v7.0.73
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
 	github.com/prometheus/client_golang v1.14.0

go.sum

@@ -610,6 +610,8 @@ github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZz
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13 h1:79fAk5rUFNl7A5RhYUlxcZTHAY31lygULC6QUvpBrUo=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b h1:Q4f3a6Y+HZFEadEx3pUXTZmBwrS3QRvkEHWNd8fjDtI=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70 h1:Z+sp64fmAOxAG7mU0dfVOXvAXlwRB0c8a96rIM5HevI=
 github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
 github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=

internal/parser/planparserv2/Plan.g4

@@ -5,19 +5,20 @@ expr:
 	| FloatingConstant										                     # Floating
 	| BooleanConstant										                     # Boolean
 	| StringLiteral											                     # String
-	| Identifier											                     # Identifier
+	| (Identifier|Meta)           			      							     # Identifier
 	| JSONIdentifier                                                             # JSONIdentifier
+	| LBRACE Identifier RBRACE                                                   # TemplateVariable
 	| '(' expr ')'											                     # Parens
 	| '[' expr (',' expr)* ','? ']'                                              # Array
+	| EmptyArray                                                                 # EmptyArray
 	| expr LIKE StringLiteral                                                    # Like
 	| expr POW expr											                     # Power
 	| op = (ADD | SUB | BNOT | NOT) expr					                     # Unary
 //	| '(' typeName ')' expr									                     # Cast
 	| expr op = (MUL | DIV | MOD) expr						                     # MulDivMod
 	| expr op = (ADD | SUB) expr							                     # AddSub
 	| expr op = (SHL | SHR) expr							                     # Shift
-	| expr op = (IN | NIN) ('[' expr (',' expr)* ','? ']')                       # Term
-	| expr op = (IN | NIN) EmptyTerm                                             # EmptyTerm
+	| expr op = NOT? IN expr                                                     # Term
 	| (JSONContains | ArrayContains)'('expr',' expr')'                           # JSONContains
 	| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')'                     # JSONContainsAll
 	| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')'                     # JSONContainsAny
@@ -42,6 +43,8 @@ expr:
 // INT64: 'int64';
 // FLOAT: 'float';
 // DOUBLE: 'double';
+LBRACE: '{';
+RBRACE: '}';
 
 LT: '<';
 LE: '<=';
@@ -71,9 +74,8 @@ OR: '||' | 'or';
 BNOT: '~';
 NOT: '!' | 'not';
 
-IN: 'in';
-NIN: 'not in';
-EmptyTerm: '[' (Whitespace | Newline)* ']';
+IN: 'in' | 'IN';
+EmptyArray: '[' (Whitespace | Newline)* ']';
 
 JSONContains: 'json_contains' | 'JSON_CONTAINS';
 JSONContainsAll: 'json_contains_all' | 'JSON_CONTAINS_ALL';
@@ -96,10 +98,11 @@ FloatingConstant:
 	DecimalFloatingConstant
 	| HexadecimalFloatingConstant;
 
-Identifier: Nondigit (Nondigit | Digit)* | '$meta';
+Identifier: Nondigit (Nondigit | Digit)*;
+Meta: '$meta';
 
 StringLiteral: EncodingPrefix? ('"' DoubleSCharSequence? '"' | '\'' SingleSCharSequence? '\'');
-JSONIdentifier: Identifier('[' (StringLiteral | DecimalConstant) ']')+;
+JSONIdentifier: (Identifier | Meta)('[' (StringLiteral | DecimalConstant) ']')+;
 
 fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';
 

internal/parser/planparserv2/check_identical_test.go

@@ -17,9 +17,9 @@ func TestCheckIdentical(t *testing.T) {
 	exprStr1 := `not (((Int64Field > 0) and (FloatField <= 20.0)) or ((Int32Field in [1, 2, 3]) and (VarCharField < "str")))`
 	exprStr2 := `Int32Field in [1, 2, 3]`
 
-	expr1, err := ParseExpr(helper, exprStr1)
+	expr1, err := ParseExpr(helper, exprStr1, nil)
 	assert.NoError(t, err)
-	expr2, err := ParseExpr(helper, exprStr2)
+	expr2, err := ParseExpr(helper, exprStr2, nil)
 	assert.NoError(t, err)
 
 	assert.True(t, CheckPredicatesIdentical(expr1, expr1))

internal/parser/planparserv2/convert_field_data_to_generic_value.go

@@ -0,0 +1,73 @@
+package planparserv2
+
+import (
+	"fmt"
+
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
+	"github.com/milvus-io/milvus/internal/proto/planpb"
+)
+
+func ConvertToGenericValue(templateName string, templateValue *schemapb.TemplateValue) (*planpb.GenericValue, error) {
+	if templateValue == nil {
+		return nil, fmt.Errorf("expression template variable value is nil, template name: {%s}", templateName)
+	}
+	switch templateValue.GetType() {
+	case schemapb.DataType_Bool:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_BoolVal{
+				BoolVal: templateValue.GetBoolVal(),
+			},
+		}, nil
+	case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32, schemapb.DataType_Int64:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_Int64Val{
+				Int64Val: templateValue.GetInt64Val(),
+			},
+		}, nil
+	case schemapb.DataType_Float, schemapb.DataType_Double:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_FloatVal{
+				FloatVal: templateValue.GetFloatVal(),
+			},
+		}, nil
+	case schemapb.DataType_String, schemapb.DataType_VarChar:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_StringVal{
+				StringVal: templateValue.GetStringVal(),
+			},
+		}, nil
+	case schemapb.DataType_Array:
+		elements := templateValue.GetArrayVal().GetArray()
+		arrayValues := make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			arrayElement, err := ConvertToGenericValue(templateName, element)
+			if err != nil {
+				return nil, err
+			}
+			arrayValues[i] = arrayElement
+		}
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_ArrayVal{
+				ArrayVal: &planpb.Array{
+					Array:       arrayValues,
+					SameType:    templateValue.GetArrayVal().GetSameType(),
+					ElementType: templateValue.GetArrayVal().GetElementType(),
+				},
+			},
+		}, nil
+	default:
+		return nil, fmt.Errorf("expression elements can only be scalars")
+	}
+}
+
+func UnmarshalExpressionValues(input map[string]*schemapb.TemplateValue) (map[string]*planpb.GenericValue, error) {
+	result := make(map[string]*planpb.GenericValue, len(input))
+	for name, value := range input {
+		rv, err := ConvertToGenericValue(name, value)
+		if err != nil {
+			return nil, err
+		}
+		result[name] = rv
+	}
+	return result, nil
+}