Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: update err msg and max partition num #778

Merged
merged 2 commits into from
Jul 3, 2024
Merged

fix: update err msg and max partition num #778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b62fdb2
fix: update err msg and max partition num
ThreadDao Jun 24, 2024
6a92a2f
fix upsert/load cases
ThreadDao Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/test-ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ jobs:
working-directory: ci/scripts
run: |
url="https://registry.hub.docker.com/v2/repositories/$IMAGE_REPO/tags?page=1&name=$TAG_PREFIX"
echo "IMAGE_TAG=$(curl -s $url | jq -r '."results"[]["name"] | select(test("amd64$"))' | head -n 1)" >> $GITHUB_ENV
echo "IMAGE_TAG=$(curl -s $url | jq -r '."results"[]["name"]' | head -n 1)" >> $GITHUB_ENV
# echo "IMAGE_TAG=$(curl -s $url | jq -r '."results"[]["name"] | select(test("amd64$"))' | head -n 1)" >> $GITHUB_ENV
# echo "IMAGE_TAG=$(./docker_image_find_tag.sh -n ${IMAGE_REPO} -t ${TAG_PREFIX}latest -f ${TAG_PREFIX} -F -L -q)" >> $GITHUB_ENV
# export IMAGE_TAG=$IMAGE_TAG
# export IMAGE_REPO=$IMAGE_REPO
Expand Down
2 changes: 1 addition & 1 deletion test/common/response_check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func CheckErr(t *testing.T, actualErr error, expErrNil bool, expErrorMsg ...stri
}
}
if !contains {
t.FailNow()
t.Fatalf("CheckErr failed, actualErr doesn't contains any expErrorMsg, please check test cases!")
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions test/common/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ const (

// const default value from milvus
const (
MaxPartitionNum = 4096
MaxPartitionNum = 1024
DefaultDynamicFieldName = "$meta"
QueryCountFieldName = "count(*)"
DefaultPartition = "_default"
Expand Down Expand Up @@ -1257,7 +1257,7 @@ func GenDynamicFieldData(start int, nb int) []entity.Column {
return data
}

func MergeColumnsToDynamic(nb int, columns []entity.Column) *entity.ColumnJSONBytes {
func MergeColumnsToDynamic(nb int, columns []entity.Column, columnName string) *entity.ColumnJSONBytes {
values := make([][]byte, 0, nb)
for i := 0; i < nb; i++ {
m := make(map[string]interface{})
Expand All @@ -1271,7 +1271,7 @@ func MergeColumnsToDynamic(nb int, columns []entity.Column) *entity.ColumnJSONBy
}
values = append(values, bs)
}
jsonColumn := entity.NewColumnJSONBytes(DefaultDynamicFieldName, values)
jsonColumn := entity.NewColumnJSONBytes(columnName, values)

var jsonData []string
for i := 0; i < jsonColumn.Len(); i++ {
Expand Down Expand Up @@ -1320,7 +1320,6 @@ func GenAllFloatIndex(metricTypes ...entity.MetricType) []entity.Index {
nlist := 128
var allFloatIndex []entity.Index
var allMetricTypes []entity.MetricType
log.Println(metricTypes)
if len(metricTypes) == 0 {
allMetricTypes = SupportFloatMetricType
} else {
Expand Down Expand Up @@ -1385,6 +1384,7 @@ type InvalidExprStruct struct {
}

var InvalidExpressions = []InvalidExprStruct{
// https://github.com/milvus-io/milvus-sdk-go/issues/777
{Expr: "id in [0]", ErrNil: true, ErrMsg: "fieldName(id) not found"}, // not exist field but no error
{Expr: "int64 in not [0]", ErrNil: false, ErrMsg: "cannot parse expression"}, // wrong term expr keyword
{Expr: "int64 > 10 AND int64 < 100", ErrNil: false, ErrMsg: "cannot parse expression"}, // AND isn't supported
Expand Down
73 changes: 53 additions & 20 deletions test/testcases/index_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ func TestCreateIndexMultiVectors(t *testing.T) {
common.CheckErr(t, err, true)
for _, idx := range common.GenAllFloatIndex() {
for _, fieldName := range []string{common.DefaultFloat16VecFieldName, common.DefaultBFloat16VecFieldName} {
log.Printf("index name=%s, index type=%v, index params=%v", idx.Name(), idx.IndexType(), idx.Params())
err := mc.CreateIndex(ctx, collName, fieldName, idx, false, client.WithIndexName(fieldName))
common.CheckErr(t, err, true)

Expand Down Expand Up @@ -360,8 +361,7 @@ func TestCreateScalarIndexVectorField(t *testing.T) {
idx := entity.NewScalarIndexWithType(ip)
for _, fieldName := range common.AllVectorsFieldsName {
err := mc.CreateIndex(ctx, collName, fieldName, idx, false)
common.CheckErr(t, err, false, "STL_SORT are only supported on numeric field",
"TRIE are only supported on varchar field", "INVERTED are not supported on")
common.CheckErr(t, err, false, "metric type not set for vector index")
}
}
for _, fieldName := range common.AllFloatVectorsFieldNames {
Expand Down Expand Up @@ -547,7 +547,7 @@ func TestCreateBinaryIndexNotSupportedMetricsType(t *testing.T) {
// create BinFlat
idxBinFlat, _ := entity.NewIndexBinFlat(metricType, 128)
err := mc.CreateIndex(ctx, collName, common.DefaultBinaryVecFieldName, idxBinFlat, false, client.WithIndexName("my_index"))
common.CheckErr(t, err, false, "supported: [HAMMING JACCARD SUBSTRUCTURE SUPERSTRUCTURE]")
common.CheckErr(t, err, false, fmt.Sprintf("binary vector index does not support metric type: %v", metricType))
}

invalidMetricTypes2 := []entity.MetricType{
Expand All @@ -563,7 +563,8 @@ func TestCreateBinaryIndexNotSupportedMetricsType(t *testing.T) {
// create BinIvfFlat index
idxBinIvfFlat, _ := entity.NewIndexBinIvfFlat(metricType, 128)
errIvf := mc.CreateIndex(ctx, collName, common.DefaultBinaryVecFieldName, idxBinIvfFlat, false, client.WithIndexName("my_index2"))
common.CheckErr(t, errIvf, false, fmt.Sprintf("metric type %v not found or not supported", metricType))
common.CheckErr(t, errIvf, false, fmt.Sprintf("metric type %s not found or not supported, supported: [HAMMING JACCARD]", metricType),
"binary vector index does not support metric type")
}

}
Expand Down Expand Up @@ -607,15 +608,18 @@ func TestCreateIndexWithoutIndexTypeParams(t *testing.T) {
common.CheckErr(t, err, true)

for _, fieldName := range common.AllVectorsFieldsName {
idx, _ := entity.NewIndexAUTOINDEX(entity.COSINE)
if fieldName == common.DefaultBinaryVecFieldName {
idx, _ := entity.NewIndexAUTOINDEX(entity.JACCARD)
err = mc.CreateIndex(ctx, collName, fieldName, idx, false)
common.CheckErr(t, err, false, "HNSW only support float vector data type")
// create binary index
idxBinary, _ := entity.NewIndexBinIvfFlat(entity.JACCARD, 64)
err = mc.CreateIndex(ctx, collName, fieldName, idxBinary, false)
common.CheckErr(t, err, true)

// describe and check index
indexes, _ := mc.DescribeIndex(ctx, collName, fieldName)
expIndex := entity.NewGenericIndex(fieldName, entity.AUTOINDEX, map[string]string{"metric_type": string(entity.JACCARD)})
common.CheckIndexResult(t, indexes, expIndex)

} else {
idx, _ := entity.NewIndexAUTOINDEX(entity.COSINE)
// create index
err = mc.CreateIndex(ctx, collName, fieldName, idx, false)
common.CheckErr(t, err, true)
Expand Down Expand Up @@ -843,25 +847,54 @@ func TestCreateSparseUnsupportedIndex(t *testing.T) {
mc.Flush(ctx, collName, false)

// create unsupported vector index on sparse field
autoIdx, _ := entity.NewIndexAUTOINDEX(entity.IP)
vectorIndex := append(common.GenAllFloatIndex(entity.IP), autoIdx)
vectorIndex := append(common.GenAllFloatIndex(entity.IP))
for _, idx := range vectorIndex {
err := mc.CreateIndex(ctx, collName, common.DefaultSparseVecFieldName, idx, false)
common.CheckErr(t, err, false, "data type should be FloatVector, Float16Vector or BFloat16Vector",
"HNSW only support float vector data type")
common.CheckErr(t, err, false, "data type 104 can't build with this index")
}

// create scalar index on sparse vector
for _, idx := range []entity.Index{
entity.NewScalarIndex(),
entity.NewScalarIndexWithType(entity.Trie),
entity.NewScalarIndexWithType(entity.Sorted),
entity.NewScalarIndexWithType(entity.Inverted),
} {
err := mc.CreateIndex(ctx, collName, common.DefaultSparseVecFieldName, idx, false)
common.CheckErr(t, err, false, "TRIE are only supported on varchar field",
"STL_SORT are only supported on numeric field", "HNSW only support float vector data type",
"INVERTED are not supported on SparseFloatVector field")
common.CheckErr(t, err, false, "metric type not set for vector index")
}
}

// create sparse auto / scalar index
func TestCreateSparseAutoIndex(t *testing.T) {
ctx := createContext(t, time.Second*common.DefaultTimeout)
//connect
mc := createMilvusClient(ctx, t)

// create collection with all datatype
cp := CollectionParams{CollectionFieldsType: Int64VarcharSparseVec, AutoID: false, EnableDynamicField: true,
ShardsNum: common.DefaultShards, Dim: common.DefaultDim, MaxLength: 300}
collName := createCollection(ctx, t, mc, cp)

// insert
dp := DataParams{CollectionName: collName, PartitionName: "", CollectionFieldsType: Int64VarcharSparseVec,
start: 0, nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: true, WithRows: false}
_, _ = insertData(ctx, t, mc, dp, common.WithSparseVectorLen(100))
mc.Flush(ctx, collName, false)

// create scalar index on sparse vector
autoIdx, _ := entity.NewIndexAUTOINDEX(entity.IP)
for _, idx := range []entity.Index{
entity.NewScalarIndex(),
autoIdx,
} {
err := mc.CreateIndex(ctx, collName, common.DefaultSparseVecFieldName, idx, false)
common.CheckErr(t, err, true)
idxes, err := mc.DescribeIndex(ctx, collName, common.DefaultSparseVecFieldName)
common.CheckErr(t, err, true)
expIndex := entity.NewGenericIndex(common.DefaultSparseVecFieldName, autoIdx.IndexType(), map[string]string{"index_type": "AUTOINDEX", "metric_type": "IP"})
common.CheckIndexResult(t, idxes, expIndex)
err = mc.DropIndex(ctx, collName, common.DefaultSparseVecFieldName)
common.CheckErr(t, err, true)
}
}

Expand Down Expand Up @@ -924,7 +957,7 @@ func TestCreateIndexNotSupportedField(t *testing.T) {
// create index
idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96)
err := mc.CreateIndex(ctx, collName, common.DefaultFloatFieldName, idx, false)
common.CheckErr(t, err, false, "HNSW only support float vector data type")
common.CheckErr(t, err, false, "can't build hnsw in not vector type")

// create scann index
indexScann, _ := entity.NewIndexSCANN(entity.L2, 8, true)
Expand Down Expand Up @@ -989,14 +1022,14 @@ func TestCreateIndexInvalidParams(t *testing.T) {
common.CheckErr(t, errScann2, true)
err := mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idxScann, false)
common.CheckErr(t, err, false,
fmt.Sprintf("metric type %s not found or not supported, supported: [L2 IP COSINE]", mt))
fmt.Sprintf("float vector index does not support metric type: %s", mt))
}

// invalid flat metric type jaccard for flat index
idx, _ := entity.NewIndexFlat(entity.JACCARD)
errMetricType := mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false)
common.CheckErr(t, errMetricType, false,
"metric type JACCARD not found or not supported, supported: [L2 IP COSINE]")
"float vector index does not support metric type: JACCARD")
}

// test create index with nil index
Expand Down
22 changes: 14 additions & 8 deletions test/testcases/load_release_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,16 +247,22 @@ func TestLoadMultiPartitions(t *testing.T) {
idx, _ := entity.NewIndexHNSW(entity.L2, 8, 96)
mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false)

// load partition
errLoad := mc.LoadPartitions(ctx, collName, []string{partitionName, common.DefaultPartition}, false)
// load default partition
errLoad := mc.LoadPartitions(ctx, collName, []string{common.DefaultPartition}, false)
common.CheckErr(t, errLoad, true)

//query nb from partition
queryIds := entity.NewColumnInt64(common.DefaultIntFieldName, []int64{0, common.DefaultNb})
queryResultPartition, _ := mc.QueryByPks(ctx, collName, []string{}, queryIds, []string{common.DefaultIntFieldName})
common.CheckQueryResult(t, queryResultPartition, []entity.Column{
entity.NewColumnInt64(common.DefaultIntFieldName, []int64{0, common.DefaultNb}),
})
//query nb from default partition
resDef, _ := mc.Query(ctx, collName, []string{common.DefaultPartition}, "", []string{common.QueryCountFieldName})
require.EqualValues(t, common.DefaultNb, resDef.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0])

// load partition and query -> actually not loaded
errLoad = mc.LoadPartitions(ctx, collName, []string{partitionName}, false)
common.CheckErr(t, errLoad, true)
resPar, _ := mc.Query(ctx, collName, []string{partitionName}, "", []string{common.QueryCountFieldName})
require.EqualValues(t, common.DefaultNb, resPar.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0])

res, _ := mc.Query(ctx, collName, []string{}, "", []string{common.QueryCountFieldName})
require.EqualValues(t, common.DefaultNb*2, res.GetColumn(common.QueryCountFieldName).(*entity.ColumnInt64).Data()[0])
}

// test load partitions repeatedly
Expand Down
4 changes: 2 additions & 2 deletions test/testcases/partition_key_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ func TestPartitionKeyInvalidNumPartition(t *testing.T) {
numPartitions int64
errMsg string
}{
{common.MaxPartitionNum + 1, "exceeds max configuration (4096)"},
{common.MaxPartitionNum + 1, fmt.Sprintf("exceeds max configuration (%d)", common.MaxPartitionNum)},
{-1, "the specified partitions should be greater than 0 if partition key is used"},
}
for _, npStruct := range invalidNumPartitionStruct {
Expand All @@ -215,7 +215,7 @@ func TestPartitionKeyNumPartition(t *testing.T) {
1,
128,
64,
4096,
common.MaxPartitionNum,
}
for _, numPartitionsValue := range numPartitionsValues {
ctx := createContext(t, time.Second*common.DefaultTimeout)
Expand Down
Loading
Loading