Skip to content

Commit

Permalink
test: updata v2.2 nightly to v2.4 and add mmap scalar index cases (#812)
Browse files Browse the repository at this point in the history 
test: updata v2.2 nightly to v2.4
test: update cases for mmap scalar index

Signed-off-by: ThreadDao <[email protected]>
  • Loading branch information
@ThreadDao
ThreadDao authored Aug 28, 2024
1 parent 0c84a71 commit bd99cd1
Show file tree
Hide file tree
Showing 4 changed files with 174 additions and 56 deletions.
12 changes: 6 additions & 6 deletions .github/workflows/nightly_ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,23 +132,23 @@ jobs:
run: |
nc -vz 127.0.0.1 19530
curl -sSL "https://github.com/gotestyourself/gotestsum/releases/download/v1.8.2/gotestsum_1.8.2_linux_amd64.tar.gz" | sudo tar -xz -C /usr/local/bin gotestsum
gotestsum --format testname --hide-summary=output ./testcases/... --tags L0,L1,L2,L3 --addr=127.0.0.1:19530 -timeout=60m
gotestsum --format testname --hide-summary=output ./testcases/... --tags L0,L1,L2,L3 --addr=127.0.0.1:19530 -timeout=60m
nightly-v2_2_x:
name: Run Nightly CI(v2.2.x)
nightly-v2_4_x:
name: Run Nightly CI(v2.4.x)
runs-on: ubuntu-latest
timeout-minutes: 90
env:
IMAGE_REPO: "milvusdb/milvus"
TAG_PREFIX: "2.2.0-"
TAG_PREFIX: "2.4-"
RELEASE_NAME: "go-test"
IMAGE_TAG: "2.2.0-latest"
IMAGE_TAG: "2.4-latest"
QA_NAMESPACE: "qa"
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "v2.2.x"
ref: "master"
fetch-depth: '0'

- name: Get the latest of Milvus dev image tag
Expand Down
46 changes: 22 additions & 24 deletions test/testcases/index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,16 @@ import (
"github.com/milvus-io/milvus-sdk-go/v2/test/common"
)

func supportScalarIndexFieldType(field entity.FieldType) bool {
vectorFieldTypes := []entity.FieldType{
entity.FieldTypeBinaryVector, entity.FieldTypeFloatVector,
entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector, entity.FieldTypeSparseVector,
entity.FieldTypeJSON,
}
for _, vectorFieldType := range vectorFieldTypes {
if field == vectorFieldType {
return false
}
}
return true
}
/*
Info about scalar index
TRIE: only support varchar
STL_SORT: only support numeric (not include Array field)
INVERTED: all supported except Json
Bitmap: all supported except Json, float, double. (If Array field, according to its ElementType)
ScalarAutoIndex: {"int_*": "HYBRID","varchar": "HYBRID","bool": "BITMAP", "float/double": "INVERTED"}
- except Json
- if Array field, according to its ElementType
*/

// test create index with supported float vector index, L2 metric type
func TestCreateIndex(t *testing.T) {
Expand Down Expand Up @@ -193,7 +190,7 @@ func TestCreateScalarIndex(t *testing.T) {
common.PrintAllFieldNames(collName, coll.Schema)
idx := entity.NewScalarIndex()
for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
err := mc.CreateIndex(ctx, collName, field.Name, idx, false, client.WithIndexName(field.Name))
common.CheckErr(t, err, true)

Expand Down Expand Up @@ -254,7 +251,7 @@ func TestCreateIndexOnLoadedCollection(t *testing.T) {
coll, _ := mc.DescribeCollection(ctx, collName)
idx := entity.NewScalarIndex()
for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
err := mc.CreateIndex(ctx, collName, field.Name, idx, false, client.WithIndexName(field.Name))
// need check failed
common.CheckErr(t, err, true, "")
Expand All @@ -265,7 +262,7 @@ func TestCreateIndexOnLoadedCollection(t *testing.T) {
common.CheckErr(t, err, true)

for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
_, err := mc.DescribeIndex(ctx, collName, field.Name)
common.CheckErr(t, err, true, "")
}
Expand Down Expand Up @@ -295,7 +292,7 @@ func TestCreateTrieScalarIndexUnsupportedDataType(t *testing.T) {
coll, _ := mc.DescribeCollection(ctx, collName)
idx := entity.NewScalarIndexWithType(entity.Trie)
for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
if field.DataType == entity.FieldTypeVarChar {
err := mc.CreateIndex(ctx, collName, field.Name, idx, false, client.WithIndexName(field.Name))
common.CheckErr(t, err, true)
Expand Down Expand Up @@ -335,7 +332,7 @@ func TestCreateSortScalarIndexUnsupportedDataType(t *testing.T) {
coll, _ := mc.DescribeCollection(ctx, collName)
idx := entity.NewScalarIndexWithType(entity.Sorted)
for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
if field.DataType == entity.FieldTypeVarChar || field.DataType == entity.FieldTypeBool ||
field.DataType == entity.FieldTypeArray {
err := mc.CreateIndex(ctx, collName, field.Name, idx, false, client.WithIndexName(field.Name))
Expand Down Expand Up @@ -376,7 +373,7 @@ func TestCreateInvertedScalarIndex(t *testing.T) {
coll, _ := mc.DescribeCollection(ctx, collName)
idx := entity.NewScalarIndexWithType(entity.Inverted)
for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
err := mc.CreateIndex(ctx, collName, field.Name, idx, false, client.WithIndexName(field.Name))
common.CheckErr(t, err, true)

Expand Down Expand Up @@ -424,7 +421,7 @@ func TestCreateBitmapScalarIndex(t *testing.T) {
idx := entity.NewScalarIndexWithType(entity.Bitmap)
BitmapNotSupport := []interface{}{entity.FieldTypeJSON, entity.FieldTypeDouble, entity.FieldTypeFloat}
for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
log.Println(field.Name, field.DataType, field.ElementType)
if common.CheckContainsValue(BitmapNotSupport, field.DataType) || (field.DataType == entity.FieldTypeArray && common.CheckContainsValue(BitmapNotSupport, field.ElementType)) {
err := mc.CreateIndex(ctx, collName, field.Name, idx, false, client.WithIndexName(field.Name))
Expand Down Expand Up @@ -548,6 +545,10 @@ func TestCreateIndexJsonField(t *testing.T) {
err := mc.CreateIndex(ctx, collName, common.DefaultJSONFieldName, entity.NewScalarIndexWithType(ip.indexType), false, client.WithIndexName("json_index"))
common.CheckErr(t, err, false, ip.errMsg)
}

autoIndex, _ := entity.NewIndexAUTOINDEX(entity.COSINE)
err = mc.CreateIndex(ctx, collName, common.DefaultJSONFieldName, autoIndex, false, client.WithIndexName("json_index"))
common.CheckErr(t, err, false, "create auto index on type:JSON is not supported")
}

func TestCreateIndexArrayField(t *testing.T) {
Expand Down Expand Up @@ -576,9 +577,6 @@ func TestCreateIndexArrayField(t *testing.T) {
inxError := []scalarIndexError{
{entity.Sorted, "STL_SORT are only supported on numeric field"},
{entity.Trie, "TRIE are only supported on varchar field"},

// AutoIndex support building on all array dtype fileds
// {entity.Scalar, "create auto index on Array field is not supported"},
}

// create scalar and vector index on array field
Expand Down Expand Up @@ -892,7 +890,7 @@ func TestCreateAutoIndexScalarFields(t *testing.T) {
indexAuto, _ := entity.NewIndexAUTOINDEX(entity.L2)
coll, _ := mc.DescribeCollection(ctx, collName)
for _, field := range coll.Schema.Fields {
if supportScalarIndexFieldType(field.DataType) {
if SupportScalarIndexFieldType(field.DataType) {
if field.DataType == entity.FieldTypeJSON {
err := mc.CreateIndex(ctx, collName, field.Name, indexAuto, false, client.WithIndexName(field.Name))
common.CheckErr(t, err, false, fmt.Sprintf("create auto index on %v field is not supported", field.DataType))
Expand Down
157 changes: 131 additions & 26 deletions test/testcases/load_release_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ func TestLoadMultiPartitions(t *testing.T) {
mc := createMilvusClient(ctx, t)

// create collection and insert [0, nb) into default partition, [nb, nb*2) into new partition
collName := createDefaultCollection(ctx, t, mc, false, common.DefaultShards)
collName := createDefaultCollection(ctx, t, mc, false, common.DefaultShards, client.WithConsistencyLevel(entity.ClStrong))
partitionName, _, _ := createInsertTwoPartitions(ctx, t, mc, collName, common.DefaultNb)

// create index
Expand Down Expand Up @@ -540,7 +540,6 @@ func TestMmapCollectionIndexDefault(t *testing.T) {
_ = mc.Flush(ctx, collName, false)

// create vector index with mmap
GenDefaultIndexParamsForAllVectors()
indexHnsw, _ := entity.NewIndexHNSW(entity.L2, 8, 96)
indexBinary, _ := entity.NewIndexBinIvfFlat(entity.JACCARD, 64)
for _, fieldName := range common.AllVectorsFieldsName {
Expand Down Expand Up @@ -713,12 +712,11 @@ func TestMmapCollectionScalarIndexed(t *testing.T) {

dp := DataParams{DoInsert: true, CollectionFieldsType: Int64FloatVec, start: 0, nb: common.DefaultNb,
dim: common.DefaultDim, EnableDynamicField: false}

collName := prepareCollection(ctx, t, mc, cp, WithDataParams(dp), WithCreateOption(client.WithConsistencyLevel(entity.ClStrong)))

// create scalar index
for _, fName := range []string{common.DefaultIntFieldName, common.DefaultFloatFieldName} {
err := mc.CreateIndex(ctx, collName, fName, entity.NewScalarIndexWithType(entity.Inverted), false)
err := mc.CreateIndex(ctx, collName, fName, entity.NewScalarIndex(), false)
common.CheckErr(t, err, true)
}

Expand All @@ -728,59 +726,166 @@ func TestMmapCollectionScalarIndexed(t *testing.T) {
common.CheckErr(t, err, true)
err = mc.LoadCollection(ctx, collName, false)
common.CheckErr(t, err, true)

sp, _ := entity.NewIndexHNSWSearchParam(32)
queryVec := common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
resSearch, errSearch := mc.Search(ctx, collName, []string{}, "", []string{"*"}, queryVec, common.DefaultFloatVecFieldName,
entity.L2, common.DefaultTopK, sp)
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultTopK)
common.CheckOutputFields(t, resSearch[0].Fields, []string{common.DefaultIntFieldName, common.DefaultFloatFieldName, common.DefaultFloatVecFieldName})
}

// test mmap scalar index: inverted
func TestMmapScalarInvertedIndex(t *testing.T) {
// vector index
ctx := createContext(t, time.Second*common.DefaultTimeout*2)
// connect
mc := createMilvusClient(ctx, t)

// create -> insert [0, 3000) -> flush -> index -> load
cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: true,
ShardsNum: common.DefaultShards, Dim: common.DefaultDim}

dp := DataParams{DoInsert: true, CollectionFieldsType: AllFields, start: 0, nb: common.DefaultNb,
dim: common.DefaultDim, EnableDynamicField: true}

// build vector's indexes
ips := GenDefaultIndexParamsForAllVectors()
lp := LoadParams{DoLoad: false}
collName := prepareCollection(ctx, t, mc, cp, WithDataParams(dp), WithIndexParams(ips), WithLoadParams(lp),
WithCreateOption(client.WithConsistencyLevel(entity.ClStrong)))

//create scalar index with mmap
collection, _ := mc.DescribeCollection(ctx, collName)
for _, field := range collection.Schema.Fields {
if SupportScalarIndexFieldType(field.DataType) {
err := mc.CreateIndex(ctx, collName, field.Name, entity.NewScalarIndexWithType(entity.Inverted), false, client.WithMmap(true))
common.CheckErr(t, err, true)
}
}

// load and search
mc.LoadCollection(ctx, collName, false)
// query
queryRes, err := mc.Query(ctx, collName, []string{}, fmt.Sprintf("%s < 10", common.DefaultIntFieldName), []string{"*"})
common.CheckErr(t, err, true)
require.Equal(t, queryRes.GetColumn(common.DefaultIntFieldName).Len(), 10)
common.CheckOutputFields(t, queryRes, common.GetAllFieldsName(true, false))

// search
queryVec1 := common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
sp, _ := entity.NewIndexHNSWSearchParam(74)
expr := fmt.Sprintf("%s > 10", common.DefaultIntFieldName)
searchRes, _ := mc.Search(ctx, collName, []string{}, expr, []string{"*"}, queryVec1, common.DefaultFloatVecFieldName,
entity.L2, common.DefaultTopK, sp)
common.CheckOutputFields(t, searchRes[0].Fields, common.GetAllFieldsName(true, false))

// hybrid search
queryVec2 := common.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeBinaryVector)
sReqs := []*client.ANNSearchRequest{
client.NewANNSearchRequest(common.DefaultFloatVecFieldName, entity.L2, "", queryVec1, sp, common.DefaultTopK, client.WithOffset(3)),
client.NewANNSearchRequest(common.DefaultBinaryVecFieldName, entity.JACCARD, "", queryVec2, sp, common.DefaultTopK),
}
_, errSearch := mc.HybridSearch(ctx, collName, []string{}, common.DefaultTopK, []string{}, client.NewRRFReranker(), sReqs)
common.CheckErr(t, errSearch, true)
}

// test mmap unsupported index: DiskANN, GPU-class
// test mmap unsupported index: DiskANN, GPU-class, scalar index except inverted
func TestMmapIndexUnsupported(t *testing.T) {
t.Skip("https://github.com/milvus-io/milvus-sdk-go/issues/714")
ctx := createContext(t, time.Second*common.DefaultTimeout*2)
// connect
mc := createMilvusClient(ctx, t)

// create -> insert -> flush -> index -> load
cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: false,
cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: false,
ShardsNum: common.DefaultShards, Dim: common.DefaultDim}
collName := createCollection(ctx, t, mc, cp)

dp := DataParams{DoInsert: true, CollectionName: collName, CollectionFieldsType: Int64FloatVec, start: 0,
nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false}
insertData(ctx, t, mc, dp)
mc.Flush(ctx, collName, false)

//create index with mmap
idx, _ := entity.NewIndexDISKANN(entity.COSINE)
err := mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idx, false, client.WithMmap(true))
common.CheckErr(t, err, false, "index type DISKANN does not support mmap")

//create scalar index with mmap
for _, idx := range []entity.Index{entity.NewScalarIndex(), entity.NewScalarIndexWithType(entity.Inverted)} {
err := mc.CreateIndex(ctx, collName, common.DefaultFloatFieldName, idx, false, client.WithMmap(true))
common.CheckErr(t, err, false, "does not support mmap")
}
// bitmap index with mmap
err1 := mc.CreateIndex(ctx, collName, common.DefaultIntFieldName, entity.NewScalarIndexWithType(entity.Bitmap), false, client.WithMmap(true))
common.CheckErr(t, err1, false, "index type BITMAP does not support mmap")

err1 = mc.CreateIndex(ctx, collName, common.DefaultIntFieldName, entity.NewScalarIndex(), false, client.WithMmap(true))
common.CheckErr(t, err1, false, "index type HYBRID does not support mmap")

err1 = mc.CreateIndex(ctx, collName, common.DefaultVarcharFieldName, entity.NewScalarIndexWithType(entity.Trie), false, client.WithMmap(true))
common.CheckErr(t, err1, false, "index type Trie does not support mmap")

err1 = mc.CreateIndex(ctx, collName, common.DefaultFloatFieldName, entity.NewScalarIndexWithType(entity.Sorted), false, client.WithMmap(true))
common.CheckErr(t, err1, false, "index type STL_SORT does not support mmap")
}

func TestAlterIndexMmapUnsupportedIndex(t *testing.T) {
// test mmap unsupported index: DiskANN, GPU-class， HYBRID， BITMAP
func TestMmapScalarAutoIndex(t *testing.T) {
ctx := createContext(t, time.Second*common.DefaultTimeout*2)
// connect
mc := createMilvusClient(ctx, t)

// create -> insert -> flush -> index -> load
cp := CollectionParams{CollectionFieldsType: Int64FloatVec, AutoID: false, EnableDynamicField: false,
cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: false,
ShardsNum: common.DefaultShards, Dim: common.DefaultDim}
collName := createCollection(ctx, t, mc, cp)

dp := DataParams{DoInsert: true, CollectionName: collName, CollectionFieldsType: Int64FloatVec, start: 0,
dp := DataParams{DoInsert: true, CollectionName: collName, CollectionFieldsType: AllFields, start: 0,
nb: common.DefaultNb, dim: common.DefaultDim, EnableDynamicField: false}
insertData(ctx, t, mc, dp)
mc.Flush(ctx, collName, false)

idxHnsw, _ := entity.NewIndexDISKANN(entity.IP)
err := mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idxHnsw, false)
// mmap not supported HYBRID index
err1 := mc.CreateIndex(ctx, collName, common.DefaultVarcharFieldName, entity.NewScalarIndex(), false, client.WithMmap(true))
common.CheckErr(t, err1, false, "index type HYBRID does not support mmap")

// mmap not supported HYBRID index
err1 = mc.CreateIndex(ctx, collName, common.DefaultBoolFieldName, entity.NewScalarIndexWithType(entity.Bitmap), false, client.WithMmap(true))
common.CheckErr(t, err1, false, "index type BITMAP does not support mmap")
}

func TestAlterIndexMmapUnsupportedIndex(t *testing.T) {
ctx := createContext(t, time.Second*common.DefaultTimeout*2)
// connect
mc := createMilvusClient(ctx, t)

// create -> insert -> flush -> index -> load
cp := CollectionParams{CollectionFieldsType: AllFields, AutoID: false, EnableDynamicField: false,
ShardsNum: common.DefaultShards, Dim: common.DefaultDim}
collName := createCollection(ctx, t, mc, cp)

// diskAnn
idxDiskAnn, _ := entity.NewIndexDISKANN(entity.IP)
err := mc.CreateIndex(ctx, collName, common.DefaultFloatVecFieldName, idxDiskAnn, false)
common.CheckErr(t, err, true)
for _, mmap := range []bool{true, false} {
err = mc.AlterIndex(ctx, collName, common.DefaultFloatVecFieldName, client.WithMmap(mmap))
common.CheckErr(t, err, false, "index type DISKANN does not support mmap: invalid parameter")
}
err = mc.AlterIndex(ctx, collName, common.DefaultFloatVecFieldName, client.WithMmap(true))
common.CheckErr(t, err, false, "index type DISKANN does not support mmap")

// bitmap index with mmap
err = mc.CreateIndex(ctx, collName, common.DefaultIntFieldName, entity.NewScalarIndexWithType(entity.Bitmap), false)
common.CheckErr(t, err, true)
errAlert := mc.AlterIndex(ctx, collName, common.DefaultIntFieldName, client.WithMmap(true))
common.CheckErr(t, errAlert, false, "index type BITMAP does not support mmap")

// HYBRID index
err = mc.CreateIndex(ctx, collName, common.DefaultInt32FieldName, entity.NewScalarIndex(), false)
common.CheckErr(t, err, true)
errAlert = mc.AlterIndex(ctx, collName, common.DefaultInt32FieldName, client.WithMmap(true))
common.CheckErr(t, errAlert, false, "index type HYBRID does not support mmap")

// Trie index
err = mc.CreateIndex(ctx, collName, common.DefaultVarcharFieldName, entity.NewScalarIndexWithType(entity.Trie), false)
common.CheckErr(t, err, true)
errAlert = mc.AlterIndex(ctx, collName, common.DefaultVarcharFieldName, client.WithMmap(true))
common.CheckErr(t, errAlert, false, "index type Trie does not support mmap")

// STL_SORT
err = mc.CreateIndex(ctx, collName, common.DefaultFloatFieldName, entity.NewScalarIndexWithType(entity.Sorted), false)
common.CheckErr(t, err, true)
errAlert = mc.AlterIndex(ctx, collName, common.DefaultFloatFieldName, client.WithMmap(true))
common.CheckErr(t, errAlert, false, "index type STL_SORT does not support mmap")
}

func TestMmapAlterIndex(t *testing.T) {
Expand Down
15 changes: 15 additions & 0 deletions test/testcases/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,21 @@ func GenDefaultIndexParamsForAllVectors() []IndexParams {
return ips
}

// inverted or scalar index not supported json
func SupportScalarIndexFieldType(field entity.FieldType) bool {
vectorFieldTypes := []entity.FieldType{
entity.FieldTypeBinaryVector, entity.FieldTypeFloatVector,
entity.FieldTypeFloat16Vector, entity.FieldTypeBFloat16Vector, entity.FieldTypeSparseVector,
entity.FieldTypeJSON,
}
for _, vectorFieldType := range vectorFieldTypes {
if field == vectorFieldType {
return false
}
}
return true
}

func TestMain(m *testing.M) {
flag.Parse()
log.Printf("parse addr=%s", *addr)
Expand Down

0 comments on commit bd99cd1

Please sign in to comment.