[close #175] Add br apiv2 integration test

.github/config/br_rawkv.toml

@@ -7,12 +7,11 @@ pd-heartbeat-tick-interval = "2s"
 pd-store-heartbeat-tick-interval = "5s"
 split-region-check-tick-interval = "1s"
 
-[storage]
-enable-ttl = true
-
 [rocksdb]
 max-open-files = 10000
 
 [raftdb]
 max-open-files = 10000
 
+[storage]
+enable-ttl = true

.github/workflows/ci-br.yml

@@ -60,11 +60,12 @@ jobs:
           cd br
           make test
   br-integration-test:
-    name: br-integration-test-${{ matrix.tikv_version }}
+    name: br-integration-test-${{ matrix.tikv_version }}-apiv${{ matrix.api_version }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
         tikv_version: [nightly]
+        api_version: [1, 2]
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
@@ -74,18 +75,24 @@ jobs:
         run: curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
       - name: start tikv cluster
         run: |
-          # start tikv in apiv1ttl
+          # start tikv
+          echo -e "\napi-version = ${{ matrix.api_version }}\n" >> /home/runner/work/migration/migration/.github/config/br_rawkv.toml
           /home/runner/.tiup/bin/tiup playground ${{ matrix.tikv_version }} --mode tikv-slim --kv 1 --without-monitor --kv.config /home/runner/work/migration/migration/.github/config/br_rawkv.toml --pd.config /home/runner/work/migration/migration/.github/config/br_pd.toml &> raw.out 2>&1 &
           # The first run of `tiup` has to download all components so it'll take longer.
           sleep 1m 30s
           # Parse PD address from `tiup` output
           echo "PD_ADDR=$(cat raw.out | grep -oP '(?<=PD client endpoints: \[)[0-9\.:]+(?=\])')" >> $GITHUB_ENV
+          # Output the api version
+          echo "API_VERSION=${{ matrix.api_version }}" >> $GITHUB_ENV
           # Log the output
           echo "$(cat raw.out)" >&2
       - name: run integration test
         run: |
           cd br
           make test/integration
+      - name: display backup log
+        run: |
+          cat /tmp/br.log*
 # comments low version CI as they fail without checksum & split region capability.
 # Plan to support low version with feature gate in the future.
 #  br-integration-test-5X:

br/Makefile

@@ -28,6 +28,7 @@ COVERAGE_DIR   ?= build
 TEST_PARALLEL  ?= 8
 PD_ADDR        ?= 127.0.0.1:2379
 BR_LOCAL_STORE ?= /tmp/backup_restore_test
+API_VERSION    ?= 1
 
 LDFLAGS += -X "github.com/tikv/migration/br/pkg/version/build.ReleaseVersion=$(shell git describe --tags --dirty --always)"
 LDFLAGS += -X "github.com/tikv/migration/br/pkg/version/build.BuildTS=$(shell date -u '+%Y-%m-%d %H:%M:%S')"
@@ -46,7 +47,7 @@ check/golangci-lint: tools/bin/golangci-lint
 	GO111MODULE=on CGO_ENABLED=0 tools/bin/golangci-lint run -v $$($(DIRECTORIES)) --config ../.golangci.yml --timeout 5m
 
 check/gosec:
-	$(GO) install github.com/securego/gosec/v2/cmd/gosec@v2.11.0
+	$(GO) install github.com/securego/gosec/v2/cmd/gosec@v2.12.0
 	gosec -fmt=junit-xml -out=results.xml -stdout -verbose=text -exclude=G103,G104,G204,G304,G307,G401,G404,G501,G505,G601 ./...
 
 test: tools/bin/gocov tools/bin/gocov-xml
@@ -57,10 +58,11 @@ test: tools/bin/gocov tools/bin/gocov-xml
 	tools/bin/gocov convert $(COVERAGE_DIR)/coverage.raw | tools/bin/gocov-xml > $(COVERAGE_DIR)/coverage.xml
 	make failpoint/disable
 
-test/integration: build/br-test build/rawkv-helper
-	./tests/rawkv/run.py --test-helper=bin/rawkv --pd=$(PD_ADDR) \
-		--br='$(TEST_BIN_PATH) -test.coverprofile=cov.br_rawkv.out.log DEVEL' \
-		--br-storage=local://$(BR_LOCAL_STORE)
+test/integration: build/br-test build/rawkv-integration-test
+	./bin/rawkv_test --pd=${PD_ADDR} \
+		--br='${TEST_BIN_PATH}' \
+		--br-storage=${BR_LOCAL_STORE} \
+		--api-version=${API_VERSION}
 
 failpoint/enable: tools/bin/failpoint-ctl
 	find `pwd` -type d | grep -vE "(\.git|tools)" | xargs tools/bin/failpoint-ctl enable
@@ -86,8 +88,8 @@ build:
 release:
 	CGO_ENABLED=1 $(GO) build -tags codes -ldflags '$(LDFLAGS)' -o $(BR_BIN_PATH) cmd/br/*.go
 
-build/rawkv-helper:
-	cd tests/rawkv && $(GO) build -mod=mod -o ../../bin/rawkv client.go
+build/rawkv-integration-test:
+	cd tests/rawkv && $(GO) build -mod=mod -o ../../bin/rawkv_test *.go
 
 build/br-test:
 	@make failpoint/enable

br/go.mod

@@ -16,21 +16,21 @@ require (
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/mock v1.6.0
-	github.com/google/btree v1.0.1
+	github.com/google/btree v1.1.2
 	github.com/google/uuid v1.1.2
 	github.com/jarcoal/httpmock v1.1.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
 	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00
-	github.com/pingcap/kvproto v0.0.0-20220517085838-12e2f5a9d167
+	github.com/pingcap/kvproto v0.0.0-20220705053936-aa9c2d20cd2a
 	github.com/pingcap/log v0.0.0-20211215031037-e024ba4eb0ee
 	github.com/pingcap/tidb v1.1.0-beta.0.20220222031143-5988d0b2f46e
 	github.com/prometheus/client_golang v1.11.0
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
-	github.com/tikv/client-go/v2 v2.0.1-0.20220518162527-de7ca289ac77
+	github.com/tikv/client-go/v2 v2.0.1-0.20220721031657-e38d2b07de3f
 	github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710
 	go.uber.org/goleak v1.1.12
 	go.uber.org/multierr v1.7.0
@@ -81,6 +81,7 @@ require (
 	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/stathat/consistent v1.0.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.2 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect

br/go.sum

@@ -290,8 +290,9 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
+github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -532,8 +533,8 @@ github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO
 github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/kvproto v0.0.0-20220215045702-d229fcc888c8/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
 github.com/pingcap/kvproto v0.0.0-20220302110454-c696585a961b/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20220517085838-12e2f5a9d167 h1:dsMpneacHyuVslSVndgUfJKrXFNG7VPdXip2ulG6glo=
-github.com/pingcap/kvproto v0.0.0-20220517085838-12e2f5a9d167/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20220705053936-aa9c2d20cd2a h1:nP2wmyw9JTRsk5rm+tZtfAso6c/1FvuaFNbXTaYz3FE=
+github.com/pingcap/kvproto v0.0.0-20220705053936-aa9c2d20cd2a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
@@ -624,6 +625,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
+github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -636,8 +639,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU=
 github.com/tikv/client-go/v2 v2.0.0-rc.0.20220217140116-589cf79b2ab5/go.mod h1:gaHSp8rnxZ0w36qb6QPPNPh9P0Mu5vAEwCQcc0Brni4=
-github.com/tikv/client-go/v2 v2.0.1-0.20220518162527-de7ca289ac77 h1:baRGhZtEp4wY3qh/0bscTCZ+97MVTZekBg5rOdgopQc=
-github.com/tikv/client-go/v2 v2.0.1-0.20220518162527-de7ca289ac77/go.mod h1:VTlli8fRRpcpISj9I2IqroQmcAFfaTyBquiRhofOcDs=
+github.com/tikv/client-go/v2 v2.0.1-0.20220721031657-e38d2b07de3f h1:fCAWVzUJYee1pDqYTBt5j2G1WuUfOL7A2mZoONoaHgs=
+github.com/tikv/client-go/v2 v2.0.1-0.20220721031657-e38d2b07de3f/go.mod h1:UmDQEoeHXza8RSHBXVFERpxH54VBOf8yKZVphyel3l4=
 github.com/tikv/pd/client v0.0.0-20220216070739-26c668271201/go.mod h1:fEvI5fhAuJn1Fn87VJF8ByE9Vc16EzWGoePZB21/nL8=
 github.com/tikv/pd/client v0.0.0-20220216080339-1b8f82378ee7/go.mod h1:fEvI5fhAuJn1Fn87VJF8ByE9Vc16EzWGoePZB21/nL8=
 github.com/tikv/pd/client v0.0.0-20220307081149-841fa61e9710 h1:jxgmKOscXSjaFEKQGRyY5qOpK8hLqxs2irb/uDJMtwk=
@@ -1231,3 +1234,4 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
 sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k=
+stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=

br/pkg/rtree/rtree.go

@@ -79,12 +79,14 @@ var _ btree.Item = &Range{}
 // All the ranges it stored do not overlap.
 type RangeTree struct {
 	*btree.BTree
+	hasValidRange bool
 }
 
 // NewRangeTree returns an empty range tree.
 func NewRangeTree() RangeTree {
 	return RangeTree{
-		BTree: btree.New(32),
+		BTree:         btree.New(32),
+		hasValidRange: false,
 	}
 }
 
@@ -140,6 +142,7 @@ func (rangeTree *RangeTree) Update(rg Range) {
 		rangeTree.Delete(item)
 	}
 	rangeTree.ReplaceOrInsert(&rg)
+	rangeTree.hasValidRange = true
 }
 
 // Put forms a range and inserts it into tree.
@@ -215,5 +218,10 @@ func (rangeTree *RangeTree) GetIncompleteRange(
 				append(incomplete, Range{StartKey: start, EndKey: end})
 		}
 	}
+	// is no valid range in Btree, and startKey/endKey are both empty, incomplete is empty too.
+	// This is not the result we need, append the whole input range.
+	if !rangeTree.hasValidRange && len(incomplete) == 0 {
+		incomplete = append(incomplete, Range{StartKey: startKey, EndKey: endKey})
+	}
 	return incomplete
 }

br/pkg/rtree/rtree_test.go

@@ -17,6 +17,14 @@ func newRange(start, end []byte) *rtree.Range {
 	}
 }
 
+func TestEmptyRange(t *testing.T) {
+	rangeTree := rtree.NewRangeTree()
+	inCompelete := rangeTree.GetIncompleteRange([]byte(""), []byte(""))
+	require.Equal(t, len(inCompelete), 1)
+	require.Equal(t, inCompelete[0].StartKey, []byte(""))
+	require.Equal(t, inCompelete[0].EndKey, []byte(""))
+}
+
 func TestRangeTree(t *testing.T) {
 	rangeTree := rtree.NewRangeTree()
 	require.Nil(t, rangeTree.Get(newRange([]byte(""), []byte(""))))