From b90ea1ef2e6b6c5d28535101d464f7c263f2bcd6 Mon Sep 17 00:00:00 2001
From: Ping Yu <yuping@pingcap.com>
Date: Sun, 21 Jan 2024 22:35:47 +0800
Subject: [PATCH] improve download

Signed-off-by: Ping Yu <yuping@pingcap.com>
---
 cdc/Makefile                                  |  6 ++--
 .../download-integration-test-binaries.sh     | 33 ++++++++++++++-----
 2 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/cdc/Makefile b/cdc/Makefile
index 9b34d07e..387531f4 100644
--- a/cdc/Makefile
+++ b/cdc/Makefile
@@ -248,8 +248,10 @@ integration_test_by_group: prepare_test_binaries check_third_party_binary integr
 	tests/integration_tests/run_group.sh others
 
 prepare_test_binaries:
-	cd scripts && ./download-integration-test-binaries.sh "$(TEST_ON_BRANCH)" && cd ..
-	touch prepare_test_binaries
+	cd scripts && \
+		./download-integration-test-binaries.sh "$(TEST_ON_BRANCH)" && \
+		cd .. && \
+		touch prepare_test_binaries
 
 check_third_party_binary:
 	@which scripts/bin/tidb-server
diff --git a/cdc/scripts/download-integration-test-binaries.sh b/cdc/scripts/download-integration-test-binaries.sh
index c237da3a..a11ff321 100755
--- a/cdc/scripts/download-integration-test-binaries.sh
+++ b/cdc/scripts/download-integration-test-binaries.sh
@@ -28,6 +28,19 @@ color-green() { # Green
 	echo -e "\x1B[1;32m${*}\x1B[0m"
 }
 
+function download() {
+	local url=$1
+	local file_name=$2
+	local file_path=$3
+	if [[ -f "${file_path}" ]]; then
+		echo "file ${file_name} already exists, skip download"
+		return
+	fi
+	echo ">>>"
+	echo "download ${file_name} from ${url}"
+	wget --no-verbose --retry-connrefused --waitretry=1 -t 3 -O "${file_path}" "${url}"
+}
+
 # Specify the download branch.
 branch=$1
 
@@ -56,18 +69,22 @@ mkdir -p tmp
 mkdir -p bin
 
 color-green "Download binaries..."
-curl "${tidb_download_url}" | tar xz -C tmp bin/tidb-server
-curl "${tikv_download_url}" | tar xz -C tmp bin/tikv-server
-curl "${pd_download_url}" | tar xz --wildcards -C tmp bin/*
-mv tmp/bin/* third_bin
-
-curl "${go_ycsb_download_url}" -o third_bin/go-ycsb
-curl -L "${etcd_download_url}" | tar xz -C tmp
-mv tmp/etcd-v3.4.7-linux-amd64/etcdctl third_bin
+
+download "$tidb_download_url" "tidb-server.tar.gz" "tmp/tidb-server.tar.gz"
+tar -xz -C third_bin bin/tidb-server -f tmp/tidb-server.tar.gz && mv third_bin/bin/tidb-server third_bin/
+download "$pd_download_url" "pd-server.tar.gz" "tmp/pd-server.tar.gz"
+tar -xz --wildcards -C third_bin 'bin/*' -f tmp/pd-server.tar.gz && mv third_bin/bin/* third_bin/
+download "$tikv_download_url" "tikv-server.tar.gz" "tmp/tikv-server.tar.gz"
+tar -xz -C third_bin bin/tikv-server -f tmp/tikv-server.tar.gz && mv third_bin/bin/tikv-server third_bin/
+download "$go_ycsb_download_url" "go-ycsb" "third_bin/go-ycsb"
+download "$etcd_download_url" "etcd.tar.gz" "tmp/etcd.tar.gz"
+tar -xz -C third_bin etcd-v3.4.7-linux-amd64/etcdctl -f tmp/etcd.tar.gz && mv third_bin/etcd-v3.4.7-linux-amd64/etcdctl third_bin/
+
 chmod a+x third_bin/*
 
 # Copy it to the bin directory in the root directory.
 rm -rf tmp
+rm -rf bin/bin
 mv third_bin/* ./bin
 rm -rf third_bin