From 4e0b70e534a429b5b2e342a151412d26308a9a5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= <ege.gunes@percona.com>
Date: Fri, 8 Nov 2024 22:50:37 +0300
Subject: [PATCH] Wait for resync to finish after physical restore in tests
 (#1709)

---
 e2e-tests/demand-backup-physical-sharded/run |  5 +++--
 e2e-tests/demand-backup-physical/run         |  5 +----
 e2e-tests/functions                          | 19 +++++++++++++++++++
 3 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/e2e-tests/demand-backup-physical-sharded/run b/e2e-tests/demand-backup-physical-sharded/run
index c7c8dcabe5..1ae02ff797 100755
--- a/e2e-tests/demand-backup-physical-sharded/run
+++ b/e2e-tests/demand-backup-physical-sharded/run
@@ -42,10 +42,11 @@ run_recovery_check() {
 		exit 1
 	fi
 	echo
+
 	wait_cluster_consistency ${cluster} 42
+	wait_for_pbm_operations ${cluster}
+
 	compare_mongos_cmd "find" "myApp:myPass@${cluster}-mongos.${namespace}" "-sharded"
-	echo
-	set -o xtrace
 }
 
 check_exported_mongos_service_endpoint() {
diff --git a/e2e-tests/demand-backup-physical/run b/e2e-tests/demand-backup-physical/run
index 853468d29c..2c1a1cbedc 100755
--- a/e2e-tests/demand-backup-physical/run
+++ b/e2e-tests/demand-backup-physical/run
@@ -37,7 +37,6 @@ run_recovery_check() {
 	# we don't wait for cluster readiness here because the annotation gets removed then
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1800"
 
-	kubectl_bin get psmdb ${cluster} -o yaml
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"
 		exit 1
@@ -45,13 +44,11 @@ run_recovery_check() {
 	echo
 
 	wait_cluster_consistency ${cluster}
+	wait_for_pbm_operations ${cluster}
 
 	compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-0.${cluster}-rs0.${namespace}"
 	compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-1.${cluster}-rs0.${namespace}"
 	compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-2.${cluster}-rs0.${namespace}"
-
-	echo
-	set -o xtrace
 }
 
 create_infra "${namespace}"
diff --git a/e2e-tests/functions b/e2e-tests/functions
index db1e9d4c6d..d8dde927b5 100755
--- a/e2e-tests/functions
+++ b/e2e-tests/functions
@@ -239,6 +239,25 @@ wait_backup() {
 	set_debug
 }
 
+wait_for_pbm_operations() {
+	local cluster=$1
+
+	set +o xtrace
+	echo -n "waiting for PBM operation to finish"
+	retry=0
+	until [[ $(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json -s running | jq -r .running.opID) == null ]]; do
+		if [ $retry -ge 540 ]; then
+			echo max retry count $retry reached. something went wrong with PBM operations
+			exit 1
+		fi
+		echo -n .
+		sleep 5
+	done
+	echo
+	set_debug
+}
+
+
 run_restore() {
 	local backup_name=$1