Wait for resync to finish after physical restore in tests (#1709)

e2e-tests/demand-backup-physical-sharded/run

@@ -42,10 +42,11 @@ run_recovery_check() {
 		exit 1
 	fi
 	echo
+
 	wait_cluster_consistency ${cluster} 42
+	wait_for_pbm_operations ${cluster}
+
 	compare_mongos_cmd "find" "myApp:myPass@${cluster}-mongos.${namespace}" "-sharded"
-	echo
-	set -o xtrace
 }
 
 check_exported_mongos_service_endpoint() {

e2e-tests/demand-backup-physical/run

@@ -37,21 +37,18 @@ run_recovery_check() {
 	# we don't wait for cluster readiness here because the annotation gets removed then
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1800"
 
-	kubectl_bin get psmdb ${cluster} -o yaml
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"
 		exit 1
 	fi
 	echo
 
 	wait_cluster_consistency ${cluster}
+	wait_for_pbm_operations ${cluster}
 
 	compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-0.${cluster}-rs0.${namespace}"
 	compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-1.${cluster}-rs0.${namespace}"
 	compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-2.${cluster}-rs0.${namespace}"
-
-	echo
-	set -o xtrace
 }
 
 create_infra "${namespace}"

e2e-tests/functions

@@ -239,6 +239,25 @@ wait_backup() {
 	set_debug
 }
 
+wait_for_pbm_operations() {
+	local cluster=$1
+
+	set +o xtrace
+	echo -n "waiting for PBM operation to finish"
+	retry=0
+	until [[ $(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json -s running | jq -r .running.opID) == null ]]; do
+		if [ $retry -ge 540 ]; then
+			echo max retry count $retry reached. something went wrong with PBM operations
+			exit 1
+		fi
+		echo -n .
+		sleep 5
+	done
+	echo
+	set_debug
+}
+
+
 run_restore() {
 	local backup_name=$1