Continue operation in case gravity-site or etcd is down (#2166)

* Continue operation in case gravity-site or etcd is down

tool/gravity/cli/clusterupdate.go

@@ -249,7 +249,9 @@ func executeUpdatePhaseForOperation(env *localenv.LocalEnvironment, environ Loca
 func executeOrForkPhase(env *localenv.LocalEnvironment, updater updater, params PhaseParams, operation ops.SiteOperation) error {
 	if params.isResume() {
 		if err := verifyOrDeployAgents(env); err != nil {
-			return trace.Wrap(err)
+			// Continue operation in case gravity-site or etcd is down. In these
+			// cases the agent status may not be retrievable.
+			log.WithError(err).Warn("Failed to verify or deploy agents.")
 		}
 	}
 

tool/gravity/cli/operation.go

@@ -178,7 +178,9 @@ func rollbackPlan(localEnv *localenv.LocalEnvironment, environ LocalEnvironmentF
 	}
 
 	if err := verifyOrDeployAgents(localEnv); err != nil {
-		return trace.Wrap(err)
+		// Continue operation in case gravity-site or etcd is down. In these
+		// cases the agent status may not be retrievable.
+		log.WithError(err).Warn("Failed to verify or deploy agents.")
 	}
 
 	if !confirmed && !params.DryRun {

tool/gravity/cli/rpcagent.go

@@ -43,6 +43,7 @@ import (
 	"github.com/gravitational/gravity/lib/utils"
 
 	"github.com/cenkalti/backoff"
+	"github.com/fatih/color"
 	teleclient "github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/trace"
 	"github.com/gravitational/version"
@@ -500,15 +501,16 @@ func collectAgentStatus(env *localenv.LocalEnvironment) (statusList rpc.StatusLi
 func verifyOrDeployAgents(env *localenv.LocalEnvironment) error {
 	statusList, err := collectAgentStatus(env)
 	if err != nil {
+		env.Println(color.YellowString("Couldn't verify upgrade agents status. If some are offline, they won't be redeployed automatically"))
 		return trace.Wrap(err, "failed to collect agent status")
 	}
 	if statusList.AgentsActive() {
 		return nil
 	}
 	if err := rpcAgentDeploy(env, deployOptions{}); err != nil {
-		log.WithError(err).Error("Failed to deploy agents.")
 		env.Println(statusList.String())
-		return trace.BadParameter("some agents are offline; ensure all agents are deployed with `./gravity agent deploy`")
+		env.Println(color.YellowString("Some agents are offline. Ensure all agents are deployed with `./gravity agent deploy`"))
+		return trace.Wrap(err, "failed to deploy agents")
 	}
 	return nil
 }