From 763beed7d259b16909f0d1e9bfdf5ea8a77d765c Mon Sep 17 00:00:00 2001 From: Bernard Kim Date: Tue, 29 Sep 2020 22:54:21 -0700 Subject: [PATCH] Continue operation in case gravity-site or etcd is down (#2166) * Continue operation in case gravity-site or etcd is down --- tool/gravity/cli/clusterupdate.go | 4 +++- tool/gravity/cli/operation.go | 4 +++- tool/gravity/cli/rpcagent.go | 6 ++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/tool/gravity/cli/clusterupdate.go b/tool/gravity/cli/clusterupdate.go index 2e2c6f0bea..2ce0b479e1 100644 --- a/tool/gravity/cli/clusterupdate.go +++ b/tool/gravity/cli/clusterupdate.go @@ -249,7 +249,9 @@ func executeUpdatePhaseForOperation(env *localenv.LocalEnvironment, environ Loca func executeOrForkPhase(env *localenv.LocalEnvironment, updater updater, params PhaseParams, operation ops.SiteOperation) error { if params.isResume() { if err := verifyOrDeployAgents(env); err != nil { - return trace.Wrap(err) + // Continue operation in case gravity-site or etcd is down. In these + // cases the agent status may not be retrievable. + log.WithError(err).Warn("Failed to verify or deploy agents.") } } diff --git a/tool/gravity/cli/operation.go b/tool/gravity/cli/operation.go index 78be5c94ec..c7b4c6a245 100644 --- a/tool/gravity/cli/operation.go +++ b/tool/gravity/cli/operation.go @@ -178,7 +178,9 @@ func rollbackPlan(localEnv *localenv.LocalEnvironment, environ LocalEnvironmentF } if err := verifyOrDeployAgents(localEnv); err != nil { - return trace.Wrap(err) + // Continue operation in case gravity-site or etcd is down. In these + // cases the agent status may not be retrievable. + log.WithError(err).Warn("Failed to verify or deploy agents.") } if !confirmed && !params.DryRun { diff --git a/tool/gravity/cli/rpcagent.go b/tool/gravity/cli/rpcagent.go index 61e662a235..ae0667502f 100644 --- a/tool/gravity/cli/rpcagent.go +++ b/tool/gravity/cli/rpcagent.go @@ -43,6 +43,7 @@ import ( "github.com/gravitational/gravity/lib/utils" "github.com/cenkalti/backoff" + "github.com/fatih/color" teleclient "github.com/gravitational/teleport/lib/client" "github.com/gravitational/trace" "github.com/gravitational/version" @@ -500,15 +501,16 @@ func collectAgentStatus(env *localenv.LocalEnvironment) (statusList rpc.StatusLi func verifyOrDeployAgents(env *localenv.LocalEnvironment) error { statusList, err := collectAgentStatus(env) if err != nil { + env.Println(color.YellowString("Couldn't verify upgrade agents status. If some are offline, they won't be redeployed automatically")) return trace.Wrap(err, "failed to collect agent status") } if statusList.AgentsActive() { return nil } if err := rpcAgentDeploy(env, deployOptions{}); err != nil { - log.WithError(err).Error("Failed to deploy agents.") env.Println(statusList.String()) - return trace.BadParameter("some agents are offline; ensure all agents are deployed with `./gravity agent deploy`") + env.Println(color.YellowString("Some agents are offline. Ensure all agents are deployed with `./gravity agent deploy`")) + return trace.Wrap(err, "failed to deploy agents") } return nil }