Add retry logic to join election phase. (#481)

lib/expand/phases/elect.go

@@ -18,13 +18,10 @@ package phases
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/gravitational/gravity/lib/constants"
-	"github.com/gravitational/gravity/lib/defaults"
 	"github.com/gravitational/gravity/lib/fsm"
 	"github.com/gravitational/gravity/lib/ops"
-	"github.com/gravitational/gravity/lib/utils"
 
 	"github.com/gravitational/trace"
 	"github.com/sirupsen/logrus"
@@ -56,21 +53,15 @@ type electExecutor struct {
 }
 
 // Execute executes the system phase
-func (p *electExecutor) Execute(ctx context.Context) error {
+func (p *electExecutor) Execute(ctx context.Context) (err error) {
 	p.Progress.NextStep("Enabling leader elections")
-	// TODO use etcd client?
-	cmd := "resume"
-	if !p.Phase.Data.Server.IsMaster() {
-		cmd = "pause"
+	if p.Phase.Data.Server.IsMaster() {
+		err = ops.EnableLeaderElection(ctx, p.Plan.ClusterName, *p.Phase.Data.Server, p.FieldLogger)
+	} else {
+		err = ops.PauseLeaderElection(ctx, p.Plan.ClusterName, *p.Phase.Data.Server, p.FieldLogger)
 	}
-	out, err := utils.RunPlanetCommand(ctx, p.FieldLogger, "leader", cmd,
-		fmt.Sprintf("--public-ip=%v", p.Phase.Data.Server.AdvertiseIP),
-		fmt.Sprintf("--election-key=/planet/cluster/%v/election", p.Plan.ClusterName),
-		fmt.Sprintf("--etcd-cafile=%v", defaults.Secret(defaults.RootCertFilename)),
-		fmt.Sprintf("--etcd-certfile=%v", defaults.Secret(defaults.EtcdCertFilename)),
-		fmt.Sprintf("--etcd-keyfile=%v", defaults.Secret(defaults.EtcdKeyFilename)))
 	if err != nil {
-		return trace.Wrap(err, "failed to enable leader election: %s", out)
+		return trace.Wrap(err)
 	}
 	p.Info("Reset leader election.")
 	return nil

lib/install/phases/election.go

@@ -18,17 +18,12 @@ package phases
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/gravitational/gravity/lib/constants"
-	"github.com/gravitational/gravity/lib/defaults"
 	"github.com/gravitational/gravity/lib/fsm"
 	"github.com/gravitational/gravity/lib/ops"
 	"github.com/gravitational/gravity/lib/schema"
-	"github.com/gravitational/gravity/lib/storage"
-	"github.com/gravitational/gravity/lib/utils"
 
-	"github.com/cenkalti/backoff"
 	"github.com/gravitational/trace"
 	"github.com/sirupsen/logrus"
 )
@@ -68,15 +63,10 @@ func (p *enableElectionExecutor) Execute(ctx context.Context) error {
 
 	for _, server := range p.Plan.Servers {
 		if server.ClusterRole == string(schema.ServiceRoleMaster) {
-			b := backoff.NewExponentialBackOff()
-			b.MaxElapsedTime = defaults.ElectionWaitTimeout
-			err := utils.RetryTransient(ctx, b, func() error {
-				return p.resumeLeader(ctx, server)
-			})
+			err := ops.EnableLeaderElection(ctx, p.Plan.ClusterName, server, p.FieldLogger)
 			if err != nil {
 				return trace.Wrap(err)
 			}
-
 		}
 	}
 
@@ -89,30 +79,11 @@ func (*enableElectionExecutor) Rollback(ctx context.Context) error {
 }
 
 // PreCheck is no-op for this phase
-func (p *enableElectionExecutor) PreCheck(ctx context.Context) error {
+func (*enableElectionExecutor) PreCheck(ctx context.Context) error {
 	return nil
 }
 
 // PostCheck is no-op for this phase
 func (*enableElectionExecutor) PostCheck(ctx context.Context) error {
 	return nil
 }
-
-func (p *enableElectionExecutor) resumeLeader(ctx context.Context, server storage.Server) error {
-	out, err := utils.RunPlanetCommand(
-		ctx,
-		p.FieldLogger,
-		"leader",
-		"resume",
-		fmt.Sprintf("--public-ip=%v", server.AdvertiseIP),
-		fmt.Sprintf("--election-key=/planet/cluster/%v/election", p.Plan.ClusterName),
-		"--etcd-cafile=/var/state/root.cert",
-		"--etcd-certfile=/var/state/etcd.cert",
-		"--etcd-keyfile=/var/state/etcd.key",
-	)
-	if err != nil {
-		return trace.Wrap(err, "failed to enable election for master %v. reason: %s",
-			server.AdvertiseIP, string(out))
-	}
-	return nil
-}

lib/ops/election.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2019 Gravitational, Inc.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ops
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/gravitational/gravity/lib/defaults"
+	"github.com/gravitational/gravity/lib/storage"
+	"github.com/gravitational/gravity/lib/utils"
+
+	"github.com/cenkalti/backoff"
+	"github.com/gravitational/trace"
+	"github.com/sirupsen/logrus"
+)
+
+// EnableLeaderElection turns on leader election for the specified node.
+func EnableLeaderElection(ctx context.Context, clusterName string, node storage.Server, log logrus.FieldLogger) error {
+	return runLeaderCommandRetry(ctx, "resume", clusterName, node, log)
+}
+
+// PauseLeaderElection pauses leader election for the specified node.
+func PauseLeaderElection(ctx context.Context, clusterName string, node storage.Server, log logrus.FieldLogger) error {
+	return runLeaderCommandRetry(ctx, "pause", clusterName, node, log)
+}
+
+func runLeaderCommandRetry(ctx context.Context, command string, clusterName string, node storage.Server, log logrus.FieldLogger) error {
+	b := backoff.NewExponentialBackOff()
+	b.MaxElapsedTime = defaults.ElectionWaitTimeout
+	return utils.RetryTransient(ctx, b, func() error {
+		return runLeaderCommand(ctx, command, clusterName, node, log)
+	})
+}
+
+func runLeaderCommand(ctx context.Context, command string, clusterName string, node storage.Server, log logrus.FieldLogger) error {
+	out, err := utils.RunPlanetCommand(ctx, log, "leader", command,
+		fmt.Sprintf("--public-ip=%v", node.AdvertiseIP),
+		fmt.Sprintf("--election-key=/planet/cluster/%v/election", clusterName),
+		"--etcd-cafile=/var/state/root.cert",
+		"--etcd-certfile=/var/state/etcd.cert",
+		"--etcd-keyfile=/var/state/etcd.key")
+	if err != nil {
+		return trace.Wrap(err, "failed to enable election for %v: %s",
+			node.AdvertiseIP, string(out))
+	}
+	return nil
+}