Support multi-control plane - HA clusters

.gitignore

@@ -25,6 +25,7 @@ _testmain.go
 *.exe
 *.test
 *.prof
+*.pprof
 
 /deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/usr/bin/auto-pause
 /deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/usr/bin/auto-pause

cmd/minikube/cmd/config/profile_list.go

@@ -83,7 +83,7 @@ func printProfilesTable() {
 	}
 
 	if len(validProfiles) == 0 {
-		exit.Message(reason.UsageNoProfileRunning, "No minikube profile was found. ")
+		exit.Message(reason.UsageNoProfileRunning, "No minikube profile was found.")
 	}
 
 	updateProfilesStatus(validProfiles)
@@ -111,45 +111,81 @@ func updateProfilesStatus(profiles []*config.Profile) {
 }
 
 func profileStatus(p *config.Profile, api libmachine.API) string {
-	cp, err := config.PrimaryControlPlane(p.Config)
-	if err != nil {
-		exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
+	cps := config.ControlPlanes(*p.Config)
+	if len(cps) == 0 {
+		exit.Message(reason.GuestCpConfig, "No control-plane nodes found.")
 	}
 
-	host, err := machine.LoadHost(api, config.MachineName(*p.Config, cp))
-	if err != nil {
-		klog.Warningf("error loading profiles: %v", err)
-		return "Unknown"
-	}
+	status := "Unknown"
+	healthyCPs := 0
+	for _, cp := range cps {
+		machineName := config.MachineName(*p.Config, cp)
 
-	// The machine isn't running, no need to check inside
-	s, err := host.Driver.GetState()
-	if err != nil {
-		klog.Warningf("error getting host state: %v", err)
-		return "Unknown"
-	}
-	if s != state.Running {
-		return s.String()
-	}
+		ms, err := machine.Status(api, machineName)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): machine status for %s: %v", machineName, err)
+			continue
+		}
+		if ms != state.Running.String() {
+			klog.Warningf("error loading profile (will continue): machine %s is not running: %q", machineName, ms)
+			status = ms
+			continue
+		}
 
-	cr, err := machine.CommandRunner(host)
-	if err != nil {
-		klog.Warningf("error loading profiles: %v", err)
-		return "Unknown"
-	}
+		host, err := machine.LoadHost(api, machineName)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): load host for %s: %v", machineName, err)
+			continue
+		}
 
-	hostname, _, port, err := driver.ControlPlaneEndpoint(p.Config, &cp, host.DriverName)
-	if err != nil {
-		klog.Warningf("error loading profiles: %v", err)
-		return "Unknown"
+		hs, err := host.Driver.GetState()
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): host state for %s: %v", machineName, err)
+			continue
+		}
+		if hs != state.Running {
+			klog.Warningf("error loading profile (will continue): host %s is not running: %q", machineName, hs)
+			status = hs.String()
+			continue
+		}
+
+		cr, err := machine.CommandRunner(host)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): command runner for %s: %v", machineName, err)
+			continue
+		}
+
+		hostname, _, port, err := driver.ControlPlaneEndpoint(p.Config, &cp, host.DriverName)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): control-plane endpoint for %s: %v", machineName, err)
+			continue
+		}
+
+		as, err := kverify.APIServerStatus(cr, hostname, port)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): apiserver status for %s: %v", machineName, err)
+			continue
+		}
+		status = as.String()
+		if as != state.Running {
+			klog.Warningf("error loading profile (will continue): apiserver %s is not running: %q", machineName, hs)
+			continue
+		}
+
+		healthyCPs++
 	}
 
-	status, err := kverify.APIServerStatus(cr, hostname, port)
-	if err != nil {
-		klog.Warningf("error getting apiserver status for %s: %v", p.Name, err)
-		return "Unknown"
+	if config.IsHA(*p.Config) {
+		switch {
+		case healthyCPs < 2:
+			return state.Stopped.String()
+		case healthyCPs == 2:
+			return "Degraded"
+		default:
+			return "HAppy"
+		}
 	}
-	return status.String()
+	return status
 }
 
 func renderProfilesTable(ps [][]string) {
@@ -166,9 +202,15 @@ func profilesToTableData(profiles []*config.Profile) [][]string {
 	var data [][]string
 	currentProfile := ClusterFlagValue()
 	for _, p := range profiles {
-		cp, err := config.PrimaryControlPlane(p.Config)
-		if err != nil {
-			exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
+		cpIP := p.Config.KubernetesConfig.APIServerHAVIP
+		cpPort := p.Config.APIServerPort
+		if !config.IsHA(*p.Config) {
+			cp, err := config.ControlPlane(*p.Config)
+			if err != nil {
+				exit.Error(reason.GuestCpConfig, "error getting control-plane node", err)
+			}
+			cpIP = cp.IP
+			cpPort = cp.Port
 		}
 
 		k8sVersion := p.Config.KubernetesConfig.KubernetesVersion
@@ -179,7 +221,7 @@ func profilesToTableData(profiles []*config.Profile) [][]string {
 		if p.Name == currentProfile {
 			c = "*"
 		}
-		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), k8sVersion, p.Status, strconv.Itoa(len(p.Config.Nodes)), c})
+		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cpIP, strconv.Itoa(cpPort), k8sVersion, p.Status, strconv.Itoa(len(p.Config.Nodes)), c})
 	}
 	return data
 }

cmd/minikube/cmd/cp.go

@@ -71,7 +71,7 @@ Example Command : "minikube cp a.txt /home/docker/b.txt" +
 			runner = remoteCommandRunner(&co, dst.node)
 		} else if src.node == "" {
 			// if node name not explicitly specified in both of source and target,
-			// consider target is controlpanel node for backward compatibility.
+			// consider target is control-plane node for backward compatibility.
 			runner = co.CP.Runner
 		} else {
 			runner = command.NewExecRunner(false)
@@ -84,9 +84,6 @@ Example Command : "minikube cp a.txt /home/docker/b.txt" +
 	},
 }
 
-func init() {
-}
-
 // setDstFileNameFromSrc sets the src filename as dst filename
 // when the dst file name is not provided and ends with a `/`.
 // Otherwise this function is a no-op and returns the passed dst.
@@ -211,7 +208,7 @@ func validateArgs(src, dst *remotePath) {
 	}
 
 	// if node name not explicitly specified in both of source and target,
-	// consider target node is controlpanel for backward compatibility.
+	// consider target node is control-plane for backward compatibility.
 	if src.node == "" && dst.node == "" && !strings.HasPrefix(dst.path, "/") {
 		exit.Message(reason.Usage, `Target <remote file path> must be an absolute Path. Relative Path is not allowed (example: "minikube:/home/docker/copied.txt")`)
 	}

cmd/minikube/cmd/docker-env.go

@@ -228,8 +228,7 @@ func mustRestartDockerd(name string, runner command.Runner) {
 		// will need to wait for apisever container to come up, this usually takes 5 seconds
 		// verifying apisever using kverify would add code complexity for a rare case.
 		klog.Warningf("waiting to ensure apisever container is up...")
-		startTime := time.Now()
-		if err = waitForAPIServerProcess(runner, startTime, time.Second*30); err != nil {
+		if err = waitForAPIServerProcess(runner, time.Now(), time.Second*30); err != nil {
 			klog.Warningf("apiserver container isn't up, error: %v", err)
 		}
 	}

cmd/minikube/cmd/logs.go

@@ -138,7 +138,7 @@ func shouldSilentFail() bool {
 
 	api, cc := mustload.Partial(ClusterFlagValue())
 
-	cp, err := config.PrimaryControlPlane(cc)
+	cp, err := config.ControlPlane(*cc)
 	if err != nil {
 		return false
 	}

cmd/minikube/cmd/node_add.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
 	"k8s.io/minikube/pkg/minikube/cni"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/driver"
@@ -32,8 +33,9 @@ import (
 )
 
 var (
-	cp     bool
-	worker bool
+	cpNode              bool
+	workerNode          bool
+	deleteNodeOnFailure bool
 )
 
 var nodeAddCmd = &cobra.Command{
@@ -48,20 +50,31 @@ var nodeAddCmd = &cobra.Command{
 			out.FailureT("none driver does not support multi-node clusters")
 		}
 
-		name := node.Name(len(cc.Nodes) + 1)
+		if cpNode && !config.IsHA(*cc) {
+			out.FailureT("Adding a control-plane node to a non-HA (non-multi-control plane) cluster is not currently supported. Please first delete the cluster and use 'minikube start --ha' to create new one.")
+		}
+
+		roles := []string{}
+		if workerNode {
+			roles = append(roles, "worker")
+		}
+		if cpNode {
+			roles = append(roles, "control-plane")
+		}
 
-		// for now control-plane feature is not supported
-		if cp {
-			out.Step(style.Unsupported, "Adding a control-plane node is not yet supported, setting control-plane flag to false")
-			cp = false
+		// calculate appropriate new node name with id following the last existing one
+		lastID, err := node.ID(cc.Nodes[len(cc.Nodes)-1].Name)
+		if err != nil {
+			lastID = len(cc.Nodes)
+			out.ErrLn("determining last node index (will assume %d): %v", lastID, err)
 		}
+		name := node.Name(lastID + 1)
 
-		out.Step(style.Happy, "Adding node {{.name}} to cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
-		// TODO: Deal with parameters better. Ideally we should be able to acceot any node-specific minikube start params here.
+		out.Step(style.Happy, "Adding node {{.name}} to cluster {{.cluster}} as {{.roles}}", out.V{"name": name, "cluster": cc.Name, "roles": roles})
 		n := config.Node{
 			Name:              name,
-			Worker:            worker,
-			ControlPlane:      cp,
+			Worker:            workerNode,
+			ControlPlane:      cpNode,
 			KubernetesVersion: cc.KubernetesConfig.KubernetesVersion,
 		}
 
@@ -77,7 +90,7 @@ var nodeAddCmd = &cobra.Command{
 		}
 
 		register.Reg.SetStep(register.InitialSetup)
-		if err := node.Add(cc, n, false); err != nil {
+		if err := node.Add(cc, n, deleteNodeOnFailure); err != nil {
 			_, err := maybeDeleteAndRetry(cmd, *cc, n, nil, err)
 			if err != nil {
 				exit.Error(reason.GuestNodeAdd, "failed to add node", err)
@@ -93,10 +106,9 @@ var nodeAddCmd = &cobra.Command{
 }
 
 func init() {
-	// TODO(https://github.com/kubernetes/minikube/issues/7366): We should figure out which minikube start flags to actually import
-	nodeAddCmd.Flags().BoolVar(&cp, "control-plane", false, "This flag is currently unsupported.")
-	nodeAddCmd.Flags().BoolVar(&worker, "worker", true, "If true, the added node will be marked for work. Defaults to true.")
-	nodeAddCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
+	nodeAddCmd.Flags().BoolVar(&cpNode, "control-plane", false, "If set, added node will become a control-plane. Defaults to false. Currently only supported for existing HA (multi-control plane) clusters.")
+	nodeAddCmd.Flags().BoolVar(&workerNode, "worker", true, "If set, added node will be available as worker. Defaults to true.")
+	nodeAddCmd.Flags().BoolVar(&deleteNodeOnFailure, "delete-on-failure", false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
 
 	nodeCmd.AddCommand(nodeAddCmd)
 }

cmd/minikube/cmd/node_start.go

@@ -56,7 +56,7 @@ var nodeStartCmd = &cobra.Command{
 		}
 
 		register.Reg.SetStep(register.InitialSetup)
-		r, p, m, h, err := node.Provision(cc, n, n.ControlPlane, viper.GetBool(deleteOnFailure))
+		r, p, m, h, err := node.Provision(cc, n, viper.GetBool(deleteOnFailure))
 		if err != nil {
 			exit.Error(reason.GuestNodeProvision, "provisioning host for node", err)
 		}
@@ -71,10 +71,8 @@ var nodeStartCmd = &cobra.Command{
 			ExistingAddons: cc.Addons,
 		}
 
-		_, err = node.Start(s, n.ControlPlane)
-		if err != nil {
-			_, err := maybeDeleteAndRetry(cmd, *cc, *n, nil, err)
-			if err != nil {
+		if _, err = node.Start(s); err != nil {
+			if _, err := maybeDeleteAndRetry(cmd, *cc, *n, nil, err); err != nil {
 				node.ExitIfFatal(err, false)
 				exit.Error(reason.GuestNodeStart, "failed to start node", err)
 			}