Merge branch 'kubernetes:master' into qemu-root

.github/workflows/functional_verified.yml

@@ -118,7 +118,7 @@ jobs:
           hostname || true
           echo "--------------------------"
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -202,7 +202,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: download all extra reports
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
       - name: upload all extra reports
         shell: bash {0}
         continue-on-error: true

.github/workflows/master.yml

@@ -140,7 +140,7 @@ jobs:
         run: |
           go install github.com/medyagh/gopogh/cmd/[email protected]
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -238,7 +238,7 @@ jobs:
         run: |
           go install github.com/medyagh/gopogh/cmd/[email protected]
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -340,7 +340,7 @@ jobs:
         run: |
           go install github.com/medyagh/gopogh/cmd/[email protected]
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -440,7 +440,7 @@ jobs:
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -545,7 +545,7 @@ jobs:
         run: |
           sudo sysctl fs.protected_regular=0
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -619,7 +619,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: download all reports
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
       - name: upload all reports
         shell: bash {0}
         continue-on-error: true

.github/workflows/pr.yml

@@ -138,7 +138,7 @@ jobs:
         run: |
           go install github.com/medyagh/gopogh/cmd/[email protected]
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -237,7 +237,7 @@ jobs:
         run: |
           go install github.com/medyagh/gopogh/cmd/[email protected]
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -354,7 +354,7 @@ jobs:
         run: |
           go install github.com/medyagh/gopogh/cmd/[email protected]
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -457,7 +457,7 @@ jobs:
         run: |
           go install github.com/medyagh/gopogh/cmd/[email protected]
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -558,7 +558,7 @@ jobs:
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
           sudo /usr/libexec/ApplicationFirewall/socketfilterfw -k
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -664,7 +664,7 @@ jobs:
         run: |
           sudo sysctl fs.protected_regular=0
       - name: Download Binaries
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
         with:
           name: minikube_binaries
           path: minikube_binaries
@@ -740,7 +740,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: download all reports
-        uses: actions/download-artifact@87c55149d96e628cc2ef7e6fc2aab372015aec85
+        uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427
       - name: upload all reports
         shell: bash {0}
         continue-on-error: true

.gitignore

@@ -25,6 +25,7 @@ _testmain.go
 *.exe
 *.test
 *.prof
+*.pprof
 
 /deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/usr/bin/auto-pause
 /deploy/iso/minikube-iso/board/minikube/aarch64/rootfs-overlay/usr/bin/auto-pause

cmd/minikube/cmd/config/kubernetes_version.go

@@ -20,7 +20,7 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/google/go-github/v59/github"
+	"github.com/google/go-github/v60/github"
 	"golang.org/x/mod/semver"
 	"k8s.io/minikube/pkg/minikube/constants"
 )

cmd/minikube/cmd/config/profile_list.go

@@ -83,7 +83,7 @@ func printProfilesTable() {
 	}
 
 	if len(validProfiles) == 0 {
-		exit.Message(reason.UsageNoProfileRunning, "No minikube profile was found. ")
+		exit.Message(reason.UsageNoProfileRunning, "No minikube profile was found.")
 	}
 
 	updateProfilesStatus(validProfiles)
@@ -111,45 +111,81 @@ func updateProfilesStatus(profiles []*config.Profile) {
 }
 
 func profileStatus(p *config.Profile, api libmachine.API) string {
-	cp, err := config.PrimaryControlPlane(p.Config)
-	if err != nil {
-		exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
+	cps := config.ControlPlanes(*p.Config)
+	if len(cps) == 0 {
+		exit.Message(reason.GuestCpConfig, "No control-plane nodes found.")
 	}
 
-	host, err := machine.LoadHost(api, config.MachineName(*p.Config, cp))
-	if err != nil {
-		klog.Warningf("error loading profiles: %v", err)
-		return "Unknown"
-	}
+	status := "Unknown"
+	healthyCPs := 0
+	for _, cp := range cps {
+		machineName := config.MachineName(*p.Config, cp)
 
-	// The machine isn't running, no need to check inside
-	s, err := host.Driver.GetState()
-	if err != nil {
-		klog.Warningf("error getting host state: %v", err)
-		return "Unknown"
-	}
-	if s != state.Running {
-		return s.String()
-	}
+		ms, err := machine.Status(api, machineName)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): machine status for %s: %v", machineName, err)
+			continue
+		}
+		if ms != state.Running.String() {
+			klog.Warningf("error loading profile (will continue): machine %s is not running: %q", machineName, ms)
+			status = ms
+			continue
+		}
 
-	cr, err := machine.CommandRunner(host)
-	if err != nil {
-		klog.Warningf("error loading profiles: %v", err)
-		return "Unknown"
-	}
+		host, err := machine.LoadHost(api, machineName)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): load host for %s: %v", machineName, err)
+			continue
+		}
 
-	hostname, _, port, err := driver.ControlPlaneEndpoint(p.Config, &cp, host.DriverName)
-	if err != nil {
-		klog.Warningf("error loading profiles: %v", err)
-		return "Unknown"
+		hs, err := host.Driver.GetState()
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): host state for %s: %v", machineName, err)
+			continue
+		}
+		if hs != state.Running {
+			klog.Warningf("error loading profile (will continue): host %s is not running: %q", machineName, hs)
+			status = hs.String()
+			continue
+		}
+
+		cr, err := machine.CommandRunner(host)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): command runner for %s: %v", machineName, err)
+			continue
+		}
+
+		hostname, _, port, err := driver.ControlPlaneEndpoint(p.Config, &cp, host.DriverName)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): control-plane endpoint for %s: %v", machineName, err)
+			continue
+		}
+
+		as, err := kverify.APIServerStatus(cr, hostname, port)
+		if err != nil {
+			klog.Warningf("error loading profile (will continue): apiserver status for %s: %v", machineName, err)
+			continue
+		}
+		status = as.String()
+		if as != state.Running {
+			klog.Warningf("error loading profile (will continue): apiserver %s is not running: %q", machineName, hs)
+			continue
+		}
+
+		healthyCPs++
 	}
 
-	status, err := kverify.APIServerStatus(cr, hostname, port)
-	if err != nil {
-		klog.Warningf("error getting apiserver status for %s: %v", p.Name, err)
-		return "Unknown"
+	if config.IsHA(*p.Config) {
+		switch {
+		case healthyCPs < 2:
+			return state.Stopped.String()
+		case healthyCPs == 2:
+			return "Degraded"
+		default:
+			return "HAppy"
+		}
 	}
-	return status.String()
+	return status
 }
 
 func renderProfilesTable(ps [][]string) {
@@ -166,9 +202,15 @@ func profilesToTableData(profiles []*config.Profile) [][]string {
 	var data [][]string
 	currentProfile := ClusterFlagValue()
 	for _, p := range profiles {
-		cp, err := config.PrimaryControlPlane(p.Config)
-		if err != nil {
-			exit.Error(reason.GuestCpConfig, "error getting primary control plane", err)
+		cpIP := p.Config.KubernetesConfig.APIServerHAVIP
+		cpPort := p.Config.APIServerPort
+		if !config.IsHA(*p.Config) {
+			cp, err := config.ControlPlane(*p.Config)
+			if err != nil {
+				exit.Error(reason.GuestCpConfig, "error getting control-plane node", err)
+			}
+			cpIP = cp.IP
+			cpPort = cp.Port
 		}
 
 		k8sVersion := p.Config.KubernetesConfig.KubernetesVersion
@@ -179,7 +221,7 @@ func profilesToTableData(profiles []*config.Profile) [][]string {
 		if p.Name == currentProfile {
 			c = "*"
 		}
-		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), k8sVersion, p.Status, strconv.Itoa(len(p.Config.Nodes)), c})
+		data = append(data, []string{p.Name, p.Config.Driver, p.Config.KubernetesConfig.ContainerRuntime, cpIP, strconv.Itoa(cpPort), k8sVersion, p.Status, strconv.Itoa(len(p.Config.Nodes)), c})
 	}
 	return data
 }

cmd/minikube/cmd/cp.go

@@ -71,7 +71,7 @@ Example Command : "minikube cp a.txt /home/docker/b.txt" +
 			runner = remoteCommandRunner(&co, dst.node)
 		} else if src.node == "" {
 			// if node name not explicitly specified in both of source and target,
-			// consider target is controlpanel node for backward compatibility.
+			// consider target is control-plane node for backward compatibility.
 			runner = co.CP.Runner
 		} else {
 			runner = command.NewExecRunner(false)
@@ -84,9 +84,6 @@ Example Command : "minikube cp a.txt /home/docker/b.txt" +
 	},
 }
 
-func init() {
-}
-
 // setDstFileNameFromSrc sets the src filename as dst filename
 // when the dst file name is not provided and ends with a `/`.
 // Otherwise this function is a no-op and returns the passed dst.
@@ -211,7 +208,7 @@ func validateArgs(src, dst *remotePath) {
 	}
 
 	// if node name not explicitly specified in both of source and target,
-	// consider target node is controlpanel for backward compatibility.
+	// consider target node is control-plane for backward compatibility.
 	if src.node == "" && dst.node == "" && !strings.HasPrefix(dst.path, "/") {
 		exit.Message(reason.Usage, `Target <remote file path> must be an absolute Path. Relative Path is not allowed (example: "minikube:/home/docker/copied.txt")`)
 	}

cmd/minikube/cmd/docker-env.go

@@ -228,8 +228,7 @@ func mustRestartDockerd(name string, runner command.Runner) {
 		// will need to wait for apisever container to come up, this usually takes 5 seconds
 		// verifying apisever using kverify would add code complexity for a rare case.
 		klog.Warningf("waiting to ensure apisever container is up...")
-		startTime := time.Now()
-		if err = waitForAPIServerProcess(runner, startTime, time.Second*30); err != nil {
+		if err = waitForAPIServerProcess(runner, time.Now(), time.Second*30); err != nil {
 			klog.Warningf("apiserver container isn't up, error: %v", err)
 		}
 	}

cmd/minikube/cmd/logs.go

@@ -138,7 +138,7 @@ func shouldSilentFail() bool {
 
 	api, cc := mustload.Partial(ClusterFlagValue())
 
-	cp, err := config.PrimaryControlPlane(cc)
+	cp, err := config.ControlPlane(*cc)
 	if err != nil {
 		return false
 	}