Handle kubevirt control plane NodePlacement on hypershift clusters (#…

…3219) Since kubevirt v1.3, the kubevirt control plane components (e.g. virt-controller and virt-api) are getting schdeuled by default on the control-plane/master nodes, not regular worker nodes. In hypershift (Hosted Control Planes) clusters, there are no control plane nodes at all (the control plane is hosted as pods on the management cluster). Therefore, in case there are no control plane nodes, we're setting the kubevirt's node placement as an empty struct to avoid any predefined scheduling/affinity rules that might prevent the pods to be scheduled on the cluster. Signed-off-by: Oren Cohen <[email protected]>
kubevirt · Dec 22, 2024 · d70d3c3 · d70d3c3
1 parent 68b8238
commit d70d3c3
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 14 deletions.
diff --git a/controllers/commontestutils/testUtils.go b/controllers/commontestutils/testUtils.go
@@ -294,6 +294,9 @@ func (ClusterInfoMock) IsManagedByOLM() bool {
 func (ClusterInfoMock) IsControlPlaneHighlyAvailable() bool {
 	return true
 }
+func (ClusterInfoMock) IsControlPlaneNodeExists() bool {
+	return true
+}
 func (ClusterInfoMock) IsInfrastructureHighlyAvailable() bool {
 	return true
 }
@@ -366,6 +369,9 @@ func (ClusterInfoSNOMock) IsManagedByOLM() bool {
 func (ClusterInfoSNOMock) IsControlPlaneHighlyAvailable() bool {
 	return false
 }
+func (ClusterInfoSNOMock) IsControlPlaneNodeExists() bool {
+	return true
+}
 func (ClusterInfoSNOMock) IsInfrastructureHighlyAvailable() bool {
 	return false
 }
@@ -438,6 +444,9 @@ func (ClusterInfoSRCPHAIMock) IsManagedByOLM() bool {
 func (ClusterInfoSRCPHAIMock) IsControlPlaneHighlyAvailable() bool {
 	return false
 }
+func (ClusterInfoSRCPHAIMock) IsControlPlaneNodeExists() bool {
+	return true
+}
 func (ClusterInfoSRCPHAIMock) IsInfrastructureHighlyAvailable() bool {
 	return true
 }

diff --git a/controllers/operands/kubevirt.go b/controllers/operands/kubevirt.go
@@ -282,7 +282,8 @@ func NewKubeVirt(hc *hcov1beta1.HyperConverged, opts ...string) (*kubevirtcorev1
 
 	kvCertConfig := hcoCertConfig2KvCertificateRotateStrategy(hc.Spec.CertConfig)
 
-	infrastructureHighlyAvailable := hcoutil.GetClusterInfo().IsInfrastructureHighlyAvailable()
+	controlPlaneHighlyAvailable := hcoutil.GetClusterInfo().IsControlPlaneHighlyAvailable()
+	controlPlaneNodeExists := hcoutil.GetClusterInfo().IsControlPlaneNodeExists()
 
 	uninstallStrategy := kubevirtcorev1.KubeVirtUninstallStrategyBlockUninstallIfWorkloadsExist
 	if hc.Spec.UninstallStrategy == hcov1beta1.HyperConvergedUninstallStrategyRemoveWorkloads {
@@ -291,8 +292,8 @@ func NewKubeVirt(hc *hcov1beta1.HyperConverged, opts ...string) (*kubevirtcorev1
 
 	spec := kubevirtcorev1.KubeVirtSpec{
 		UninstallStrategy:           uninstallStrategy,
-		Infra:                       hcoConfig2KvConfig(hc.Spec.Infra, infrastructureHighlyAvailable),
-		Workloads:                   hcoConfig2KvConfig(hc.Spec.Workloads, true),
+		Infra:                       hcoConfig2KvConfig(hc.Spec.Infra, controlPlaneHighlyAvailable, controlPlaneNodeExists),
+		Workloads:                   hcoConfig2KvConfig(hc.Spec.Workloads, true, true),
 		Configuration:               *config,
 		CertificateRotationStrategy: *kvCertConfig,
 		WorkloadUpdateStrategy:      hcWorkloadUpdateStrategyToKv(&hc.Spec.WorkloadUpdateStrategy),
@@ -802,16 +803,24 @@ func NewKubeVirtWithNameOnly(hc *hcov1beta1.HyperConverged, opts ...string) *kub
 	}
 }
 
-func hcoConfig2KvConfig(hcoConfig hcov1beta1.HyperConvergedConfig, infrastructureHighlyAvailable bool) *kubevirtcorev1.ComponentConfig {
-	if hcoConfig.NodePlacement == nil && infrastructureHighlyAvailable {
+func hcoConfig2KvConfig(hcoConfig hcov1beta1.HyperConvergedConfig, controlPlaneHighlyAvailable, controlPlaneNodeExists bool) *kubevirtcorev1.ComponentConfig {
+	if hcoConfig.NodePlacement == nil && controlPlaneHighlyAvailable {
 		return nil
 	}
 
 	kvConfig := &kubevirtcorev1.ComponentConfig{}
-	if !infrastructureHighlyAvailable {
+	if !controlPlaneHighlyAvailable {
 		kvConfig.Replicas = ptr.To[uint8](1)
 	}
 
+	// In case there are no worker nodes, we're setting an empty struct for NodePlacement
+	// so that kubevirt control plane pods won't have any affinity rules, and they could get
+	// scheduled onto worker nodes.
+	if hcoConfig.NodePlacement == nil && !controlPlaneNodeExists {
+		kvConfig.NodePlacement = &kubevirtcorev1.NodePlacement{}
+		return kvConfig
+	}
+
 	if hcoConfig.NodePlacement != nil {
 		kvConfig.NodePlacement = &kubevirtcorev1.NodePlacement{}
 

diff --git a/controllers/operands/kubevirt_test.go b/controllers/operands/kubevirt_test.go
@@ -3015,14 +3015,14 @@ Version: 1.2.3`)
 					Expect(kv.Spec.Workloads).To(BeNil())
 				})
 
-				It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() {
+				It("should set replica=1 with SingleReplica ControlPlane and HighAvailable Infrastructure ", func() {
 					hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo {
 						return &commontestutils.ClusterInfoSRCPHAIMock{}
 					}
 					kv, err := NewKubeVirt(hco)
 					Expect(err).ToNot(HaveOccurred())
 					Expect(kv.Spec.Infra).To(Not(BeNil()))
-					Expect(kv.Spec.Infra.Replicas).To(BeNil())
+					Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1)))
 					Expect(kv.Spec.Workloads).To(BeNil())
 				})
 
@@ -3059,13 +3059,15 @@ Version: 1.2.3`)
 					Expect(kv.Spec.Workloads.Replicas).To(BeNil())
 				})
 
-				It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() {
+				It("should set replica=1 with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() {
 					hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo {
 						return &commontestutils.ClusterInfoSRCPHAIMock{}
 					}
 					kv, err := NewKubeVirt(hco)
 					Expect(err).ToNot(HaveOccurred())
-					Expect(kv.Spec.Infra).To(BeNil())
+					Expect(kv.Spec.Infra).To(Not(BeNil()))
+					Expect(kv.Spec.Infra.Replicas).To(Not(BeNil()))
+					Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1)))
 					Expect(kv.Spec.Workloads).To(Not(BeNil()))
 					Expect(kv.Spec.Workloads.Replicas).To(BeNil())
 				})
@@ -3098,13 +3100,15 @@ Version: 1.2.3`)
 					Expect(kv.Spec.Workloads).To(BeNil())
 				})
 
-				It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() {
+				It("should set replica=1 with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() {
 					hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo {
 						return &commontestutils.ClusterInfoSRCPHAIMock{}
 					}
 					kv, err := NewKubeVirt(hco)
 					Expect(err).ToNot(HaveOccurred())
-					Expect(kv.Spec.Infra).To(BeNil())
+					Expect(kv.Spec.Infra).To(Not(BeNil()))
+					Expect(kv.Spec.Infra.Replicas).To(Not(BeNil()))
+					Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1)))
 					Expect(kv.Spec.Workloads).To(BeNil())
 				})
 
@@ -3143,14 +3147,14 @@ Version: 1.2.3`)
 					Expect(kv.Spec.Workloads.Replicas).To(BeNil())
 				})
 
-				It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() {
+				It("should set replica=1 with SingleReplica ControlPlane and HighAvailable Infrastructure ", func() {
 					hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo {
 						return &commontestutils.ClusterInfoSRCPHAIMock{}
 					}
 					kv, err := NewKubeVirt(hco)
 					Expect(err).ToNot(HaveOccurred())
 					Expect(kv.Spec.Infra).To(Not(BeNil()))
-					Expect(kv.Spec.Infra.Replicas).To(BeNil())
+					Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1)))
 					Expect(kv.Spec.Workloads).To(Not(BeNil()))
 					Expect(kv.Spec.Workloads.Replicas).To(BeNil())
 				})

diff --git a/pkg/util/cluster.go b/pkg/util/cluster.go
@@ -32,6 +32,7 @@ type ClusterInfo interface {
 	GetBaseDomain() string
 	IsManagedByOLM() bool
 	IsControlPlaneHighlyAvailable() bool
+	IsControlPlaneNodeExists() bool
 	IsInfrastructureHighlyAvailable() bool
 	SetHighAvailabilityMode(ctx context.Context, cl client.Client) error
 	IsConsolePluginImageProvided() bool
@@ -53,6 +54,7 @@ type ClusterInfoImp struct {
 	managedByOLM                  bool
 	runningLocally                bool
 	controlPlaneHighlyAvailable   atomic.Bool
+	controlPlaneNodeExist         atomic.Bool
 	infrastructureHighlyAvailable atomic.Bool
 	consolePluginImageProvided    bool
 	monitoringAvailable           bool
@@ -205,6 +207,10 @@ func (c *ClusterInfoImp) IsControlPlaneHighlyAvailable() bool {
 	return c.controlPlaneHighlyAvailable.Load()
 }
 
+func (c *ClusterInfoImp) IsControlPlaneNodeExists() bool {
+	return c.controlPlaneNodeExist.Load()
+}
+
 func (c *ClusterInfoImp) IsInfrastructureHighlyAvailable() bool {
 	return c.infrastructureHighlyAvailable.Load()
 }
@@ -217,6 +223,7 @@ func (c *ClusterInfoImp) SetHighAvailabilityMode(ctx context.Context, cl client.
 	}
 
 	c.controlPlaneHighlyAvailable.Store(masterNodeCount >= 3)
+	c.controlPlaneNodeExist.Store(masterNodeCount >= 1)
 	c.infrastructureHighlyAvailable.Store(workerNodeCount >= 2)
 	return nil
 }