From d70d3c336d22816c35360a7169a7e3b328ef23ce Mon Sep 17 00:00:00 2001 From: Oren Cohen Date: Sun, 22 Dec 2024 10:36:09 +0200 Subject: [PATCH] Handle kubevirt control plane NodePlacement on hypershift clusters (#3219) Since kubevirt v1.3, the kubevirt control plane components (e.g. virt-controller and virt-api) are getting schdeuled by default on the control-plane/master nodes, not regular worker nodes. In hypershift (Hosted Control Planes) clusters, there are no control plane nodes at all (the control plane is hosted as pods on the management cluster). Therefore, in case there are no control plane nodes, we're setting the kubevirt's node placement as an empty struct to avoid any predefined scheduling/affinity rules that might prevent the pods to be scheduled on the cluster. Signed-off-by: Oren Cohen --- controllers/commontestutils/testUtils.go | 9 +++++++++ controllers/operands/kubevirt.go | 21 +++++++++++++++------ controllers/operands/kubevirt_test.go | 20 ++++++++++++-------- pkg/util/cluster.go | 7 +++++++ 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/controllers/commontestutils/testUtils.go b/controllers/commontestutils/testUtils.go index 00f7a76b9..8f4c17870 100644 --- a/controllers/commontestutils/testUtils.go +++ b/controllers/commontestutils/testUtils.go @@ -294,6 +294,9 @@ func (ClusterInfoMock) IsManagedByOLM() bool { func (ClusterInfoMock) IsControlPlaneHighlyAvailable() bool { return true } +func (ClusterInfoMock) IsControlPlaneNodeExists() bool { + return true +} func (ClusterInfoMock) IsInfrastructureHighlyAvailable() bool { return true } @@ -366,6 +369,9 @@ func (ClusterInfoSNOMock) IsManagedByOLM() bool { func (ClusterInfoSNOMock) IsControlPlaneHighlyAvailable() bool { return false } +func (ClusterInfoSNOMock) IsControlPlaneNodeExists() bool { + return true +} func (ClusterInfoSNOMock) IsInfrastructureHighlyAvailable() bool { return false } @@ -438,6 +444,9 @@ func (ClusterInfoSRCPHAIMock) IsManagedByOLM() bool { func (ClusterInfoSRCPHAIMock) IsControlPlaneHighlyAvailable() bool { return false } +func (ClusterInfoSRCPHAIMock) IsControlPlaneNodeExists() bool { + return true +} func (ClusterInfoSRCPHAIMock) IsInfrastructureHighlyAvailable() bool { return true } diff --git a/controllers/operands/kubevirt.go b/controllers/operands/kubevirt.go index 718f2dfc9..fb873d307 100644 --- a/controllers/operands/kubevirt.go +++ b/controllers/operands/kubevirt.go @@ -282,7 +282,8 @@ func NewKubeVirt(hc *hcov1beta1.HyperConverged, opts ...string) (*kubevirtcorev1 kvCertConfig := hcoCertConfig2KvCertificateRotateStrategy(hc.Spec.CertConfig) - infrastructureHighlyAvailable := hcoutil.GetClusterInfo().IsInfrastructureHighlyAvailable() + controlPlaneHighlyAvailable := hcoutil.GetClusterInfo().IsControlPlaneHighlyAvailable() + controlPlaneNodeExists := hcoutil.GetClusterInfo().IsControlPlaneNodeExists() uninstallStrategy := kubevirtcorev1.KubeVirtUninstallStrategyBlockUninstallIfWorkloadsExist if hc.Spec.UninstallStrategy == hcov1beta1.HyperConvergedUninstallStrategyRemoveWorkloads { @@ -291,8 +292,8 @@ func NewKubeVirt(hc *hcov1beta1.HyperConverged, opts ...string) (*kubevirtcorev1 spec := kubevirtcorev1.KubeVirtSpec{ UninstallStrategy: uninstallStrategy, - Infra: hcoConfig2KvConfig(hc.Spec.Infra, infrastructureHighlyAvailable), - Workloads: hcoConfig2KvConfig(hc.Spec.Workloads, true), + Infra: hcoConfig2KvConfig(hc.Spec.Infra, controlPlaneHighlyAvailable, controlPlaneNodeExists), + Workloads: hcoConfig2KvConfig(hc.Spec.Workloads, true, true), Configuration: *config, CertificateRotationStrategy: *kvCertConfig, WorkloadUpdateStrategy: hcWorkloadUpdateStrategyToKv(&hc.Spec.WorkloadUpdateStrategy), @@ -802,16 +803,24 @@ func NewKubeVirtWithNameOnly(hc *hcov1beta1.HyperConverged, opts ...string) *kub } } -func hcoConfig2KvConfig(hcoConfig hcov1beta1.HyperConvergedConfig, infrastructureHighlyAvailable bool) *kubevirtcorev1.ComponentConfig { - if hcoConfig.NodePlacement == nil && infrastructureHighlyAvailable { +func hcoConfig2KvConfig(hcoConfig hcov1beta1.HyperConvergedConfig, controlPlaneHighlyAvailable, controlPlaneNodeExists bool) *kubevirtcorev1.ComponentConfig { + if hcoConfig.NodePlacement == nil && controlPlaneHighlyAvailable { return nil } kvConfig := &kubevirtcorev1.ComponentConfig{} - if !infrastructureHighlyAvailable { + if !controlPlaneHighlyAvailable { kvConfig.Replicas = ptr.To[uint8](1) } + // In case there are no worker nodes, we're setting an empty struct for NodePlacement + // so that kubevirt control plane pods won't have any affinity rules, and they could get + // scheduled onto worker nodes. + if hcoConfig.NodePlacement == nil && !controlPlaneNodeExists { + kvConfig.NodePlacement = &kubevirtcorev1.NodePlacement{} + return kvConfig + } + if hcoConfig.NodePlacement != nil { kvConfig.NodePlacement = &kubevirtcorev1.NodePlacement{} diff --git a/controllers/operands/kubevirt_test.go b/controllers/operands/kubevirt_test.go index a3ef5c278..361352761 100644 --- a/controllers/operands/kubevirt_test.go +++ b/controllers/operands/kubevirt_test.go @@ -3015,14 +3015,14 @@ Version: 1.2.3`) Expect(kv.Spec.Workloads).To(BeNil()) }) - It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() { + It("should set replica=1 with SingleReplica ControlPlane and HighAvailable Infrastructure ", func() { hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo { return &commontestutils.ClusterInfoSRCPHAIMock{} } kv, err := NewKubeVirt(hco) Expect(err).ToNot(HaveOccurred()) Expect(kv.Spec.Infra).To(Not(BeNil())) - Expect(kv.Spec.Infra.Replicas).To(BeNil()) + Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1))) Expect(kv.Spec.Workloads).To(BeNil()) }) @@ -3059,13 +3059,15 @@ Version: 1.2.3`) Expect(kv.Spec.Workloads.Replicas).To(BeNil()) }) - It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() { + It("should set replica=1 with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() { hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo { return &commontestutils.ClusterInfoSRCPHAIMock{} } kv, err := NewKubeVirt(hco) Expect(err).ToNot(HaveOccurred()) - Expect(kv.Spec.Infra).To(BeNil()) + Expect(kv.Spec.Infra).To(Not(BeNil())) + Expect(kv.Spec.Infra.Replicas).To(Not(BeNil())) + Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1))) Expect(kv.Spec.Workloads).To(Not(BeNil())) Expect(kv.Spec.Workloads.Replicas).To(BeNil()) }) @@ -3098,13 +3100,15 @@ Version: 1.2.3`) Expect(kv.Spec.Workloads).To(BeNil()) }) - It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() { + It("should set replica=1 with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() { hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo { return &commontestutils.ClusterInfoSRCPHAIMock{} } kv, err := NewKubeVirt(hco) Expect(err).ToNot(HaveOccurred()) - Expect(kv.Spec.Infra).To(BeNil()) + Expect(kv.Spec.Infra).To(Not(BeNil())) + Expect(kv.Spec.Infra.Replicas).To(Not(BeNil())) + Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1))) Expect(kv.Spec.Workloads).To(BeNil()) }) @@ -3143,14 +3147,14 @@ Version: 1.2.3`) Expect(kv.Spec.Workloads.Replicas).To(BeNil()) }) - It("should not set replica with SingleReplica ControlPlane but HighAvailable Infrastructure ", func() { + It("should set replica=1 with SingleReplica ControlPlane and HighAvailable Infrastructure ", func() { hcoutil.GetClusterInfo = func() hcoutil.ClusterInfo { return &commontestutils.ClusterInfoSRCPHAIMock{} } kv, err := NewKubeVirt(hco) Expect(err).ToNot(HaveOccurred()) Expect(kv.Spec.Infra).To(Not(BeNil())) - Expect(kv.Spec.Infra.Replicas).To(BeNil()) + Expect(*kv.Spec.Infra.Replicas).To(Equal(uint8(1))) Expect(kv.Spec.Workloads).To(Not(BeNil())) Expect(kv.Spec.Workloads.Replicas).To(BeNil()) }) diff --git a/pkg/util/cluster.go b/pkg/util/cluster.go index 3c3416acd..2ef462a9e 100644 --- a/pkg/util/cluster.go +++ b/pkg/util/cluster.go @@ -32,6 +32,7 @@ type ClusterInfo interface { GetBaseDomain() string IsManagedByOLM() bool IsControlPlaneHighlyAvailable() bool + IsControlPlaneNodeExists() bool IsInfrastructureHighlyAvailable() bool SetHighAvailabilityMode(ctx context.Context, cl client.Client) error IsConsolePluginImageProvided() bool @@ -53,6 +54,7 @@ type ClusterInfoImp struct { managedByOLM bool runningLocally bool controlPlaneHighlyAvailable atomic.Bool + controlPlaneNodeExist atomic.Bool infrastructureHighlyAvailable atomic.Bool consolePluginImageProvided bool monitoringAvailable bool @@ -205,6 +207,10 @@ func (c *ClusterInfoImp) IsControlPlaneHighlyAvailable() bool { return c.controlPlaneHighlyAvailable.Load() } +func (c *ClusterInfoImp) IsControlPlaneNodeExists() bool { + return c.controlPlaneNodeExist.Load() +} + func (c *ClusterInfoImp) IsInfrastructureHighlyAvailable() bool { return c.infrastructureHighlyAvailable.Load() } @@ -217,6 +223,7 @@ func (c *ClusterInfoImp) SetHighAvailabilityMode(ctx context.Context, cl client. } c.controlPlaneHighlyAvailable.Store(masterNodeCount >= 3) + c.controlPlaneNodeExist.Store(masterNodeCount >= 1) c.infrastructureHighlyAvailable.Store(workerNodeCount >= 2) return nil }