Skip to content

Commit

Permalink
Ensure correct Server power state and boot config removal
Browse files Browse the repository at this point in the history
  • Loading branch information
@afritzler
afritzler committed Apr 30, 2024
1 parent d45bf73 commit 310aa50
Show file tree
Hide file tree
Showing 6 changed files with 77 additions and 143 deletions.
2 changes: 1 addition & 1 deletion internal/controller/bmc_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (r *BMCReconciler) reconcile(ctx context.Context, log logr.Logger, bmcObj *
}
log.V(1).Info("Updated BMC status")

if err := r.discoverServers(ctx, bmcObj); err != nil {
if err := r.discoverServers(ctx, bmcObj); err != nil && !errors.IsNotFound(err) {
return ctrl.Result{}, fmt.Errorf("failed to discover servers: %w", err)
}
log.V(1).Info("Discovered servers")
Expand Down
97 changes: 68 additions & 29 deletions internal/controller/server_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ import (
"net/http"
"time"

"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/handler"

apierrors "k8s.io/apimachinery/pkg/api/errors"

"github.com/afritzler/metal-operator/internal/api/registry"
Expand All @@ -35,11 +38,9 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/handler"
)

const (
Expand Down Expand Up @@ -149,11 +150,16 @@ func (r *ServerReconciler) reconcile(ctx context.Context, log logr.Logger, serve
}
}

if err := r.updateServerStatus(ctx, log, server); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to update server status: %w", err)
}
log.V(1).Info("Updated Server status")

requeue, err := r.ensureServerStateTransition(ctx, log, server)
if requeue && err == nil {
return ctrl.Result{Requeue: requeue, RequeueAfter: r.RequeueInterval}, nil
}
if err != nil {
if err != nil && !apierrors.IsNotFound(err) {
return ctrl.Result{}, fmt.Errorf("failed to ensure server state transition: %w", err)
}

Expand Down Expand Up @@ -188,21 +194,15 @@ func (r *ServerReconciler) reconcile(ctx context.Context, log logr.Logger, serve
func (r *ServerReconciler) ensureServerStateTransition(ctx context.Context, log logr.Logger, server *metalv1alpha1.Server) (bool, error) {
switch server.Status.State {
case metalv1alpha1.ServerStateInitial:
if err := r.updateServerStatus(ctx, log, server); err != nil {
return false, err
}
log.V(1).Info("Updated Server status")

// apply boot configuration
config, err := r.applyBootConfigurationAndIgnitionForDiscovery(ctx, server)
if err != nil {
if err := r.applyBootConfigurationAndIgnitionForDiscovery(ctx, server); err != nil {
return false, fmt.Errorf("failed to apply server boot configuration: %w", err)
}
log.V(1).Info("Applied Server boot configuration")

if ready, err := r.serverBootConfigurationIsReady(ctx, server); err != nil || !ready {
log.V(1).Info("Server boot configuration is not ready. Retrying ...")
return false, err
return true, err
}
log.V(1).Info("Server boot configuration is ready")

Expand All @@ -222,13 +222,13 @@ func (r *ServerReconciler) ensureServerStateTransition(ctx context.Context, log
}
log.V(1).Info("Extracted Server details")

if err := r.ensureInitialBootConfigurationIsDeleted(ctx, config); err != nil {
return false, fmt.Errorf("failed to ensure server initial boot configuration is deleted: %w", err)
serverBase := server.DeepCopy()
server.Spec.Power = metalv1alpha1.PowerOff
if err := r.Patch(ctx, server, client.MergeFrom(serverBase)); err != nil {
return false, fmt.Errorf("failed to update server power state: %w", err)
}
log.V(1).Info("Ensured initial boot configuration is deleted")
log.V(1).Info("Updated Server power state", "PowerState", metalv1alpha1.PowerOff)

// TODO: fix that by providing the power state to the ensure method
server.Spec.Power = metalv1alpha1.PowerOff
if err := r.ensureServerPowerState(ctx, log, server); err != nil {
return false, fmt.Errorf("failed to ensure server power state: %w", err)
}
Expand All @@ -244,10 +244,10 @@ func (r *ServerReconciler) ensureServerStateTransition(ctx context.Context, log
return false, err
}
case metalv1alpha1.ServerStateAvailable:
if err := r.updateServerStatus(ctx, log, server); err != nil {
return false, err
if err := r.ensureInitialBootConfigurationIsDeleted(ctx, server); err != nil {
return false, fmt.Errorf("failed to ensure server initial boot configuration is deleted: %w", err)
}
log.V(1).Info("Updated Server status")
log.V(1).Info("Ensured initial boot configuration is deleted")

if err := r.ensureServerPowerState(ctx, log, server); err != nil {
return false, fmt.Errorf("failed to ensure server power state: %w", err)
Expand All @@ -257,11 +257,7 @@ func (r *ServerReconciler) ensureServerStateTransition(ctx context.Context, log
}
log.V(1).Info("Reconciled available state")
case metalv1alpha1.ServerStateReserved:
if err := r.updateServerStatus(ctx, log, server); err != nil {
return false, err
}
log.V(1).Info("Updated Server status")

// TODO: do first PXE boot
if err := r.ensureServerPowerState(ctx, log, server); err != nil {
return false, fmt.Errorf("failed to ensure server power state: %w", err)
}
Expand All @@ -273,6 +269,22 @@ func (r *ServerReconciler) ensureServerStateTransition(ctx context.Context, log
return false, nil
}

func (r *ServerReconciler) ensureServerBootConfigRef(ctx context.Context, server *metalv1alpha1.Server, config *metalv1alpha1.ServerBootConfiguration) error {
serverBase := server.DeepCopy()
server.Spec.BootConfigurationRef = &v1.ObjectReference{
Namespace: config.Namespace,
Name: config.Name,
UID: config.UID,
APIVersion: "metal.ironcore.dev/v1alpha1",
Kind: "ServerBootConfiguration",
}
if err := r.Patch(ctx, server, client.MergeFrom(serverBase)); err != nil {
return err
}

return nil
}

func (r *ServerReconciler) updateServerStatus(ctx context.Context, log logr.Logger, server *metalv1alpha1.Server) error {
if server.Spec.BMCRef == nil && server.Spec.BMC == nil {
log.V(1).Info("Server has no BMC connection configured")
Expand Down Expand Up @@ -303,7 +315,7 @@ func (r *ServerReconciler) updateServerStatus(ctx context.Context, log logr.Logg
return nil
}

func (r *ServerReconciler) applyBootConfigurationAndIgnitionForDiscovery(ctx context.Context, server *metalv1alpha1.Server) (*metalv1alpha1.ServerBootConfiguration, error) {
func (r *ServerReconciler) applyBootConfigurationAndIgnitionForDiscovery(ctx context.Context, server *metalv1alpha1.Server) error {
// apply server boot configuration
bootConfig := &metalv1alpha1.ServerBootConfiguration{
TypeMeta: metav1.TypeMeta{
Expand All @@ -326,14 +338,18 @@ func (r *ServerReconciler) applyBootConfigurationAndIgnitionForDiscovery(ctx con
}

if err := r.Patch(ctx, bootConfig, client.Apply, fieldOwner, client.ForceOwnership); err != nil {
return nil, fmt.Errorf("failed to apply server boot configuration: %w", err)
return err
}

if err := r.ensureServerBootConfigRef(ctx, server, bootConfig); err != nil {
return err
}

if err := r.applyDefaultIgnitionForServer(ctx, server, bootConfig, r.RegistryURL); err != nil {
return nil, fmt.Errorf("failed to apply default server ignitionSecret: %w", err)
return err
}

return bootConfig, nil
return nil
}

func (r *ServerReconciler) applyDefaultIgnitionForServer(
Expand Down Expand Up @@ -517,10 +533,33 @@ func (r *ServerReconciler) ensureIndicatorLED(ctx context.Context, log logr.Logg
return nil
}

func (r *ServerReconciler) ensureInitialBootConfigurationIsDeleted(ctx context.Context, config *metalv1alpha1.ServerBootConfiguration) error {
func (r *ServerReconciler) ensureInitialBootConfigurationIsDeleted(ctx context.Context, server *metalv1alpha1.Server) error {
if server.Spec.BootConfigurationRef == nil {
return nil
}

if server.Spec.BootConfigurationRef.Namespace != r.ManagerNamespace && server.Spec.BootConfigurationRef.Name != server.Name {
// hit a non initial boot config
return nil
}

config := &metalv1alpha1.ServerBootConfiguration{
ObjectMeta: metav1.ObjectMeta{
Namespace: server.Spec.BootConfigurationRef.Namespace,
Name: server.Spec.BootConfigurationRef.Name,
},
}

if err := r.Delete(ctx, config); !apierrors.IsNotFound(err) {
return err
}

serverBase := server.DeepCopy()
server.Spec.BootConfigurationRef = nil
if err := r.Patch(ctx, server, client.MergeFrom(serverBase)); err != nil {
return err
}

return nil
}

Expand Down
21 changes: 7 additions & 14 deletions internal/controller/server_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import (
. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
)

var _ = Describe("Server Controller", func() {
var _ = FDescribe("Server Controller", func() {
ns := SetupTest()

var endpoint *metalv1alpha1.Endpoint
Expand Down Expand Up @@ -75,7 +75,6 @@ var _ = Describe("Server Controller", func() {
},
}
Eventually(Object(bootConfig)).Should(SatisfyAll(
HaveField("Finalizers", ContainElement(ServerBootConfigurationFinalizer)),
HaveField("Spec.ServerRef", v1.LocalObjectReference{Name: server.Name}),
HaveField("Spec.Image", "fooOS:latest"),
HaveField("Spec.IgnitionSecretRef", &v1.LocalObjectReference{Name: server.Name}),
Expand Down Expand Up @@ -131,18 +130,18 @@ var _ = Describe("Server Controller", func() {
HaveField("Status.State", metalv1alpha1.ServerStateInitial),
))

By("Patching the boot configuration to a Ready state")
Eventually(UpdateStatus(bootConfig, func() {
bootConfig.Status.State = metalv1alpha1.ServerBootConfigurationStateReady
})).Should(Succeed())

By("Starting the probe agent")
probeAgent := probe.NewAgent(server.Spec.UUID, registryURL)
go func() {
defer GinkgoRecover()
Expect(probeAgent.Start(ctx)).To(Succeed(), "failed to start probe agent")
}()

By("Patching the boot configuration to a Ready state")
Eventually(UpdateStatus(bootConfig, func() {
bootConfig.Status.State = metalv1alpha1.ServerBootConfigurationStateReady
})).Should(Succeed())

By("Ensuring that the server is set to available and powered off")
Eventually(Object(server)).Should(SatisfyAll(
HaveField("Spec.BootConfigurationRef", BeNil()),
Expand All @@ -152,13 +151,7 @@ var _ = Describe("Server Controller", func() {
))

By("Ensuring that the boot configuration has been removed")
config := &metalv1alpha1.ServerBootConfiguration{
ObjectMeta: metav1.ObjectMeta{
Namespace: ns.Name,
Name: server.Name,
},
}
Consistently(Get(config)).Should(Satisfy(apierrors.IsNotFound))
Consistently(Get(bootConfig)).Should(Satisfy(apierrors.IsNotFound))

By("Ensuring that the server is removed from the registry")
response, err := http.Get(registryURL + "/systems/" + server.Spec.UUID)
Expand Down
87 changes: 0 additions & 87 deletions internal/controller/serverbootconfiguration_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,6 @@ package controller

import (
"context"
"fmt"

"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/handler"

"github.com/ironcore-dev/controller-utils/clientutils"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"

metalv1alpha1 "github.com/afritzler/metal-operator/api/v1alpha1"
"github.com/go-logr/logr"
Expand All @@ -34,10 +26,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
ServerBootConfigurationFinalizer = "metal.ironcore.dev/serverbootconfiguration"
)

// ServerBootConfigurationReconciler reconciles a ServerBootConfiguration object
type ServerBootConfigurationReconciler struct {
client.Client
Expand Down Expand Up @@ -70,16 +58,6 @@ func (r *ServerBootConfigurationReconciler) reconcileExists(ctx context.Context,
func (r *ServerBootConfigurationReconciler) delete(ctx context.Context, log logr.Logger, config *metalv1alpha1.ServerBootConfiguration) (ctrl.Result, error) {
log.V(1).Info("Deleting ServerBootConfiguration")

if err := r.removeServerBootConfigRef(ctx, config); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to remove ServerBootConfigRef from server: %w", err)
}
log.V(1).Info("Ensured no server boot config is set on server")

if modified, err := clientutils.PatchEnsureNoFinalizer(ctx, r.Client, config, ServerBootConfigurationFinalizer); !apierrors.IsNotFound(err) || modified {
return ctrl.Result{}, err
}
log.V(1).Info("Ensured that the finalizer has been removed")

log.V(1).Info("Deleted ServerBootConfiguration")
return ctrl.Result{}, nil
}
Expand All @@ -93,16 +71,6 @@ func (r *ServerBootConfigurationReconciler) reconcile(ctx context.Context, log l
}
log.V(1).Info("Patched state")

if modified, err := clientutils.PatchEnsureFinalizer(ctx, r.Client, config, ServerBootConfigurationFinalizer); err != nil || modified {
return ctrl.Result{}, err
}
log.V(1).Info("Ensured finalizer has been added")

if err := r.patchServerBootConfigRef(ctx, config); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to patch server boot config ref: %w", err)
}
log.V(1).Info("Patched server boot config ref")

log.V(1).Info("Reconciled ServerBootConfiguration")
return ctrl.Result{}, nil
}
Expand All @@ -119,64 +87,9 @@ func (r *ServerBootConfigurationReconciler) patchState(ctx context.Context, conf
return true, nil
}

func (r *ServerBootConfigurationReconciler) patchServerBootConfigRef(ctx context.Context, config *metalv1alpha1.ServerBootConfiguration) error {
server := &metalv1alpha1.Server{}
if err := r.Get(ctx, client.ObjectKey{Name: config.Spec.ServerRef.Name}, server); err != nil {
return err
}

serverBase := server.DeepCopy()
server.Spec.BootConfigurationRef = &v1.ObjectReference{
Namespace: config.Namespace,
Name: config.Name,
UID: config.UID,
APIVersion: "metal.ironcore.dev/v1alpha1",
Kind: "ServerBootConfiguration",
}
if err := r.Patch(ctx, server, client.MergeFrom(serverBase)); err != nil {
return err
}

return nil
}

func (r *ServerBootConfigurationReconciler) removeServerBootConfigRef(ctx context.Context, config *metalv1alpha1.ServerBootConfiguration) error {
server := &metalv1alpha1.Server{}
if err := r.Get(ctx, client.ObjectKey{Name: config.Spec.ServerRef.Name}, server); err != nil {
if apierrors.IsNotFound(err) {
// server is gone
return nil
}
return err
}

serverBase := server.DeepCopy()
server.Spec.BootConfigurationRef = nil
if err := r.Patch(ctx, server, client.MergeFrom(serverBase)); err != nil {
return err
}

return nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *ServerBootConfigurationReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&metalv1alpha1.ServerBootConfiguration{}).
Watches(&metalv1alpha1.Server{}, r.enqueueServerBootConfigByServerRef()).
Complete(r)
}

func (r *ServerBootConfigurationReconciler) enqueueServerBootConfigByServerRef() handler.EventHandler {
return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []ctrl.Request {
server := obj.(*metalv1alpha1.Server)
if server.Spec.BootConfigurationRef != nil {
return []ctrl.Request{
{
NamespacedName: types.NamespacedName{Namespace: server.Spec.BootConfigurationRef.Namespace, Name: server.Spec.BootConfigurationRef.Name},
},
}
}
return nil
})
}
Loading

0 comments on commit 310aa50

Please sign in to comment.