From 061a407bc153147f6659b5b489e0ea2429f3f2e0 Mon Sep 17 00:00:00 2001
From: Ashley Dumaine <ashley.dumaine@gmail.com>
Date: Mon, 29 Jul 2024 16:04:58 -0400
Subject: [PATCH] add more checks for retrying on transient error during
 machine creation

---
 controller/linodemachine_controller.go | 23 ++++++++++++++---------
 util/reconciler/defaults.go            |  2 +-
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go
index 068c11df4..c9b3bf918 100644
--- a/controller/linodemachine_controller.go
+++ b/controller/linodemachine_controller.go
@@ -56,7 +56,6 @@ import (
 
 const (
 	linodeBusyCode        = 400
-	linodeTooManyRequests = 429
 	defaultDiskFilesystem = string(linodego.FilesystemExt4)
 
 	// conditions for preflight instance creation
@@ -262,6 +261,14 @@ func (r *LinodeMachineReconciler) reconcile(
 	return
 }
 
+func retryIfTransient(err error) (ctrl.Result, error) {
+	if util.IsTransientError(err) {
+		return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
+	}
+
+	return ctrl.Result{}, err
+}
+
 func (r *LinodeMachineReconciler) reconcileCreate(
 	ctx context.Context,
 	logger logr.Logger,
@@ -302,15 +309,11 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 		createOpts, err := r.newCreateConfig(ctx, machineScope, tags, logger)
 		if err != nil {
 			logger.Error(err, "Failed to create Linode machine InstanceCreateOptions")
-			if util.IsTransientError(err) {
-				return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
-			}
-
-			return ctrl.Result{}, err
+			return retryIfTransient(err)
 		}
 		linodeInstance, err = machineScope.LinodeClient.CreateInstance(ctx, *createOpts)
 		if err != nil {
-			if linodego.ErrHasStatus(err, linodeTooManyRequests) || linodego.ErrHasStatus(err, linodego.ErrorFromError) {
+			if linodego.ErrHasStatus(err, http.StatusTooManyRequests) || linodego.ErrHasStatus(err, linodego.ErrorFromError) {
 				logger.Error(err, "Failed to create Linode instance due to API error, requeing")
 				return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
 			}
@@ -362,11 +365,13 @@ func (r *LinodeMachineReconciler) reconcileInstanceCreate(
 		instanceConfig, err := r.getDefaultInstanceConfig(ctx, machineScope, linodeInstance.ID)
 		if err != nil {
 			logger.Error(err, "Failed to get default instance configuration")
-			return ctrl.Result{}, err
+			return retryIfTransient(err)
+
 		}
 
 		if _, err := machineScope.LinodeClient.UpdateInstanceConfig(ctx, linodeInstance.ID, instanceConfig.ID, linodego.InstanceConfigUpdateOptions{Kernel: machineScope.LinodeMachine.Spec.Configuration.Kernel}); err != nil {
-			return ctrl.Result{}, err
+			logger.Error(err, "Failed to get update instance configuration")
+			return retryIfTransient(err)
 		}
 	}
 
diff --git a/util/reconciler/defaults.go b/util/reconciler/defaults.go
index 3d0f9d21c..a8c5b739f 100644
--- a/util/reconciler/defaults.go
+++ b/util/reconciler/defaults.go
@@ -37,7 +37,7 @@ const (
 	// DefaultMachineControllerWaitForRunningTimeout is the default timeout if instance is not running.
 	DefaultMachineControllerWaitForRunningTimeout = 20 * time.Minute
 	// DefaultMachineControllerRetryDelay is the default requeue delay if there is an error.
-	DefaultMachineControllerRetryDelay = 10 * time.Second
+	DefaultMachineControllerRetryDelay = 30 * time.Second
 
 	// DefaultVPCControllerReconcileDelay is the default requeue delay when a reconcile operation fails.
 	DefaultVPCControllerReconcileDelay = 5 * time.Second