add more checks for retrying on transient error during machine creation

linode · Jul 29, 2024 · 061a407 · 061a407
1 parent 3119045
commit 061a407
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 10 deletions.
diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go
@@ -56,7 +56,6 @@ import (
 
 const (
 	linodeBusyCode        = 400
-	linodeTooManyRequests = 429
 	defaultDiskFilesystem = string(linodego.FilesystemExt4)
 
 	// conditions for preflight instance creation
@@ -262,6 +261,14 @@ func (r *LinodeMachineReconciler) reconcile(
 	return
 }
 
+func retryIfTransient(err error) (ctrl.Result, error) {
+	if util.IsTransientError(err) {
+		return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
+	}
+
+	return ctrl.Result{}, err
+}
+
 func (r *LinodeMachineReconciler) reconcileCreate(
 	ctx context.Context,
 	logger logr.Logger,
@@ -302,15 +309,11 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 		createOpts, err := r.newCreateConfig(ctx, machineScope, tags, logger)
 		if err != nil {
 			logger.Error(err, "Failed to create Linode machine InstanceCreateOptions")
-			if util.IsTransientError(err) {
-				return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
-			}
-
-			return ctrl.Result{}, err
+			return retryIfTransient(err)
 		}
 		linodeInstance, err = machineScope.LinodeClient.CreateInstance(ctx, *createOpts)
 		if err != nil {
-			if linodego.ErrHasStatus(err, linodeTooManyRequests) || linodego.ErrHasStatus(err, linodego.ErrorFromError) {
+			if linodego.ErrHasStatus(err, http.StatusTooManyRequests) || linodego.ErrHasStatus(err, linodego.ErrorFromError) {
 				logger.Error(err, "Failed to create Linode instance due to API error, requeing")
 				return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
 			}
@@ -362,11 +365,13 @@ func (r *LinodeMachineReconciler) reconcileInstanceCreate(
 		instanceConfig, err := r.getDefaultInstanceConfig(ctx, machineScope, linodeInstance.ID)
 		if err != nil {
 			logger.Error(err, "Failed to get default instance configuration")
-			return ctrl.Result{}, err
+			return retryIfTransient(err)
+
 		}
 
 		if _, err := machineScope.LinodeClient.UpdateInstanceConfig(ctx, linodeInstance.ID, instanceConfig.ID, linodego.InstanceConfigUpdateOptions{Kernel: machineScope.LinodeMachine.Spec.Configuration.Kernel}); err != nil {
-			return ctrl.Result{}, err
+			logger.Error(err, "Failed to get update instance configuration")
+			return retryIfTransient(err)
 		}
 	}
 

diff --git a/util/reconciler/defaults.go b/util/reconciler/defaults.go
@@ -37,7 +37,7 @@ const (
 	// DefaultMachineControllerWaitForRunningTimeout is the default timeout if instance is not running.
 	DefaultMachineControllerWaitForRunningTimeout = 20 * time.Minute
 	// DefaultMachineControllerRetryDelay is the default requeue delay if there is an error.
-	DefaultMachineControllerRetryDelay = 10 * time.Second
+	DefaultMachineControllerRetryDelay = 30 * time.Second
 
 	// DefaultVPCControllerReconcileDelay is the default requeue delay when a reconcile operation fails.
 	DefaultVPCControllerReconcileDelay = 5 * time.Second