Skip to content

Commit

Permalink
account for retryable errors (#422)
Browse files Browse the repository at this point in the history
  • Loading branch information
AshleyDumaine authored Jul 24, 2024
1 parent f66718b commit 27f5949
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 0 deletions.
3 changes: 3 additions & 0 deletions controller/linodemachine_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,9 @@ func (r *LinodeMachineReconciler) reconcileCreate(
createOpts, err := r.newCreateConfig(ctx, machineScope, tags, logger)
if err != nil {
logger.Error(err, "Failed to create Linode machine InstanceCreateOptions")
if util.IsTransientError(err) {
return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerRetryDelay}, nil
}

return ctrl.Result{}, err
}
Expand Down
23 changes: 23 additions & 0 deletions util/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package util

import (
"errors"
"io"
"net/http"
"os"

"github.com/linode/linodego"
)
Expand Down Expand Up @@ -30,3 +33,23 @@ func UnwrapError(err error) error {

return err
}

// IsTransientError determines if the error is transient, meaning a controller that
// encounters this error should requeue reconciliation to try again later
func IsTransientError(err error) bool {
if linodego.ErrHasStatus(
err,
http.StatusTooManyRequests,
http.StatusInternalServerError,
http.StatusBadGateway,
http.StatusGatewayTimeout,
http.StatusServiceUnavailable) {
return true
}

if errors.Is(err, http.ErrHandlerTimeout) || errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}

return false
}
40 changes: 40 additions & 0 deletions util/helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package util

import (
"errors"
"io"
"net/http"
"testing"

"github.com/linode/linodego"
Expand Down Expand Up @@ -49,3 +51,41 @@ func TestIgnoreLinodeAPIError(t *testing.T) {
})
}
}

func TestIsTransientError(t *testing.T) {
t.Parallel()
tests := []struct {
name string
err error
shouldRetry bool
}{{
name: "unexpected EOF",
err: io.ErrUnexpectedEOF,
shouldRetry: true,
}, {
name: "not found Linode API error",
err: &linodego.Error{
Response: nil,
Code: http.StatusNotFound,
Message: "not found",
},
shouldRetry: false,
}, {
name: "Rate limiting Linode API error",
err: &linodego.Error{
Response: nil,
Code: http.StatusTooManyRequests,
Message: "rate limited",
},
shouldRetry: true,
}}
for _, tt := range tests {
testcase := tt
t.Run(testcase.name, func(t *testing.T) {
t.Parallel()
if testcase.shouldRetry != IsTransientError(testcase.err) {
t.Errorf("wanted %v, got %v", testcase.shouldRetry, IsTransientError(testcase.err))
}
})
}
}

0 comments on commit 27f5949

Please sign in to comment.