From e5fbf86157bbad14668509eca01c7fe91fc5c952 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Sat, 14 May 2016 14:25:47 -0400 Subject: [PATCH 1/4] Allow StatusErrors to be modified after creation --- pkg/api/errors/errors.go | 30 +++++++++---------- pkg/api/errors/errors_test.go | 2 +- .../unversioned/testclient/testclient_test.go | 4 +-- pkg/kubectl/cmd/delete_test.go | 4 +-- pkg/storage/cacher_test.go | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pkg/api/errors/errors.go b/pkg/api/errors/errors.go index 345ad0e04355b..89e83c2e3a6c0 100644 --- a/pkg/api/errors/errors.go +++ b/pkg/api/errors/errors.go @@ -93,7 +93,7 @@ func FromObject(obj runtime.Object) error { } // NewNotFound returns a new error which indicates that the resource of the kind and the name was not found. -func NewNotFound(qualifiedResource unversioned.GroupResource, name string) error { +func NewNotFound(qualifiedResource unversioned.GroupResource, name string) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusNotFound, @@ -108,7 +108,7 @@ func NewNotFound(qualifiedResource unversioned.GroupResource, name string) error } // NewAlreadyExists returns an error indicating the item requested exists by that identifier. -func NewAlreadyExists(qualifiedResource unversioned.GroupResource, name string) error { +func NewAlreadyExists(qualifiedResource unversioned.GroupResource, name string) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusConflict, @@ -124,7 +124,7 @@ func NewAlreadyExists(qualifiedResource unversioned.GroupResource, name string) // NewUnauthorized returns an error indicating the client is not authorized to perform the requested // action. -func NewUnauthorized(reason string) error { +func NewUnauthorized(reason string) *StatusError { message := reason if len(message) == 0 { message = "not authorized" @@ -138,7 +138,7 @@ func NewUnauthorized(reason string) error { } // NewForbidden returns an error indicating the requested action was forbidden -func NewForbidden(qualifiedResource unversioned.GroupResource, name string, err error) error { +func NewForbidden(qualifiedResource unversioned.GroupResource, name string, err error) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusForbidden, @@ -153,7 +153,7 @@ func NewForbidden(qualifiedResource unversioned.GroupResource, name string, err } // NewConflict returns an error indicating the item can't be updated as provided. -func NewConflict(qualifiedResource unversioned.GroupResource, name string, err error) error { +func NewConflict(qualifiedResource unversioned.GroupResource, name string, err error) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusConflict, @@ -168,7 +168,7 @@ func NewConflict(qualifiedResource unversioned.GroupResource, name string, err e } // NewGone returns an error indicating the item no longer available at the server and no forwarding address is known. -func NewGone(message string) error { +func NewGone(message string) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusGone, @@ -178,7 +178,7 @@ func NewGone(message string) error { } // NewInvalid returns an error indicating the item is invalid and cannot be processed. -func NewInvalid(qualifiedKind unversioned.GroupKind, name string, errs field.ErrorList) error { +func NewInvalid(qualifiedKind unversioned.GroupKind, name string, errs field.ErrorList) *StatusError { causes := make([]unversioned.StatusCause, 0, len(errs)) for i := range errs { err := errs[i] @@ -203,7 +203,7 @@ func NewInvalid(qualifiedKind unversioned.GroupKind, name string, errs field.Err } // NewBadRequest creates an error that indicates that the request is invalid and can not be processed. -func NewBadRequest(reason string) error { +func NewBadRequest(reason string) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusBadRequest, @@ -213,7 +213,7 @@ func NewBadRequest(reason string) error { } // NewServiceUnavailable creates an error that indicates that the requested service is unavailable. -func NewServiceUnavailable(reason string) error { +func NewServiceUnavailable(reason string) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusServiceUnavailable, @@ -223,7 +223,7 @@ func NewServiceUnavailable(reason string) error { } // NewMethodNotSupported returns an error indicating the requested action is not supported on this kind. -func NewMethodNotSupported(qualifiedResource unversioned.GroupResource, action string) error { +func NewMethodNotSupported(qualifiedResource unversioned.GroupResource, action string) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusMethodNotAllowed, @@ -238,7 +238,7 @@ func NewMethodNotSupported(qualifiedResource unversioned.GroupResource, action s // NewServerTimeout returns an error indicating the requested action could not be completed due to a // transient error, and the client should try again. -func NewServerTimeout(qualifiedResource unversioned.GroupResource, operation string, retryAfterSeconds int) error { +func NewServerTimeout(qualifiedResource unversioned.GroupResource, operation string, retryAfterSeconds int) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusInternalServerError, @@ -255,12 +255,12 @@ func NewServerTimeout(qualifiedResource unversioned.GroupResource, operation str // NewServerTimeoutForKind should not exist. Server timeouts happen when accessing resources, the Kind is just what we // happened to be looking at when the request failed. This delegates to keep code sane, but we should work towards removing this. -func NewServerTimeoutForKind(qualifiedKind unversioned.GroupKind, operation string, retryAfterSeconds int) error { +func NewServerTimeoutForKind(qualifiedKind unversioned.GroupKind, operation string, retryAfterSeconds int) *StatusError { return NewServerTimeout(unversioned.GroupResource{Group: qualifiedKind.Group, Resource: qualifiedKind.Kind}, operation, retryAfterSeconds) } // NewInternalError returns an error indicating the item is invalid and cannot be processed. -func NewInternalError(err error) error { +func NewInternalError(err error) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: http.StatusInternalServerError, @@ -274,7 +274,7 @@ func NewInternalError(err error) error { // NewTimeoutError returns an error indicating that a timeout occurred before the request // could be completed. Clients may retry, but the operation may still complete. -func NewTimeoutError(message string, retryAfterSeconds int) error { +func NewTimeoutError(message string, retryAfterSeconds int) *StatusError { return &StatusError{unversioned.Status{ Status: unversioned.StatusFailure, Code: StatusServerTimeout, @@ -287,7 +287,7 @@ func NewTimeoutError(message string, retryAfterSeconds int) error { } // NewGenericServerResponse returns a new error for server responses that are not in a recognizable form. -func NewGenericServerResponse(code int, verb string, qualifiedResource unversioned.GroupResource, name, serverMessage string, retryAfterSeconds int, isUnexpectedResponse bool) error { +func NewGenericServerResponse(code int, verb string, qualifiedResource unversioned.GroupResource, name, serverMessage string, retryAfterSeconds int, isUnexpectedResponse bool) *StatusError { reason := unversioned.StatusReasonUnknown message := fmt.Sprintf("the server responded with the status code %d but did not return more information", code) switch code { diff --git a/pkg/api/errors/errors_test.go b/pkg/api/errors/errors_test.go index 066bb25f2747a..55928f2318a5a 100644 --- a/pkg/api/errors/errors_test.go +++ b/pkg/api/errors/errors_test.go @@ -152,7 +152,7 @@ func TestNewInvalid(t *testing.T) { vErr, expected := testCase.Err, testCase.Details expected.Causes[0].Message = vErr.ErrorBody() err := NewInvalid(api.Kind("Kind"), "name", field.ErrorList{vErr}) - status := err.(*StatusError).ErrStatus + status := err.ErrStatus if status.Code != 422 || status.Reason != unversioned.StatusReasonInvalid { t.Errorf("%d: unexpected status: %#v", i, status) } diff --git a/pkg/client/unversioned/testclient/testclient_test.go b/pkg/client/unversioned/testclient/testclient_test.go index b31bdc213ee51..4a799df71db25 100644 --- a/pkg/client/unversioned/testclient/testclient_test.go +++ b/pkg/client/unversioned/testclient/testclient_test.go @@ -56,9 +56,9 @@ func TestErrors(t *testing.T) { o.Add(&api.List{ Items: []runtime.Object{ // This first call to List will return this error - &(errors.NewNotFound(api.Resource("ServiceList"), "").(*errors.StatusError).ErrStatus), + &(errors.NewNotFound(api.Resource("ServiceList"), "").ErrStatus), // The second call to List will return this error - &(errors.NewForbidden(api.Resource("ServiceList"), "", nil).(*errors.StatusError).ErrStatus), + &(errors.NewForbidden(api.Resource("ServiceList"), "", nil).ErrStatus), }, }) client := &Fake{} diff --git a/pkg/kubectl/cmd/delete_test.go b/pkg/kubectl/cmd/delete_test.go index 1f2122ab01323..1e8c964838a4c 100644 --- a/pkg/kubectl/cmd/delete_test.go +++ b/pkg/kubectl/cmd/delete_test.go @@ -192,7 +192,7 @@ func TestDeleteAllNotFound(t *testing.T) { // Add an item to the list which will result in a 404 on delete svc.Items = append(svc.Items, api.Service{ObjectMeta: api.ObjectMeta{Name: "foo"}}) - notFoundError := &errors.NewNotFound(api.Resource("services"), "foo").(*errors.StatusError).ErrStatus + notFoundError := &errors.NewNotFound(api.Resource("services"), "foo").ErrStatus tf.Printer = &testPrinter{} tf.Client = &fake.RESTClient{ @@ -234,7 +234,7 @@ func TestDeleteAllIgnoreNotFound(t *testing.T) { // Add an item to the list which will result in a 404 on delete svc.Items = append(svc.Items, api.Service{ObjectMeta: api.ObjectMeta{Name: "foo"}}) - notFoundError := &errors.NewNotFound(api.Resource("services"), "foo").(*errors.StatusError).ErrStatus + notFoundError := &errors.NewNotFound(api.Resource("services"), "foo").ErrStatus tf.Printer = &testPrinter{} tf.Client = &fake.RESTClient{ diff --git a/pkg/storage/cacher_test.go b/pkg/storage/cacher_test.go index 21cf571f74481..37408f876c9b1 100644 --- a/pkg/storage/cacher_test.go +++ b/pkg/storage/cacher_test.go @@ -239,7 +239,7 @@ func TestWatch(t *testing.T) { } defer tooOldWatcher.Stop() // Ensure we get a "Gone" error - expectedGoneError := errors.NewGone("").(*errors.StatusError).ErrStatus + expectedGoneError := errors.NewGone("").ErrStatus verifyWatchEvent(t, tooOldWatcher, watch.Error, &expectedGoneError) initialWatcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", fooCreated.ResourceVersion, storage.Everything) From 88b39cadf877402b6fe7bfba78dbf5743fae76ce Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Thu, 5 May 2016 17:28:54 -0400 Subject: [PATCH 2/4] Have the service account controller force retry Service account controller, when API token not found, now sends 500 with Retry-After: 1s. Also change the apiserver to actually write the error. --- pkg/apiserver/apiserver.go | 6 +++++ pkg/apiserver/apiserver_test.go | 25 ++++++++++++++++++- .../pkg/admission/serviceaccount/admission.go | 9 +++++-- .../serviceaccount/admission_test.go | 5 ++-- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 49db40d618de1..e96369c297587 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -26,6 +26,7 @@ import ( "net/http" "path" rt "runtime" + "strconv" "strings" "time" @@ -449,6 +450,11 @@ func writeNegotiated(s runtime.NegotiatedSerializer, gv unversioned.GroupVersion func errorNegotiated(err error, s runtime.NegotiatedSerializer, gv unversioned.GroupVersion, w http.ResponseWriter, req *http.Request) int { status := errToAPIStatus(err) code := int(status.Code) + // when writing an error, check to see if the status indicates a retry after period + if status.Details != nil && status.Details.RetryAfterSeconds > 0 { + delay := strconv.Itoa(int(status.Details.RetryAfterSeconds)) + w.Header().Set("Retry-After", delay) + } writeNegotiated(s, gv, w, req, code, status) return code } diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index cc89ddc0555fd..4408c16124d69 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -1562,6 +1562,7 @@ func TestGetNamespaceSelfLink(t *testing.T) { t.Errorf("Never set self link") } } + func TestGetMissing(t *testing.T) { storage := map[string]rest.Storage{} simpleStorage := SimpleRESTStorage{ @@ -1572,7 +1573,7 @@ func TestGetMissing(t *testing.T) { server := httptest.NewServer(handler) defer server.Close() - resp, err := http.Get(server.URL + "/" + prefix + "/" + testGroupVersion.Group + "/" + testGroupVersion.Version + "/simple/id") + resp, err := http.Get(server.URL + "/" + prefix + "/" + testGroupVersion.Group + "/" + testGroupVersion.Version + "/namespaces/default/simple/id") if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1582,6 +1583,28 @@ func TestGetMissing(t *testing.T) { } } +func TestGetRetryAfter(t *testing.T) { + storage := map[string]rest.Storage{} + simpleStorage := SimpleRESTStorage{ + errors: map[string]error{"get": apierrs.NewServerTimeout(api.Resource("simples"), "id", 2)}, + } + storage["simple"] = &simpleStorage + handler := handle(storage) + server := httptest.NewServer(handler) + defer server.Close() + + resp, err := http.Get(server.URL + "/" + prefix + "/" + testGroupVersion.Group + "/" + testGroupVersion.Version + "/namespaces/default/simple/id") + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("Unexpected response %#v", resp) + } + if resp.Header.Get("Retry-After") != "2" { + t.Errorf("Unexpected Retry-After header: %v", resp.Header) + } +} + func TestConnect(t *testing.T) { responseText := "Hello World" itemID := "theID" diff --git a/plugin/pkg/admission/serviceaccount/admission.go b/plugin/pkg/admission/serviceaccount/admission.go index 55b7bfaa6fca1..7be0dded2b23f 100644 --- a/plugin/pkg/admission/serviceaccount/admission.go +++ b/plugin/pkg/admission/serviceaccount/admission.go @@ -28,6 +28,7 @@ import ( "k8s.io/kubernetes/pkg/admission" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/errors" + "k8s.io/kubernetes/pkg/api/unversioned" "k8s.io/kubernetes/pkg/client/cache" "k8s.io/kubernetes/pkg/fields" kubelet "k8s.io/kubernetes/pkg/kubelet/types" @@ -199,6 +200,9 @@ func (s *serviceAccount) Admit(a admission.Attributes) (err error) { if s.MountServiceAccountToken { if err := s.mountServiceAccountToken(serviceAccount, pod); err != nil { + if _, ok := err.(errors.APIStatus); ok { + return err + } return admission.NewForbidden(a, err) } } @@ -357,8 +361,9 @@ func (s *serviceAccount) mountServiceAccountToken(serviceAccount *api.ServiceAcc // We don't have an API token to mount, so return if s.RequireAPIToken { // If a token is required, this is considered an error - // TODO: convert to a ServerTimeout error (or other error that sends a Retry-After header) - return fmt.Errorf("no API token found for service account %s/%s, retry after the token is automatically created and added to the service account", serviceAccount.Namespace, serviceAccount.Name) + err := errors.NewServerTimeout(unversioned.GroupResource{Resource: "serviceaccounts"}, "create pod", 1) + err.ErrStatus.Message = fmt.Sprintf("No API token found for service account %q, retry after the token is automatically created and added to the service account", serviceAccount.Name) + return err } return nil } diff --git a/plugin/pkg/admission/serviceaccount/admission_test.go b/plugin/pkg/admission/serviceaccount/admission_test.go index f6d9addbbf119..9640b1fd789c8 100644 --- a/plugin/pkg/admission/serviceaccount/admission_test.go +++ b/plugin/pkg/admission/serviceaccount/admission_test.go @@ -23,6 +23,7 @@ import ( "k8s.io/kubernetes/pkg/admission" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" kubelet "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/pkg/types" @@ -168,8 +169,8 @@ func TestAssignsDefaultServiceAccountAndRejectsMissingAPIToken(t *testing.T) { pod := &api.Pod{} attrs := admission.NewAttributesRecord(pod, api.Kind("Pod").WithVersion("version"), ns, "myname", api.Resource("pods").WithVersion("version"), "", admission.Create, nil) err := admit.Admit(attrs) - if err == nil { - t.Errorf("Expected admission error for missing API token") + if err == nil || !errors.IsServerTimeout(err) { + t.Errorf("Expected server timeout error for missing API token: %v", err) } } From a8ce27709c217b89ac7263e0fa62c8a1ec0abfe4 Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Mon, 9 May 2016 11:54:43 -0400 Subject: [PATCH 3/4] Print more data about an error for debugging Sometimes clients send unintelligible data to the server, provide a bit more debugging in the returned error to make it easier to pin down where the problem is from the user side. --- pkg/apiserver/resthandler.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/pkg/apiserver/resthandler.go b/pkg/apiserver/resthandler.go index 8bf9f4e7d2ca2..4a1e970067674 100644 --- a/pkg/apiserver/resthandler.go +++ b/pkg/apiserver/resthandler.go @@ -17,6 +17,7 @@ limitations under the License. package apiserver import ( + "encoding/hex" "encoding/json" "fmt" "math/rand" @@ -376,7 +377,7 @@ func createHandler(r rest.NamedCreater, scope RequestScope, typer runtime.Object trace.Step("About to convert to expected version") obj, gvk, err := decoder.Decode(body, &defaultGVK, original) if err != nil { - err = transformDecodeError(typer, err, original, gvk) + err = transformDecodeError(typer, err, original, gvk, body) scope.err(err, res.ResponseWriter, req.Request) return } @@ -650,7 +651,7 @@ func UpdateResource(r rest.Updater, scope RequestScope, typer runtime.ObjectType trace.Step("About to convert to expected version") obj, gvk, err := scope.Serializer.DecoderToVersion(s, defaultGVK.GroupVersion()).Decode(body, &defaultGVK, original) if err != nil { - err = transformDecodeError(typer, err, original, gvk) + err = transformDecodeError(typer, err, original, gvk, body) scope.err(err, res.ResponseWriter, req.Request) return } @@ -938,7 +939,7 @@ func finishRequest(timeout time.Duration, fn resultFunc) (result runtime.Object, } // transformDecodeError adds additional information when a decode fails. -func transformDecodeError(typer runtime.ObjectTyper, baseErr error, into runtime.Object, gvk *unversioned.GroupVersionKind) error { +func transformDecodeError(typer runtime.ObjectTyper, baseErr error, into runtime.Object, gvk *unversioned.GroupVersionKind, body []byte) error { objGVK, err := typer.ObjectKind(into) if err != nil { return err @@ -946,7 +947,8 @@ func transformDecodeError(typer runtime.ObjectTyper, baseErr error, into runtime if gvk != nil && len(gvk.Kind) > 0 { return errors.NewBadRequest(fmt.Sprintf("%s in version %q cannot be handled as a %s: %v", gvk.Kind, gvk.Version, objGVK.Kind, baseErr)) } - return errors.NewBadRequest(fmt.Sprintf("the object provided is unrecognized (must be of type %s): %v", objGVK.Kind, baseErr)) + summary := summarizeData(body, 30) + return errors.NewBadRequest(fmt.Sprintf("the object provided is unrecognized (must be of type %s): %v (%s)", objGVK.Kind, baseErr, summary)) } // setSelfLink sets the self link of an object (or the child items in a list) to the base URL of the request @@ -1038,3 +1040,20 @@ func getPatchedJS(patchType api.PatchType, originalJS, patchJS []byte, obj runti return nil, fmt.Errorf("unknown Content-Type header for patch: %v", patchType) } } + +func summarizeData(data []byte, maxLength int) string { + switch { + case len(data) == 0: + return "" + case data[0] == '{': + if len(data) > maxLength { + return string(data[:maxLength]) + " ..." + } + return string(data) + default: + if len(data) > maxLength { + return hex.EncodeToString(data[:maxLength]) + " ..." + } + return hex.EncodeToString(data) + } +} From eeb04e6826db551386c8424160560a3e4447461c Mon Sep 17 00:00:00 2001 From: Clayton Coleman Date: Sat, 14 May 2016 14:06:57 -0400 Subject: [PATCH 4/4] Reset input buffer on retry Retries were previously sending empty bodies to the server. --- pkg/client/restclient/request.go | 15 ++++++++++++--- pkg/client/restclient/request_test.go | 7 +++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/client/restclient/request.go b/pkg/client/restclient/request.go index a063f0898d88f..9fd3f0ddb0917 100644 --- a/pkg/client/restclient/request.go +++ b/pkg/client/restclient/request.go @@ -539,10 +539,10 @@ func (r *Request) Body(obj interface{}) *Request { return r } glog.V(8).Infof("Request Body: %s", string(data)) - r.body = bytes.NewBuffer(data) + r.body = bytes.NewReader(data) case []byte: glog.V(8).Infof("Request Body: %s", string(t)) - r.body = bytes.NewBuffer(t) + r.body = bytes.NewReader(t) case io.Reader: r.body = t case runtime.Object: @@ -556,7 +556,7 @@ func (r *Request) Body(obj interface{}) *Request { return r } glog.V(8).Infof("Request Body: %s", string(data)) - r.body = bytes.NewBuffer(data) + r.body = bytes.NewReader(data) r.SetHeader("Content-Type", r.content.ContentType) default: r.err = fmt.Errorf("unknown type used for body: %+v", obj) @@ -823,6 +823,15 @@ func (r *Request) request(fn func(*http.Request, *http.Response)) error { retries++ if seconds, wait := checkWait(resp); wait && retries < maxRetries { + if seeker, ok := r.body.(io.Seeker); ok && r.body != nil { + _, err := seeker.Seek(0, 0) + if err != nil { + glog.V(4).Infof("Could not retry request, can't Seek() back to beginning of body for %T", r.body) + fn(req, resp) + return true + } + } + glog.V(4).Infof("Got a Retry-After %s response for attempt %d to %v", seconds, retries, url) r.backoffMgr.Sleep(time.Duration(seconds) * time.Second) return false diff --git a/pkg/client/restclient/request_test.go b/pkg/client/restclient/request_test.go index 236d9eef8bace..e770a1ca083df 100644 --- a/pkg/client/restclient/request_test.go +++ b/pkg/client/restclient/request_test.go @@ -874,6 +874,13 @@ func TestCheckRetryHandles429And5xx(t *testing.T) { count := 0 ch := make(chan struct{}) testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + data, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Fatalf("unable to read request body: %v", err) + } + if !bytes.Equal(data, []byte(strings.Repeat("abcd", 1000))) { + t.Fatalf("retry did not send a complete body: %s", data) + } t.Logf("attempt %d", count) if count >= 4 { w.WriteHeader(http.StatusOK)