From d3b60543e7df924881854108984593aafb557d3c Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 19 Apr 2024 18:00:51 +0000 Subject: [PATCH] Fix 10 second etcd-snapshot request timeout The default clientaccess request timeout is too short. Wait longer by default, and add the s3 timeout if s3 is enabled. Signed-off-by: Brad Davidson --- pkg/cli/etcdsnapshot/etcd_snapshot.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index 93cb66c16a90..e4e880243c26 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -2,6 +2,7 @@ package etcdsnapshot import ( "bytes" + "context" "encoding/json" "fmt" "os" @@ -26,6 +27,8 @@ import ( "k8s.io/cli-runtime/pkg/printers" ) +var timeout = 2 * time.Minute + // commandSetup setups up common things needed // for each etcd command. func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *clientaccess.Info, error) { @@ -58,6 +61,8 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c sr.S3.SecretKey = cfg.EtcdS3SecretKey sr.S3.SkipSSLVerify = cfg.EtcdS3SkipSSLVerify sr.S3.Timeout = metav1.Duration{Duration: cfg.EtcdS3Timeout} + // extend request timeout to allow the S3 operation to complete + timeout += cfg.EtcdS3Timeout } dataDir, err := server.ResolveDataDir(cfg.DataDir) @@ -78,6 +83,11 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c } func wrapServerError(err error) error { + if errors.Is(err, context.DeadlineExceeded) { + // if the request timed out the server log likely won't contain anything useful, + // since the operation may have actualy succeeded despite the client timing out the request. + return err + } return errors.Wrap(err, "see server log for details") } @@ -110,7 +120,7 @@ func save(app *cli.Context, cfg *cmds.Server) error { if err != nil { return err } - r, err := info.Post("/db/snapshot", b) + r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout)) if err != nil { return wrapServerError(err) } @@ -151,7 +161,7 @@ func delete(app *cli.Context, cfg *cmds.Server) error { if err != nil { return err } - r, err := info.Post("/db/snapshot", b) + r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout)) if err != nil { return wrapServerError(err) } @@ -206,7 +216,7 @@ func list(app *cli.Context, cfg *cmds.Server) error { if err != nil { return err } - r, err := info.Post("/db/snapshot", b) + r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout)) if err != nil { return wrapServerError(err) } @@ -269,7 +279,7 @@ func prune(app *cli.Context, cfg *cmds.Server) error { if err != nil { return err } - r, err := info.Post("/db/snapshot", b) + r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout)) if err != nil { return wrapServerError(err) }