From 0a53ffd67fd298e37faf2825fd4ddb7803f75f47 Mon Sep 17 00:00:00 2001 From: Gabriel Handford Date: Sat, 6 Mar 2021 12:35:01 -0800 Subject: [PATCH] http: Client#Request changes; JSON --- http/client.go | 90 +++++++++++++++++++++++++++------------- http/client_test.go | 1 + http/github_test.go | 2 +- http/reddit_test.go | 2 +- http/request.go | 63 +++++++++++++++++++++++++--- user/services/github.go | 10 ++--- user/services/reddit.go | 4 +- user/services/request.go | 7 +++- users/github_test.go | 6 +-- users/reddit_test.go | 2 +- users/search_test.go | 20 ++++----- users/twitter_test.go | 8 ++-- users/users_test.go | 8 ++-- 13 files changed, 154 insertions(+), 69 deletions(-) create mode 100644 http/client_test.go diff --git a/http/client.go b/http/client.go index 1902d91..46a93a8 100644 --- a/http/client.go +++ b/http/client.go @@ -4,6 +4,7 @@ package http import ( "context" + "encoding/json" "fmt" "io/ioutil" "net" @@ -11,6 +12,7 @@ import ( "net/url" "strings" "time" + "unicode/utf8" ) // ErrTimeout is a timeout error. @@ -18,13 +20,17 @@ type ErrTimeout struct { error } -// Error is an HTTP Error. -type Error struct { - StatusCode int +// Err is an HTTP Error. +type Err struct { + Code int + Message string } -func (e Error) Error() string { - return fmt.Sprintf("http error %d", e.StatusCode) +func (e Err) Error() string { + if e.Message != "" { + return e.Message + } + return fmt.Sprintf("http error %d", e.Code) } func httpClient() *http.Client { @@ -55,13 +61,28 @@ func httpClient() *http.Client { return client } -func doRequest(client *http.Client, req *Request, headers []Header, options ...func(*http.Request)) (http.Header, []byte, error) { - logger.Debugf("Requesting %s %s", req.Method, req.URL) +// JSON request. +func JSON(req *Request, v interface{}) error { + hcl := &http.Client{ + Timeout: time.Second * 30, + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 10 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + }, + } - req.Header.Set("User-Agent", "keys.pub") - for _, header := range headers { - req.Header.Set(header.Name, header.Value) + b, err := doRequest(hcl, req) + if err != nil { + return err } + return json.Unmarshal(b, v) +} + +// Do HTTP request. +func doRequest(client *http.Client, req *Request, options ...func(*http.Request)) ([]byte, error) { + logger.Debugf("Requesting %s %s", req.Method, req.URL) for _, opt := range options { opt(req) @@ -70,7 +91,7 @@ func doRequest(client *http.Client, req *Request, headers []Header, options ...f resp, err := client.Do(req) switch err := err.(type) { default: - return nil, nil, err + return nil, err case nil: // no error @@ -79,33 +100,44 @@ func doRequest(client *http.Client, req *Request, headers []Header, options ...f // when exceeding it's `Transport`'s `ResponseHeadersTimeout` e1, ok := err.Err.(net.Error) if ok && e1.Timeout() { - return nil, nil, ErrTimeout{err} + return nil, ErrTimeout{err} } - return nil, nil, err + return nil, err case net.Error: // `http.Client.Do` will return a `net.Error` directly when Dial times // out, or when the Client's RoundTripper otherwise returns an err if err.Timeout() { - return nil, nil, ErrTimeout{err} + return nil, ErrTimeout{err} } - return nil, nil, err + return nil, err } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + logger.Debugf("Response body (len=%d)", len(body)) + defer resp.Body.Close() if resp.StatusCode/200 != 1 { - return resp.Header, nil, Error{StatusCode: resp.StatusCode} - } + var errMsg string + if len(body) > 1024 { + body = body[0:1024] + } + if utf8.Valid(body) { + errMsg = string(body) + } - respBody, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, nil, err + return nil, Err{ + Code: resp.StatusCode, + Message: errMsg, + } } - logger.Debugf("Response body (len=%d)", len(respBody)) - return resp.Header, respBody, nil + return body, nil } // ErrTemporary means there was a temporary error @@ -135,7 +167,7 @@ type Header struct { // Client for HTTP. type Client interface { - Request(ctx context.Context, req *Request, headers []Header) ([]byte, error) + Request(ctx context.Context, req *Request) ([]byte, error) SetProxy(urs string, fn ProxyFn) } @@ -149,7 +181,7 @@ func NewClient() Client { } // ProxyFn for proxy. -type ProxyFn func(ctx context.Context, req *Request, headers []Header) ProxyResponse +type ProxyFn func(ctx context.Context, req *Request) ProxyResponse // ProxyResponse ... type ProxyResponse struct { @@ -167,24 +199,26 @@ func (c *client) SetProxy(urs string, fn ProxyFn) { } // Request an URL. -func (c *client) Request(ctx context.Context, req *Request, headers []Header) ([]byte, error) { +func (c *client) Request(ctx context.Context, req *Request) ([]byte, error) { if c.proxies != nil { fn := c.proxies[req.URL.String()] if fn != nil { - pr := fn(ctx, req, headers) + pr := fn(ctx, req) if !pr.Skip { return pr.Body, pr.Err } } fn = c.proxies[""] if fn != nil { - pr := fn(ctx, req, headers) + pr := fn(ctx, req) if !pr.Skip { return pr.Body, pr.Err } } } - _, body, err := doRequest(httpClient(), req, headers) + + req.Header.Set("User-Agent", "keys.pub") + body, err := doRequest(httpClient(), req) if err != nil { logger.Warningf("Failed request: %s", err) } diff --git a/http/client_test.go b/http/client_test.go new file mode 100644 index 0000000..c5a0068 --- /dev/null +++ b/http/client_test.go @@ -0,0 +1 @@ +package http_test diff --git a/http/github_test.go b/http/github_test.go index 05b37bb..38aa589 100644 --- a/http/github_test.go +++ b/http/github_test.go @@ -14,7 +14,7 @@ func TestGithubRequest(t *testing.T) { urs := "https://gist.github.com/gabriel/ceea0f3b675bac03425472692273cf52" req, err := http.NewRequest("GET", urs, nil) require.NoError(t, err) - res, err := client.Request(context.TODO(), req, nil) + res, err := client.Request(context.TODO(), req) require.NoError(t, err) out, brand := encoding.FindSaltpack(string(res), true) diff --git a/http/reddit_test.go b/http/reddit_test.go index ce8adfc..8c7b483 100644 --- a/http/reddit_test.go +++ b/http/reddit_test.go @@ -14,7 +14,7 @@ func TestReddit(t *testing.T) { urs := "https://old.reddit.com/r/keyspubmsgs/comments/f8g9vd/gabrlh.json" req, err := http.NewRequest("GET", urs, nil) require.NoError(t, err) - res, err := client.Request(context.TODO(), req, nil) + res, err := client.Request(context.TODO(), req) require.NoError(t, err) var red reddit diff --git a/http/request.go b/http/request.go index 377cf41..7b76bd6 100644 --- a/http/request.go +++ b/http/request.go @@ -1,6 +1,8 @@ package http import ( + "bytes" + "encoding/json" "io" "net/http" "time" @@ -14,13 +16,13 @@ var NewRequest = http.NewRequest // NewRequestWithContext alias. var NewRequestWithContext = http.NewRequestWithContext -// NewAuthRequest returns new authorized/signed HTTP request from keys. -func NewAuthRequest(method string, urs string, body io.Reader, contentHash string, tm time.Time, auth *keys.EdX25519Key) (*http.Request, error) { - return newRequest(method, urs, body, contentHash, tm, keys.RandBytes(24), auth) +// NewAuthRequest returns new authorized/signed HTTP request using auth key. +func NewAuthRequest(method string, urs string, body io.Reader, contentHash string, ts time.Time, key *keys.EdX25519Key) (*http.Request, error) { + return newRequest(method, urs, body, contentHash, ts, keys.RandBytes(24), key) } -func newRequest(method string, urs string, body io.Reader, contentHash string, tm time.Time, nonce []byte, auth *keys.EdX25519Key) (*http.Request, error) { - ur, err := authURL(urs, tm, nonce) +func newRequest(method string, urs string, body io.Reader, contentHash string, ts time.Time, nonce []byte, key *keys.EdX25519Key) (*http.Request, error) { + ur, err := authURL(urs, ts, nonce) if err != nil { return nil, err } @@ -28,10 +30,59 @@ func newRequest(method string, urs string, body io.Reader, contentHash string, t if err != nil { return nil, err } - a, err := newAuthWithURL(method, ur, contentHash, auth) + a, err := newAuthWithURL(method, ur, contentHash, key) if err != nil { return nil, err } req.Header.Set("Authorization", a.Header()) return req, nil } + +// NewJSONRequest ... +func NewJSONRequest(method string, urs string, i interface{}, opt ...RequestOption) (*http.Request, error) { + opts := NewRequestOptions(opt...) + b, err := json.Marshal(i) + if err != nil { + return nil, err + } + if opts.Key != nil { + ts := opts.Timestamp + if ts.IsZero() { + ts = time.Now() + } + return NewAuthRequest(method, urs, bytes.NewReader(b), ContentHash(b), ts, opts.Key) + } + return NewRequest(method, urs, bytes.NewReader(b)) +} + +// RequestOptions ... +type RequestOptions struct { + Timestamp time.Time + Key *keys.EdX25519Key +} + +// RequestOption ... +type RequestOption func(*RequestOptions) + +// NewRequestOptions parses RequestOption. +func NewRequestOptions(opts ...RequestOption) RequestOptions { + var options RequestOptions + for _, o := range opts { + o(&options) + } + return options +} + +// WithTimestamp to overwride timestamp. +func WithTimestamp(ts time.Time) RequestOption { + return func(o *RequestOptions) { + o.Timestamp = ts + } +} + +// SignedWith key. +func SignedWith(key *keys.EdX25519Key) RequestOption { + return func(o *RequestOptions) { + o.Key = key + } +} diff --git a/user/services/github.go b/user/services/github.go index 7777e7a..1d9ebd9 100644 --- a/user/services/github.go +++ b/user/services/github.go @@ -55,12 +55,10 @@ func (s *github) Verify(ctx context.Context, b []byte, usr *user.User) (user.Sta } func (s *github) headers() []http.Header { - return []http.Header{ - { - Name: "Accept", - Value: "application/vnd.github.v3+json", - }, - } + return []http.Header{{ + Name: "Accept", + Value: "application/vnd.github.v3+json", + }} } type file struct { diff --git a/user/services/reddit.go b/user/services/reddit.go index e2d28c7..6bb283b 100644 --- a/user/services/reddit.go +++ b/user/services/reddit.go @@ -79,9 +79,7 @@ func (s *reddit) headers(urs string) ([]http.Header, error) { } // Not sure if this is required anymore. if strings.HasSuffix(ur.Host, ".reddit.com") { - return []http.Header{ - {Name: "Host", Value: "reddit.com"}, - }, nil + return []http.Header{{Name: "Host", Value: "reddit.com"}}, nil } return nil, nil } diff --git a/user/services/request.go b/user/services/request.go index 07f1db1..2324773 100644 --- a/user/services/request.go +++ b/user/services/request.go @@ -15,9 +15,12 @@ func Request(ctx context.Context, client http.Client, urs string, headers []http if err != nil { return user.StatusFailure, nil, err } - b, err := client.Request(ctx, req, headers) + for _, h := range headers { + req.Header.Set(h.Name, h.Value) + } + b, err := client.Request(ctx, req) if err != nil { - if errHTTP, ok := errors.Cause(err).(http.Error); ok && errHTTP.StatusCode == 404 { + if errHTTP, ok := errors.Cause(err).(http.Err); ok && errHTTP.Code == 404 { return user.StatusResourceNotFound, nil, errors.Errorf("resource not found") } return user.StatusConnFailure, nil, err diff --git a/users/github_test.go b/users/github_test.go index b71543a..b4f2933 100644 --- a/users/github_test.go +++ b/users/github_test.go @@ -43,7 +43,7 @@ func TestResultGithub(t *testing.T) { _, err = user.NewSigchainStatement(sc, stu, sk, clock.Now()) require.EqualError(t, err, "user set in sigchain already") - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(githubMock("alice", "1", msg))} }) @@ -94,7 +94,7 @@ func TestResultGithubWrongName(t *testing.T) { sc := keys.NewSigchain(sk.ID()) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(githubMock("alice", "1", msg))} }) @@ -127,7 +127,7 @@ func TestResultGithubWrongService(t *testing.T) { msg, err := invalid.Sign(sk) require.NoError(t, err) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(githubMock("alice", "1", msg))} }) diff --git a/users/reddit_test.go b/users/reddit_test.go index 6b3346b..fec5a4c 100644 --- a/users/reddit_test.go +++ b/users/reddit_test.go @@ -40,7 +40,7 @@ func TestResultReddit(t *testing.T) { _, err = user.NewSigchainStatement(sc, stu, sk, clock.Now()) require.EqualError(t, err, "user set in sigchain already") - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: testdata(t, "testdata/reddit/charlie.json")} }) diff --git a/users/search_test.go b/users/search_test.go index c3c2256..ce776ba 100644 --- a/users/search_test.go +++ b/users/search_test.go @@ -278,7 +278,7 @@ func TestUserValidateUpdateInvalid(t *testing.T) { } smsg, err := usr.Sign(key) require.NoError(t, err) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(redditMock("Testing", smsg, "keyspubmsgs"))} }) @@ -337,7 +337,7 @@ func TestReddit(t *testing.T) { smsg, err := usr.Sign(key) require.NoError(t, err) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(redditMock("alice", smsg, "keyspubmsgs"))} }) @@ -346,7 +346,7 @@ func TestReddit(t *testing.T) { require.Equal(t, user.StatusOK, result.Status) // Different name - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(redditMock("alice2", smsg, "keyspubmsgs"))} }) result, err = usrs.Update(ctx, key.ID()) @@ -354,7 +354,7 @@ func TestReddit(t *testing.T) { require.Equal(t, user.StatusContentInvalid, result.Status) // Different subreddit - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(redditMock("alice", smsg, "keyspubmsgs2"))} }) result, err = usrs.Update(ctx, key.ID()) @@ -393,8 +393,8 @@ func TestSearchUsersRequestErrors(t *testing.T) { require.Equal(t, int64(1234567890004), results[0].Result.VerifiedAt) // Set 500 error for alice@github - usrs.Client().SetProxy(aliceUser.URL, func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { - return http.ProxyResponse{Err: http.Error{StatusCode: 500}} + usrs.Client().SetProxy(aliceUser.URL, func(ctx context.Context, req *http.Request) http.ProxyResponse { + return http.ProxyResponse{Err: http.Err{Code: 500}} }) _, err = usrs.Update(ctx, alice.ID()) require.NoError(t, err) @@ -424,8 +424,8 @@ func TestSearchUsersRequestErrors(t *testing.T) { require.Equal(t, keys.ID("kex132yw8ht5p8cetl2jmvknewjawt9xwzdlrk2pyxlnwjyqrdq0dawqqph077"), fail[0]) // Set 404 error for alice@github - usrs.Client().SetProxy(aliceUser.URL, func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { - return http.ProxyResponse{Err: http.Error{StatusCode: 404}} + usrs.Client().SetProxy(aliceUser.URL, func(ctx context.Context, req *http.Request) http.ProxyResponse { + return http.ProxyResponse{Err: http.Err{Code: 404}} }) res, err := usrs.Update(ctx, alice.ID()) require.NoError(t, err) @@ -437,7 +437,7 @@ func TestSearchUsersRequestErrors(t *testing.T) { require.Equal(t, 0, len(results)) // Reset proxy - usrs.Client().SetProxy(aliceUser.URL, func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy(aliceUser.URL, func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(aliceUser.Response)} }) _, err = usrs.Update(ctx, alice.ID()) @@ -570,7 +570,7 @@ func saveUser(users *users.Users, scs *keys.Sigchains, key *keys.EdX25519Key, na resp = githubMock(name, id, msg) } - client.SetProxy(murl, func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + client.SetProxy(murl, func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(resp)} }) diff --git a/users/twitter_test.go b/users/twitter_test.go index 1ff6d8b..5cf7beb 100644 --- a/users/twitter_test.go +++ b/users/twitter_test.go @@ -43,7 +43,7 @@ func TestResultTwitter(t *testing.T) { require.NoError(t, err) // Set error response - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Err: errors.Errorf("testing")} }) @@ -62,7 +62,7 @@ func TestResultTwitter(t *testing.T) { require.EqualError(t, err, "user set in sigchain already") // Set valid response - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: testdata(t, "testdata/twitter/1205589994380783616.json")} }) @@ -77,7 +77,7 @@ func TestResultTwitter(t *testing.T) { require.Equal(t, int64(1234567890004), result.Timestamp) // Set error response again - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Err: errors.Errorf("testing2")} }) @@ -136,7 +136,7 @@ func TestResultTwitterInvalidStatement(t *testing.T) { err = scs.Save(sc) require.NoError(t, err) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: testdata(t, "testdata/twitter/1205589994380783616.json")} }) diff --git a/users/users_test.go b/users/users_test.go index b27f112..d450c30 100644 --- a/users/users_test.go +++ b/users/users_test.go @@ -95,7 +95,7 @@ func TestSigchainUsersUpdate(t *testing.T) { usrs := users.New(ds, scs, users.Clock(clock)) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(testdata(t, "testdata/twitter/1222706272849391616.json"))} }) @@ -133,7 +133,7 @@ func TestSigchainRevokeUpdate(t *testing.T) { err = sc.Add(st) require.NoError(t, err) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(twitterMock("gabriel", "1", msg))} }) @@ -159,7 +159,7 @@ func TestSigchainRevokeUpdate(t *testing.T) { err = sc.Add(st2) require.NoError(t, err) - usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + usrs.Client().SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { return http.ProxyResponse{Body: []byte(twitterMock("gabriel", "2", msg))} }) @@ -236,7 +236,7 @@ func mockStatement(key *keys.EdX25519Key, sc *keys.Sigchain, name string, servic return nil, err } - client.SetProxy("", func(ctx context.Context, req *http.Request, headers []http.Header) http.ProxyResponse { + client.SetProxy("", func(ctx context.Context, req *http.Request) http.ProxyResponse { // TODO: Set based on url return http.ProxyResponse{Body: []byte(msg)} })