diff --git a/demo/httpreq/curl_test.go b/demo/httpreq/post-file_test.go similarity index 71% rename from demo/httpreq/curl_test.go rename to demo/httpreq/post-file_test.go index 8498c0c..bc9474a 100644 --- a/demo/httpreq/curl_test.go +++ b/demo/httpreq/post-file_test.go @@ -7,15 +7,15 @@ import ( "github.com/ahuigo/gohttptool/httpreq" ) -func TestCurl(t *testing.T) { +func TestPostFile(t *testing.T) { curl, err := httpreq.R(). - SetParams(map[string]string{"p": "1"}). - SetData(map[string]string{"key": "xx"}). + SetQueryParams(map[string]string{"p": "1"}). + SetFormData(map[string]string{"key": "xx"}). SetAuthBasic("user", "pass"). SetHeader("header1", "value1"). AddCookieKV("count", "1"). AddFileHeader("file", "test.txt", []byte("hello world")). - AddFileHeader("file2", "test.txt", []byte("hello world2")). + AddFile("file2", getTestDataPath("a.txt")). SetReq("GET", "/path"). ToCurl() if err != nil { diff --git a/demo/httpreq/post_test.go b/demo/httpreq/post_test.go new file mode 100644 index 0000000..3dd4783 --- /dev/null +++ b/demo/httpreq/post_test.go @@ -0,0 +1,103 @@ +package httpreq + +import ( + "strings" + "testing" + + "github.com/ahuigo/gohttptool/httpreq" +) + +type MapString = map[string]string + +// POST: curl -X POST -d ” 'http://local/post?p=1' +func TestPostParams(t *testing.T) { + curl, err := httpreq.R(). + SetQueryParams(map[string]string{"p": "1"}). + SetReq("POST", "http://local/post"). + ToCurl() + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(curl, "curl ") || + !strings.Contains(curl, "?p=1") { + t.Fatal("bad curl: ", curl) + } else { + t.Log(curl) + } +} + +// POST: curl -X POST -H 'Content-Type: application/json' -d '{"name":"ahuigo"}' http://localhost/path +func TestPostJson(t *testing.T) { + curl, err := httpreq.R(). + SetJson(map[string]string{"name": "ahuigo"}). + SetReq("POST", "/path"). + ToCurl() + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(curl, "curl ") || + !strings.Contains(curl, `'Content-Type: application/json`) || + !strings.Contains(curl, `{"name":"ahuigo"}`) { + t.Fatal("bad curl: ", curl) + } else { + t.Log(curl) + } +} + +// Post Data: curl -H 'Content-Type: application/x-www-form-urlencoded' http://local/post -d 'name=Alex' +func TestPostFormUrlEncode(t *testing.T) { + curl, err := httpreq.R(). + SetFormData(map[string]string{"name": "Alex"}). + SetReq("POST", "http://local/post"). + ToCurl() + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(curl, "curl ") || + !strings.Contains(curl, `'Content-Type: application/x-www-form-urlencode`) || + !strings.Contains(curl, "name=Alex") { + t.Fatal("bad curl: ", curl) + } else { + t.Log(curl) + } + +} + +// POST FormData: multipart/form-data; boundary=.... +// curl http://local/post -F 'name=Alex' +func TestPostFormMultipart(t *testing.T) { + curl, err := httpreq.R(). + SetIsMultiPart(true). + SetFormData(map[string]string{"name": "Alex"}). + SetReq("POST", "http://local/post"). + ToCurl() + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(curl, "curl ") || + !strings.Contains(curl, `'Content-Type: multipart/form-data`) || + !strings.Contains(curl, `name="name"`) { + t.Fatal("bad curl: ", curl) + } else { + println(curl) + } +} + +// POST: curl -X POST -H 'Content-Type: text/plain' -d 'hello!' http://local/post +func TestPostPlainData(t *testing.T) { + curl, err := httpreq.R(). + SetContentType(httpreq.ContentTypePlain). + SetBody([]byte("hello!")). + SetReq("POST", "http://local/post"). + ToCurl() + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(curl, "curl ") || + !strings.Contains(curl, `'Content-Type: text/plain`) || + !strings.Contains(curl, `-d 'hello!'`) { + t.Fatal("bad curl: ", curl) + } else { + println(curl) + } +} diff --git a/demo/httpreq/testdata/a.txt b/demo/httpreq/testdata/a.txt new file mode 100644 index 0000000..a3828ba --- /dev/null +++ b/demo/httpreq/testdata/a.txt @@ -0,0 +1,2 @@ + +I'm a.txt \ No newline at end of file diff --git a/demo/httpreq/util_test.go b/demo/httpreq/util_test.go new file mode 100644 index 0000000..23b1eae --- /dev/null +++ b/demo/httpreq/util_test.go @@ -0,0 +1,11 @@ +package httpreq + +import ( + "os" + "path/filepath" +) + +func getTestDataPath(filename string) string { + pwd, _ := os.Getwd() + return filepath.Join(pwd, "./testdata", filename) +} diff --git a/go.mod b/go.mod index 517ba10..81f020e 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/ahuigo/gohttptool -go 1.22.1 +go 1.21.1 require github.com/pkg/errors v0.9.1 diff --git a/httpreq/curl.go b/httpreq/curl.go index 1cc491c..b9fa9ed 100644 --- a/httpreq/curl.go +++ b/httpreq/curl.go @@ -15,6 +15,7 @@ import ( func (r *RequestBuilder) FromCurl(curl string) { } + func (r *RequestBuilder) ToCurl() (curl string, err error) { if httpreq, err := r.ToRequest(); err != nil { return "", err @@ -24,36 +25,39 @@ func (r *RequestBuilder) ToCurl() (curl string, err error) { } } -func buildCurlRequest(req *http.Request, httpCookiejar http.CookieJar) (curl string) { +func buildCurlRequest(req *http.Request, httpCookiejar http.CookieJar) (curlString string) { + buf := acquireBuffer() + defer releaseBuffer(buf) + // 1. Generate curl raw headers - curl = "curl -X " + req.Method + " " + buf.WriteString("curl -X " + req.Method + " ") // req.Host + req.URL.Path + "?" + req.URL.RawQuery + " " + req.Proto + " " headers := dumpCurlHeaders(req) for _, kv := range *headers { - curl += `-H ` + shell.Quote(kv[0]+": "+kv[1]) + ` ` + buf.WriteString(`-H ` + shell.Quote(kv[0]+": "+kv[1]) + ` `) } // 2. Generate curl cookies if cookieJar, ok := httpCookiejar.(*cookiejar.Jar); ok { cookies := cookieJar.Cookies(req.URL) if len(cookies) > 0 { - curl += ` -H ` + shell.Quote(dumpCurlCookies(cookies)) + " " + buf.WriteString(` -H ` + shell.Quote(dumpCurlCookies(cookies)) + " ") } } // 3. Generate curl body if req.Body != nil { - buf, _ := io.ReadAll(req.Body) - req.Body = io.NopCloser(bytes.NewBuffer(buf)) // important!! - curl += `-d ` + shell.Quote(string(buf)) + buf2, _ := io.ReadAll(req.Body) + req.Body = io.NopCloser(bytes.NewBuffer(buf2)) // important!! + buf.WriteString(`-d ` + shell.Quote(string(buf2))) } urlString := shell.Quote(req.URL.String()) if urlString == "''" { urlString = "'http://unexecuted-request'" } - curl += " " + urlString - return curl + buf.WriteString(" " + urlString) + return buf.String() } // dumpCurlCookies dumps cookies to curl format @@ -64,6 +68,7 @@ func dumpCurlCookies(cookies []*http.Cookie) string { sb.WriteString(cookie.Name + "=" + url.QueryEscape(cookie.Value) + "&") } return strings.TrimRight(sb.String(), "&") + } // dumpCurlHeaders dumps headers to curl format diff --git a/httpreq/req-builder.go b/httpreq/req-builder.go index 9f38527..192a527 100644 --- a/httpreq/req-builder.go +++ b/httpreq/req-builder.go @@ -16,8 +16,15 @@ import ( func (rb *RequestBuilder) ToRequest() (*http.Request, error) { var dataType = ContentType(rb.rawreq.Header.Get("Content-Type")) var origurl = rb.url - if len(rb.files) > 0 || len(rb.fileHeaders) > 0 { + if rb.isMultiPart || len(rb.files) > 0 || len(rb.fileHeaders) > 0 { dataType = ContentTypeFormData + } else if rb.json != nil { + dataType = ContentTypeJson + } else if len(rb.formData) > 0 { + dataType = ContentTypeFormEncode + } + if dataType != "" { + rb.rawreq.Header.Set("Content-Type", string(dataType)) } URL, err := rb.buildURLParams(origurl) @@ -30,14 +37,16 @@ func (rb *RequestBuilder) ToRequest() (*http.Request, error) { } switch dataType { + case ContentTypeJson: + rb.setBodyJson() case ContentTypeFormEncode: - if len(rb.datas) > 0 { - formEncodeValues := rb.buildFormEncode(rb.datas) - rb.setBodyFormEncode(formEncodeValues) + if len(rb.formData) > 0 { + rb.setBodyFormEncode(rb.formData) } case ContentTypeFormData: // multipart/form-data rb.buildFilesAndForms() + default: } if rb.rawreq.Body == nil && rb.rawreq.Method != "GET" { @@ -49,25 +58,26 @@ func (rb *RequestBuilder) ToRequest() (*http.Request, error) { return rb.rawreq, nil } -// build post Form encode -func (rb *RequestBuilder) buildFormEncode(datas map[string]string) (Forms url.Values) { - Forms = url.Values{} - for key, value := range datas { - Forms.Add(key, value) - } - return Forms -} - // set form urlencode -func (rb *RequestBuilder) setBodyFormEncode(Forms url.Values) { - data := Forms.Encode() +func (rb *RequestBuilder) setBodyFormEncode(formData url.Values) { + data := formData.Encode() rb.rawreq.Body = io.NopCloser(strings.NewReader(data)) rb.rawreq.ContentLength = int64(len(data)) } +func (rb *RequestBuilder) setBodyJson() { + bodyBuf, err := noescapeJSONMarshalIndent(&rb.json) + if err == nil { + prtBodyBytes := bodyBuf.Bytes() + plen := len(prtBodyBytes) + if plen > 0 && prtBodyBytes[plen-1] == '\n' { + prtBodyBytes = prtBodyBytes[:plen-1] + } + rb.rawreq.Body = io.NopCloser(bytes.NewReader(prtBodyBytes)) + } +} + func (rb *RequestBuilder) buildURLParams(userURL string) (*url.URL, error) { - params := rb.params - paramsArray := rb.paramsList if strings.HasPrefix(userURL, "/") { userURL = "http://localhost" + userURL } else if userURL == "" { @@ -81,13 +91,9 @@ func (rb *RequestBuilder) buildURLParams(userURL string) (*url.URL, error) { values := parsedURL.Query() - for key, value := range params { - values.Set(key, value) - } - for key, vals := range paramsArray { - for _, v := range vals { - values.Add(key, v) - } + for key, value := range rb.queryParam { + values[key] = value + // values.Set(key, value[0]) } parsedURL.RawQuery = values.Encode() return parsedURL, nil @@ -95,14 +101,16 @@ func (rb *RequestBuilder) buildURLParams(userURL string) (*url.URL, error) { func (rb *RequestBuilder) buildFilesAndForms() error { files := rb.files - datas := rb.datas + formData := rb.formData filesHeaders := rb.fileHeaders //handle file multipart var b bytes.Buffer w := multipart.NewWriter(&b) - for k, v := range datas { - w.WriteField(k, v) + for k, v := range formData { + for _, vv := range v { + w.WriteField(k, vv) + } } for field, path := range files { diff --git a/httpreq/req.go b/httpreq/req.go index d422027..644f4c8 100644 --- a/httpreq/req.go +++ b/httpreq/req.go @@ -1,9 +1,12 @@ package httpreq import ( + "bytes" "context" + "io" "net/http" "net/textproto" + "net/url" ) type ContentType string @@ -27,13 +30,15 @@ type fileHeader struct { } type RequestBuilder struct { - rawreq *http.Request - url string + rawreq *http.Request + url string + + queryParam url.Values + formData url.Values + isMultiPart bool + json any files map[string]string // field -> path fileHeaders map[string]fileHeader // field -> contents - datas map[string]string // key -> value - params map[string]string // key -> value - paramsList map[string][]string // key -> value list } func R() *RequestBuilder { @@ -45,11 +50,11 @@ func R() *RequestBuilder { ProtoMajor: 1, ProtoMinor: 1, }, + queryParam: make(url.Values), + formData: make(map[string][]string), + // paramsList: make(map[string][]string), files: make(map[string]string), fileHeaders: make(map[string]fileHeader), - datas: make(map[string]string), - params: make(map[string]string), - paramsList: make(map[string][]string), } } @@ -69,6 +74,11 @@ func (r *RequestBuilder) SetAuthBearer(token string) *RequestBuilder { return r } +func (r *RequestBuilder) SetContentType(ct ContentType) *RequestBuilder { + r.rawreq.Header.Set("Content-Type", string(ct)) + return r +} + func (r *RequestBuilder) AddCookies(cookies []*http.Cookie) *RequestBuilder { for _, cookie := range cookies { r.rawreq.AddCookie(cookie) @@ -84,6 +94,7 @@ func (r *RequestBuilder) AddCookieKV(name, value string) *RequestBuilder { return r } +/************** params **********************/ /************** file **********************/ func (r *RequestBuilder) AddFile(fieldname, path string) *RequestBuilder { r.files[fieldname] = path @@ -110,16 +121,67 @@ func (r *RequestBuilder) SetReq(method string, url string) *RequestBuilder { return r } -func (r *RequestBuilder) SetParams(params map[string]string) *RequestBuilder { - r.params = params +/************** params **********************/ +func (r *RequestBuilder) SetQueryParams(params map[string]string) *RequestBuilder { + for p, v := range params { + r.SetQueryParam(p, v) + } + return r +} +func (r *RequestBuilder) SetQueryParam(param, value string) *RequestBuilder { + r.queryParam.Set(param, value) return r } -func (r *RequestBuilder) SetData(data map[string]string) *RequestBuilder { - r.datas = data +func (r *RequestBuilder) SetQueryParamsFromValues(params url.Values) *RequestBuilder { + for p, v := range params { + for _, pv := range v { + r.queryParam.Add(p, pv) + } + } + return r +} + +/************** body(bytes) **********************/ +func (r *RequestBuilder) SetBody(body []byte) *RequestBuilder { + r.rawreq.Body = io.NopCloser(bytes.NewReader(body)) return r } +/************** body(form) **********************/ +// Set Form data(encode or multipart) +func (r *RequestBuilder) SetIsMultiPart(b bool) *RequestBuilder { + r.isMultiPart = b + return r +} +func (r *RequestBuilder) SetFormData(data map[string]string) *RequestBuilder { + for k, v := range data { + r.formData.Set(k, v) + } + return r +} + +// SetFormDataFromValues method appends multiple form parameters with multi-value +// +// SetFormDataFromValues(url.Values{"words": []string{"book", "glass", "pencil"},}) +func (r *RequestBuilder) SetFormDataFromValues(data url.Values) *RequestBuilder { + for k, v := range data { + for _, kv := range v { + r.formData.Add(k, kv) + } + } + return r +} + +/************** body(json) **********************/ +func (r *RequestBuilder) SetJson(data any) *RequestBuilder { + r.json = data + return r +} + +/************** body(plain) **********************/ + +/************** utils **********************/ func (r *RequestBuilder) GetRawreq() *http.Request { return r.rawreq } diff --git a/httpreq/utils.go b/httpreq/utils.go new file mode 100644 index 0000000..e6a090b --- /dev/null +++ b/httpreq/utils.go @@ -0,0 +1,35 @@ +package httpreq + +import ( + "bytes" + "encoding/json" + "sync" +) + +var ( + bufPool = &sync.Pool{New: func() interface{} { return &bytes.Buffer{} }} +) + +var noescapeJSONMarshalIndent = func(v interface{}) (*bytes.Buffer, error) { + buf := acquireBuffer() + encoder := json.NewEncoder(buf) + encoder.SetEscapeHTML(false) + // encoder.SetIndent("", " ") + + if err := encoder.Encode(v); err != nil { + releaseBuffer(buf) + return nil, err + } + return buf, nil +} + +func acquireBuffer() *bytes.Buffer { + return bufPool.Get().(*bytes.Buffer) +} + +func releaseBuffer(buf *bytes.Buffer) { + if buf != nil { + buf.Reset() + bufPool.Put(buf) + } +} diff --git a/version b/version index 837042c..254a9f2 100644 --- a/version +++ b/version @@ -1 +1 @@ -v0.0.5 \ No newline at end of file +v0.0.6 \ No newline at end of file