Merge pull request #2 from iarsham/develop

add:set headers & update:transport skip verify

headers.go

@@ -0,0 +1,32 @@
+package scrapify
+
+import "net/http"
+
+var defaultHeaders = map[string]string{
+	"Accept":                    "*/*",
+	"Accept-Encoding":           "gzip, deflate, br",
+	"Accept-Language":           "en-US,en;q=0.9",
+	"Cache-Control":             "no-cache",
+	"Connection":                "keep-alive",
+	"User-Agent":                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0",
+	"Upgrade-Insecure-Requests": "1",
+	"sec-ch-ua":                 `"Not)A;Brand";v="99", "Google Chrome";v="127", "Chromium";v="127"`,
+	"sec-ch-ua-mobile":          "?0",
+	"sec-ch-ua-platform":        "Windows",
+	"sec-fetch-user":            "?1",
+	"sec-fetch-dest":            "empty",
+	"sec-fetch-mode":            "cors",
+	"sec-fetch-site":            "same-origin",
+	"Priority":                  "u=0, i",
+}
+
+func SetHeaders(req *http.Request, headers M) {
+	for k, v := range defaultHeaders {
+		req.Header.Set(k, v)
+	}
+	if headers != nil {
+		for k, v := range headers {
+			req.Header.Set(k, v)
+		}
+	}
+}

transport.go

@@ -5,12 +5,12 @@ import (
 	"net/http"
 )
 
-func NewTransport(browser Browser, debug bool) http.RoundTripper {
+func NewTransport(browser Browser) http.RoundTripper {
 	tlsConfig := &tls.Config{
 		MinVersion:         tls.VersionTLS12,
 		MaxVersion:         tls.VersionTLS13,
 		CipherSuites:       MapBrowserToCipher[browser],
-		InsecureSkipVerify: debug,
+		InsecureSkipVerify: true,
 	}
 	return &http.Transport{
 		TLSClientConfig: tlsConfig,

utils.go

@@ -0,0 +1,3 @@
+package scrapify
+
+type M map[string]string