redact debug/trace output

Signed-off-by: Jesse Stuart <[email protected]>

audible/audible.go

@@ -88,7 +88,7 @@ func NewClient(opts ...Option) (*Client, error) {
 				return http.ErrUseLastResponse
 			}
 			log.TraceFn(func() []interface{} {
-				return []interface{}{fmt.Sprintf("Redirect: %s", req.URL)}
+				return []interface{}{fmt.Sprintf("Redirect: %s", redactURL(req.URL))}
 			})
 			c.lastURL = req.URL
 			return nil

audible/round_tripper.go

@@ -6,10 +6,14 @@ import (
 	"io"
 	"mime"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 
+	"github.com/antchfx/htmlquery"
+	"github.com/antchfx/xpath"
 	"github.com/jvatic/audible-downloader/internal/config"
 	log "github.com/sirupsen/logrus"
 )
@@ -34,15 +38,15 @@ func (rt *roundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	req.Header.Set("DNT", "1")
 	req.Header.Set("Upgrade-Insecure-Requests", "1")
 
-	log.Debugf("%s %s", req.Method, req.URL)
+	log.Debugf("%s %s", req.Method, redactURL(req.URL))
 	log.TraceFn(logHeader(req.Header, "User-Agent"))
 
 	resp, err := http.DefaultTransport.RoundTrip(req)
 	if err != nil {
 		return resp, err
 	}
 
-	log.Debugf("-> %s: %s", resp.Request.URL, resp.Status)
+	log.Debugf("-> %s: %s", redactURL(resp.Request.URL), resp.Status)
 	log.TraceFn(logHeader(resp.Header, "Content-Type"))
 	log.TraceFn(logResponseBody(resp))
 
@@ -83,7 +87,11 @@ func logResponseBody(resp *http.Response) log.LogFunction {
 		if err != nil {
 			return []interface{}{fmt.Sprintf("error saving response body to %q: %s", p, err)}
 		}
-		io.Copy(file, bytes.NewReader(buf.Bytes()))
+		if strings.HasPrefix(exts[0], ".htm") {
+			io.Copy(file, bytes.NewReader(redactHTML(buf.Bytes())))
+		} else {
+			io.Copy(file, bytes.NewReader(buf.Bytes()))
+		}
 		return []interface{}{fmt.Sprintf("response body saved to %q", p)}
 	}
 }
@@ -93,3 +101,52 @@ func logHeader(h http.Header, name string) log.LogFunction {
 		return []interface{}{fmt.Sprintf("%s: %s", name, h.Get(name))}
 	}
 }
+
+func redactURL(u *url.URL) *url.URL {
+	redacted := &url.URL{
+		Scheme:   u.Scheme,
+		Opaque:   u.Opaque,
+		Host:     u.Host,
+		Path:     u.Path,
+		RawQuery: redactQuery(u.Query()).Encode(),
+	}
+	return redacted
+}
+
+var redactQueryAllowlist = map[string]bool{"ipRedirectOverride": true}
+
+func redactQuery(q url.Values) url.Values {
+	redacted := make(url.Values, len(q))
+	for k, v := range q {
+		if allowed, ok := redactQueryAllowlist[k]; allowed && ok {
+			redacted[k] = v
+		} else {
+			redacted[k] = []string{"REDACTED"}
+		}
+	}
+	return redacted
+}
+
+var (
+	findScriptTags   = xpath.MustCompile("//script")
+	findHiddenInputs = xpath.MustCompile(`//input[@type="hidden"]`)
+)
+
+func redactHTML(data []byte) []byte {
+	doc, err := htmlquery.Parse(bytes.NewReader(data))
+	if err != nil {
+		return data
+	}
+
+	// remove all <script /> elements
+	for _, node := range htmlquery.QuerySelectorAll(doc, findScriptTags) {
+		node.Parent.RemoveChild(node)
+	}
+
+	// remove all hidden <input /> elements
+	for _, node := range htmlquery.QuerySelectorAll(doc, findHiddenInputs) {
+		node.Parent.RemoveChild(node)
+	}
+
+	return []byte(htmlquery.OutputHTML(doc, true))
+}

go.mod

@@ -5,11 +5,11 @@ go 1.14
 require (
 	fyne.io/fyne v1.4.2
 	github.com/antchfx/htmlquery v1.2.3
+	github.com/antchfx/xpath v1.1.6
 	github.com/dhowden/tag v0.0.0-20201120070457-d52dcb253c63
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/sirupsen/logrus v1.7.0
 	github.com/vbauerster/mpb/v5 v5.3.0
 	golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
-	src.techknowlogick.com/xgo v1.2.1-0.20210120013813-c3fef1e88e60 // indirect
 )

go.sum

@@ -108,5 +108,3 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-src.techknowlogick.com/xgo v1.2.1-0.20210120013813-c3fef1e88e60 h1:vc5HnN8W+M1F6wabNIwrxVUH8FQpzE4dq/N8yryMfX8=
-src.techknowlogick.com/xgo v1.2.1-0.20210120013813-c3fef1e88e60/go.mod h1:31CE1YKtDOrKTk9PSnjTpe6YbO6W/0LTYZ1VskL09oU=

vendor/modules.txt

@@ -31,6 +31,7 @@ github.com/acarl005/stripansi
 ## explicit
 github.com/antchfx/htmlquery
 # github.com/antchfx/xpath v1.1.6
+## explicit
 github.com/antchfx/xpath
 # github.com/davecgh/go-spew v1.1.1
 github.com/davecgh/go-spew/spew
@@ -137,5 +138,3 @@ golang.org/x/text/runes
 golang.org/x/text/transform
 # gopkg.in/yaml.v2 v2.2.8
 gopkg.in/yaml.v2
-# src.techknowlogick.com/xgo v1.2.1-0.20210120013813-c3fef1e88e60
-## explicit