BackingImage does not download URL correctly in some situation

Remove the `Referer` header that is automatically added by the http client. Without this header it is possible to download files that are redirected several times. This mimics the behaviour of `curl` and `wget` which do not submit the `Referer` header by default.

Signed-off-by: Volker Theile <[email protected]>

pkg/sync/handler.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -39,7 +40,7 @@ func (h *HTTPHandler) GetSizeFromURL(url string) (size int64, err error) {
 		return 0, err
 	}
 
-	client := http.Client{}
+	client := NewDownloadHttpClient()
 	resp, err := client.Do(rr)
 	if err != nil {
 		return 0, err
@@ -73,7 +74,7 @@ func (h *HTTPHandler) DownloadFromURL(ctx context.Context, url, filePath string,
 		return 0, err
 	}
 
-	client := http.Client{}
+	client := NewDownloadHttpClient()
 	resp, err := client.Do(rr)
 	if err != nil {
 		return 0, err
@@ -177,6 +178,27 @@ func IdleTimeoutCopy(ctx context.Context, cancel context.CancelFunc, src io.Read
 	return copied, err
 }
 
+func removeReferer(req *http.Request) {
+	for k := range req.Header {
+		if strings.ToLower(k) == "referer" {
+			delete(req.Header, k)
+		}
+	}
+}
+
+func NewDownloadHttpClient() http.Client {
+	return http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// Remove the "Referer" header to enable downloads of files
+			// that are delivered via CDN and therefore may be redirected
+			// several times. This is the same behaviour of curl or wget
+			// in their default configuration.
+			removeReferer(req)
+			return nil
+		},
+	}
+}
+
 const (
 	MockFileSize = 4096
 )

pkg/sync/handler_test.go

@@ -0,0 +1,20 @@
+package sync
+
+import (
+	. "gopkg.in/check.v1"
+	"net/http"
+)
+
+type TestSuite struct{}
+
+var _ = Suite(&TestSuite{})
+
+func (s *TestSuite) TestRemoveReferer(c *C) {
+	req, err := http.NewRequest("HEAD", "https://foo.bar", nil)
+	c.Assert(err, IsNil)
+	req.Header.Set("Referer", "https://foo.bar")
+	req.Header.Set("Foo", "foo")
+	removeReferer(req)
+	c.Assert(req.Referer(), Equals, "")
+	c.Assert(req.Header, HasLen, 1)
+}