diff --git a/customTLSSpec.go b/customTLSSpec.go new file mode 100644 index 0000000..98024d9 --- /dev/null +++ b/customTLSSpec.go @@ -0,0 +1,79 @@ +package warc + +import tls "github.com/refraction-networking/utls" + +// Taken from https://github.com/refraction-networking/utls/blob/master/u_parrots.go#L215 as the default Chrome config and modified to fit our needs. +func getCustomTLSSpec() *tls.ClientHelloSpec { + return &tls.ClientHelloSpec{ + CipherSuites: []uint16{ + tls.GREASE_PLACEHOLDER, + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + }, + CompressionMethods: []byte{ + 0x00, // compressionNone + }, + Extensions: []tls.TLSExtension{ + &tls.UtlsGREASEExtension{}, + &tls.SNIExtension{}, + &tls.UtlsExtendedMasterSecretExtension{}, + &tls.RenegotiationInfoExtension{Renegotiation: tls.RenegotiateOnceAsClient}, + &tls.SupportedCurvesExtension{Curves: []tls.CurveID{ + tls.CurveID(tls.GREASE_PLACEHOLDER), + tls.X25519, + tls.CurveP256, + tls.CurveP384, + }}, + &tls.SupportedPointsExtension{SupportedPoints: []byte{ + 0x00, // pointFormatUncompressed + }}, + &tls.SessionTicketExtension{}, + // changed IMPORTANT!!! default ALPN is "h2", "http/1.1". This could get servers to automatically send us HTTP2, which we can't parse or handle. We could be profiled based on this. + &tls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}}, + &tls.StatusRequestExtension{}, + &tls.SignatureAlgorithmsExtension{SupportedSignatureAlgorithms: []tls.SignatureScheme{ + tls.ECDSAWithP256AndSHA256, + tls.PSSWithSHA256, + tls.PKCS1WithSHA256, + tls.ECDSAWithP384AndSHA384, + tls.PSSWithSHA384, + tls.PKCS1WithSHA384, + tls.PSSWithSHA512, + tls.PKCS1WithSHA512, + }}, + &tls.SCTExtension{}, + &tls.KeyShareExtension{KeyShares: []tls.KeyShare{ + {Group: tls.CurveID(tls.GREASE_PLACEHOLDER), Data: []byte{0}}, + {Group: tls.X25519}, + }}, + &tls.PSKKeyExchangeModesExtension{Modes: []uint8{ + tls.PskModeDHE, + }}, + &tls.SupportedVersionsExtension{Versions: []uint16{ + tls.GREASE_PLACEHOLDER, + tls.VersionTLS13, + tls.VersionTLS12, + tls.VersionTLS11, + tls.VersionTLS10, + }}, + &tls.UtlsCompressCertExtension{Algorithms: []tls.CertCompressionAlgo{ + tls.CertCompressionBrotli, + }}, + &tls.UtlsGREASEExtension{}, + &tls.UtlsPaddingExtension{GetPaddingLen: tls.BoringPaddingStyle}, + }, + } +} diff --git a/dialer.go b/dialer.go index ae1fa28..4a4c65c 100644 --- a/dialer.go +++ b/dialer.go @@ -3,7 +3,6 @@ package warc import ( "bufio" "context" - "crypto/tls" "errors" "io" "net" @@ -13,6 +12,7 @@ import ( "sync" "time" + tls "github.com/refraction-networking/utls" uuid "github.com/satori/go.uuid" "golang.org/x/sync/errgroup" ) @@ -89,10 +89,12 @@ func (d *customDialer) CustomDialTLS(network, address string) (net.Conn, error) serverName := address[:strings.LastIndex(address, ":")] cfg.ServerName = serverName cfg.InsecureSkipVerify = d.client.verifyCerts - cfg.PreferServerCipherSuites = false - cfg.CurvePreferences = []tls.CurveID{tls.CurveP256, tls.CurveP384, tls.CurveP521, tls.X25519} - tlsConn := tls.Client(plainConn, cfg) + tlsConn := tls.UClient(plainConn, cfg, tls.HelloCustom) + + if err := tlsConn.ApplyPreset(getCustomTLSSpec()); err != nil { + return nil, err + } errc := make(chan error, 2) timer := time.AfterFunc(time.Second, func() { diff --git a/go.mod b/go.mod index 83b496f..5685edf 100644 --- a/go.mod +++ b/go.mod @@ -5,15 +5,22 @@ go 1.18 require ( github.com/klauspost/compress v1.15.1 github.com/klauspost/pgzip v1.2.5 + github.com/refraction-networking/utls v1.1.0 github.com/satori/go.uuid v1.2.0 go.uber.org/goleak v1.1.12 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) require ( + github.com/andybalholm/brotli v1.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/kr/pretty v0.3.0 // indirect + golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect golang.org/x/lint v0.0.0-20200130185559-910be7a94367 // indirect + golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect golang.org/x/tools v0.1.7 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) + +// this is required as the upstream repo hasn't merged this PR in yet and is required for compressed certificates. +replace github.com/refraction-networking/utls v1.1.0 => github.com/getlantern/utls v0.0.0-20220209202455-048f18ca06ae diff --git a/go.sum b/go.sum index 463bcfe..25a78ac 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,12 @@ +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/getlantern/utls v0.0.0-20220209202455-048f18ca06ae h1:3m2E1+vxqYXIdK9Cgh1dE0KI3fz+MWWB0x4NYbr0bSE= +github.com/getlantern/utls v0.0.0-20220209202455-048f18ca06ae/go.mod h1:+D89TUtA8+NKVFj1IXWr0p3tSdX1+SqUB7rL0QnGqyg= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= @@ -28,6 +33,8 @@ go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa h1:idItI2DDfCokpg0N51B2VtiLdJ4vAuXC9fnCb2gACo4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -36,7 +43,9 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211111160137-58aab5ef257a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -44,10 +53,15 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=