From 5bcb0f2766cd696d235aee72c5059d2b762e2f62 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Sun, 18 Feb 2024 00:32:15 -0600 Subject: [PATCH 01/10] Add test steps to build --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de72f088..2c7b7595 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,3 +17,7 @@ jobs: run: | go get -t ./... make + + - name: Test + run: | + make test From 23d0141434bc11895f8c2a60079763508fa676d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 22:39:56 +0000 Subject: [PATCH 02/10] --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/integration-test.yml | 2 +- .github/workflows/linter.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index c4060256..d43d7503 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.18 diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 3f6e3958..0ca5ce6a 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -21,7 +21,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.18 - name: Run go fmt From 03b3ec5f35487190b5e9fd5ce8e139ccb698008d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 22:40:30 +0000 Subject: [PATCH 03/10] --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index ac0f403a..7cbc6cae 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6 golang.org/x/crypto v0.17.0 golang.org/x/net v0.10.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.20.0 golang.org/x/text v0.14.0 google.golang.org/protobuf v1.30.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 91047293..5b2ea179 100644 --- a/go.sum +++ b/go.sum @@ -408,8 +408,9 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From bbadc3055ed92873db1464c0e4d31ace0619b0a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 18:41:26 -0700 Subject: [PATCH 04/10] Bump alpine from 3.16 to 3.19 updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d398e866..223c6ab4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . . RUN make all ## Runtime image ## -FROM alpine:3.16 as run +FROM alpine:3.19 as run COPY --from=build /usr/src/zgrab2/cmd/zgrab2/zgrab2 /usr/bin/zgrab2 From 50a122591af5905fe1d75335dc4225a5cf3001cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 01:49:54 +0000 Subject: [PATCH 05/10] --- (#435) --- go.mod | 6 +++--- go.sum | 17 +++++++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 7cbc6cae..a34603f9 100644 --- a/go.mod +++ b/go.mod @@ -12,10 +12,10 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/zmap/zcrypto v0.0.0-20230310154051-c8b263fd8300 github.com/zmap/zflags v1.4.0-beta.1.0.20200204220219-9d95409821b6 - golang.org/x/crypto v0.17.0 - golang.org/x/net v0.10.0 + golang.org/x/crypto v0.23.0 + golang.org/x/net v0.25.0 golang.org/x/sys v0.20.0 - golang.org/x/text v0.14.0 + golang.org/x/text v0.15.0 google.golang.org/protobuf v1.30.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 diff --git a/go.sum b/go.sum index 5b2ea179..50550654 100644 --- a/go.sum +++ b/go.sum @@ -263,8 +263,9 @@ golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -337,8 +338,10 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -408,7 +411,7 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -417,7 +420,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -429,8 +433,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 990c9c36e43b0739e2951255568cf165825b4fa8 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Tue, 21 May 2024 22:58:46 -0400 Subject: [PATCH 06/10] Fix tests and boundary condition (#410) Fix two failing tests Write a bunch more tests to expand coverage and check for errors Fix a bug where we can have no response and trigger a success --- lib/http/transport.go | 2 +- modules/http/http_readlimit_test.go | 208 +++++++++++++++++++--------- 2 files changed, 146 insertions(+), 64 deletions(-) diff --git a/lib/http/transport.go b/lib/http/transport.go index d9288e6e..f8cfdf54 100644 --- a/lib/http/transport.go +++ b/lib/http/transport.go @@ -1553,7 +1553,7 @@ func (pc *persistConn) readLoop() { closeErr = err } - if err != nil && (!pc.sawEOF || resp == nil) { + if err != nil && (!pc.sawEOF || resp == nil || resp.Status == "") { if pc.readLimit <= 0 { err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) } diff --git a/modules/http/http_readlimit_test.go b/modules/http/http_readlimit_test.go index 7ed76caf..90eaaeaf 100644 --- a/modules/http/http_readlimit_test.go +++ b/modules/http/http_readlimit_test.go @@ -13,6 +13,7 @@ import ( "github.com/zmap/zcrypto/tls" "github.com/zmap/zgrab2" + "github.com/zmap/zgrab2/lib/http" ) // BEGIN Taken from handshake_server_test.go -- certs for TLS server @@ -107,7 +108,13 @@ func (cfg *readLimitTestConfig) runFakeHTTPServer(t *testing.T) { } head := "HTTP/1.0 200 OK\r\nBogus-Header: X" + if cfg.customHeader != nil { + head = *cfg.customHeader + } headSuffix := fmt.Sprintf("\r\nContent-Length: %d\r\n\r\n", cfg.bodySize) + if cfg.customSuffix != nil { + headSuffix = *cfg.customSuffix + } size := cfg.headerSize - len(head) - len(headSuffix) if size < 0 { t.Fatalf("Header size %d too small: must be at least %d bytes", cfg.headerSize, len(head)+len(headSuffix)) @@ -191,6 +198,11 @@ type readLimitTestConfig struct { // If set, the error returned by the scan must contain this. expectedError string + + // If set, return a custom header + customHeader *string + + customSuffix *string } const ( @@ -198,6 +210,8 @@ const ( readLimitTestConfigHTTPSBasePort = 0x7bbc ) +func adr(s string) *string {return &s} + var readLimitTestConfigs = map[string]*readLimitTestConfig{ // The socket truncates the connection while reading the body. To the client it looks as if the // server closed the connection prior to sending Content-Length bytes; the result is success, @@ -219,84 +233,140 @@ var readLimitTestConfigs = map[string]*readLimitTestConfig{ // and the truncated body. // maxReadSize > headerSize + bodySize > bodySize > maxBodySize "truncate_body": { - tls: false, - port: readLimitTestConfigHTTPBasePort + 1, - maxBodySize: 2048, - maxReadSize: 8192, - headerSize: 64, - bodySize: 4096, + tls: false, + port: readLimitTestConfigHTTPBasePort + 1, + maxBodySize: 2048, + maxReadSize: 8192, + headerSize: 64, + bodySize: 4096, expectedStatus: zgrab2.SCAN_SUCCESS, }, "tls_truncate_body": { - tls: true, - port: readLimitTestConfigHTTPSBasePort + 1, - maxBodySize: 2048, - maxReadSize: 8192, - headerSize: 64, - bodySize: 4096, + tls: true, + port: readLimitTestConfigHTTPSBasePort + 1, + maxBodySize: 2048, + maxReadSize: 8192, + headerSize: 64, + bodySize: 4096, expectedStatus: zgrab2.SCAN_SUCCESS, }, - // The socket truncates the connection while reading the headers. The result isn't a valid HTTP - // response, so the library returns an unexpected EOF error. + // The socket truncates the connection while reading the headers. The result isn't a completely valid HTTP + // response, but we capture the output regardless // headerSize > maxReadSize "truncate_read_header": { - tls: false, - port: readLimitTestConfigHTTPBasePort + 2, - maxBodySize: 1024, - maxReadSize: 2048, - headerSize: 3072, - bodySize: 8, - expectedError: "unexpected EOF", - expectedStatus: zgrab2.SCAN_UNKNOWN_ERROR, + tls: false, + port: readLimitTestConfigHTTPBasePort + 2, + maxBodySize: 1024, + maxReadSize: 2048, + headerSize: 3072, + bodySize: 0, + expectedStatus: zgrab2.SCAN_SUCCESS, }, "tls_truncate_read_header": { - tls: true, - port: readLimitTestConfigHTTPSBasePort + 2, - maxBodySize: 1024, - maxReadSize: 2048, - headerSize: 3072, - bodySize: 8, - expectedError: "unexpected EOF", + tls: true, + port: readLimitTestConfigHTTPSBasePort + 2, + maxBodySize: 1024, + maxReadSize: 2048, + headerSize: 3072, + bodySize: 0, + expectedStatus: zgrab2.SCAN_SUCCESS, + }, + + // The socket truncates the connection while reading the status code. The result isn't a valid HTTP + // response + // headerSize > maxReadSize + "invalid_status_code": { + tls: false, + port: readLimitTestConfigHTTPBasePort + 2, + maxBodySize: 8192, + maxReadSize: 8192, + headerSize: 1024, + bodySize: 1024, + customHeader: adr("HTTP/1.0 200"), + expectedError: "malformed HTTP status code", + expectedStatus: zgrab2.SCAN_UNKNOWN_ERROR, + }, + "tls_invalid_status_code": { + tls: true, + port: readLimitTestConfigHTTPSBasePort + 2, + maxBodySize: 8192, + maxReadSize: 8192, + headerSize: 1024, + bodySize: 1024, + customHeader: adr("HTTP/1.0 200"), + expectedError: "malformed HTTP status code", + expectedStatus: zgrab2.SCAN_UNKNOWN_ERROR, + }, + + "invalid_no_status": { + tls: false, + port: readLimitTestConfigHTTPBasePort + 2, + maxBodySize: 8192, + maxReadSize: 8192, + headerSize: 1024, + bodySize: 1024, + customHeader: adr(""), + customSuffix: adr(""), + expectedError: "malformed HTTP response", + expectedStatus: zgrab2.SCAN_UNKNOWN_ERROR, + }, + + "invalid_response": { + tls: false, + port: readLimitTestConfigHTTPBasePort + 2, + maxBodySize: 8192, + maxReadSize: 8192, + headerSize: 1024, + bodySize: 1024, + customHeader: adr(""), + expectedError: "malformed HTTP response", + expectedStatus: zgrab2.SCAN_UNKNOWN_ERROR, + }, + + "invalid_low_read_limit": { + tls: false, + port: readLimitTestConfigHTTPBasePort + 2, + maxBodySize: 8192, + maxReadSize: 1, + headerSize: 1024, + bodySize: 1024, + expectedError: "malformed HTTP response", expectedStatus: zgrab2.SCAN_UNKNOWN_ERROR, }, // Happy case. None of the limits are hit. // maxReadSize >= maxBodySize > bodySize + headerSize "happy_case": { - tls: false, - port: readLimitTestConfigHTTPBasePort + 3, - maxBodySize: 8192, - maxReadSize: 8192, - headerSize: 1024, - bodySize: 1024, + tls: false, + port: readLimitTestConfigHTTPBasePort + 3, + maxBodySize: 8192, + maxReadSize: 8192, + headerSize: 1024, + bodySize: 1024, expectedStatus: zgrab2.SCAN_SUCCESS, }, "tls_happy_case": { - tls: true, - port: readLimitTestConfigHTTPSBasePort + 3, - maxBodySize: 8192, - maxReadSize: 8192, - headerSize: 1024, - bodySize: 1024, + tls: true, + port: readLimitTestConfigHTTPSBasePort + 3, + maxBodySize: 8192, + maxReadSize: 8192, + headerSize: 1024, + bodySize: 1024, expectedStatus: zgrab2.SCAN_SUCCESS, }, } // Try to get the HTTP body from a result; otherwise return the empty string. -func getBody(result interface{}) string { +func getResponse(result interface{}) *http.Response { if result == nil { - return "" + return nil } httpResult, ok := result.(*Results) if !ok { - return "" - } - response := httpResult.Response - if response == nil { - return "" + return nil } - return response.BodyText + return httpResult.Response } // Run a single test with the given configuration. @@ -307,31 +377,43 @@ func (cfg *readLimitTestConfig) runTest(t *testing.T, testName string) { IP: net.ParseIP("127.0.0.1"), } status, ret, err := scanner.Scan(target) + response := getResponse(ret) if status != cfg.expectedStatus { - t.Errorf("Wrong status: expected %s, got %s", cfg.expectedStatus, status) + t.Errorf("Wrong status: expected %s, got %s with %+v", cfg.expectedStatus, status, response) } if err != nil { if !strings.Contains(err.Error(), cfg.expectedError) { - t.Errorf("Wrong error: expected %s, got %s", err.Error(), cfg.expectedError) + t.Errorf("Wrong error: expected %s, got %s", cfg.expectedError, err.Error()) } } else if len(cfg.expectedError) > 0 { t.Errorf("Expected error '%s' but got none", cfg.expectedError) } if cfg.expectedStatus == zgrab2.SCAN_SUCCESS { - body := getBody(ret) - if body == "" { - t.Errorf("Expected success, but got no body") - } else { - if len(body) > cfg.maxBodySize || len(body) > cfg.maxReadSize { - t.Errorf("Body exceeds max size: len(body)=%d; maxBodySize=%d, maxReadSize=%d", len(body), cfg.maxBodySize, cfg.maxReadSize) - } - if !cfg.tls { - if len(body)+cfg.headerSize > cfg.maxReadSize { - t.Errorf("Body and header exceed max read size: len(body)=%d, headerSize=%d, maxReadSize=%d", len(body), cfg.headerSize, cfg.maxReadSize) - } - } + if response == nil { + t.Errorf("Expected response, but got none") + } + + statusCode := response.Status + if statusCode != "200 OK" { + t.Errorf("Expected status %s, but got %s", "200 OK", statusCode) + } + + body := response.BodyText + if body == "" { + if cfg.bodySize != 0 { + t.Errorf("Expected success, but got no body") + } + } else { + if len(body) > cfg.maxBodySize || len(body) > cfg.maxReadSize { + t.Errorf("Body exceeds max size: len(body)=%d; maxBodySize=%d, maxReadSize=%d", len(body), cfg.maxBodySize, cfg.maxReadSize) + } + if !cfg.tls { + if len(body)+cfg.headerSize > cfg.maxReadSize { + t.Errorf("Body and header exceed max read size: len(body)=%d, headerSize=%d, maxReadSize=%d", len(body), cfg.headerSize, cfg.maxReadSize) + } } + } } } From 6106c1ad2186502a73e5b4e9acae5ffb03cf546d Mon Sep 17 00:00:00 2001 From: guoguangwu Date: Wed, 22 May 2024 14:04:04 +0800 Subject: [PATCH 07/10] fix: close CPU profile Signed-off-by: guoguangwu --- bin/bin.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bin/bin.go b/bin/bin.go index dc490461..50f7a055 100644 --- a/bin/bin.go +++ b/bin/bin.go @@ -61,7 +61,7 @@ func dumpHeapProfile() { // If CPU profiling is enabled (ZGRAB2_CPUPROFILE is not empty), start tracking // CPU profiling in the configured file. Caller is responsible for invoking // stopCPUProfile() when finished. -func startCPUProfile() { +func startCPUProfile() *os.File { if file := getCPUProfileFile(); file != "" { now := time.Now() fullFile := getFormattedFile(file, now) @@ -72,15 +72,21 @@ func startCPUProfile() { if err := pprof.StartCPUProfile(f); err != nil { log.Fatal("could not start CPU profile: ", err) } + return f } + + return nil } // If CPU profiling is enabled (ZGRAB2_CPUPROFILE is not empty), stop profiling // CPU usage. -func stopCPUProfile() { +func stopCPUProfile(f *os.File) { if getCPUProfileFile() != "" { pprof.StopCPUProfile() } + if f != nil { + f.Close() + } } // ZGrab2Main should be called by func main() in a binary. The caller is @@ -88,8 +94,8 @@ func stopCPUProfile() { // include custom sets of scan modules by creating new main packages with custom // sets of ZGrab modules imported with side-effects. func ZGrab2Main() { - startCPUProfile() - defer stopCPUProfile() + f := startCPUProfile() + defer stopCPUProfile(f) defer dumpHeapProfile() _, moduleType, flag, err := zgrab2.ParseCommandLine(os.Args[1:]) From ab1db71e5aa3b09c717de5a0cbbdddc973cb6df3 Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Wed, 22 May 2024 15:54:23 -0400 Subject: [PATCH 08/10] Add TLS support for fox (#403) * Add TLS support for fox module * Add TLS to the schema --------- Co-authored-by: Raymond Nook <59678453+developStorm@users.noreply.github.com> --- modules/fox/log.go | 6 ++++++ modules/fox/scanner.go | 29 ++++++++++++++++++++++++++++- zgrab2_schemas/zgrab2/fox.py | 1 + 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/modules/fox/log.go b/modules/fox/log.go index 3b8c860c..d715d0ba 100644 --- a/modules/fox/log.go +++ b/modules/fox/log.go @@ -1,5 +1,9 @@ package fox +import ( + "github.com/zmap/zgrab2" +) + // FoxLog is the struct returned to the caller. type FoxLog struct { // IsFox should always be true (otherwise, the result should have been nil). @@ -58,4 +62,6 @@ type FoxLog struct { // AuthAgentType corresponds to the "authAgentTypeSpecs" field. AuthAgentType string `json:"auth_agent_type,omitempty"` + + TLSLog *zgrab2.TLSLog `json:"tls,omitempty"` } diff --git a/modules/fox/scanner.go b/modules/fox/scanner.go index a2ac69cb..a2abf241 100644 --- a/modules/fox/scanner.go +++ b/modules/fox/scanner.go @@ -9,6 +9,7 @@ import ( "errors" log "github.com/sirupsen/logrus" "github.com/zmap/zgrab2" + "net" ) // Flags holds the command-line configuration for the fox scan module. @@ -17,6 +18,8 @@ type Flags struct { zgrab2.BaseFlags Verbose bool `long:"verbose" description:"More verbose logging, include debug fields in the scan results"` + UseTLS bool `long:"use-tls" description:"Sends probe with a TLS connection. Loads TLS module command options."` + zgrab2.TLSFlags } // Module implements the zgrab2.Module interface. @@ -98,12 +101,36 @@ func (scanner *Scanner) Protocol() string { // 4. If the response has the Fox response prefix, mark the scan as having detected the service. // 5. Attempt to read any / all of the data fields from the Log struct func (scanner *Scanner) Scan(target zgrab2.ScanTarget) (zgrab2.ScanStatus, interface{}, error) { - conn, err := target.Open(&scanner.config.BaseFlags) + + var ( + conn net.Conn + tlsConn *zgrab2.TLSConnection + err error + ) + + conn, err = target.Open(&scanner.config.BaseFlags) + if scanner.config.UseTLS { + tlsConn, err = scanner.config.TLSFlags.GetTLSConnection(conn) + if err != nil { + return zgrab2.TryGetScanStatus(err), nil, err + } + if err := tlsConn.Handshake(); err != nil { + return zgrab2.TryGetScanStatus(err), nil, err + } + conn = tlsConn + } else { + conn, err = target.Open(&scanner.config.BaseFlags) + } + if err != nil { return zgrab2.TryGetScanStatus(err), nil, err } + defer conn.Close() result := new(FoxLog) + if tlsConn != nil { + result.TLSLog = tlsConn.GetLog() + } err = GetFoxBanner(result, conn) if !result.IsFox { diff --git a/zgrab2_schemas/zgrab2/fox.py b/zgrab2_schemas/zgrab2/fox.py index 212ecd85..fcf0ee0e 100644 --- a/zgrab2_schemas/zgrab2/fox.py +++ b/zgrab2_schemas/zgrab2/fox.py @@ -30,6 +30,7 @@ "brand_id": String(), "sys_info": String(), "agent_auth_type": String(), + "tls": zgrab2.tls_log, } ) }, From 6993f20b8988eff6f752702bee780bc3793d6c4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 15:56:24 -0700 Subject: [PATCH 09/10] build: bump golang from 1.18.3-alpine3.16 to 1.20.4-alpine3.16 (#433) updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 223c6ab4..d5150101 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ## Build image ## -FROM golang:1.18.3-alpine3.16 as build +FROM golang:1.20.4-alpine3.16 as build # System dependencies RUN apk add --no-cache make From 0ad07a915e7ec79638bcd3510e4033f47fdd985f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 03:00:47 +0000 Subject: [PATCH 10/10] Bump google.golang.org/protobuf from 1.30.0 to 1.33.0 (#414) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a34603f9..5c1e5265 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( golang.org/x/net v0.25.0 golang.org/x/sys v0.20.0 golang.org/x/text v0.15.0 - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 50550654..8236b487 100644 --- a/go.sum +++ b/go.sum @@ -564,8 +564,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=