diff --git a/pkg/common/http.go b/pkg/common/http.go index 48b404536dc7..ba0532ef8186 100644 --- a/pkg/common/http.go +++ b/pkg/common/http.go @@ -127,3 +127,11 @@ func SaneHttpClient() *http.Client { httpClient.Transport = NewCustomTransport(nil) return httpClient } + +//custom timeout for some scanners +func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client { + httpClient := &http.Client{} + httpClient.Timeout = time.Second * time.Duration(timeOutSeconds) + httpClient.Transport = NewCustomTransport(nil) + return httpClient +} diff --git a/pkg/detectors/allsports/allsports.go b/pkg/detectors/allsports/allsports.go index 5acbd0ba3f5d..91dbc8a5e390 100644 --- a/pkg/detectors/allsports/allsports.go +++ b/pkg/detectors/allsports/allsports.go @@ -58,7 +58,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if strings.Contains(body, "success") { diff --git a/pkg/detectors/amadeus/amadeus.go b/pkg/detectors/amadeus/amadeus.go index 83a26d18954a..0748d1e3458a 100644 --- a/pkg/detectors/amadeus/amadeus.go +++ b/pkg/detectors/amadeus/amadeus.go @@ -65,7 +65,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") { s1.Verified = true diff --git a/pkg/detectors/auth0oauth/auth0oauth.go b/pkg/detectors/auth0oauth/auth0oauth.go index 99ca0c70c3ad..962ae2e8e6a3 100644 --- a/pkg/detectors/auth0oauth/auth0oauth.go +++ b/pkg/detectors/auth0oauth/auth0oauth.go @@ -90,7 +90,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} diff --git a/pkg/detectors/baseapiio/baseapiio.go b/pkg/detectors/baseapiio/baseapiio.go index 4966a0980beb..17e8802ecbb1 100644 --- a/pkg/detectors/baseapiio/baseapiio.go +++ b/pkg/detectors/baseapiio/baseapiio.go @@ -57,7 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if strings.Contains(body, "items") { diff --git a/pkg/detectors/besttime/besttime.go b/pkg/detectors/besttime/besttime.go index e0a1c50371c4..50ad2b8a874a 100644 --- a/pkg/detectors/besttime/besttime.go +++ b/pkg/detectors/besttime/besttime.go @@ -55,7 +55,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if strings.Contains(body, `"status": "OK"`) { diff --git a/pkg/detectors/borgbase/borgbase.go b/pkg/detectors/borgbase/borgbase.go index bd8f53e3d09f..18fedd47d1e8 100644 --- a/pkg/detectors/borgbase/borgbase.go +++ b/pkg/detectors/borgbase/borgbase.go @@ -64,13 +64,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"errors"`) + validResponse := strings.Contains(bodyString, `"sshList":[]`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/bulbul/bulbul.go b/pkg/detectors/bulbul/bulbul.go index b5f9ce2cb6b4..0709151590ed 100644 --- a/pkg/detectors/bulbul/bulbul.go +++ b/pkg/detectors/bulbul/bulbul.go @@ -55,7 +55,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } res, err := client.Do(req) if err == nil { - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + + if err != nil { + continue + } bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"message":"Successful",`) diff --git a/pkg/detectors/cexio/cexio.go b/pkg/detectors/cexio/cexio.go index 92b67962383b..22a5269af915 100644 --- a/pkg/detectors/cexio/cexio.go +++ b/pkg/detectors/cexio/cexio.go @@ -92,6 +92,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result defer res.Body.Close() body, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } + bodyString := string(body) + validResponse := strings.Contains(bodyString, `timestamp`) if err != nil { fmt.Print(err.Error()) } @@ -99,7 +104,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result var responseObject Response json.Unmarshal(body, &responseObject) - if res.StatusCode >= 200 && res.StatusCode < 300 && responseObject.Error == "" { + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { s1.Verified = true } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/coinlayer/coinlayer.go b/pkg/detectors/coinlayer/coinlayer.go index 2662ff088a45..a8f71a24ca7d 100644 --- a/pkg/detectors/coinlayer/coinlayer.go +++ b/pkg/detectors/coinlayer/coinlayer.go @@ -49,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.coinlayer.com/api/live?access_key=%s", resMatch), nil) + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.coinlayer.com/api/livelive?access_key=%s", resMatch), nil) if err != nil { continue } @@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `"success": true`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/currencycloud/currencycloud.go b/pkg/detectors/currencycloud/currencycloud.go index 70c6978c1a44..11bab2b8dc3a 100644 --- a/pkg/detectors/currencycloud/currencycloud.go +++ b/pkg/detectors/currencycloud/currencycloud.go @@ -66,7 +66,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if strings.Contains(body, "auth_token") { s1.Verified = true diff --git a/pkg/detectors/currencylayer/currencylayer.go b/pkg/detectors/currencylayer/currencylayer.go index 86d84ab9b3c4..cae43a0ad636 100644 --- a/pkg/detectors/currencylayer/currencylayer.go +++ b/pkg/detectors/currencylayer/currencylayer.go @@ -59,13 +59,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err2 := ioutil.ReadAll(res.Body) if err2 == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `"success": true`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/dyspatch/dyspatch.go b/pkg/detectors/dyspatch/dyspatch.go index b128c9dba6ba..cb28e908799f 100644 --- a/pkg/detectors/dyspatch/dyspatch.go +++ b/pkg/detectors/dyspatch/dyspatch.go @@ -58,10 +58,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) + validResponse := strings.Contains(body, "limited_usage") || strings.Contains(body, "data") - if !strings.Contains(body, "unauthenticated") { + if validResponse { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/elasticemail/elasticemail.go b/pkg/detectors/elasticemail/elasticemail.go index 9d9a53288daf..d8261beaf86b 100644 --- a/pkg/detectors/elasticemail/elasticemail.go +++ b/pkg/detectors/elasticemail/elasticemail.go @@ -67,8 +67,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) - if err != nil { + data, readErr := ioutil.ReadAll(res.Body) + if readErr != nil { continue } var ResVar struct { diff --git a/pkg/detectors/fetchrss/fetchrss.go b/pkg/detectors/fetchrss/fetchrss.go index 5e873f41fb86..3a3decad4dea 100644 --- a/pkg/detectors/fetchrss/fetchrss.go +++ b/pkg/detectors/fetchrss/fetchrss.go @@ -48,14 +48,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://fetchrss.com/api/v1/feed/create?auth="+resMatch, nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://fetchrss.com/api/v1/feed/list?auth="+resMatch, nil) if err != nil { continue } res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if !strings.Contains(body, "Not authorised") { diff --git a/pkg/detectors/fetchrss/fetchrss_test.go b/pkg/detectors/fetchrss/fetchrss_test.go index 1e9bff2feb4d..e485142a7d33 100644 --- a/pkg/detectors/fetchrss/fetchrss_test.go +++ b/pkg/detectors/fetchrss/fetchrss_test.go @@ -20,7 +20,7 @@ func TestFetchrss_FromChunk(t *testing.T) { if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("FETCHRSS") + secret := testSecrets.MustGetField("FETCHRSS_TOKEN") inactiveSecret := testSecrets.MustGetField("FETCHRSS_INACTIVE") type args struct { diff --git a/pkg/detectors/financialmodelingprep/financialmodelingprep.go b/pkg/detectors/financialmodelingprep/financialmodelingprep.go index 4dacb3b79e23..9f61ce64bfe6 100644 --- a/pkg/detectors/financialmodelingprep/financialmodelingprep.go +++ b/pkg/detectors/financialmodelingprep/financialmodelingprep.go @@ -56,16 +56,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { bodyBytes, err := ioutil.ReadAll(res.Body) + bodyString := string(bodyBytes) if err == nil { - bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"Error Message"`) + // valid response should be an array of currencies + // error response is in json + validResponse := strings.Contains(bodyString, `[ "`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/fixerio/fixerio.go b/pkg/detectors/fixerio/fixerio.go index fb3c30d8a941..5a4c9d80be42 100644 --- a/pkg/detectors/fixerio/fixerio.go +++ b/pkg/detectors/fixerio/fixerio.go @@ -56,14 +56,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} // ingenious! - if !strings.Contains(body, "invalid_access_key") { + validResponse := strings.Contains(body, `"success": true`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) + defer res.Body.Close() + + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/flickr/flickr.go b/pkg/detectors/flickr/flickr.go index 5abc59bd4fe5..b3fb21ce3959 100644 --- a/pkg/detectors/flickr/flickr.go +++ b/pkg/detectors/flickr/flickr.go @@ -57,7 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "owner=") { s1.Verified = true diff --git a/pkg/detectors/flightstats/flightstats.go b/pkg/detectors/flightstats/flightstats.go index 5265d1e9301d..6775b4bc43db 100644 --- a/pkg/detectors/flightstats/flightstats.go +++ b/pkg/detectors/flightstats/flightstats.go @@ -65,9 +65,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 403 && strings.Contains(body, "application is not active")) { + validResponse := (res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "id")) || (res.StatusCode == 403 && strings.Contains(body, "application is not active")) + if validResponse { s1.Verified = true } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/flowflu/flowflu.go b/pkg/detectors/flowflu/flowflu.go index 09716fabf9bc..6e735f284a22 100644 --- a/pkg/detectors/flowflu/flowflu.go +++ b/pkg/detectors/flowflu/flowflu.go @@ -22,7 +22,8 @@ var ( client = common.SaneHttpClient() //Make sure that your group is surrounded in boundry characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"flowflu"}) + `\b([a-zA-Z0-9]{51})\b`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"flowflu"}) + `\b([a-zA-Z0-9]{51})\b`) + accountPat = regexp.MustCompile(detectors.PrefixRegex([]string{"flowflu", "account"}) + `\b([a-zA-Z0-9]{4,30})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -36,6 +37,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result dataStr := string(data) matches := keyPat.FindAllStringSubmatch(dataStr, -1) + accountMatches := accountPat.FindAllStringSubmatch(dataStr, -1) for _, match := range matches { if len(match) != 2 { @@ -43,43 +45,51 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } resMatch := strings.TrimSpace(match[1]) - s1 := detectors.Result{ - DetectorType: detectorspb.DetectorType_FlowFlu, - Raw: []byte(resMatch), - } - - if verify { - req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://secretscanner.flowlu.com/api/v1/module/crm/lead/list?api_key=%s", resMatch), nil) - if err != nil { + for _, accountMatch := range accountMatches { + if len(accountMatch) != 2 { continue } - res, err := client.Do(req) - if err == nil { - bodyBytes, err := ioutil.ReadAll(res.Body) - if err == nil { + + resAccount := strings.TrimSpace(accountMatch[1]) + + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_FlowFlu, + Raw: []byte(resMatch), + } + + if verify { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s.flowlu.com/api/v1/module/crm/lead/list?api_key=%s", resAccount, resMatch), nil) + if err != nil { continue } + res, err := client.Do(req) + if err == nil { + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } - bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `error_code":11`) + bodyString := string(bodyBytes) + validResponse := strings.Contains(bodyString, `total_result`) - defer res.Body.Close() - if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + if validResponse { + s1.Verified = true + } else { + s1.Verified = false + } } else { - s1.Verified = true - } - } else { - //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue + //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key + if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue + } } } } - } - results = append(results, s1) + results = append(results, s1) + } } return detectors.CleanResults(results), nil diff --git a/pkg/detectors/flowflu/flowflu_test.go b/pkg/detectors/flowflu/flowflu_test.go index ce9af6b03ee6..559cc24ffba9 100644 --- a/pkg/detectors/flowflu/flowflu_test.go +++ b/pkg/detectors/flowflu/flowflu_test.go @@ -20,6 +20,7 @@ func TestFlowFlu_FromChunk(t *testing.T) { if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } + account := testSecrets.MustGetField("FLOWFLU_ACCOUNT") secret := testSecrets.MustGetField("FLOWFLU_TOKEN") inactiveSecret := testSecrets.MustGetField("FLOWFLU_INACTIVE") @@ -40,7 +41,7 @@ func TestFlowFlu_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a flowflu secret %s within", secret)), + data: []byte(fmt.Sprintf("You can find a flowflu secret %s within flowflu account %s", secret, account)), verify: true, }, want: []detectors.Result{ @@ -56,7 +57,7 @@ func TestFlowFlu_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a flowflu secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a flowflu secret %s within flowflu account %s but not valid", inactiveSecret, account)), // the secret would satisfy the regex but not pass validation verify: true, }, want: []detectors.Result{ diff --git a/pkg/detectors/formbucket/formbucket.go b/pkg/detectors/formbucket/formbucket.go index 158d9ce0d27b..cc5b337aac27 100644 --- a/pkg/detectors/formbucket/formbucket.go +++ b/pkg/detectors/formbucket/formbucket.go @@ -2,7 +2,6 @@ package formbucket import ( "context" - "encoding/json" "fmt" "io/ioutil" "net/http" @@ -23,7 +22,7 @@ var ( client = common.SaneHttpClient() //Make sure that your group is surrounded in boundry characters such as below to reduce false positives - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"formbucket"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{171}\.[0-9A-Z-a-z\-_]{43}[ \r\n]{1})`) + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"formbucket"}) + `\b([0-9A-Za-z]{1,}.[0-9A-Za-z]{1,}\.[0-9A-Z-a-z\-_]{1,})`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -60,12 +59,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { defer res.Body.Close() body, errBody := ioutil.ReadAll(res.Body) - - var result Response + if errBody != nil { + continue + } + bodyString := string(body) + validResponse := strings.Contains(bodyString, `created_on`) + defer res.Body.Close() if errBody == nil { - json.Unmarshal(body, &result) - - if res.StatusCode >= 200 && res.StatusCode < 300 && !result.Anonymous { + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { s1.Verified = true } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.go index 48c8598fc26d..5c3f53382af7 100644 --- a/pkg/detectors/freshbooks/freshbooks.go +++ b/pkg/detectors/freshbooks/freshbooks.go @@ -63,7 +63,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "Log In to FreshBooks") { s1.Verified = true diff --git a/pkg/detectors/getemail/getemail.go b/pkg/detectors/getemail/getemail.go index e191772c2c8a..d63202fa267b 100644 --- a/pkg/detectors/getemail/getemail.go +++ b/pkg/detectors/getemail/getemail.go @@ -19,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = common.SaneHttpClientTimeOut(5) //Make sure that your group is surrounded in boundry characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`) diff --git a/pkg/detectors/gocanvas/gocanvas_test.go b/pkg/detectors/gocanvas/gocanvas_test.go index 22b331a3ad0e..2668e7d5f87b 100644 --- a/pkg/detectors/gocanvas/gocanvas_test.go +++ b/pkg/detectors/gocanvas/gocanvas_test.go @@ -20,7 +20,7 @@ func TestGoCanvas_FromChunk(t *testing.T) { if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - username := testSecrets.MustGetField("GOCANVAS_USERNAME") + username := testSecrets.MustGetField("SCANNERS_EMAIL") secret := testSecrets.MustGetField("GOCANVAS") inactiveSecret := testSecrets.MustGetField("GOCANVAS_INACTIVE") diff --git a/pkg/detectors/humanity/humanity.go b/pkg/detectors/humanity/humanity.go index 0beb35545cb7..9b74dfd70374 100644 --- a/pkg/detectors/humanity/humanity.go +++ b/pkg/detectors/humanity/humanity.go @@ -55,10 +55,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if !strings.Contains(body, "Invalid token key - Please re-authenticate") { + validResponse := strings.Contains(body, "name") + + if validResponse { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/ipapi/ipapi.go b/pkg/detectors/ipapi/ipapi.go index aad5143a8c76..77e38ad47f32 100644 --- a/pkg/detectors/ipapi/ipapi.go +++ b/pkg/detectors/ipapi/ipapi.go @@ -59,7 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err == nil { bodyString := string(bodyBytes) - valid := strings.Contains(bodyString, "continent_code") + valid := strings.Contains(bodyString, "continent_code") || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { diff --git a/pkg/detectors/ipquality/ipquality.go b/pkg/detectors/ipquality/ipquality.go index f7049b453068..4207c68b1b3c 100644 --- a/pkg/detectors/ipquality/ipquality.go +++ b/pkg/detectors/ipquality/ipquality.go @@ -57,9 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) && !strings.Contains(body, "Invalid or unauthorized key") { + validResponse := strings.Contains(body, "insufficient credits") || strings.Contains(body, `"success":true`) + if (res.StatusCode >= 200 && res.StatusCode < 300) && validResponse { s1.Verified = true } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/ipstack/ipstack.go b/pkg/detectors/ipstack/ipstack.go index 18f415a21804..951d623d0e12 100644 --- a/pkg/detectors/ipstack/ipstack.go +++ b/pkg/detectors/ipstack/ipstack.go @@ -54,10 +54,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) + validResponse := strings.Contains(body, "continent_code") || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) - if !strings.Contains(body, "invalid_access_key") { + if validResponse { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/languagelayer/languagelayer.go b/pkg/detectors/languagelayer/languagelayer.go index 51cbaf309376..8a9becb8b1df 100644 --- a/pkg/detectors/languagelayer/languagelayer.go +++ b/pkg/detectors/languagelayer/languagelayer.go @@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `results`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/linearapi/linearapi.go b/pkg/detectors/linearapi/linearapi.go index c18294940330..820ff9741779 100644 --- a/pkg/detectors/linearapi/linearapi.go +++ b/pkg/detectors/linearapi/linearapi.go @@ -59,7 +59,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"data":`) { s1.Verified = true diff --git a/pkg/detectors/mailboxlayer/mailboxlayer.go b/pkg/detectors/mailboxlayer/mailboxlayer.go index 1555655d23d5..ba14d314fa70 100644 --- a/pkg/detectors/mailboxlayer/mailboxlayer.go +++ b/pkg/detectors/mailboxlayer/mailboxlayer.go @@ -55,14 +55,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) + validResponse := strings.Contains(body, `email`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} // if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"} // ingenious! - if !strings.Contains(body, "invalid_access_key") { + if validResponse { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/mesibo/mesibo.go b/pkg/detectors/mesibo/mesibo.go index fcbf7fdaec6e..7e535ec79dc1 100644 --- a/pkg/detectors/mesibo/mesibo.go +++ b/pkg/detectors/mesibo/mesibo.go @@ -55,7 +55,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if !strings.Contains(body, "INVALIDTOKEN") { diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go index 5a3172a2902b..675258f4a362 100644 --- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go +++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = common.SaneHttpClientTimeOut(5) //Make sure that your group is surrounded in boundry characters such as below to reduce false positives keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`) @@ -59,7 +59,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) + body, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } if res.StatusCode >= 200 && string(body) == "1" { s1.Verified = true diff --git a/pkg/detectors/moosend/moosend.go b/pkg/detectors/moosend/moosend.go index 33e01f8fb3e7..bab4f30892e0 100644 --- a/pkg/detectors/moosend/moosend.go +++ b/pkg/detectors/moosend/moosend.go @@ -55,9 +55,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if !strings.Contains(body, "USER_NOT_FOUND") { + if strings.Contains(body, "CreatedOn") { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/mrticktock/mrticktock.go b/pkg/detectors/mrticktock/mrticktock.go index 2371ed0602a8..28f3a8b368f2 100644 --- a/pkg/detectors/mrticktock/mrticktock.go +++ b/pkg/detectors/mrticktock/mrticktock.go @@ -66,7 +66,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"errors":[]`) { s1.Verified = true diff --git a/pkg/detectors/numverify/numverify.go b/pkg/detectors/numverify/numverify.go index 8df5f112989f..a56a1f2feaf8 100644 --- a/pkg/detectors/numverify/numverify.go +++ b/pkg/detectors/numverify/numverify.go @@ -53,13 +53,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `country_code`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/onedesk/onedesk.go b/pkg/detectors/onedesk/onedesk.go index 9a8f5b3c4217..00370e8ebf7f 100644 --- a/pkg/detectors/onedesk/onedesk.go +++ b/pkg/detectors/onedesk/onedesk.go @@ -65,7 +65,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"code":"SUCCESS"`) { s1.Verified = true diff --git a/pkg/detectors/paralleldots/paralleldots.go b/pkg/detectors/paralleldots/paralleldots.go index 0d1d3b073336..7f008cf7482d 100644 --- a/pkg/detectors/paralleldots/paralleldots.go +++ b/pkg/detectors/paralleldots/paralleldots.go @@ -78,7 +78,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "intent") { s1.Verified = true diff --git a/pkg/detectors/planyo/planyo.go b/pkg/detectors/planyo/planyo.go index 65c38485b23f..906999d0be32 100644 --- a/pkg/detectors/planyo/planyo.go +++ b/pkg/detectors/planyo/planyo.go @@ -57,9 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Invalid method or API key") { + validResponse := strings.Contains(body, "data") || strings.Contains(body, "Your Planyo site has expired") + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { s1.Verified = true } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/scrapersite/scrapersite.go b/pkg/detectors/scrapersite/scrapersite.go index e00f40dc7fb6..1985e4c549ad 100644 --- a/pkg/detectors/scrapersite/scrapersite.go +++ b/pkg/detectors/scrapersite/scrapersite.go @@ -7,7 +7,6 @@ import ( "net/http" "regexp" "strings" - "time" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -20,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = common.SaneHttpClientTimeOut(10) //Make sure that your group is surrounded in boundry characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`) @@ -50,15 +49,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - timeout := 10 * time.Second - client.Timeout = timeout req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://scrapersite.com/api-v1?api_key=%s&url=https://google.com", resMatch), nil) if err != nil { continue } res, err := client.Do(req) if err == nil { - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } bodyString := string(bodyBytes) validResponse := strings.Contains(bodyString, `"status":true`) defer res.Body.Close() diff --git a/pkg/detectors/scrapestack/scrapestack.go b/pkg/detectors/scrapestack/scrapestack.go index 03513b5022b0..2f748c6e2d10 100644 --- a/pkg/detectors/scrapestack/scrapestack.go +++ b/pkg/detectors/scrapestack/scrapestack.go @@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `html`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/screenshotlayer/screenshotlayer.go b/pkg/detectors/screenshotlayer/screenshotlayer.go index bb460e9488f9..d68a1a66ccc8 100644 --- a/pkg/detectors/screenshotlayer/screenshotlayer.go +++ b/pkg/detectors/screenshotlayer/screenshotlayer.go @@ -7,7 +7,6 @@ import ( "net/http" "regexp" "strings" - "time" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -20,7 +19,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = common.SaneHttpClientTimeOut(10) //Make sure that your group is surrounded in boundry characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`) @@ -50,8 +49,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - timeout := 10 * time.Second - client.Timeout = timeout req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.screenshotlayer.com/api/capture?access_key=%s&url=https://google.com", resMatch), nil) if err != nil { continue @@ -61,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `PNG`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/semaphore/semaphore.go b/pkg/detectors/semaphore/semaphore.go index 295b87f39457..f6def3645a7c 100644 --- a/pkg/detectors/semaphore/semaphore.go +++ b/pkg/detectors/semaphore/semaphore.go @@ -61,7 +61,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "account_id") { s1.Verified = true diff --git a/pkg/detectors/serpstack/serpstack.go b/pkg/detectors/serpstack/serpstack.go index d625840ecf93..19a36a197acd 100644 --- a/pkg/detectors/serpstack/serpstack.go +++ b/pkg/detectors/serpstack/serpstack.go @@ -61,13 +61,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `search_url`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/shortcut/shortcut.go b/pkg/detectors/shortcut/shortcut.go index 3547f01b05fa..c6a33c999f4e 100644 --- a/pkg/detectors/shortcut/shortcut.go +++ b/pkg/detectors/shortcut/shortcut.go @@ -48,7 +48,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.app.shortcut.com/api/v3/categories", nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.app.shortcut.com/api/v3/projects", nil) if err != nil { continue } @@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } body := string(bodyBytes) - if !strings.Contains(body, "Unauthorized") { + if strings.Contains(body, "app_url") { s1.Verified = true } else { // if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/slackwebhook/slackwebhook.go b/pkg/detectors/slackwebhook/slackwebhook.go index a20d86252dc3..6cb6c2baa85f 100644 --- a/pkg/detectors/slackwebhook/slackwebhook.go +++ b/pkg/detectors/slackwebhook/slackwebhook.go @@ -57,7 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "missing_text")) { s1.Verified = true diff --git a/pkg/detectors/text2data/text2data.go b/pkg/detectors/text2data/text2data.go index 9a2da399f942..4f1877b5239e 100644 --- a/pkg/detectors/text2data/text2data.go +++ b/pkg/detectors/text2data/text2data.go @@ -2,7 +2,6 @@ package text2data import ( "context" - "encoding/json" "io/ioutil" "net/http" "net/url" @@ -55,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result data.Add("DocumentText", "Excellent location, opposite a very large mall with wide variety of shops, restaurants and more.") data.Add("PrivateKey", resMatch) - req, err := http.NewRequestWithContext(ctx, "POST", "https://api.text2data.com/v3/analyze", strings.NewReader(data.Encode())) + req, err := http.NewRequestWithContext(ctx, "POST", "http://api.text2data.com/v3/Analyze", strings.NewReader(data.Encode())) if err != nil { continue } @@ -67,11 +66,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result body, errBody := ioutil.ReadAll(res.Body) if errBody == nil { + bodyString := string(body) + validResponse := strings.Contains(bodyString, `"DocSentimentResultString":"positive"`) - var response Response - json.Unmarshal(body, &response) - - if res.StatusCode >= 200 && res.StatusCode < 300 && response.Status == 1 && response.ErrorMessage == "" { + if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse { s1.Verified = true } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/thinkific/thinkific.go b/pkg/detectors/thinkific/thinkific.go index d01dbcb57ad2..62911bcd1134 100644 --- a/pkg/detectors/thinkific/thinkific.go +++ b/pkg/detectors/thinkific/thinkific.go @@ -57,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if verify { domainRes := fmt.Sprintf("%s-s-school", resDomainMatch) - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.thinkific.com/api/public/v1/users", nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.thinkific.com/api/public/v1/collections", nil) if err != nil { continue } @@ -67,7 +67,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if strings.Contains(body, "API Access is not available") { diff --git a/pkg/detectors/twelvedata/twelvedata.go b/pkg/detectors/twelvedata/twelvedata.go index e3bb4088aaeb..5b19120cc359 100644 --- a/pkg/detectors/twelvedata/twelvedata.go +++ b/pkg/detectors/twelvedata/twelvedata.go @@ -56,7 +56,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) // if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"} diff --git a/pkg/detectors/unifyid/unifyid.go b/pkg/detectors/unifyid/unifyid.go index 14341cfed9c0..9025c545c4fa 100644 --- a/pkg/detectors/unifyid/unifyid.go +++ b/pkg/detectors/unifyid/unifyid.go @@ -58,7 +58,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "invalid token")) { s1.Verified = true diff --git a/pkg/detectors/userstack/userstack.go b/pkg/detectors/userstack/userstack.go index 3e7db2379115..a157e672a971 100644 --- a/pkg/detectors/userstack/userstack.go +++ b/pkg/detectors/userstack/userstack.go @@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - valid := strings.Contains(bodyString, `is_mobile_device`) + valid := strings.Contains(bodyString, `is_mobile_device`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if valid { s1.Verified = true diff --git a/pkg/detectors/vatlayer/vatlayer.go b/pkg/detectors/vatlayer/vatlayer.go index bfe82dc68dbc..df1c2078a73a 100644 --- a/pkg/detectors/vatlayer/vatlayer.go +++ b/pkg/detectors/vatlayer/vatlayer.go @@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result bodyBytes, err := ioutil.ReadAll(res.Body) if err == nil { bodyString := string(bodyBytes) - errCode := strings.Contains(bodyString, `"code":101`) + validResponse := strings.Contains(bodyString, `vat_number`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { - if errCode { - s1.Verified = false - } else { + if validResponse { s1.Verified = true + } else { + s1.Verified = false } } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/walkscore/walkscore.go b/pkg/detectors/walkscore/walkscore.go index b6c33df7504b..86f4fdb566f8 100644 --- a/pkg/detectors/walkscore/walkscore.go +++ b/pkg/detectors/walkscore/walkscore.go @@ -57,7 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, `distance`) { s1.Verified = true diff --git a/pkg/detectors/weatherstack/weatherstack.go b/pkg/detectors/weatherstack/weatherstack.go index d6d5bd689a66..1c3a0d77e97f 100644 --- a/pkg/detectors/weatherstack/weatherstack.go +++ b/pkg/detectors/weatherstack/weatherstack.go @@ -57,9 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if (res.StatusCode >= 200 && res.StatusCode < 300) && !strings.Contains(body, "invalid_access_key") { + validResponse := strings.Contains(body, `location`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`) + if (res.StatusCode >= 200 && res.StatusCode < 300) && validResponse { s1.Verified = true } else { //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key diff --git a/pkg/detectors/webex/webex.go b/pkg/detectors/webex/webex.go index 5f0f5754a035..3d57a6582c40 100644 --- a/pkg/detectors/webex/webex.go +++ b/pkg/detectors/webex/webex.go @@ -70,7 +70,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } defer res.Body.Close() - body, _ := ioutil.ReadAll(res.Body) + body, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } var message struct { Message string `json:"message"` diff --git a/pkg/detectors/webscraping/webscraping.go b/pkg/detectors/webscraping/webscraping.go index 12d4dbb29e36..f15fca6aed31 100644 --- a/pkg/detectors/webscraping/webscraping.go +++ b/pkg/detectors/webscraping/webscraping.go @@ -55,7 +55,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if strings.Contains(body, "key `url` required.") { diff --git a/pkg/detectors/whoxy/whoxy.go b/pkg/detectors/whoxy/whoxy.go index 6ca6532844e7..c4e2511c4b8a 100644 --- a/pkg/detectors/whoxy/whoxy.go +++ b/pkg/detectors/whoxy/whoxy.go @@ -57,7 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"status": 1`) { s1.Verified = true diff --git a/pkg/detectors/zenscrape/zenscrape.go b/pkg/detectors/zenscrape/zenscrape.go index bea8f4dc9629..6987eadc6467 100644 --- a/pkg/detectors/zenscrape/zenscrape.go +++ b/pkg/detectors/zenscrape/zenscrape.go @@ -2,7 +2,6 @@ package zenscrape import ( "context" - "fmt" "io/ioutil" "net/http" "regexp" @@ -53,11 +52,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } - req.Header.Add("apikey", fmt.Sprintf("%s", resMatch)) + req.Header.Add("apikey", resMatch) res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) if !strings.Contains(body, "Not enough requests.") { diff --git a/pkg/detectors/zenserp/zenserp.go b/pkg/detectors/zenserp/zenserp.go index cc42b1cc4478..dc2184103bf0 100644 --- a/pkg/detectors/zenserp/zenserp.go +++ b/pkg/detectors/zenserp/zenserp.go @@ -18,7 +18,7 @@ type Scanner struct{} var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + client = common.SaneHttpClientTimeOut(5) //Make sure that your group is surrounded in boundry characters such as below to reduce false positives keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`) @@ -47,16 +47,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://app.zenserp.com/api/v2/search?apikey="+resMatch, nil) + req, err := http.NewRequestWithContext(ctx, "GET", `https://app.zenserp.com/api/v2/search?q="test"&apikey=`+resMatch, nil) if err != nil { continue } res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if !strings.Contains(body, "Not enough requests.") { + if strings.Contains(body, "query") { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { diff --git a/pkg/detectors/zerobounce/zerobounce.go b/pkg/detectors/zerobounce/zerobounce.go index e335b3af2dd1..42685eb1d72c 100644 --- a/pkg/detectors/zerobounce/zerobounce.go +++ b/pkg/detectors/zerobounce/zerobounce.go @@ -48,18 +48,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.zerobounce.net/v2/getcredits?api_key="+resMatch, nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.zerobounce.net/v1/activity?email=testemail@email.com&api_key="+resMatch, nil) if err != nil { continue } - res, err := client.Do(req) if err == nil { defer res.Body.Close() - bodyBytes, _ := ioutil.ReadAll(res.Body) + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + continue + } body := string(bodyBytes) - if !strings.Contains(body, "-1") { + if strings.Contains(body, "found") { s1.Verified = true } else { if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {