Detector improvements (trufflesecurity#125)

* New tokens and endpoints  (trufflesecurity#115)

Co-authored-by: dmarquero <[email protected]>

pkg/common/http.go

@@ -127,3 +127,11 @@ func SaneHttpClient() *http.Client {
 	httpClient.Transport = NewCustomTransport(nil)
 	return httpClient
 }
+
+//custom timeout for some scanners
+func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
+	httpClient := &http.Client{}
+	httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
+	httpClient.Transport = NewCustomTransport(nil)
+	return httpClient
+}

pkg/detectors/allsports/allsports.go

@@ -58,7 +58,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
 				body := string(bodyBytes)
 
 				if strings.Contains(body, "success") {

pkg/detectors/amadeus/amadeus.go

@@ -65,7 +65,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
-					bodyBytes, _ := ioutil.ReadAll(res.Body)
+					bodyBytes, err := ioutil.ReadAll(res.Body)
+					if err != nil {
+						continue
+					}
 					body := string(bodyBytes)
 					if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") {
 						s1.Verified = true

pkg/detectors/auth0oauth/auth0oauth.go

@@ -90,7 +90,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					res, err := client.Do(req)
 					if err == nil {
 						defer res.Body.Close()
-						bodyBytes, _ := ioutil.ReadAll(res.Body)
+						bodyBytes, err := ioutil.ReadAll(res.Body)
+						if err != nil {
+							continue
+						}
 						body := string(bodyBytes)
 
 						// if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"}

pkg/detectors/baseapiio/baseapiio.go

@@ -57,7 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
 				body := string(bodyBytes)
 
 				if strings.Contains(body, "items") {

pkg/detectors/besttime/besttime.go

@@ -55,7 +55,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
 				body := string(bodyBytes)
 
 				if strings.Contains(body, `"status": "OK"`) {

pkg/detectors/borgbase/borgbase.go

@@ -64,13 +64,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				bodyBytes, err := ioutil.ReadAll(res.Body)
 				if err == nil {
 					bodyString := string(bodyBytes)
-					errCode := strings.Contains(bodyString, `"errors"`)
+					validResponse := strings.Contains(bodyString, `"sshList":[]`)
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						if errCode {
-							s1.Verified = false
-						} else {
+						if validResponse {
 							s1.Verified = true
+						} else {
+							s1.Verified = false
 						}
 					} else {
 						//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key

pkg/detectors/bulbul/bulbul.go

@@ -55,7 +55,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			res, err := client.Do(req)
 			if err == nil {
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+
+				if err != nil {
+					continue
+				}
 
 				bodyString := string(bodyBytes)
 				validResponse := strings.Contains(bodyString, `"message":"Successful",`)

pkg/detectors/cexio/cexio.go

@@ -92,14 +92,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						defer res.Body.Close()
 
 						body, err := ioutil.ReadAll(res.Body)
+						if err != nil {
+							continue
+						}
+						bodyString := string(body)
+						validResponse := strings.Contains(bodyString, `timestamp`)
 						if err != nil {
 							fmt.Print(err.Error())
 						}
 
 						var responseObject Response
 						json.Unmarshal(body, &responseObject)
 
-						if res.StatusCode >= 200 && res.StatusCode < 300 && responseObject.Error == "" {
+						if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
 							s1.Verified = true
 						} else {
 							//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key

pkg/detectors/coinlayer/coinlayer.go

@@ -49,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.coinlayer.com/api/live?access_key=%s", resMatch), nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.coinlayer.com/api/livelive?access_key=%s", resMatch), nil)
 			if err != nil {
 				continue
 			}
@@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				bodyBytes, err := ioutil.ReadAll(res.Body)
 				if err == nil {
 					bodyString := string(bodyBytes)
-					errCode := strings.Contains(bodyString, `"code":101`)
+					validResponse := strings.Contains(bodyString, `"success": true`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`)
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						if errCode {
-							s1.Verified = false
-						} else {
+						if validResponse {
 							s1.Verified = true
+						} else {
+							s1.Verified = false
 						}
 					} else {
 						//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key

pkg/detectors/currencycloud/currencycloud.go

@@ -66,7 +66,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
-					bodyBytes, _ := ioutil.ReadAll(res.Body)
+					bodyBytes, err := ioutil.ReadAll(res.Body)
+					if err != nil {
+						continue
+					}
 					body := string(bodyBytes)
 					if strings.Contains(body, "auth_token") {
 						s1.Verified = true

pkg/detectors/currencylayer/currencylayer.go

@@ -59,13 +59,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				bodyBytes, err2 := ioutil.ReadAll(res.Body)
 				if err2 == nil {
 					bodyString := string(bodyBytes)
-					errCode := strings.Contains(bodyString, `"code":101`)
+					validResponse := strings.Contains(bodyString, `"success": true`) || strings.Contains(bodyString, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`)
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						if errCode {
-							s1.Verified = false
-						} else {
+						if validResponse {
 							s1.Verified = true
+						} else {
+							s1.Verified = false
 						}
 					} else {
 						//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key

pkg/detectors/dyspatch/dyspatch.go

@@ -58,10 +58,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
 				body := string(bodyBytes)
+				validResponse := strings.Contains(body, "limited_usage") || strings.Contains(body, "data")
 
-				if !strings.Contains(body, "unauthenticated") {
+				if validResponse {
 					s1.Verified = true
 				} else {
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {

pkg/detectors/elasticemail/elasticemail.go

@@ -67,8 +67,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			defer res.Body.Close()
-			data, err := ioutil.ReadAll(res.Body)
-			if err != nil {
+			data, readErr := ioutil.ReadAll(res.Body)
+			if readErr != nil {
 				continue
 			}
 			var ResVar struct {

pkg/detectors/fetchrss/fetchrss.go

@@ -48,14 +48,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://fetchrss.com/api/v1/feed/create?auth="+resMatch, nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://fetchrss.com/api/v1/feed/list?auth="+resMatch, nil)
 			if err != nil {
 				continue
 			}
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
 				body := string(bodyBytes)
 
 				if !strings.Contains(body, "Not authorised") {

pkg/detectors/fetchrss/fetchrss_test.go

@@ -20,7 +20,7 @@ func TestFetchrss_FromChunk(t *testing.T) {
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("FETCHRSS")
+	secret := testSecrets.MustGetField("FETCHRSS_TOKEN")
 	inactiveSecret := testSecrets.MustGetField("FETCHRSS_INACTIVE")
 
 	type args struct {

pkg/detectors/financialmodelingprep/financialmodelingprep.go

@@ -56,16 +56,18 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				bodyBytes, err := ioutil.ReadAll(res.Body)
+				bodyString := string(bodyBytes)
 				if err == nil {
-					bodyString := string(bodyBytes)
-					errCode := strings.Contains(bodyString, `"Error Message"`)
+					// valid response should be an array of currencies
+					// error response is in json
+					validResponse := strings.Contains(bodyString, `[ "`)
 
 					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
-						if errCode {
-							s1.Verified = false
-						} else {
+						if validResponse {
 							s1.Verified = true
+						} else {
+							s1.Verified = false
 						}
 					} else {
 						//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key

pkg/detectors/fixerio/fixerio.go

@@ -56,14 +56,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
 				body := string(bodyBytes)
 
 				// if client_id and client_secret is valid -> 403 {"error":"invalid_grant","error_description":"Invalid authorization code"}
 				// if invalid -> 401 {"error":"access_denied","error_description":"Unauthorized"}
 				// ingenious!
 
-				if !strings.Contains(body, "invalid_access_key") {
+				validResponse := strings.Contains(body, `"success": true`) || strings.Contains(body, `"info":"Access Restricted - Your current Subscription Plan does not support HTTPS Encryption."`)
+				defer res.Body.Close()
+
+				if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
 					s1.Verified = true
 				} else {
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {

pkg/detectors/flickr/flickr.go

@@ -57,7 +57,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				bodyBytes, _ := ioutil.ReadAll(res.Body)
+				bodyBytes, err := ioutil.ReadAll(res.Body)
+				if err != nil {
+					continue
+				}
 				body := string(bodyBytes)
 				if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "owner=") {
 					s1.Verified = true

pkg/detectors/flightstats/flightstats.go

@@ -65,9 +65,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
-					bodyBytes, _ := ioutil.ReadAll(res.Body)
+					bodyBytes, err := ioutil.ReadAll(res.Body)
+					if err != nil {
+						continue
+					}
 					body := string(bodyBytes)
-					if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 403 && strings.Contains(body, "application is not active")) {
+					validResponse := (res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "id")) || (res.StatusCode == 403 && strings.Contains(body, "application is not active"))
+					if validResponse {
 						s1.Verified = true
 					} else {
 						//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key