From 94e2ffd541d3feec637d1cfa25480cfa8d4a4934 Mon Sep 17 00:00:00 2001 From: Vernell Parker Date: Tue, 15 Jun 2021 20:48:35 -0400 Subject: [PATCH 1/3] Refactored logic for GetCloudAccountEvaluations to predetermine clould account types and only make one call to qualys --- .gitignore | 1 - go.mod | 1 + go.sum | 6 ++++ pkg/qualys/api_cloudview.go | 59 ++++++++++++++++---------------- pkg/qualys/api_cloudview_test.go | 23 +++++++++++++ 5 files changed, 60 insertions(+), 30 deletions(-) create mode 100644 pkg/qualys/api_cloudview_test.go diff --git a/.gitignore b/.gitignore index 0a76837..4bcfed1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ dev.json exceptions/ prod.json *app.json -prod.json *.xml *.csv scaffold.json diff --git a/go.mod b/go.mod index 999c74e..d41479c 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/nortonlifelock/scaffold v1.0.1-0.20200128220520-41da8a42d6d5 github.com/pkg/errors v0.9.1 github.com/rs/cors v1.7.0 + github.com/stretchr/testify v1.7.0 // indirect github.com/trivago/tgo v1.0.7 golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d gopkg.in/korylprince/go-ad-auth.v2 v2.2.0 diff --git a/go.sum b/go.sum index 8232fc2..0d8a737 100644 --- a/go.sum +++ b/go.sum @@ -79,9 +79,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/trivago/tgo v1.0.1/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM= github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc= @@ -101,7 +104,10 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/korylprince/go-ad-auth.v2 v2.2.0 h1:8shjo9Maud/IwUiuWNAaBPLweuX62rQ2Y+gUQFaS6qI= gopkg.in/korylprince/go-ad-auth.v2 v2.2.0/go.mod h1:CTCIfxRcFmT6LwxKFX83iXjvaz/gn10veNfiCbOtY74= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/qualys/api_cloudview.go b/pkg/qualys/api_cloudview.go index 03c22a9..80d0bf6 100644 --- a/pkg/qualys/api_cloudview.go +++ b/pkg/qualys/api_cloudview.go @@ -7,6 +7,7 @@ import ( "github.com/nortonlifelock/aegis/pkg/log" "io/ioutil" "net/http" + "regexp" "strings" "time" ) @@ -102,36 +103,31 @@ const ( GOOGLE_CLOUD_ACCOUNT = "gcp" ) +// getAccountType determines the account type +func getAccountType(accountID string) string { + matched := false + // AWS + if matched, _ = regexp.MatchString(`^\d{12}$`, accountID); matched{ + return AWS_CLOUD_ACCOUNT + } + // Azure + if matched, _ = regexp.MatchString(`^[0-9A-Fa-f]{8}-([0-9A-Fa-f]{4}-){3}[0-9A-Fa-f]{12}$`, accountID); matched { + return AZURE_CLOUD_ACCOUNT + } + //GCP + return GOOGLE_CLOUD_ACCOUNT +} + func (session *Session) GetCloudAccountEvaluations(accountID string) (evaluations []AccountEvaluationContent, cloudAccountType string, err error) { - var possibleAccountTypes = []string{AWS_CLOUD_ACCOUNT, AZURE_CLOUD_ACCOUNT, GOOGLE_CLOUD_ACCOUNT} + accountType := getAccountType(accountID) - // from looking at the API documentation, I don't see a way to find the cloud account type by using the cloud account ID alone - // so we just check all three and use one if it's present - for _, possibleCloudAccountType := range possibleAccountTypes { - if len(cloudAccountType) > 0 { - break - } - var possibleEvals []AccountEvaluationContent - - if possibleEvals, err = session.GetCloudAccountEvaluationsWithCloudAccountType(accountID, possibleCloudAccountType); err == nil { - for _, eval := range possibleEvals { - if eval.FailedResources > 0 || eval.PassedResources > 0 { - evaluations = possibleEvals - cloudAccountType = possibleCloudAccountType - break - } - } - } else { - err = fmt.Errorf("error while determining cloud account type for evaluation gathering [%s|%s] - %s", accountID, possibleCloudAccountType, err.Error()) - break - } - } + var evals []AccountEvaluationContent - if err == nil && len(cloudAccountType) == 0 { - err = fmt.Errorf("could not determine cloud account type for accountID [%s]", accountID) + if evals, err = session.GetCloudAccountEvaluationsWithCloudAccountType(accountID, accountType); err != nil { + err = fmt.Errorf("error while gathering evaluations [%s|%s] - %s", accountID, accountType, err.Error()) } - return evaluations, cloudAccountType, err + return evals, accountType, err } func (session *Session) GetCloudAccountEvaluationsWithCloudAccountType(accountID string, cloudAccountType string) (evaluations []AccountEvaluationContent, err error) { @@ -143,8 +139,12 @@ func (session *Session) GetCloudAccountEvaluationsWithCloudAccountType(accountID for !lastAccPage { var req *http.Request - req, err = http.NewRequest(http.MethodGet, session.Config.Address()+fmt.Sprintf("/cloudview-api/rest/v1/%s/evaluations/%s?pageNo=%d&sortOrder=asc&filter=evaluatedOn%3A%5Bnow-24h%20..%20now-1s%5D%20and%20(policy.name%3ACIS%20Amazon%20Web%20Services%20Foundations%20Benchmark%20or%20policy.name%3AAegis%20AWS%20Benchmark%20or%20policy.name%3ACIS%20Microsoft%20Azure%20Foundations%20Benchmark%20or%20policy.name%3ACIS%20Google%20Cloud%20Platform%20Foundation%20Benchmark)", cloudAccountType, accountID, accPage), nil) - if err == nil { + // URL was split as percent signs in url affected Sprintf when running unit test + req, err = http.NewRequest(http.MethodGet, session.Config.Address()+fmt.Sprintf("/cloudview-api/rest/v1/%s/evaluations/%s?pageNo=%d&sortOrder=asc&filter=evaluatedOn", cloudAccountType, accountID, accPage) + + "%3A%5Bnow-24h%20..%20now-1s%5D%20and%20(policy.name%3ACIS%20Amazon%20Web%20Services%20Foundations%20Benchmark%20or%20policy.name%3AAegis%20AWS%20Benchmark%20or%20policy.name%3ACIS%20Microsoft%20Azure%20Foundations%20Benchmark%20or%20policy.name%3ACIS%20Google%20Cloud%20Platform%20Foundation%20Benchmark)", + nil) + + if err != nil { err = session.makeRequest(false, req, func(resp *http.Response) (err error) { var body []byte body, err = ioutil.ReadAll(resp.Body) @@ -179,9 +179,10 @@ func (session *Session) GetCloudEvaluationFindings(accountID string, content Acc for !last { evaluationResult := &EvaluationResult{} - var req *http.Request - req, err = http.NewRequest(http.MethodGet, session.Config.Address()+fmt.Sprintf("/cloudview-api/rest/v1/%s/evaluations/%s/resources/%s?pageNo=%d&pageSize=10000&sortOrder=asc&filter=evaluatedOn%3A%5Bnow-24h%20..%20now-1s%5D", cloudAccountType, accountID, content.ControlID, page), nil) // TODO qualys sorting does not seem to be working + // URL was split as percent signs in url affected Sprintf when running unit test + req, err = http.NewRequest(http.MethodGet, session.Config.Address()+fmt.Sprintf("/cloudview-api/rest/v1/%s/evaluations/%s/resources/%s?pageNo=%d", cloudAccountType, accountID, content.ControlID, page) + + "&pageSize=10000&sortOrder=asc&filter=evaluatedOn%3A%5Bnow-24h%20..%20now-1s%5D", nil) // TODO qualys sorting does not seem to be working if err == nil { err = session.makeRequest(false, req, func(resp *http.Response) (err error) { var body []byte diff --git a/pkg/qualys/api_cloudview_test.go b/pkg/qualys/api_cloudview_test.go new file mode 100644 index 0000000..0906be0 --- /dev/null +++ b/pkg/qualys/api_cloudview_test.go @@ -0,0 +1,23 @@ +package qualys + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestGetAccountType(t *testing.T) { + t.Run("Should return as an AWS account", func(t *testing.T) { + mockAWS := "123456789012" + assert.Equal(t,AWS_CLOUD_ACCOUNT,getAccountType(mockAWS)) + }) + t.Run("Should return as an Azure account", func(t *testing.T) { + mockAzure := "20ff7fc3-e762-44dd-bd96-b71116dcdc23" + assert.Equal(t,AZURE_CLOUD_ACCOUNT,getAccountType(mockAzure)) + }) + t.Run("Should return as an GCP account", func(t *testing.T) { + mockGCP := "12we" + assert.Equal(t,GOOGLE_CLOUD_ACCOUNT,getAccountType(mockGCP)) + }) + +} + From d162f1a89e0bd283f5b78da38bf816568169e5a8 Mon Sep 17 00:00:00 2001 From: Vernell Parker Date: Wed, 16 Jun 2021 10:28:20 -0400 Subject: [PATCH 2/3] Added comments to explain chances that were made --- pkg/qualys/api_cloudview.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/qualys/api_cloudview.go b/pkg/qualys/api_cloudview.go index 80d0bf6..3a3727c 100644 --- a/pkg/qualys/api_cloudview.go +++ b/pkg/qualys/api_cloudview.go @@ -103,7 +103,7 @@ const ( GOOGLE_CLOUD_ACCOUNT = "gcp" ) -// getAccountType determines the account type +// getAccountType determines the account type. AccountID are determined through regexp filters and can be added to this func func getAccountType(accountID string) string { matched := false // AWS @@ -117,7 +117,7 @@ func getAccountType(accountID string) string { //GCP return GOOGLE_CLOUD_ACCOUNT } - +// GetCloudAccountEvaluations was refactored with the ability to determine the account type via the getAccountType function. This removes unnecessary calls to the qualys api func (session *Session) GetCloudAccountEvaluations(accountID string) (evaluations []AccountEvaluationContent, cloudAccountType string, err error) { accountType := getAccountType(accountID) From bde11319182880a7409f24dadf84987f9ac40d86 Mon Sep 17 00:00:00 2001 From: Vernell Parker Date: Wed, 16 Jun 2021 17:40:26 -0400 Subject: [PATCH 3/3] Added and removed some error handling --- pkg/qualys/api_cloudview.go | 40 +++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/pkg/qualys/api_cloudview.go b/pkg/qualys/api_cloudview.go index 3a3727c..705a0cb 100644 --- a/pkg/qualys/api_cloudview.go +++ b/pkg/qualys/api_cloudview.go @@ -120,7 +120,6 @@ func getAccountType(accountID string) string { // GetCloudAccountEvaluations was refactored with the ability to determine the account type via the getAccountType function. This removes unnecessary calls to the qualys api func (session *Session) GetCloudAccountEvaluations(accountID string) (evaluations []AccountEvaluationContent, cloudAccountType string, err error) { accountType := getAccountType(accountID) - var evals []AccountEvaluationContent if evals, err = session.GetCloudAccountEvaluationsWithCloudAccountType(accountID, accountType); err != nil { @@ -144,29 +143,26 @@ func (session *Session) GetCloudAccountEvaluationsWithCloudAccountType(accountID "%3A%5Bnow-24h%20..%20now-1s%5D%20and%20(policy.name%3ACIS%20Amazon%20Web%20Services%20Foundations%20Benchmark%20or%20policy.name%3AAegis%20AWS%20Benchmark%20or%20policy.name%3ACIS%20Microsoft%20Azure%20Foundations%20Benchmark%20or%20policy.name%3ACIS%20Google%20Cloud%20Platform%20Foundation%20Benchmark)", nil) - if err != nil { - err = session.makeRequest(false, req, func(resp *http.Response) (err error) { - var body []byte - body, err = ioutil.ReadAll(resp.Body) - if err == nil { - err = json.Unmarshal(body, accountEvaluation) - } else { - err = fmt.Errorf("error while reading response body - %s", err.Error()) - } - - return err - }) - } else { - err = fmt.Errorf("error while making request - %s", err.Error()) + if err != nil{ + err = fmt.Errorf("error creating url for request to Qualys - %s", err.Error()) + return nil, err } - if err == nil { - evaluations = append(evaluations, accountEvaluation.Content...) - lastAccPage = accountEvaluation.Last - accPage++ - } else { - break - } + err = session.makeRequest(false, req, func(resp *http.Response) (err error) { + var body []byte + body, err = ioutil.ReadAll(resp.Body) + if err == nil { + err = json.Unmarshal(body, accountEvaluation) + } else { + err = fmt.Errorf("error while reading response body - %s", err.Error()) + } + + return err + }) + + evaluations = append(evaluations, accountEvaluation.Content...) + lastAccPage = accountEvaluation.Last + accPage++ } return evaluations, err