feat: dora config (#8103)

* feat: github/graphql dora config and adjust regex function (#8026) * feat: adjust the part of github scope config cicd * fix: github dora config api (#8032) * feat: jenkins dora config (#8033) * feat: circleci dora config (#8038) * feat: bitbucket dora config (#8043) * fix: check match items cannot iterable * fix: some style for github transformation cicd * feat: bamboo dora config (#8045) * feat: gitlab dora config + bamboo fix (#8047) * fix: adjust the return value for api * feat: adjust the github transformation cicd style * feat: regex match nil value (#8051) * fix: check match items total * fix: useCustom just defined onece * chore: upgrade antd and @ant-design/icons * fix: check match items total logic error * feat: add github workflow run guideline * fix: return values (#8052) * feat: azure dora config (#8053) * fix: prod match when is nil (#8056) * fix: some bugs for dora config * fix: adjust the deploymentPattern and productionPattern default value * feat: add a new component ShowMore * refactor: use ShowMore component in github transformation * feat: imporve the jenkins transformation cicd configuration * feat: imporve the gitlab transformation cicd configuration * feat: improve the bitbucket transformation cicd configuration * feat: improve the azuredevops transformation cicd configuration * feat: improve the circleci transformation cicd configuration * fix: update github trans-to-deployment scope config to pointer (#8060) * fix: update gitlab trans-to-deployment scope config to pointer (#8062) * fix: update bamboo/bitbuctet/jenkins trans-to-deployment scope config… (#8065) * fix: update bamboo/bitbuctet/jenkins trans-to-deployment scope config to pointer * fix: update circleci trans-to-deployment scope config to pointer * fix: bamboo lint * feat: improve the bamboo transformation cicd configuration * fix: not reset state in check matched items * fix: remove default value about jenkins cicd * fix: cannot check in bitbucket cicd * fix: error default value in deployment * fix: jenkins can not get scope config (#8080) * fix: jenkins can not get scope config * fix: jenkins can not get scope config * fix: some error about miller column * fix: deployment doesn't set value * fix: bitbucket sql (#8085) * fix: missed cicd default value in circleci * fix: throw the error when check matched items request failed * fix: enrich with regex can not match (#8090) * fix: enrich with regex can not match * fix: cannot check in bamboo cicd * fix: bamboo using api client in tool/domain layer (#8094) * fix: not set value for envNamePattern * fix: bamboo repo url * fix: type error * fix: bitbucket e2e * fix: bitbucket e2e * fix: circle ci e2e test * fix: circle ci e2e test * fix: github e2e test * fix: gitlab e2e test * fix: jenkins e2e test * fix: jenkins e2e test * fix: jenkins e2e test * fix: jenkins e2e test --------- Co-authored-by: mintsweet <[email protected]>
apache · Sep 25, 2024 · fb0dd2e · fb0dd2e
1 parent 323b98e
commit fb0dd2e
Show file tree

Hide file tree

Showing 125 changed files with 3,266 additions and 803 deletions.
diff --git a/backend/core/models/domainlayer/devops/cicd_task.go b/backend/core/models/domainlayer/devops/cicd_task.go
@@ -36,6 +36,16 @@ const (
 
 const ENV_NAME_PATTERN = "ENV_NAME_PATTERN"
 
+type TransformDeployment struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
+type TransformDeploymentResponse struct {
+	Total int                   `json:"total"`
+	Data  []TransformDeployment `json:"data"`
+}
+
 type CICDTask struct {
 	domainlayer.DomainEntity
 	Name              string `gorm:"type:varchar(255)"`

diff --git a/backend/helpers/pluginhelper/api/enrich_with_regex.go b/backend/helpers/pluginhelper/api/enrich_with_regex.go
@@ -28,12 +28,13 @@ import (
 // TODO: remove Enricher from naming since it is more like a util function
 type RegexEnricher struct {
 	// This field will store compiled regular expression for every pattern
-	regexpMap map[string]*regexp.Regexp
+	regexpMap    map[string]*regexp.Regexp
+	regexMapList map[string][]*regexp.Regexp
 }
 
 // NewRegexEnricher initialize a regexEnricher
 func NewRegexEnricher() *RegexEnricher {
-	return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp)}
+	return &RegexEnricher{regexpMap: make(map[string]*regexp.Regexp), regexMapList: make(map[string][]*regexp.Regexp)}
 }
 
 // AddRegexp will add compiled regular expression for pattern to regexpMap
@@ -105,3 +106,51 @@ func (r *RegexEnricher) ReturnNameIfOmittedOrMatched(name string, targets ...str
 	}
 	return r.ReturnNameIfMatched(name, targets...)
 }
+
+// TryAdd a named regexp if given pattern is not empty
+func (r *RegexEnricher) TryAddList(name string, patterns ...string) errors.Error {
+	if _, ok := r.regexMapList[name]; ok {
+		return errors.Default.New(fmt.Sprintf("Regex pattern with name: %s already exists", name))
+	}
+	var regexList []*regexp.Regexp
+	for _, pattern := range patterns {
+		if pattern == "" {
+			continue
+		}
+		regex, err := errors.Convert01(regexp.Compile(pattern))
+		if err != nil {
+			return errors.BadInput.Wrap(err, fmt.Sprintf("Fail to compile pattern for regex pattern: %s", pattern))
+		}
+		regexList = append(regexList, regex)
+	}
+
+	// Only save non-empty regexList
+	if len(regexList) > 0 {
+		r.regexMapList[name] = regexList
+	}
+	return nil
+}
+
+// ReturnNameIfMatched will return name if any of the targets matches the regex associated with the given name
+func (r *RegexEnricher) ReturnNameIfMatchedList(name string, targets ...string) string {
+	if regexList, ok := r.regexMapList[name]; !ok {
+		return ""
+	} else {
+		for _, regex := range regexList {
+			for _, target := range targets {
+				if regex.MatchString(target) {
+					return name
+				}
+			}
+		}
+		return "" // If any regex fails to match, return ""
+	}
+}
+
+// ReturnNameIfOmittedOrMatched returns the given name if regex of the given name is omitted or fallback to ReturnNameIfMatched
+func (r *RegexEnricher) ReturnNameIfOmittedOrMatchedList(name string, targets ...string) string {
+	if _, ok := r.regexMapList[name]; !ok {
+		return name
+	}
+	return r.ReturnNameIfMatched(name, targets...)
+}
diff --git a/backend/helpers/pluginhelper/api/pagenation.go b/backend/helpers/pluginhelper/api/pagenation.go
@@ -20,6 +20,8 @@ package api
 import (
 	"net/url"
 	"strconv"
+
+	"github.com/go-errors/errors"
 )
 
 const pageSize = 50
@@ -47,3 +49,25 @@ func GetLimitOffset(q url.Values, pageSizeKey, pageKey string) (limit int, offse
 	offset = (page - 1) * limit
 	return limit, offset
 }
+
+func ParsePageParam(body map[string]interface{}, paramName string, defaultValue int) (int, error) {
+	value, exists := body[paramName]
+	if !exists {
+		return defaultValue, nil
+	}
+
+	switch v := value.(type) {
+	case int:
+		return v, nil
+	case float64:
+		return int(v), nil
+	case string:
+		parsedValue, err := strconv.Atoi(v)
+		if err != nil {
+			return 0, errors.New("invalid " + paramName + " value")
+		}
+		return parsedValue, nil
+	default:
+		return 0, errors.New(paramName + " must be int or string")
+	}
+}
diff --git a/backend/plugins/bamboo/api/connection_api.go b/backend/plugins/bamboo/api/connection_api.go
@@ -21,10 +21,12 @@ import (
 	"context"
 	"net/http"
 
+	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/bamboo/models"
+	"github.com/apache/incubator-devlake/plugins/bamboo/tasks"
 	"github.com/apache/incubator-devlake/server/api/shared"
 )
 
@@ -162,3 +164,128 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
 func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
 	return dsHelper.ConnApi.GetDetail(input)
 }
+
+// GetConnectionDeployments return one connection deployments
+// @Summary return one connection deployments
+// @Description return one connection deployments
+// @Tags plugins/bamboo
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200  {array} string "List of Environment Names"
+// @Failure 400  {object} shared.ApiBody "Bad Request"
+// @Failure 500  {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bamboo/connections/{connectionId}/deployments [GET]
+func GetConnectionDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	db := basicRes.GetDal()
+	connectionId := input.Params["connectionId"]
+	var environments []string
+	err := db.All(&environments,
+		dal.From(&models.BambooDeployBuild{}),
+		dal.Where("connection_id = ?", connectionId),
+		dal.Select("DISTINCT environment"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &plugin.ApiResourceOutput{
+		Body: environments,
+	}, nil
+}
+
+// GetConnectionTransformToDeployments return one connection deployments
+// @Summary return one connection deployments
+// @Description return one connection deployments
+// @Tags plugins/bamboo
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200  {object} map[string]interface{}
+// @Failure 400  {object} shared.ApiBody "Bad Request"
+// @Failure 500  {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bamboo/connections/{connectionId}/transform-to-deployments [POST]
+func GetConnectionTransformToDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	db := basicRes.GetDal()
+	connectionId := input.Params["connectionId"]
+	deploymentPattern := input.Body["deploymentPattern"]
+	productionPattern := input.Body["productionPattern"]
+	page, err := api.ParsePageParam(input.Body, "page", 1)
+	if err != nil {
+		return nil, errors.Default.New("invalid page value")
+	}
+	pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10)
+	if err != nil {
+		return nil, errors.Default.New("invalid pageSize value")
+	}
+
+	cursor, err := db.RawCursor(`
+		SELECT DISTINCT plan_build_key, link_href, build_started_time
+		FROM(
+			SELECT plan_build_key, link_href, build_started_time
+			FROM _tool_bamboo_plan_builds
+			WHERE connection_id = ? 
+			    AND (plan_name REGEXP ?)
+    			AND (? = '' OR plan_name REGEXP ?)
+			UNION
+			SELECT p.plan_build_key, p.link_href, p.build_started_time
+			FROM _tool_bamboo_job_builds j
+			LEFT JOIN _tool_bamboo_plan_builds p on p.plan_build_key = j.plan_build_key
+			WHERE j.connection_id = ? 
+			    AND (j.job_name REGEXP ?)
+    			AND (? = '' OR j.job_name REGEXP ?)
+			ORDER BY build_started_time DESC
+		) AS t
+		ORDER BY build_started_time DESC
+	`, connectionId, deploymentPattern, productionPattern, productionPattern, connectionId, deploymentPattern, productionPattern, productionPattern)
+	if err != nil {
+		return nil, errors.Default.Wrap(err, "error on get")
+	}
+	defer cursor.Close()
+
+	type selectFileds struct {
+		PlanBuildKey string
+		LinkHref     string
+	}
+	type transformedFields struct {
+		Name string `json:"name"`
+		URL  string `json:"url"`
+	}
+	var allRuns []transformedFields
+	for cursor.Next() {
+		sf := &selectFileds{}
+		err = db.Fetch(cursor, sf)
+		if err != nil {
+			return nil, errors.Default.Wrap(err, "error on fetch")
+		}
+		// Directly transform and append to allRuns
+		url, err := tasks.GetBambooHomePage(sf.LinkHref)
+		if err != nil {
+			url = sf.LinkHref
+		}
+		transformed := transformedFields{
+			Name: sf.PlanBuildKey,
+			URL:  url + "/browse/" + sf.PlanBuildKey,
+		}
+		allRuns = append(allRuns, transformed)
+	}
+	// Calculate total count
+	totalCount := len(allRuns)
+
+	// Paginate in memory
+	start := (page - 1) * pageSize
+	end := start + pageSize
+	if start > totalCount {
+		start = totalCount
+	}
+	if end > totalCount {
+		end = totalCount
+	}
+	pagedRuns := allRuns[start:end]
+
+	// Return result containing paged runs and total count
+	result := map[string]interface{}{
+		"total": totalCount,
+		"data":  pagedRuns,
+	}
+	return &plugin.ApiResourceOutput{
+		Body: result,
+	}, nil
+}
diff --git a/backend/plugins/bamboo/e2e/deploy_build_test.go b/backend/plugins/bamboo/e2e/deploy_build_test.go
@@ -41,20 +41,22 @@ func getFakeAPIClient() *helper.ApiAsyncClient {
 func TestBambooDeployBuildDataFlow(t *testing.T) {
 	var bamboo impl.Bamboo
 	dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
+	dPattern := "(?i)release"
+	pPattern := "(?i)release"
 	taskData := &tasks.BambooOptions{
 		Options: &models.BambooOptions{
 			ConnectionId: 1,
 			PlanKey:      "TEST-PLA2",
 			BambooScopeConfig: &models.BambooScopeConfig{
-				DeploymentPattern: "(?i)release",
-				ProductionPattern: "(?i)release",
+				DeploymentPattern: &dPattern,
+				ProductionPattern: &pPattern,
 			},
 		},
 		RegexEnricher: helper.NewRegexEnricher(),
 		ApiClient:     getFakeAPIClient(),
 	}
-	taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern)
-	taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern)
+	taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, *taskData.Options.DeploymentPattern)
+	taskData.RegexEnricher.TryAdd(devops.PRODUCTION, *taskData.Options.ProductionPattern)
 	// import raw data table
 	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bamboo_api_deploy_builds.csv", "_raw_bamboo_api_deploy_builds")
 
@@ -102,7 +104,7 @@ func TestBambooDeployBuildDataFlow(t *testing.T) {
 	dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{}, e2ehelper.TableOptions{
 		CSVRelPath:   "./snapshot_tables/cicd_deployment_commits.csv",
 		IgnoreTypes:  []interface{}{common.NoPKModel{}},
-		IgnoreFields: []string{"created_date", "queued_date", "started_date", "finished_date"},
+		IgnoreFields: []string{"created_date", "queued_date", "started_date", "finished_date", "repo_url"},
 	})
 	dataflowTester.VerifyTableWithOptions(&devops.CicdDeploymentCommit{}, e2ehelper.TableOptions{
 		CSVRelPath:   "./snapshot_tables/cicd_deployments.csv",

diff --git a/backend/plugins/bamboo/e2e/job_build_test.go b/backend/plugins/bamboo/e2e/job_build_test.go
@@ -30,22 +30,23 @@ import (
 )
 
 func TestBambooJobBuildDataFlow(t *testing.T) {
-
 	var bamboo impl.Bamboo
 	dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
+	dPattern := "(?i)release"
+	pPattern := "(?i)release"
 	taskData := &tasks.BambooOptions{
 		Options: &models.BambooOptions{
 			ConnectionId: 1,
 			PlanKey:      "TEST-PLA3",
 			BambooScopeConfig: &models.BambooScopeConfig{
-				DeploymentPattern: "(?i)compile",
-				ProductionPattern: "(?i)compile",
+				DeploymentPattern: &dPattern,
+				ProductionPattern: &pPattern,
 			},
 		},
 		RegexEnricher: helper.NewRegexEnricher(),
 		ApiClient:     getFakeAPIClient(),
 	}
-	taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, taskData.Options.DeploymentPattern)
+	taskData.RegexEnricher.TryAdd(devops.DEPLOYMENT, *taskData.Options.DeploymentPattern)
 	// import raw data table
 	// SELECT * FROM _raw_bamboo_api_job_build INTO OUTFILE "/tmp/_raw_bamboo_api_job_builds.csv" FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' LINES TERMINATED BY '\r\n';
 	dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bamboo_api_job_builds.csv", "_raw_bamboo_api_job_builds")
@@ -63,7 +64,7 @@ func TestBambooJobBuildDataFlow(t *testing.T) {
 	)
 
 	// verify extraction
-	taskData.RegexEnricher.TryAdd(devops.PRODUCTION, taskData.Options.ProductionPattern)
+	taskData.RegexEnricher.TryAdd(devops.PRODUCTION, *taskData.Options.ProductionPattern)
 	dataflowTester.FlushTabler(&models.BambooJobBuild{})
 	dataflowTester.Subtask(tasks.ExtractJobBuildMeta, taskData)
 	dataflowTester.VerifyTable(

diff --git a/backend/plugins/bamboo/e2e/job_test.go b/backend/plugins/bamboo/e2e/job_test.go
@@ -30,14 +30,15 @@ func TestBambooJobDataFlow(t *testing.T) {
 
 	var bamboo impl.Bamboo
 	dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
-
+	dPattern := "(?i)release"
+	pPattern := "(?i)release"
 	taskData := &tasks.BambooOptions{
 		Options: &models.BambooOptions{
 			ConnectionId: 1,
 			PlanKey:      "TEST-PLA1",
 			BambooScopeConfig: &models.BambooScopeConfig{
-				DeploymentPattern: "(?i)compile",
-				ProductionPattern: "(?i)compile",
+				DeploymentPattern: &dPattern,
+				ProductionPattern: &pPattern,
 			},
 		},
 		ApiClient: getFakeAPIClient(),

diff --git a/backend/plugins/bamboo/e2e/plan_build_commits_test.go b/backend/plugins/bamboo/e2e/plan_build_commits_test.go
@@ -29,17 +29,17 @@ import (
 )
 
 func TestBambooPlanBuildCommitsDataFlow(t *testing.T) {
-
 	var bamboo impl.Bamboo
 	dataflowTester := e2ehelper.NewDataFlowTester(t, "bamboo", bamboo)
-
+	dPattern := "(?i)release"
+	pPattern := "(?i)release"
 	taskData := &tasks.BambooOptions{
 		Options: &models.BambooOptions{
 			ConnectionId: 1,
 			PlanKey:      "TEST-PLA2",
 			BambooScopeConfig: &models.BambooScopeConfig{
-				DeploymentPattern: "(?i)compile",
-				ProductionPattern: "(?i)compile",
+				DeploymentPattern: &dPattern,
+				ProductionPattern: &pPattern,
 			},
 		},
 		ApiClient: getFakeAPIClient(),