feat: gitlab dora config + bamboo fix (#8047)

backend/plugins/bamboo/impl/impl.go

@@ -189,8 +189,15 @@ func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
 	if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ProductionPattern); err != nil {
 		return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`")
 	}
-	if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.EnvNamePattern); err != nil {
-		return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
+	if len(op.BambooScopeConfig.EnvNameList) > 0 || (len(op.BambooScopeConfig.EnvNameList) == 0 && op.BambooScopeConfig.EnvNamePattern == "") {
+		if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN, op.BambooScopeConfig.EnvNameList...); err != nil {
+			return nil, errors.BadInput.Wrap(err, "invalid value for `envNameList`")
+		}
+	} else {
+		if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.BambooScopeConfig.EnvNamePattern); err != nil {
+			return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
+		}
+
 	}
 	return &tasks.BambooOptions{
 		Options:       op,

backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go

@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addEnvNameListToScopeConfig)(nil)
+
+type scopeConfig20240906 struct {
+	EnvNameList []string `gorm:"type:json;serializer:json" json:"env_name_list" mapstructure:"env_name_list"`
+}
+
+func (scopeConfig20240906) TableName() string {
+	return "_tool_bamboo_scope_configs"
+}
+
+type addEnvNameListToScopeConfig struct{}
+
+func (*addEnvNameListToScopeConfig) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if err := db.AutoMigrate(&scopeConfig20240906{}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (*addEnvNameListToScopeConfig) Version() uint64 {
+	return 20240906142102
+}
+
+func (*addEnvNameListToScopeConfig) Name() string {
+	return "add env_name_list to _tool_bamboo_scope_configs"
+}

backend/plugins/bamboo/models/migrationscripts/register.go

@@ -38,5 +38,6 @@ func All() []plugin.MigrationScript {
 		new(addMissingPrimaryKeyForBambooPlanBuildVcsRevision),
 		new(addQueuedFieldsInJobBuild20231128),
 		new(addLinkHrefToBambooPlanBuild),
+		new(addEnvNameListToScopeConfig),
 	}
 }

backend/plugins/bamboo/models/scope_config.go

@@ -27,6 +27,7 @@ type BambooScopeConfig struct {
 	DeploymentPattern  string           `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern" gorm:"type:varchar(255)"`
 	ProductionPattern  string           `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"`
 	EnvNamePattern     string           `mapstructure:"envNamePattern,omitempty" json:"envNamePattern" gorm:"type:varchar(255)"`
+	EnvNameList        []string         `gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"`
 }
 
 func (BambooScopeConfig) TableName() string {

backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go

@@ -105,7 +105,7 @@ func ConvertDeployBuildsToDeployments(taskCtx plugin.SubTaskContext) errors.Erro
 				},
 				DisplayTitle: name,
 			}
-			if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, input.Environment) != "" {
+			if data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN, input.Environment) != "" {
 				deployment.Environment = devops.PRODUCTION
 			}
 			if input.FinishedDate != nil && input.ExecutedDate != nil {

backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go

@@ -48,5 +48,5 @@ func (*addEnvNameListToScopeConfig) Version() uint64 {
 }
 
 func (*addEnvNameListToScopeConfig) Name() string {
-	return "add is_draft to _tool_github_pull_requests"
+	return "add env_name_list to _tool_github_scope_configs"
 }

backend/plugins/gitlab/api/connection_api.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/url"
 
+	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -174,3 +175,124 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
 func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
 	return dsHelper.ConnApi.GetDetail(input)
 }
+
+// GetConnectionDeployments return one connection deployments
+// @Summary return one connection deployments
+// @Description return one connection deployments
+// @Tags plugins/gitlab
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200  {array} string "List of Environment Names"
+// @Failure 400  {object} shared.ApiBody "Bad Request"
+// @Failure 500  {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/deployments [GET]
+func GetConnectionDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	db := basicRes.GetDal()
+	connectionId := input.Params["connectionId"]
+	var environments []string
+	err := db.All(&environments,
+		dal.From(&models.GitlabDeployment{}),
+		dal.Where("connection_id = ?", connectionId),
+		dal.Select("DISTINCT environment"))
+	if err != nil {
+		return nil, err
+	}
+
+	return &plugin.ApiResourceOutput{
+		Body: environments,
+	}, nil
+}
+
+// GetConnectionTransformToDeployments return one connection deployments
+// @Summary return one connection deployments
+// @Description return one connection deployments
+// @Tags plugins/gitlab
+// @Param id path int true "id"
+// @Param connectionId path int true "connectionId"
+// @Success 200  {object} map[string]interface{}
+// @Failure 400  {object} shared.ApiBody "Bad Request"
+// @Failure 500  {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/transform-to-deployments [POST]
+func GetConnectionTransformToDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	db := basicRes.GetDal()
+	connectionId := input.Params["connectionId"]
+	deploymentPattern := input.Body["deploymentPattern"]
+	productionPattern := input.Body["productionPattern"]
+	page, err := api.ParsePageParam(input.Body, "page", 1)
+	if err != nil {
+		return nil, errors.Default.New("invalid page value")
+	}
+	pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10)
+	if err != nil {
+		return nil, errors.Default.New("invalid pageSize value")
+	}
+
+	cursor, err := db.RawCursor(`
+		SELECT DISTINCT name, gitlab_id, web_url, started_at
+		FROM (
+			SELECT r.name, p.gitlab_id, p.web_url, p.started_at
+			FROM _tool_gitlab_pipelines p
+			LEFT JOIN _tool_gitlab_projects r on r.gitlab_id = p.project_id
+			WHERE p.connection_id = ? AND ref REGEXP ?
+			AND ref REGEXP ?
+			UNION
+			SELECT r.name, p.gitlab_id, p.web_url, p.started_at
+			FROM _tool_gitlab_pipelines p
+			LEFT JOIN _tool_gitlab_projects r on r.gitlab_id = p.project_id
+			LEFT JOIN _tool_gitlab_jobs j on j.pipeline_id = p.gitlab_id
+			WHERE j.connection_id = ? AND j.name REGEXP ?
+			AND j.name REGEXP ?
+		) r
+		ORDER BY r.started_at DESC
+	`, connectionId, deploymentPattern, productionPattern, connectionId, deploymentPattern, productionPattern)
+	if err != nil {
+		return nil, errors.Default.Wrap(err, "error on get")
+	}
+	defer cursor.Close()
+
+	type selectFileds struct {
+		Name     string
+		GitlabId int
+		WebUrl   string
+	}
+	type transformedFields struct {
+		Name string
+		URL  string
+	}
+	var allRuns []transformedFields
+	for cursor.Next() {
+		sf := &selectFileds{}
+		err = db.Fetch(cursor, sf)
+		if err != nil {
+			return nil, errors.Default.Wrap(err, "error on fetch")
+		}
+		// Directly transform and append to allRuns
+		transformed := transformedFields{
+			Name: fmt.Sprintf("%s-#%d", sf.Name, sf.GitlabId),
+			URL:  sf.WebUrl,
+		}
+		allRuns = append(allRuns, transformed)
+	}
+	// Calculate total count
+	totalCount := len(allRuns)
+
+	// Paginate in memory
+	start := (page - 1) * pageSize
+	end := start + pageSize
+	if start > totalCount {
+		start = totalCount
+	}
+	if end > totalCount {
+		end = totalCount
+	}
+	pagedRuns := allRuns[start:end]
+
+	// Return result containing paged runs and total count
+	result := map[string]interface{}{
+		"total": totalCount,
+		"data":  pagedRuns,
+	}
+	return &plugin.ApiResourceOutput{
+		Body: result,
+	}, nil
+}

backend/plugins/gitlab/impl/impl.go

@@ -212,8 +212,15 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
 	if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ScopeConfig.ProductionPattern); err != nil {
 		return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`")
 	}
-	if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
-		return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
+	if len(op.ScopeConfig.EnvNameList) > 0 || (len(op.ScopeConfig.EnvNameList) == 0 && op.ScopeConfig.EnvNamePattern == "") {
+		if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNameList...); err != nil {
+			return nil, errors.BadInput.Wrap(err, "invalid value for `envNameList`")
+		}
+	} else {
+		if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
+			return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
+		}
+
 	}
 
 	taskData := tasks.GitlabTaskData{
@@ -268,6 +275,12 @@ func (p Gitlab) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
 			"GET": api.GetScopeList,
 			"PUT": api.PutScopes,
 		},
+		"connections/:connectionId/deployments": {
+			"GET": api.GetConnectionDeployments,
+		},
+		"connections/:connectionId/transform-to-deployments": {
+			"POST": api.GetConnectionTransformToDeployments,
+		},
 		"connections/:connectionId/scope-configs": {
 			"POST": api.CreateScopeConfig,
 			"GET":  api.GetScopeConfigList,

backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go

@@ -0,0 +1,52 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addEnvNameListToScopeConfig)(nil)
+
+type scopeConfig20240906 struct {
+	EnvNameList []string `gorm:"type:json;serializer:json" json:"env_name_list" mapstructure:"env_name_list"`
+}
+
+func (scopeConfig20240906) TableName() string {
+	return "_tool_gitlab_scope_configs"
+}
+
+type addEnvNameListToScopeConfig struct{}
+
+func (*addEnvNameListToScopeConfig) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+	if err := db.AutoMigrate(&scopeConfig20240906{}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (*addEnvNameListToScopeConfig) Version() uint64 {
+	return 20240906142101
+}
+
+func (*addEnvNameListToScopeConfig) Name() string {
+	return "add env_name_list to _tool_gitlab_scope_configs"
+}

backend/plugins/gitlab/models/migrationscripts/register.go

@@ -52,5 +52,6 @@ func All() []plugin.MigrationScript {
 		new(addGitlabAssigneeAndReviewerPrimaryKey),
 		new(changeIssueComponentType),
 		new(addIsChildToPipelines240906),
+		new(addEnvNameListToScopeConfig),
 	}
 }

backend/plugins/gitlab/models/scope_config.go

@@ -36,6 +36,7 @@ type GitlabScopeConfig struct {
 	DeploymentPattern    string            `mapstructure:"deploymentPattern" json:"deploymentPattern"`
 	ProductionPattern    string            `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"`
 	EnvNamePattern       string            `mapstructure:"envNamePattern,omitempty" json:"envNamePattern" gorm:"type:varchar(255)"`
+	EnvNameList          []string          `gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"`
 	Refdiff              datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"`
 }
 

backend/plugins/gitlab/tasks/deployment_convertor.go

@@ -129,7 +129,7 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
 				Url:               repo.WebUrl + "/environments",
 			}
 			if data.RegexEnricher != nil {
-				if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, gitlabDeployment.Environment) != "" {
+				if data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN, gitlabDeployment.Environment) != "" {
 					domainDeployCommit.Environment = devops.PRODUCTION
 				}
 			}