From 30c6736e0cef0f62f8ce0d3e08b574a7a34e7881 Mon Sep 17 00:00:00 2001 From: abeizn Date: Fri, 13 Sep 2024 14:43:07 +0800 Subject: [PATCH] feat: gitlab dora config + bamboo fix (#8047) --- backend/plugins/bamboo/impl/impl.go | 11 +- ...40906_add_env_name_list_to_scope_config.go | 52 ++++++++ .../models/migrationscripts/register.go | 1 + backend/plugins/bamboo/models/scope_config.go | 1 + .../deploy_build_to_deployment_convertor.go | 2 +- ...40906_add_env_name_list_to_scope_config.go | 2 +- backend/plugins/gitlab/api/connection_api.go | 122 ++++++++++++++++++ backend/plugins/gitlab/impl/impl.go | 17 ++- ...40906_add_env_name_list_to_scope_config.go | 52 ++++++++ .../models/migrationscripts/register.go | 1 + backend/plugins/gitlab/models/scope_config.go | 1 + .../gitlab/tasks/deployment_convertor.go | 2 +- 12 files changed, 257 insertions(+), 7 deletions(-) create mode 100644 backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go create mode 100644 backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go index 41950364668..15a14016524 100644 --- a/backend/plugins/bamboo/impl/impl.go +++ b/backend/plugins/bamboo/impl/impl.go @@ -189,8 +189,15 @@ func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ProductionPattern); err != nil { return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`") } - if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.EnvNamePattern); err != nil { - return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`") + if len(op.BambooScopeConfig.EnvNameList) > 0 || (len(op.BambooScopeConfig.EnvNameList) == 0 && op.BambooScopeConfig.EnvNamePattern == "") { + if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN, op.BambooScopeConfig.EnvNameList...); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid value for `envNameList`") + } + } else { + if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.BambooScopeConfig.EnvNamePattern); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`") + } + } return &tasks.BambooOptions{ Options: op, diff --git a/backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go b/backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go new file mode 100644 index 00000000000..e0a439e060d --- /dev/null +++ b/backend/plugins/bamboo/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addEnvNameListToScopeConfig)(nil) + +type scopeConfig20240906 struct { + EnvNameList []string `gorm:"type:json;serializer:json" json:"env_name_list" mapstructure:"env_name_list"` +} + +func (scopeConfig20240906) TableName() string { + return "_tool_bamboo_scope_configs" +} + +type addEnvNameListToScopeConfig struct{} + +func (*addEnvNameListToScopeConfig) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&scopeConfig20240906{}); err != nil { + return err + } + return nil +} + +func (*addEnvNameListToScopeConfig) Version() uint64 { + return 20240906142102 +} + +func (*addEnvNameListToScopeConfig) Name() string { + return "add env_name_list to _tool_bamboo_scope_configs" +} diff --git a/backend/plugins/bamboo/models/migrationscripts/register.go b/backend/plugins/bamboo/models/migrationscripts/register.go index d7ef35c7e55..435368026f3 100644 --- a/backend/plugins/bamboo/models/migrationscripts/register.go +++ b/backend/plugins/bamboo/models/migrationscripts/register.go @@ -38,5 +38,6 @@ func All() []plugin.MigrationScript { new(addMissingPrimaryKeyForBambooPlanBuildVcsRevision), new(addQueuedFieldsInJobBuild20231128), new(addLinkHrefToBambooPlanBuild), + new(addEnvNameListToScopeConfig), } } diff --git a/backend/plugins/bamboo/models/scope_config.go b/backend/plugins/bamboo/models/scope_config.go index 587ba991e4d..1ec217baadb 100644 --- a/backend/plugins/bamboo/models/scope_config.go +++ b/backend/plugins/bamboo/models/scope_config.go @@ -27,6 +27,7 @@ type BambooScopeConfig struct { DeploymentPattern string `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern" gorm:"type:varchar(255)"` ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"` EnvNamePattern string `mapstructure:"envNamePattern,omitempty" json:"envNamePattern" gorm:"type:varchar(255)"` + EnvNameList []string `gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"` } func (BambooScopeConfig) TableName() string { diff --git a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go index badf42a530c..ffa6568b230 100644 --- a/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go +++ b/backend/plugins/bamboo/tasks/deploy_build_to_deployment_convertor.go @@ -105,7 +105,7 @@ func ConvertDeployBuildsToDeployments(taskCtx plugin.SubTaskContext) errors.Erro }, DisplayTitle: name, } - if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, input.Environment) != "" { + if data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN, input.Environment) != "" { deployment.Environment = devops.PRODUCTION } if input.FinishedDate != nil && input.ExecutedDate != nil { diff --git a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go b/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go index fe08552adb2..20226b10e97 100644 --- a/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go +++ b/backend/plugins/github/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go @@ -48,5 +48,5 @@ func (*addEnvNameListToScopeConfig) Version() uint64 { } func (*addEnvNameListToScopeConfig) Name() string { - return "add is_draft to _tool_github_pull_requests" + return "add env_name_list to _tool_github_scope_configs" } diff --git a/backend/plugins/gitlab/api/connection_api.go b/backend/plugins/gitlab/api/connection_api.go index 598d5b975e6..1a485d58963 100644 --- a/backend/plugins/gitlab/api/connection_api.go +++ b/backend/plugins/gitlab/api/connection_api.go @@ -23,6 +23,7 @@ import ( "net/http" "net/url" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -174,3 +175,124 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ConnApi.GetDetail(input) } + +// GetConnectionDeployments return one connection deployments +// @Summary return one connection deployments +// @Description return one connection deployments +// @Tags plugins/gitlab +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 {array} string "List of Environment Names" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/gitlab/connections/{connectionId}/deployments [GET] +func GetConnectionDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + db := basicRes.GetDal() + connectionId := input.Params["connectionId"] + var environments []string + err := db.All(&environments, + dal.From(&models.GitlabDeployment{}), + dal.Where("connection_id = ?", connectionId), + dal.Select("DISTINCT environment")) + if err != nil { + return nil, err + } + + return &plugin.ApiResourceOutput{ + Body: environments, + }, nil +} + +// GetConnectionTransformToDeployments return one connection deployments +// @Summary return one connection deployments +// @Description return one connection deployments +// @Tags plugins/gitlab +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 {object} map[string]interface{} +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/gitlab/connections/{connectionId}/transform-to-deployments [POST] +func GetConnectionTransformToDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + db := basicRes.GetDal() + connectionId := input.Params["connectionId"] + deploymentPattern := input.Body["deploymentPattern"] + productionPattern := input.Body["productionPattern"] + page, err := api.ParsePageParam(input.Body, "page", 1) + if err != nil { + return nil, errors.Default.New("invalid page value") + } + pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10) + if err != nil { + return nil, errors.Default.New("invalid pageSize value") + } + + cursor, err := db.RawCursor(` + SELECT DISTINCT name, gitlab_id, web_url, started_at + FROM ( + SELECT r.name, p.gitlab_id, p.web_url, p.started_at + FROM _tool_gitlab_pipelines p + LEFT JOIN _tool_gitlab_projects r on r.gitlab_id = p.project_id + WHERE p.connection_id = ? AND ref REGEXP ? + AND ref REGEXP ? + UNION + SELECT r.name, p.gitlab_id, p.web_url, p.started_at + FROM _tool_gitlab_pipelines p + LEFT JOIN _tool_gitlab_projects r on r.gitlab_id = p.project_id + LEFT JOIN _tool_gitlab_jobs j on j.pipeline_id = p.gitlab_id + WHERE j.connection_id = ? AND j.name REGEXP ? + AND j.name REGEXP ? + ) r + ORDER BY r.started_at DESC + `, connectionId, deploymentPattern, productionPattern, connectionId, deploymentPattern, productionPattern) + if err != nil { + return nil, errors.Default.Wrap(err, "error on get") + } + defer cursor.Close() + + type selectFileds struct { + Name string + GitlabId int + WebUrl string + } + type transformedFields struct { + Name string + URL string + } + var allRuns []transformedFields + for cursor.Next() { + sf := &selectFileds{} + err = db.Fetch(cursor, sf) + if err != nil { + return nil, errors.Default.Wrap(err, "error on fetch") + } + // Directly transform and append to allRuns + transformed := transformedFields{ + Name: fmt.Sprintf("%s-#%d", sf.Name, sf.GitlabId), + URL: sf.WebUrl, + } + allRuns = append(allRuns, transformed) + } + // Calculate total count + totalCount := len(allRuns) + + // Paginate in memory + start := (page - 1) * pageSize + end := start + pageSize + if start > totalCount { + start = totalCount + } + if end > totalCount { + end = totalCount + } + pagedRuns := allRuns[start:end] + + // Return result containing paged runs and total count + result := map[string]interface{}{ + "total": totalCount, + "data": pagedRuns, + } + return &plugin.ApiResourceOutput{ + Body: result, + }, nil +} diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go index 2d4dbb4fc49..8bf6f9235ab 100644 --- a/backend/plugins/gitlab/impl/impl.go +++ b/backend/plugins/gitlab/impl/impl.go @@ -212,8 +212,15 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ScopeConfig.ProductionPattern); err != nil { return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`") } - if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil { - return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`") + if len(op.ScopeConfig.EnvNameList) > 0 || (len(op.ScopeConfig.EnvNameList) == 0 && op.ScopeConfig.EnvNamePattern == "") { + if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNameList...); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid value for `envNameList`") + } + } else { + if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`") + } + } taskData := tasks.GitlabTaskData{ @@ -268,6 +275,12 @@ func (p Gitlab) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "GET": api.GetScopeList, "PUT": api.PutScopes, }, + "connections/:connectionId/deployments": { + "GET": api.GetConnectionDeployments, + }, + "connections/:connectionId/transform-to-deployments": { + "POST": api.GetConnectionTransformToDeployments, + }, "connections/:connectionId/scope-configs": { "POST": api.CreateScopeConfig, "GET": api.GetScopeConfigList, diff --git a/backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go b/backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go new file mode 100644 index 00000000000..88fa893644f --- /dev/null +++ b/backend/plugins/gitlab/models/migrationscripts/20240906_add_env_name_list_to_scope_config.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addEnvNameListToScopeConfig)(nil) + +type scopeConfig20240906 struct { + EnvNameList []string `gorm:"type:json;serializer:json" json:"env_name_list" mapstructure:"env_name_list"` +} + +func (scopeConfig20240906) TableName() string { + return "_tool_gitlab_scope_configs" +} + +type addEnvNameListToScopeConfig struct{} + +func (*addEnvNameListToScopeConfig) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&scopeConfig20240906{}); err != nil { + return err + } + return nil +} + +func (*addEnvNameListToScopeConfig) Version() uint64 { + return 20240906142101 +} + +func (*addEnvNameListToScopeConfig) Name() string { + return "add env_name_list to _tool_gitlab_scope_configs" +} diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go b/backend/plugins/gitlab/models/migrationscripts/register.go index 1d89b250512..2fb32db0b6c 100644 --- a/backend/plugins/gitlab/models/migrationscripts/register.go +++ b/backend/plugins/gitlab/models/migrationscripts/register.go @@ -52,5 +52,6 @@ func All() []plugin.MigrationScript { new(addGitlabAssigneeAndReviewerPrimaryKey), new(changeIssueComponentType), new(addIsChildToPipelines240906), + new(addEnvNameListToScopeConfig), } } diff --git a/backend/plugins/gitlab/models/scope_config.go b/backend/plugins/gitlab/models/scope_config.go index 78cfd7f2d1d..63f6ab7d8f2 100644 --- a/backend/plugins/gitlab/models/scope_config.go +++ b/backend/plugins/gitlab/models/scope_config.go @@ -36,6 +36,7 @@ type GitlabScopeConfig struct { DeploymentPattern string `mapstructure:"deploymentPattern" json:"deploymentPattern"` ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"` EnvNamePattern string `mapstructure:"envNamePattern,omitempty" json:"envNamePattern" gorm:"type:varchar(255)"` + EnvNameList []string `gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"` Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"` } diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go b/backend/plugins/gitlab/tasks/deployment_convertor.go index d7ff0d7741f..06ed5923de0 100644 --- a/backend/plugins/gitlab/tasks/deployment_convertor.go +++ b/backend/plugins/gitlab/tasks/deployment_convertor.go @@ -129,7 +129,7 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error { Url: repo.WebUrl + "/environments", } if data.RegexEnricher != nil { - if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, gitlabDeployment.Environment) != "" { + if data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN, gitlabDeployment.Environment) != "" { domainDeployCommit.Environment = devops.PRODUCTION } }