Skip to content

Commit

Permalink
feat: gitlab dora config + bamboo fix (#8047)
Browse files Browse the repository at this point in the history
  • Loading branch information
abeizn authored and mintsweet committed Sep 20, 2024
1 parent da6c673 commit 30c6736
Show file tree
Hide file tree
Showing 12 changed files with 257 additions and 7 deletions.
11 changes: 9 additions & 2 deletions backend/plugins/bamboo/impl/impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,15 @@ func (p Bamboo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`")
}
if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.EnvNamePattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
if len(op.BambooScopeConfig.EnvNameList) > 0 || (len(op.BambooScopeConfig.EnvNameList) == 0 && op.BambooScopeConfig.EnvNamePattern == "") {
if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN, op.BambooScopeConfig.EnvNameList...); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNameList`")
}
} else {
if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.BambooScopeConfig.EnvNamePattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
}

}
return &tasks.BambooOptions{
Options: op,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
)

var _ plugin.MigrationScript = (*addEnvNameListToScopeConfig)(nil)

type scopeConfig20240906 struct {
EnvNameList []string `gorm:"type:json;serializer:json" json:"env_name_list" mapstructure:"env_name_list"`
}

func (scopeConfig20240906) TableName() string {
return "_tool_bamboo_scope_configs"
}

type addEnvNameListToScopeConfig struct{}

func (*addEnvNameListToScopeConfig) Up(basicRes context.BasicRes) errors.Error {
db := basicRes.GetDal()
if err := db.AutoMigrate(&scopeConfig20240906{}); err != nil {
return err
}
return nil
}

func (*addEnvNameListToScopeConfig) Version() uint64 {
return 20240906142102
}

func (*addEnvNameListToScopeConfig) Name() string {
return "add env_name_list to _tool_bamboo_scope_configs"
}
1 change: 1 addition & 0 deletions backend/plugins/bamboo/models/migrationscripts/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,6 @@ func All() []plugin.MigrationScript {
new(addMissingPrimaryKeyForBambooPlanBuildVcsRevision),
new(addQueuedFieldsInJobBuild20231128),
new(addLinkHrefToBambooPlanBuild),
new(addEnvNameListToScopeConfig),
}
}
1 change: 1 addition & 0 deletions backend/plugins/bamboo/models/scope_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type BambooScopeConfig struct {
DeploymentPattern string `mapstructure:"deploymentPattern,omitempty" json:"deploymentPattern" gorm:"type:varchar(255)"`
ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"`
EnvNamePattern string `mapstructure:"envNamePattern,omitempty" json:"envNamePattern" gorm:"type:varchar(255)"`
EnvNameList []string `gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"`
}

func (BambooScopeConfig) TableName() string {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func ConvertDeployBuildsToDeployments(taskCtx plugin.SubTaskContext) errors.Erro
},
DisplayTitle: name,
}
if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, input.Environment) != "" {
if data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN, input.Environment) != "" {
deployment.Environment = devops.PRODUCTION
}
if input.FinishedDate != nil && input.ExecutedDate != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,5 @@ func (*addEnvNameListToScopeConfig) Version() uint64 {
}

func (*addEnvNameListToScopeConfig) Name() string {
return "add is_draft to _tool_github_pull_requests"
return "add env_name_list to _tool_github_scope_configs"
}
122 changes: 122 additions & 0 deletions backend/plugins/gitlab/api/connection_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"net/http"
"net/url"

"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
Expand Down Expand Up @@ -174,3 +175,124 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
return dsHelper.ConnApi.GetDetail(input)
}

// GetConnectionDeployments return one connection deployments
// @Summary return one connection deployments
// @Description return one connection deployments
// @Tags plugins/gitlab
// @Param id path int true "id"
// @Param connectionId path int true "connectionId"
// @Success 200 {array} string "List of Environment Names"
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/deployments [GET]
func GetConnectionDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
db := basicRes.GetDal()
connectionId := input.Params["connectionId"]
var environments []string
err := db.All(&environments,
dal.From(&models.GitlabDeployment{}),
dal.Where("connection_id = ?", connectionId),
dal.Select("DISTINCT environment"))
if err != nil {
return nil, err
}

return &plugin.ApiResourceOutput{
Body: environments,
}, nil
}

// GetConnectionTransformToDeployments return one connection deployments
// @Summary return one connection deployments
// @Description return one connection deployments
// @Tags plugins/gitlab
// @Param id path int true "id"
// @Param connectionId path int true "connectionId"
// @Success 200 {object} map[string]interface{}
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/transform-to-deployments [POST]
func GetConnectionTransformToDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
db := basicRes.GetDal()
connectionId := input.Params["connectionId"]
deploymentPattern := input.Body["deploymentPattern"]
productionPattern := input.Body["productionPattern"]
page, err := api.ParsePageParam(input.Body, "page", 1)
if err != nil {
return nil, errors.Default.New("invalid page value")
}
pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10)
if err != nil {
return nil, errors.Default.New("invalid pageSize value")
}

cursor, err := db.RawCursor(`
SELECT DISTINCT name, gitlab_id, web_url, started_at
FROM (
SELECT r.name, p.gitlab_id, p.web_url, p.started_at
FROM _tool_gitlab_pipelines p
LEFT JOIN _tool_gitlab_projects r on r.gitlab_id = p.project_id
WHERE p.connection_id = ? AND ref REGEXP ?
AND ref REGEXP ?
UNION
SELECT r.name, p.gitlab_id, p.web_url, p.started_at
FROM _tool_gitlab_pipelines p
LEFT JOIN _tool_gitlab_projects r on r.gitlab_id = p.project_id
LEFT JOIN _tool_gitlab_jobs j on j.pipeline_id = p.gitlab_id
WHERE j.connection_id = ? AND j.name REGEXP ?
AND j.name REGEXP ?
) r
ORDER BY r.started_at DESC
`, connectionId, deploymentPattern, productionPattern, connectionId, deploymentPattern, productionPattern)
if err != nil {
return nil, errors.Default.Wrap(err, "error on get")
}
defer cursor.Close()

type selectFileds struct {
Name string
GitlabId int
WebUrl string
}
type transformedFields struct {
Name string
URL string
}
var allRuns []transformedFields
for cursor.Next() {
sf := &selectFileds{}
err = db.Fetch(cursor, sf)
if err != nil {
return nil, errors.Default.Wrap(err, "error on fetch")
}
// Directly transform and append to allRuns
transformed := transformedFields{
Name: fmt.Sprintf("%s-#%d", sf.Name, sf.GitlabId),
URL: sf.WebUrl,
}
allRuns = append(allRuns, transformed)
}
// Calculate total count
totalCount := len(allRuns)

// Paginate in memory
start := (page - 1) * pageSize
end := start + pageSize
if start > totalCount {
start = totalCount
}
if end > totalCount {
end = totalCount
}
pagedRuns := allRuns[start:end]

// Return result containing paged runs and total count
result := map[string]interface{}{
"total": totalCount,
"data": pagedRuns,
}
return &plugin.ApiResourceOutput{
Body: result,
}, nil
}
17 changes: 15 additions & 2 deletions backend/plugins/gitlab/impl/impl.go
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,15 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
if err := regexEnricher.TryAdd(devops.PRODUCTION, op.ScopeConfig.ProductionPattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `productionPattern`")
}
if err := regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
if len(op.ScopeConfig.EnvNameList) > 0 || (len(op.ScopeConfig.EnvNameList) == 0 && op.ScopeConfig.EnvNamePattern == "") {
if err = regexEnricher.TryAddList(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNameList...); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNameList`")
}
} else {
if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
}

}

taskData := tasks.GitlabTaskData{
Expand Down Expand Up @@ -268,6 +275,12 @@ func (p Gitlab) ApiResources() map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScopes,
},
"connections/:connectionId/deployments": {
"GET": api.GetConnectionDeployments,
},
"connections/:connectionId/transform-to-deployments": {
"POST": api.GetConnectionTransformToDeployments,
},
"connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package migrationscripts

import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
)

var _ plugin.MigrationScript = (*addEnvNameListToScopeConfig)(nil)

type scopeConfig20240906 struct {
EnvNameList []string `gorm:"type:json;serializer:json" json:"env_name_list" mapstructure:"env_name_list"`
}

func (scopeConfig20240906) TableName() string {
return "_tool_gitlab_scope_configs"
}

type addEnvNameListToScopeConfig struct{}

func (*addEnvNameListToScopeConfig) Up(basicRes context.BasicRes) errors.Error {
db := basicRes.GetDal()
if err := db.AutoMigrate(&scopeConfig20240906{}); err != nil {
return err
}
return nil
}

func (*addEnvNameListToScopeConfig) Version() uint64 {
return 20240906142101
}

func (*addEnvNameListToScopeConfig) Name() string {
return "add env_name_list to _tool_gitlab_scope_configs"
}
1 change: 1 addition & 0 deletions backend/plugins/gitlab/models/migrationscripts/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,5 +52,6 @@ func All() []plugin.MigrationScript {
new(addGitlabAssigneeAndReviewerPrimaryKey),
new(changeIssueComponentType),
new(addIsChildToPipelines240906),
new(addEnvNameListToScopeConfig),
}
}
1 change: 1 addition & 0 deletions backend/plugins/gitlab/models/scope_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ type GitlabScopeConfig struct {
DeploymentPattern string `mapstructure:"deploymentPattern" json:"deploymentPattern"`
ProductionPattern string `mapstructure:"productionPattern,omitempty" json:"productionPattern" gorm:"type:varchar(255)"`
EnvNamePattern string `mapstructure:"envNamePattern,omitempty" json:"envNamePattern" gorm:"type:varchar(255)"`
EnvNameList []string `gorm:"type:json;serializer:json" json:"envNameList" mapstructure:"envNameList"`
Refdiff datatypes.JSONMap `mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object" format:"json"`
}

Expand Down
2 changes: 1 addition & 1 deletion backend/plugins/gitlab/tasks/deployment_convertor.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
Url: repo.WebUrl + "/environments",
}
if data.RegexEnricher != nil {
if data.RegexEnricher.ReturnNameIfMatched(devops.ENV_NAME_PATTERN, gitlabDeployment.Environment) != "" {
if data.RegexEnricher.ReturnNameIfMatchedList(devops.ENV_NAME_PATTERN, gitlabDeployment.Environment) != "" {
domainDeployCommit.Environment = devops.PRODUCTION
}
}
Expand Down

0 comments on commit 30c6736

Please sign in to comment.