Kw 5519 dshelper refactor (#7345)

* refactor: opsgenie adopts the new dshelpers

* fix: typo of circleci proxy endpoint tag

* refactor: teambition adopts the new dshelpers

* refactor: trello adopts the new dshelpers

* refactor: zentao adopts the new dshelpers

* fix: linting

* fix: teambition e2e

* fix: teambition e2e

backend/plugins/circleci/api/remote_api.go

@@ -103,7 +103,7 @@ func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er
 // @Description Forward API requests to the specified remote server
 // @Param connectionId path int true "connection ID"
 // @Param path path string true "path to a API endpoint"
-// @Tags plugins/circle
+// @Tags plugins/circleci
 // @Router /plugins/bitbucket/connections/{connectionId}/proxy/{path} [GET]
 func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
 	return raProxy.Proxy(input)

backend/plugins/teambition/api/blueprint_v200.go

@@ -0,0 +1,105 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/plugins/teambition/models"
+	"github.com/apache/incubator-devlake/plugins/teambition/tasks"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+	"github.com/apache/incubator-devlake/core/plugin"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+)
+
+func MakeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	connectionId uint64,
+	bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
+	if err != nil {
+		return nil, nil, err
+	}
+	plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopes, err := makeScopesV200(scopeDetails, connection)
+	return plan, scopes, err
+}
+
+func makePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	scopeDetails []*srvhelper.ScopeDetail[models.TeambitionProject, srvhelper.NoScopeConfig],
+	connection *models.TeambitionConnection,
+) (coreModels.PipelinePlan, errors.Error) {
+	plan := make(coreModels.PipelinePlan, len(scopeDetails))
+	for i, scopeDetail := range scopeDetails {
+		stage := plan[i]
+		if stage == nil {
+			stage = coreModels.PipelineStage{}
+		}
+
+		scope := scopeDetail.Scope
+		// construct task options for circleci
+		task, err := helper.MakePipelinePlanTask(
+			"teambition",
+			subtaskMetas,
+			nil,
+			tasks.TeambitionOptions{
+				ConnectionId: connection.ID,
+				ProjectId:    scope.Id,
+			},
+		)
+		if err != nil {
+			return nil, err
+		}
+		stage = append(stage, task)
+		plan[i] = stage
+	}
+
+	return plan, nil
+}
+
+func makeScopesV200(
+	scopeDetails []*srvhelper.ScopeDetail[models.TeambitionProject, srvhelper.NoScopeConfig],
+	connection *models.TeambitionConnection,
+) ([]plugin.Scope, errors.Error) {
+	scopes := make([]plugin.Scope, 0, len(scopeDetails))
+
+	idgen := didgen.NewDomainIdGenerator(&models.TeambitionProject{})
+	for _, scopeDetail := range scopeDetails {
+		scope, _ := scopeDetail.Scope, scopeDetail.ScopeConfig
+		id := idgen.Generate(connection.ID, scope.Id)
+
+		// add cicd_scope to scopes
+		// if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) {
+		scopes = append(scopes, devops.NewCicdScope(id, scope.Name))
+		// }
+	}
+
+	return scopes, nil
+}

backend/plugins/teambition/api/connection.go → backend/plugins/teambition/api/connection_api.go

@@ -114,12 +114,8 @@ func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
 // @Failure 500  {string} errcode.Error "Internal Error"
 // @Router /plugins/teambition/connections/{connectionId}/test [POST]
 func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
-	connection := &models.TeambitionConnection{}
-	err := connectionHelper.First(connection, input.Params)
+	connection, err := dsHelper.ConnApi.GetMergedConnection(input)
 	if err != nil {
-		return nil, errors.BadInput.Wrap(err, "find connection from db")
-	}
-	if err := api.DecodeMapStruct(input.Body, connection, false); err != nil {
 		return nil, err
 	}
 	testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection.TeambitionConn)
@@ -138,13 +134,7 @@ func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResource
 // @Failure 500  {string} errcode.Error "Internal Error"
 // @Router /plugins/teambition/connections [POST]
 func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
-	// update from request and save to database
-	connection := &models.TeambitionConnection{}
-	err := connectionHelper.Create(connection, input)
-	if err != nil {
-		return nil, err
-	}
-	return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil
+	return dsHelper.ConnApi.Post(input)
 }
 
 // PatchConnection @Summary patch teambition connection
@@ -156,17 +146,7 @@ func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
 // @Failure 500  {string} errcode.Error "Internal Error"
 // @Router /plugins/teambition/connections/{connectionId} [PATCH]
 func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
-	connection := &models.TeambitionConnection{}
-	if err := connectionHelper.First(&connection, input.Params); err != nil {
-		return nil, err
-	}
-	if err := (&models.TeambitionConnection{}).MergeFromRequest(connection, input.Body); err != nil {
-		return nil, errors.Convert(err)
-	}
-	if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil {
-		return nil, err
-	}
-	return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil
+	return dsHelper.ConnApi.Patch(input)
 }
 
 // DeleteConnection @Summary delete a teambition connection
@@ -178,14 +158,7 @@ func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
 // @Failure 500  {string} errcode.Error "Internal Error"
 // @Router /plugins/teambition/connections/{connectionId} [DELETE]
 func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
-	conn := &models.TeambitionConnection{}
-	output, err := connectionHelper.Delete(conn, input)
-	if err != nil {
-		return output, err
-	}
-	output.Body = conn.Sanitize()
-	return output, nil
-
+	return dsHelper.ConnApi.Delete(input)
 }
 
 // ListConnections @Summary get all teambition connections
@@ -196,15 +169,7 @@ func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput
 // @Failure 500  {string} errcode.Error "Internal Error"
 // @Router /plugins/teambition/connections [GET]
 func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
-	var connections []models.TeambitionConnection
-	err := connectionHelper.List(&connections)
-	if err != nil {
-		return nil, err
-	}
-	for idx, c := range connections {
-		connections[idx] = c.Sanitize()
-	}
-	return &plugin.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil
+	return dsHelper.ConnApi.GetAll(input)
 }
 
 // GetConnection @Summary get teambition connection detail
@@ -215,7 +180,5 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
 // @Failure 500  {string} errcode.Error "Internal Error"
 // @Router /plugins/teambition/connections/{connectionId} [GET]
 func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
-	connection := &models.TeambitionConnection{}
-	err := connectionHelper.First(connection, input.Params)
-	return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, err
+	return dsHelper.ConnApi.GetDetail(input)
 }

backend/plugins/teambition/api/init.go

@@ -20,21 +20,31 @@ package api
 import (
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/plugin"
-	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+	"github.com/apache/incubator-devlake/plugins/teambition/models"
 	"github.com/go-playground/validator/v10"
 )
 
 var vld *validator.Validate
-var connectionHelper *helper.ConnectionApiHelper
+var dsHelper *api.DsHelper[models.TeambitionConnection, models.TeambitionProject, srvhelper.NoScopeConfig]
 var basicRes context.BasicRes
 
 func Init(br context.BasicRes, p plugin.PluginMeta) {
-
 	basicRes = br
 	vld = validator.New()
-	connectionHelper = helper.NewConnectionHelper(
-		basicRes,
-		vld,
+	dsHelper = api.NewDataSourceHelper[
+		models.TeambitionConnection,
+		models.TeambitionProject,
+		srvhelper.NoScopeConfig,
+	](
+		br,
 		p.Name(),
+		[]string{"name"},
+		func(c models.TeambitionConnection) models.TeambitionConnection {
+			return c.Sanitize()
+		},
+		nil,
+		nil,
 	)
 }

backend/plugins/teambition/e2e/snapshot_tables/_tool_teambition_projects.csv

@@ -1,2 +1,2 @@
-connection_id,id,name,logo,description,organization_id,visibility,is_template,creator_id,is_archived,is_suspended,unique_id_prefix,created,updated,start_date,end_date,customfields,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,64132c94f0d59df1c9825ab8,缺陷管理,https://tcs-ga.teambition.net/thumbnail/312qf26a0a3a77f700a4fded1f2a4c19b257/w/600/h/300,"",640b1c30c933fd85bb11ca31,project,0,5f27709685e4266322e2690a,0,0,"",2023-03-16 14:49:56.415,2023-03-16 14:49:56.415,,,[],2023-03-23 14:24:57.135,2023-03-23 14:24:57.135,"{""ConnectionId"":1,""OrganizationId"":"""",""ProjectId"":""64132c94f0d59df1c9825ab8""}",_raw_teambition_api_projects,1,""
+connection_id,id,name,logo,description,organization_id,visibility,is_template,creator_id,is_archived,is_suspended,unique_id_prefix,created,updated,start_date,end_date,customfields,created_at,updated_at,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,scope_config_id
+1,64132c94f0d59df1c9825ab8,缺陷管理,https://tcs-ga.teambition.net/thumbnail/312qf26a0a3a77f700a4fded1f2a4c19b257/w/600/h/300,"",640b1c30c933fd85bb11ca31,project,0,5f27709685e4266322e2690a,0,0,"",2023-03-16 14:49:56.415,2023-03-16 14:49:56.415,,,[],2023-03-23 14:24:57.135,2023-03-23 14:24:57.135,"{""ConnectionId"":1,""OrganizationId"":"""",""ProjectId"":""64132c94f0d59df1c9825ab8""}",_raw_teambition_api_projects,1,"",0