fix(gitextractor): subtask Clone Git Repo ended unexpectedly (#8136)

* feat: generating new github access token to every gitextractor task Signed-off-by: Caio Queiroz <[email protected]> * feat: using DynamicGitUrl interface to implement the git url logic * refactor: remove unused code * fix: unit test * fix: lint --------- Signed-off-by: Caio Queiroz <[email protected]>
apache · Oct 22, 2024 · edd7dc0 · edd7dc0
1 parent 98bc98b
commit edd7dc0
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 12 deletions.
diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go
@@ -18,6 +18,7 @@ limitations under the License.
 package impl
 
 import (
+	"fmt"
 	"net/url"
 
 	"github.com/apache/incubator-devlake/core/dal"
@@ -37,6 +38,10 @@ var _ interface {
 
 type GitExtractor struct{}
 
+type DynamicGitUrl interface {
+	GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error)
+}
+
 func (p GitExtractor) GetTablesInfo() []dal.Tabler {
 	return []dal.Tabler{}
 }
@@ -68,6 +73,24 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st
 		return nil, err
 	}
 
+	if op.PluginName != "" {
+		pluginInstance, err := plugin.GetPlugin(op.PluginName)
+		if err != nil {
+			return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to get plugin instance for plugin: %s", op.PluginName))
+		}
+
+		if pluginGit, ok := pluginInstance.(DynamicGitUrl); ok {
+			gitUrl, err := pluginGit.GetDynamicGitUrl(taskCtx, op.ConnectionId, op.Url)
+			if err != nil {
+				return nil, errors.Default.Wrap(err, "failed to get Git URL")
+			}
+
+			op.Url = gitUrl
+		} else {
+			log.Printf("Plugin does not implement DynamicGitUrl interface for plugin: %s", op.PluginName)
+		}
+	}
+
 	parsedURL, err := giturls.Parse(op.Url)
 	if err != nil {
 		return nil, errors.BadInput.Wrap(err, "failed to parse git url")

diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go
@@ -45,4 +45,6 @@ type GitExtractorOptions struct {
 	SkipCommitStat        *bool  `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"`
 	SkipCommitFiles       *bool  `json:"skipCommitFiles" mapstructure:"skipCommitFiles"`
 	NoShallowClone        bool   `json:"noShallowClone" mapstructure:"noShallowClone"`
+	ConnectionId          uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"`
+	PluginName            string `json:"pluginName" mapstructure:"pluginName,omitempty"`
 }
diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go
@@ -131,11 +131,13 @@ func makeDataSourcePipelinePlanV200(
 			stage = append(stage, &coreModels.PipelineTask{
 				Plugin: "gitextractor",
 				Options: map[string]interface{}{
-					"url":      cloneUrl.String(),
-					"name":     githubRepo.FullName,
-					"fullName": githubRepo.FullName,
-					"repoId":   didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId),
-					"proxy":    connection.Proxy,
+					"url":          cloneUrl.String(),
+					"name":         githubRepo.FullName,
+					"fullName":     githubRepo.FullName,
+					"repoId":       didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId),
+					"proxy":        connection.Proxy,
+					"connectionId": githubRepo.ConnectionId,
+					"pluginName":   "github",
 				},
 			})
 

diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go
@@ -19,6 +19,7 @@ package impl
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter"
 
@@ -176,7 +177,6 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i
 		if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil {
 			return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`")
 		}
-
 	}
 	taskData.RegexEnricher = regexEnricher
 
@@ -273,9 +273,41 @@ func (p Github) Close(taskCtx plugin.TaskContext) errors.Error {
 	return nil
 }
 
+func (p Github) GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error) {
+	connectionHelper := helper.NewConnectionHelper(
+		taskCtx,
+		nil,
+		p.Name(),
+	)
+
+	connection := &models.GithubConnection{}
+	err := connectionHelper.FirstById(connection, connectionId)
+	if err != nil {
+		return "", errors.Default.Wrap(err, "unable to get github connection by the given connection ID")
+	}
+
+	apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.GetEndpoint(), nil, 0, connection.GetProxy(), taskCtx)
+	if err != nil {
+		return "", err
+	}
+
+	err = connection.PrepareApiClient(apiClient)
+	if err != nil {
+		return "", err
+	}
+
+	newUrl, err := replaceAcessTokenInUrl(repoUrl, connection.Token)
+	if err != nil {
+		return "", err
+	}
+
+	return newUrl, nil
+}
+
 func EnrichOptions(taskCtx plugin.TaskContext,
 	op *tasks.GithubOptions,
-	apiClient *helper.ApiClient) errors.Error {
+	apiClient *helper.ApiClient,
+) errors.Error {
 	var githubRepo models.GithubRepo
 	// validate the op and set name=owner/repo if this is from advanced mode or bpV100
 	err := tasks.ValidateTaskOptions(op)
@@ -342,3 +374,24 @@ func convertApiRepoToScope(repo *tasks.GithubApiRepo, connectionId uint64) *mode
 	scope.CloneUrl = repo.CloneUrl
 	return &scope
 }
+
+func replaceAcessTokenInUrl(gitURL, newCredential string) (string, errors.Error) {
+	atIndex := strings.Index(gitURL, "@")
+	if atIndex == -1 {
+		return "", errors.Default.New("Invalid Git URL")
+	}
+
+	protocolIndex := strings.Index(gitURL, "://")
+	if protocolIndex == -1 {
+		return "", errors.Default.New("Invalid Git URL")
+	}
+
+	// Extract the base URL (e.g., "https://git:")
+	baseURL := gitURL[:protocolIndex+7]
+
+	repoURL := gitURL[atIndex+1:]
+
+	modifiedURL := fmt.Sprintf("%s%s@%s", baseURL, newCredential, repoURL)
+
+	return modifiedURL, nil
+}
diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go b/backend/plugins/gitlab/api/blueprint_V200_test.go
@@ -130,6 +130,9 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 					Name:              gitlabProjectName,
 					PathWithNamespace: pathWithNamespace,
 					HttpUrlToRepo:     httpUrlToRepo,
+					Scope: common.Scope{
+						ConnectionId:		connectionID,
+					},
 				},
 				ScopeConfig: scopeConfig,
 			},
@@ -166,6 +169,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
 					"name":     gitlabProjectName,
 					"fullName": pathWithNamespace,
 					"url":      "https://git:nddtf@this_is_cloneUrl",
+					"connectionId": connectionID,
+					"pluginName": pluginName,
 				},
 			},
 		},

diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -136,11 +136,13 @@ func makePipelinePlanV200(
 			stage = append(stage, &coreModels.PipelineTask{
 				Plugin: "gitextractor",
 				Options: map[string]interface{}{
-					"url":      cloneUrl.String(),
-					"name":     gitlabProject.Name,
-					"fullName": gitlabProject.PathWithNamespace,
-					"repoId":   didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId),
-					"proxy":    connection.Proxy,
+					"url":          cloneUrl.String(),
+					"name":         gitlabProject.Name,
+					"fullName":     gitlabProject.PathWithNamespace,
+					"repoId":       didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId),
+					"proxy":        connection.Proxy,
+					"connectionId": gitlabProject.ConnectionId,
+					"pluginName":   "gitlab",
 				},
 			})
 		}