From edd7dc09740d171cbf14b2a5479b6bd154072f3e Mon Sep 17 00:00:00 2001 From: Caio Queiroz Date: Tue, 22 Oct 2024 06:21:21 -0300 Subject: [PATCH] fix(gitextractor): subtask Clone Git Repo ended unexpectedly (#8136) * feat: generating new github access token to every gitextractor task Signed-off-by: Caio Queiroz * feat: using DynamicGitUrl interface to implement the git url logic * refactor: remove unused code * fix: unit test * fix: lint --------- Signed-off-by: Caio Queiroz --- backend/plugins/gitextractor/impl/impl.go | 23 ++++++++ .../plugins/gitextractor/parser/taskdata.go | 2 + backend/plugins/github/api/blueprint_v200.go | 12 ++-- backend/plugins/github/impl/impl.go | 57 ++++++++++++++++++- .../plugins/gitlab/api/blueprint_V200_test.go | 5 ++ backend/plugins/gitlab/api/blueprint_v200.go | 12 ++-- 6 files changed, 99 insertions(+), 12 deletions(-) diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index 45c84ab6e17..18d2054bad3 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -18,6 +18,7 @@ limitations under the License. package impl import ( + "fmt" "net/url" "github.com/apache/incubator-devlake/core/dal" @@ -37,6 +38,10 @@ var _ interface { type GitExtractor struct{} +type DynamicGitUrl interface { + GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error) +} + func (p GitExtractor) GetTablesInfo() []dal.Tabler { return []dal.Tabler{} } @@ -68,6 +73,24 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st return nil, err } + if op.PluginName != "" { + pluginInstance, err := plugin.GetPlugin(op.PluginName) + if err != nil { + return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to get plugin instance for plugin: %s", op.PluginName)) + } + + if pluginGit, ok := pluginInstance.(DynamicGitUrl); ok { + gitUrl, err := pluginGit.GetDynamicGitUrl(taskCtx, op.ConnectionId, op.Url) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get Git URL") + } + + op.Url = gitUrl + } else { + log.Printf("Plugin does not implement DynamicGitUrl interface for plugin: %s", op.PluginName) + } + } + parsedURL, err := giturls.Parse(op.Url) if err != nil { return nil, errors.BadInput.Wrap(err, "failed to parse git url") diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index 1b22c98855e..8dccf5ffe9f 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -45,4 +45,6 @@ type GitExtractorOptions struct { SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"` SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` } diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go index 3016db8cb4a..4367951a0d0 100644 --- a/backend/plugins/github/api/blueprint_v200.go +++ b/backend/plugins/github/api/blueprint_v200.go @@ -131,11 +131,13 @@ func makeDataSourcePipelinePlanV200( stage = append(stage, &coreModels.PipelineTask{ Plugin: "gitextractor", Options: map[string]interface{}{ - "url": cloneUrl.String(), - "name": githubRepo.FullName, - "fullName": githubRepo.FullName, - "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId), - "proxy": connection.Proxy, + "url": cloneUrl.String(), + "name": githubRepo.FullName, + "fullName": githubRepo.FullName, + "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId), + "proxy": connection.Proxy, + "connectionId": githubRepo.ConnectionId, + "pluginName": "github", }, }) diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go index e030923706f..7cab40b5ade 100644 --- a/backend/plugins/github/impl/impl.go +++ b/backend/plugins/github/impl/impl.go @@ -19,6 +19,7 @@ package impl import ( "fmt" + "strings" "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter" @@ -176,7 +177,6 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil { return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`") } - } taskData.RegexEnricher = regexEnricher @@ -273,9 +273,41 @@ func (p Github) Close(taskCtx plugin.TaskContext) errors.Error { return nil } +func (p Github) GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error) { + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + p.Name(), + ) + + connection := &models.GithubConnection{} + err := connectionHelper.FirstById(connection, connectionId) + if err != nil { + return "", errors.Default.Wrap(err, "unable to get github connection by the given connection ID") + } + + apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.GetEndpoint(), nil, 0, connection.GetProxy(), taskCtx) + if err != nil { + return "", err + } + + err = connection.PrepareApiClient(apiClient) + if err != nil { + return "", err + } + + newUrl, err := replaceAcessTokenInUrl(repoUrl, connection.Token) + if err != nil { + return "", err + } + + return newUrl, nil +} + func EnrichOptions(taskCtx plugin.TaskContext, op *tasks.GithubOptions, - apiClient *helper.ApiClient) errors.Error { + apiClient *helper.ApiClient, +) errors.Error { var githubRepo models.GithubRepo // validate the op and set name=owner/repo if this is from advanced mode or bpV100 err := tasks.ValidateTaskOptions(op) @@ -342,3 +374,24 @@ func convertApiRepoToScope(repo *tasks.GithubApiRepo, connectionId uint64) *mode scope.CloneUrl = repo.CloneUrl return &scope } + +func replaceAcessTokenInUrl(gitURL, newCredential string) (string, errors.Error) { + atIndex := strings.Index(gitURL, "@") + if atIndex == -1 { + return "", errors.Default.New("Invalid Git URL") + } + + protocolIndex := strings.Index(gitURL, "://") + if protocolIndex == -1 { + return "", errors.Default.New("Invalid Git URL") + } + + // Extract the base URL (e.g., "https://git:") + baseURL := gitURL[:protocolIndex+7] + + repoURL := gitURL[atIndex+1:] + + modifiedURL := fmt.Sprintf("%s%s@%s", baseURL, newCredential, repoURL) + + return modifiedURL, nil +} diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go b/backend/plugins/gitlab/api/blueprint_V200_test.go index b4a345fa423..dc8bdb33ebc 100644 --- a/backend/plugins/gitlab/api/blueprint_V200_test.go +++ b/backend/plugins/gitlab/api/blueprint_V200_test.go @@ -130,6 +130,9 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) { Name: gitlabProjectName, PathWithNamespace: pathWithNamespace, HttpUrlToRepo: httpUrlToRepo, + Scope: common.Scope{ + ConnectionId: connectionID, + }, }, ScopeConfig: scopeConfig, }, @@ -166,6 +169,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) { "name": gitlabProjectName, "fullName": pathWithNamespace, "url": "https://git:nddtf@this_is_cloneUrl", + "connectionId": connectionID, + "pluginName": pluginName, }, }, }, diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go index e8d43184176..bf870a8e8ba 100644 --- a/backend/plugins/gitlab/api/blueprint_v200.go +++ b/backend/plugins/gitlab/api/blueprint_v200.go @@ -136,11 +136,13 @@ func makePipelinePlanV200( stage = append(stage, &coreModels.PipelineTask{ Plugin: "gitextractor", Options: map[string]interface{}{ - "url": cloneUrl.String(), - "name": gitlabProject.Name, - "fullName": gitlabProject.PathWithNamespace, - "repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId), - "proxy": connection.Proxy, + "url": cloneUrl.String(), + "name": gitlabProject.Name, + "fullName": gitlabProject.PathWithNamespace, + "repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId), + "proxy": connection.Proxy, + "connectionId": gitlabProject.ConnectionId, + "pluginName": "gitlab", }, }) }