From 512c748f7f836bc970226e8da3ff4643b16297c3 Mon Sep 17 00:00:00 2001 From: Caio Queiroz Date: Wed, 9 Oct 2024 08:57:15 -0300 Subject: [PATCH 1/5] feat: generating new github access token to every gitextractor task Signed-off-by: Caio Queiroz --- backend/plugins/gitextractor/impl/impl.go | 52 ++++++++++++++++++- .../plugins/gitextractor/parser/taskdata.go | 1 + backend/plugins/github/api/blueprint_v200.go | 1 + 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index 45c84ab6e17..df18a55a689 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -18,7 +18,9 @@ limitations under the License. package impl import ( + "fmt" "net/url" + "strings" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -26,6 +28,7 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitextractor/parser" "github.com/apache/incubator-devlake/plugins/gitextractor/tasks" + "github.com/apache/incubator-devlake/plugins/github/models" giturls "github.com/chainguard-dev/git-urls" ) @@ -68,9 +71,33 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st return nil, err } - parsedURL, err := giturls.Parse(op.Url) + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + p.Name(), + ) + connection := &models.GithubConnection{} + err := connectionHelper.FirstById(connection, op.ConnectionId) if err != nil { - return nil, errors.BadInput.Wrap(err, "failed to parse git url") + return nil, errors.Default.Wrap(err, "unable to get github connection by the given connection ID") + } + + apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.GetEndpoint(), nil, 0, connection.GetProxy(), taskCtx) + if err != nil { + return nil, err + } + + connection.PrepareApiClient(apiClient) + + newUrl, err := replaceAcessTokenInUrl(op.Url, connection.Token) + if err != nil { + return nil, err + } + op.Url = newUrl + + parsedURL, errParse := giturls.Parse(op.Url) + if errParse != nil { + return nil, errors.BadInput.Wrap(errParse, "failed to parse git url") } // append username to the git url @@ -126,3 +153,24 @@ func (p GitExtractor) RootPkgPath() string { func (p GitExtractor) TestConnection(id uint64) errors.Error { return nil } + +func replaceAcessTokenInUrl(gitURL, newCredential string) (string, errors.Error) { + atIndex := strings.Index(gitURL, "@") + if atIndex == -1 { + return "", errors.Default.New("Invalid Git URL") + } + + protocolIndex := strings.Index(gitURL, "://") + if protocolIndex == -1 { + return "", errors.Default.New("Invalid Git URL") + } + + // Extract the base URL (e.g., "https://git:") + baseURL := gitURL[:protocolIndex+7] + + repoURL := gitURL[atIndex+1:] + + modifiedURL := fmt.Sprintf("%s%s@%s", baseURL, newCredential, repoURL) + + return modifiedURL, nil +} diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index 1b22c98855e..57f5b4b4870 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -45,4 +45,5 @@ type GitExtractorOptions struct { SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"` SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` } diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go index 3016db8cb4a..20fd9b7dcd0 100644 --- a/backend/plugins/github/api/blueprint_v200.go +++ b/backend/plugins/github/api/blueprint_v200.go @@ -136,6 +136,7 @@ func makeDataSourcePipelinePlanV200( "fullName": githubRepo.FullName, "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId), "proxy": connection.Proxy, + "connectionId": githubRepo.ConnectionId, }, }) From 26b716f9880aff1ff58c1730f1c4260112cef110 Mon Sep 17 00:00:00 2001 From: Caio Queiroz Date: Thu, 17 Oct 2024 16:59:04 -0300 Subject: [PATCH 2/5] feat: using DynamicGitUrl interface to implement the git url logic --- backend/plugins/gitextractor/impl/impl.go | 45 ++++++++--------- .../plugins/gitextractor/parser/taskdata.go | 1 + backend/plugins/github/api/blueprint_v200.go | 1 + backend/plugins/github/impl/impl.go | 50 +++++++++++++++++++ backend/plugins/gitlab/api/blueprint_v200.go | 2 + 5 files changed, 76 insertions(+), 23 deletions(-) diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index df18a55a689..ef274910a9c 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -28,7 +28,6 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitextractor/parser" "github.com/apache/incubator-devlake/plugins/gitextractor/tasks" - "github.com/apache/incubator-devlake/plugins/github/models" giturls "github.com/chainguard-dev/git-urls" ) @@ -40,6 +39,10 @@ var _ interface { type GitExtractor struct{} +type DynamicGitUrl interface { + GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error) +} + func (p GitExtractor) GetTablesInfo() []dal.Tabler { return []dal.Tabler{} } @@ -71,33 +74,29 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st return nil, err } - connectionHelper := helper.NewConnectionHelper( - taskCtx, - nil, - p.Name(), - ) - connection := &models.GithubConnection{} - err := connectionHelper.FirstById(connection, op.ConnectionId) - if err != nil { - return nil, errors.Default.Wrap(err, "unable to get github connection by the given connection ID") - } - apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.GetEndpoint(), nil, 0, connection.GetProxy(), taskCtx) - if err != nil { - return nil, err - } + if op.PluginName != "" { + pluginInstance, err := plugin.GetPlugin(op.PluginName) + if err != nil { + return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to get plugin instance for plugin: %s", op.PluginName)) - connection.PrepareApiClient(apiClient) + } - newUrl, err := replaceAcessTokenInUrl(op.Url, connection.Token) - if err != nil { - return nil, err + if pluginGit, ok := pluginInstance.(DynamicGitUrl); ok { + gitUrl, err := pluginGit.GetDynamicGitUrl(taskCtx, op.ConnectionId, op.Url) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get Git URL") + } + + op.Url = gitUrl + } else { + log.Printf("Plugin does not implement DynamicGitUrl interface for plugin: %s", op.PluginName) + } } - op.Url = newUrl - parsedURL, errParse := giturls.Parse(op.Url) - if errParse != nil { - return nil, errors.BadInput.Wrap(errParse, "failed to parse git url") + parsedURL, err := giturls.Parse(op.Url) + if err != nil { + return nil, errors.BadInput.Wrap(err, "failed to parse git url") } // append username to the git url diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index 57f5b4b4870..7cc7b31f88d 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -46,4 +46,5 @@ type GitExtractorOptions struct { SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` } diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go index 20fd9b7dcd0..27c900f7631 100644 --- a/backend/plugins/github/api/blueprint_v200.go +++ b/backend/plugins/github/api/blueprint_v200.go @@ -137,6 +137,7 @@ func makeDataSourcePipelinePlanV200( "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId), "proxy": connection.Proxy, "connectionId": githubRepo.ConnectionId, + "pluginName": "github", }, }) diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go index e030923706f..a299fbd7ff1 100644 --- a/backend/plugins/github/impl/impl.go +++ b/backend/plugins/github/impl/impl.go @@ -19,6 +19,7 @@ package impl import ( "fmt" + "strings" "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter" @@ -273,6 +274,34 @@ func (p Github) Close(taskCtx plugin.TaskContext) errors.Error { return nil } +func (p Github) GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64, repoUrl string) (string, errors.Error) { + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + p.Name(), + ) + + connection := &models.GithubConnection{} + err := connectionHelper.FirstById(connection, connectionId) + if err != nil { + return "", errors.Default.Wrap(err, "unable to get github connection by the given connection ID") + } + + apiClient, err := helper.NewApiClient(taskCtx.GetContext(), connection.GetEndpoint(), nil, 0, connection.GetProxy(), taskCtx) + if err != nil { + return "", err + } + + connection.PrepareApiClient(apiClient) + + newUrl, err := replaceAcessTokenInUrl(repoUrl, connection.Token) + if err != nil { + return "", err + } + + return newUrl, nil +} + func EnrichOptions(taskCtx plugin.TaskContext, op *tasks.GithubOptions, apiClient *helper.ApiClient) errors.Error { @@ -342,3 +371,24 @@ func convertApiRepoToScope(repo *tasks.GithubApiRepo, connectionId uint64) *mode scope.CloneUrl = repo.CloneUrl return &scope } + +func replaceAcessTokenInUrl(gitURL, newCredential string) (string, errors.Error) { + atIndex := strings.Index(gitURL, "@") + if atIndex == -1 { + return "", errors.Default.New("Invalid Git URL") + } + + protocolIndex := strings.Index(gitURL, "://") + if protocolIndex == -1 { + return "", errors.Default.New("Invalid Git URL") + } + + // Extract the base URL (e.g., "https://git:") + baseURL := gitURL[:protocolIndex+7] + + repoURL := gitURL[atIndex+1:] + + modifiedURL := fmt.Sprintf("%s%s@%s", baseURL, newCredential, repoURL) + + return modifiedURL, nil +} diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go index e8d43184176..c51ba221070 100644 --- a/backend/plugins/gitlab/api/blueprint_v200.go +++ b/backend/plugins/gitlab/api/blueprint_v200.go @@ -141,6 +141,8 @@ func makePipelinePlanV200( "fullName": gitlabProject.PathWithNamespace, "repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId), "proxy": connection.Proxy, + "connectionId": gitlabProject.ConnectionId, + "pluginName": "gitlab", }, }) } From cf89afd3a0689c4f13797e81322b4c8bb2d569a1 Mon Sep 17 00:00:00 2001 From: Caio Queiroz Date: Thu, 17 Oct 2024 17:05:09 -0300 Subject: [PATCH 3/5] refactor: remove unused code --- backend/plugins/gitextractor/impl/impl.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index ef274910a9c..93a4f40df1a 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -20,7 +20,6 @@ package impl import ( "fmt" "net/url" - "strings" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -152,24 +151,3 @@ func (p GitExtractor) RootPkgPath() string { func (p GitExtractor) TestConnection(id uint64) errors.Error { return nil } - -func replaceAcessTokenInUrl(gitURL, newCredential string) (string, errors.Error) { - atIndex := strings.Index(gitURL, "@") - if atIndex == -1 { - return "", errors.Default.New("Invalid Git URL") - } - - protocolIndex := strings.Index(gitURL, "://") - if protocolIndex == -1 { - return "", errors.Default.New("Invalid Git URL") - } - - // Extract the base URL (e.g., "https://git:") - baseURL := gitURL[:protocolIndex+7] - - repoURL := gitURL[atIndex+1:] - - modifiedURL := fmt.Sprintf("%s%s@%s", baseURL, newCredential, repoURL) - - return modifiedURL, nil -} From 66891a8ea9cfcb73c079450927bfa5aa5cc5b692 Mon Sep 17 00:00:00 2001 From: Caio Queiroz Date: Mon, 21 Oct 2024 20:51:53 -0300 Subject: [PATCH 4/5] fix: unit test --- backend/plugins/gitlab/api/blueprint_V200_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go b/backend/plugins/gitlab/api/blueprint_V200_test.go index b4a345fa423..dc8bdb33ebc 100644 --- a/backend/plugins/gitlab/api/blueprint_V200_test.go +++ b/backend/plugins/gitlab/api/blueprint_V200_test.go @@ -130,6 +130,9 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) { Name: gitlabProjectName, PathWithNamespace: pathWithNamespace, HttpUrlToRepo: httpUrlToRepo, + Scope: common.Scope{ + ConnectionId: connectionID, + }, }, ScopeConfig: scopeConfig, }, @@ -166,6 +169,8 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) { "name": gitlabProjectName, "fullName": pathWithNamespace, "url": "https://git:nddtf@this_is_cloneUrl", + "connectionId": connectionID, + "pluginName": pluginName, }, }, }, From 58de05ed1ce581190b827dcdffcf82561d138191 Mon Sep 17 00:00:00 2001 From: Caio Queiroz Date: Mon, 21 Oct 2024 21:03:19 -0300 Subject: [PATCH 5/5] fix: lint --- backend/plugins/gitextractor/impl/impl.go | 4 +--- backend/plugins/gitextractor/parser/taskdata.go | 4 ++-- backend/plugins/github/api/blueprint_v200.go | 12 ++++++------ backend/plugins/github/impl/impl.go | 9 ++++++--- backend/plugins/gitlab/api/blueprint_v200.go | 12 ++++++------ 5 files changed, 21 insertions(+), 20 deletions(-) diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index 93a4f40df1a..18d2054bad3 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -73,18 +73,16 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st return nil, err } - if op.PluginName != "" { pluginInstance, err := plugin.GetPlugin(op.PluginName) if err != nil { return nil, errors.Default.Wrap(err, fmt.Sprintf("failed to get plugin instance for plugin: %s", op.PluginName)) - } if pluginGit, ok := pluginInstance.(DynamicGitUrl); ok { gitUrl, err := pluginGit.GetDynamicGitUrl(taskCtx, op.ConnectionId, op.Url) if err != nil { - return nil, errors.Default.Wrap(err, "failed to get Git URL") + return nil, errors.Default.Wrap(err, "failed to get Git URL") } op.Url = gitUrl diff --git a/backend/plugins/gitextractor/parser/taskdata.go b/backend/plugins/gitextractor/parser/taskdata.go index 7cc7b31f88d..8dccf5ffe9f 100644 --- a/backend/plugins/gitextractor/parser/taskdata.go +++ b/backend/plugins/gitextractor/parser/taskdata.go @@ -45,6 +45,6 @@ type GitExtractorOptions struct { SkipCommitStat *bool `json:"skipCommitStat" mapstructure:"skipCommitStat" comment:"skip all commit stat including added/deleted lines and commit files as well"` SkipCommitFiles *bool `json:"skipCommitFiles" mapstructure:"skipCommitFiles"` NoShallowClone bool `json:"noShallowClone" mapstructure:"noShallowClone"` - ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` - PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + PluginName string `json:"pluginName" mapstructure:"pluginName,omitempty"` } diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go index 27c900f7631..4367951a0d0 100644 --- a/backend/plugins/github/api/blueprint_v200.go +++ b/backend/plugins/github/api/blueprint_v200.go @@ -131,13 +131,13 @@ func makeDataSourcePipelinePlanV200( stage = append(stage, &coreModels.PipelineTask{ Plugin: "gitextractor", Options: map[string]interface{}{ - "url": cloneUrl.String(), - "name": githubRepo.FullName, - "fullName": githubRepo.FullName, - "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId), - "proxy": connection.Proxy, + "url": cloneUrl.String(), + "name": githubRepo.FullName, + "fullName": githubRepo.FullName, + "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, githubRepo.GithubId), + "proxy": connection.Proxy, "connectionId": githubRepo.ConnectionId, - "pluginName": "github", + "pluginName": "github", }, }) diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go index a299fbd7ff1..7cab40b5ade 100644 --- a/backend/plugins/github/impl/impl.go +++ b/backend/plugins/github/impl/impl.go @@ -177,7 +177,6 @@ func (p Github) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i if err = regexEnricher.TryAdd(devops.ENV_NAME_PATTERN, op.ScopeConfig.EnvNamePattern); err != nil { return nil, errors.BadInput.Wrap(err, "invalid value for `envNamePattern`") } - } taskData.RegexEnricher = regexEnricher @@ -292,7 +291,10 @@ func (p Github) GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64 return "", err } - connection.PrepareApiClient(apiClient) + err = connection.PrepareApiClient(apiClient) + if err != nil { + return "", err + } newUrl, err := replaceAcessTokenInUrl(repoUrl, connection.Token) if err != nil { @@ -304,7 +306,8 @@ func (p Github) GetDynamicGitUrl(taskCtx plugin.TaskContext, connectionId uint64 func EnrichOptions(taskCtx plugin.TaskContext, op *tasks.GithubOptions, - apiClient *helper.ApiClient) errors.Error { + apiClient *helper.ApiClient, +) errors.Error { var githubRepo models.GithubRepo // validate the op and set name=owner/repo if this is from advanced mode or bpV100 err := tasks.ValidateTaskOptions(op) diff --git a/backend/plugins/gitlab/api/blueprint_v200.go b/backend/plugins/gitlab/api/blueprint_v200.go index c51ba221070..bf870a8e8ba 100644 --- a/backend/plugins/gitlab/api/blueprint_v200.go +++ b/backend/plugins/gitlab/api/blueprint_v200.go @@ -136,13 +136,13 @@ func makePipelinePlanV200( stage = append(stage, &coreModels.PipelineTask{ Plugin: "gitextractor", Options: map[string]interface{}{ - "url": cloneUrl.String(), - "name": gitlabProject.Name, - "fullName": gitlabProject.PathWithNamespace, - "repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId), - "proxy": connection.Proxy, + "url": cloneUrl.String(), + "name": gitlabProject.Name, + "fullName": gitlabProject.PathWithNamespace, + "repoId": didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID, gitlabProject.GitlabId), + "proxy": connection.Proxy, "connectionId": gitlabProject.ConnectionId, - "pluginName": "gitlab", + "pluginName": "gitlab", }, }) }