From 6f1cd04f1bfb048a26a3c11d9610c00f463d683a Mon Sep 17 00:00:00 2001 From: Sergii Kabashniuk Date: Mon, 29 Apr 2024 13:58:01 +0300 Subject: [PATCH] Renovate support of Gitlab/Github OnPrem (#281) * Renovate support of Gitlab/Github OnPrem --- controllers/git_tekton_resources_renovater.go | 2 + .../git_tekton_resources_renovater_test.go | 5 +- pkg/git/apiendpoint.go | 48 +++++++++++++++++ pkg/git/apiendpoint_test.go | 54 +++++++++++++++++++ pkg/k8s/credentials.go | 19 ++++--- pkg/renovate/basicauth.go | 32 +++++------ pkg/renovate/basicauth_test.go | 10 ++-- pkg/renovate/config.go | 5 +- pkg/renovate/githubapp.go | 12 ++--- pkg/renovate/job.go | 4 +- pkg/renovate/task.go | 14 ++--- 11 files changed, 156 insertions(+), 49 deletions(-) create mode 100644 pkg/git/apiendpoint.go create mode 100644 pkg/git/apiendpoint_test.go diff --git a/controllers/git_tekton_resources_renovater.go b/controllers/git_tekton_resources_renovater.go index 50c6f23d..f77b3edf 100644 --- a/controllers/git_tekton_resources_renovater.go +++ b/controllers/git_tekton_resources_renovater.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "reflect" "time" @@ -124,6 +125,7 @@ func (r *GitTektonResourcesRenovater) Reconcile(ctx context.Context, req ctrl.Re var tasks []*renovate.Task for _, taskProvider := range r.taskProviders { newTasks := taskProvider.GetNewTasks(ctx, scmComponents) + log.Info("found new tasks", "tasks", len(newTasks), "provider", reflect.TypeOf(taskProvider).String()) if len(newTasks) > 0 { tasks = append(tasks, newTasks...) } diff --git a/controllers/git_tekton_resources_renovater_test.go b/controllers/git_tekton_resources_renovater_test.go index e49041aa..47159903 100644 --- a/controllers/git_tekton_resources_renovater_test.go +++ b/controllers/git_tekton_resources_renovater_test.go @@ -24,10 +24,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + . "github.com/redhat-appstudio/build-service/pkg/common" "github.com/redhat-appstudio/build-service/pkg/git/github" "github.com/redhat-appstudio/build-service/pkg/renovate" - "k8s.io/apimachinery/pkg/types" ) var _ = Describe("Git tekton resources renovater", func() { @@ -134,7 +135,7 @@ var _ = Describe("Git tekton resources renovater", func() { deleteSecret(pacSecretKey) createDefaultBuildPipelineRunSelector(defaultSelectorKey) - Eventually(listEvents).WithArguments("default").WithTimeout(timeout).ShouldNot(HaveLen(0)) + Eventually(listEvents).WithArguments("default").WithTimeout(timeout).ShouldNot(BeEmpty()) allEvents := listEvents("default") Expect(allEvents[0].Reason).To(Equal("ErrorReadingPaCSecret")) deleteComponent(componentNamespacedName) diff --git a/pkg/git/apiendpoint.go b/pkg/git/apiendpoint.go new file mode 100644 index 00000000..f6986341 --- /dev/null +++ b/pkg/git/apiendpoint.go @@ -0,0 +1,48 @@ +package git + +import "fmt" + +// APIEndpoint interface defines the method to get the API endpoint url for +// the source code providers. +type APIEndpoint interface { + APIEndpoint(host string) string +} + +// GithubAPIEndpoint represents an API endpoint for GitHub. +type GithubAPIEndpoint struct { +} + +// APIEndpoint returns the GitHub API endpoint. +func (g *GithubAPIEndpoint) APIEndpoint(host string) string { + return fmt.Sprintf("https://api.%s/", host) +} + +// GitlabAPIEndpoint represents an API endpoint for GitLab. +type GitlabAPIEndpoint struct { +} + +// APIEndpoint returns the API GitLab endpoint. +func (g *GitlabAPIEndpoint) APIEndpoint(host string) string { + return fmt.Sprintf("https://%s/api/v4/", host) +} + +// UnknownAPIEndpoint represents an endpoint for unknown or non existed provider. It returns empty string for api endpoint. +type UnknownAPIEndpoint struct { +} + +// APIEndpoint returns the GitLab endpoint. +func (g *UnknownAPIEndpoint) APIEndpoint(host string) string { + return "" +} + +// BuildAPIEndpoint constructs and returns an endpoint object based on the type provided type. +func BuildAPIEndpoint(endpointType string) APIEndpoint { + switch endpointType { + case "github": + return &GithubAPIEndpoint{} + case "gitlab": + return &GitlabAPIEndpoint{} + default: + return &UnknownAPIEndpoint{} + } +} diff --git a/pkg/git/apiendpoint_test.go b/pkg/git/apiendpoint_test.go new file mode 100644 index 00000000..5e38e4e3 --- /dev/null +++ b/pkg/git/apiendpoint_test.go @@ -0,0 +1,54 @@ +package git + +import ( + "testing" +) + +func TestBuildEndpoint(t *testing.T) { + + tests := []struct { + name string + endpointType string + host string + wantEndpoint string + }{ + { + name: "Github SAAS", + endpointType: "github", + host: "github.com", + wantEndpoint: "https://api.github.com/", + }, + { + name: "Github On-Prem", + endpointType: "github", + host: "github.umbrella.com", + wantEndpoint: "https://api.github.umbrella.com/", + }, + { + name: "Gitlab SAAS", + endpointType: "gitlab", + host: "gitlab.com", + wantEndpoint: "https://gitlab.com/api/v4/", + }, + { + name: "Gitlab On-Prem", + endpointType: "gitlab", + host: "gitlab.umbrella.com", + wantEndpoint: "https://gitlab.umbrella.com/api/v4/", + }, + { + name: "Unknown provider", + endpointType: "bibi", + host: "bibi.umbrella.com", + wantEndpoint: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := BuildAPIEndpoint(tt.endpointType).APIEndpoint(tt.host); got != tt.wantEndpoint { + t.Errorf("BuildAPIEndpoint() = %v, want %v", got, tt.wantEndpoint) + } + }) + } +} diff --git a/pkg/k8s/credentials.go b/pkg/k8s/credentials.go index 9b39761d..655d49d5 100644 --- a/pkg/k8s/credentials.go +++ b/pkg/k8s/credentials.go @@ -18,7 +18,6 @@ import ( . "github.com/redhat-appstudio/build-service/pkg/common" "github.com/redhat-appstudio/build-service/pkg/git" . "github.com/redhat-appstudio/build-service/pkg/git/credentials" - "github.com/redhat-appstudio/build-service/pkg/logs" bslices "github.com/redhat-appstudio/build-service/pkg/slices" ) @@ -89,7 +88,7 @@ func (k *GitCredentialProvider) GetSSHCredentials(ctx context.Context, component func (k *GitCredentialProvider) LookupSecret(ctx context.Context, component *git.ScmComponent, secretType corev1.SecretType) (*corev1.Secret, error) { log := ctrllog.FromContext(ctx) - log.V(logs.DebugLevel).Info("looking for scm secret", "component", component) + log.Info("looking for scm secret", "component", component) secretList := &corev1.SecretList{} opts := client.ListOption(&client.MatchingLabels{ @@ -100,7 +99,7 @@ func (k *GitCredentialProvider) LookupSecret(ctx context.Context, component *git if err := k.client.List(ctx, secretList, client.InNamespace(component.NamespaceName()), opts); err != nil { return nil, fmt.Errorf("failed to list Pipelines as Code secrets in %s namespace: %w", component.NamespaceName(), err) } - log.V(logs.DebugLevel).Info("found secrets", "count", len(secretList.Items)) + log.Info("found secrets", "count", len(secretList.Items)) secretsWithCredentialsCandidates := bslices.Filter(secretList.Items, func(secret corev1.Secret) bool { return secret.Type == secretType && len(secret.Data) > 0 }) @@ -108,7 +107,7 @@ func (k *GitCredentialProvider) LookupSecret(ctx context.Context, component *git if secretWithCredential != nil { return secretWithCredential, nil } - log.V(logs.DebugLevel).Info("no matching secret found for component", "component", component) + log.Info("no matching secret found for component", "component", component) return nil, boerrors.NewBuildOpError(boerrors.EComponentGitSecretMissing, nil) } @@ -128,24 +127,24 @@ func bestMatchingSecret(ctx context.Context, componentRepository string, secrets for index, secret := range secrets { repositoryAnnotation, exists := secret.Annotations[ScmSecretRepositoryAnnotation] - log.V(logs.DebugLevel).Info("found secret", "secret", secret.Name, "repositoryAnnotation", repositoryAnnotation, "exists", exists) + log.Info("found secret", "secret", secret.Name, "repositoryAnnotation", repositoryAnnotation, "exists", exists) if !exists || repositoryAnnotation == "" { hostOnlySecrets = append(hostOnlySecrets, secret) continue } secretRepositories := strings.Split(repositoryAnnotation, ",") - log.V(logs.DebugLevel).Info("found secret repositories", "repositories", secretRepositories) + log.Info("found secret repositories", "repositories", secretRepositories) //trim possible slashes at the beginning of the repository path for i, repository := range secretRepositories { secretRepositories[i] = strings.TrimPrefix(repository, "/") } // Direct repository match, return secret - log.V(logs.DebugLevel).Info("checking for direct match", "componentRepository", componentRepository, "secretRepositories", secretRepositories) + log.Info("checking for direct match", "componentRepository", componentRepository, "secretRepositories", secretRepositories) if slices.Contains(secretRepositories, componentRepository) { return &secret } - log.V(logs.DebugLevel).Info("no direct match found", "componentRepository", componentRepository, "secretRepositories", secretRepositories) + log.Info("no direct match found", "componentRepository", componentRepository, "secretRepositories", secretRepositories) // No direct match, check for wildcard match, i.e. org/repo/* matches org/repo/foo, org/repo/bar, etc. componentRepoParts := strings.Split(componentRepository, "/") @@ -160,14 +159,14 @@ func bestMatchingSecret(ctx context.Context, componentRepository string, secrets } } } - log.V(logs.DebugLevel).Info("potential matches", "count", len(potentialMatches)) + log.Info("potential matches", "count", len(potentialMatches)) if len(potentialMatches) == 0 { if len(hostOnlySecrets) == 0 { return nil // Nothing matched } return &hostOnlySecrets[0] // Return first host-only secret } - log.V(logs.DebugLevel).Info("host only secrets", "count", len(hostOnlySecrets), "potentialMatches", potentialMatches) + log.Info("host only secrets", "count", len(hostOnlySecrets), "potentialMatches", potentialMatches) // find the best matching secret var bestIndex, bestCount int for i, count := range potentialMatches { diff --git a/pkg/renovate/basicauth.go b/pkg/renovate/basicauth.go index 28695ad4..d6b7a465 100644 --- a/pkg/renovate/basicauth.go +++ b/pkg/renovate/basicauth.go @@ -9,7 +9,6 @@ import ( "github.com/redhat-appstudio/build-service/pkg/boerrors" "github.com/redhat-appstudio/build-service/pkg/git" "github.com/redhat-appstudio/build-service/pkg/git/credentials" - "github.com/redhat-appstudio/build-service/pkg/logs" ) // BasicAuthTaskProvider is an implementation of the renovate.TaskProvider that creates the renovate.Task for the components @@ -37,22 +36,23 @@ func (g BasicAuthTaskProvider) GetNewTasks(ctx context.Context, components []*gi log := ctrllog.FromContext(ctx) // Step 1 componentNamespaceMap := git.NamespaceToComponentMap(components) - log.V(logs.DebugLevel).Info("generating new renovate task in user's namespace for components", "count", len(components)) + log.Info("generating new renovate task in user's namespace for components", "count", len(components)) var newTasks []*Task for namespace, componentsInNamespace := range componentNamespaceMap { - log.V(logs.DebugLevel).Info("found components", "namespace", namespace, "count", len(componentsInNamespace)) + log.Info("found components", "namespace", namespace, "count", len(componentsInNamespace)) // Step 2 platformToComponentMap := git.PlatformToComponentMap(componentsInNamespace) - log.V(logs.DebugLevel).Info("found git platform on namespace", "namespace", namespace, "count", len(platformToComponentMap)) + log.Info("found git platform on namespace", "namespace", namespace, "count", len(platformToComponentMap)) for platform, componentsOnPlatform := range platformToComponentMap { - log.V(logs.DebugLevel).Info("processing components on platform", "platform", platform, "count", len(componentsOnPlatform)) + log.Info("processing components on platform", "platform", platform, "count", len(componentsOnPlatform)) // Step 3 hostToComponentsMap := git.HostToComponentMap(componentsOnPlatform) - log.V(logs.DebugLevel).Info("found hosts on platform", "namespace", namespace, "platform", platform, "count", len(hostToComponentsMap)) + log.Info("found hosts on platform", "namespace", namespace, "platform", platform, "count", len(hostToComponentsMap)) // Step 4 var tasksOnHost []*Task for host, componentsOnHost := range hostToComponentsMap { - log.V(logs.DebugLevel).Info("processing components on host", "namespace", namespace, "platform", platform, "host", host, "count", len(componentsOnHost)) + endpoint := git.BuildAPIEndpoint(platform).APIEndpoint(host) + log.Info("processing components on host", "namespace", namespace, "platform", platform, "host", host, "endpoint", endpoint, "count", len(componentsOnHost)) for _, component := range componentsOnHost { // Step 5 if !AddNewBranchToTheExistedRepositoryTasksOnTheSameHosts(tasksOnHost, component) { @@ -66,7 +66,7 @@ func (g BasicAuthTaskProvider) GetNewTasks(ctx context.Context, components []*gi // Step 6 if !AddNewRepoToTasksOnTheSameHostsWithSameCredentials(tasksOnHost, component, creds) { // Step 7 - tasksOnHost = append(tasksOnHost, NewBasicAuthTask(platform, host, creds, []*Repository{ + tasksOnHost = append(tasksOnHost, NewBasicAuthTask(platform, host, endpoint, creds, []*Repository{ { Repository: component.Repository(), BaseBranches: []string{component.Branch()}, @@ -81,17 +81,17 @@ func (g BasicAuthTaskProvider) GetNewTasks(ctx context.Context, components []*gi } } - log.V(logs.DebugLevel).Info("generated new renovate tasks", "count", len(newTasks)) + log.Info("generated new renovate tasks", "count", len(newTasks)) return newTasks } -func NewBasicAuthTask(platform string, host string, credentials *credentials.BasicAuthCredentials, repositories []*Repository) *Task { +func NewBasicAuthTask(platform, host, endpoint string, credentials *credentials.BasicAuthCredentials, repositories []*Repository) *Task { return &Task{ - Platform: platform, - Username: credentials.Username, - GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.%s>", credentials.Username, credentials.Username, host), - RenovatePattern: GetRenovatePatternConfiguration(), - Token: credentials.Password, - Repositories: repositories, + Platform: platform, + Username: credentials.Username, + GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.%s>", credentials.Username, credentials.Username, host), + Token: credentials.Password, + Endpoint: endpoint, + Repositories: repositories, } } diff --git a/pkg/renovate/basicauth_test.go b/pkg/renovate/basicauth_test.go index 4ee51e5f..d14c2886 100644 --- a/pkg/renovate/basicauth_test.go +++ b/pkg/renovate/basicauth_test.go @@ -36,7 +36,7 @@ func TestNewTasks(t *testing.T) { credentialsFunc: StaticCredentialsFunc, components: []*git.ScmComponent{ignoreError(git.NewScmComponent("github", "https://github.com/umbrellacorp/devfile-sample-go-basic", "main", "devfile-sample-go-basic", "umbrellacorp-tenant")).(*git.ScmComponent)}, expected: []*Task{ - NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{ + NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{ { Repository: "umbrellacorp/devfile-sample-go-basic", BaseBranches: []string{"main"}, @@ -65,7 +65,7 @@ func TestNewTasks(t *testing.T) { "umbrellacorp-tenant")).(*git.ScmComponent), }, expected: []*Task{ - NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{ + NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{ { Repository: "umbrellacorp/devfile-sample-python-basic", BaseBranches: []string{"develop"}, @@ -105,7 +105,7 @@ func TestNewTasks(t *testing.T) { "umbrellacorp-tenant")).(*git.ScmComponent), }, expected: []*Task{ - NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{ + NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{ { Repository: "umbrellacorp/devfile-sample-python-basic", BaseBranches: []string{"develop", "main"}, @@ -138,13 +138,13 @@ func TestNewTasks(t *testing.T) { "umbrellacorp-tenant")).(*git.ScmComponent), }, expected: []*Task{ - NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{ + NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{ { Repository: "umbrellacorp/devfile-sample-python-basic", BaseBranches: []string{"develop"}, }, }), - NewBasicAuthTask("gitlab", "gitlab.com", staticCredentials, []*Repository{ + NewBasicAuthTask("gitlab", "gitlab.com", "https://gitlab.com/api/v4/", staticCredentials, []*Repository{ { Repository: "umbrellacorp/devfile-sample-go-basic", BaseBranches: []string{"main"}, diff --git a/pkg/renovate/config.go b/pkg/renovate/config.go index 5a6dea3e..1d2154a4 100644 --- a/pkg/renovate/config.go +++ b/pkg/renovate/config.go @@ -27,6 +27,7 @@ type JobConfig struct { Tekton Tekton `json:"tekton"` ForkProcessing string `json:"forkProcessing"` DependencyDashboard bool `json:"dependencyDashboard"` + Endpoint string `json:"endpoint,omitempty"` } type Repository struct { @@ -63,7 +64,8 @@ type PackageRule struct { RebaseWhen string `json:"rebaseWhen,omitempty"` } -func NewTektonJobConfig(platform, username, gitAuthor, renovatePattern string, repositories []*Repository) JobConfig { +func NewTektonJobConfig(platform, endpoint, username, gitAuthor string, repositories []*Repository) JobConfig { + renovatePattern := GetRenovatePatternConfiguration() return JobConfig{ Platform: platform, Username: username, @@ -71,6 +73,7 @@ func NewTektonJobConfig(platform, username, gitAuthor, renovatePattern string, r Onboarding: false, RequireConfig: "ignored", EnabledManagers: []string{"tekton"}, + Endpoint: endpoint, Repositories: repositories, Tekton: Tekton{FileMatch: []string{"\\.yaml$", "\\.yml$"}, IncludePaths: []string{".tekton/**"}, PackageRules: []PackageRule{DisableAllPackageRules, { MatchPackagePatterns: []string{renovatePattern}, diff --git a/pkg/renovate/githubapp.go b/pkg/renovate/githubapp.go index 38bce724..b2d8acd9 100644 --- a/pkg/renovate/githubapp.go +++ b/pkg/renovate/githubapp.go @@ -68,11 +68,11 @@ func (g GithubAppRenovaterTaskProvider) GetNewTasks(ctx context.Context, compone func newGithubTask(slug string, token string, repositories []*Repository) *Task { return &Task{ - Platform: "github", - Username: fmt.Sprintf("%s[bot]", slug), - GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.github.com>", slug, slug), - RenovatePattern: GetRenovatePatternConfiguration(), - Token: token, - Repositories: repositories, + Platform: "github", + Endpoint: git.BuildAPIEndpoint("github").APIEndpoint("github.com"), + Username: fmt.Sprintf("%s[bot]", slug), + GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.github.com>", slug, slug), + Token: token, + Repositories: repositories, } } diff --git a/pkg/renovate/job.go b/pkg/renovate/job.go index 176ec84e..a9ce22da 100644 --- a/pkg/renovate/job.go +++ b/pkg/renovate/job.go @@ -67,7 +67,7 @@ func (j *JobCoordinator) Execute(ctx context.Context, tasks []*Task) error { timestamp := time.Now().Unix() name := fmt.Sprintf("renovate-job-%d-%s", timestamp, RandomString(5)) - log.V(logs.DebugLevel).Info(fmt.Sprintf("Creating renovate job %s for %d unique sets of scm repositories", name, len(tasks))) + log.Info(fmt.Sprintf("Creating renovate job %s for %d unique sets of scm repositories", name, len(tasks))) secretTokens := map[string]string{} configMapData := map[string]string{} @@ -82,7 +82,7 @@ func (j *JobCoordinator) Execute(ctx context.Context, tasks []*Task) error { } configMapData[fmt.Sprintf("%s.json", taskId)] = string(config) - log.V(logs.DebugLevel).Info(fmt.Sprintf("Creating renovate config map entry with length %d and value %s", len(config), config)) + log.Info(fmt.Sprintf("Creating renovate config map entry with length %d and value %s", len(config), config)) renovateCmd = append(renovateCmd, fmt.Sprintf("RENOVATE_TOKEN=$TOKEN_%s RENOVATE_CONFIG_FILE=/configs/%s.json renovate", taskId, taskId), ) diff --git a/pkg/renovate/task.go b/pkg/renovate/task.go index 9207f746..b2fc64b4 100644 --- a/pkg/renovate/task.go +++ b/pkg/renovate/task.go @@ -9,12 +9,12 @@ import ( // Task represents a task to be executed by Renovate with credentials and repositories type Task struct { - Platform string - Username string - GitAuthor string - RenovatePattern string - Token string - Repositories []*Repository + Platform string + Username string + GitAuthor string + Token string + Endpoint string + Repositories []*Repository } // AddNewBranchToTheExistedRepositoryTasksOnTheSameHosts iterates over the tasks and adds a new branch to the repository if it already exists @@ -58,5 +58,5 @@ type TaskProvider interface { } func (t *Task) JobConfig() JobConfig { - return NewTektonJobConfig(t.Platform, t.Username, t.GitAuthor, t.RenovatePattern, t.Repositories) + return NewTektonJobConfig(t.Platform, t.Endpoint, t.Username, t.GitAuthor, t.Repositories) }