Skip to content

Commit

Permalink
Renovate support of Gitlab/Github OnPrem (#281)
Browse files Browse the repository at this point in the history
* Renovate support of Gitlab/Github OnPrem
  • Loading branch information
skabashnyuk authored Apr 29, 2024
1 parent f834c1b commit 6f1cd04
Show file tree
Hide file tree
Showing 11 changed files with 156 additions and 49 deletions.
2 changes: 2 additions & 0 deletions controllers/git_tekton_resources_renovater.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package controllers

import (
"context"
"reflect"

"time"

Expand Down Expand Up @@ -124,6 +125,7 @@ func (r *GitTektonResourcesRenovater) Reconcile(ctx context.Context, req ctrl.Re
var tasks []*renovate.Task
for _, taskProvider := range r.taskProviders {
newTasks := taskProvider.GetNewTasks(ctx, scmComponents)
log.Info("found new tasks", "tasks", len(newTasks), "provider", reflect.TypeOf(taskProvider).String())
if len(newTasks) > 0 {
tasks = append(tasks, newTasks...)
}
Expand Down
5 changes: 3 additions & 2 deletions controllers/git_tekton_resources_renovater_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,11 @@ import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

"k8s.io/apimachinery/pkg/types"

. "github.com/redhat-appstudio/build-service/pkg/common"
"github.com/redhat-appstudio/build-service/pkg/git/github"
"github.com/redhat-appstudio/build-service/pkg/renovate"
"k8s.io/apimachinery/pkg/types"
)

var _ = Describe("Git tekton resources renovater", func() {
Expand Down Expand Up @@ -134,7 +135,7 @@ var _ = Describe("Git tekton resources renovater", func() {

deleteSecret(pacSecretKey)
createDefaultBuildPipelineRunSelector(defaultSelectorKey)
Eventually(listEvents).WithArguments("default").WithTimeout(timeout).ShouldNot(HaveLen(0))
Eventually(listEvents).WithArguments("default").WithTimeout(timeout).ShouldNot(BeEmpty())
allEvents := listEvents("default")
Expect(allEvents[0].Reason).To(Equal("ErrorReadingPaCSecret"))
deleteComponent(componentNamespacedName)
Expand Down
48 changes: 48 additions & 0 deletions pkg/git/apiendpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package git

import "fmt"

// APIEndpoint interface defines the method to get the API endpoint url for
// the source code providers.
type APIEndpoint interface {
APIEndpoint(host string) string
}

// GithubAPIEndpoint represents an API endpoint for GitHub.
type GithubAPIEndpoint struct {
}

// APIEndpoint returns the GitHub API endpoint.
func (g *GithubAPIEndpoint) APIEndpoint(host string) string {
return fmt.Sprintf("https://api.%s/", host)
}

// GitlabAPIEndpoint represents an API endpoint for GitLab.
type GitlabAPIEndpoint struct {
}

// APIEndpoint returns the API GitLab endpoint.
func (g *GitlabAPIEndpoint) APIEndpoint(host string) string {
return fmt.Sprintf("https://%s/api/v4/", host)
}

// UnknownAPIEndpoint represents an endpoint for unknown or non existed provider. It returns empty string for api endpoint.
type UnknownAPIEndpoint struct {
}

// APIEndpoint returns the GitLab endpoint.
func (g *UnknownAPIEndpoint) APIEndpoint(host string) string {
return ""
}

// BuildAPIEndpoint constructs and returns an endpoint object based on the type provided type.
func BuildAPIEndpoint(endpointType string) APIEndpoint {
switch endpointType {
case "github":
return &GithubAPIEndpoint{}
case "gitlab":
return &GitlabAPIEndpoint{}
default:
return &UnknownAPIEndpoint{}
}
}
54 changes: 54 additions & 0 deletions pkg/git/apiendpoint_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package git

import (
"testing"
)

func TestBuildEndpoint(t *testing.T) {

tests := []struct {
name string
endpointType string
host string
wantEndpoint string
}{
{
name: "Github SAAS",
endpointType: "github",
host: "github.com",
wantEndpoint: "https://api.github.com/",
},
{
name: "Github On-Prem",
endpointType: "github",
host: "github.umbrella.com",
wantEndpoint: "https://api.github.umbrella.com/",
},
{
name: "Gitlab SAAS",
endpointType: "gitlab",
host: "gitlab.com",
wantEndpoint: "https://gitlab.com/api/v4/",
},
{
name: "Gitlab On-Prem",
endpointType: "gitlab",
host: "gitlab.umbrella.com",
wantEndpoint: "https://gitlab.umbrella.com/api/v4/",
},
{
name: "Unknown provider",
endpointType: "bibi",
host: "bibi.umbrella.com",
wantEndpoint: "",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := BuildAPIEndpoint(tt.endpointType).APIEndpoint(tt.host); got != tt.wantEndpoint {
t.Errorf("BuildAPIEndpoint() = %v, want %v", got, tt.wantEndpoint)
}
})
}
}
19 changes: 9 additions & 10 deletions pkg/k8s/credentials.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (
. "github.com/redhat-appstudio/build-service/pkg/common"
"github.com/redhat-appstudio/build-service/pkg/git"
. "github.com/redhat-appstudio/build-service/pkg/git/credentials"
"github.com/redhat-appstudio/build-service/pkg/logs"
bslices "github.com/redhat-appstudio/build-service/pkg/slices"
)

Expand Down Expand Up @@ -89,7 +88,7 @@ func (k *GitCredentialProvider) GetSSHCredentials(ctx context.Context, component
func (k *GitCredentialProvider) LookupSecret(ctx context.Context, component *git.ScmComponent, secretType corev1.SecretType) (*corev1.Secret, error) {
log := ctrllog.FromContext(ctx)

log.V(logs.DebugLevel).Info("looking for scm secret", "component", component)
log.Info("looking for scm secret", "component", component)

secretList := &corev1.SecretList{}
opts := client.ListOption(&client.MatchingLabels{
Expand All @@ -100,15 +99,15 @@ func (k *GitCredentialProvider) LookupSecret(ctx context.Context, component *git
if err := k.client.List(ctx, secretList, client.InNamespace(component.NamespaceName()), opts); err != nil {
return nil, fmt.Errorf("failed to list Pipelines as Code secrets in %s namespace: %w", component.NamespaceName(), err)
}
log.V(logs.DebugLevel).Info("found secrets", "count", len(secretList.Items))
log.Info("found secrets", "count", len(secretList.Items))
secretsWithCredentialsCandidates := bslices.Filter(secretList.Items, func(secret corev1.Secret) bool {
return secret.Type == secretType && len(secret.Data) > 0
})
secretWithCredential := bestMatchingSecret(ctx, component.Repository(), secretsWithCredentialsCandidates)
if secretWithCredential != nil {
return secretWithCredential, nil
}
log.V(logs.DebugLevel).Info("no matching secret found for component", "component", component)
log.Info("no matching secret found for component", "component", component)
return nil, boerrors.NewBuildOpError(boerrors.EComponentGitSecretMissing, nil)
}

Expand All @@ -128,24 +127,24 @@ func bestMatchingSecret(ctx context.Context, componentRepository string, secrets

for index, secret := range secrets {
repositoryAnnotation, exists := secret.Annotations[ScmSecretRepositoryAnnotation]
log.V(logs.DebugLevel).Info("found secret", "secret", secret.Name, "repositoryAnnotation", repositoryAnnotation, "exists", exists)
log.Info("found secret", "secret", secret.Name, "repositoryAnnotation", repositoryAnnotation, "exists", exists)
if !exists || repositoryAnnotation == "" {
hostOnlySecrets = append(hostOnlySecrets, secret)
continue
}
secretRepositories := strings.Split(repositoryAnnotation, ",")
log.V(logs.DebugLevel).Info("found secret repositories", "repositories", secretRepositories)
log.Info("found secret repositories", "repositories", secretRepositories)
//trim possible slashes at the beginning of the repository path
for i, repository := range secretRepositories {
secretRepositories[i] = strings.TrimPrefix(repository, "/")
}

// Direct repository match, return secret
log.V(logs.DebugLevel).Info("checking for direct match", "componentRepository", componentRepository, "secretRepositories", secretRepositories)
log.Info("checking for direct match", "componentRepository", componentRepository, "secretRepositories", secretRepositories)
if slices.Contains(secretRepositories, componentRepository) {
return &secret
}
log.V(logs.DebugLevel).Info("no direct match found", "componentRepository", componentRepository, "secretRepositories", secretRepositories)
log.Info("no direct match found", "componentRepository", componentRepository, "secretRepositories", secretRepositories)
// No direct match, check for wildcard match, i.e. org/repo/* matches org/repo/foo, org/repo/bar, etc.
componentRepoParts := strings.Split(componentRepository, "/")

Expand All @@ -160,14 +159,14 @@ func bestMatchingSecret(ctx context.Context, componentRepository string, secrets
}
}
}
log.V(logs.DebugLevel).Info("potential matches", "count", len(potentialMatches))
log.Info("potential matches", "count", len(potentialMatches))
if len(potentialMatches) == 0 {
if len(hostOnlySecrets) == 0 {
return nil // Nothing matched
}
return &hostOnlySecrets[0] // Return first host-only secret
}
log.V(logs.DebugLevel).Info("host only secrets", "count", len(hostOnlySecrets), "potentialMatches", potentialMatches)
log.Info("host only secrets", "count", len(hostOnlySecrets), "potentialMatches", potentialMatches)
// find the best matching secret
var bestIndex, bestCount int
for i, count := range potentialMatches {
Expand Down
32 changes: 16 additions & 16 deletions pkg/renovate/basicauth.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"github.com/redhat-appstudio/build-service/pkg/boerrors"
"github.com/redhat-appstudio/build-service/pkg/git"
"github.com/redhat-appstudio/build-service/pkg/git/credentials"
"github.com/redhat-appstudio/build-service/pkg/logs"
)

// BasicAuthTaskProvider is an implementation of the renovate.TaskProvider that creates the renovate.Task for the components
Expand Down Expand Up @@ -37,22 +36,23 @@ func (g BasicAuthTaskProvider) GetNewTasks(ctx context.Context, components []*gi
log := ctrllog.FromContext(ctx)
// Step 1
componentNamespaceMap := git.NamespaceToComponentMap(components)
log.V(logs.DebugLevel).Info("generating new renovate task in user's namespace for components", "count", len(components))
log.Info("generating new renovate task in user's namespace for components", "count", len(components))
var newTasks []*Task
for namespace, componentsInNamespace := range componentNamespaceMap {
log.V(logs.DebugLevel).Info("found components", "namespace", namespace, "count", len(componentsInNamespace))
log.Info("found components", "namespace", namespace, "count", len(componentsInNamespace))
// Step 2
platformToComponentMap := git.PlatformToComponentMap(componentsInNamespace)
log.V(logs.DebugLevel).Info("found git platform on namespace", "namespace", namespace, "count", len(platformToComponentMap))
log.Info("found git platform on namespace", "namespace", namespace, "count", len(platformToComponentMap))
for platform, componentsOnPlatform := range platformToComponentMap {
log.V(logs.DebugLevel).Info("processing components on platform", "platform", platform, "count", len(componentsOnPlatform))
log.Info("processing components on platform", "platform", platform, "count", len(componentsOnPlatform))
// Step 3
hostToComponentsMap := git.HostToComponentMap(componentsOnPlatform)
log.V(logs.DebugLevel).Info("found hosts on platform", "namespace", namespace, "platform", platform, "count", len(hostToComponentsMap))
log.Info("found hosts on platform", "namespace", namespace, "platform", platform, "count", len(hostToComponentsMap))
// Step 4
var tasksOnHost []*Task
for host, componentsOnHost := range hostToComponentsMap {
log.V(logs.DebugLevel).Info("processing components on host", "namespace", namespace, "platform", platform, "host", host, "count", len(componentsOnHost))
endpoint := git.BuildAPIEndpoint(platform).APIEndpoint(host)
log.Info("processing components on host", "namespace", namespace, "platform", platform, "host", host, "endpoint", endpoint, "count", len(componentsOnHost))
for _, component := range componentsOnHost {
// Step 5
if !AddNewBranchToTheExistedRepositoryTasksOnTheSameHosts(tasksOnHost, component) {
Expand All @@ -66,7 +66,7 @@ func (g BasicAuthTaskProvider) GetNewTasks(ctx context.Context, components []*gi
// Step 6
if !AddNewRepoToTasksOnTheSameHostsWithSameCredentials(tasksOnHost, component, creds) {
// Step 7
tasksOnHost = append(tasksOnHost, NewBasicAuthTask(platform, host, creds, []*Repository{
tasksOnHost = append(tasksOnHost, NewBasicAuthTask(platform, host, endpoint, creds, []*Repository{
{
Repository: component.Repository(),
BaseBranches: []string{component.Branch()},
Expand All @@ -81,17 +81,17 @@ func (g BasicAuthTaskProvider) GetNewTasks(ctx context.Context, components []*gi
}

}
log.V(logs.DebugLevel).Info("generated new renovate tasks", "count", len(newTasks))
log.Info("generated new renovate tasks", "count", len(newTasks))
return newTasks
}

func NewBasicAuthTask(platform string, host string, credentials *credentials.BasicAuthCredentials, repositories []*Repository) *Task {
func NewBasicAuthTask(platform, host, endpoint string, credentials *credentials.BasicAuthCredentials, repositories []*Repository) *Task {
return &Task{
Platform: platform,
Username: credentials.Username,
GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.%s>", credentials.Username, credentials.Username, host),
RenovatePattern: GetRenovatePatternConfiguration(),
Token: credentials.Password,
Repositories: repositories,
Platform: platform,
Username: credentials.Username,
GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.%s>", credentials.Username, credentials.Username, host),
Token: credentials.Password,
Endpoint: endpoint,
Repositories: repositories,
}
}
10 changes: 5 additions & 5 deletions pkg/renovate/basicauth_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestNewTasks(t *testing.T) {
credentialsFunc: StaticCredentialsFunc,
components: []*git.ScmComponent{ignoreError(git.NewScmComponent("github", "https://github.com/umbrellacorp/devfile-sample-go-basic", "main", "devfile-sample-go-basic", "umbrellacorp-tenant")).(*git.ScmComponent)},
expected: []*Task{
NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{
NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{
{
Repository: "umbrellacorp/devfile-sample-go-basic",
BaseBranches: []string{"main"},
Expand Down Expand Up @@ -65,7 +65,7 @@ func TestNewTasks(t *testing.T) {
"umbrellacorp-tenant")).(*git.ScmComponent),
},
expected: []*Task{
NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{
NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{
{
Repository: "umbrellacorp/devfile-sample-python-basic",
BaseBranches: []string{"develop"},
Expand Down Expand Up @@ -105,7 +105,7 @@ func TestNewTasks(t *testing.T) {
"umbrellacorp-tenant")).(*git.ScmComponent),
},
expected: []*Task{
NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{
NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{
{
Repository: "umbrellacorp/devfile-sample-python-basic",
BaseBranches: []string{"develop", "main"},
Expand Down Expand Up @@ -138,13 +138,13 @@ func TestNewTasks(t *testing.T) {
"umbrellacorp-tenant")).(*git.ScmComponent),
},
expected: []*Task{
NewBasicAuthTask("github", "github.com", staticCredentials, []*Repository{
NewBasicAuthTask("github", "github.com", "https://api.github.com/", staticCredentials, []*Repository{
{
Repository: "umbrellacorp/devfile-sample-python-basic",
BaseBranches: []string{"develop"},
},
}),
NewBasicAuthTask("gitlab", "gitlab.com", staticCredentials, []*Repository{
NewBasicAuthTask("gitlab", "gitlab.com", "https://gitlab.com/api/v4/", staticCredentials, []*Repository{
{
Repository: "umbrellacorp/devfile-sample-go-basic",
BaseBranches: []string{"main"},
Expand Down
5 changes: 4 additions & 1 deletion pkg/renovate/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type JobConfig struct {
Tekton Tekton `json:"tekton"`
ForkProcessing string `json:"forkProcessing"`
DependencyDashboard bool `json:"dependencyDashboard"`
Endpoint string `json:"endpoint,omitempty"`
}

type Repository struct {
Expand Down Expand Up @@ -63,14 +64,16 @@ type PackageRule struct {
RebaseWhen string `json:"rebaseWhen,omitempty"`
}

func NewTektonJobConfig(platform, username, gitAuthor, renovatePattern string, repositories []*Repository) JobConfig {
func NewTektonJobConfig(platform, endpoint, username, gitAuthor string, repositories []*Repository) JobConfig {
renovatePattern := GetRenovatePatternConfiguration()
return JobConfig{
Platform: platform,
Username: username,
GitAuthor: gitAuthor,
Onboarding: false,
RequireConfig: "ignored",
EnabledManagers: []string{"tekton"},
Endpoint: endpoint,
Repositories: repositories,
Tekton: Tekton{FileMatch: []string{"\\.yaml$", "\\.yml$"}, IncludePaths: []string{".tekton/**"}, PackageRules: []PackageRule{DisableAllPackageRules, {
MatchPackagePatterns: []string{renovatePattern},
Expand Down
12 changes: 6 additions & 6 deletions pkg/renovate/githubapp.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,11 @@ func (g GithubAppRenovaterTaskProvider) GetNewTasks(ctx context.Context, compone

func newGithubTask(slug string, token string, repositories []*Repository) *Task {
return &Task{
Platform: "github",
Username: fmt.Sprintf("%s[bot]", slug),
GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.github.com>", slug, slug),
RenovatePattern: GetRenovatePatternConfiguration(),
Token: token,
Repositories: repositories,
Platform: "github",
Endpoint: git.BuildAPIEndpoint("github").APIEndpoint("github.com"),
Username: fmt.Sprintf("%s[bot]", slug),
GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.github.com>", slug, slug),
Token: token,
Repositories: repositories,
}
}
Loading

0 comments on commit 6f1cd04

Please sign in to comment.