Don't iterate all GitHub repositories

controllers/component_dependency_update_controller.go

@@ -171,6 +171,13 @@ func (r *ComponentDependencyUpdateReconciler) Reconcile(ctx context.Context, req
 
 	if pipelineRun.IsDone() || pipelineRun.Status.CompletionTime != nil || pipelineRun.DeletionTimestamp != nil {
 		log.Info("PipelineRun complete")
+		//these objects are so heavily contented that we always grab the latest copy from the
+		//api server to reduce the chance of conflicts
+		//and we update in a loop
+		err = r.ApiReader.Get(ctx, req.NamespacedName, pipelineRun)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
 		if controllerutil.ContainsFinalizer(pipelineRun, NudgeFinalizer) || pipelineRun.Annotations == nil || pipelineRun.Annotations[NudgeProcessedAnnotationName] == "" {
 			log.Info("running renovate job")
 			// Pipeline run is done and we have not cleared the finalizer yet
@@ -281,7 +288,7 @@ func (r *ComponentDependencyUpdateReconciler) handleCompletedBuild(ctx context.C
 		}
 		failureCount, err := strconv.Atoi(existing)
 		if err != nil {
-			log.Error(err, "failed to pass retry count, not retrying")
+			log.Error(err, "failed to parse retry count, not retrying")
 			return r.removePipelineFinalizer(ctx, pipelineRun)
 		}
 		failureCount = failureCount + 1
@@ -340,7 +347,6 @@ func (r *ComponentDependencyUpdateReconciler) handleCompletedBuild(ctx context.C
 			if possiblyStalePr.Annotations == nil {
 				possiblyStalePr.Annotations = map[string]string{}
 			}
-			possiblyStalePr.Annotations[NudgeProcessedAnnotationName] = "true"
 			_, err := r.removePipelineFinalizer(ctx, &possiblyStalePr)
 			if err != nil {
 				finalizerError = err
@@ -357,18 +363,12 @@ func (r *ComponentDependencyUpdateReconciler) handleCompletedBuild(ctx context.C
 // Run until it is completed, e.g. if the controller was down
 func (r *ComponentDependencyUpdateReconciler) removePipelineFinalizer(ctx context.Context, pipelineRun *tektonapi.PipelineRun) (ctrl.Result, error) {
 
-	//these objects are so heavily contented that we always grab the latest copy from the
-	//api server to reduce the chance of conflicts
-	err := r.ApiReader.Get(ctx, types.NamespacedName{Namespace: pipelineRun.Namespace, Name: pipelineRun.Name}, pipelineRun)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
 	if pipelineRun.Annotations == nil {
 		pipelineRun.Annotations = map[string]string{}
 	}
 	pipelineRun.Annotations[NudgeProcessedAnnotationName] = "true"
 	controllerutil.RemoveFinalizer(pipelineRun, NudgeFinalizer)
-	err = r.Client.Update(ctx, pipelineRun)
+	err := r.Client.Update(ctx, pipelineRun)
 	if err != nil {
 		if !errors.IsConflict(err) {
 			log := ctrllog.FromContext(ctx)
@@ -420,8 +420,8 @@ func GetComponentFromPipelineRun(c client.Client, ctx context.Context, pipelineR
 }
 func DefaultDependenciesUpdate(ctx context.Context, client client.Client, scheme *runtime.Scheme, eventRecorder record.EventRecorder, downstreamComponents []applicationapi.Component, result *BuildResult) (immediateRetry bool, err error) {
 	log := ctrllog.FromContext(ctx)
-	log.Info("reading github installations")
-	slug, installationsToUpdate, err := GetGithubInstallations(ctx, client, eventRecorder, downstreamComponents)
+	log.Info(fmt.Sprintf("reading github installations for %d components", len(downstreamComponents)))
+	slug, installationsToUpdate, err := GetGithubInstallationsForComponents(ctx, client, eventRecorder, downstreamComponents)
 	if err != nil || slug == "" {
 		return false, err
 	}

controllers/renovate_util.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"os"
+	"regexp"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	log2 "sigs.k8s.io/controller-runtime/pkg/log"
@@ -99,8 +100,94 @@ func GetGithubInstallations(ctx context.Context, client client.Client, eventReco
 	return slug, installationsToUpdate, nil
 }
 
+// This method avoids iterating over all installations, it is intended to be called when the component list is small
+func GetGithubInstallationsForComponents(ctx context.Context, client client.Client, eventRecorder record.EventRecorder, componentList []v1alpha1.Component) (string, []installationStruct, error) {
+	log := log2.FromContext(ctx)
+	// Check if GitHub Application is used, if not then skip
+	pacSecret := v1.Secret{}
+	globalPaCSecretKey := types.NamespacedName{Namespace: buildServiceNamespaceName, Name: prepare.PipelinesAsCodeSecretName}
+	if err := client.Get(ctx, globalPaCSecretKey, &pacSecret); err != nil {
+		eventRecorder.Event(&pacSecret, "Warning", "ErrorReadingPaCSecret", err.Error())
+		if errors.IsNotFound(err) {
+			log.Error(err, "not found Pipelines as Code secret in %s namespace: %w", globalPaCSecretKey.Namespace, err, logs.Action, logs.ActionView)
+		} else {
+			log.Error(err, "failed to get Pipelines as Code secret in %s namespace: %w", globalPaCSecretKey.Namespace, err, logs.Action, logs.ActionView)
+		}
+		return "", nil, nil
+	}
+	isApp := gitops.IsPaCApplicationConfigured("github", pacSecret.Data)
+	if !isApp {
+		log.Info("GitHub App is not set")
+		return "", nil, nil
+	}
+
+	// Load GitHub App and get GitHub Installations
+	githubAppIdStr := string(pacSecret.Data[gitops.PipelinesAsCode_githubAppIdKey])
+	privateKey := pacSecret.Data[gitops.PipelinesAsCode_githubPrivateKey]
+
+	ghUrlRegex, err := regexp.Compile(`github.com/([^/]+)/([^/]+)(\.git)?$`)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// Match installed repositories with Components and get custom branch if defined
+	installationsToUpdate := []installationStruct{}
+	var slug string
+	for _, component := range componentList {
+		if component.Spec.Source.GitSource == nil {
+			continue
+		}
+
+		gitSource := component.Spec.Source.GitSource
+		match := ghUrlRegex.FindStringSubmatch(gitSource.URL)
+		if match == nil {
+			log.Info(fmt.Sprintf("Unable to nudge %s as it is not a github repo", gitSource.URL))
+			continue
+		}
+
+		githubAppInstallation, slugTmp, err := github.GetAppInstallationsForRepository(githubAppIdStr, privateKey, match[1], match[2])
+		if slug == "" {
+			slug = slugTmp
+		}
+		if err != nil {
+			log.Error(err, fmt.Sprintf("Failed to get GitHub app installation for component %s/%s", component.Namespace, component.Name))
+			continue
+		}
+
+		branch := gitSource.Revision
+		if branch == "" {
+			branch = InternalDefaultBranch
+		}
+
+		repositories := []renovateRepository{}
+		for _, repository := range githubAppInstallation.Repositories {
+			if branch == InternalDefaultBranch {
+				branch = repository.GetDefaultBranch()
+			}
+
+			repositories = append(repositories, renovateRepository{
+				BaseBranches: []string{branch},
+				Repository:   repository.GetFullName(),
+			})
+		}
+		// Do not add intatallation which has no matching repositories
+		if len(repositories) == 0 {
+			continue
+		}
+		installationsToUpdate = append(installationsToUpdate,
+			installationStruct{
+				id:           int(githubAppInstallation.ID),
+				token:        githubAppInstallation.Token,
+				repositories: repositories,
+			})
+	}
+
+	return slug, installationsToUpdate, nil
+}
+
 func CreateRenovaterJob(ctx context.Context, client client.Client, scheme *runtime.Scheme, installations []installationStruct, slug string, debug bool, js func(slug string, repositories []renovateRepository, info interface{}) (string, error), info interface{}) error {
 	log := log2.FromContext(ctx)
+	log.Info(fmt.Sprintf("Creating renovate job for %d installations", len(installations)))
 
 	if len(installations) == 0 {
 		return nil
@@ -117,6 +204,8 @@ func CreateRenovaterJob(ctx context.Context, client client.Client, scheme *runti
 			return err
 		}
 		configmaps[fmt.Sprintf("%d.js", installation.id)] = config
+
+		log.Info(fmt.Sprintf("Creating renovate config map entry for %d installation with length %d and value %s", installation.id, len(config), config))
 		renovateCmds = append(renovateCmds,
 			fmt.Sprintf("RENOVATE_TOKEN=$TOKEN_%d RENOVATE_CONFIG_FILE=/configs/%d.js renovate", installation.id, installation.id),
 		)

go.mod

@@ -3,10 +3,10 @@ module github.com/redhat-appstudio/build-service
 go 1.20
 
 require (
-	github.com/bradleyfalzon/ghinstallation/v2 v2.1.1-0.20221216144751-8f41e6541ca6
+	github.com/bradleyfalzon/ghinstallation/v2 v2.2.0
 	github.com/devfile/api/v2 v2.2.1-alpha.0.20230413012049-a6c32fca0dbd
 	github.com/go-logr/logr v1.3.0
-	github.com/google/go-containerregistry v0.13.0
+	github.com/google/go-containerregistry v0.15.2
 	github.com/h2non/gock v1.2.0
 	github.com/onsi/ginkgo/v2 v2.13.1
 	github.com/onsi/gomega v1.29.0
@@ -22,13 +22,11 @@ require (
 	k8s.io/apimachinery v0.28.3
 	k8s.io/client-go v1.5.2
 	k8s.io/klog/v2 v2.110.1
-	knative.dev/pkg v0.0.0-20230125083639-408ad0773f47
+	knative.dev/pkg v0.0.0-20230320014357-4c84b1b51ee8
 	sigs.k8s.io/controller-runtime v0.16.3
 	sigs.k8s.io/yaml v1.4.0
 )
 
-require k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
-
 // If you update dependencies below you must also update controllers/suite_test.go
 require (
 	github.com/openshift-pipelines/pipelines-as-code v0.18.0
@@ -47,20 +45,20 @@ require (
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blendle/zapdriver v1.3.1 // indirect
-	github.com/bluekeyes/go-gitdiff v0.7.0 // indirect
-	github.com/bradleyfalzon/ghinstallation/v2 v2.2.0
+	github.com/bluekeyes/go-gitdiff v0.7.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/containerd/containerd v1.6.19 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/devfile/library/v2 v2.2.1-0.20230418160146-e75481b7eebd // indirect
 	github.com/devfile/registry-support/index/generator v0.0.0-20230123181701-4de4dadb13e7 // indirect
 	github.com/devfile/registry-support/registry-library v0.0.0-20230123181701-4de4dadb13e7 // indirect
-	github.com/docker/cli v23.0.1+incompatible // indirect
+	github.com/docker/cli v23.0.5+incompatible // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/docker/docker v23.0.1+incompatible // indirect
+	github.com/docker/docker v23.0.5+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
@@ -89,8 +87,8 @@ require (
 	github.com/google/btree v1.1.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
-	github.com/google/go-containerregistry v0.14.0
 	github.com/google/go-github/v45 v45.2.0
+	github.com/google/go-github/v50 v50.2.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20231101202521-4ca4178f5c7a // indirect
@@ -107,11 +105,11 @@ require (
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/jenkins-x/go-scm v1.13.9 // indirect
+	github.com/jenkins-x/go-scm v1.13.13 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
-	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/klauspost/compress v1.16.5 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
@@ -127,7 +125,7 @@ require (
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
+	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -147,6 +145,7 @@ require (
 	github.com/spf13/afero v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
@@ -179,15 +178,10 @@ require (
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20231113174909-778a5567bc1e // indirect
 	k8s.io/pod-security-admission v0.26.1 // indirect
-<<<<<<< HEAD
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
-	knative.dev/pkg v0.0.0-20230320014357-4c84b1b51ee8 // indirect
-=======
->>>>>>> 263d9ea (Add renovate job)
 	oras.land/oras-go v1.1.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	github.com/google/go-github/v50 v50.2.0 // indirect
 )
 
 replace k8s.io/client-go v1.5.2 => k8s.io/client-go v0.28.3