diff --git a/controllers/component_build_controller.go b/controllers/component_build_controller.go index e51d7664..9f01e4b4 100644 --- a/controllers/component_build_controller.go +++ b/controllers/component_build_controller.go @@ -174,6 +174,15 @@ func (r *ComponentBuildReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } + // don't recreate SA upon component deletion + if component.ObjectMeta.DeletionTimestamp.IsZero() { + // Ensure pipeline service account exists + _, err = r.ensurePipelineServiceAccount(ctx, component.Namespace) + if err != nil { + return ctrl.Result{}, err + } + } + if getContainerImageRepositoryForComponent(&component) == "" { // Container image must be set. It's not possible to proceed without it. log.Info("Waiting for ContainerImage to be set") @@ -230,12 +239,6 @@ func (r *ComponentBuildReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, nil } - // Ensure pipeline service account exists - _, err = r.ensurePipelineServiceAccount(ctx, component.Namespace) - if err != nil { - return ctrl.Result{}, err - } - _, err = r.GetBuildPipelineFromComponentAnnotation(ctx, &component) if err != nil { buildStatus := readBuildStatus(&component) diff --git a/controllers/component_dependency_update_controller.go b/controllers/component_dependency_update_controller.go index ca8c8cac..0d73f25c 100644 --- a/controllers/component_dependency_update_controller.go +++ b/controllers/component_dependency_update_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "encoding/base64" "encoding/json" "fmt" "regexp" @@ -96,6 +97,20 @@ type BuildResult struct { Component *applicationapi.Component } +type RepositoryCredentials struct { + SecretName string + RepoName string + UserName string + Password string +} + +type RepositoryConfigAuth struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Email string `json:"email,omitempty"` + Auth string `json:"auth,omitempty"` +} + // SetupController creates a new Integration reconciler and adds it to the Manager. func (r *ComponentDependencyUpdateReconciler) SetupWithManager(manager ctrl.Manager) error { return setupControllerWithManager(manager, r) @@ -194,6 +209,27 @@ func (r *ComponentDependencyUpdateReconciler) Reconcile(ctx context.Context, req } log.Info("component has BuildNudgesRef set", "ComponentName", component.Name, "BuildNudgesRef", component.Spec.BuildNudgesRef) + // verify that there exist some components to be nudged + allComponents := applicationapi.ComponentList{} + err = r.Client.List(ctx, &allComponents, client.InNamespace(pipelineRun.Namespace)) + if err != nil { + log.Error(err, "failed to list components in namespace") + return ctrl.Result{}, err + } + + nudgedComponentsCount := 0 + for i := range allComponents.Items { + comp := allComponents.Items[i] + if slices.Contains(component.Spec.BuildNudgesRef, comp.Name) { + nudgedComponentsCount++ + log.Info("component in BuildNudgesRef exist", "ComponentName", comp.Name) + } + } + if nudgedComponentsCount == 0 { + log.Info("no components in BuildNudgesRef exist", "BuildNudgesRef", component.Spec.BuildNudgesRef) + return ctrl.Result{}, nil + } + if pipelineRun.IsDone() || pipelineRun.Status.CompletionTime != nil || pipelineRun.DeletionTimestamp != nil { result, err := r.verifyUpToDate(ctx, pipelineRun) if err != nil { @@ -351,16 +387,31 @@ func (r *ComponentDependencyUpdateReconciler) handleCompletedBuild(ctx context.C componentsToUpdate = append(componentsToUpdate, comp) } } - var nudgeErr error + updatedOutputImage := updatedComponent.Spec.ContainerImage + imageRepositoryHost := strings.Split(updatedOutputImage, "/")[0] + + imageRepositoryUsername, imageRepositoryPassword, err := r.getImageRepositoryCredentials(ctx, pipelineRun.Namespace, updatedOutputImage) + if err != nil { + // when we can't find credential for repository, remove pipeline finalizer and return error + _, errRemoveFinalizer := r.removePipelineFinalizer(ctx, pipelineRun, patch) + if errRemoveFinalizer != nil { + return ctrl.Result{}, errRemoveFinalizer + } + + return ctrl.Result{}, err + } + + var nudgeErr error var targets []updateTarget - newTargets := r.ComponentDependenciesUpdater.GetUpdateTargetsGithubApp(ctx, componentsToUpdate) + + newTargets := r.ComponentDependenciesUpdater.GetUpdateTargetsGithubApp(ctx, componentsToUpdate, imageRepositoryHost, imageRepositoryUsername, imageRepositoryPassword) log.Info("found new targets for GitHub app", "targets", len(newTargets)) if len(newTargets) > 0 { targets = append(targets, newTargets...) } - newTargets = r.ComponentDependenciesUpdater.GetUpdateTargetsBasicAuth(ctx, componentsToUpdate) + newTargets = r.ComponentDependenciesUpdater.GetUpdateTargetsBasicAuth(ctx, componentsToUpdate, imageRepositoryHost, imageRepositoryUsername, imageRepositoryPassword) log.Info("found new targets for basic auth", "targets", len(newTargets)) if len(newTargets) > 0 { targets = append(targets, newTargets...) @@ -488,6 +539,154 @@ func (r *ComponentDependencyUpdateReconciler) removePipelineFinalizer(ctx contex return ctrl.Result{}, nil } +// getImageRepositoryCredentials returns username and password for image repository +// it is searching all dockerconfigjson type secrets which are linked to the service account +func (r *ComponentDependencyUpdateReconciler) getImageRepositoryCredentials(ctx context.Context, namespace, updatedOutputImage string) (string, string, error) { + log := ctrllog.FromContext(ctx) + + // get service account and gather linked secrets + pipelinesServiceAccount := &corev1.ServiceAccount{} + err := r.Client.Get(ctx, types.NamespacedName{Name: buildPipelineServiceAccountName, Namespace: namespace}, pipelinesServiceAccount) + if err != nil { + log.Error(err, fmt.Sprintf("Failed to read service account %s in namespace %s", buildPipelineServiceAccountName, namespace), l.Action, l.ActionView) + return "", "", err + } + + linkedSecretNames := []string{} + for _, secret := range pipelinesServiceAccount.Secrets { + linkedSecretNames = append(linkedSecretNames, secret.Name) + } + + if len(linkedSecretNames) == 0 { + err = fmt.Errorf("No secrets linked to service account %s in namespace %s", buildPipelineServiceAccountName, namespace) + log.Error(err, "no linked secrets") + return "", "", err + } + log.Info("secrets linked to service account", "count", len(linkedSecretNames)) + + // get all docker config json secrets + allImageRepoSecrets := &corev1.SecretList{} + opts := client.ListOption(&client.MatchingFields{"type": string(corev1.SecretTypeDockerConfigJson)}) + + if err := r.Client.List(ctx, allImageRepoSecrets, client.InNamespace(namespace), opts); err != nil { + return "", "", fmt.Errorf("failed to list secrets of type %s in %s namespace: %w", corev1.SecretTypeDockerConfigJson, namespace, err) + } + log.Info("found docker config secrets secrets", "count", len(allImageRepoSecrets.Items)) + + type DockerConfigJson struct { + ConfigAuths map[string]RepositoryConfigAuth `json:"auths"` + } + + filteredSecretsData := []RepositoryCredentials{} + for _, secret := range allImageRepoSecrets.Items { + isSecretLinked := false + + for _, linkedSecret := range linkedSecretNames { + if secret.Name == linkedSecret { + isSecretLinked = true + break + } + } + if !isSecretLinked { + continue + } + + dockerConfigObject := &DockerConfigJson{} + if err = json.Unmarshal(secret.Data[corev1.DockerConfigJsonKey], dockerConfigObject); err != nil { + log.Error(err, fmt.Sprintf("unable to parse docker json config in the secret %s", secret.Name)) + continue + } + + for repoName, repoAuth := range dockerConfigObject.ConfigAuths { + if repoAuth.Username != "" && repoAuth.Password != "" { + filteredSecretsData = append(filteredSecretsData, RepositoryCredentials{SecretName: secret.Name, RepoName: repoName, UserName: repoAuth.Username, Password: repoAuth.Password}) + } else { + if repoAuth.Auth == "" { + log.Error(fmt.Errorf("password and username and auth are empty in auth config for repository %s", repoName), "no valid auth") + continue + } else { + decodedAuth, err := base64.StdEncoding.DecodeString(repoAuth.Auth) + if err != nil { + log.Error(err, fmt.Sprintf("unable to decode docker config json auth for repository %s in the secret %s", repoName, secret.Name)) + continue + } + authParts := strings.Split(string(decodedAuth), ":") + filteredSecretsData = append(filteredSecretsData, RepositoryCredentials{SecretName: secret.Name, RepoName: repoName, UserName: authParts[0], Password: authParts[1]}) + } + } + } + } + + imageRepositoryUsername, imageRepositoryPassword, err := GetMatchedCredentialForImageRepository(ctx, updatedOutputImage, filteredSecretsData) + if err != nil { + log.Error(err, fmt.Sprintf("unable to find credential for repository %s", updatedOutputImage)) + return "", "", err + } + return imageRepositoryUsername, imageRepositoryPassword, nil +} + +// GetMatchedCredentialForImageRepository returns credentials for image repository +// it is trying to search for credential for the given image repository from all provided credentials +// first it tries to find exact repo match +// then it tries to find the best (the longest) partial match +func GetMatchedCredentialForImageRepository(ctx context.Context, outputImage string, imageRepoSecrets []RepositoryCredentials) (string, string, error) { + log := ctrllog.FromContext(ctx) + + repoPath := outputImage + if strings.Contains(outputImage, "@") { + repoPath = strings.Split(outputImage, "@")[0] + } else { + if strings.Contains(outputImage, ":") { + repoPath = strings.Split(outputImage, ":")[0] + } + } + repoPath = strings.TrimSuffix(repoPath, "/") + + username := "" + password := "" + // first check for credential which matches full repository path + for _, credential := range imageRepoSecrets { + credentialRepo := strings.TrimSuffix(credential.RepoName, "/") + if repoPath == credentialRepo { + log.Info("found full match of repository in auth", "repo", repoPath, "secretName", credential.SecretName) + username = credential.UserName + password = credential.Password + break + } + } + if username != "" && password != "" { + return username, password, nil + + } + + // check for partial match, get the most complete match + // if there is multiple secrets for registry and some partial match, upload sbom would fail anyway, because it chooses them randomly (until cosign fixes it) + repoParts := strings.Split(repoPath, "/") + for { + repoParts = repoParts[:len(repoParts)-1] + if len(repoParts) == 0 { + break + } + partialRepo := strings.Join(repoParts, "/") + + for _, credential := range imageRepoSecrets { + credentialRepo := strings.TrimSuffix(credential.RepoName, "/") + if partialRepo == credentialRepo { + log.Info("partial match found of repository in auth", "repo", partialRepo, "secretName", credential.SecretName) + if credential.UserName != "" && credential.Password != "" { + username = credential.UserName + password = credential.Password + return username, password, nil + } + log.Info("credential in the auth for repository is missing password or username", "repo", partialRepo, "secretName", credential.SecretName) + } + } + } + + log.Info("no credentials found for repo", "repo", repoPath) + return "", "", fmt.Errorf("No credentials found for repository %s ", repoPath) +} + func IsBuildPushPipelineRun(object client.Object) bool { if pipelineRun, ok := object.(*tektonapi.PipelineRun); ok { diff --git a/controllers/component_dependency_update_controller_test.go b/controllers/component_dependency_update_controller_test.go index 7069a434..a285dbba 100644 --- a/controllers/component_dependency_update_controller_test.go +++ b/controllers/component_dependency_update_controller_test.go @@ -17,6 +17,7 @@ package controllers import ( "context" + "encoding/base64" "encoding/json" "fmt" "strings" @@ -57,7 +58,10 @@ var _ = Describe("Component nudge controller", func() { BeforeEach(func() { createNamespace(UserNamespace) baseComponentName := types.NamespacedName{Namespace: UserNamespace, Name: BaseComponent} - createComponent(baseComponentName) + createCustomComponentWithoutBuildRequest(componentConfig{ + componentKey: baseComponentName, + containerImage: "quay.io/organization/repo:tag", + }) createComponent(types.NamespacedName{Namespace: UserNamespace, Name: Operator1}) createComponent(types.NamespacedName{Namespace: UserNamespace, Name: Operator2}) baseComponent := applicationapi.Component{} @@ -81,6 +85,18 @@ var _ = Describe("Component nudge controller", func() { } createSCMSecret(basicSecretKey, basicSecretData, v1.SecretTypeBasicAuth, map[string]string{}) + imageRepoSecretKey := types.NamespacedName{Name: "dockerconfigjsonsecret", Namespace: UserNamespace} + dockerConfigJson := fmt.Sprintf(`{"auths":{"quay.io/organization/repo":{"auth":"%s"}}}`, base64.StdEncoding.EncodeToString([]byte("image_repo_username:image_repo_password"))) + imageRepoSecretData := map[string]string{ + ".dockerconfigjson": dockerConfigJson, + } + createSCMSecret(imageRepoSecretKey, imageRepoSecretData, v1.SecretTypeDockerConfigJson, map[string]string{}) + + serviceAccountKey := types.NamespacedName{Name: buildPipelineServiceAccountName, Namespace: UserNamespace} + sa := waitServiceAccount(serviceAccountKey) + sa.Secrets = []v1.ObjectReference{{Name: "dockerconfigjsonsecret"}} + Expect(k8sClient.Update(ctx, &sa)).To(Succeed()) + github.GetAppInstallationsForRepository = func(githubAppIdStr string, appPrivateKeyPem []byte, repoUrl string) (*github.ApplicationInstallation, string, error) { repo_name := "repo_name" repo_fullname := "repo_fullname" @@ -198,12 +214,139 @@ var _ = Describe("Component nudge controller", func() { // check that no nudgeerror event was reported failureCount := getRenovateFailedEventCount() // check that renovate config was created - renovateConfigsCreated := getRenovateConfigMapCount() + renovateConfigsCreated := len(getRenovateConfigMapList()) // check that renovate pipeline run was created renovatePipelinesCreated := getRenovatePipelineRunCount() + return renovatePipelinesCreated == 1 && renovateConfigsCreated == 1 && failureCount == 0 + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + + renovateConfigMaps := getRenovateConfigMapList() + Expect(len(renovateConfigMaps)).Should(Equal(1)) + for _, renovateConfig := range renovateConfigMaps { + for _, renovateConfigData := range renovateConfig.Data { + Expect(strings.Contains(renovateConfigData, `"username": "image_repo_username"`)).Should(BeTrue()) + } + } + }) + + It("Test build performs nudge on success, image repository partial auth", func() { + imageRepoSecretKey := types.NamespacedName{Name: "dockerconfigjsonsecret", Namespace: UserNamespace} + // create secret with only partial repository match + dockerConfigJson := fmt.Sprintf(`{"auths":{"quay.io":{"auth":"%s"}}}`, base64.StdEncoding.EncodeToString([]byte("image_repo_username_partial:image_repo_password"))) + imageRepoSecretData := map[string]string{ + ".dockerconfigjson": dockerConfigJson, + } + deleteSecret(imageRepoSecretKey) + createSCMSecret(imageRepoSecretKey, imageRepoSecretData, v1.SecretTypeDockerConfigJson, map[string]string{}) + + serviceAccountKey := types.NamespacedName{Name: buildPipelineServiceAccountName, Namespace: UserNamespace} + sa := waitServiceAccount(serviceAccountKey) + sa.Secrets = []v1.ObjectReference{{Name: "dockerconfigjsonsecret"}} + Expect(k8sClient.Update(ctx, &sa)).To(Succeed()) + + createBuildPipelineRun("test-pipeline-1", UserNamespace, BaseComponent) + Eventually(func() bool { + pr := getPipelineRun("test-pipeline-1", UserNamespace) + return controllerutil.ContainsFinalizer(pr, NudgeFinalizer) + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + pr := getPipelineRun("test-pipeline-1", UserNamespace) + pr.Status.SetCondition(&apis.Condition{ + Type: apis.ConditionSucceeded, + Status: "True", + LastTransitionTime: apis.VolatileTime{Inner: metav1.Time{Time: time.Now()}}, + }) + pr.Status.Results = []tektonapi.PipelineRunResult{ + {Name: ImageDigestParamName, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "sha256:12345"}}, + {Name: ImageUrl, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "quay.io.foo/bar:latest"}}, + } + pr.Status.CompletionTime = &metav1.Time{Time: time.Now()} + Expect(k8sClient.Status().Update(ctx, pr)).Should(BeNil()) + Eventually(func() bool { + // check that no nudgeerror event was reported + failureCount := getRenovateFailedEventCount() + // check that renovate config was created + renovateConfigsCreated := len(getRenovateConfigMapList()) + // check that renovate pipeline run was created + renovatePipelinesCreated := getRenovatePipelineRunCount() return renovatePipelinesCreated == 1 && renovateConfigsCreated == 1 && failureCount == 0 }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + + renovateConfigMaps := getRenovateConfigMapList() + Expect(len(renovateConfigMaps)).Should(Equal(1)) + for _, renovateConfig := range renovateConfigMaps { + for _, renovateConfigData := range renovateConfig.Data { + Expect(strings.Contains(renovateConfigData, `"username": "image_repo_username_partial"`)).Should(BeTrue()) + } + } + }) + + It("Test build performs nudge on success, image repository partial auth from multiple secrets", func() { + // create secret with one partial repository match + dockerConfigJson := fmt.Sprintf(`{"auths":{"quay.io":{"auth":"%s"}}}`, base64.StdEncoding.EncodeToString([]byte("image_repo_username_1:image_repo_password"))) + imageRepoSecretData := map[string]string{ + ".dockerconfigjson": dockerConfigJson, + } + imageRepoSecretKey := types.NamespacedName{Name: "dockerconfigjsonsecret", Namespace: UserNamespace} + deleteSecret(imageRepoSecretKey) + createSCMSecret(imageRepoSecretKey, imageRepoSecretData, v1.SecretTypeDockerConfigJson, map[string]string{}) + + // create secret with another partial repository match, this one will be used (because has more complete path) + dockerConfigJson = fmt.Sprintf(`{"auths":{"quay.io/organization":{"auth":"%s"}}}`, base64.StdEncoding.EncodeToString([]byte("image_repo_username_2:image_repo_password"))) + imageRepoSecretData = map[string]string{ + ".dockerconfigjson": dockerConfigJson, + } + imageRepoSecretKey = types.NamespacedName{Name: "dockerconfigjsonsecret2", Namespace: UserNamespace} + createSCMSecret(imageRepoSecretKey, imageRepoSecretData, v1.SecretTypeDockerConfigJson, map[string]string{}) + + // create secret with path which doesn't match at all + dockerConfigJson = fmt.Sprintf(`{"auths":{"registry.io":{"auth":"%s"}}}`, base64.StdEncoding.EncodeToString([]byte("image_repo_username_3:image_repo_password"))) + imageRepoSecretData = map[string]string{ + ".dockerconfigjson": dockerConfigJson, + } + imageRepoSecretKey = types.NamespacedName{Name: "dockerconfigjsonsecret3", Namespace: UserNamespace} + createSCMSecret(imageRepoSecretKey, imageRepoSecretData, v1.SecretTypeDockerConfigJson, map[string]string{}) + + serviceAccountKey := types.NamespacedName{Name: buildPipelineServiceAccountName, Namespace: UserNamespace} + sa := waitServiceAccount(serviceAccountKey) + sa.Secrets = []v1.ObjectReference{{Name: "dockerconfigjsonsecret"}, {Name: "dockerconfigjsonsecret2"}} + Expect(k8sClient.Update(ctx, &sa)).To(Succeed()) + + createBuildPipelineRun("test-pipeline-1", UserNamespace, BaseComponent) + Eventually(func() bool { + pr := getPipelineRun("test-pipeline-1", UserNamespace) + return controllerutil.ContainsFinalizer(pr, NudgeFinalizer) + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + pr := getPipelineRun("test-pipeline-1", UserNamespace) + pr.Status.SetCondition(&apis.Condition{ + Type: apis.ConditionSucceeded, + Status: "True", + LastTransitionTime: apis.VolatileTime{Inner: metav1.Time{Time: time.Now()}}, + }) + pr.Status.Results = []tektonapi.PipelineRunResult{ + {Name: ImageDigestParamName, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "sha256:12345"}}, + {Name: ImageUrl, Value: tektonapi.ResultValue{Type: tektonapi.ParamTypeString, StringVal: "quay.io.foo/bar:latest"}}, + } + pr.Status.CompletionTime = &metav1.Time{Time: time.Now()} + Expect(k8sClient.Status().Update(ctx, pr)).Should(BeNil()) + + Eventually(func() bool { + // check that no nudgeerror event was reported + failureCount := getRenovateFailedEventCount() + // check that renovate config was created + renovateConfigsCreated := len(getRenovateConfigMapList()) + // check that renovate pipeline run was created + renovatePipelinesCreated := getRenovatePipelineRunCount() + return renovatePipelinesCreated == 1 && renovateConfigsCreated == 1 && failureCount == 0 + }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) + + renovateConfigMaps := getRenovateConfigMapList() + Expect(len(renovateConfigMaps)).Should(Equal(1)) + for _, renovateConfig := range renovateConfigMaps { + for _, renovateConfigData := range renovateConfig.Data { + Expect(strings.Contains(renovateConfigData, `"username": "image_repo_username_2"`)).Should(BeTrue()) + } + } }) It("Test stale pipeline not nudged", func() { @@ -235,13 +378,14 @@ var _ = Describe("Component nudge controller", func() { } failureCount := getRenovateFailedEventCount() // check that renovate config was created - renovateConfigsCreated := getRenovateConfigMapCount() + renovateConfigsCreated := len(getRenovateConfigMapList()) // check that renovate pipeline run was created renovatePipelinesCreated := getRenovatePipelineRunCount() return renovatePipelinesCreated == 1 && renovateConfigsCreated == 1 && failureCount == 0 }, timeout, interval).WithTimeout(ensureTimeout).Should(BeTrue()) }) + }) Context("Test nudge failure handling", func() { @@ -284,7 +428,7 @@ var _ = Describe("Component nudge controller", func() { // after one error it will retry and create pipeline Eventually(func() bool { // check that renovate config was created - renovateConfigsCreated := getRenovateConfigMapCount() + renovateConfigsCreated := len(getRenovateConfigMapList()) // check that renovate pipeline run was created renovatePipelinesCreated := getRenovatePipelineRunCount() @@ -344,6 +488,8 @@ var _ = Describe("Component nudge controller", func() { gitAuthor := "renovate-gitauthor" gitToken := "renovate-token" gitEndpoint := "https://api.github.com/" + imageRepositoryHost := "quay.io" + imageRepositoryUsername := "repository_username" fileMatches := "file1, file2, file3" fileMatchesList := `["file1","file2","file3"]` repositories := []renovateRepository{{BaseBranches: []string{"base_branch"}, Repository: "some_repository/something"}} @@ -362,21 +508,26 @@ var _ = Describe("Component nudge controller", func() { FileMatches: fileMatches, } renovateTarget := updateTarget{ - ComponentName: "nudged component", - GitProvider: gitProvider, - Username: gitUsername, - GitAuthor: gitAuthor, - Token: gitToken, - Endpoint: gitEndpoint, - Repositories: repositories, + ComponentName: "nudged component", + GitProvider: gitProvider, + Username: gitUsername, + GitAuthor: gitAuthor, + Token: gitToken, + Endpoint: gitEndpoint, + Repositories: repositories, + ImageRepositoryHost: imageRepositoryHost, + ImageRepositoryUsername: imageRepositoryUsername, } result, err := generateRenovateConfigForNudge(renovateTarget, &buildResult) Expect(err).Should(Succeed()) + Expect(strings.Contains(result, fmt.Sprintf(`platform: "%s"`, gitProvider))).Should(BeTrue()) Expect(strings.Contains(result, fmt.Sprintf(`username: "%s"`, gitUsername))).Should(BeTrue()) Expect(strings.Contains(result, fmt.Sprintf(`gitAuthor: "%s"`, gitAuthor))).Should(BeTrue()) Expect(strings.Contains(result, fmt.Sprintf(`repositories: %s`, repositoriesData))).Should(BeTrue()) + Expect(strings.Contains(result, fmt.Sprintf(`"username": "%s"`, imageRepositoryUsername))).Should(BeTrue()) + Expect(strings.Contains(result, fmt.Sprintf(`"matchHost": "%s"`, imageRepositoryHost))).Should(BeTrue()) Expect(strings.Contains(result, fmt.Sprintf(`endpoint: "%s"`, gitEndpoint))).Should(BeTrue()) Expect(strings.Contains(result, fmt.Sprintf(`"fileMatch": %s`, fileMatchesList))).Should(BeTrue()) Expect(strings.Contains(result, fmt.Sprintf(`"currentValueTemplate": "%s"`, buildResult.BuiltImageTag))).Should(BeTrue()) @@ -446,18 +597,18 @@ func createBuildPipelineRun(name string, namespace string, component string) *te return &run } -func getRenovateConfigMapCount() int { - renovateConfigsCreated := 0 +func getRenovateConfigMapList() []v1.ConfigMap { configMapList := &v1.ConfigMapList{} + renovateConfigMapList := []v1.ConfigMap{} err := k8sClient.List(context.TODO(), configMapList, &client.ListOptions{Namespace: UserNamespace}) Expect(err).ToNot(HaveOccurred()) for _, configMap := range configMapList.Items { if strings.HasPrefix(configMap.ObjectMeta.Name, "renovate-pipeline") { log.Info(configMap.ObjectMeta.Name) - renovateConfigsCreated += 1 + renovateConfigMapList = append(renovateConfigMapList, configMap) } } - return renovateConfigsCreated + return renovateConfigMapList } func getRenovatePipelineRunCount() int { diff --git a/controllers/renovate_util.go b/controllers/renovate_util.go index 7849b8b7..db4395f6 100644 --- a/controllers/renovate_util.go +++ b/controllers/renovate_util.go @@ -42,13 +42,16 @@ type renovateRepository struct { // UpdateTarget represents a target source code repository to be executed by Renovate with credentials and repositories type updateTarget struct { - ComponentName string - GitProvider string - Username string - GitAuthor string - Token string - Endpoint string - Repositories []renovateRepository + ComponentName string + GitProvider string + Username string + GitAuthor string + Token string + Endpoint string + Repositories []renovateRepository + ImageRepositoryHost string + ImageRepositoryUsername string + ImageRepositoryPassword string } type ComponentDependenciesUpdater struct { @@ -65,7 +68,7 @@ func NewComponentDependenciesUpdater(client client.Client, scheme *runtime.Schem } // GetUpdateTargetsBasicAuth This method returns targets for components based on basic auth -func (u ComponentDependenciesUpdater) GetUpdateTargetsBasicAuth(ctx context.Context, componentList []v1alpha1.Component) []updateTarget { +func (u ComponentDependenciesUpdater) GetUpdateTargetsBasicAuth(ctx context.Context, componentList []v1alpha1.Component, imageRepositoryHost, imageRepositoryUsername, imageRepositoryPassword string) []updateTarget { log := logger.FromContext(ctx) targetsToUpdate := []updateTarget{} @@ -125,13 +128,16 @@ func (u ComponentDependenciesUpdater) GetUpdateTargetsBasicAuth(ctx context.Cont } targetsToUpdate = append(targetsToUpdate, updateTarget{ - ComponentName: component.Name, - GitProvider: gitProvider, - Username: username, - GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.%s>", username, username, scmComponent.RepositoryHost()), - Token: creds.Password, - Endpoint: git.BuildAPIEndpoint(gitProvider).APIEndpoint(scmComponent.RepositoryHost()), - Repositories: repositories, + ComponentName: component.Name, + GitProvider: gitProvider, + Username: username, + GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.%s>", username, username, scmComponent.RepositoryHost()), + Token: creds.Password, + Endpoint: git.BuildAPIEndpoint(gitProvider).APIEndpoint(scmComponent.RepositoryHost()), + Repositories: repositories, + ImageRepositoryHost: imageRepositoryHost, + ImageRepositoryUsername: imageRepositoryUsername, + ImageRepositoryPassword: imageRepositoryPassword, }) log.Info("component to update for basic auth", "component", component.Name, "repositories", repositories) } @@ -140,7 +146,7 @@ func (u ComponentDependenciesUpdater) GetUpdateTargetsBasicAuth(ctx context.Cont } // GetUpdateTargetsGithubApp This method returns targets for components based on github app -func (u ComponentDependenciesUpdater) GetUpdateTargetsGithubApp(ctx context.Context, componentList []v1alpha1.Component) []updateTarget { +func (u ComponentDependenciesUpdater) GetUpdateTargetsGithubApp(ctx context.Context, componentList []v1alpha1.Component, imageRepositoryHost, imageRepositoryUsername, imageRepositoryPassword string) []updateTarget { log := logger.FromContext(ctx) // Check if GitHub Application is used, if not then skip pacSecret := corev1.Secret{} @@ -215,14 +221,18 @@ func (u ComponentDependenciesUpdater) GetUpdateTargetsGithubApp(ctx context.Cont log.Info("no repositories found in the installation", "ComponentName", component.Name, "ComponentNamespace", component.Namespace) continue } + targetsToUpdate = append(targetsToUpdate, updateTarget{ - ComponentName: component.Name, - GitProvider: gitProvider, - Username: fmt.Sprintf("%s[bot]", slug), - GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.github.com>", slug, slug), - Token: githubAppInstallation.Token, - Endpoint: git.BuildAPIEndpoint("github").APIEndpoint("github.com"), - Repositories: repositories, + ComponentName: component.Name, + GitProvider: gitProvider, + Username: fmt.Sprintf("%s[bot]", slug), + GitAuthor: fmt.Sprintf("%s <123456+%s[bot]@users.noreply.github.com>", slug, slug), + Token: githubAppInstallation.Token, + Endpoint: git.BuildAPIEndpoint("github").APIEndpoint("github.com"), + Repositories: repositories, + ImageRepositoryHost: imageRepositoryHost, + ImageRepositoryUsername: imageRepositoryUsername, + ImageRepositoryPassword: imageRepositoryPassword, }) log.Info("component to update for installations", "component", component.Name, "repositories", repositories) } @@ -287,12 +297,20 @@ func generateRenovateConfigForNudge(target updateTarget, buildResult *BuildResul followTag: "{{.BuiltImageTag}}" } ], + hostRules: [ + { + "matchHost": "{{.ImageRepositoryHost}}", + "username": "{{.ImageRepositoryUsername}}", + "password": process.env.RENOVATE_REPO_PASS, + } + ], forkProcessing: "enabled", extends: [":gitSignOff"], dependencyDashboard: false } {{end}} ` + data := struct { GitProvider string Username string @@ -305,6 +323,8 @@ func generateRenovateConfigForNudge(target updateTarget, buildResult *BuildResul Digest string DistributionRepositories []string FileMatches string + ImageRepositoryHost string + ImageRepositoryUsername string }{ GitProvider: target.GitProvider, Username: target.Username, @@ -317,6 +337,8 @@ func generateRenovateConfigForNudge(target updateTarget, buildResult *BuildResul Digest: buildResult.Digest, DistributionRepositories: buildResult.DistributionRepositories, FileMatches: string(fileMatch), + ImageRepositoryHost: target.ImageRepositoryHost, + ImageRepositoryUsername: target.ImageRepositoryUsername, } configTemplate, err := template.New("renovate").Parse(body) @@ -354,7 +376,9 @@ func (u ComponentDependenciesUpdater) CreateRenovaterPipeline(ctx context.Contex for _, target := range targets { randomStr1 := RandomString(5) randomStr2 := RandomString(10) + randomStr3 := RandomString(10) secretTokens[randomStr2] = target.Token + secretTokens[randomStr3] = target.ImageRepositoryPassword config, err := GenerateRenovateConfigForNudge(target, buildResult) if err != nil { return err @@ -363,7 +387,7 @@ func (u ComponentDependenciesUpdater) CreateRenovaterPipeline(ctx context.Contex log.Info(fmt.Sprintf("Creating renovate config map entry for %s component with length %d and value %s", target.ComponentName, len(config), config)) renovateCmds = append(renovateCmds, - fmt.Sprintf("RENOVATE_TOKEN=$TOKEN_%s RENOVATE_CONFIG_FILE=/configs/%s-%s.js renovate", randomStr2, target.ComponentName, randomStr1), + fmt.Sprintf("RENOVATE_PR_HOURLY_LIMIT=0 RENOVATE_PR_CONCURRENT_LIMIT=0 RENOVATE_TOKEN=$TOKEN_%s RENOVATE_REPO_PASS=$TOKEN_%s RENOVATE_CONFIG_FILE=/configs/%s-%s.js renovate", randomStr2, randomStr3, target.ComponentName, randomStr1), ) } if len(renovateCmds) == 0 { diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 530f3863..33b7ef3f 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -65,6 +65,13 @@ func TestAPIs(t *testing.T) { RunSpecs(t, "Controller Suite") } +func getCacheExcludedObjectsTypes() []client.Object { + return []client.Object{ + &corev1.Secret{}, + &corev1.ConfigMap{}, + } +} + var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) @@ -129,8 +136,15 @@ var _ = BeforeSuite(func() { Expect(patchPipelineRunCRD()).Should(Succeed()) + clientOpts := client.Options{ + Cache: &client.CacheOptions{ + DisableFor: getCacheExcludedObjectsTypes(), + }, + } + k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, + Client: clientOpts, }) Expect(err).ToNot(HaveOccurred()) diff --git a/controllers/suite_util_test.go b/controllers/suite_util_test.go index b25a0923..4c7b5493 100644 --- a/controllers/suite_util_test.go +++ b/controllers/suite_util_test.go @@ -592,14 +592,16 @@ func deleteBuildPipelineConfigMap(configMapKey types.NamespacedName) { } } -func waitServiceAccount(serviceAccountKey types.NamespacedName) { - serviceAccount := &corev1.ServiceAccount{} +func waitServiceAccount(serviceAccountKey types.NamespacedName) corev1.ServiceAccount { + serviceAccount := corev1.ServiceAccount{} Eventually(func() bool { - if err := k8sClient.Get(ctx, serviceAccountKey, serviceAccount); err != nil { + if err := k8sClient.Get(ctx, serviceAccountKey, &serviceAccount); err != nil { return false } return serviceAccount.ResourceVersion != "" }, timeout, interval).Should(BeTrue()) + + return serviceAccount } func deleteServiceAccount(serviceAccountKey types.NamespacedName) { @@ -626,7 +628,7 @@ func deleteServiceAccount(serviceAccountKey types.NamespacedName) { func waitPipelineServiceAccount(namespace string) { pipelineServiceAccountKey := types.NamespacedName{Name: buildPipelineServiceAccountName, Namespace: namespace} - waitServiceAccount(pipelineServiceAccountKey) + _ = waitServiceAccount(pipelineServiceAccountKey) } func deletePipelineServiceAccount(namespace string) {