From 17ce65f917dd7df14ef9fdc656edd3cf0eea99de Mon Sep 17 00:00:00 2001 From: Somtochi Onyekwere Date: Wed, 22 Nov 2023 02:05:27 +0100 Subject: [PATCH] Bootstrap healthcheck: Check GitRepository and return Ready condition message on timeout Signed-off-by: Somtochi Onyekwere --- cmd/flux/events_test.go | 28 +- pkg/bootstrap/bootstrap.go | 94 ++++-- pkg/bootstrap/bootstrap_plain_git.go | 57 +++- pkg/bootstrap/bootstrap_test.go | 469 +++++++++++++++++++++++++++ pkg/status/status.go | 2 +- 5 files changed, 595 insertions(+), 55 deletions(-) create mode 100644 pkg/bootstrap/bootstrap_test.go diff --git a/cmd/flux/events_test.go b/cmd/flux/events_test.go index a6b5e11f28..7737f7c40a 100644 --- a/cmd/flux/events_test.go +++ b/cmd/flux/events_test.go @@ -27,20 +27,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - helmv2beta1 "github.com/fluxcd/helm-controller/api/v2beta1" - autov1 "github.com/fluxcd/image-automation-controller/api/v1beta1" - imagev1 "github.com/fluxcd/image-reflector-controller/api/v1beta2" - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" - notificationv1 "github.com/fluxcd/notification-controller/api/v1" - notificationv1b2 "github.com/fluxcd/notification-controller/api/v1beta2" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/ssa" - sourcev1 "github.com/fluxcd/source-controller/api/v1" - sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/fluxcd/flux2/v2/internal/utils" ) @@ -172,7 +163,7 @@ func Test_getObjectRef(t *testing.T) { objs, err := ssa.ReadObjects(strings.NewReader(objects)) g.Expect(err).To(Not(HaveOccurred())) - builder := fake.NewClientBuilder().WithScheme(getScheme()) + builder := fake.NewClientBuilder().WithScheme(utils.NewScheme()) for _, obj := range objs { builder = builder.WithObjects(obj) } @@ -256,7 +247,7 @@ func Test_getRows(t *testing.T) { objs, err := ssa.ReadObjects(strings.NewReader(objects)) g.Expect(err).To(Not(HaveOccurred())) - builder := fake.NewClientBuilder().WithScheme(getScheme()) + builder := fake.NewClientBuilder().WithScheme(utils.NewScheme()) for _, obj := range objs { builder = builder.WithObjects(obj) } @@ -410,21 +401,6 @@ func getTestListOpt(kind, name string) client.ListOption { return client.MatchingFieldsSelector{Selector: sel} } -func getScheme() *runtime.Scheme { - newscheme := runtime.NewScheme() - corev1.AddToScheme(newscheme) - kustomizev1.AddToScheme(newscheme) - helmv2beta1.AddToScheme(newscheme) - notificationv1.AddToScheme(newscheme) - notificationv1b2.AddToScheme(newscheme) - imagev1.AddToScheme(newscheme) - autov1.AddToScheme(newscheme) - sourcev1.AddToScheme(newscheme) - sourcev1b2.AddToScheme(newscheme) - - return newscheme -} - func createEvent(obj client.Object, eventType, msg, reason string) corev1.Event { return corev1.Event{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index 1111f6e9ac..88a4a65bd7 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "k8s.io/apimachinery/pkg/util/wait" "strings" "time" @@ -28,12 +27,17 @@ import ( apierr "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + apierrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" "github.com/fluxcd/pkg/apis/meta" - sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" + sourcev1 "github.com/fluxcd/source-controller/api/v1" + sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/fluxcd/flux2/v2/pkg/manifestgen/install" "github.com/fluxcd/flux2/v2/pkg/manifestgen/sourcesecret" @@ -44,6 +48,10 @@ var ( ErrReconciledWithWarning = errors.New("reconciled with warning") ) +// Reconciler reconciles and reports the health of different +// components and kubernetes resources involved in the installation of Flux. +// +// It is recommended use the `ReconcilerWithSyncCheck` interface that also reports the health of the GitRepository. type Reconciler interface { // ReconcileComponents reconciles the components by generating the // manifests with the provided values, committing them to Git and @@ -62,8 +70,7 @@ type Reconciler interface { // and pushing to remote if there are any changes. ReconcileSyncConfig(ctx context.Context, options sync.Options) error - // ReportKustomizationHealth reports about the health of the - // Kustomization synchronizing the components. + // ReportKustomizationHealth reports about the health of the Kustomization synchronizing the components. ReportKustomizationHealth(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error // ReportComponentsHealth reports about the health for the components @@ -76,6 +83,14 @@ type RepositoryReconciler interface { ReconcileRepository(ctx context.Context) error } +// ReconcilerWithSyncCheck extends the Reconciler interface to also report the health of the GitReposiotry +// that syncs Flux on the cluster +type ReconcilerWithSyncCheck interface { + Reconciler + // ReportGitRepoHealth reports about the health of the GitRepository synchronizing the components. + ReportGitRepoHealth(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error +} + type PostGenerateSecretFunc func(ctx context.Context, secret corev1.Secret, options sourcesecret.Options) error func Run(ctx context.Context, reconciler Reconciler, manifestsBase string, @@ -99,18 +114,25 @@ func Run(ctx context.Context, reconciler Reconciler, manifestsBase string, return err } - var healthErrCount int + var errs []error + if r, ok := reconciler.(ReconcilerWithSyncCheck); ok { + if err := r.ReportGitRepoHealth(ctx, syncOpts, pollInterval, timeout); err != nil { + errs = append(errs, err) + } + } + if err := reconciler.ReportKustomizationHealth(ctx, syncOpts, pollInterval, timeout); err != nil { - healthErrCount++ + errs = append(errs, err) } + if err := reconciler.ReportComponentsHealth(ctx, installOpts, timeout); err != nil { - healthErrCount++ + errs = append(errs, err) } - if healthErrCount > 0 { + if len(errs) > 0 { // Composing a "smart" error message here from the returned // errors does not result in any useful information for the // user, as both methods log the failures they run into. - err = fmt.Errorf("bootstrap failed with %d health check failure(s)", healthErrCount) + err = fmt.Errorf("bootstrap failed with %d health check failure(s): %w", len(errs), apierrors.NewAggregate(errs)) } return err @@ -173,32 +195,47 @@ func kustomizationPathDiffers(ctx context.Context, kube client.Client, objKey cl return k.Spec.Path, nil } -func kustomizationReconciled(kube client.Client, objKey client.ObjectKey, kustomization *kustomizev1.Kustomization, expectRevision string) wait.ConditionWithContextFunc { +type objectWithConditions interface { + client.Object + GetConditions() []metav1.Condition +} + +func objectReconciled(kube client.Client, objKey client.ObjectKey, clientObject objectWithConditions, expectRevision string) wait.ConditionWithContextFunc { return func(ctx context.Context) (bool, error) { - if err := kube.Get(ctx, objKey, kustomization); err != nil { + // for some reason, TypeMeta gets unset after kube.Get so we want to store the GVK and set it after + // ref https://github.com/kubernetes-sigs/controller-runtime/issues/1517#issuecomment-844703142 + gvk := clientObject.GetObjectKind().GroupVersionKind() + if err := kube.Get(ctx, objKey, clientObject); err != nil { return false, err } + clientObject.GetObjectKind().SetGroupVersionKind(gvk) - // Detect suspended Kustomization, as this would result in an endless wait - if kustomization.Spec.Suspend { - return false, fmt.Errorf("Kustomization is suspended") + kind := gvk.Kind + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(clientObject) + if err != nil { + return false, err } - // Confirm the state we are observing is for the current generation - if kustomization.Generation != kustomization.Status.ObservedGeneration { - return false, nil + // Detect suspended object, as this would result in an endless wait + if suspended, ok, _ := unstructured.NestedBool(obj, "spec", "suspend"); ok && suspended { + return false, fmt.Errorf("%s '%s' is suspended", kind, objKey.String()) } - // Confirm the given revision has been attempted by the controller - if sourcev1.TransformLegacyRevision(kustomization.Status.LastAttemptedRevision) != expectRevision { + // Confirm the state we are observing is for the current generation + if generation, ok, _ := unstructured.NestedInt64(obj, "status", "observedGeneration"); ok && generation != clientObject.GetGeneration() { return false, nil } // Confirm the resource is healthy - if c := apimeta.FindStatusCondition(kustomization.Status.Conditions, meta.ReadyCondition); c != nil { + if c := apimeta.FindStatusCondition(clientObject.GetConditions(), meta.ReadyCondition); c != nil { switch c.Status { case metav1.ConditionTrue: - return true, nil + // Confirm the given revision has been attempted by the controller + hasRev, err := hasRevision(kind, obj, expectRevision) + if err != nil { + return false, err + } + return hasRev, nil case metav1.ConditionFalse: return false, fmt.Errorf(c.Message) } @@ -207,6 +244,21 @@ func kustomizationReconciled(kube client.Client, objKey client.ObjectKey, kustom } } +// hasRevision checks that the reconciled revision (for Kustomization this is `.status.lastAttemptedRevision` +// and for Source APIs, it is stored in `.status.artifact.revision`) is the same as the expectedRev +func hasRevision(kind string, obj map[string]interface{}, expectedRev string) (bool, error) { + var rev string + switch kind { + case sourcev1.GitRepositoryKind, sourcev1b2.OCIRepositoryKind, sourcev1b2.BucketKind, sourcev1b2.HelmChartKind: + rev, _, _ = unstructured.NestedString(obj, "status", "artifact", "revision") + case kustomizev1.KustomizationKind: + rev, _, _ = unstructured.NestedString(obj, "status", "lastAttemptedRevision") + default: + return false, fmt.Errorf("cannot get status revision for kind: '%s'", kind) + } + return sourcev1b2.TransformLegacyRevision(rev) == expectedRev, nil +} + func retry(retries int, wait time.Duration, fn func() error) (err error) { for i := 0; ; i++ { err = fn() diff --git a/pkg/bootstrap/bootstrap_plain_git.go b/pkg/bootstrap/bootstrap_plain_git.go index 1be3d2659b..c8a1de04c4 100644 --- a/pkg/bootstrap/bootstrap_plain_git.go +++ b/pkg/bootstrap/bootstrap_plain_git.go @@ -29,6 +29,8 @@ import ( "github.com/ProtonMail/go-crypto/openpgp" gogit "github.com/go-git/go-git/v5" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -38,10 +40,12 @@ import ( "github.com/fluxcd/cli-utils/pkg/object" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" + "github.com/fluxcd/pkg/apis/meta" "github.com/fluxcd/pkg/git" "github.com/fluxcd/pkg/git/repository" "github.com/fluxcd/pkg/kustomize/filesys" runclient "github.com/fluxcd/pkg/runtime/client" + sourcev1 "github.com/fluxcd/source-controller/api/v1" "github.com/fluxcd/flux2/v2/internal/utils" "github.com/fluxcd/flux2/v2/pkg/log" @@ -397,20 +401,59 @@ func (b *PlainGitBootstrapper) ReportKustomizationHealth(ctx context.Context, op objKey := client.ObjectKey{Name: options.Name, Namespace: options.Namespace} - b.logger.Waitingf("waiting for Kustomization %q to be reconciled", objKey.String()) - expectRevision := fmt.Sprintf("%s@%s", options.Branch, git.Hash(head).Digest()) - var k kustomizev1.Kustomization + b.logger.Waitingf("waiting for Kustomization %q to be reconciled", objKey.String()) + k := &kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: kustomizev1.KustomizationKind, + }, + } if err := wait.PollUntilContextTimeout(ctx, pollInterval, timeout, true, - kustomizationReconciled(b.kube, objKey, &k, expectRevision)); err != nil { - b.logger.Failuref(err.Error()) - return err + objectReconciled(b.kube, objKey, k, expectRevision)); err != nil { + // if the poll timed out, we want to log the ready condition message as + // that likely contains the reason + if errors.Is(err, context.DeadlineExceeded) { + readyCondition := apimeta.FindStatusCondition(k.Status.Conditions, meta.ReadyCondition) + if readyCondition != nil && readyCondition.Status != metav1.ConditionTrue { + err = fmt.Errorf("kustomization '%s' not ready: '%s'", objKey, readyCondition.Message) + } + } + return fmt.Errorf("error while waiting for Kustomization to be ready: '%s'", err) } - b.logger.Successf("Kustomization reconciled successfully") + return nil } +func (b *PlainGitBootstrapper) ReportGitRepoHealth(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { + head, err := b.gitClient.Head() + if err != nil { + return err + } + + objKey := client.ObjectKey{Name: options.Name, Namespace: options.Namespace} + + b.logger.Waitingf("waiting for GitRepository %q to be reconciled", objKey.String()) + expectRevision := fmt.Sprintf("%s@%s", options.Branch, git.Hash(head).Digest()) + g := &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + } + if err := wait.PollUntilContextTimeout(ctx, pollInterval, timeout, true, + objectReconciled(b.kube, objKey, g, expectRevision)); err != nil { + if errors.Is(err, context.DeadlineExceeded) { + readyCondition := apimeta.FindStatusCondition(g.Status.Conditions, meta.ReadyCondition) + if readyCondition != nil && readyCondition.Status != metav1.ConditionTrue { + return fmt.Errorf("gitrepository '%s' not ready: '%s'", objKey, readyCondition.Message) + } + } + return fmt.Errorf("error while waiting for gitrepository to be ready: '%s'", err) + } + b.logger.Successf("GitRepsoitory reconciled successfully") + return nil +} func (b *PlainGitBootstrapper) ReportComponentsHealth(ctx context.Context, install install.Options, timeout time.Duration) error { cfg, err := utils.KubeConfig(b.restClientGetter, b.restClientOptions) if err != nil { diff --git a/pkg/bootstrap/bootstrap_test.go b/pkg/bootstrap/bootstrap_test.go new file mode 100644 index 0000000000..47bf07f98d --- /dev/null +++ b/pkg/bootstrap/bootstrap_test.go @@ -0,0 +1,469 @@ +/* +Copyright 2023 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "testing" + + "github.com/fluxcd/pkg/apis/meta" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/fluxcd/flux2/v2/internal/utils" + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1" + notificationv1 "github.com/fluxcd/notification-controller/api/v1beta2" + sourcev1 "github.com/fluxcd/source-controller/api/v1" + sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2" +) + +func Test_hasRevision(t *testing.T) { + var revision = "main@sha1:5bf3a8f9bb0aa5ae8afd6208f43757ab73fc033a" + + tests := []struct { + name string + obj objectWithConditions + rev string + expectErr bool + expectedBool bool + }{ + { + name: "Kustomization revision", + obj: &kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: kustomizev1.KustomizationKind, + }, + Status: kustomizev1.KustomizationStatus{ + LastAttemptedRevision: "main@sha1:5bf3a8f9bb0aa5ae8afd6208f43757ab73fc033a", + }, + }, + expectedBool: true, + }, + { + name: "GitRepository revision", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + Status: sourcev1.GitRepositoryStatus{ + Artifact: &sourcev1.Artifact{ + Revision: "main@sha1:5bf3a8f9bb0aa5ae8afd6208f43757ab73fc033a", + }, + }, + }, + expectedBool: true, + }, + { + name: "GitRepository revision (wrong revision)", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + Status: sourcev1.GitRepositoryStatus{ + Artifact: &sourcev1.Artifact{ + Revision: "main@sha1:e7f3a8f9bb0aa5ae8afd6208f43757ab73fc043a", + }, + }, + }, + }, + { + name: "Kustomization revision (empty revision)", + obj: &kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: kustomizev1.KustomizationKind, + }, + Status: kustomizev1.KustomizationStatus{ + LastAttemptedRevision: "", + }, + }, + }, + { + name: "OCIRepository revision", + obj: &sourcev1b2.OCIRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1b2.OCIRepositoryKind, + }, + Status: sourcev1b2.OCIRepositoryStatus{ + Artifact: &sourcev1.Artifact{ + Revision: "main@sha1:5bf3a8f9bb0aa5ae8afd6208f43757ab73fc033a", + }, + }, + }, + expectedBool: true, + }, + { + name: "Alert revision(Not supported)", + obj: ¬ificationv1.Alert{ + TypeMeta: metav1.TypeMeta{ + Kind: notificationv1.AlertKind, + }, + Status: notificationv1.AlertStatus{ + ObservedGeneration: 1, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tt.obj) + g.Expect(err).To(BeNil()) + got, err := hasRevision(tt.obj.GetObjectKind().GroupVersionKind().Kind, obj, revision) + if tt.expectErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(got).To(Equal(tt.expectedBool)) + }) + } +} + +func Test_objectReconciled(t *testing.T) { + expectedRev := "main@sha1:5bf3a8f9bb0aa5ae8afd6208f43757ab73fc033a" + + type updateStatus struct { + statusFn func(o client.Object) + expectedErr bool + expectedBool bool + } + tests := []struct { + name string + obj objectWithConditions + statuses []updateStatus + }{ + { + name: "GitRepository with no status", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + }, + }, + statuses: []updateStatus{ + { + expectedErr: false, + expectedBool: false, + }, + }, + }, + { + name: "suspended Kustomization", + obj: &kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: kustomizev1.KustomizationKind, + APIVersion: kustomizev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + }, + Spec: kustomizev1.KustomizationSpec{ + Suspend: true, + }, + }, + statuses: []updateStatus{ + { + expectedErr: true, + }, + }, + }, + { + name: "Kustomization - status with old generation", + obj: &kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: kustomizev1.KustomizationKind, + APIVersion: kustomizev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + Status: kustomizev1.KustomizationStatus{ + ObservedGeneration: -1, + }, + }, + statuses: []updateStatus{ + { + expectedErr: false, + expectedBool: false, + }, + }, + }, + { + name: "GitRepository - status with same generation but no conditions", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + Status: sourcev1.GitRepositoryStatus{ + ObservedGeneration: 1, + }, + }, + statuses: []updateStatus{ + { + expectedErr: false, + expectedBool: false, + }, + }, + }, + { + name: "GitRepository - status with conditions but no ready condition", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + Status: sourcev1.GitRepositoryStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReconcilingCondition, Status: metav1.ConditionTrue, ObservedGeneration: 1, Reason: "Progressing", Message: "Progressing"}, + }, + }, + }, + statuses: []updateStatus{ + { + expectedErr: false, + expectedBool: false, + }, + }, + }, + { + name: "Kustomization - status with false ready condition", + obj: &kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: kustomizev1.KustomizationKind, + APIVersion: kustomizev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + Status: kustomizev1.KustomizationStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReadyCondition, Status: metav1.ConditionFalse, ObservedGeneration: 1, Reason: "Failing", Message: "Failed to clone"}, + }, + }, + }, + statuses: []updateStatus{ + { + expectedErr: true, + expectedBool: false, + }, + }, + }, + { + name: "Kustomization - status with true ready condition but different revision", + obj: &kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: kustomizev1.KustomizationKind, + APIVersion: kustomizev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + Status: kustomizev1.KustomizationStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: 1, Reason: "Passing", Message: "Applied revision"}, + }, + LastAttemptedRevision: "main@sha1:e7f3a8f9bb0aa5ae8afd6208f43757ab73fc043a", + }, + }, + statuses: []updateStatus{ + { + expectedErr: false, + expectedBool: false, + }, + }, + }, + { + name: "GitRepository - status with true ready condition but different revision", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + Status: sourcev1.GitRepositoryStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: 1, Reason: "Readyyy", Message: "Cloned successfully"}, + }, + Artifact: &sourcev1.Artifact{ + Revision: "main@sha1:e7f3a8f9bb0aa5ae8afd6208f43757ab73fc043a", + }, + }, + }, + statuses: []updateStatus{ + { + expectedErr: false, + expectedBool: false, + }, + }, + }, + { + name: "GitRepository - ready with right revision", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + Status: sourcev1.GitRepositoryStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: 1, Reason: "Readyyy", Message: "Cloned successfully"}, + }, + Artifact: &sourcev1.Artifact{ + Revision: expectedRev, + }, + }, + }, + statuses: []updateStatus{ + { + expectedErr: false, + expectedBool: true, + }, + }, + }, + { + name: "GitRepository - sequence of status updates before ready", + obj: &sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: sourcev1.GitRepositoryKind, + APIVersion: sourcev1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "flux-system", + Namespace: "flux-system", + Generation: 1, + }, + }, + statuses: []updateStatus{ + { + // observed gen different + statusFn: func(o client.Object) { + gitRepo := o.(*sourcev1.GitRepository) + gitRepo.Status = sourcev1.GitRepositoryStatus{ + ObservedGeneration: -1, + } + }, + }, + { + // ready failing + statusFn: func(o client.Object) { + gitRepo := o.(*sourcev1.GitRepository) + gitRepo.Status = sourcev1.GitRepositoryStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReadyCondition, Status: metav1.ConditionFalse, ObservedGeneration: 1, Reason: "Not Ready", Message: "Transient connection issue"}, + }, + } + }, + expectedErr: true, + }, + { + // updated to a different revision + statusFn: func(o client.Object) { + gitRepo := o.(*sourcev1.GitRepository) + gitRepo.Status = sourcev1.GitRepositoryStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: 1, Reason: "Readyyy", Message: "Cloned successfully"}, + }, + Artifact: &sourcev1.Artifact{ + Revision: "wrong rev", + }, + } + }, + }, + { + // updated to the expected revision + statusFn: func(o client.Object) { + gitRepo := o.(*sourcev1.GitRepository) + gitRepo.Status = sourcev1.GitRepositoryStatus{ + ObservedGeneration: 1, + Conditions: []metav1.Condition{ + {Type: meta.ReadyCondition, Status: metav1.ConditionTrue, ObservedGeneration: 1, Reason: "Readyyy", Message: "Cloned successfully"}, + }, + Artifact: &sourcev1.Artifact{ + Revision: expectedRev, + }, + } + }, + expectedBool: true, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + builder := fake.NewClientBuilder().WithScheme(utils.NewScheme()) + builder.WithObjects(tt.obj) + + kubeClient := builder.Build() + + for _, updates := range tt.statuses { + if updates.statusFn != nil { + updates.statusFn(tt.obj) + g.Expect(kubeClient.Update(context.TODO(), tt.obj)).To(Succeed()) + } + + waitFunc := objectReconciled(kubeClient, client.ObjectKeyFromObject(tt.obj), tt.obj, expectedRev) + got, err := waitFunc(context.TODO()) + g.Expect(err != nil).To(Equal(updates.expectedErr)) + g.Expect(got).To(Equal(updates.expectedBool)) + } + }) + } +} diff --git a/pkg/status/status.go b/pkg/status/status.go index 8b61881c47..baef9d7f92 100644 --- a/pkg/status/status.go +++ b/pkg/status/status.go @@ -94,7 +94,7 @@ func (sc *StatusChecker) Assess(identifiers ...object.ObjMetadata) error { } if coll.Error != nil || ctx.Err() == context.DeadlineExceeded { - return fmt.Errorf("timed out waiting for condition") + return fmt.Errorf("timed out waiting for all resources to be ready") } return nil }