From 72f69e429add2a1a9397ebc65419af77a847565f Mon Sep 17 00:00:00 2001 From: Artem Miroshnychenko Date: Thu, 8 Feb 2024 02:31:38 +0100 Subject: [PATCH] feat: additional events when reconciliation starts Signed-off-by: Artem Miroshnychenko --- api/v2beta2/condition_types.go | 20 +++ internal/reconcile/install.go | 14 ++ internal/reconcile/install_test.go | 102 +++++++++++++- internal/reconcile/rollback_remediation.go | 19 ++- .../reconcile/rollback_remediation_test.go | 72 +++++++++- internal/reconcile/test.go | 14 ++ internal/reconcile/test_test.go | 72 +++++++++- internal/reconcile/uninstall.go | 14 ++ internal/reconcile/uninstall_remediation.go | 16 +++ .../reconcile/uninstall_remediation_test.go | 88 +++++++++++- internal/reconcile/uninstall_test.go | 133 +++++++++++++++++- internal/reconcile/unlock.go | 15 ++ internal/reconcile/unlock_test.go | 46 +++++- internal/reconcile/upgrade.go | 14 ++ internal/reconcile/upgrade_test.go | 117 ++++++++++++++- 15 files changed, 740 insertions(+), 16 deletions(-) diff --git a/api/v2beta2/condition_types.go b/api/v2beta2/condition_types.go index 10172dfb1..c6ba5c23a 100644 --- a/api/v2beta2/condition_types.go +++ b/api/v2beta2/condition_types.go @@ -32,6 +32,10 @@ const ( ) const ( + // InstallStartedReason represents the fact that the Helm install for the + // HelmRelease started. + InstallStartedReason string = "InstallStarted" + // InstallSucceededReason represents the fact that the Helm install for the // HelmRelease succeeded. InstallSucceededReason string = "InstallSucceeded" @@ -40,6 +44,10 @@ const ( // HelmRelease failed. InstallFailedReason string = "InstallFailed" + // UpgradeStartedReason represents the fact that the Helm upgrade for the + // HelmRelease started. + UpgradeStartedReason string = "UpgradeStarted" + // UpgradeSucceededReason represents the fact that the Helm upgrade for the // HelmRelease succeeded. UpgradeSucceededReason string = "UpgradeSucceeded" @@ -48,6 +56,10 @@ const ( // HelmRelease failed. UpgradeFailedReason string = "UpgradeFailed" + // TestStartedReason represents the fact that the Helm tests for the + // HelmRelease started. + TestStartedReason string = "TestStarted" + // TestSucceededReason represents the fact that the Helm tests for the // HelmRelease succeeded. TestSucceededReason string = "TestSucceeded" @@ -56,6 +68,10 @@ const ( // failed. TestFailedReason string = "TestFailed" + // RollbackStartedReason represents the fact that the Helm rollback for the + // HelmRelease started. + RollbackStartedReason string = "RollbackStarted" + // RollbackSucceededReason represents the fact that the Helm rollback for the // HelmRelease succeeded. RollbackSucceededReason string = "RollbackSucceeded" @@ -64,6 +80,10 @@ const ( // HelmRelease failed. RollbackFailedReason string = "RollbackFailed" + // UninstallStartedReason represents the fact that the Helm uninstall for the + // HelmRelease started. + UninstallStartedReason string = "UninstallStarted" + // UninstallSucceededReason represents the fact that the Helm uninstall for the // HelmRelease succeeded. UninstallSucceededReason string = "UninstallSucceeded" diff --git a/internal/reconcile/install.go b/internal/reconcile/install.go index 0567b549d..5236188d7 100644 --- a/internal/reconcile/install.go +++ b/internal/reconcile/install.go @@ -41,6 +41,7 @@ import ( // cleared to mark the start of a new release lifecycle. This ensures we never // attempt to roll back to a previous release before the install. // +// When the installation starts, the object emits a corresponding event. // During the installation process, the writes to the Helm storage are // observed and recorded in the Status.History field of the Request.Object. // @@ -88,6 +89,17 @@ func (r *Install) Reconcile(ctx context.Context, req *Request) error { conditions.Delete(req.Object, v2.TestSuccessCondition) conditions.Delete(req.Object, v2.RemediatedCondition) + // Compose started message. + msg := fmt.Sprintf(fmtInstallStarted, req.Object.GetReleaseName(), req.Chart.Name()) + // Record event. + r.eventRecorder.AnnotatedEventf( + req.Object, + eventMeta(req.Chart.Metadata.Version, chartutil.DigestValues(digest.Canonical, req.Values).String()), + corev1.EventTypeNormal, + v2.InstallStartedReason, + msg, + ) + // Run the Helm install action. _, err := action.Install(ctx, cfg, req.Object, req.Chart, req.Values) @@ -126,6 +138,8 @@ func (r *Install) Type() ReconcilerType { } const ( + // fmtInstallStarted is the message format for an installation start. + fmtInstallStarted = "Helm install started for release %s with chart %s" // fmtInstallFailure is the message format for an installation failure. fmtInstallFailure = "Helm install failed for release %s/%s with chart %s@%s: %s" // fmtInstallSuccess is the message format for a successful installation. diff --git a/internal/reconcile/install_test.go b/internal/reconcile/install_test.go index d156e45ca..071b114b2 100644 --- a/internal/reconcile/install_test.go +++ b/internal/reconcile/install_test.go @@ -32,7 +32,6 @@ import ( helmdriver "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/apis/meta" @@ -68,6 +67,9 @@ func TestInstall_Reconcile(t *testing.T) { // expectedConditions are the conditions that are expected to be set on // the HelmRelease after install. expectConditions []metav1.Condition + // expectEvents is the expected Events of the HelmRelease + // after install. + expectEvents func(chart *chart.Chart) []corev1.Event // expectHistory is the expected History of the HelmRelease after // install. expectHistory func(releases []*helmrelease.Release) v2.Snapshots @@ -94,6 +96,21 @@ func TestInstall_Reconcile(t *testing.T) { release.ObservedToSnapshot(release.ObserveRelease(releases[0])), } }, + expectEvents: func(chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.InstallStartedReason, + Message: fmt.Sprintf(fmtInstallStarted, mockReleaseName, chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, }, { name: "install failure", @@ -109,6 +126,21 @@ func TestInstall_Reconcile(t *testing.T) { release.ObservedToSnapshot(release.ObserveRelease(releases[0])), } }, + expectEvents: func(chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.InstallStartedReason, + Message: fmt.Sprintf(fmtInstallStarted, mockReleaseName, chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectFailures: 1, expectInstallFailures: 1, }, @@ -128,6 +160,21 @@ func TestInstall_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.InstallFailedReason, "storage create error"), }, + expectEvents: func(chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.InstallStartedReason, + Message: fmt.Sprintf(fmtInstallStarted, mockReleaseName, chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectFailures: 1, expectInstallFailures: 0, }, @@ -163,6 +210,21 @@ func TestInstall_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.ReleasedCondition, v2.InstallSucceededReason, "Helm install succeeded"), }, + expectEvents: func(chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.InstallStartedReason, + Message: fmt.Sprintf(fmtInstallStarted, mockReleaseName, chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[1])), @@ -191,6 +253,21 @@ func TestInstall_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.ReleasedCondition, v2.InstallSucceededReason, "Helm install succeeded"), }, + expectEvents: func(chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.InstallStartedReason, + Message: fmt.Sprintf(fmtInstallStarted, mockReleaseName, chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -214,6 +291,21 @@ func TestInstall_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.ReleasedCondition, v2.InstallSucceededReason, "Helm install succeeded"), }, + expectEvents: func(chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.InstallStartedReason, + Message: fmt.Sprintf(fmtInstallStarted, mockReleaseName, chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -270,7 +362,7 @@ func TestInstall_Reconcile(t *testing.T) { cfg.Driver = tt.driver(cfg.Driver) } - recorder := new(record.FakeRecorder) + recorder := testutil.NewFakeRecorder(10, false) got := (NewInstall(cfg, recorder)).Reconcile(context.TODO(), &Request{ Object: obj, Chart: tt.chart, @@ -282,6 +374,12 @@ func TestInstall_Reconcile(t *testing.T) { g.Expect(got).ToNot(HaveOccurred()) } + if tt.expectEvents != nil { + for _, event := range tt.expectEvents(tt.chart) { + g.Expect(recorder.GetEvents()).To(ContainElement(event)) + } + } + g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions)) releases, _ = store.History(mockReleaseName) diff --git a/internal/reconcile/rollback_remediation.go b/internal/reconcile/rollback_remediation.go index e614f5a30..c51f019fb 100644 --- a/internal/reconcile/rollback_remediation.go +++ b/internal/reconcile/rollback_remediation.go @@ -44,6 +44,7 @@ import ( // The writes to the Helm storage during the rollback are observed, and update // the Status.History field. // +// When rollback starts, the object emits a corresponding event. // After a successful rollback, the object is marked with Remediated=True and // an event is emitted. When the rollback fails, the object is marked with // Remediated=False and a warning event is emitted. @@ -94,8 +95,21 @@ func (r *RollbackRemediation) Reconcile(ctx context.Context, req *Request) error ErrReleaseMismatch, prev.FullReleaseName(), cur.FullReleaseName()) } + // Compose started message. + msg := fmt.Sprintf(fmtRollbackRemediationStarted, prev.FullReleaseName(), prev.VersionedChartName()) + // Record event. + r.eventRecorder.AnnotatedEventf( + req.Object, + eventMeta(cur.ChartVersion, chartutil.DigestValues(digest.Canonical, req.Values).String()), + corev1.EventTypeNormal, + v2.RollbackStartedReason, + msg, + ) + // Run the Helm rollback action. - if err := action.Rollback(cfg, req.Object, prev.Name, action.RollbackToVersion(prev.Version)); err != nil { + err := action.Rollback(cfg, req.Object, prev.Name, action.RollbackToVersion(prev.Version)) + + if err != nil { r.failure(req, prev, logBuf, err) // Return error if we did not store a release, as this does not @@ -120,6 +134,9 @@ func (r *RollbackRemediation) Type() ReconcilerType { } const ( + // fmtRollbackRemediationStarted is the message format for a rollback + // remediation start. + fmtRollbackRemediationStarted = "Helm rollback to previous release %s with chart %s started" // fmtRollbackRemediationFailure is the message format for a rollback // remediation failure. fmtRollbackRemediationFailure = "Helm rollback to previous release %s with chart %s failed: %s" diff --git a/internal/reconcile/rollback_remediation_test.go b/internal/reconcile/rollback_remediation_test.go index 2300aefc3..6c2aa35a7 100644 --- a/internal/reconcile/rollback_remediation_test.go +++ b/internal/reconcile/rollback_remediation_test.go @@ -31,7 +31,6 @@ import ( helmdriver "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/apis/meta" @@ -68,6 +67,9 @@ func TestRollbackRemediation_Reconcile(t *testing.T) { // expectedConditions are the conditions that are expected to be set on // the HelmRelease after rolling back. expectConditions []metav1.Condition + // expectEvents is the expected Events of the HelmRelease + // after rolling back. + expectEvents func(prev *v2.Snapshot, release *v2.HelmRelease) []corev1.Event // expectHistory is the expected History on the HelmRelease after // rolling back. expectHistory func(releases []*helmrelease.Release) v2.Snapshots @@ -112,6 +114,21 @@ func TestRollbackRemediation_Reconcile(t *testing.T) { *conditions.FalseCondition(meta.ReadyCondition, v2.RollbackSucceededReason, "succeeded"), *conditions.TrueCondition(v2.RemediatedCondition, v2.RollbackSucceededReason, "succeeded"), }, + expectEvents: func(prev *v2.Snapshot, release *v2.HelmRelease) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.RollbackStartedReason, + Message: fmt.Sprintf(fmtRollbackRemediationStarted, prev.FullReleaseName(), prev.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): prev.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, release.GetValues()).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[2])), @@ -188,6 +205,21 @@ func TestRollbackRemediation_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.RemediatedCondition, v2.RollbackFailedReason, "timed out waiting for the condition"), }, + expectEvents: func(prev *v2.Snapshot, release *v2.HelmRelease) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.RollbackStartedReason, + Message: fmt.Sprintf(fmtRollbackRemediationStarted, prev.FullReleaseName(), prev.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): prev.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, release.GetValues()).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[2])), @@ -238,6 +270,21 @@ func TestRollbackRemediation_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.RemediatedCondition, v2.RollbackFailedReason, mockCreateErr.Error()), }, + expectEvents: func(prev *v2.Snapshot, release *v2.HelmRelease) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.RollbackStartedReason, + Message: fmt.Sprintf(fmtRollbackRemediationStarted, prev.FullReleaseName(), prev.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): prev.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, release.GetValues()).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[1])), @@ -286,6 +333,21 @@ func TestRollbackRemediation_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.RemediatedCondition, v2.RollbackFailedReason, "storage update error"), }, + expectEvents: func(prev *v2.Snapshot, release *v2.HelmRelease) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.RollbackStartedReason, + Message: fmt.Sprintf(fmtRollbackRemediationStarted, prev.FullReleaseName(), prev.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): prev.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, release.GetValues()).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[2])), @@ -342,7 +404,7 @@ func TestRollbackRemediation_Reconcile(t *testing.T) { cfg.Driver = tt.driver(cfg.Driver) } - recorder := new(record.FakeRecorder) + recorder := testutil.NewFakeRecorder(10, true) got := (NewRollbackRemediation(cfg, recorder)).Reconcile(context.TODO(), &Request{ Object: obj, }) @@ -351,6 +413,12 @@ func TestRollbackRemediation_Reconcile(t *testing.T) { } else { g.Expect(got).ToNot(HaveOccurred()) } + if tt.expectEvents != nil { + prev := obj.Status.History.Previous(obj.GetUpgrade().GetRemediation().MustIgnoreTestFailures(obj.GetTest().IgnoreFailures)) + for _, event := range tt.expectEvents(prev, obj) { + g.Expect(recorder.GetEvents()).To(ContainElement(event)) + } + } g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions)) diff --git a/internal/reconcile/test.go b/internal/reconcile/test.go index f40a3d4cf..48aae7f83 100644 --- a/internal/reconcile/test.go +++ b/internal/reconcile/test.go @@ -42,6 +42,7 @@ import ( // TestHooks field of the latest Snapshot in the Status.History to be updated // if it matches the target of the test. // +// When the test starts, the object emits a corresponding event. // When all test hooks for the release succeed, the object is marked with // TestSuccess=True and an event is emitted. When one of the test hooks fails, // Helm stops running the remaining tests, and the object is marked with @@ -83,6 +84,17 @@ func (r *Test) Reconcile(ctx context.Context, req *Request) error { return fmt.Errorf("%w: required for test", ErrNoLatest) } + // Compose started message. + msg := fmt.Sprintf(fmtTestStarted, cur.FullReleaseName(), cur.VersionedChartName()) + // Record event. + r.eventRecorder.AnnotatedEventf( + req.Object, + eventMeta(cur.ChartVersion, cur.ConfigDigest), + corev1.EventTypeNormal, + v2.TestStartedReason, + msg, + ) + // Run the Helm test action. rls, err := action.Test(ctx, cfg, req.Object) @@ -119,6 +131,8 @@ func (r *Test) Type() ReconcilerType { } const ( + // fmtTestStarted is the message format used when tests are started. + fmtTestStarted = "Helm test %s with chart %s is started" // fmtTestPending is the message format used when awaiting tests to be run. fmtTestPending = "Helm release %s with chart %s is awaiting tests" // fmtTestFailure is the message format for a test failure. diff --git a/internal/reconcile/test_test.go b/internal/reconcile/test_test.go index d97dbe0c9..1bb01f1fe 100644 --- a/internal/reconcile/test_test.go +++ b/internal/reconcile/test_test.go @@ -30,7 +30,6 @@ import ( helmdriver "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/apis/meta" @@ -96,6 +95,9 @@ func TestTest_Reconcile(t *testing.T) { // expectedConditions are the conditions that are expected to be set on // the HelmRelease after running test. expectConditions []metav1.Condition + // expectEvents is the expected Events of the HelmRelease + // after running tests. + expectEvents func(cur *v2.Snapshot) []corev1.Event // expectHistory is the expected History on the HelmRelease after // running test. expectHistory func(releases []*helmrelease.Release) v2.Snapshots @@ -134,6 +136,21 @@ func TestTest_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.TestSuccessCondition, v2.TestSucceededReason, "1 test hook completed successfully"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.TestStartedReason, + Message: fmt.Sprintf(fmtTestStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { withTests := release.ObservedToSnapshot(release.ObserveRelease(releases[0])) withTests.SetTestHooks(release.TestHooksFromRelease(releases[0])) @@ -166,6 +183,21 @@ func TestTest_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.TestSuccessCondition, v2.TestSucceededReason, "no test hooks"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.TestStartedReason, + Message: fmt.Sprintf(fmtTestStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { withTests := release.ObservedToSnapshot(release.ObserveRelease(releases[0])) withTests.SetTestHooks(release.TestHooksFromRelease(releases[0])) @@ -200,6 +232,21 @@ func TestTest_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.TestSuccessCondition, v2.TestFailedReason, "timed out waiting for the condition"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.TestStartedReason, + Message: fmt.Sprintf(fmtTestStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { withTests := release.ObservedToSnapshot(release.ObserveRelease(releases[0])) withTests.SetTestHooks(release.TestHooksFromRelease(releases[0])) @@ -257,6 +304,21 @@ func TestTest_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.TestSuccessCondition, v2.TestFailedReason, ErrReleaseMismatch.Error()), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.TestStartedReason, + Message: fmt.Sprintf(fmtTestStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -318,7 +380,7 @@ func TestTest_Reconcile(t *testing.T) { cfg.Driver = tt.driver(cfg.Driver) } - recorder := new(record.FakeRecorder) + recorder := testutil.NewFakeRecorder(10, true) got := (NewTest(cfg, recorder)).Reconcile(context.TODO(), &Request{ Object: obj, }) @@ -328,6 +390,12 @@ func TestTest_Reconcile(t *testing.T) { g.Expect(got).ToNot(HaveOccurred()) } + if tt.expectEvents != nil { + cur := obj.Status.History.Latest().DeepCopy() + for _, event := range tt.expectEvents(cur) { + g.Expect(recorder.GetEvents()).To(ContainElement(event)) + } + } g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions)) releases, _ = store.History(mockReleaseName) diff --git a/internal/reconcile/uninstall.go b/internal/reconcile/uninstall.go index 8c8158745..256eb14dd 100644 --- a/internal/reconcile/uninstall.go +++ b/internal/reconcile/uninstall.go @@ -43,6 +43,7 @@ import ( // The writes to the Helm storage during the uninstallation are observed, and // update the Status.History field. // +// When uninstall starts, the object emits a corresponding event. // After a successful uninstall, the object is marked with Released=False and // an event is emitted. When the uninstallation fails, the object is marked // with Released=False and a warning event is emitted. @@ -92,6 +93,17 @@ func (r *Uninstall) Reconcile(ctx context.Context, req *Request) error { return fmt.Errorf("%w: required to uninstall", ErrNoLatest) } + // Compose started message + msg := fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()) + // Record event + r.eventRecorder.AnnotatedEventf( + req.Object, + eventMeta(cur.ChartVersion, cur.ConfigDigest), + corev1.EventTypeNormal, + v2.UninstallStartedReason, + msg, + ) + // Run the Helm uninstall action. res, err := action.Uninstall(ctx, cfg, req.Object, cur.Name) @@ -158,6 +170,8 @@ func (r *Uninstall) Type() ReconcilerType { } const ( + // fmtUninstallStarted is the message format for an uninstall start. + fmtUninstallStarted = "Helm uninstall started for release %s with chart %s" // fmtUninstallFailed is the message format for an uninstall failure. fmtUninstallFailure = "Helm uninstall failed for release %s with chart %s: %s" // fmtUninstallSuccess is the message format for a successful uninstall. diff --git a/internal/reconcile/uninstall_remediation.go b/internal/reconcile/uninstall_remediation.go index 4e244cdc0..d6df627ec 100644 --- a/internal/reconcile/uninstall_remediation.go +++ b/internal/reconcile/uninstall_remediation.go @@ -44,6 +44,8 @@ var ( // The writes to the Helm storage during the rollback are observed, and update // the Status.History field. // +// When uninstall starts, the object emits a corresponding event. +// // After a successful uninstall, the object is marked with Remediated=True and // an event is emitted. When the uninstallation fails, the object is marked // with Remediated=False and a warning event is emitted. @@ -90,6 +92,17 @@ func (r *UninstallRemediation) Reconcile(ctx context.Context, req *Request) erro return fmt.Errorf("%w: required to uninstall", ErrNoLatest) } + // Compose started message. + msg := fmt.Sprintf(fmtUninstallRemediationStarted, cur.FullReleaseName(), cur.VersionedChartName()) + // Record event. + r.eventRecorder.AnnotatedEventf( + req.Object, + eventMeta(cur.ChartVersion, cur.ConfigDigest), + corev1.EventTypeNormal, + v2.UninstallStartedReason, + msg, + ) + // Run the Helm uninstall action. res, err := action.Uninstall(ctx, cfg, req.Object, cur.Name) @@ -130,6 +143,9 @@ func (r *UninstallRemediation) Type() ReconcilerType { } const ( + // fmtUninstallRemediationStarted is the message format for an uninstall + // remediation start. + fmtUninstallRemediationStarted = "Helm uninstall remediation for release %s with chart %s started" // fmtUninstallRemediationFailure is the message format for an uninstall // remediation failure. fmtUninstallRemediationFailure = "Helm uninstall remediation for release %s with chart %s failed: %s" diff --git a/internal/reconcile/uninstall_remediation_test.go b/internal/reconcile/uninstall_remediation_test.go index f6abe2745..32a25af92 100644 --- a/internal/reconcile/uninstall_remediation_test.go +++ b/internal/reconcile/uninstall_remediation_test.go @@ -30,7 +30,6 @@ import ( helmdriver "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/runtime/conditions" @@ -66,6 +65,9 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { // expectedConditions are the conditions that are expected to be set on // the HelmRelease after running rollback. expectConditions []metav1.Condition + // expectEvents is the expected Events of the HelmRelease + // after running rollback. + expectEvents func(cur *v2.Snapshot) []corev1.Event // expectHistory is the expected History of the HelmRelease after // uninstall. expectHistory func(releases []*helmrelease.Release) v2.Snapshots @@ -107,6 +109,21 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.RemediatedCondition, v2.UninstallSucceededReason, "succeeded"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallRemediationStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -142,6 +159,21 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.RemediatedCondition, v2.UninstallFailedReason, "uninstallation completed with 1 error(s): 1 error occurred:\n\t* timed out waiting for the condition"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallRemediationStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -189,6 +221,21 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.RemediatedCondition, v2.UninstallFailedReason, ErrNoStorageUpdate.Error()), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallRemediationStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -231,6 +278,21 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { expectConditions: []metav1.Condition{ *conditions.FalseCondition(v2.RemediatedCondition, v2.UninstallFailedReason, mockDeleteErr.Error()), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallRemediationStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -290,6 +352,21 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.RemediatedCondition, v2.UninstallFailedReason, ErrReleaseMismatch.Error()), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallRemediationStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -348,7 +425,7 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { cfg.Driver = tt.driver(cfg.Driver) } - recorder := new(record.FakeRecorder) + recorder := testutil.NewFakeRecorder(10, true) got := NewUninstallRemediation(cfg, recorder).Reconcile(context.TODO(), &Request{ Object: obj, }) @@ -358,6 +435,13 @@ func TestUninstallRemediation_Reconcile(t *testing.T) { g.Expect(got).ToNot(HaveOccurred()) } + if tt.expectEvents != nil { + cur := obj.Status.History.Latest().DeepCopy() + for _, event := range tt.expectEvents(cur) { + g.Expect(recorder.GetEvents()).To(ContainElement(event)) + } + } + g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions)) releases, _ = store.History(mockReleaseName) diff --git a/internal/reconcile/uninstall_test.go b/internal/reconcile/uninstall_test.go index ec0a9e23a..cd581f9c6 100644 --- a/internal/reconcile/uninstall_test.go +++ b/internal/reconcile/uninstall_test.go @@ -30,7 +30,6 @@ import ( helmdriver "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/apis/meta" @@ -64,6 +63,9 @@ func TestUninstall_Reconcile(t *testing.T) { // expectedConditions are the conditions that are expected to be set on // the HelmRelease after running rollback. expectConditions []metav1.Condition + // expectEvents is the expected Events of the HelmRelease + // after running rollback. + expectEvents func(cur *v2.Snapshot) []corev1.Event // expectHistory is the expected History of the HelmRelease after // uninstall. expectHistory func(namespace string, releases []*helmrelease.Release) v2.Snapshots @@ -107,6 +109,21 @@ func TestUninstall_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UninstallSucceededReason, "succeeded"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(namespace string, releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -144,6 +161,21 @@ func TestUninstall_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UninstallFailedReason, "uninstallation completed with 1 error(s): 1 error occurred:\n\t* timed out waiting for the condition"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(namespace string, releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -198,6 +230,21 @@ func TestUninstall_Reconcile(t *testing.T) { release.ObservedToSnapshot(release.ObserveRelease(releases[0])), } }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectFailures: 1, wantErr: ErrNoStorageUpdate, }, @@ -243,6 +290,21 @@ func TestUninstall_Reconcile(t *testing.T) { release.ObservedToSnapshot(release.ObserveRelease(releases[0])), } }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectFailures: 1, }, { @@ -299,6 +361,21 @@ func TestUninstall_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UninstallFailedReason, ErrReleaseMismatch.Error()), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(namespace string, releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -338,6 +415,21 @@ func TestUninstall_Reconcile(t *testing.T) { }, } }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectConditions: []metav1.Condition{ *conditions.FalseCondition(meta.ReadyCondition, v2.UninstallSucceededReason, "assuming it is uninstalled"), @@ -376,6 +468,21 @@ func TestUninstall_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UninstallSucceededReason, "succeeded"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(namespace string, releases []*helmrelease.Release) v2.Snapshots { rls := testutil.BuildRelease(&helmrelease.MockReleaseOptions{ Name: mockReleaseName, @@ -420,6 +527,21 @@ func TestUninstall_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UninstallSucceededReason, "was already uninstalled"), }, + expectEvents: func(cur *v2.Snapshot) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UninstallStartedReason, + Message: fmt.Sprintf(fmtUninstallStarted, cur.FullReleaseName(), cur.VersionedChartName()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(namespace string, releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -476,7 +598,7 @@ func TestUninstall_Reconcile(t *testing.T) { cfg.Driver = tt.driver(cfg.Driver) } - recorder := new(record.FakeRecorder) + recorder := testutil.NewFakeRecorder(10, true) got := NewUninstall(cfg, recorder).Reconcile(context.TODO(), &Request{ Object: obj, }) @@ -486,6 +608,13 @@ func TestUninstall_Reconcile(t *testing.T) { g.Expect(got).ToNot(HaveOccurred()) } + if tt.expectEvents != nil { + cur := obj.Status.History.Latest().DeepCopy() + for _, event := range tt.expectEvents(cur) { + g.Expect(recorder.GetEvents()).To(ContainElement(event)) + } + } + g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions)) releases, _ = store.History(mockReleaseName) diff --git a/internal/reconcile/unlock.go b/internal/reconcile/unlock.go index 7d045856c..aab9a7b82 100644 --- a/internal/reconcile/unlock.go +++ b/internal/reconcile/unlock.go @@ -38,6 +38,8 @@ import ( // for a Request.Object in the Helm storage if stuck in a pending state, by // setting the status to release.StatusFailed and persisting it. // +// When unlock starts, the object emits a corresponding event. +// // This write to the Helm storage is observed, and updates the Status.History // field if the persisted object targets the same release version. // @@ -79,6 +81,17 @@ func (r *Unlock) Reconcile(_ context.Context, req *Request) error { // Ensure the release is in a pending state. cur := release.ObservedToSnapshot(release.ObserveRelease(rls)) if status := rls.Info.Status; status.IsPending() { + // Compose started message. + msg := fmt.Sprintf(fmtUnlockStarted, cur.FullReleaseName(), cur.VersionedChartName(), status.String()) + // Record event. + r.eventRecorder.AnnotatedEventf( + req.Object, + eventMeta(cur.ChartVersion, cur.ConfigDigest), + corev1.EventTypeNormal, + "PendingRelease", + msg, + ) + // Update pending status to failed and persist. rls.SetStatus(helmrelease.StatusFailed, fmt.Sprintf("Release unlocked from stale '%s' state", status.String())) if err = cfg.Releases.Update(rls); err != nil { @@ -99,6 +112,8 @@ func (r *Unlock) Type() ReconcilerType { } const ( + // fmtUnlockStarted is the message format for an unlock start. + fmtUnlockStarted = "Unlock of Helm release %s with chart %s in %s state started" // fmtUnlockFailure is the message format for an unlock failure. fmtUnlockFailure = "Unlock of Helm release %s with chart %s in %s state failed: %s" // fmtUnlockSuccess is the message format for a successful unlock. diff --git a/internal/reconcile/unlock_test.go b/internal/reconcile/unlock_test.go index 6799fe198..33c31a352 100644 --- a/internal/reconcile/unlock_test.go +++ b/internal/reconcile/unlock_test.go @@ -30,7 +30,6 @@ import ( helmdriver "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/apis/meta" @@ -67,6 +66,9 @@ func TestUnlock_Reconcile(t *testing.T) { // expectedConditions are the conditions that are expected to be set on // the HelmRelease after running rollback. expectConditions []metav1.Condition + // expectEvents is the expected Events of the HelmRelease after + // running unlock. + expectEvents func(cur *v2.Snapshot, status *helmrelease.Status) []corev1.Event // expectHistory is the expected History of the HelmRelease after // unlock. expectHistory func(releases []*helmrelease.Release) v2.Snapshots @@ -103,6 +105,21 @@ func TestUnlock_Reconcile(t *testing.T) { *conditions.FalseCondition(meta.ReadyCondition, "PendingRelease", "Unlocked Helm release"), *conditions.FalseCondition(v2.ReleasedCondition, "PendingRelease", "Unlocked Helm release"), }, + expectEvents: func(cur *v2.Snapshot, status *helmrelease.Status) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: "PendingRelease", + Message: fmt.Sprintf(fmtUnlockStarted, cur.FullReleaseName(), cur.VersionedChartName(), status), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -140,6 +157,21 @@ func TestUnlock_Reconcile(t *testing.T) { *conditions.FalseCondition(meta.ReadyCondition, "PendingRelease", "in pending-rollback state failed: storage update error"), *conditions.FalseCondition(v2.ReleasedCondition, "PendingRelease", "in pending-rollback state failed: storage update error"), }, + expectEvents: func(cur *v2.Snapshot, status *helmrelease.Status) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: "PendingRelease", + Message: fmt.Sprintf(fmtUnlockStarted, cur.FullReleaseName(), cur.VersionedChartName(), status), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): cur.ChartVersion, + eventMetaGroupKey(eventv1.MetaTokenKey): cur.ConfigDigest, + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -336,7 +368,11 @@ func TestUnlock_Reconcile(t *testing.T) { cfg.Driver = tt.driver(cfg.Driver) } - recorder := new(record.FakeRecorder) + recorder := testutil.NewFakeRecorder(10, true) + // Retrieve last release object. + rls, _ := action.LastRelease(cfg.Build(nil, observeUnlock(obj)), obj.GetReleaseName()) + cur := release.ObservedToSnapshot(release.ObserveRelease(rls)) + got := NewUnlock(cfg, recorder).Reconcile(context.TODO(), &Request{ Object: obj, }) @@ -345,7 +381,11 @@ func TestUnlock_Reconcile(t *testing.T) { } else { g.Expect(got).ToNot(HaveOccurred()) } - + if tt.expectEvents != nil { + for _, event := range tt.expectEvents(cur, &rls.Info.Status) { + g.Expect(recorder.GetEvents()).To(ContainElement(event)) + } + } g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions)) releases, _ = store.History(mockReleaseName) diff --git a/internal/reconcile/upgrade.go b/internal/reconcile/upgrade.go index 8cdbb0828..f45ad76c9 100644 --- a/internal/reconcile/upgrade.go +++ b/internal/reconcile/upgrade.go @@ -40,6 +40,7 @@ import ( // The writes to the Helm storage during the upgrade process are observed, // and update the Status.History field. // +// When upgrade starts, the object emits a corresponding event. // On upgrade success, the object is marked with Released=True and emits an // event. In addition, the object is marked with TestSuccess=False if tests // are enabled to indicate we are awaiting the results. @@ -79,6 +80,17 @@ func (r *Upgrade) Reconcile(ctx context.Context, req *Request) error { conditions.Delete(req.Object, v2.TestSuccessCondition) conditions.Delete(req.Object, v2.RemediatedCondition) + // Compose started message. + msg := fmt.Sprintf(fmtUpgradeStarted, req.Object.GetReleaseName(), req.Chart.Name()) + // Record event. + r.eventRecorder.AnnotatedEventf( + req.Object, + eventMeta(req.Chart.Metadata.Version, chartutil.DigestValues(digest.Canonical, req.Values).String()), + corev1.EventTypeNormal, + v2.UpgradeStartedReason, + msg, + ) + // Run the Helm upgrade action. _, err := action.Upgrade(ctx, cfg, req.Object, req.Chart, req.Values) @@ -117,6 +129,8 @@ func (r *Upgrade) Type() ReconcilerType { } const ( + // fmtUpgradeStarted is the message format for an upgrade started. + fmtUpgradeStarted = "Helm upgrade started for release %s with chart %s" // fmtUpgradeFailure is the message format for an upgrade failure. fmtUpgradeFailure = "Helm upgrade failed for release %s/%s with chart %s@%s: %s" // fmtUpgradeSuccess is the message format for a successful upgrade. diff --git a/internal/reconcile/upgrade_test.go b/internal/reconcile/upgrade_test.go index dbfb85b7c..cd44310d7 100644 --- a/internal/reconcile/upgrade_test.go +++ b/internal/reconcile/upgrade_test.go @@ -24,6 +24,7 @@ import ( "time" . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/chart" helmchart "helm.sh/helm/v3/pkg/chart" helmchartutil "helm.sh/helm/v3/pkg/chartutil" helmrelease "helm.sh/helm/v3/pkg/release" @@ -32,7 +33,6 @@ import ( helmdriver "helm.sh/helm/v3/pkg/storage/driver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" "github.com/fluxcd/pkg/apis/meta" @@ -73,6 +73,9 @@ func TestUpgrade_Reconcile(t *testing.T) { // expectedConditions are the conditions that are expected to be set on // the HelmRelease after upgrade. expectConditions []metav1.Condition + // expectEvents is the expected Events of the HelmRelease + // after upgrade. + expectEvents func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event // expectHistory returns the expected History of the HelmRelease after // upgrade. expectHistory func(releases []*helmrelease.Release) v2.Snapshots @@ -110,6 +113,21 @@ func TestUpgrade_Reconcile(t *testing.T) { *conditions.TrueCondition(meta.ReadyCondition, v2.UpgradeSucceededReason, "Helm upgrade succeeded"), *conditions.TrueCondition(v2.ReleasedCondition, v2.UpgradeSucceededReason, "Helm upgrade succeeded"), }, + expectEvents: func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UpgradeStartedReason, + Message: fmt.Sprintf(fmtUpgradeStarted, release.GetReleaseName(), chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[1])), @@ -144,6 +162,21 @@ func TestUpgrade_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UpgradeFailedReason, "post-upgrade hooks failed: 1 error occurred:\n\t* timed out waiting for the condition"), }, + expectEvents: func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UpgradeStartedReason, + Message: fmt.Sprintf(fmtUpgradeStarted, release.GetReleaseName(), chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[1])), @@ -186,6 +219,21 @@ func TestUpgrade_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UpgradeFailedReason, mockCreateErr.Error()), }, + expectEvents: func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UpgradeStartedReason, + Message: fmt.Sprintf(fmtUpgradeStarted, release.GetReleaseName(), chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[0])), @@ -228,6 +276,21 @@ func TestUpgrade_Reconcile(t *testing.T) { *conditions.FalseCondition(v2.ReleasedCondition, v2.UpgradeFailedReason, mockUpdateErr.Error()), }, + expectEvents: func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UpgradeStartedReason, + Message: fmt.Sprintf(fmtUpgradeStarted, release.GetReleaseName(), chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[1])), @@ -260,6 +323,21 @@ func TestUpgrade_Reconcile(t *testing.T) { *conditions.TrueCondition(meta.ReadyCondition, v2.UpgradeSucceededReason, "Helm upgrade succeeded"), *conditions.TrueCondition(v2.ReleasedCondition, v2.UpgradeSucceededReason, "Helm upgrade succeeded"), }, + expectEvents: func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UpgradeStartedReason, + Message: fmt.Sprintf(fmtUpgradeStarted, release.GetReleaseName(), chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[1])), @@ -305,6 +383,21 @@ func TestUpgrade_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.ReleasedCondition, v2.UpgradeSucceededReason, "Helm upgrade succeeded"), }, + expectEvents: func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UpgradeStartedReason, + Message: fmt.Sprintf(fmtUpgradeStarted, release.GetReleaseName(), chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[2])), @@ -345,6 +438,21 @@ func TestUpgrade_Reconcile(t *testing.T) { *conditions.TrueCondition(v2.ReleasedCondition, v2.UpgradeSucceededReason, "Helm upgrade succeeded"), }, + expectEvents: func(release *v2.HelmRelease, chart *chart.Chart) []corev1.Event { + return []corev1.Event{ + { + Type: corev1.EventTypeNormal, + Reason: v2.UpgradeStartedReason, + Message: fmt.Sprintf(fmtUpgradeStarted, release.GetReleaseName(), chart.Name()), + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + eventMetaGroupKey(eventv1.MetaRevisionKey): chart.Metadata.Version, + eventMetaGroupKey(eventv1.MetaTokenKey): chartutil.DigestValues(digest.Canonical, chart.Values).String(), + }, + }, + }, + } + }, expectHistory: func(releases []*helmrelease.Release) v2.Snapshots { return v2.Snapshots{ release.ObservedToSnapshot(release.ObserveRelease(releases[1])), @@ -401,7 +509,7 @@ func TestUpgrade_Reconcile(t *testing.T) { cfg.Driver = tt.driver(cfg.Driver) } - recorder := new(record.FakeRecorder) + recorder := testutil.NewFakeRecorder(10, true) got := NewUpgrade(cfg, recorder).Reconcile(context.TODO(), &Request{ Object: obj, Chart: tt.chart, @@ -412,6 +520,11 @@ func TestUpgrade_Reconcile(t *testing.T) { } else { g.Expect(got).ToNot(HaveOccurred()) } + if tt.expectEvents != nil { + for _, event := range tt.expectEvents(obj, tt.chart) { + g.Expect(recorder.GetEvents()).To(ContainElement(event)) + } + } g.Expect(obj.Status.Conditions).To(conditions.MatchConditions(tt.expectConditions))