From 66e1e38fa95e81a1ebbf85d2ca88c1333203b0e4 Mon Sep 17 00:00:00 2001 From: Oleksandr Saulyak Date: Sat, 5 Oct 2024 17:18:39 +0300 Subject: [PATCH 1/2] feat: revisions info reporting with multi-sourced apps support (#333) * event-reporter: added utils methods to retrieve revisions metadata for application * event-reporter: report all revisions metadata instead single to support multisourced apps * event-reporter: added revision sha to reported value in anotations - "app.meta.revisions-metadata" * event-reporter: added change revisions sha to reported value in anotations - app.meta.revisions-metadata * event-reporter: updated changelog * event-reporter: changes to anotations repoting - app.meta.revisions-metadata, report only revision in case of helm chart * event-reporter: changes after pr review * event-reporter: fixed unit tests * event-reporter: fix lint issues * event-reporter: changes after pr reviev, fixing typo, added dedicated func a.Spec.IsHelmSource(idx), removed legacy code * event-reporter: refactoring of getApplicationLegacyRevisionDetails method * event-reporter / app_revision_test.go: added some tests to AddCommitsDetailsToAnnotations, AddCommitsDetailsToAppAnnotations, getRevisions, getOperationSyncRevisions * event-reporter / app_revision_test.go: added tests for GetRevisionsDetails method * event-reporter: updated app client to support sourceIndex param in non-grpc mode * event-reporter / app_revision.go: added sourceIndex param to applicationServiceClient.RevisionMetadata in order to properly support multisourced apps * event-reporter: lint fix * event-reporter: fix lint issues * event-reporter: fix lint issues * event-reporter: added back regacy logic with setting of commit details to labels as new runtimes should work on old on-prem versions * event-reporter: added condition to not send empty array for ChangeRevisions metadata --------- Co-authored-by: pashakostohrys --- changelog/CHANGELOG.md | 2 +- event_reporter/application/client.go | 3 + event_reporter/reporter/app_revision.go | 89 +++++- event_reporter/reporter/app_revision_test.go | 168 +++++++++++ .../reporter/application_event_reporter.go | 32 +- event_reporter/reporter/event_payload.go | 64 ++-- event_reporter/reporter/event_payload_test.go | 30 +- event_reporter/utils/app_revision.go | 135 +++++++++ event_reporter/utils/app_revision_test.go | 280 +++++++++++++++++- .../acr-controller-deployment.yaml | 6 + .../application/v1alpha1/types_codefresh.go | 23 ++ 11 files changed, 763 insertions(+), 69 deletions(-) create mode 100644 event_reporter/reporter/app_revision_test.go diff --git a/changelog/CHANGELOG.md b/changelog/CHANGELOG.md index db818c0e9351f..e4fe910a101a9 100644 --- a/changelog/CHANGELOG.md +++ b/changelog/CHANGELOG.md @@ -1,2 +1,2 @@ ### Features -- fix: change revision controller should verify that revision already exists \ No newline at end of file +- feat: event-reporter: report change revisions metadata in app annotations \ No newline at end of file diff --git a/event_reporter/application/client.go b/event_reporter/application/client.go index 1efe8c7962b83..cf8367ce472e4 100644 --- a/event_reporter/application/client.go +++ b/event_reporter/application/client.go @@ -107,6 +107,9 @@ func (c *httpApplicationClient) RevisionMetadata(ctx context.Context, in *appcli params := fmt.Sprintf("?appNamespace=%s&project=%s", *in.AppNamespace, *in.Project) + if in.SourceIndex != nil { + params += fmt.Sprintf("&sourceIndex=%d", *in.SourceIndex) + } url := fmt.Sprintf("%s/api/v1/applications/%s/revisions/%s/metadata%s", c.baseUrl, *in.Name, *in.Revision, params) revisionMetadata := &v1alpha1.RevisionMetadata{} err := c.execute(ctx, url, revisionMetadata) diff --git a/event_reporter/reporter/app_revision.go b/event_reporter/reporter/app_revision.go index 82cf90371b6f7..b4c68f346325f 100644 --- a/event_reporter/reporter/app_revision.go +++ b/event_reporter/reporter/app_revision.go @@ -3,16 +3,89 @@ package reporter import ( "context" + "github.com/argoproj/argo-cd/v2/event_reporter/utils" "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" - appv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + + log "github.com/sirupsen/logrus" ) -func (s *applicationEventReporter) getApplicationRevisionDetails(ctx context.Context, a *appv1.Application, revision string) (*appv1.RevisionMetadata, error) { +// treats multi-sourced apps as single source and gets first revision details +func getApplicationLegacyRevisionDetails(a *v1alpha1.Application, revisionsWithMetadata *utils.AppSyncRevisionsMetadata) *v1alpha1.RevisionMetadata { + if revisionsWithMetadata.SyncRevisions == nil || len(revisionsWithMetadata.SyncRevisions) == 0 { + return nil + } + + sourceIdx := 0 + + if a.Spec.HasMultipleSources() { + _, sourceIdx = a.Spec.GetNonRefSource() + } + + if revisionWithMetadata := revisionsWithMetadata.SyncRevisions[sourceIdx]; revisionWithMetadata != nil { + return revisionWithMetadata.Metadata + } + + return nil +} + +func (s *applicationEventReporter) getRevisionsDetails(ctx context.Context, a *v1alpha1.Application, revisions []string) ([]*utils.RevisionWithMetadata, error) { project := a.Spec.GetProject() - return s.applicationServiceClient.RevisionMetadata(ctx, &application.RevisionMetadataQuery{ - Name: &a.Name, - AppNamespace: &a.Namespace, - Revision: &revision, - Project: &project, - }) + rms := make([]*utils.RevisionWithMetadata, 0) + + for idx, revision := range revisions { + // report just revision for helm sources + if a.Spec.SourceUnderIdxIsHelm(idx) { + rms = append(rms, &utils.RevisionWithMetadata{ + Revision: revision, + }) + continue + } + + sourceIndex := int32(idx) + + rm, err := s.applicationServiceClient.RevisionMetadata(ctx, &application.RevisionMetadataQuery{ + Name: &a.Name, + AppNamespace: &a.Namespace, + Revision: &revision, + Project: &project, + SourceIndex: &sourceIndex, + }) + if err != nil { + return nil, err + } + rms = append(rms, &utils.RevisionWithMetadata{ + Revision: revision, + Metadata: rm, + }) + } + + return rms, nil +} + +func (s *applicationEventReporter) getApplicationRevisionsMetadata(ctx context.Context, logCtx *log.Entry, a *v1alpha1.Application) (*utils.AppSyncRevisionsMetadata, error) { + result := &utils.AppSyncRevisionsMetadata{} + + if a.Status.Sync.Revision != "" || a.Status.Sync.Revisions != nil || (a.Status.History != nil && len(a.Status.History) > 0) { + // can be the latest revision of repository + operationSyncRevisionsMetadata, err := s.getRevisionsDetails(ctx, a, utils.GetOperationSyncRevisions(a)) + if err != nil { + logCtx.WithError(err).Warnf("failed to get application(%s) sync revisions metadata, resuming", a.GetName()) + } + + if err == nil && operationSyncRevisionsMetadata != nil { + result.SyncRevisions = operationSyncRevisionsMetadata + } + // latest revision of repository where changes to app resource were actually made; empty if no changeRevision(-s) present + operationChangeRevisionsMetadata, err := s.getRevisionsDetails(ctx, a, utils.GetOperationChangeRevisions(a)) + if err != nil { + logCtx.WithError(err).Warnf("failed to get application(%s) change revisions metadata, resuming", a.GetName()) + } + + if err == nil && operationChangeRevisionsMetadata != nil && len(operationChangeRevisionsMetadata) > 0 { + result.ChangeRevisions = operationChangeRevisionsMetadata + } + } + + return result, nil } diff --git a/event_reporter/reporter/app_revision_test.go b/event_reporter/reporter/app_revision_test.go new file mode 100644 index 0000000000000..4b9611738432f --- /dev/null +++ b/event_reporter/reporter/app_revision_test.go @@ -0,0 +1,168 @@ +package reporter + +import ( + "context" + "testing" + + "github.com/argoproj/argo-cd/v2/event_reporter/application/mocks" + "github.com/argoproj/argo-cd/v2/event_reporter/metrics" + "github.com/argoproj/argo-cd/v2/event_reporter/utils" + "github.com/argoproj/argo-cd/v2/pkg/apiclient/application" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" + "github.com/argoproj/argo-cd/v2/server/cache" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestGetRevisionsDetails(t *testing.T) { + t.Run("should return revisions for single source app", func(t *testing.T) { + expectedRevision := "expected-revision" + expectedResult := []*utils.RevisionWithMetadata{{ + Revision: expectedRevision, + Metadata: &v1alpha1.RevisionMetadata{ + Author: "Test Author", + Message: "first commit", + }, + }} + + app := v1alpha1.Application{ + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + RepoURL: "https://my-site.com", + TargetRevision: "HEAD", + Path: ".", + }, + }, + } + + appServiceClient := mocks.NewApplicationClient(t) + project := app.Spec.GetProject() + sourceIdx1 := int32(0) + + appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{ + Name: &app.Name, + AppNamespace: &app.Namespace, + Revision: &expectedResult[0].Revision, + Project: &project, + SourceIndex: &sourceIdx1, + }).Return(expectedResult[0].Metadata, nil) + + reporter := &applicationEventReporter{ + &cache.Cache{}, + &MockCodefreshClient{}, + newAppLister(), + appServiceClient, + &metrics.MetricsServer{}, + } + + result, _ := reporter.getRevisionsDetails(context.Background(), &app, []string{expectedRevision}) + + assert.Equal(t, expectedResult, result) + }) + + t.Run("should return revisions for multi sourced apps", func(t *testing.T) { + expectedRevision1 := "expected-revision-1" + expectedRevision2 := "expected-revision-2" + expectedResult := []*utils.RevisionWithMetadata{{ + Revision: expectedRevision1, + Metadata: &v1alpha1.RevisionMetadata{ + Author: "Repo1 Author", + Message: "first commit repo 1", + }, + }, { + Revision: expectedRevision2, + Metadata: &v1alpha1.RevisionMetadata{ + Author: "Repo2 Author", + Message: "first commit repo 2", + }, + }} + + app := v1alpha1.Application{ + Spec: v1alpha1.ApplicationSpec{ + Sources: []v1alpha1.ApplicationSource{{ + RepoURL: "https://my-site.com/repo-1", + TargetRevision: "branch1", + Path: ".", + }, { + RepoURL: "https://my-site.com/repo-2", + TargetRevision: "branch2", + Path: ".", + }}, + }, + } + + project := app.Spec.GetProject() + + appServiceClient := mocks.NewApplicationClient(t) + sourceIdx1 := int32(0) + sourceIdx2 := int32(1) + appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{ + Name: &app.Name, + AppNamespace: &app.Namespace, + Revision: &expectedRevision1, + Project: &project, + SourceIndex: &sourceIdx1, + }).Return(expectedResult[0].Metadata, nil) + appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{ + Name: &app.Name, + AppNamespace: &app.Namespace, + Revision: &expectedRevision2, + Project: &project, + SourceIndex: &sourceIdx2, + }).Return(expectedResult[1].Metadata, nil) + + reporter := &applicationEventReporter{ + &cache.Cache{}, + &MockCodefreshClient{}, + newAppLister(), + appServiceClient, + &metrics.MetricsServer{}, + } + + result, _ := reporter.getRevisionsDetails(context.Background(), &app, []string{expectedRevision1, expectedRevision2}) + + assert.Equal(t, expectedResult, result) + }) + + t.Run("should return only revision because of helm single source app", func(t *testing.T) { + expectedRevision := "expected-revision" + expectedResult := []*utils.RevisionWithMetadata{{ + Revision: expectedRevision, + }} + + app := v1alpha1.Application{ + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + RepoURL: "https://my-site.com", + TargetRevision: "HEAD", + Path: ".", + }, + }, + } + + appServiceClient := mocks.NewApplicationClient(t) + project := app.Spec.GetProject() + sourceIdx1 := int32(0) + + appServiceClient.On("RevisionMetadata", mock.Anything, &application.RevisionMetadataQuery{ + Name: &app.Name, + AppNamespace: &app.Namespace, + Revision: &expectedResult[0].Revision, + Project: &project, + SourceIndex: &sourceIdx1, + }).Return(expectedResult[0].Metadata, nil) + + reporter := &applicationEventReporter{ + &cache.Cache{}, + &MockCodefreshClient{}, + newAppLister(), + appServiceClient, + &metrics.MetricsServer{}, + } + + result, _ := reporter.getRevisionsDetails(context.Background(), &app, []string{expectedRevision}) + + assert.Equal(t, expectedResult, result) + }) +} diff --git a/event_reporter/reporter/application_event_reporter.go b/event_reporter/reporter/application_event_reporter.go index 6524655ab22eb..6feabf1e569e2 100644 --- a/event_reporter/reporter/application_event_reporter.go +++ b/event_reporter/reporter/application_event_reporter.go @@ -166,14 +166,13 @@ func (s *applicationEventReporter) StreamApplicationEvents( // helm app hasnt revision // TODO: add check if it helm application - parentOperationRevision := utils.GetOperationRevision(parentApplicationEntity) - parentRevisionMetadata, err := s.getApplicationRevisionDetails(ctx, parentApplicationEntity, parentOperationRevision) + parentAppSyncRevisionsMetadata, err := s.getApplicationRevisionsMetadata(ctx, logCtx, parentApplicationEntity) if err != nil { logCtx.WithError(err).Warn("failed to get parent application's revision metadata, resuming") } utils.SetHealthStatusIfMissing(rs) - err = s.processResource(ctx, *rs, parentApplicationEntity, logCtx, ts, parentDesiredManifests, appTree, manifestGenErr, a, parentRevisionMetadata, appInstanceLabelKey, trackingMethod, desiredManifests.ApplicationVersions) + err = s.processResource(ctx, *rs, parentApplicationEntity, logCtx, ts, parentDesiredManifests, appTree, manifestGenErr, a, parentAppSyncRevisionsMetadata, appInstanceLabelKey, trackingMethod, desiredManifests.ApplicationVersions) if err != nil { s.metricsServer.IncErroredEventsCounter(metrics.MetricChildAppEventType, metrics.MetricEventUnknownErrorType, a.Name) return err @@ -203,7 +202,7 @@ func (s *applicationEventReporter) StreamApplicationEvents( s.metricsServer.ObserveEventProcessingDurationHistogramDuration(a.Name, metrics.MetricParentAppEventType, reconcileDuration) } - revisionMetadata, _ := s.getApplicationRevisionDetails(ctx, a, utils.GetOperationRevision(a)) + revisionsMetadata, _ := s.getApplicationRevisionsMetadata(ctx, logCtx, a) // for each resource in the application get desired and actual state, // then stream the event for _, rs := range a.Status.Resources { @@ -215,7 +214,7 @@ func (s *applicationEventReporter) StreamApplicationEvents( s.metricsServer.IncCachedIgnoredEventsCounter(metrics.MetricResourceEventType, a.Name) continue } - err := s.processResource(ctx, rs, a, logCtx, ts, desiredManifests, appTree, manifestGenErr, nil, revisionMetadata, appInstanceLabelKey, trackingMethod, nil) + err := s.processResource(ctx, rs, a, logCtx, ts, desiredManifests, appTree, manifestGenErr, nil, revisionsMetadata, appInstanceLabelKey, trackingMethod, nil) if err != nil { s.metricsServer.IncErroredEventsCounter(metrics.MetricResourceEventType, metrics.MetricEventUnknownErrorType, a.Name) return err @@ -227,21 +226,22 @@ func (s *applicationEventReporter) StreamApplicationEvents( func (s *applicationEventReporter) getAppForResourceReporting( rs appv1.ResourceStatus, ctx context.Context, + logCtx *log.Entry, a *appv1.Application, - revisionMetadata *appv1.RevisionMetadata, -) (*appv1.Application, *appv1.RevisionMetadata) { + syncRevisionsMetadata *utils.AppSyncRevisionsMetadata, +) (*appv1.Application, *utils.AppSyncRevisionsMetadata) { if rs.Kind != "Rollout" { // for rollout it's crucial to report always correct operationSyncRevision - return a, revisionMetadata + return a, syncRevisionsMetadata } latestAppStatus, err := s.appLister.Applications(a.Namespace).Get(a.Name) if err != nil { - return a, revisionMetadata + return a, syncRevisionsMetadata } - revisionMetadataToReport, err := s.getApplicationRevisionDetails(ctx, latestAppStatus, utils.GetOperationRevision(latestAppStatus)) + revisionMetadataToReport, err := s.getApplicationRevisionsMetadata(ctx, logCtx, latestAppStatus) if err != nil { - return a, revisionMetadata + return a, syncRevisionsMetadata } return latestAppStatus, revisionMetadataToReport @@ -257,7 +257,7 @@ func (s *applicationEventReporter) processResource( appTree *appv1.ApplicationTree, manifestGenErr bool, originalApplication *appv1.Application, - revisionMetadata *appv1.RevisionMetadata, + revisionsMetadata *utils.AppSyncRevisionsMetadata, appInstanceLabelKey string, trackingMethod appv1.TrackingMethod, applicationVersions *apiclient.ApplicationVersions, @@ -283,12 +283,12 @@ func (s *applicationEventReporter) processResource( return nil } - parentApplicationToReport, revisionMetadataToReport := s.getAppForResourceReporting(rs, ctx, parentApplication, revisionMetadata) + parentApplicationToReport, revisionMetadataToReport := s.getAppForResourceReporting(rs, ctx, logCtx, parentApplication, revisionsMetadata) - var originalAppRevisionMetadata *appv1.RevisionMetadata = nil + var originalAppRevisionMetadata *utils.AppSyncRevisionsMetadata = nil if originalApplication != nil { - originalAppRevisionMetadata, _ = s.getApplicationRevisionDetails(ctx, originalApplication, utils.GetOperationRevision(originalApplication)) + originalAppRevisionMetadata, _ = s.getApplicationRevisionsMetadata(ctx, logCtx, originalApplication) } ev, err := getResourceEventPayload(parentApplicationToReport, &rs, actualState, desiredState, appTree, manifestGenErr, ts, originalApplication, revisionMetadataToReport, originalAppRevisionMetadata, appInstanceLabelKey, trackingMethod, applicationVersions) @@ -305,7 +305,7 @@ func (s *applicationEventReporter) processResource( appName = appRes.Name } else { utils.LogWithResourceStatus(logCtx, rs).Info("streaming resource event") - appName = rs.Name + appName = parentApplication.Name } if err := s.codefreshClient.SendEvent(ctx, appName, ev); err != nil { diff --git a/event_reporter/reporter/event_payload.go b/event_reporter/reporter/event_payload.go index ecaa2f45c3e39..cbac9a9031d32 100644 --- a/event_reporter/reporter/event_payload.go +++ b/event_reporter/reporter/event_payload.go @@ -28,8 +28,8 @@ func getResourceEventPayload( manifestGenErr bool, ts string, originalApplication *appv1.Application, // passed when rs is application - revisionMetadata *appv1.RevisionMetadata, - originalAppRevisionMetadata *appv1.RevisionMetadata, // passed when rs is application + revisionsMetadata *utils.AppSyncRevisionsMetadata, + originalAppRevisionsMetadata *utils.AppSyncRevisionsMetadata, // passed when rs is application appInstanceLabelKey string, trackingMethod appv1.TrackingMethod, applicationVersions *apiclient.ApplicationVersions, @@ -50,11 +50,15 @@ func getResourceEventPayload( object := []byte(*actualState.Manifest) - if originalAppRevisionMetadata != nil && len(object) != 0 { + if originalAppRevisionsMetadata != nil && len(object) != 0 { actualObject, err := appv1.UnmarshalToUnstructured(*actualState.Manifest) if err == nil { - actualObject = utils.AddCommitDetailsToLabels(actualObject, originalAppRevisionMetadata) + actualObject = utils.AddCommitsDetailsToAnnotations(actualObject, originalAppRevisionsMetadata) + if originalApplication != nil { + actualObject = utils.AddCommitDetailsToLabels(actualObject, getApplicationLegacyRevisionDetails(originalApplication, originalAppRevisionsMetadata)) + } + object, err = actualObject.MarshalJSON() if err != nil { return nil, fmt.Errorf("failed to marshal unstructured object: %w", err) @@ -74,8 +78,11 @@ func getResourceEventPayload( u.SetKind(rs.Kind) u.SetName(rs.Name) u.SetNamespace(rs.Namespace) - if originalAppRevisionMetadata != nil { - u = utils.AddCommitDetailsToLabels(u, originalAppRevisionMetadata) + if originalAppRevisionsMetadata != nil { + u = utils.AddCommitsDetailsToAnnotations(u, originalAppRevisionsMetadata) + if originalApplication != nil { + u = utils.AddCommitDetailsToLabels(u, getApplicationLegacyRevisionDetails(originalApplication, originalAppRevisionsMetadata)) + } } object, err = u.MarshalJSON() @@ -88,8 +95,11 @@ func getResourceEventPayload( if err != nil { return nil, fmt.Errorf("failed to add destination namespace to manifest: %w", err) } - if originalAppRevisionMetadata != nil { - unstructuredWithNamespace = utils.AddCommitDetailsToLabels(unstructuredWithNamespace, originalAppRevisionMetadata) + if originalAppRevisionsMetadata != nil { + unstructuredWithNamespace = utils.AddCommitsDetailsToAnnotations(unstructuredWithNamespace, originalAppRevisionsMetadata) + if originalApplication != nil { + unstructuredWithNamespace = utils.AddCommitDetailsToLabels(unstructuredWithNamespace, getApplicationLegacyRevisionDetails(originalApplication, originalAppRevisionsMetadata)) + } } object, _ = unstructuredWithNamespace.MarshalJSON() @@ -166,10 +176,13 @@ func getResourceEventPayload( TrackingMethod: string(trackingMethod), } - if revisionMetadata != nil { - source.CommitMessage = revisionMetadata.Message - source.CommitAuthor = revisionMetadata.Author - source.CommitDate = &revisionMetadata.Date + if revisionsMetadata != nil && revisionsMetadata.SyncRevisions != nil { + revisionMetadata := getApplicationLegacyRevisionDetails(parentApplication, revisionsMetadata) + if revisionMetadata != nil { + source.CommitMessage = revisionMetadata.Message + source.CommitAuthor = revisionMetadata.Author + source.CommitDate = &revisionMetadata.Date + } } if rs.Health != nil { @@ -225,27 +238,18 @@ func (s *applicationEventReporter) getApplicationEventPayload( syncFinished = a.Status.OperationState.FinishedAt } - applicationSource := a.Spec.GetSource() - if !applicationSource.IsHelm() && (a.Status.Sync.Revision != "" || (a.Status.History != nil && len(a.Status.History) > 0)) { - revisionMetadata, err := s.getApplicationRevisionDetails(ctx, a, utils.GetOperationRevision(a)) - - if err != nil { - if !strings.Contains(err.Error(), "not found") { - return nil, fmt.Errorf("failed to get revision metadata: %w", err) - } - - logCtx.Warnf("failed to get revision metadata: %s, reporting application deletion event", err.Error()) - } else { - if obj.ObjectMeta.Labels == nil { - obj.ObjectMeta.Labels = map[string]string{} - } - - obj.ObjectMeta.Labels["app.meta.commit-date"] = revisionMetadata.Date.Format("2006-01-02T15:04:05.000Z") - obj.ObjectMeta.Labels["app.meta.commit-author"] = revisionMetadata.Author - obj.ObjectMeta.Labels["app.meta.commit-message"] = revisionMetadata.Message + revisionsMetadata, err := s.getApplicationRevisionsMetadata(ctx, logCtx, a) + if err != nil { + if !strings.Contains(err.Error(), "not found") { + return nil, fmt.Errorf("failed to get revision metadata: %w", err) } + + logCtx.Warnf("failed to get revision metadata: %s, reporting application deletion event", err.Error()) } + utils.AddCommitsDetailsToAppAnnotations(obj, revisionsMetadata) + utils.AddCommitsDetailsToAppLabels(&obj, getApplicationLegacyRevisionDetails(&obj, revisionsMetadata)) + object, err := json.Marshal(&obj) if err != nil { return nil, fmt.Errorf("failed to marshal application event") diff --git a/event_reporter/reporter/event_payload_test.go b/event_reporter/reporter/event_payload_test.go index 25b8a8971b722..dc305a099c897 100644 --- a/event_reporter/reporter/event_payload_test.go +++ b/event_reporter/reporter/event_payload_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "testing" + "github.com/argoproj/argo-cd/v2/event_reporter/utils" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,7 +20,13 @@ import ( func TestGetResourceEventPayload(t *testing.T) { t.Run("Deleting timestamp is empty", func(t *testing.T) { - app := v1alpha1.Application{} + app := v1alpha1.Application{ + Spec: v1alpha1.ApplicationSpec{ + Source: &v1alpha1.ApplicationSource{ + RepoURL: "test", + }, + }, + } rs := v1alpha1.ResourceStatus{} man := "{ \"key\" : \"manifest\" }" @@ -30,10 +38,14 @@ func TestGetResourceEventPayload(t *testing.T) { CompiledManifest: "{ \"key\" : \"manifest\" }", } appTree := v1alpha1.ApplicationTree{} - revisionMetadata := v1alpha1.RevisionMetadata{ - Author: "demo usert", - Date: metav1.Time{}, - Message: "some message", + revisionMetadata := utils.AppSyncRevisionsMetadata{ + SyncRevisions: []*utils.RevisionWithMetadata{{ + Metadata: &v1alpha1.RevisionMetadata{ + Author: "demo usert", + Date: metav1.Time{}, + Message: "some message", + }, + }}, } event, err := getResourceEventPayload(&app, &rs, &actualState, &desiredState, &appTree, true, "", nil, &revisionMetadata, nil, common.LabelKeyAppInstance, argo.TrackingMethodLabel, &repoApiclient.ApplicationVersions{}) @@ -48,7 +60,7 @@ func TestGetResourceEventPayload(t *testing.T) { assert.Equal(t, "{ \"key\" : \"manifest\" }", eventPayload.Source.ActualManifest) }) - t.Run("Deleting timestamp is empty", func(t *testing.T) { + t.Run("Deleting timestamp not empty", func(t *testing.T) { app := v1alpha1.Application{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &metav1.Time{}, @@ -64,10 +76,8 @@ func TestGetResourceEventPayload(t *testing.T) { CompiledManifest: "{ \"key\" : \"manifest\" }", } appTree := v1alpha1.ApplicationTree{} - revisionMetadata := v1alpha1.RevisionMetadata{ - Author: "demo usert", - Date: metav1.Time{}, - Message: "some message", + revisionMetadata := utils.AppSyncRevisionsMetadata{ + SyncRevisions: []*utils.RevisionWithMetadata{}, } event, err := getResourceEventPayload(&app, &rs, &actualState, &desiredState, &appTree, true, "", nil, &revisionMetadata, nil, common.LabelKeyAppInstance, argo.TrackingMethodLabel, &repoApiclient.ApplicationVersions{}) diff --git a/event_reporter/utils/app_revision.go b/event_reporter/utils/app_revision.go index 0f0027eacbd4c..161f37f20d5a2 100644 --- a/event_reporter/utils/app_revision.go +++ b/event_reporter/utils/app_revision.go @@ -1,11 +1,30 @@ package utils import ( + "encoding/json" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" appv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" ) +type AppSyncRevisionsMetadata struct { + SyncRevisions []*RevisionWithMetadata `json:"syncRevisions" protobuf:"bytes,1,name=syncRevisions"` + ChangeRevisions []*RevisionWithMetadata `json:"changeRevisions" protobuf:"bytes,2,name=changeRevisions"` +} + +type RevisionWithMetadata struct { + Revision string `json:"revision" protobuf:"bytes,1,name=revision"` + Metadata *appv1.RevisionMetadata `json:"metadata,omitempty" protobuf:"bytes,2,name=metadata"` +} + +type RevisionsData struct { + Revision string `json:"revision,omitempty" protobuf:"bytes,1,opt,name=revision"` + Revisions []string `json:"revisions,omitempty" protobuf:"bytes,2,opt,name=revisions"` +} + +const annotationRevisionKey = "app.meta.revisions-metadata" + func GetLatestAppHistoryId(a *appv1.Application) int64 { if lastHistory := getLatestAppHistoryItem(a); lastHistory != nil { return lastHistory.ID @@ -46,6 +65,106 @@ func GetOperationRevision(a *appv1.Application) string { return revision } +func GetOperationSyncRevisions(a *appv1.Application) []string { + if a == nil { + return []string{} + } + + // this value will be used in case if application hasn't resources, like empty gitsource + revisions := getRevisions(RevisionsData{ + Revision: a.Status.Sync.Revision, + Revisions: a.Status.Sync.Revisions, + }) + + if a.Status.OperationState != nil && a.Status.OperationState.Operation.Sync != nil { + revisions = getRevisions(RevisionsData{ + Revision: a.Status.OperationState.Operation.Sync.Revision, + Revisions: a.Status.OperationState.Operation.Sync.Revisions, + }) + } else if a.Operation != nil && a.Operation.Sync != nil { + revisions = getRevisions(RevisionsData{ + Revision: a.Operation.Sync.Revision, + Revisions: a.Operation.Sync.Revisions, + }) + } + + return revisions +} + +// for monorepo support: list with revisions where actual changes to source directory were committed +func GetOperationChangeRevisions(a *appv1.Application) []string { + var revisions []string + + if a == nil { + return revisions + } + + // this value will be used in case if application hasn't resources, like empty gitsource + if a.Status.OperationState != nil && a.Status.OperationState.Operation.Sync != nil { + if a.Status.OperationState.Operation.Sync.ChangeRevision != "" || a.Status.OperationState.Operation.Sync.ChangeRevisions != nil { + revisions = getRevisions(RevisionsData{ + Revision: a.Status.OperationState.Operation.Sync.ChangeRevision, + Revisions: a.Status.OperationState.Operation.Sync.ChangeRevisions, + }) + } + } else if a.Operation != nil && a.Operation.Sync != nil { + if a.Operation.Sync.ChangeRevision != "" || a.Operation.Sync.ChangeRevisions != nil { + revisions = getRevisions(RevisionsData{ + Revision: a.Operation.Sync.ChangeRevision, + Revisions: a.Operation.Sync.ChangeRevisions, + }) + } + } + + return revisions +} + +func getRevisions(rd RevisionsData) []string { + if rd.Revisions != nil { + return rd.Revisions + } + + return []string{rd.Revision} +} + +func AddCommitsDetailsToAnnotations(unstrApp *unstructured.Unstructured, revisionsMetadata *AppSyncRevisionsMetadata) *unstructured.Unstructured { + if revisionsMetadata == nil || unstrApp == nil { + return unstrApp + } + + if field, _, _ := unstructured.NestedFieldCopy(unstrApp.Object, "metadata", "annotations"); field == nil { + _ = unstructured.SetNestedStringMap(unstrApp.Object, map[string]string{}, "metadata", "annotations") + } + + jsonRevisionsMetadata, err := json.Marshal(revisionsMetadata) + if err != nil { + return unstrApp + } + + _ = unstructured.SetNestedField(unstrApp.Object, string(jsonRevisionsMetadata), "metadata", "annotations", annotationRevisionKey) + + return unstrApp +} + +func AddCommitsDetailsToAppAnnotations(app appv1.Application, revisionsMetadata *AppSyncRevisionsMetadata) appv1.Application { + if revisionsMetadata == nil { + return app + } + + if app.ObjectMeta.Annotations == nil { + app.ObjectMeta.Annotations = map[string]string{} + } + + jsonRevisionsMetadata, err := json.Marshal(revisionsMetadata) + if err != nil { + return app + } + + app.ObjectMeta.Annotations[annotationRevisionKey] = string(jsonRevisionsMetadata) + + return app +} + func AddCommitDetailsToLabels(u *unstructured.Unstructured, revisionMetadata *appv1.RevisionMetadata) *unstructured.Unstructured { if revisionMetadata == nil || u == nil { return u @@ -61,3 +180,19 @@ func AddCommitDetailsToLabels(u *unstructured.Unstructured, revisionMetadata *ap return u } + +func AddCommitsDetailsToAppLabels(app *appv1.Application, revisionMetadata *appv1.RevisionMetadata) *appv1.Application { + if revisionMetadata == nil { + return app + } + + if app.ObjectMeta.Labels == nil { + app.ObjectMeta.Labels = map[string]string{} + } + + app.ObjectMeta.Labels["app.meta.commit-date"] = revisionMetadata.Date.Format("2006-01-02T15:04:05.000Z") + app.ObjectMeta.Labels["app.meta.commit-author"] = revisionMetadata.Author + app.ObjectMeta.Labels["app.meta.commit-message"] = revisionMetadata.Message + + return app +} diff --git a/event_reporter/utils/app_revision_test.go b/event_reporter/utils/app_revision_test.go index 852fb69ff2775..370edecdb2d5a 100644 --- a/event_reporter/utils/app_revision_test.go +++ b/event_reporter/utils/app_revision_test.go @@ -3,10 +3,11 @@ package utils import ( "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1" "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" ) @@ -102,7 +103,7 @@ func TestGetApplicationLatestRevision(t *testing.T) { }) } -func StrToUnstructured(jsonStr string) *unstructured.Unstructured { +func yamlToUnstructured(jsonStr string) *unstructured.Unstructured { obj := make(map[string]interface{}) err := yaml.Unmarshal([]byte(jsonStr), &obj) if err != nil { @@ -111,6 +112,277 @@ func StrToUnstructured(jsonStr string) *unstructured.Unstructured { return &unstructured.Unstructured{Object: obj} } +func jsonToAppSyncRevision(jsonStr string) *AppSyncRevisionsMetadata { + var obj AppSyncRevisionsMetadata + err := yaml.Unmarshal([]byte(jsonStr), &obj) + if err != nil { + panic(err) + } + return &obj +} + +func TestAddCommitsDetailsToAnnotations(t *testing.T) { + revisionMetadata := AppSyncRevisionsMetadata{ + SyncRevisions: []*RevisionWithMetadata{{ + Metadata: &v1alpha1.RevisionMetadata{ + Author: "demo usert", + Date: metav1.Time{}, + Message: "some message", + }, + }}, + } + + t.Run("set annotation when annotations object missing", func(t *testing.T) { + resource := yamlToUnstructured(` + apiVersion: v1 + kind: Service + metadata: + name: helm-guestbook + namespace: default + resourceVersion: "123" + uid: "4" + spec: + selector: + app: guestbook + type: LoadBalancer + status: + loadBalancer: + ingress: + - hostname: localhost`, + ) + + result := AddCommitsDetailsToAnnotations(resource, &revisionMetadata) + + revMetadatUnstructured := jsonToAppSyncRevision(result.GetAnnotations()[annotationRevisionKey]) + + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Author, revMetadatUnstructured.SyncRevisions[0].Metadata.Author) + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Message, revMetadatUnstructured.SyncRevisions[0].Metadata.Message) + }) + + t.Run("set annotation when annotations present", func(t *testing.T) { + resource := yamlToUnstructured(` + apiVersion: v1 + kind: Service + metadata: + name: helm-guestbook + namespace: default + annotations: + link: http://my-grafana.com/pre-generated-link + spec: + selector: + app: guestbook + type: LoadBalancer + status: + loadBalancer: + ingress: + - hostname: localhost`, + ) + + result := AddCommitsDetailsToAnnotations(resource, &revisionMetadata) + + revMetadatUnstructured := jsonToAppSyncRevision(result.GetAnnotations()[annotationRevisionKey]) + + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Author, revMetadatUnstructured.SyncRevisions[0].Metadata.Author) + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Message, revMetadatUnstructured.SyncRevisions[0].Metadata.Message) + }) +} + +func TestAddCommitsDetailsToAppAnnotations(t *testing.T) { + revisionMetadata := AppSyncRevisionsMetadata{ + SyncRevisions: []*RevisionWithMetadata{{ + Metadata: &v1alpha1.RevisionMetadata{ + Author: "demo usert", + Date: metav1.Time{}, + Message: "some message", + }, + }}, + } + + t.Run("set annotation when annotations object missing", func(t *testing.T) { + resource := v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{}, + } + + result := AddCommitsDetailsToAppAnnotations(resource, &revisionMetadata) + + revMetadatUnstructured := jsonToAppSyncRevision(result.GetAnnotations()[annotationRevisionKey]) + + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Author, revMetadatUnstructured.SyncRevisions[0].Metadata.Author) + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Message, revMetadatUnstructured.SyncRevisions[0].Metadata.Message) + }) + + t.Run("set annotation when annotations present", func(t *testing.T) { + resource := v1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "test": "value", + }, + }, + } + + result := AddCommitsDetailsToAppAnnotations(resource, &revisionMetadata) + + revMetadatUnstructured := jsonToAppSyncRevision(result.GetAnnotations()[annotationRevisionKey]) + + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Author, revMetadatUnstructured.SyncRevisions[0].Metadata.Author) + assert.Equal(t, revisionMetadata.SyncRevisions[0].Metadata.Message, revMetadatUnstructured.SyncRevisions[0].Metadata.Message) + }) +} + +func TestGetRevisions(t *testing.T) { + t.Run("should return revisions when only they passed", func(t *testing.T) { + val := "test" + result := getRevisions(RevisionsData{ + Revisions: []string{val}, + }) + assert.Len(t, result, 1) + assert.Equal(t, val, result[0]) + }) + t.Run("should return revisions when revision also passed", func(t *testing.T) { + val := "test" + result := getRevisions(RevisionsData{ + Revisions: []string{val, "test2"}, + Revision: "fail", + }) + assert.Len(t, result, 2) + assert.Equal(t, val, result[0]) + }) + t.Run("should return revision", func(t *testing.T) { + val := "test" + result := getRevisions(RevisionsData{ + Revision: val, + }) + assert.Len(t, result, 1) + assert.Equal(t, val, result[0]) + }) +} + +func TestGetOperationSyncRevisions(t *testing.T) { + t.Run("should return Status.Sync.Revision like for new apps", func(t *testing.T) { + expectedResult := "test" + app := v1alpha1.Application{ + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Revision: expectedResult, + }, + }, + } + result := GetOperationSyncRevisions(&app) + + assert.Len(t, result, 1) + assert.Equal(t, expectedResult, result[0]) + }) + + t.Run("should return Status.Sync.Revisions like for new apps", func(t *testing.T) { + expectedResult := "multi-1" + app := v1alpha1.Application{ + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Revisions: []string{expectedResult, "multi-2"}, + Revision: "single", + }, + }, + } + + result := GetOperationSyncRevisions(&app) + + assert.Len(t, result, 2) + assert.Equal(t, expectedResult, result[0]) + }) + + t.Run("should return a.Status.OperationState.Operation.Sync.Revision", func(t *testing.T) { + expectedResult := "multi-1" + app := v1alpha1.Application{ + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Revision: "fallack", + }, + OperationState: &v1alpha1.OperationState{ + Operation: v1alpha1.Operation{ + Sync: &v1alpha1.SyncOperation{ + Revision: expectedResult, + }, + }, + }, + }, + } + + result := GetOperationSyncRevisions(&app) + + assert.Len(t, result, 1) + assert.Equal(t, expectedResult, result[0]) + }) + + t.Run("should return a.Status.OperationState.Operation.Sync.Revisions", func(t *testing.T) { + expectedResult := "multi-1" + + app := v1alpha1.Application{ + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Revision: "fallack", + }, + OperationState: &v1alpha1.OperationState{ + Operation: v1alpha1.Operation{ + Sync: &v1alpha1.SyncOperation{ + Revisions: []string{expectedResult, "multi-2"}, + Revision: "single", + }, + }, + }, + }, + } + + result := GetOperationSyncRevisions(&app) + + assert.Len(t, result, 2) + assert.Equal(t, expectedResult, result[0]) + }) + + t.Run("should return a.Operation.Sync.Revision for first app sync", func(t *testing.T) { + expectedResult := "multi-1" + app := v1alpha1.Application{ + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Revision: "fallack", + }, + }, + Operation: &v1alpha1.Operation{ + Sync: &v1alpha1.SyncOperation{ + Revision: expectedResult, + }, + }, + } + + result := GetOperationSyncRevisions(&app) + + assert.Len(t, result, 1) + assert.Equal(t, expectedResult, result[0]) + }) + + t.Run("should return a.Operation.Sync.Revisions for first app sync", func(t *testing.T) { + expectedResult := "multi-1" + + app := v1alpha1.Application{ + Status: v1alpha1.ApplicationStatus{ + Sync: v1alpha1.SyncStatus{ + Revision: "fallack", + }, + }, + Operation: &v1alpha1.Operation{ + Sync: &v1alpha1.SyncOperation{ + Revisions: []string{expectedResult, "multi-2"}, + Revision: "single", + }, + }, + } + + result := GetOperationSyncRevisions(&app) + + assert.Len(t, result, 2) + assert.Equal(t, expectedResult, result[0]) + }) +} + func TestAddCommitDetailsToLabels(t *testing.T) { revisionMetadata := v1alpha1.RevisionMetadata{ Author: "demo usert", @@ -119,7 +391,7 @@ func TestAddCommitDetailsToLabels(t *testing.T) { } t.Run("set labels when lable object missing", func(t *testing.T) { - resource := StrToUnstructured(` + resource := yamlToUnstructured(` apiVersion: v1 kind: Service metadata: @@ -144,7 +416,7 @@ func TestAddCommitDetailsToLabels(t *testing.T) { }) t.Run("set labels when labels present", func(t *testing.T) { - resource := StrToUnstructured(` + resource := yamlToUnstructured(` apiVersion: v1 kind: Service metadata: diff --git a/manifests/base/acr-controller/acr-controller-deployment.yaml b/manifests/base/acr-controller/acr-controller-deployment.yaml index baf85faaf85df..d75ab9f05b55f 100644 --- a/manifests/base/acr-controller/acr-controller-deployment.yaml +++ b/manifests/base/acr-controller/acr-controller-deployment.yaml @@ -31,6 +31,12 @@ spec: secretKeyRef: key: token name: argocd-token + - name: ARGOCD_SERVER_ROOTPATH + valueFrom: + configMapKeyRef: + key: server.rootpath + name: argocd-cmd-params-cm + optional: true - name: ARGOCD_APPLICATION_NAMESPACES valueFrom: configMapKeyRef: diff --git a/pkg/apis/application/v1alpha1/types_codefresh.go b/pkg/apis/application/v1alpha1/types_codefresh.go index ad46d19bbe208..1e2d922b407a4 100644 --- a/pkg/apis/application/v1alpha1/types_codefresh.go +++ b/pkg/apis/application/v1alpha1/types_codefresh.go @@ -16,3 +16,26 @@ func (a *Application) SetDefaultTypeMeta() { APIVersion: SchemeGroupVersion.String(), } } + +func (spec *ApplicationSpec) GetNonRefSource() (*ApplicationSource, int) { + if spec.HasMultipleSources() { + for idx, source := range spec.Sources { + if !source.IsRef() { + return &source, idx + } + } + } + + if spec.Source == nil { + return nil, -2 + } + + // single source app + return spec.Source, -1 +} + +func (spec *ApplicationSpec) SourceUnderIdxIsHelm(idx int) bool { + source := spec.GetSourcePtrByIndex(idx) + + return source != nil && source.IsHelm() +} From b98b02ab4948e8f04dd3f70da203d4bce41bef6e Mon Sep 17 00:00:00 2001 From: oleksandr-codefresh Date: Sat, 5 Oct 2024 17:19:50 +0300 Subject: [PATCH 2/2] chore: update version to 2.12-2024.10.5-66e1e38fa --- VERSION | 2 +- changelog/CHANGELOG-2.12-2024.10.5-66e1e38fa.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog/CHANGELOG-2.12-2024.10.5-66e1e38fa.md diff --git a/VERSION b/VERSION index 040aa068fb3d2..3dffe49094c29 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.12-2024.9.30-995854136 \ No newline at end of file +2.12-2024.10.5-66e1e38fa \ No newline at end of file diff --git a/changelog/CHANGELOG-2.12-2024.10.5-66e1e38fa.md b/changelog/CHANGELOG-2.12-2024.10.5-66e1e38fa.md new file mode 100644 index 0000000000000..e4fe910a101a9 --- /dev/null +++ b/changelog/CHANGELOG-2.12-2024.10.5-66e1e38fa.md @@ -0,0 +1,2 @@ +### Features +- feat: event-reporter: report change revisions metadata in app annotations \ No newline at end of file