From acc4b6aae366abdc724dd406d33b2b902ac9e28b Mon Sep 17 00:00:00 2001 From: Martin Basti Date: Tue, 5 Sep 2023 14:04:29 +0200 Subject: [PATCH 1/2] feat(STONEINTG-523): functions to manipulate with test status Functions and data types to manipulate with status of test scenarios in snapshots. Annotation test.appstudio.openshift.io/status is used to keep metadata about integration tests in snapshots. This commit provides functions to manipulate with them: udpating, deleting, writing, reading statuses Example: ``` test.appstudio.openshift.io/status: [ { "scenario": "scenario-1", "status": "EnvironmentProvisionError", "lastUpdateTimep": "2023-07-26T16:57:49+02:00", "details": "Failed ..." } ] ``` Signed-off-by: Martin Basti --- gitops/snapshot.go | 3 + gitops/snapshot_integration_tests_status.go | 269 ++++++++++++ .../snapshot_integration_tests_status_test.go | 391 ++++++++++++++++++ gitops/snapshot_test.go | 12 + 4 files changed, 675 insertions(+) create mode 100644 gitops/snapshot_integration_tests_status.go create mode 100644 gitops/snapshot_integration_tests_status_test.go diff --git a/gitops/snapshot.go b/gitops/snapshot.go index 02fb74f06..0e7056b78 100644 --- a/gitops/snapshot.go +++ b/gitops/snapshot.go @@ -32,6 +32,9 @@ const ( // SnapshotTestScenarioLabel contains the name of the Snapshot test scenario. SnapshotTestScenarioLabel = "test.appstudio.openshift.io/scenario" + // SnapshotTestScenarioLabel contains json data with test results of the particular snapshot + SnapshotTestsStatusAnnotation = "test.appstudio.openshift.io/status" + // BuildPipelineRunPrefix contains the build pipeline run related labels and annotations BuildPipelineRunPrefix = "build.appstudio" diff --git a/gitops/snapshot_integration_tests_status.go b/gitops/snapshot_integration_tests_status.go new file mode 100644 index 000000000..caa5938f9 --- /dev/null +++ b/gitops/snapshot_integration_tests_status.go @@ -0,0 +1,269 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions andF +limitations under the License. +*/ + +package gitops + +import ( + "context" + "encoding/json" + "fmt" + "time" + + applicationapiv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1" + "github.com/redhat-appstudio/integration-service/api/v1beta1" + "github.com/redhat-appstudio/operator-toolkit/metadata" + "github.com/santhosh-tekuri/jsonschema/v5" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const integrationTestStatusesSchema = `{ + "$schema": "http://json-schema.org/draft/2020-12/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "scenario": { + "type": "string" + }, + "status": { + "type": "string" + }, + "lastUpdateTime": { + "type": "string" + }, + "details": { + "type": "string" + } + }, + "required": ["scenario", "status", "lastUpdateTime"] + } + }` + +// IntegrationTestStatusDetail contains metadata about the particular scenario testing status +type IntegrationTestStatusDetail struct { + // ScenarioName name + ScenarioName string `json:"scenario"` + // The status summary for the ITS and Snapshot + Status IntegrationTestStatus `json:"status"` + // The time of reporting the status + LastUpdateTime time.Time `json:"lastUpdateTime"` + // The details of reported status + Details string `json:"details"` +} + +// SnapshotIntegrationTestStatuses type handles details about snapshot tests +// Please note that internal representation differs from marshalled representation +// Data are not written directly into snapshot, they are just cached in this structure +type SnapshotIntegrationTestStatuses struct { + // map scenario name to test details + statuses map[string]*IntegrationTestStatusDetail + // flag if any updates have been done + dirty bool +} + +// IsDirty returns boolean if there are any changes +func (sits *SnapshotIntegrationTestStatuses) IsDirty() bool { + return sits.dirty +} + +// ResetDirty reset repo back to clean, i.e. no changes to data +func (sits *SnapshotIntegrationTestStatuses) ResetDirty() { + sits.dirty = false +} + +// UpdateTestStatusIfChanged updates status of scenario test when status or details changed +func (sits *SnapshotIntegrationTestStatuses) UpdateTestStatusIfChanged(scenarioName string, status IntegrationTestStatus, details string) { + detail, ok := sits.statuses[scenarioName] + timestamp := time.Now().UTC() + if ok { + // update only when status or details changed, otherwise it's a no-op + // to preserve timestamps + if detail.Status != status || detail.Details != details { + detail.Status = status + detail.Details = details + detail.LastUpdateTime = timestamp + sits.dirty = true + } + } else { + newDetail := IntegrationTestStatusDetail{ + ScenarioName: scenarioName, + Status: status, + Details: details, + LastUpdateTime: timestamp, + } + sits.statuses[scenarioName] = &newDetail + sits.dirty = true + } + +} + +// InitStatuses creates initial representation all scenarios +// This function also removes scenarios which are not defined in scenarios param +func (sits *SnapshotIntegrationTestStatuses) InitStatuses(scenarios *[]v1beta1.IntegrationTestScenario) { + var expectedScenarios map[string]struct{} = make(map[string]struct{}) // map as a set + + // if given scenario doesn't exist, create it in pending state + for _, s := range *scenarios { + expectedScenarios[s.Name] = struct{}{} + _, ok := sits.statuses[s.Name] + if !ok { + // init test statuses only if they doesn't exist + sits.UpdateTestStatusIfChanged(s.Name, IntegrationTestStatusPending, "Pending") + } + } + + // remove old scenarios which are not defined anymore + for _, detail := range sits.statuses { + _, ok := expectedScenarios[detail.ScenarioName] + if !ok { + sits.DeleteStatus(detail.ScenarioName) + } + } +} + +// DeleteStatus deletes status of the particular scenario +func (sits *SnapshotIntegrationTestStatuses) DeleteStatus(scenarioName string) { + _, ok := sits.statuses[scenarioName] + if ok { + delete(sits.statuses, scenarioName) + sits.dirty = true + } +} + +// GetStatuses returns snapshot test statuses in external format +func (sits *SnapshotIntegrationTestStatuses) GetStatuses() []*IntegrationTestStatusDetail { + // transform map to list of structs + result := make([]*IntegrationTestStatusDetail, 0, len(sits.statuses)) + for _, v := range sits.statuses { + result = append(result, v) + } + return result +} + +// GetScenarioStatus returns detail of status for the requested scenario +// Second return value represents if result was found +func (sits *SnapshotIntegrationTestStatuses) GetScenarioStatus(scenarioName string) (*IntegrationTestStatusDetail, bool) { + detail, ok := sits.statuses[scenarioName] + if !ok { + return nil, false + } + return detail, true +} + +// MarshalJSON converts data to JSON +// Please note that internal representation of data differs from marshalled output +// Example: +// [ +// +// { +// "scenario": "scenario-1", +// "status": "EnvironmentProvisionError", +// "lastUpdateTime": "2023-07-26T16:57:49+02:00", +// "details": "Failed ..." +// } +// +// ] + +func (sits *SnapshotIntegrationTestStatuses) MarshalJSON() ([]byte, error) { + result := sits.GetStatuses() + return json.Marshal(result) +} + +// UnmarshalJSON load data from JSON +func (sits *SnapshotIntegrationTestStatuses) UnmarshalJSON(b []byte) error { + var inputData []*IntegrationTestStatusDetail + + sch, err := jsonschema.CompileString("schema.json", integrationTestStatusesSchema) + if err != nil { + return fmt.Errorf("error while compiling json data for schema validation: %w", err) + } + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return fmt.Errorf("failed to unmarshal json data raw: %w", err) + } + if err = sch.Validate(v); err != nil { + return fmt.Errorf("error validating test status: %w", err) + } + err = json.Unmarshal(b, &inputData) + if err != nil { + return fmt.Errorf("failed to unmarshal json data: %w", err) + } + + // keep data in map for easier manipulation + for _, v := range inputData { + sits.statuses[v.ScenarioName] = v + } + + return nil +} + +// NewSnapshotIntegrationTestStatuses creates empty SnapshotTestStatus struct +func NewSnapshotIntegrationTestStatuses() *SnapshotIntegrationTestStatuses { + sits := SnapshotIntegrationTestStatuses{ + statuses: make(map[string]*IntegrationTestStatusDetail, 1), + dirty: false, + } + return &sits +} + +// NewSnapshotIntegrationTestStatusesFromSnapshot creates new SnapshotTestStatus struct from snapshot annotation +func NewSnapshotIntegrationTestStatusesFromSnapshot(s *applicationapiv1alpha1.Snapshot) (*SnapshotIntegrationTestStatuses, error) { + annotations := map[string]string{} + if s.ObjectMeta.GetAnnotations() != nil { + annotations = s.ObjectMeta.GetAnnotations() + } + sits := NewSnapshotIntegrationTestStatuses() + + statusAnnotation, ok := annotations[SnapshotTestsStatusAnnotation] + if ok { + err := json.Unmarshal([]byte(statusAnnotation), sits) + if err != nil { + return nil, fmt.Errorf("failed to load tests statuses from the scenario annotation: %w", err) + } + } + + return sits, nil +} + +// WriteIntegrationTestStatusesIntoSnapshot writes data to snapshot by updating CR +// Data are written only when new changes are detected +func WriteIntegrationTestStatusesIntoSnapshot(s *applicationapiv1alpha1.Snapshot, sts *SnapshotIntegrationTestStatuses, c client.Client, ctx context.Context) error { + if !sts.IsDirty() { + // No updates were done, we don't need to update snapshot + return nil + } + patch := client.MergeFrom(s.DeepCopy()) + + value, err := json.Marshal(sts) + if err != nil { + return fmt.Errorf("failed to marshal test results into JSON: %w", err) + } + + newAnnotations := map[string]string{ + SnapshotTestsStatusAnnotation: string(value), + } + if err := metadata.AddAnnotations(&s.ObjectMeta, newAnnotations); err != nil { + return fmt.Errorf("failed to add annotations: %w", err) + } + + err = c.Patch(ctx, s, patch) + if err != nil { + // don't return wrapped err, so we can use RetryOnConflict + return err + } + sts.ResetDirty() + return nil +} diff --git a/gitops/snapshot_integration_tests_status_test.go b/gitops/snapshot_integration_tests_status_test.go new file mode 100644 index 000000000..7fdd477c8 --- /dev/null +++ b/gitops/snapshot_integration_tests_status_test.go @@ -0,0 +1,391 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions andF +limitations under the License. +*/ + +package gitops_test + +import ( + "encoding/json" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + applicationapiv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1" + "github.com/redhat-appstudio/integration-service/api/v1beta1" + "github.com/redhat-appstudio/integration-service/gitops" + "github.com/redhat-appstudio/operator-toolkit/metadata" +) + +var _ = Describe("Snapshot integration test statuses", func() { + + Context("TestStatusDetail", func() { + var ( + statusDetailPending gitops.IntegrationTestStatusDetail + ) + + BeforeEach(func() { + statusDetailPending = gitops.IntegrationTestStatusDetail{Status: gitops.IntegrationTestStatusPending} + }) + + Describe("JSON operations", func() { + It("Struct can be transformed to JSON", func() { + jsonData, err := json.Marshal(statusDetailPending) + Expect(err).To(BeNil()) + Expect(jsonData).Should(ContainSubstring("Pending")) + }) + + It("From JSON back to struct", func() { + jsonData, err := json.Marshal(statusDetailPending) + Expect(err).To(BeNil()) + var statusDetailFromJSON gitops.IntegrationTestStatusDetail + err = json.Unmarshal(jsonData, &statusDetailFromJSON) + Expect(err).To(BeNil()) + Expect(statusDetailFromJSON).Should(Equal(statusDetailPending)) + }) + }) + }) + + Context("SnapshotTestsStatus", func() { + const ( + testScenarioName = "test-scenario" + testDetails = "test-details" + namespace = "default" + applicationName = "application-sample" + componentName = "component-sample" + snapshotName = "snapshot-sample" + ) + var ( + sits *gitops.SnapshotIntegrationTestStatuses + integrationTestScenario *v1beta1.IntegrationTestScenario + snapshot *applicationapiv1alpha1.Snapshot + ) + + BeforeEach(func() { + sits = gitops.NewSnapshotIntegrationTestStatuses() + + integrationTestScenario = &v1beta1.IntegrationTestScenario{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-pass", + Namespace: "default", + + Labels: map[string]string{ + "test.appstudio.openshift.io/optional": "false", + }, + }, + Spec: v1beta1.IntegrationTestScenarioSpec{ + Application: "application-sample", + ResolverRef: v1beta1.ResolverRef{ + Resolver: "git", + Params: []v1beta1.ResolverParameter{ + { + Name: "url", + Value: "https://github.com/redhat-appstudio/integration-examples.git", + }, + { + Name: "revision", + Value: "main", + }, + { + Name: "pathInRepo", + Value: "pipelineruns/integration_pipelinerun_pass.yaml", + }, + }, + }, + Environment: v1beta1.TestEnvironment{ + Name: "envname", + Type: "POC", + Configuration: &applicationapiv1alpha1.EnvironmentConfiguration{ + Env: []applicationapiv1alpha1.EnvVarPair{}, + }, + }, + }, + } + + snapshot = &applicationapiv1alpha1.Snapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: snapshotName, + Namespace: namespace, + Labels: map[string]string{ + gitops.SnapshotTypeLabel: gitops.SnapshotComponentType, + gitops.SnapshotComponentLabel: componentName, + gitops.BuildPipelineRunFinishTimeLabel: "1675992257", + }, + }, + Spec: applicationapiv1alpha1.SnapshotSpec{ + Application: applicationName, + Components: []applicationapiv1alpha1.SnapshotComponent{ + { + Name: componentName, + ContainerImage: "quay.io/redhat-appstudio/sample-image:latest", + }, + }, + }, + } + }) + + It("Can add new scenario test status", func() { + Expect(len(sits.GetStatuses())).To(Equal(0)) + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusPending, testDetails) + Expect(len(sits.GetStatuses())).To(Equal(1)) + + detail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + Expect(detail.ScenarioName).To(Equal(testScenarioName)) + Expect(detail.Status).To(Equal(gitops.IntegrationTestStatusPending)) + }) + + It("Can export valid JSON", func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusPending, testDetails) + detail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + + expectedFormatStr := `[ + { + "scenario": "%s", + "status": "Pending", + "lastUpdateTime": "%s", + "details": "%s" + } + ]` + marshaledTime, err := detail.LastUpdateTime.MarshalText() + Expect(err).To(BeNil()) + expectedStr := fmt.Sprintf(expectedFormatStr, testScenarioName, marshaledTime, testDetails) + expected := []byte(expectedStr) + + Expect(json.Marshal(sits)).To(MatchJSON(expected)) + }) + + When("Contains updates to status", func() { + + BeforeEach(func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusPending, testDetails) + }) + + It("Status is marked as dirty", func() { + Expect(sits.IsDirty()).To(BeTrue()) + }) + + It("Status can be reseted as non-dirty", func() { + sits.ResetDirty() + Expect(sits.IsDirty()).To(BeFalse()) + }) + + It("Adding the same update keeps status non-dirty", func() { + sits.ResetDirty() + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusPending, testDetails) + Expect(sits.IsDirty()).To(BeFalse()) + }) + + It("Updating status of scenario is reflected", func() { + oldSt, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + + oldTimestamp := oldSt.LastUpdateTime + + sits.ResetDirty() + // needs different status + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, testDetails) + Expect(sits.IsDirty()).To(BeTrue()) + + newSt, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + Expect(newSt.Status).To(Equal(gitops.IntegrationTestStatusInProgress)) + // timestamp must be updated too + Expect(newSt.LastUpdateTime).NotTo(Equal(oldTimestamp)) + + // no changes to nuber of records + Expect(len(sits.GetStatuses())).To(Equal(1)) + }) + + It("Updating details of scenario is reflected", func() { + newDetails := "_Testing details_" + oldSt, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + + oldTimestamp := oldSt.LastUpdateTime + + sits.ResetDirty() + // needs the same status but different details + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusPending, newDetails) + Expect(sits.IsDirty()).To(BeTrue()) + + newSt, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + Expect(newSt.Details).To(Equal(newDetails)) + // timestamp must be updated too + Expect(newSt.LastUpdateTime).NotTo(Equal(oldTimestamp)) + + // no changes to nuber of records + Expect(len(sits.GetStatuses())).To(Equal(1)) + }) + + It("Scenario can be deleted", func() { + sits.ResetDirty() + sits.DeleteStatus(testScenarioName) + Expect(len(sits.GetStatuses())).To(Equal(0)) + Expect(sits.IsDirty()).To(BeTrue()) + }) + + It("Initialization with empty scneario list will remove data", func() { + sits.ResetDirty() + sits.InitStatuses(&[]v1beta1.IntegrationTestScenario{}) + Expect(len(sits.GetStatuses())).To(Equal(0)) + Expect(sits.IsDirty()).To(BeTrue()) + }) + + }) + + It("Initialization with new test scenario creates pending status", func() { + sits.ResetDirty() + sits.InitStatuses(&[]v1beta1.IntegrationTestScenario{*integrationTestScenario}) + + Expect(sits.IsDirty()).To(BeTrue()) + Expect(len(sits.GetStatuses())).To(Equal(1)) + + statusDetail, ok := sits.GetScenarioStatus(integrationTestScenario.Name) + Expect(ok).To(BeTrue()) + Expect(statusDetail.ScenarioName).To(Equal(integrationTestScenario.Name)) + Expect(statusDetail.Status).To(Equal(gitops.IntegrationTestStatusPending)) + }) + + It("Creates empty statuses when a snaphost doesn't have test status annotation", func() { + statuses, err := gitops.NewSnapshotIntegrationTestStatusesFromSnapshot(snapshot) + Expect(err).To(BeNil()) + Expect(len(statuses.GetStatuses())).To(Equal(0)) + }) + + When("Snapshot contains empty test status annotation", func() { + + BeforeEach(func() { + err := metadata.AddAnnotations( + snapshot, + map[string]string{gitops.SnapshotTestsStatusAnnotation: "[]"}, + ) + Expect(err).To(BeNil()) + }) + + It("Returns empty test statuses", func() { + statuses, err := gitops.NewSnapshotIntegrationTestStatusesFromSnapshot(snapshot) + Expect(err).To(BeNil()) + Expect(len(statuses.GetStatuses())).To(Equal(0)) + }) + }) + + When("Snapshot contains valid test status annotation", func() { + BeforeEach(func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, testDetails) + testAnnotation, err := json.Marshal(sits) + Expect(err).To(BeNil()) + err = metadata.AddAnnotations( + snapshot, + map[string]string{gitops.SnapshotTestsStatusAnnotation: string(testAnnotation)}, + ) + Expect(err).To(BeNil()) + + }) + + It("Returns expected test statuses", func() { + statuses, err := gitops.NewSnapshotIntegrationTestStatusesFromSnapshot(snapshot) + Expect(err).To(BeNil()) + Expect(len(statuses.GetStatuses())).To(Equal(1)) + + statusDetail := statuses.GetStatuses()[0] + Expect(statusDetail.Status).To(Equal(gitops.IntegrationTestStatusInProgress)) + Expect(statusDetail.ScenarioName).To(Equal(testScenarioName)) + Expect(statusDetail.Details).To(Equal(testDetails)) + }) + + }) + + When("Snapshot contains invalid test status annotation", func() { + BeforeEach(func() { + err := metadata.AddAnnotations( + snapshot, + map[string]string{gitops.SnapshotTestsStatusAnnotation: "[{\"invalid\":\"data\"}]"}, + ) + Expect(err).To(BeNil()) + }) + + It("Returns error", func() { + _, err := gitops.NewSnapshotIntegrationTestStatusesFromSnapshot(snapshot) + Expect(err).NotTo(BeNil()) + }) + }) + + When("Snapshot contains invalid JSON test status annotation", func() { + BeforeEach(func() { + err := metadata.AddAnnotations( + snapshot, + map[string]string{gitops.SnapshotTestsStatusAnnotation: "{}"}, + ) + Expect(err).To(BeNil()) + }) + + It("Returns error", func() { + _, err := gitops.NewSnapshotIntegrationTestStatusesFromSnapshot(snapshot) + Expect(err).NotTo(BeNil()) + }) + }) + + Context("Writes data into snapshot", func() { + + // Make sure that snapshot is written to k8s for following tests + BeforeEach(func() { + Expect(k8sClient.Create(ctx, snapshot)).Should(Succeed()) + + Eventually(func() error { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: snapshot.Name, + Namespace: namespace, + }, snapshot) + return err + }, time.Second*10).ShouldNot(HaveOccurred()) + }) + + AfterEach(func() { + err := k8sClient.Delete(ctx, snapshot) + Expect(err == nil || errors.IsNotFound(err)).To(BeTrue()) + }) + + It("Test results are written into snapshot", func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, testDetails) + + err := gitops.WriteIntegrationTestStatusesIntoSnapshot(snapshot, sits, k8sClient, ctx) + Expect(err).To(BeNil()) + Expect(sits.IsDirty()).To(BeFalse()) + + // fetch updated snapshot + Eventually(func() error { + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: snapshot.Name, + Namespace: namespace, + }, snapshot) + return err + }, time.Second*10).ShouldNot(HaveOccurred()) + + statuses, err := gitops.NewSnapshotIntegrationTestStatusesFromSnapshot(snapshot) + Expect(err).To(BeNil()) + Expect(len(statuses.GetStatuses())).To(Equal(1)) + }) + }) + + }) + +}) diff --git a/gitops/snapshot_test.go b/gitops/snapshot_test.go index cce78bb27..9d3f05fef 100644 --- a/gitops/snapshot_test.go +++ b/gitops/snapshot_test.go @@ -370,5 +370,17 @@ var _ = Describe("Gitops functions for managing Snapshots", Ordered, func() { Entry("When status is TestFail", gitops.IntegrationTestStatusTestFail, "TestFail"), Entry("When status is TestPass", gitops.IntegrationTestStatusTestPassed, "TestPassed"), ) + + It("Invalid status to type fails with error", func() { + _, err := gitops.IntegrationTestStatusString("Unknown") + Expect(err).NotTo(BeNil()) + }) + + It("Invalid JSON status to type fails with error", func() { + const unknownJson = "\"Unknown\"" + var controlStatus gitops.IntegrationTestStatus + err := controlStatus.UnmarshalJSON([]byte(unknownJson)) + Expect(err).NotTo(BeNil()) + }) }) }) From 5b53aeebd61085e541a70f72f195a8c04ac96f34 Mon Sep 17 00:00:00 2001 From: Martin Basti Date: Tue, 5 Sep 2023 19:01:56 +0200 Subject: [PATCH 2/2] feat(STONEINTG-523): extend details with start and completion time This commit extends IntegrationTestStatusDetail with: * StartTime: when testing started * CompletionTime: when testing stopped Signed-off-by: Martin Basti --- gitops/snapshot_integration_tests_status.go | 80 ++++++--- .../snapshot_integration_tests_status_test.go | 163 +++++++++++++++++- 2 files changed, 216 insertions(+), 27 deletions(-) diff --git a/gitops/snapshot_integration_tests_status.go b/gitops/snapshot_integration_tests_status.go index caa5938f9..a3585da34 100644 --- a/gitops/snapshot_integration_tests_status.go +++ b/gitops/snapshot_integration_tests_status.go @@ -46,7 +46,13 @@ const integrationTestStatusesSchema = `{ }, "details": { "type": "string" - } + }, + "startTime": { + "type": "string" + }, + "completionTime": { + "type": "string" + } }, "required": ["scenario", "status", "lastUpdateTime"] } @@ -62,6 +68,10 @@ type IntegrationTestStatusDetail struct { LastUpdateTime time.Time `json:"lastUpdateTime"` // The details of reported status Details string `json:"details"` + // Startime when we moved to inProgress + StartTime *time.Time `json:"startTime,omitempty"` // pointer to make omitempty work + // Completion time when test failed or passed + CompletionTime *time.Time `json:"completionTime,omitempty"` // pointer to make omitempty work } // SnapshotIntegrationTestStatuses type handles details about snapshot tests @@ -86,25 +96,49 @@ func (sits *SnapshotIntegrationTestStatuses) ResetDirty() { // UpdateTestStatusIfChanged updates status of scenario test when status or details changed func (sits *SnapshotIntegrationTestStatuses) UpdateTestStatusIfChanged(scenarioName string, status IntegrationTestStatus, details string) { + var detail *IntegrationTestStatusDetail detail, ok := sits.statuses[scenarioName] timestamp := time.Now().UTC() - if ok { - // update only when status or details changed, otherwise it's a no-op - // to preserve timestamps - if detail.Status != status || detail.Details != details { - detail.Status = status - detail.Details = details - detail.LastUpdateTime = timestamp - sits.dirty = true - } - } else { + if !ok { newDetail := IntegrationTestStatusDetail{ ScenarioName: scenarioName, - Status: status, + Status: -1, // undefined, must be udpated within function Details: details, LastUpdateTime: timestamp, } - sits.statuses[scenarioName] = &newDetail + detail = &newDetail + sits.statuses[scenarioName] = detail + sits.dirty = true + } + + // update only when status or details changed, otherwise it's a no-op + // to preserve timestamps + if detail.Status != status { + detail.Status = status + detail.LastUpdateTime = timestamp + sits.dirty = true + + // update start and completion time if needed, only when status changed + switch status { + case IntegrationTestStatusInProgress: + detail.StartTime = ×tamp + // null CompletionTime because testing started again + detail.CompletionTime = nil + case IntegrationTestStatusPending: + // null all timestamps as test is not inProgress neither in final state + detail.StartTime = nil + detail.CompletionTime = nil + case IntegrationTestStatusDeploymentError, + IntegrationTestStatusEnvironmentProvisionError, + IntegrationTestStatusTestFail, + IntegrationTestStatusTestPassed: + detail.CompletionTime = ×tamp + } + } + + if detail.Details != details { + detail.Details = details + detail.LastUpdateTime = timestamp sits.dirty = true } @@ -166,17 +200,17 @@ func (sits *SnapshotIntegrationTestStatuses) GetScenarioStatus(scenarioName stri // MarshalJSON converts data to JSON // Please note that internal representation of data differs from marshalled output // Example: -// [ // -// { -// "scenario": "scenario-1", -// "status": "EnvironmentProvisionError", -// "lastUpdateTime": "2023-07-26T16:57:49+02:00", -// "details": "Failed ..." -// } -// -// ] - +// [ +// { +// "scenario": "scenario-1", +// "status": "EnvironmentProvisionError", +// "lastUpdateTime": "2023-07-26T16:57:49+02:00", +// "details": "Failed ...", +// "startTime": "2023-07-26T14:57:49+02:00", +// "completionTime": "2023-07-26T16:57:49+02:00" +// } +// ] func (sits *SnapshotIntegrationTestStatuses) MarshalJSON() ([]byte, error) { result := sits.GetStatuses() return json.Marshal(result) diff --git a/gitops/snapshot_integration_tests_status_test.go b/gitops/snapshot_integration_tests_status_test.go index 7fdd477c8..09ac3c291 100644 --- a/gitops/snapshot_integration_tests_status_test.go +++ b/gitops/snapshot_integration_tests_status_test.go @@ -151,7 +151,76 @@ var _ = Describe("Snapshot integration test statuses", func() { Expect(detail.Status).To(Equal(gitops.IntegrationTestStatusPending)) }) - It("Can export valid JSON", func() { + DescribeTable("Test expected additons of startTime", + func(st gitops.IntegrationTestStatus, shouldAdd bool) { + sits.UpdateTestStatusIfChanged(testScenarioName, st, testDetails) + detail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + if shouldAdd { + Expect(detail.StartTime).NotTo(BeNil()) + } else { + Expect(detail.StartTime).To(BeNil()) + } + }, + Entry("When status is Pending", gitops.IntegrationTestStatusPending, false), + Entry("When status is InProgress", gitops.IntegrationTestStatusInProgress, true), + Entry("When status is EnvironmentProvisionError", gitops.IntegrationTestStatusEnvironmentProvisionError, false), + Entry("When status is DeploymentError", gitops.IntegrationTestStatusDeploymentError, false), + Entry("When status is TestFail", gitops.IntegrationTestStatusTestFail, false), + Entry("When status is TestPass", gitops.IntegrationTestStatusTestPassed, false), + ) + + DescribeTable("Test expected additons of completionTime", + func(st gitops.IntegrationTestStatus, shouldAdd bool) { + sits.UpdateTestStatusIfChanged(testScenarioName, st, testDetails) + detail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + if shouldAdd { + Expect(detail.CompletionTime).NotTo(BeNil()) + } else { + Expect(detail.CompletionTime).To(BeNil()) + } + }, + Entry("When status is Pending", gitops.IntegrationTestStatusPending, false), + Entry("When status is InProgress", gitops.IntegrationTestStatusInProgress, false), + Entry("When status is EnvironmentProvisionError", gitops.IntegrationTestStatusEnvironmentProvisionError, true), + Entry("When status is DeploymentError", gitops.IntegrationTestStatusDeploymentError, true), + Entry("When status is TestFail", gitops.IntegrationTestStatusTestFail, true), + Entry("When status is TestPass", gitops.IntegrationTestStatusTestPassed, true), + ) + + It("Change back to InProgress updates timestamps accordingly", func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusTestPassed, testDetails) + originalDetail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + Expect(originalDetail.CompletionTime).ToNot(BeNil()) + originalStartTime := originalDetail.StartTime // copy time, it's all in pointers + + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, testDetails) + newDetail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + Expect(originalDetail.CompletionTime).To(BeNil()) + Expect(newDetail.StartTime).NotTo(Equal(originalStartTime)) + }) + + It("not changing status keeps starting time the same", func() { + newDetails := "something important" + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, testDetails) + originalDetail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + originalStartTime := originalDetail.StartTime // copy time, it's all in pointers + originalLastUpdateTime := originalDetail.LastUpdateTime + + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, newDetails) + newDetail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + Expect(newDetail.StartTime).To(Equal(originalStartTime)) + // but details and lastUpdateTimestamp must changed + Expect(newDetail.Details).To(Equal(newDetails)) + Expect(newDetail.LastUpdateTime).NotTo(Equal(originalLastUpdateTime)) + }) + + It("Can export valid JSON without start and completion time (Pending)", func() { sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusPending, testDetails) detail, ok := sits.GetScenarioStatus(testScenarioName) Expect(ok).To(BeTrue()) @@ -172,6 +241,85 @@ var _ = Describe("Snapshot integration test statuses", func() { Expect(json.Marshal(sits)).To(MatchJSON(expected)) }) + It("Can export valid JSON with start time (InProgress)", func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, testDetails) + + detail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + + expectedFormatStr := `[ + { + "scenario": "%s", + "status": "InProgress", + "lastUpdateTime": "%s", + "details": "%s", + "startTime": "%s" + } + ]` + marshaledTime, err := detail.LastUpdateTime.MarshalText() + Expect(err).To(BeNil()) + marshaledStartTime, err := detail.StartTime.MarshalText() + Expect(err).To(BeNil()) + expectedStr := fmt.Sprintf(expectedFormatStr, testScenarioName, marshaledTime, testDetails, marshaledStartTime) + expected := []byte(expectedStr) + + Expect(json.Marshal(sits)).To(MatchJSON(expected)) + }) + + It("Can export valid JSON with completion time (TestFailed)", func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusTestFail, testDetails) + + detail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + + expectedFormatStr := `[ + { + "scenario": "%s", + "status": "TestFail", + "lastUpdateTime": "%s", + "details": "%s", + "completionTime": "%s" + } + ]` + marshaledTime, err := detail.LastUpdateTime.MarshalText() + Expect(err).To(BeNil()) + marshaledCompletionTime, err := detail.CompletionTime.MarshalText() + Expect(err).To(BeNil()) + expectedStr := fmt.Sprintf(expectedFormatStr, testScenarioName, marshaledTime, testDetails, marshaledCompletionTime) + expected := []byte(expectedStr) + + Expect(json.Marshal(sits)).To(MatchJSON(expected)) + }) + + It("Can export valid JSON with startTime and completion time", func() { + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusInProgress, "yolo") + sits.UpdateTestStatusIfChanged(testScenarioName, gitops.IntegrationTestStatusTestFail, testDetails) + + detail, ok := sits.GetScenarioStatus(testScenarioName) + Expect(ok).To(BeTrue()) + + expectedFormatStr := `[ + { + "scenario": "%s", + "status": "TestFail", + "lastUpdateTime": "%s", + "details": "%s", + "startTime": "%s", + "completionTime": "%s" + } + ]` + marshaledTime, err := detail.LastUpdateTime.MarshalText() + Expect(err).To(BeNil()) + marshaledStartTime, err := detail.StartTime.MarshalText() + Expect(err).To(BeNil()) + marshaledCompletionTime, err := detail.CompletionTime.MarshalText() + Expect(err).To(BeNil()) + expectedStr := fmt.Sprintf(expectedFormatStr, testScenarioName, marshaledTime, testDetails, marshaledStartTime, marshaledCompletionTime) + expected := []byte(expectedStr) + + Expect(json.Marshal(sits)).To(MatchJSON(expected)) + }) + When("Contains updates to status", func() { BeforeEach(func() { @@ -373,11 +521,18 @@ var _ = Describe("Snapshot integration test statuses", func() { // fetch updated snapshot Eventually(func() error { - err := k8sClient.Get(ctx, types.NamespacedName{ + if err := k8sClient.Get(ctx, types.NamespacedName{ Name: snapshot.Name, Namespace: namespace, - }, snapshot) - return err + }, snapshot); err != nil { + return err + } + // race condition, sometimes it fetched old object + annotations := snapshot.GetAnnotations() + if _, ok := annotations[gitops.SnapshotTestsStatusAnnotation]; ok != true { + return fmt.Errorf("Snapshot doesn't contain the expected annotation") + } + return nil }, time.Second*10).ShouldNot(HaveOccurred()) statuses, err := gitops.NewSnapshotIntegrationTestStatusesFromSnapshot(snapshot)