From 2416e37a83682c0b078692cc5c9109b7cf488f48 Mon Sep 17 00:00:00 2001 From: Karl Isenberg Date: Thu, 19 Dec 2024 11:29:11 -0800 Subject: [PATCH] e2e: Improve rendering status validation (#310) - Test RSync status, instead of using an internal nomos CLI function. - Validate source, rendering, and sync status, not just rendering, to ensure no other errors are being returned. - Validate Syncing status condition too. Change-Id: I127929f45aa672ddf66dc7db354025c5d1a7cd44 --- e2e/nomostest/testpredicates/predicates.go | 23 -- e2e/testcases/hydration_test.go | 354 +++++++++++++++++++-- 2 files changed, 320 insertions(+), 57 deletions(-) diff --git a/e2e/nomostest/testpredicates/predicates.go b/e2e/nomostest/testpredicates/predicates.go index d26b58225..960cd69ad 100644 --- a/e2e/nomostest/testpredicates/predicates.go +++ b/e2e/nomostest/testpredicates/predicates.go @@ -32,7 +32,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - nomosstatus "kpt.dev/configsync/cmd/nomos/status" "kpt.dev/configsync/e2e/nomostest/retry" "kpt.dev/configsync/e2e/nomostest/testkubeclient" "kpt.dev/configsync/e2e/nomostest/testlogger" @@ -1497,28 +1496,6 @@ func ValidateError(errs []v1beta1.ConfigSyncError, code, message string, resourc return fmt.Errorf("error %s not present: %s", code, log.AsJSON(errs)) } -// RootSyncHasNomosStatus returns an error if the RootSync does not have the -// specified commit or status according to `nomos status` (RootRepoStatus). -func RootSyncHasNomosStatus(expectedCommit, expectedStatus string) Predicate { - return func(o client.Object) error { - if o == nil { - return ErrObjectNotFound - } - rs, ok := o.(*v1beta1.RootSync) - if !ok { - return WrongTypeErr(o, &v1beta1.RootSync{}) - } - repoStatus := nomosstatus.RootRepoStatus(rs, nil, false) - commit := repoStatus.GetCommit() - status := repoStatus.GetStatus() - if commit != expectedCommit || status != expectedStatus { - return fmt.Errorf("expected RepoStatus commit %q and status %q, but found %q and %q: error summary: %v", - expectedCommit, expectedStatus, commit, status, repoStatus.GetErrorSummary()) - } - return nil - } -} - // ConfigMapHasData returns an error if the ConfigMap doesn't contain the given key value pair func ConfigMapHasData(key string, value string) Predicate { return func(o client.Object) error { diff --git a/e2e/testcases/hydration_test.go b/e2e/testcases/hydration_test.go index 5bef515c1..00bfc57a7 100644 --- a/e2e/testcases/hydration_test.go +++ b/e2e/testcases/hydration_test.go @@ -18,9 +18,12 @@ import ( "fmt" "testing" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/assert" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "kpt.dev/configsync/e2e/nomostest" "kpt.dev/configsync/e2e/nomostest/gitproviders" "kpt.dev/configsync/e2e/nomostest/kustomizecomponents" @@ -37,8 +40,11 @@ import ( "kpt.dev/configsync/pkg/importer/analyzer/validation/nonhierarchical" "kpt.dev/configsync/pkg/kinds" "kpt.dev/configsync/pkg/metadata" + "kpt.dev/configsync/pkg/parse" "kpt.dev/configsync/pkg/reconcilermanager" + "kpt.dev/configsync/pkg/rootsync" "kpt.dev/configsync/pkg/status" + "sigs.k8s.io/cli-utils/pkg/testutil" ) var expectedBuiltinOrigin = "configuredIn: kustomization.yaml\nconfiguredBy:\n apiVersion: builtin\n kind: HelmChartInflationGenerator\n" @@ -103,51 +109,65 @@ func TestHydrateKustomizeComponents(t *testing.T) { }) nt.Must(tg.Wait()) - // Validate nomos status - latestCommit := rootSyncGitRepo.MustHash(nt.T) - nt.Must(nt.Validate(rootSyncID.Name, rootSyncID.Namespace, &v1beta1.RootSync{}, - testpredicates.RootSyncHasNomosStatus(latestCommit, "SYNCED"))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSyncCompleted(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + parse.RenderingSucceeded) kustomizecomponents.ValidateAllTenants(nt, string(declared.RootScope), "base", "tenant-a", "tenant-b", "tenant-c") nt.T.Log("Remove kustomization.yaml to make the sync fail") nt.Must(rootSyncGitRepo.Remove("./kustomize-components/kustomization.yml")) nt.Must(rootSyncGitRepo.CommitAndPush("remove the Kustomize configuration to make the sync fail")) - latestCommit = rootSyncGitRepo.MustHash(nt.T) + nt.Must(nt.Watcher.WatchObject(kinds.RootSyncV1Beta1(), rootSyncID.Name, rootSyncID.Namespace, testwatcher.WatchPredicates( testpredicates.RootSyncHasRenderingError(status.ActionableHydrationErrorCode, "Kustomization config file is missing from the sync directory"), - testpredicates.RootSyncHasNomosStatus(latestCommit, "ERROR"), ))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncRenderingErrors(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + status.ActionableHydrationErrorCode) + nt.T.Log("Add kustomization.yaml back") nt.Must(rootSyncGitRepo.Copy("../testdata/hydration/kustomize-components/kustomization.yml", "./kustomize-components/kustomization.yml")) nt.Must(rootSyncGitRepo.CommitAndPush("add kustomization.yml back")) nt.Must(nt.WatchForAllSyncs()) - // Validate nomos status - latestCommit = rootSyncGitRepo.MustHash(nt.T) - nt.Must(nt.Validate(rootSyncID.Name, rootSyncID.Namespace, &v1beta1.RootSync{}, - testpredicates.RootSyncHasNomosStatus(latestCommit, "SYNCED"))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSyncCompleted(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + parse.RenderingSucceeded) nt.T.Log("Make kustomization.yaml invalid") nt.Must(rootSyncGitRepo.Copy("../testdata/hydration/invalid-kustomization.yaml", "./kustomize-components/kustomization.yml")) nt.Must(rootSyncGitRepo.CommitAndPush("update kustomization.yaml to make it invalid")) - latestCommit = rootSyncGitRepo.MustHash(nt.T) + nt.Must(nt.Watcher.WatchObject(kinds.RootSyncV1Beta1(), rootSyncID.Name, rootSyncID.Namespace, testwatcher.WatchPredicates( testpredicates.RootSyncHasRenderingError(status.ActionableHydrationErrorCode, "failed to run kustomize build"), - testpredicates.RootSyncHasNomosStatus(latestCommit, "ERROR"), ))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncRenderingErrors(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + status.ActionableHydrationErrorCode) + // one final validation to ensure hydration-controller can be re-disabled nt.T.Log("Remove all dry configs") nt.Must(rootSyncGitRepo.Remove("./kustomize-components")) nt.Must(rootSyncGitRepo.Copy("../testdata/hydration/compiled/kustomize-components", ".")) nt.Must(rootSyncGitRepo.CommitAndPush("Replace dry configs with wet configs")) + nt.Must(nt.WatchForAllSyncs()) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSyncCompleted(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + parse.RenderingSkipped) + nt.T.Log("Verify the hydration-controller is omitted after dry configs were removed") // Dry configs removed from repo, assert that hydration is disabled again tg = taskgroup.New() @@ -194,7 +214,6 @@ func TestHydrateExternalFiles(t *testing.T) { tg.Go(func() error { return nt.Validate("test-configmap", "test-namespace", &corev1.ConfigMap{}, testpredicates.ConfigMapHasData("external-data.txt", "Foo")) }) - tg.Go(func() error { return nt.Validate("test-namespace", "", &corev1.Namespace{}, testpredicates.HasAnnotation(metadata.KustomizeOrigin, "path: namespace.yaml\n")) }) @@ -225,10 +244,10 @@ func TestHydrateHelmComponents(t *testing.T) { nt.Must(nt.WatchForAllSyncs()) - // Validate nomos status - latestCommit := rootSyncGitRepo.MustHash(nt.T) - nt.Must(nt.Validate(rootSyncID.Name, rootSyncID.Namespace, &v1beta1.RootSync{}, - testpredicates.RootSyncHasNomosStatus(latestCommit, "SYNCED"))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSyncCompleted(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + parse.RenderingSucceeded) nt.T.Log("Validate resources are synced") tg := taskgroup.New() @@ -253,15 +272,15 @@ func TestHydrateHelmComponents(t *testing.T) { nt.Must(nt.WatchForAllSyncs()) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSyncCompleted(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + parse.RenderingSucceeded) + nt.Must(nt.Validate("my-coredns-coredns", "coredns", &appsv1.Deployment{}, testpredicates.DeploymentContainerPullPolicyEquals("coredns", "Always"), testpredicates.DeploymentContainerImageEquals("coredns", "coredns/coredns:1.8.4"), testpredicates.HasAnnotation(metadata.KustomizeOrigin, expectedBuiltinOrigin))) - - // Validate nomos status - latestCommit = rootSyncGitRepo.MustHash(nt.T) - nt.Must(nt.Validate(rootSyncID.Name, rootSyncID.Namespace, &v1beta1.RootSync{}, - testpredicates.RootSyncHasNomosStatus(latestCommit, "SYNCED"))) } func TestHydrateHelmOverlay(t *testing.T) { @@ -297,31 +316,39 @@ func TestHydrateHelmOverlay(t *testing.T) { testpredicates.HasLabel("test-case", "hydration"), testpredicates.DeploymentContainerPullPolicyEquals("coredns", "Always"))) - // Validate nomos status - latestCommit := rootSyncGitRepo.MustHash(nt.T) - nt.Must(nt.Validate(rootSyncID.Name, rootSyncID.Namespace, &v1beta1.RootSync{}, - testpredicates.RootSyncHasNomosStatus(latestCommit, "SYNCED"))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSyncCompleted(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + parse.RenderingSucceeded) nt.T.Log("Make the hydration fail by checking in an invalid kustomization.yaml") nt.Must(rootSyncGitRepo.Copy("../testdata/hydration/resource-duplicate/kustomization.yaml", "./helm-overlay/kustomization.yaml")) nt.Must(rootSyncGitRepo.Copy("../testdata/hydration/resource-duplicate/namespace_tenant-a.yaml", "./helm-overlay/namespace_tenant-a.yaml")) nt.Must(rootSyncGitRepo.CommitAndPush("Update kustomization.yaml with duplicated resources")) - latestCommit = rootSyncGitRepo.MustHash(nt.T) + nt.Must(nt.Watcher.WatchObject(kinds.RootSyncV1Beta1(), rootSyncID.Name, rootSyncID.Namespace, testwatcher.WatchPredicates( testpredicates.RootSyncHasRenderingError(status.ActionableHydrationErrorCode, "failed to run kustomize build"), - testpredicates.RootSyncHasNomosStatus(latestCommit, "ERROR"), ))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncRenderingErrors(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + status.ActionableHydrationErrorCode) + nt.T.Log("Make the parsing fail by checking in a deprecated group and kind") nt.Must(rootSyncGitRepo.Copy("../testdata/hydration/deprecated-GK/kustomization.yaml", "./helm-overlay/kustomization.yaml")) nt.Must(rootSyncGitRepo.CommitAndPush("Update kustomization.yaml to render a deprecated group and kind")) - latestCommit = rootSyncGitRepo.MustHash(nt.T) + nt.Must(nt.Watcher.WatchObject(kinds.RootSyncV1Beta1(), rootSyncID.Name, rootSyncID.Namespace, testwatcher.WatchPredicates( testpredicates.RootSyncHasSourceError(nonhierarchical.DeprecatedGroupKindErrorCode, "The config is using a deprecated Group and Kind"), - testpredicates.RootSyncHasNomosStatus(latestCommit, "ERROR"), ))) + + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSourceErrors(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + nonhierarchical.DeprecatedGroupKindErrorCode) } func TestHydrateRemoteResources(t *testing.T) { @@ -420,10 +447,10 @@ func TestHydrateResourcesInRelativePath(t *testing.T) { nt.Must(nt.WatchForAllSyncs()) - // Validate nomos status - latestCommit := rootSyncGitRepo.MustHash(nt.T) - nt.Must(nt.Validate(rootSyncID.Name, rootSyncID.Namespace, &v1beta1.RootSync{}, - testpredicates.RootSyncHasNomosStatus(latestCommit, "SYNCED"))) + rs = getRootSync(nt, rootSyncID.Name, rootSyncID.Namespace) + validateRootSyncSyncCompleted(nt, rs, + rootSyncGitRepo.MustHash(nt.T), + parse.RenderingSucceeded) nt.T.Log("Validating resources are synced") tg := taskgroup.New() @@ -445,3 +472,262 @@ func TestHydrateResourcesInRelativePath(t *testing.T) { }) nt.Must(tg.Wait()) } + +// getRootSync gets the RootSync from the cluster +func getRootSync(nt *nomostest.NT, name string, namespace string) *v1beta1.RootSync { + rs := &v1beta1.RootSync{} + nt.Must(nt.KubeClient.Get(name, namespace, rs)) + return rs +} + +// gitRevisionOrDefault returns the specified Revision or the default value. +func gitRevisionOrDefault(git v1beta1.Git) string { + if git.Revision == "" { + return "HEAD" + } + return git.Revision +} + +func validateRootSyncSyncCompleted(nt *nomostest.NT, rs *v1beta1.RootSync, commit, renderingMessage string) { + nt.T.Helper() + + // Use a custom asserter so we can ignore hard-to-test fields. + // Testing whole structs makes debugging easier by printing the full + // expected and actual values, but it will also print any ignored fields. + asserter := testutil.NewAsserter( + cmpopts.IgnoreFields(v1beta1.SourceStatus{}, "LastUpdate"), + cmpopts.IgnoreFields(v1beta1.RenderingStatus{}, "LastUpdate"), + cmpopts.IgnoreFields(v1beta1.SyncStatus{}, "LastUpdate"), + cmpopts.IgnoreFields(v1beta1.RootSyncCondition{}, "LastUpdateTime", "LastTransitionTime", "Message"), + cmpopts.IgnoreFields(v1beta1.ConfigSyncError{}, "ErrorMessage", "Resources"), + ) + + expectedRootSyncStatus := v1beta1.Status{ + ObservedGeneration: rs.Generation, + Reconciler: core.RootReconcilerName(rs.Name), + LastSyncedCommit: commit, + Source: v1beta1.SourceStatus{ + Git: &v1beta1.GitStatus{ + Repo: rs.Spec.Repo, + Revision: gitRevisionOrDefault(*rs.Spec.Git), + Branch: rs.Spec.Git.Branch, + Dir: rs.Spec.Git.Dir, + }, + // LastUpdate ignored + Commit: commit, + Errors: nil, + ErrorSummary: &v1beta1.ErrorSummary{}, + }, + Rendering: v1beta1.RenderingStatus{ + Git: &v1beta1.GitStatus{ + Repo: rs.Spec.Repo, + Revision: gitRevisionOrDefault(*rs.Spec.Git), + Branch: rs.Spec.Git.Branch, + Dir: rs.Spec.Git.Dir, + }, + // LastUpdate ignored + Message: renderingMessage, // RenderingSucceeded/RenderingSkipped + Commit: commit, + Errors: nil, + ErrorSummary: &v1beta1.ErrorSummary{}, + }, + Sync: v1beta1.SyncStatus{ + Git: &v1beta1.GitStatus{ + Repo: rs.Spec.Repo, + Revision: gitRevisionOrDefault(*rs.Spec.Git), + Branch: rs.Spec.Git.Branch, + Dir: rs.Spec.Git.Dir, + }, + // LastUpdate ignored + Commit: commit, + Errors: nil, + ErrorSummary: &v1beta1.ErrorSummary{}, + }, + } + assertEqual(nt, asserter, expectedRootSyncStatus, rs.Status.Status, + "RootSync .status") + + // Validate Syncing condition fields + rsSyncingCondition := rootsync.GetCondition(rs.Status.Conditions, v1beta1.RootSyncSyncing) + expectedSyncingCondition := &v1beta1.RootSyncCondition{ + Type: v1beta1.RootSyncSyncing, + Status: metav1.ConditionFalse, + // LastUpdateTime ignored + // LastTransitionTime ignored + Reason: "Sync", + Message: "Sync Completed", + Commit: commit, + // Errors unused by the Syncing condition (always nil) + ErrorSourceRefs: nil, + ErrorSummary: &v1beta1.ErrorSummary{}, + } + assertEqual(nt, asserter, expectedSyncingCondition, rsSyncingCondition, + "RootSync .status.conditions[.status=%q]", v1beta1.RootSyncSyncing) + + if nt.T.Failed() { + nt.T.FailNow() + } +} + +func validateRootSyncRenderingErrors(nt *nomostest.NT, rs *v1beta1.RootSync, commit string, errCodes ...string) { + nt.T.Helper() + + if len(errCodes) == 0 { + nt.T.Fatal("Invalid test: expected specific errors to validate, but none were specified") + } + + // Use a custom asserter so we can ignore hard-to-test fields. + // Testing whole structs makes debugging easier by printing the full + // expected and actual values, but it will also print any ignored fields. + asserter := testutil.NewAsserter( + cmpopts.IgnoreFields(v1beta1.RenderingStatus{}, "LastUpdate"), + cmpopts.IgnoreFields(v1beta1.RootSyncCondition{}, "LastUpdateTime", "LastTransitionTime", "Message"), + cmpopts.IgnoreFields(v1beta1.ConfigSyncError{}, "ErrorMessage", "Resources"), + // Ignore the current Syncing condition status. Retry will flip it back to True. + cmpopts.IgnoreFields(v1beta1.RootSyncCondition{}, "Status"), + ) + + // Build untruncated ErrorSummary & fake Errors list from error codes + var errorList []v1beta1.ConfigSyncError + var errorSources []v1beta1.ErrorSource + + errorSources = append(errorSources, v1beta1.RenderingError) + errorSummary := &v1beta1.ErrorSummary{ + TotalCount: len(errCodes), + Truncated: false, + ErrorCountAfterTruncation: len(errCodes), + } + for _, errCode := range errCodes { + errorList = append(errorList, v1beta1.ConfigSyncError{ + Code: errCode, + // ErrorMessage ignored + // Resources ignored + }) + } + + // Validate .status.rendering fields. + expectedRenderingStatus := v1beta1.RenderingStatus{ + Git: &v1beta1.GitStatus{ + Repo: rs.Spec.Repo, + Revision: gitRevisionOrDefault(*rs.Spec.Git), + Branch: rs.Spec.Git.Branch, + Dir: rs.Spec.Git.Dir, + }, + // LastUpdate ignored + Message: parse.RenderingFailed, + Commit: commit, + Errors: errorList, + ErrorSummary: errorSummary, + } + assertEqual(nt, asserter, expectedRenderingStatus, rs.Status.Rendering, + "RootSync .status.rendering") + + // Validate Syncing condition fields + rsSyncingCondition := rootsync.GetCondition(rs.Status.Conditions, v1beta1.RootSyncSyncing) + expectedSyncingCondition := &v1beta1.RootSyncCondition{ + Type: v1beta1.RootSyncSyncing, + // Status ignored + // LastUpdateTime ignored + // LastTransitionTime ignored + Reason: "Rendering", + Message: "Rendering failed", + Commit: commit, + // Errors unused by the Syncing condition (always nil) + ErrorSourceRefs: errorSources, + ErrorSummary: errorSummary, + } + assertEqual(nt, asserter, expectedSyncingCondition, rsSyncingCondition, + "RootSync .status.conditions[.status=%q]", v1beta1.RootSyncSyncing) + + if nt.T.Failed() { + nt.T.FailNow() + } +} + +func validateRootSyncSourceErrors(nt *nomostest.NT, rs *v1beta1.RootSync, commit string, errCodes ...string) { + nt.T.Helper() + + if len(errCodes) == 0 { + nt.T.Fatal("Invalid test: expected specific errors to validate, but none were specified") + } + + // Use a custom asserter so we can ignore hard-to-test fields. + // Testing whole structs makes debugging easier by printing the full + // expected and actual values, but it will also print any ignored fields. + asserter := testutil.NewAsserter( + cmpopts.IgnoreFields(v1beta1.SourceStatus{}, "LastUpdate"), + cmpopts.IgnoreFields(v1beta1.RootSyncCondition{}, "LastUpdateTime", "LastTransitionTime", "Message"), + cmpopts.IgnoreFields(v1beta1.ConfigSyncError{}, "ErrorMessage", "Resources"), + // Ignore the current Syncing condition status. Retries will cause flapping between True & False. + cmpopts.IgnoreFields(v1beta1.RootSyncCondition{}, "Status"), + ) + + // Build untruncated ErrorSummary & fake Errors list from error codes + var errorList []v1beta1.ConfigSyncError + var errorSources []v1beta1.ErrorSource + + errorSources = append(errorSources, v1beta1.SourceError) + errorSummary := &v1beta1.ErrorSummary{ + TotalCount: len(errCodes), + Truncated: false, + ErrorCountAfterTruncation: len(errCodes), + } + for _, errCode := range errCodes { + errorList = append(errorList, v1beta1.ConfigSyncError{ + Code: errCode, + // ErrorMessage ignored + // Resources ignored + }) + } + + // Validate .status.source fields. + expectedSourceStatus := v1beta1.SourceStatus{ + Git: &v1beta1.GitStatus{ + Repo: rs.Spec.Repo, + Revision: gitRevisionOrDefault(*rs.Spec.Git), + Branch: rs.Spec.Git.Branch, + Dir: rs.Spec.Git.Dir, + }, + // LastUpdate ignored + Commit: commit, + Errors: errorList, + ErrorSummary: errorSummary, + } + assertEqual(nt, asserter, expectedSourceStatus, rs.Status.Source, + "RootSync .status.source") + + // Validate Syncing condition fields + rsSyncingCondition := rootsync.GetCondition(rs.Status.Conditions, v1beta1.RootSyncSyncing) + expectedSyncingCondition := &v1beta1.RootSyncCondition{ + Type: v1beta1.RootSyncSyncing, + // Status ignored + // LastUpdateTime ignored + // LastTransitionTime ignored + Reason: "Source", + Message: "Source", + Commit: commit, + // Errors unused by the Syncing condition (always nil) + ErrorSourceRefs: errorSources, + ErrorSummary: errorSummary, + } + assertEqual(nt, asserter, expectedSyncingCondition, rsSyncingCondition, + "RootSync .status.conditions[.status=%q]", v1beta1.RootSyncSyncing) + + if nt.T.Failed() { + nt.T.FailNow() + } +} + +// assertEqual simulates testutil.AssertEqual, but works with the nomostest.NT interface. +func assertEqual(nt *nomostest.NT, asserter *testutil.Asserter, expected, actual interface{}, msgAndArgs ...interface{}) { + nt.T.Helper() + matcher := asserter.EqualMatcher(expected) + match, err := matcher.Match(actual) + if err != nil { + nt.T.Fatalf("errored testing equality: %v", err) + return + } + if !match { + assert.Fail(nt.T, matcher.FailureMessage(actual), msgAndArgs...) + } +}