From b4dfa2001de7afb7705924c77f100fb43b7266fc Mon Sep 17 00:00:00 2001 From: Chitrang Patel Date: Fri, 20 Oct 2023 16:30:53 -0400 Subject: [PATCH] Flake Test fix: Sort TaskRunResults PR https://github.com/tektoncd/pipeline/pull/7100 introduced a flaky test. When comparing the pipelineRuns and TaskRuns, the results were not sorted. As a result, `cmp.Diff` throws an error sometimes when the order does not match. This PR sorts the TaskRunResults by providing a sort function to `cmp.Diff`. After the fix, running this 20 times consecutively, no flakes were found. --- test/propagated_results_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/propagated_results_test.go b/test/propagated_results_test.go index 6af07542851..57fee1be165 100644 --- a/test/propagated_results_test.go +++ b/test/propagated_results_test.go @@ -35,10 +35,16 @@ import ( func TestPropagatedResults(t *testing.T) { t.Parallel() + // Ignore the Results when comparing the Runs directly. Instead, check the results separately. + // This is because the order of resutls changes and makes the test very flaky. + ignoreTaskRunStatusFields := cmpopts.IgnoreFields(v1.TaskRunStatusFields{}, "Results") + sortTaskRunResults := cmpopts.SortSlices(func(x, y v1.TaskRunResult) bool { return x.Name < y.Name }) + ignorePipelineRunStatusFields := cmpopts.IgnoreFields(v1.PipelineRunStatusFields{}, "Provenance") ignoreTaskRunStatus := cmpopts.IgnoreFields(v1.TaskRunStatusFields{}, "StartTime", "CompletionTime", "Sidecars", "Provenance") requireAlphaFeatureFlag = requireAnyGate(map[string]string{ "enable-api-fields": "alpha"}) + type tests struct { name string pipelineName string @@ -89,6 +95,7 @@ func TestPropagatedResults(t *testing.T) { ignoreStepState, ignoreSAPipelineRunSpec, ignorePipelineRunStatusFields, + ignoreTaskRunStatusFields, ) if d != "" { t.Fatalf(`The resolved spec does not match the expected spec. Here is the diff: %v`, d) @@ -105,10 +112,15 @@ func TestPropagatedResults(t *testing.T) { ignoreContainerStates, ignoreStepState, ignoreSATaskRunSpec, + ignoreTaskRunStatusFields, ) if d != "" { t.Fatalf(`The expected taskrun does not match created taskrun. Here is the diff: %v`, d) } + d = cmp.Diff(tr.Status.TaskRunStatusFields.Results, taskrun.Status.TaskRunStatusFields.Results, sortTaskRunResults) + if d != "" { + t.Fatalf(`The expected TaskRunResults does not what was received. Here is the diff: %v`, d) + } } t.Logf("Successfully finished test %q", td.name) })