From 1caa451311330885aea92971ceceb86d9d1e8b35 Mon Sep 17 00:00:00 2001
From: Ben Ye <benye@amazon.com>
Date: Mon, 15 Jan 2024 08:01:29 -0800
Subject: [PATCH] improve query fuzzing tests with better promql result
 comparison (#5724)

Signed-off-by: Ben Ye <benye@amazon.com>
---
 integration/query_fuzz_test.go | 115 ++++++++++++++++++++++++---------
 1 file changed, 83 insertions(+), 32 deletions(-)

diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go
index 8ec081801b..f21768b7cf 100644
--- a/integration/query_fuzz_test.go
+++ b/integration/query_fuzz_test.go
@@ -186,7 +186,7 @@ func TestVerticalShardingFuzz(t *testing.T) {
 				t.Logf("case %d error mismatch.\n%s: %s\nerr1: %v\nerr2: %v\n", i, qt, tc.query, tc.err1, tc.err2)
 				failures++
 			}
-		} else if !sameModelValue(tc.res1, tc.res2) {
+		} else if !cmp.Equal(tc.res1, tc.res2, comparer) {
 			t.Logf("case %d results mismatch.\n%s: %s\nres1: %s\nres2: %s\n", i, qt, tc.query, tc.res1.String(), tc.res2.String())
 			failures++
 		}
@@ -196,37 +196,88 @@ func TestVerticalShardingFuzz(t *testing.T) {
 	}
 }
 
-func sameModelValue(a model.Value, b model.Value) bool {
-	if a.Type() != b.Type() {
+// comparer should be used to compare promql results between engines.
+var comparer = cmp.Comparer(func(x, y model.Value) bool {
+	if x.Type() != y.Type() {
 		return false
 	}
-	// We allow a margin for comparing floats.
-	opts := []cmp.Option{cmpopts.EquateNaNs(), cmpopts.EquateApprox(0, 1e-6)}
-	switch a.Type() {
-	case model.ValMatrix:
-		s1, _ := a.(model.Matrix)
-		s2, _ := b.(model.Matrix)
-		// Sort to make sure we are not affected by series order.
-		sort.Sort(s1)
-		sort.Sort(s2)
-		return cmp.Equal(s1, s2, opts...)
-	case model.ValVector:
-		s1, _ := a.(model.Vector)
-		s2, _ := b.(model.Vector)
-		// Sort to make sure we are not affected by series order.
-		sort.Sort(s1)
-		sort.Sort(s2)
-		return cmp.Equal(s1, s2, opts...)
-	case model.ValScalar:
-		s1, _ := a.(*model.Scalar)
-		s2, _ := b.(*model.Scalar)
-		return cmp.Equal(s1, s2, opts...)
-	case model.ValString:
-		s1, _ := a.(*model.String)
-		s2, _ := b.(*model.String)
-		return cmp.Equal(s1, s2, opts...)
-	default:
-		// model.ValNone is impossible.
-		return false
+	compareFloats := func(l, r float64) bool {
+		const epsilon = 1e-6
+		return cmp.Equal(l, r, cmpopts.EquateNaNs(), cmpopts.EquateApprox(0, epsilon))
 	}
-}
+	compareMetrics := func(l, r model.Metric) bool {
+		return l.Equal(r)
+	}
+
+	vx, xvec := x.(model.Vector)
+	vy, yvec := y.(model.Vector)
+
+	if xvec && yvec {
+		if len(vx) != len(vy) {
+			return false
+		}
+
+		// Sort vector before comparing.
+		sort.Sort(vx)
+		sort.Sort(vy)
+
+		for i := 0; i < len(vx); i++ {
+			if !compareMetrics(vx[i].Metric, vy[i].Metric) {
+				return false
+			}
+			if vx[i].Timestamp != vy[i].Timestamp {
+				return false
+			}
+			if !compareFloats(float64(vx[i].Value), float64(vy[i].Value)) {
+				return false
+			}
+		}
+		return true
+	}
+
+	mx, xmat := x.(model.Matrix)
+	my, ymat := y.(model.Matrix)
+
+	if xmat && ymat {
+		if len(mx) != len(my) {
+			return false
+		}
+		// Sort matrix before comparing.
+		sort.Sort(mx)
+		sort.Sort(my)
+		for i := 0; i < len(mx); i++ {
+			mxs := mx[i]
+			mys := my[i]
+
+			if !compareMetrics(mxs.Metric, mys.Metric) {
+				return false
+			}
+
+			xps := mxs.Values
+			yps := mys.Values
+
+			if len(xps) != len(yps) {
+				return false
+			}
+			for j := 0; j < len(xps); j++ {
+				if xps[j].Timestamp != yps[j].Timestamp {
+					return false
+				}
+				if !compareFloats(float64(xps[j].Value), float64(yps[j].Value)) {
+					return false
+				}
+			}
+		}
+		return true
+	}
+
+	sx, xscalar := x.(*model.Scalar)
+	sy, yscalar := y.(*model.Scalar)
+	if xscalar && yscalar {
+		if sx.Timestamp != sy.Timestamp {
+			return false
+		}
+		return compareFloats(float64(sx.Value), float64(sy.Value))
+	}
+	return false
+})