enable decimal types

apache · May 29, 2024 · 90aa10c · 90aa10c
1 parent 7464d7c
commit 90aa10c
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 6 deletions.
diff --git a/fuzz-testing/src/main/scala/org/apache/comet/fuzz/Meta.scala b/fuzz-testing/src/main/scala/org/apache/comet/fuzz/Meta.scala
@@ -25,20 +25,20 @@ import org.apache.spark.sql.types.DataTypes
 object Meta {
 
   val dataTypes: Seq[(DataType, Double)] = Seq(
-    (DataTypes.BooleanType, 0.2),
+    (DataTypes.BooleanType, 0.1),
     (DataTypes.ByteType, 0.2),
     (DataTypes.ShortType, 0.2),
     (DataTypes.IntegerType, 0.2),
     (DataTypes.LongType, 0.2),
     (DataTypes.FloatType, 0.2),
     (DataTypes.DoubleType, 0.2),
-    // (DataTypes.createDecimalType(10, 2), 0.2),
+    (DataTypes.createDecimalType(10, 2), 0.2),
     (DataTypes.DateType, 0.2),
     (DataTypes.TimestampType, 0.2),
     // TimestampNTZType only in Spark 3.4+
     // (DataTypes.TimestampNTZType, 0.2),
     (DataTypes.StringType, 0.2),
-    (DataTypes.BinaryType, 0.2))
+    (DataTypes.BinaryType, 0.1))
 
   val stringScalarFunc: Seq[Function] = Seq(
     Function("substring", 3),

diff --git a/fuzz-testing/src/main/scala/org/apache/comet/fuzz/QueryRunner.scala b/fuzz-testing/src/main/scala/org/apache/comet/fuzz/QueryRunner.scala
@@ -23,7 +23,7 @@ import java.io.{BufferedWriter, FileWriter}
 
 import scala.io.Source
 
-import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.{Row, SparkSession}
 
 object QueryRunner {
 
@@ -84,14 +84,15 @@ object QueryRunner {
                       case (a: Double, b: Double) if a.isInfinity => b.isInfinity
                       case (a: Double, b: Double) if a.isNaN => b.isNaN
                       case (a: Double, b: Double) => (a - b).abs <= 0.000001
+                      case (a: Array[Byte], b: Array[Byte]) => a.sameElements(b)
                       case (a, b) => a == b
                     }
                     if (!same) {
                       showSQL(w, sql)
                       showPlans(w, sparkPlan, cometPlan)
                       w.write(s"First difference at row $i:\n")
-                      w.write("Spark: `" + l.mkString(",") + "`\n")
-                      w.write("Comet: `" + r.mkString(",") + "`\n")
+                      w.write("Spark: `" + formatRow(l) + "`\n")
+                      w.write("Comet: `" + formatRow(r) + "`\n")
                       i = sparkRows.length
                     }
                   }
@@ -130,6 +131,15 @@ object QueryRunner {
     }
   }
 
+  private def formatRow(row: Row): String = {
+    row.toSeq
+      .map {
+        case v: Array[Byte] => v.mkString
+        case other => other.toString
+      }
+      .mkString(",")
+  }
+
   private def showSQL(w: BufferedWriter, sql: String, maxLength: Int = 120): Unit = {
     w.write("## SQL\n")
     w.write("```\n")