build: Enable spark-4.0 Spark tests

kazuyukitanimura · Jun 4, 2024 · 49364c0 · 49364c0
1 parent c030079
commit 49364c0
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 17 deletions.
diff --git a/core/src/errors.rs b/core/src/errors.rs
@@ -38,6 +38,7 @@ use jni::sys::{jboolean, jbyte, jchar, jdouble, jfloat, jint, jlong, jobject, js
 
 use crate::execution::operators::ExecutionError;
 use jni::JNIEnv;
+use jni::objects::{GlobalRef, JThrowable};
 use lazy_static::lazy_static;
 use parquet::errors::ParquetError;
 use thiserror::Error;
@@ -156,7 +157,7 @@ pub enum CometError {
     },
 
     #[error("{class}: {msg}")]
-    JavaException { class: String, msg: String },
+    JavaException { class: String, msg: String, throwable: GlobalRef },
 }
 
 pub fn init() {
@@ -366,19 +367,28 @@ pub fn unwrap_or_throw_default<T: JNIDefault>(
     }
 }
 
-fn throw_exception<E: ToException>(env: &mut JNIEnv, error: &E, backtrace: Option<String>) {
+fn throw_exception(env: &mut JNIEnv, error: &CometError, backtrace: Option<String>) {
     // If there isn't already an exception?
     if env.exception_check().is_ok() {
         // ... then throw new exception
-        let exception = error.to_exception();
-        match backtrace {
-            Some(backtrace_string) => env.throw_new(
-                exception.class,
-                to_stacktrace_string(exception.msg, backtrace_string).unwrap(),
-            ),
-            _ => env.throw_new(exception.class, exception.msg),
+        match error {
+            CometError::JavaException { class: _, msg: _, throwable } => {
+                let obj = env.new_local_ref(throwable).unwrap();
+                // let obj = env.alloc_object(JClass::from(obj)).unwrap();
+                env.throw(JThrowable::from(obj))
+            }
+            _ => {
+                let exception = error.to_exception();
+                match backtrace {
+                    Some(backtrace_string) => env.throw_new(
+                        exception.class,
+                        to_stacktrace_string(exception.msg, backtrace_string).unwrap(),
+                    ),
+                    _ => env.throw_new(exception.class, exception.msg),
+                }
+            }
         }
-        .expect("Thrown exception")
+            .expect("Thrown exception")
     }
 }
 

diff --git a/core/src/jvm_bridge/mod.rs b/core/src/jvm_bridge/mod.rs
@@ -382,8 +382,12 @@ pub(crate) fn convert_exception(
     let exception_class_name_str = get_throwable_class_name(env, cache, throwable)?;
     let message_str = get_throwable_message(env, cache, throwable)?;
 
+    // let obj = env.new_local_ref(throwable).unwrap();
+    // let obj = env.alloc_object(JClass::from(obj)).unwrap();
+
     Ok(CometError::JavaException {
         class: exception_class_name_str,
         msg: message_str,
+        throwable: env.new_global_ref(throwable)?,
     })
 }
diff --git a/dev/diffs/4.0.0.diff b/dev/diffs/4.0.0.diff
@@ -1,5 +1,5 @@
 diff --git a/pom.xml b/pom.xml
-index 6bbcf05b59e..59300f94bef 100644
+index 36d12b9b528..4a459825eb2 100644
 --- a/pom.xml
 +++ b/pom.xml
 @@ -147,6 +147,8 @@
@@ -11,7 +11,7 @@ index 6bbcf05b59e..59300f94bef 100644
      <!--
      If you change codahale.metrics.version, you also need to change
      the link to metrics.dropwizard.io in docs/monitoring.md.
-@@ -2844,6 +2846,25 @@
+@@ -2848,6 +2850,25 @@
          <artifactId>arpack</artifactId>
          <version>${netlib.ludovic.dev.version}</version>
        </dependency>
@@ -252,7 +252,7 @@ index f6fd6b501d7..11870c85d82 100644
            spark.range(100).write.saveAsTable(s"$dbName.$table2Name")
 
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
-index f11ad230ec1..a1271bd51cd 100644
+index 760ee802608..db4dc90475e 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
 @@ -40,7 +40,7 @@ import org.apache.spark.sql.connector.FakeV2Provider
@@ -1624,10 +1624,10 @@ index 795e9f46a8d..5306c94a686 100644
          case _ => assert(false, "Can not match ParquetTable in the query.")
        }
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-index 4fb8faa43a3..984fd1a9892 100644
+index 02e1c70cc8c..8b61d0ad2a7 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
 +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetIOSuite.scala
-@@ -1297,7 +1297,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
+@@ -1334,7 +1334,8 @@ class ParquetIOSuite extends QueryTest with ParquetTest with SharedSparkSession
      }
    }
 
@@ -2459,7 +2459,7 @@ index 4b000fff0eb..e867b6c0d9b 100644
        spark.sql(
          """
 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
-index 3f8de93b330..e96c10bf789 100644
+index 3f8de93b330..fc06e81f41a 100644
 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
 +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/test/TestHive.scala
 @@ -53,24 +53,53 @@ object TestHive
@@ -2507,7 +2507,7 @@ index 3f8de93b330..e96c10bf789 100644
 +        val v = System.getenv("ENABLE_COMET")
 +        if (v != null && v.toBoolean) {
 +          conf
-+            .set("spark.sql.extensions", "org.apache.spark.CometSparkSessionExtensions")
++            .set("spark.sql.extensions", "org.apache.comet.CometSparkSessionExtensions")
 +            .set("spark.comet.enabled", "true")
 +
 +          val v = System.getenv("ENABLE_COMET_SCAN_ONLY")