From 2f5bff203899ab6570a94215518acab9a063964d Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 10 Apr 2024 18:16:24 -0700 Subject: [PATCH] Wrap the illegal access error with a suggested solution. --- .../spark/shuffle/sort/CometShuffleExternalSorter.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java b/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java index 9fe88ecbd..55f64cc3d 100644 --- a/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java +++ b/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java @@ -431,7 +431,14 @@ class SpillSorter extends SpillWriter { // As we cannot access the address of the internal array in the sorter, so we need to // allocate the array manually and expand the pointer array in the sorter. // We don't want in-memory sorter to allocate memory but the initial size cannot be zero. - this.inMemSorter = new ShuffleInMemorySorter(allocator, 1, true); + try { + this.inMemSorter = new ShuffleInMemorySorter(allocator, 1, true); + } catch (java.lang.IllegalAccessError e) { + throw new java.lang.RuntimeException( + "Error lodaing in memory sorter check class path -- see " + + "https://github.com/apache/arrow-datafusion-comet?tab=readme-ov-file#enable-comet-shuffle", + e); + } sorterArray = allocator.allocateArray(initialSize); this.inMemSorter.expandPointerArray(sorterArray);