From c98d6905068974f533cbcfea4089baeead1fcea0 Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 26 Apr 2024 08:44:39 -0700 Subject: [PATCH 01/20] Coverage: Add a manual test for DF supporting Spark expressions directly --- doc/spark_builtin_expr_df_coverage.txt | 419 ++++++++++++++++++ .../comet/CometExpressionCoverageSuite.scala | 181 ++++---- 2 files changed, 520 insertions(+), 80 deletions(-) create mode 100644 doc/spark_builtin_expr_df_coverage.txt diff --git a/doc/spark_builtin_expr_df_coverage.txt b/doc/spark_builtin_expr_df_coverage.txt new file mode 100644 index 000000000..6f401f0b5 --- /dev/null +++ b/doc/spark_builtin_expr_df_coverage.txt @@ -0,0 +1,419 @@ ++---------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|name |details | ++---------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|! |{FAILED, [{SELECT ! true;, std_err: SQL error: ParserError("Expected an expression:, found: !")\n}]} | +|% |{PASSED, [{SELECT 2 % 1.8;, OK}]} | +|& |{PASSED, [{SELECT 3 & 5;, OK}]} | +|* |{PASSED, [{SELECT 2 * 3;, OK}]} | +|+ |{PASSED, [{SELECT 1 + 2;, OK}]} | +|- |{PASSED, [{SELECT 2 - 1;, OK}]} | +|/ |{PASSED, [{SELECT 3 / 2;, OK}]} | +|< |{PASSED, [{SELECT 1 < 2;, OK}]} | +|<= |{PASSED, [{SELECT 2 <= 2;, OK}]} | +|<=> |{FAILED, [{SELECT 2 <=> 2;, std_err: This feature is not implemented: Unsupported SQL binary operator Spaceship\n}]} | +|= |{PASSED, [{SELECT 2 = 2;, OK}]} | +|== |{PASSED, [{SELECT 2 == 2;, OK}]} | +|> |{PASSED, [{SELECT 2 > 1;, OK}]} | +|>= |{PASSED, [{SELECT 2 >= 1;, OK}]} | +|^ |{PASSED, [{SELECT 3 ^ 5;, OK}]} | +|abs |{PASSED, [{SELECT abs(-1);, OK}]} | +|acos |{PASSED, [{SELECT acos(1);, OK}]} | +|acosh |{PASSED, [{SELECT acosh(1);, OK}]} | +|add_months |{FAILED, [{SELECT add_months('2016-08-31', 1);, std_err: Error during planning: Invalid function 'add_months'.\nDid you mean 'radians'?\n}]} | +|aes_decrypt |{FAILED, [{SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333');, std_err: Error during planning: Invalid function 'aes_decrypt'.\nDid you mean 'list_except'?\n}]} | +|aes_encrypt |{FAILED, [{SELECT hex(aes_encrypt('Spark', '0000111122223333'));, std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n}]} | +|aggregate |{FAILED, [{SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|and |{PASSED, [{SELECT true and true;, OK}]} | +|any |{FAILED, [{SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'any'.\nDid you mean 'abs'?\n}]} | +|any_value |{FAILED, [{SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'any_value'.\nDid you mean 'LAST_VALUE'?\n}]} | +|approx_count_distinct |{FAILED, [{SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'approx_count_distinct'.\nDid you mean 'APPROX_DISTINCT'?\n}]} | +|approx_percentile |{FAILED, [{SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n}]} | +|array |{FAILED, [{SELECT array(1, 2, 3);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_agg |{PASSED, [{SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col);, OK}]} | +|array_append |{FAILED, [{SELECT array_append(array('b', 'd', 'c', 'a'), 'd');, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | +|array_compact |{FAILED, [{SELECT array_compact(array(1, 2, 3, null));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_contains |{FAILED, [{SELECT array_contains(array(1, 2, 3), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_distinct |{FAILED, [{SELECT array_distinct(array(1, 2, 3, null, 3));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_except |{FAILED, [{SELECT array_except(array(1, 2, 3), array(1, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_insert |{FAILED, [{SELECT array_insert(array(1, 2, 3, 4), 5, 5);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_intersect |{FAILED, [{SELECT array_intersect(array(1, 2, 3), array(1, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_join |{FAILED, [{SELECT array_join(array('hello', 'world'), ' ');, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'hello'")\n}]} | +|array_max |{FAILED, [{SELECT array_max(array(1, 20, null, 3));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_min |{FAILED, [{SELECT array_min(array(1, 20, null, 3));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_position |{FAILED, [{SELECT array_position(array(3, 2, 1), 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 3")\n}]} | +|array_remove |{FAILED, [{SELECT array_remove(array(1, 2, 3, null, 3), 3);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|array_repeat |{PASSED, [{SELECT array_repeat('123', 2);, OK}]} | +|array_size |{FAILED, [{SELECT array_size(array('b', 'd', 'c', 'a'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | +|array_sort |{FAILED, [{SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 5")\n}]} | +|array_union |{FAILED, [{SELECT array_union(array(1, 2, 3), array(1, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|arrays_overlap |{FAILED, [{SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|arrays_zip |{FAILED, [{SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|ascii |{PASSED, [{SELECT ascii('222');, OK}]} | +|asin |{PASSED, [{SELECT asin(0);, OK}]} | +|asinh |{PASSED, [{SELECT asinh(0);, OK}]} | +|assert_true |{FAILED, [{SELECT assert_true(0 < 1);, std_err: Error during planning: Invalid function 'assert_true'.\nDid you mean 'date_trunc'?\n}]} | +|atan |{PASSED, [{SELECT atan(0);, OK}]} | +|atan2 |{PASSED, [{SELECT atan2(0, 0);, OK}]} | +|atanh |{PASSED, [{SELECT atanh(0);, OK}]} | +|avg |{PASSED, [{SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | +|base64 |{FAILED, [{SELECT base64('Spark SQL');, std_err: Error during planning: Invalid function 'base64'.\nDid you mean 'asin'?\n}]} | +|bigint |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|bin |{FAILED, [{SELECT bin(13);, std_err: Error during planning: Invalid function 'bin'.\nDid you mean 'sin'?\n}]} | +|binary |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|bit_and |{PASSED, [{SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col);, OK}]} | +|bit_count |{FAILED, [{SELECT bit_count(0);, std_err: Error during planning: Invalid function 'bit_count'.\nDid you mean 'COUNT'?\n}]} | +|bit_get |{FAILED, [{SELECT bit_get(11, 0);, std_err: Error during planning: Invalid function 'bit_get'.\nDid you mean 'BIT_AND'?\n}]} | +|bit_length |{PASSED, [{SELECT bit_length('Spark SQL');, OK}]} | +|bit_or |{PASSED, [{SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col);, OK}]} | +|bit_xor |{PASSED, [{SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col);, OK}]} | +|bool_and |{PASSED, [{SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col);, OK}]} | +|bool_or |{PASSED, [{SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col);, OK}]} | +|boolean |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|bround |{FAILED, [{SELECT bround(2.5, 0);, std_err: Error during planning: Invalid function 'bround'.\nDid you mean 'round'?\n}]} | +|btrim |{PASSED, [{SELECT btrim(' SparkSQL ');, OK}]} | +|cardinality |{FAILED, [{SELECT cardinality(array('b', 'd', 'c', 'a'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | +|cast |{PASSED, [{SELECT cast('10' as int);, OK}]} | +|cbrt |{PASSED, [{SELECT cbrt(27.0);, OK}]} | +|ceil |{PASSED, [{SELECT ceil(-0.1);, OK}]} | +|ceiling |{FAILED, [{SELECT ceiling(-0.1);, std_err: Error during planning: Invalid function 'ceiling'.\nDid you mean 'ceil'?\n}]} | +|char |{FAILED, [{SELECT char(65);, std_err: Error during planning: Invalid function 'char'.\nDid you mean 'chr'?\n}]} | +|char_length |{PASSED, [{SELECT char_length('Spark SQL ');, OK}]} | +|character_length |{PASSED, [{SELECT character_length('Spark SQL ');, OK}]} | +|chr |{PASSED, [{SELECT chr(65);, OK}]} | +|coalesce |{PASSED, [{SELECT coalesce(NULL, 1, NULL);, OK}]} | +|collect_list |{FAILED, [{SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'collect_list'.\nDid you mean 'make_list'?\n}]} | +|collect_set |{FAILED, [{SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'collect_set'.\nDid you mean 'coalesce'?\n}]} | +|concat |{PASSED, [{SELECT concat('Spark', 'SQL');, OK}]} | +|concat_ws |{PASSED, [{SELECT concat_ws(' ', 'Spark', 'SQL');, OK}]} | +|contains |{FAILED, [{SELECT contains('Spark SQL', 'Spark');, std_err: Error during planning: Invalid function 'contains'.\nDid you mean 'concat'?\n}]} | +|conv |{FAILED, [{SELECT conv('100', 2, 10);, std_err: Error during planning: Invalid function 'conv'.\nDid you mean 'cos'?\n}]} | +|convert_timezone |{FAILED, [{SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00');, std_err: SQL error: ParserError("Expected ), found: '2021-12-06 00:00:00'")\n}]} | +|corr |{PASSED, [{SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2);, OK}]} | +|cos |{PASSED, [{SELECT cos(0);, OK}]} | +|cosh |{PASSED, [{SELECT cosh(0);, OK}]} | +|cot |{PASSED, [{SELECT cot(1);, OK}]} | +|count |{PASSED, [{SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col);, OK}]} | +|count_if |{FAILED, [{SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'count_if'.\nDid you mean 'COUNT'?\n}]} | +|count_min_sketch |{FAILED, [{SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected ), found: d")\n}]} | +|covar_pop |{PASSED, [{SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);, OK}]} | +|covar_samp |{PASSED, [{SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);, OK}]} | +|crc32 |{FAILED, [{SELECT crc32('Spark');, std_err: Error during planning: Invalid function 'crc32'.\nDid you mean 'log2'?\n}]} | +|csc |{FAILED, [{SELECT csc(1);, std_err: Error during planning: Invalid function 'csc'.\nDid you mean 'cosh'?\n}]} | +|cume_dist |{PASSED, [{SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | +|curdate |{FAILED, [{SELECT curdate();, std_err: Error during planning: Invalid function 'curdate'.\nDid you mean 'to_date'?\n}]} | +|current_catalog |{FAILED, [{SELECT current_catalog();, std_err: SQL error: ParserError("Expected end of statement, found: (")\n}]} | +|current_database |{FAILED, [{SELECT current_database();, std_err: Error during planning: Invalid function 'current_database'.\nDid you mean 'current_date'?\n}]} | +|current_date |{PASSED, [{SELECT current_date();, OK}]} | +|current_schema |{FAILED, [{SELECT current_schema();, std_err: Error during planning: Invalid function 'current_schema'.\nDid you mean 'current_date'?\n}]} | +|current_timestamp |{FAILED, [{SELECT current_timestamp();, std_err: Error during planning: Invalid function 'current_timestamp'.\nDid you mean 'current_time'?\n}]} | +|current_timezone |{FAILED, [{SELECT current_timezone();, std_err: Error during planning: Invalid function 'current_timezone'.\nDid you mean 'current_time'?\n}]} | +|current_user |{FAILED, [{SELECT current_user();, std_err: SQL error: ParserError("Expected end of statement, found: (")\n}]} | +|date |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|date_add |{FAILED, [{SELECT date_add('2016-07-30', 1);, std_err: Error during planning: Invalid function 'date_add'.\nDid you mean 'datepart'?\n}]} | +|date_diff |{FAILED, [{SELECT date_diff('2009-07-31', '2009-07-30');, std_err: Error during planning: Invalid function 'date_diff'.\nDid you mean 'date_bin'?\n}]} | +|date_format |{PASSED, [{SELECT date_format('2016-04-08', 'y');, OK}]} | +|date_from_unix_date |{FAILED, [{SELECT date_from_unix_date(1);, std_err: Error during planning: Invalid function 'date_from_unix_date'.\nDid you mean 'from_unixtime'?\n}]} | +|date_part |{PASSED, [{SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456');, OK}]} | +|date_sub |{FAILED, [{SELECT date_sub('2016-07-30', 1);, std_err: Error during planning: Invalid function 'date_sub'.\nDid you mean 'date_bin'?\n}]} | +|date_trunc |{PASSED, [{SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359');, OK}]} | +|dateadd |{FAILED, [{SELECT dateadd('2016-07-30', 1);, std_err: Error during planning: Invalid function 'dateadd'.\nDid you mean 'datepart'?\n}]} | +|datediff |{FAILED, [{SELECT datediff('2009-07-31', '2009-07-30');, std_err: Error during planning: Invalid function 'datediff'.\nDid you mean 'datepart'?\n}]} | +|datepart |{PASSED, [{SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456');, OK}]} | +|day |{FAILED, [{SELECT day('2009-07-30');, std_err: Error during planning: Invalid function 'day'.\nDid you mean 'today'?\n}]} | +|dayofmonth |{FAILED, [{SELECT dayofmonth('2009-07-30');, std_err: Error during planning: Invalid function 'dayofmonth'.\nDid you mean 'date_part'?\n}]} | +|dayofweek |{FAILED, [{SELECT dayofweek('2009-07-30');, std_err: Error during planning: Invalid function 'dayofweek'.\nDid you mean 'lower'?\n}]} | +|dayofyear |{FAILED, [{SELECT dayofyear('2016-04-09');, std_err: Error during planning: Invalid function 'dayofyear'.\nDid you mean 'date_part'?\n}]} | +|decimal |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|decode |{FAILED, [{SELECT decode(encode('abc', 'utf-8'), 'utf-8');, std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n}]} | +|degrees |{PASSED, [{SELECT degrees(3.141592653589793);, OK}]} | +|dense_rank |{FAILED, [{SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'DENSE_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tDENSE_RANK()\n}]} | +|div |{FAILED, [{SELECT 3 div 2;, std_err: SQL error: ParserError("No infix parser for token Word(Word { value: \"div\", quote_style: None, keyword: DIV })")\n}]} | +|double |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|e |{FAILED, [{SELECT e();, std_err: Error during planning: Invalid function 'e'.\nDid you mean 'ln'?\n}]} | +|element_at |{FAILED, [{SELECT element_at(array(1, 2, 3), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|elt |{FAILED, [{SELECT elt(1, 'scala', 'java');, std_err: Error during planning: Invalid function 'elt'.\nDid you mean 'left'?\n}]} | +|encode |{FAILED, [{SELECT encode('abc', 'utf-8');, std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n}]} | +|endswith |{FAILED, [{SELECT endswith('Spark SQL', 'SQL');, std_err: Error during planning: Invalid function 'endswith'.\nDid you mean 'ends_with'?\n}]} | +|equal_null |{FAILED, [{SELECT equal_null(3, 3);, std_err: Error during planning: Invalid function 'equal_null'.\nDid you mean 'ifnull'?\n}]} | +|every |{FAILED, [{SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'every'.\nDid you mean 'reverse'?\n}]} | +|exists |{FAILED, [{SELECT exists(array(1, 2, 3), x -> x % 2 == 0);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n}]} | +|exp |{PASSED, [{SELECT exp(0);, OK}]} | +|explode |{FAILED, [{SELECT explode(array(10, 20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | +|explode_outer |{FAILED, [{SELECT explode_outer(array(10, 20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | +|expm1 |{FAILED, [{SELECT expm1(0);, std_err: Error during planning: Invalid function 'expm1'.\nDid you mean 'exp'?\n}]} | +|extract |{PASSED, [{SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456');, OK}]} | +|factorial |{PASSED, [{SELECT factorial(5);, OK}]} | +|filter |{FAILED, [{SELECT filter(array(1, 2, 3), x -> x % 2 == 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|find_in_set |{PASSED, [{SELECT find_in_set('ab','abc,b,ab,c,def');, OK}]} | +|first |{FAILED, [{SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'first'.\nDid you mean 'digest'?\n}]} | +|first_value |{PASSED, [{SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col);, OK}]} | +|flatten |{FAILED, [{SELECT flatten(array(array(1, 2), array(3, 4)));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n}]} | +|float |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|floor |{PASSED, [{SELECT floor(-0.1);, OK}]} | +|forall |{FAILED, [{SELECT forall(array(1, 2, 3), x -> x % 2 == 0);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|format_number |{FAILED, [{SELECT format_number(12332.123456, 4);, std_err: Error during planning: Invalid function 'format_number'.\nDid you mean 'FIRST_VALUE'?\n}]} | +|format_string |{FAILED, [{SELECT format_string("Hello World %d %s", 100, "days");, std_err: Error during planning: Invalid function 'format_string'.\nDid you mean 'list_to_string'?\n}]} | +|from_csv |{FAILED, [{SELECT from_csv('1, 0.8', 'a INT, b DOUBLE');, std_err: Error during planning: Invalid function 'from_csv'.\nDid you mean 'arrow_cast'?\n}]} | +|from_json |{FAILED, [{SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE');, std_err: Error during planning: Invalid function 'from_json'.\nDid you mean 'array_join'?\n}]} | +|from_unixtime |{FAILED, [{SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss');, std_err: Error during planning: No function matches the given name and argument types 'from_unixtime(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tfrom_unixtime(Int64)\n}]} | +|from_utc_timestamp |{FAILED, [{SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul');, std_err: Error during planning: Invalid function 'from_utc_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | +|get |{FAILED, [{SELECT get(array(1, 2, 3), 0);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|get_json_object |{FAILED, [{SELECT get_json_object('{"a":"b"}', '$.a');, std_err: Error during planning: Invalid function 'get_json_object'.\nDid you mean 'get_field'?\n}]} | +|getbit |{FAILED, [{SELECT getbit(11, 0);, std_err: Error during planning: Invalid function 'getbit'.\nDid you mean 'repeat'?\n}]} | +|greatest |{FAILED, [{SELECT greatest(10, 9, 2, 4, 3);, std_err: Error during planning: Invalid function 'greatest'.\nDid you mean 'repeat'?\n}]} | +|grouping |{FAILED, [{SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name);, [MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#79, spark_grouping_id#78L], [age#75, name#79]\n+- Expand [[age#75, name#76, name#77, 0], [age#75, name#76, null, 1]], [age#75, name#76, name#79, spark_grouping_id#78L]\n +- Project [age#75, name#76, name#76 AS name#77]\n +- SubqueryAlias people\n +- LocalRelation [age#75, name#76]\n}]} | +|grouping_id |{FAILED, [{SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);, [MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#245, height#246, spark_grouping_id#244L], [age#239, name#245, height#246]\n+- Expand [[age#239, name#240, height#241, name#242, height#243, 0], [age#239, name#240, height#241, name#242, null, 1], [age#239, name#240, height#241, null, height#243, 2], [age#239, name#240, height#241, null, null, 3]], [age#239, name#240, height#241, name#245, height#246, spark_grouping_id#244L]\n +- Project [age#239, name#240, height#241, name#240 AS name#242, height#241 AS height#243]\n +- SubqueryAlias people\n +- LocalRelation [age#239, name#240, height#241]\n}]}| +|hash |{FAILED, [{SELECT hash('Spark', array(123), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n}]} | +|hex |{FAILED, [{SELECT hex(17);, std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n}]} | +|histogram_numeric |{FAILED, [{SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'histogram_numeric'.\nDid you mean 'list_remove_n'?\n}]} | +|hour |{FAILED, [{SELECT hour('2009-07-30 12:58:59');, std_err: Error during planning: Invalid function 'hour'.\nDid you mean 'CORR'?\n}]} | +|hypot |{FAILED, [{SELECT hypot(3, 4);, std_err: Error during planning: Invalid function 'hypot'.\nDid you mean 'cot'?\n}]} | +|if |{FAILED, [{SELECT if(1 < 2, 'a', 'b');, std_err: Error during planning: Invalid function 'if'.\nDid you mean 'pi'?\n}]} | +|ifnull |{FAILED, [{SELECT ifnull(NULL, array('2'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n}]} | +|ilike |{FAILED, [{SELECT ilike('Spark', '_Park');, std_err: Error during planning: Invalid function 'ilike'.\nDid you mean 'sinh'?\n}]} | +|in |{PASSED, [{SELECT 1 in(1, 2, 3);, OK}]} | +|initcap |{PASSED, [{SELECT initcap('sPark sql');, OK}]} | +|inline |{FAILED, [{SELECT inline(array(struct(1, 'a'), struct(2, 'b')));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n}]} | +|inline_outer |{FAILED, [{SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b')));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n}]} | +|input_file_block_length |{FAILED, [{SELECT input_file_block_length();, std_err: Error during planning: Invalid function 'input_file_block_length'.\nDid you mean 'octet_length'?\n}]} | +|input_file_block_start |{FAILED, [{SELECT input_file_block_start();, std_err: Error during planning: Invalid function 'input_file_block_start'.\nDid you mean 'list_replace_all'?\n}]} | +|input_file_name |{FAILED, [{SELECT input_file_name();, std_err: Error during planning: Invalid function 'input_file_name'.\nDid you mean 'list_replace'?\n}]} | +|instr |{PASSED, [{SELECT instr('SparkSQL', 'SQL');, OK}]} | +|int |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|isnan |{PASSED, [{SELECT isnan(cast('NaN' as double));, OK}]} | +|isnotnull |{FAILED, [{SELECT isnotnull(1);, std_err: Error during planning: Invalid function 'isnotnull'.\nDid you mean 'ifnull'?\n}]} | +|isnull |{FAILED, [{SELECT isnull(1);, std_err: Error during planning: Invalid function 'isnull'.\nDid you mean 'ifnull'?\n}]} | +|java_method |{FAILED, [{SELECT java_method('java.util.UUID', 'randomUUID');, std_err: Error during planning: Invalid function 'java_method'.\nDid you mean 'make_date'?\n}]} | +|json_array_length |{FAILED, [{SELECT json_array_length('[1,2,3,4]');, std_err: Error during planning: Invalid function 'json_array_length'.\nDid you mean 'array_length'?\n}]} | +|json_object_keys |{FAILED, [{SELECT json_object_keys('{}');, std_err: Error during planning: Invalid function 'json_object_keys'.\nDid you mean 'concat_ws'?\n}]} | +|json_tuple |{FAILED, [{SELECT json_tuple('{"a":1, "b":2}', 'a', 'b');, std_err: Error during planning: Invalid function 'json_tuple'.\nDid you mean 'to_date'?\n}]} | +|kurtosis |{FAILED, [{SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'kurtosis'.\nDid you mean 'rtrim'?\n}]} | +|lag |{PASSED, [{SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | +|last |{FAILED, [{SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'last'.\nDid you mean 'left'?\n}]} | +|last_day |{FAILED, [{SELECT last_day('2009-01-12');, std_err: Error during planning: Invalid function 'last_day'.\nDid you mean 'list_cat'?\n}]} | +|last_value |{PASSED, [{SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col);, OK}]} | +|lcase |{FAILED, [{SELECT lcase('SparkSql');, std_err: Error during planning: Invalid function 'lcase'.\nDid you mean 'lpad'?\n}]} | +|lead |{PASSED, [{SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | +|least |{FAILED, [{SELECT least(10, 9, 2, 4, 3);, std_err: Error during planning: Invalid function 'least'.\nDid you mean 'left'?\n}]} | +|left |{PASSED, [{SELECT left('Spark SQL', 3);, OK}]} | +|len |{FAILED, [{SELECT len('Spark SQL ');, std_err: Error during planning: Invalid function 'len'.\nDid you mean 'ln'?\n}]} | +|length |{PASSED, [{SELECT length('Spark SQL ');, OK}]} | +|levenshtein |{PASSED, [{SELECT levenshtein('kitten', 'sitting');, OK}]} | +|like |{FAILED, [{SELECT like('Spark', '_park');, std_err: Error during planning: Invalid function 'like'.\nDid you mean 'pi'?\n}]} | +|ln |{PASSED, [{SELECT ln(1);, OK}]} | +|localtimestamp |{FAILED, [{SELECT localtimestamp();, std_err: Error during planning: Invalid function 'localtimestamp'.\nDid you mean 'to_timestamp'?\n}]} | +|locate |{FAILED, [{SELECT locate('bar', 'foobarbar');, std_err: Error during planning: Invalid function 'locate'.\nDid you mean 'concat'?\n}]} | +|log |{PASSED, [{SELECT log(10, 100);, OK}]} | +|log10 |{PASSED, [{SELECT log10(10);, OK}]} | +|log1p |{FAILED, [{SELECT log1p(0);, std_err: Error during planning: Invalid function 'log1p'.\nDid you mean 'log10'?\n}]} | +|log2 |{PASSED, [{SELECT log2(2);, OK}]} | +|lower |{PASSED, [{SELECT lower('SparkSql');, OK}]} | +|lpad |{PASSED, [{SELECT lpad('hi', 5, '??');, OK}]} | +|ltrim |{PASSED, [{SELECT ltrim(' SparkSQL ');, OK}]} | +|make_date |{PASSED, [{SELECT make_date(2013, 7, 15);, OK}]} | +|make_dt_interval |{FAILED, [{SELECT make_dt_interval(1, 12, 30, 01.001001);, std_err: Error during planning: Invalid function 'make_dt_interval'.\nDid you mean 'make_date'?\n}]} | +|make_interval |{FAILED, [{SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001);, std_err: Error during planning: Invalid function 'make_interval'.\nDid you mean 'make_array'?\n}]} | +|make_timestamp |{FAILED, [{SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887);, std_err: Error during planning: Invalid function 'make_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | +|make_timestamp_ltz |{FAILED, [{SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887);, std_err: Error during planning: Invalid function 'make_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n}]} | +|make_timestamp_ntz |{FAILED, [{SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887);, std_err: Error during planning: Invalid function 'make_timestamp_ntz'.\nDid you mean 'to_timestamp_nanos'?\n}]} | +|make_ym_interval |{FAILED, [{SELECT make_ym_interval(1, 2);, std_err: Error during planning: Invalid function 'make_ym_interval'.\nDid you mean 'make_date'?\n}]} | +|map |{FAILED, [{SELECT map(1.0, '2', 3.0, '4');, std_err: Error during planning: Invalid function 'map'.\nDid you mean 'MAX'?\n}]} | +|map_concat |{FAILED, [{SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c'));, std_err: Error during planning: Invalid function 'map_concat'.\nDid you mean 'list_concat'?\n}]} | +|map_contains_key |{FAILED, [{SELECT map_contains_key(map(1, 'a', 2, 'b'), 1);, std_err: Error during planning: Invalid function 'map_contains_key'.\nDid you mean 'list_contains'?\n}]} | +|map_entries |{FAILED, [{SELECT map_entries(map(1, 'a', 2, 'b'));, std_err: Error during planning: Invalid function 'map_entries'.\nDid you mean 'make_date'?\n}]} | +|map_filter |{FAILED, [{SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v);, std_err: Error during planning: Invalid function 'map_filter'.\nDid you mean 'make_date'?\n}]} | +|map_from_arrays |{FAILED, [{SELECT map_from_arrays(array(1.0, 3.0), array('2', '4'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1.0")\n}]} | +|map_from_entries |{FAILED, [{SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b')));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n}]} | +|map_keys |{FAILED, [{SELECT map_keys(map(1, 'a', 2, 'b'));, std_err: Error during planning: Invalid function 'map_keys'.\nDid you mean 'make_date'?\n}]} | +|map_values |{FAILED, [{SELECT map_values(map(1, 'a', 2, 'b'));, std_err: Error during planning: Invalid function 'map_values'.\nDid you mean 'LAST_VALUE'?\n}]} | +|map_zip_with |{FAILED, [{SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2));, std_err: Error during planning: Invalid function 'map_zip_with'.\nDid you mean 'ends_with'?\n}]} | +|mask |{FAILED, [{SELECT mask('abcd-EFGH-8765-4321');, std_err: Error during planning: Invalid function 'mask'.\nDid you mean 'MAX'?\n}]} | +|max |{PASSED, [{SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col);, OK}]} | +|max_by |{FAILED, [{SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'max_by'.\nDid you mean 'MAX'?\n}]} | +|md5 |{PASSED, [{SELECT md5('Spark');, OK}]} | +|mean |{PASSED, [{SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | +|median |{PASSED, [{SELECT median(col) FROM VALUES (0), (10) AS tab(col);, OK}]} | +|min |{PASSED, [{SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col);, OK}]} | +|min_by |{FAILED, [{SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'min_by'.\nDid you mean 'MIN'?\n}]} | +|minute |{FAILED, [{SELECT minute('2009-07-30 12:58:59');, std_err: Error during planning: Invalid function 'minute'.\nDid you mean 'instr'?\n}]} | +|mod |{PASSED, [{SELECT 2 % 1.8;, OK}]} | +|mode |{FAILED, [{SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'mode'.\nDid you mean 'md5'?\n}]} | +|monotonically_increasing_id|{FAILED, [{SELECT monotonically_increasing_id();, std_err: Error during planning: Invalid function 'monotonically_increasing_id'.\nDid you mean 'array_prepend'?\n}]} | +|month |{FAILED, [{SELECT month('2016-07-30');, std_err: Error during planning: Invalid function 'month'.\nDid you mean 'sinh'?\n}]} | +|months_between |{FAILED, [{SELECT months_between('1997-02-28 10:30:00', '1996-10-30');, std_err: Error during planning: Invalid function 'months_between'.\nDid you mean 'NTH_VALUE'?\n}]} | +|named_struct |{FAILED, [{SELECT named_struct("a", 1, "b", 2, "c", 3);, std_err: Schema error: No field named a.\n}]} | +|nanvl |{PASSED, [{SELECT nanvl(cast('NaN' as double), 123);, OK}]} | +|negative |{FAILED, [{SELECT negative(1);, std_err: Error during planning: Invalid function 'negative'.\nDid you mean 'nanvl'?\n}]} | +|next_day |{FAILED, [{SELECT next_day('2015-01-14', 'TU');, std_err: Error during planning: Invalid function 'next_day'.\nDid you mean 'today'?\n}]} | +|not |{PASSED, [{SELECT not true;, OK}]} | +|now |{PASSED, [{SELECT now();, OK}]} | +|nth_value |{PASSED, [{SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | +|ntile |{PASSED, [{SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | +|nullif |{PASSED, [{SELECT nullif(2, 2);, OK}]} | +|nvl |{FAILED, [{SELECT nvl(NULL, array('2'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n}]} | +|nvl2 |{PASSED, [{SELECT nvl2(NULL, 2, 1);, OK}]} | +|octet_length |{PASSED, [{SELECT octet_length('Spark SQL');, OK}]} | +|or |{PASSED, [{SELECT true or false;, OK}]} | +|overlay |{PASSED, [{SELECT overlay('Spark SQL' PLACING '_' FROM 6);, OK}]} | +|parse_url |{FAILED, [{SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST');, std_err: Error during planning: Invalid function 'parse_url'.\nDid you mean 'date_part'?\n}]} | +|percent_rank |{FAILED, [{SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'PERCENT_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tPERCENT_RANK()\n}]} | +|percentile |{FAILED, [{SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'percentile'.\nDid you mean 'ceil'?\n}]} | +|percentile_approx |{FAILED, [{SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n}]} | +|pi |{PASSED, [{SELECT pi();, OK}]} | +|pmod |{FAILED, [{SELECT pmod(10, 3);, std_err: Error during planning: Invalid function 'pmod'.\nDid you mean 'pow'?\n}]} | +|posexplode |{FAILED, [{SELECT posexplode(array(10,20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | +|posexplode_outer |{FAILED, [{SELECT posexplode_outer(array(10,20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | +|position |{FAILED, [{SELECT position('bar', 'foobarbar');, std_err: SQL error: ParserError("Position function must include IN keyword")\n}]} | +|positive |{FAILED, [{SELECT positive(1);, std_err: Error during planning: Invalid function 'positive'.\nDid you mean 'position'?\n}]} | +|pow |{PASSED, [{SELECT pow(2, 3);, OK}]} | +|power |{PASSED, [{SELECT power(2, 3);, OK}]} | +|printf |{FAILED, [{SELECT printf("Hello World %d %s", 100, "days");, std_err: Error during planning: Invalid function 'printf'.\nDid you mean 'instr'?\n}]} | +|quarter |{FAILED, [{SELECT quarter('2016-08-31');, std_err: Error during planning: Invalid function 'quarter'.\nDid you mean 'upper'?\n}]} | +|radians |{PASSED, [{SELECT radians(180);, OK}]} | +|raise_error |{FAILED, [{SELECT raise_error('custom error message');, std_err: Error during planning: Invalid function 'raise_error'.\nDid you mean 'iszero'?\n}]} | +|rand |{FAILED, [{SELECT rand();, std_err: Error during planning: Invalid function 'rand'.\nDid you mean 'rpad'?\n}]} | +|randn |{FAILED, [{SELECT randn();, std_err: Error during planning: Invalid function 'randn'.\nDid you mean 'random'?\n}]} | +|random |{PASSED, [{SELECT random();, OK}]} | +|rank |{FAILED, [{SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tRANK()\n}]} | +|reduce |{FAILED, [{SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|reflect |{FAILED, [{SELECT reflect('java.util.UUID', 'randomUUID');, std_err: Error during planning: Invalid function 'reflect'.\nDid you mean 'repeat'?\n}]} | +|regexp |{FAILED, [{SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');, std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n}]} | +|regexp_count |{FAILED, [{SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en');, std_err: Error during planning: Invalid function 'regexp_count'.\nDid you mean 'REGR_COUNT'?\n}]} | +|regexp_extract |{FAILED, [{SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1);, std_err: Error during planning: Invalid function 'regexp_extract'.\nDid you mean 'regexp_match'?\n}]} | +|regexp_extract_all |{FAILED, [{SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1);, std_err: Error during planning: Invalid function 'regexp_extract_all'.\nDid you mean 'regexp_replace'?\n}]} | +|regexp_instr |{FAILED, [{SELECT regexp_instr('user@spark.apache.org', '@[^.]*');, std_err: Error during planning: Invalid function 'regexp_instr'.\nDid you mean 'regexp_match'?\n}]} | +|regexp_like |{FAILED, [{SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');, std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n}]} | +|regexp_replace |{PASSED, [{SELECT regexp_replace('100-200', '(\\d+)', 'num');, OK}]} | +|regexp_substr |{FAILED, [{SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en');, std_err: Error during planning: Invalid function 'regexp_substr'.\nDid you mean 'regexp_like'?\n}]} | +|regr_avgx |{PASSED, [{SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | +|regr_avgy |{PASSED, [{SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | +|regr_count |{PASSED, [{SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | +|regr_intercept |{PASSED, [{SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x);, OK}]} | +|regr_r2 |{PASSED, [{SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | +|regr_slope |{PASSED, [{SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x);, OK}]} | +|regr_sxx |{PASSED, [{SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | +|regr_sxy |{PASSED, [{SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | +|regr_syy |{PASSED, [{SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | +|repeat |{PASSED, [{SELECT repeat('123', 2);, OK}]} | +|replace |{PASSED, [{SELECT replace('ABCabc', 'abc', 'DEF');, OK}]} | +|reverse |{PASSED, [{SELECT reverse('Spark SQL');, OK}]} | +|right |{PASSED, [{SELECT right('Spark SQL', 3);, OK}]} | +|rint |{FAILED, [{SELECT rint(12.3456);, std_err: Error during planning: Invalid function 'rint'.\nDid you mean 'right'?\n}]} | +|rlike |{FAILED, [{SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');, std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n}]} | +|round |{PASSED, [{SELECT round(2.5, 0);, OK}]} | +|row_number |{PASSED, [{SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | +|rpad |{PASSED, [{SELECT rpad('hi', 5, '??');, OK}]} | +|rtrim |{PASSED, [{SELECT rtrim(' SparkSQL ');, OK}]} | +|schema_of_csv |{FAILED, [{SELECT schema_of_csv('1,abc');, std_err: Error during planning: Invalid function 'schema_of_csv'.\nDid you mean 'concat_ws'?\n}]} | +|schema_of_json |{FAILED, [{SELECT schema_of_json('[{"col":0}]');, std_err: Error during planning: Invalid function 'schema_of_json'.\nDid you mean 'concat_ws'?\n}]} | +|sec |{FAILED, [{SELECT sec(0);, std_err: Error during planning: Invalid function 'sec'.\nDid you mean 'sin'?\n}]} | +|second |{FAILED, [{SELECT second('2009-07-30 12:58:59');, std_err: Error during planning: Invalid function 'second'.\nDid you mean 'decode'?\n}]} | +|sentences |{FAILED, [{SELECT sentences('Hi there! Good morning.');, std_err: Error during planning: Invalid function 'sentences'.\nDid you mean 'degrees'?\n}]} | +|sequence |{FAILED, [{SELECT sequence(1, 5);, std_err: Error during planning: Invalid function 'sequence'.\nDid you mean 'reverse'?\n}]} | +|sha |{FAILED, [{SELECT sha('Spark');, std_err: Error during planning: Invalid function 'sha'.\nDid you mean 'chr'?\n}]} | +|sha1 |{FAILED, [{SELECT sha1('Spark');, std_err: Error during planning: Invalid function 'sha1'.\nDid you mean 'sha512'?\n}]} | +|sha2 |{FAILED, [{SELECT sha2('Spark', 256);, std_err: Error during planning: Invalid function 'sha2'.\nDid you mean 'sha512'?\n}]} | +|shiftleft |{FAILED, [{SELECT shiftleft(2, 1);, std_err: Error during planning: Invalid function 'shiftleft'.\nDid you mean 'left'?\n}]} | +|shiftright |{FAILED, [{SELECT shiftright(4, 1);, std_err: Error during planning: Invalid function 'shiftright'.\nDid you mean 'right'?\n}]} | +|shiftrightunsigned |{FAILED, [{SELECT shiftrightunsigned(4, 1);, std_err: Error during planning: Invalid function 'shiftrightunsigned'.\nDid you mean 'substring_index'?\n}]} | +|shuffle |{FAILED, [{SELECT shuffle(array(1, 20, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|sign |{FAILED, [{SELECT sign(40);, std_err: Error during planning: Invalid function 'sign'.\nDid you mean 'sin'?\n}]} | +|signum |{PASSED, [{SELECT signum(40);, OK}]} | +|sin |{PASSED, [{SELECT sin(0);, OK}]} | +|sinh |{PASSED, [{SELECT sinh(0);, OK}]} | +|size |{FAILED, [{SELECT size(array('b', 'd', 'c', 'a'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | +|skewness |{FAILED, [{SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'skewness'.\nDid you mean 'degrees'?\n}]} | +|slice |{FAILED, [{SELECT slice(array(1, 2, 3, 4), 2, 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|smallint |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|some |{FAILED, [{SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'some'.\nDid you mean 'SUM'?\n}]} | +|sort_array |{FAILED, [{SELECT sort_array(array('b', 'd', null, 'c', 'a'), true);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | +|soundex |{FAILED, [{SELECT soundex('Miller');, std_err: Error during planning: Invalid function 'soundex'.\nDid you mean 'round'?\n}]} | +|space |{FAILED, [{SELECT concat(space(2), '1');, std_err: Error during planning: Invalid function 'space'.\nDid you mean 'lpad'?\n}]} | +|spark_partition_id |{FAILED, [{SELECT spark_partition_id();, std_err: Error during planning: Invalid function 'spark_partition_id'.\nDid you mean 'list_position'?\n}]} | +|split |{FAILED, [{SELECT split('oneAtwoBthreeC', '[ABC]');, std_err: Error during planning: Invalid function 'split'.\nDid you mean 'pi'?\n}]} | +|split_part |{PASSED, [{SELECT split_part('11.12.13', '.', 3);, OK}]} | +|sqrt |{PASSED, [{SELECT sqrt(4);, OK}]} | +|stack |{FAILED, [{SELECT stack(2, 1, 2, 3);, std_err: Error during planning: Invalid function 'stack'.\nDid you mean 'tan'?\n}]} | +|startswith |{FAILED, [{SELECT startswith('Spark SQL', 'Spark');, std_err: Error during planning: Invalid function 'startswith'.\nDid you mean 'starts_with'?\n}]} | +|std |{FAILED, [{SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'std'.\nDid you mean 'sin'?\n}]} | +|stddev |{PASSED, [{SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | +|stddev_pop |{PASSED, [{SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | +|stddev_samp |{PASSED, [{SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | +|str_to_map |{FAILED, [{SELECT str_to_map('a:1,b:2,c:3', ',', ':');, std_err: Error during planning: Invalid function 'str_to_map'.\nDid you mean 'date_format'?\n}]} | +|string |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|struct |{PASSED, [{SELECT struct(1, 2, 3);, OK}]} | +|substr |{PASSED, [{SELECT substr('Spark SQL', 5);, OK}]} | +|substring |{FAILED, [{SELECT substring('Spark SQL', 5);, std_err: This feature is not implemented: Unsupported ast node in sqltorel: Substring { expr: Value(SingleQuotedString("Spark SQL")), substring_from: Some(Value(Number("5", false))), substring_for: None, special: true }\n}]} | +|substring_index |{PASSED, [{SELECT substring_index('www.apache.org', '.', 2);, OK}]} | +|sum |{PASSED, [{SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col);, OK}]} | +|tan |{PASSED, [{SELECT tan(0);, OK}]} | +|tanh |{PASSED, [{SELECT tanh(0);, OK}]} | +|timestamp |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|timestamp_micros |{FAILED, [{SELECT timestamp_micros(1230219000123123);, std_err: Error during planning: Invalid function 'timestamp_micros'.\nDid you mean 'to_timestamp_micros'?\n}]} | +|timestamp_millis |{FAILED, [{SELECT timestamp_millis(1230219000123);, std_err: Error during planning: Invalid function 'timestamp_millis'.\nDid you mean 'to_timestamp_millis'?\n}]} | +|timestamp_seconds |{FAILED, [{SELECT timestamp_seconds(1230219000);, std_err: Error during planning: Invalid function 'timestamp_seconds'.\nDid you mean 'to_timestamp_seconds'?\n}]} | +|tinyint |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | +|to_binary |{FAILED, [{SELECT to_binary('abc', 'utf-8');, std_err: Error during planning: Invalid function 'to_binary'.\nDid you mean 'to_char'?\n}]} | +|to_char |{FAILED, [{SELECT to_char(454, '999');, std_err: Error during planning: No function matches the given name and argument types 'to_char(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tto_char(Date32, Utf8)\n\tto_char(Date64, Utf8)\n\tto_char(Time32(Millisecond), Utf8)\n\tto_char(Time32(Second), Utf8)\n\tto_char(Time64(Microsecond), Utf8)\n\tto_char(Time64(Nanosecond), Utf8)\n\tto_char(Timestamp(Second, None), Utf8)\n\tto_char(Timestamp(Second, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Millisecond, None), Utf8)\n\tto_char(Timestamp(Millisecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Microsecond, None), Utf8)\n\tto_char(Timestamp(Microsecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Nanosecond, None), Utf8)\n\tto_char(Timestamp(Nanosecond, Some("+TZ")), Utf8)\n\tto_char(Duration(Second), Utf8)\n\tto_char(Duration(Millisecond), Utf8)\n\tto_char(Duration(Microsecond), Utf8)\n\tto_char(Duration(Nanosecond), Utf8)\n}]} | +|to_csv |{FAILED, [{SELECT to_csv(named_struct('a', 1, 'b', 2));, std_err: Error during planning: Invalid function 'to_csv'.\nDid you mean 'to_char'?\n}]} | +|to_date |{PASSED, [{SELECT to_date('2009-07-30 04:17:52');, OK}]} | +|to_json |{FAILED, [{SELECT to_json(named_struct('a', 1, 'b', 2));, std_err: Error during planning: Invalid function 'to_json'.\nDid you mean 'to_date'?\n}]} | +|to_number |{FAILED, [{SELECT to_number('454', '999');, std_err: Error during planning: Invalid function 'to_number'.\nDid you mean 'to_date'?\n}]} | +|to_timestamp |{PASSED, [{SELECT to_timestamp('2016-12-31 00:12:00');, OK}]} | +|to_timestamp_ltz |{FAILED, [{SELECT to_timestamp_ltz('2016-12-31 00:12:00');, std_err: Error during planning: Invalid function 'to_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n}]} | +|to_timestamp_ntz |{FAILED, [{SELECT to_timestamp_ntz('2016-12-31 00:12:00');, std_err: Error during planning: Invalid function 'to_timestamp_ntz'.\nDid you mean 'to_timestamp_nanos'?\n}]} | +|to_unix_timestamp |{FAILED, [{SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd');, std_err: Error during planning: Invalid function 'to_unix_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | +|to_utc_timestamp |{FAILED, [{SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul');, std_err: Error during planning: Invalid function 'to_utc_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | +|transform |{FAILED, [{SELECT transform(array(1, 2, 3), x -> x + 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|transform_keys |{FAILED, [{SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|transform_values |{FAILED, [{SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|translate |{PASSED, [{SELECT translate('AaBbCc', 'abc', '123');, OK}]} | +|trim |{PASSED, [{SELECT trim(' SparkSQL ');, OK}]} | +|trunc |{FAILED, [{SELECT trunc('2019-08-04', 'week');, std_err: Error during planning: No function matches the given name and argument types 'trunc(Utf8, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\ttrunc(Float32, Int64)\n\ttrunc(Float64, Int64)\n\ttrunc(Float64)\n\ttrunc(Float32)\n}]} | +|try_add |{FAILED, [{SELECT try_add(1, 2);, std_err: Error during planning: Invalid function 'try_add'.\nDid you mean 'rpad'?\n}]} | +|try_avg |{FAILED, [{SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'try_avg'.\nDid you mean 'AVG'?\n}]} | +|try_divide |{FAILED, [{SELECT try_divide(3, 2);, std_err: Error during planning: Invalid function 'try_divide'.\nDid you mean 'to_date'?\n}]} | +|try_element_at |{FAILED, [{SELECT try_element_at(array(1, 2, 3), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|try_multiply |{FAILED, [{SELECT try_multiply(2, 3);, std_err: Error during planning: Invalid function 'try_multiply'.\nDid you mean 'trunc'?\n}]} | +|try_subtract |{FAILED, [{SELECT try_subtract(2, 1);, std_err: Error during planning: Invalid function 'try_subtract'.\nDid you mean 'array_extract'?\n}]} | +|try_sum |{FAILED, [{SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'try_sum'.\nDid you mean 'trim'?\n}]} | +|try_to_binary |{FAILED, [{SELECT try_to_binary('abc', 'utf-8');, std_err: Error during planning: Invalid function 'try_to_binary'.\nDid you mean 'date_bin'?\n}]} | +|try_to_number |{FAILED, [{SELECT try_to_number('454', '999');, std_err: Error during planning: Invalid function 'try_to_number'.\nDid you mean 'array_to_string'?\n}]} | +|try_to_timestamp |{FAILED, [{SELECT try_to_timestamp('2016-12-31 00:12:00');, std_err: Error during planning: Invalid function 'try_to_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | +|typeof |{FAILED, [{SELECT typeof(1);, std_err: Error during planning: Invalid function 'typeof'.\nDid you mean 'repeat'?\n}]} | +|ucase |{FAILED, [{SELECT ucase('SparkSql');, std_err: Error during planning: Invalid function 'ucase'.\nDid you mean 'acos'?\n}]} | +|unbase64 |{FAILED, [{SELECT unbase64('U3BhcmsgU1FM');, std_err: Error during planning: Invalid function 'unbase64'.\nDid you mean 'asin'?\n}]} | +|unhex |{FAILED, [{SELECT decode(unhex('537061726B2053514C'), 'UTF-8');, std_err: Error during planning: Invalid function 'unhex'.\nDid you mean 'upper'?\n}]} | +|unix_date |{FAILED, [{SELECT unix_date(DATE("1970-01-02"));, std_err: Error during planning: Invalid function 'unix_date'.\nDid you mean 'to_date'?\n}]} | +|unix_micros |{FAILED, [{SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z'));, std_err: Error during planning: Invalid function 'unix_micros'.\nDid you mean 'iszero'?\n}]} | +|unix_millis |{FAILED, [{SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z'));, std_err: Error during planning: Invalid function 'unix_millis'.\nDid you mean 'nullif'?\n}]} | +|unix_seconds |{FAILED, [{SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'));, std_err: Error during planning: Invalid function 'unix_seconds'.\nDid you mean 'encode'?\n}]} | +|unix_timestamp |{FAILED, [{SELECT unix_timestamp();, std_err: Error during planning: Invalid function 'unix_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | +|upper |{PASSED, [{SELECT upper('SparkSql');, OK}]} | +|url_decode |{FAILED, [{SELECT url_decode('https%3A%2F%2Fspark.apache.org');, std_err: Error during planning: Invalid function 'url_decode'.\nDid you mean 'decode'?\n}]} | +|url_encode |{FAILED, [{SELECT url_encode('https://spark.apache.org');, std_err: Error during planning: Invalid function 'url_encode'.\nDid you mean 'encode'?\n}]} | +|user |{FAILED, [{SELECT user();, std_err: SQL error: ParserError("Expected end of statement, found: (")\n}]} | +|uuid |{PASSED, [{SELECT uuid();, OK}]} | +|var_pop |{PASSED, [{SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | +|var_samp |{PASSED, [{SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | +|variance |{FAILED, [{SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'variance'.\nDid you mean 'range'?\n}]} | +|version |{FAILED, [{SELECT version();, std_err: Error during planning: Invalid function 'version'.\nDid you mean 'asin'?\n}]} | +|weekday |{FAILED, [{SELECT weekday('2009-07-30');, std_err: Error during planning: Invalid function 'weekday'.\nDid you mean 'overlay'?\n}]} | +|weekofyear |{FAILED, [{SELECT weekofyear('2008-02-20');, std_err: Error during planning: Invalid function 'weekofyear'.\nDid you mean 'to_char'?\n}]} | +|when |{PASSED, [{SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;, OK}]} | +|width_bucket |{FAILED, [{SELECT width_bucket(5.3, 0.2, 10.6, 5);, std_err: Error during planning: Invalid function 'width_bucket'.\nDid you mean 'list_except'?\n}]} | +|xpath |{FAILED, [{SELECT xpath('b1b2b3c1c2','a/b/text()');, std_err: Error during planning: Invalid function 'xpath'.\nDid you mean 'rpad'?\n}]} | +|xpath_boolean |{FAILED, [{SELECT xpath_boolean('1','a/b');, std_err: Error during planning: Invalid function 'xpath_boolean'.\nDid you mean 'date_format'?\n}]} | +|xpath_double |{FAILED, [{SELECT xpath_double('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_double'.\nDid you mean 'date_trunc'?\n}]} | +|xpath_float |{FAILED, [{SELECT xpath_float('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_float'.\nDid you mean 'date_format'?\n}]} | +|xpath_int |{FAILED, [{SELECT xpath_int('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_int'.\nDid you mean 'date_bin'?\n}]} | +|xpath_long |{FAILED, [{SELECT xpath_long('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_long'.\nDid you mean 'date_bin'?\n}]} | +|xpath_number |{FAILED, [{SELECT xpath_number('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_number'.\nDid you mean 'date_bin'?\n}]} | +|xpath_short |{FAILED, [{SELECT xpath_short('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_short'.\nDid you mean 'list_sort'?\n}]} | +|xpath_string |{FAILED, [{SELECT xpath_string('bcc','a/c');, std_err: Error during planning: Invalid function 'xpath_string'.\nDid you mean 'array_to_string'?\n}]} | +|xxhash64 |{FAILED, [{SELECT xxhash64('Spark', array(123), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n}]} | +|year |{FAILED, [{SELECT year('2016-07-30');, std_err: Error during planning: Invalid function 'year'.\nDid you mean 'VAR'?\n}]} | +|zip_with |{FAILED, [{SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | +|| |{PASSED, [{SELECT 3 | 5;, OK}]} | +|~ |{FAILED, [{SELECT ~ 0;, std_err: SQL error: ParserError("Expected an expression:, found: ~")\n}]} | ++---------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 26c9c8fec..a3b38e6f9 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -32,21 +32,20 @@ import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.comet.CoverageResultStatus.CoverageResultStatus - /** * Manual test to calculate Spark builtin expressions coverage support by the Comet * - * The test will update files docs/spark_builtin_expr_coverage.txt + * The test will update files doc/spark_builtin_expr_coverage.txt, + * doc/spark_builtin_expr_coverage_agg.txt */ @Ignore class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ - private val projectDocFolder = "docs" - private val rawCoverageFilePath = s"$projectDocFolder/spark_builtin_expr_coverage.txt" - private val DATAFUSIONCLI_PATH_ENV_VAR = "DATAFUSIONCLI_PATH" + private val rawCoverageFilePath = "doc/spark_builtin_expr_coverage.txt" + private val aggCoverageFilePath = "doc/spark_builtin_expr_coverage_agg.txt" + private val rawCoverageFileDatafusionPath = "doc/spark_builtin_expr_df_coverage.txt" private val queryPattern = """(?i)SELECT (.+?);""".r private val valuesPattern = """(?i)FROM VALUES(.+?);""".r @@ -81,7 +80,6 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH builtinExamplesMap.foreach { case (funcName, q :: _) => - var dfMessage: Option[String] = None val queryResult = try { // Example with predefined values @@ -93,13 +91,8 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH val values = valuesPattern.findFirstMatchIn(q).map(_.group(0)) (select, values) match { case (Some(s), Some(v)) => - withTempDir { dir => - val path = new Path(dir.toURI.toString).toUri.getPath - spark.sql(s"select * $v").repartition(1).write.mode("overwrite").parquet(path) - dfMessage = runDatafusionCli(s"""$s '$path/*.parquet'""") - } - testSingleLineQuery(s"select * $v", s"$s tbl") + case _ => sys.error("Cannot parse properly") } @@ -111,108 +104,143 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH // // ConstantFolding is a operator optimization rule in Catalyst that replaces expressions // that can be statically evaluated with their equivalent literal values. - dfMessage = runDatafusionCli(q) testSingleLineQuery( "select 'dummy' x", s"${q.dropRight(1)}, x from tbl", excludedOptimizerRules = Some("org.apache.spark.sql.catalyst.optimizer.ConstantFolding")) } - CoverageResult( - q, - CoverageResultStatus.Passed, - CoverageResultDetails( - cometMessage = "OK", - datafusionMessage = dfMessage.getOrElse("OK"))) - + CoverageResult(CoverageResultStatus.Passed.toString, Seq((q, "OK"))) } catch { case e: TestFailedException - if e.getMessage.contains("Expected only Comet native operators") => - CoverageResult( - q, - CoverageResultStatus.Failed, - CoverageResultDetails( - cometMessage = - "Unsupported: Expected only Comet native operators but found Spark fallback", - datafusionMessage = dfMessage.getOrElse(""))) - + if e.message.getOrElse("").contains("Expected only Comet native operators") => + CoverageResult(CoverageResultStatus.Failed.toString, Seq((q, "Unsupported"))) case e if e.getMessage.contains("CometNativeException") => CoverageResult( - q, - CoverageResultStatus.Failed, - CoverageResultDetails( - cometMessage = "Failed on native side: found CometNativeException", - datafusionMessage = dfMessage.getOrElse(""))) - - case e => + CoverageResultStatus.Failed.toString, + Seq((q, "Failed on native side"))) + case _ => CoverageResult( - q, - CoverageResultStatus.Failed, - CoverageResultDetails( - cometMessage = e.getMessage, - datafusionMessage = dfMessage.getOrElse(""))) + CoverageResultStatus.Failed.toString, + Seq((q, "Failed on something else. Check query manually"))) } resultsMap.put(funcName, queryResult) - // Function with no examples case (funcName, List()) => resultsMap.put( funcName, CoverageResult( - "", - CoverageResultStatus.Skipped, - CoverageResultDetails( - cometMessage = "No examples found in spark.sessionState.functionRegistry", - datafusionMessage = ""))) + CoverageResultStatus.Skipped.toString, + Seq(("", "No examples found in spark.sessionState.functionRegistry")))) } // TODO: convert results into HTML or .md file resultsMap.toSeq.toDF("name", "details").createOrReplaceTempView("t") - - val str = showString( + val str_agg = showString( spark.sql( - "select name, details.query, details.result, details.details.cometMessage, details.details.datafusionMessage from t order by 1"), + "select result, d._2 as details, count(1) cnt from (select name, t.details.result, explode_outer(t.details.details) as d from t) group by 1, 2 order by 1"), 1000, 0) + Files.write(Paths.get(aggCoverageFilePath), str_agg.getBytes(StandardCharsets.UTF_8)) + + val str = showString(spark.sql("select * from t order by 1"), 1000, 0) Files.write(Paths.get(rawCoverageFilePath), str.getBytes(StandardCharsets.UTF_8)) } - // Returns execution error, None means successful execution - private def runDatafusionCli(sql: String): Option[String] = { + /** + * Manual test to calculate Spark builtin expressions coverage support by the Datafusion + * directly + * + * The test will update files doc/spark_builtin_expr_df_coverage.txt, + * + * Requires to set DATAFUSIONCLI_PATH env variable to valid path to datafusion-cli + */ + test("Test Spark builtin expressions coverage by Datafusion directly") { + val builtinExamplesMap = getExamples() + + // key - function name + // value - list of result shows if function supported by Datafusion + val resultsMap = new mutable.HashMap[String, CoverageResult]() + + builtinExamplesMap.foreach { + case (funcName, q :: _) => + val queryResult = + try { + // Example with predefined values + // e.g. SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col) + if (q.toLowerCase.contains(" from values")) { + val select = selectPattern.findFirstMatchIn(q).map(_.group(0)) + val values = valuesPattern.findFirstMatchIn(q).map(_.group(0)) + (select, values) match { + case (Some(s), Some(v)) => + withTempDir { dir => + val path = new Path(dir.toURI.toString).toUri.getPath + spark.sql(s"select * $v").repartition(1).write.mode("overwrite").parquet(path) + runDatafusionCli(s"""$s '$path/*.parquet'""") + } + + case _ => + sys.error("Cannot parse properly") + } + } else { + // Process the simple example like `SELECT cos(0);` + runDatafusionCli(q) + } + CoverageResult(CoverageResultStatus.Passed.toString, Seq((q, "OK"))) + } catch { + case e: Throwable => + CoverageResult(CoverageResultStatus.Failed.toString, Seq((q, e.getMessage))) + } + resultsMap.put(funcName, queryResult) + + case (funcName, List()) => + resultsMap.put( + funcName, + CoverageResult( + CoverageResultStatus.Skipped.toString, + Seq(("", "No examples found in spark.sessionState.functionRegistry")))) + } + + resultsMap.toSeq.toDF("name", "details").createOrReplaceTempView("t") + val str = showString(spark.sql("select * from t order by 1"), 1000, 0) + Files.write(Paths.get(rawCoverageFileDatafusionPath), str.getBytes(StandardCharsets.UTF_8)) + } + + private def runDatafusionCli(sql: String) = { val datafusionCliPath = sys.env.getOrElse( - DATAFUSIONCLI_PATH_ENV_VAR, - return Some(s"$DATAFUSIONCLI_PATH_ENV_VAR env variable not set")) + "DATAFUSIONCLI_PATH", + sys.error("DATAFUSIONCLI_PATH env variable not set")) val tempFilePath = Files.createTempFile("temp-", ".sql") + Files.write(tempFilePath, sql.getBytes) + + val command = s"""$datafusionCliPath/datafusion-cli -f $tempFilePath""" + val stdout = new StringBuilder val stderr = new StringBuilder - try { - Files.write(tempFilePath, sql.getBytes) - - val command = s"""$datafusionCliPath/datafusion-cli -f $tempFilePath""" - command.!( - ProcessLogger( - out => stdout.append(out).append("\n"), // stdout - err => stderr.append(err).append("\n") // stderr - )) - } finally { - Files.delete(tempFilePath) - } - val err = stderr.toString - val out = stdout.toString + command.!( + ProcessLogger( + out => stdout.append(out).append("\n"), // stdout + err => stderr.append(err).append("\n") // stderr + )) + + Files.delete(tempFilePath) + + val err = stderr.toString() + val out = stdout.toString() if (err.nonEmpty) - return Some(s"std_err: $err") + sys.error(s"std_err: $err") if (out.toLowerCase.contains("error")) - return Some(s"std_out: $out") - - None + sys.error(s"std_out: $out") } } +case class CoverageResult(result: String, details: Seq[(String, String)]) + object CoverageResultStatus extends Enumeration { type CoverageResultStatus = Value @@ -220,10 +248,3 @@ object CoverageResultStatus extends Enumeration { val Passed: Value = Value("PASSED") val Skipped: Value = Value("SKIPPED") } - -case class CoverageResult( - query: String, - result: CoverageResultStatus, - details: CoverageResultDetails) - -case class CoverageResultDetails(cometMessage: String, datafusionMessage: String) From 91ce3317cb7860d153e28f51d8be12d8f26addcb Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 9 May 2024 08:18:41 -0700 Subject: [PATCH 02/20] Update spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala Co-authored-by: advancedxy --- .../org/apache/comet/CometExpressionCoverageSuite.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index a3b38e6f9..45921883f 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -213,8 +213,9 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH sys.error("DATAFUSIONCLI_PATH env variable not set")) val tempFilePath = Files.createTempFile("temp-", ".sql") + try { Files.write(tempFilePath, sql.getBytes) - + val command = s"""$datafusionCliPath/datafusion-cli -f $tempFilePath""" val stdout = new StringBuilder @@ -225,8 +226,9 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH out => stdout.append(out).append("\n"), // stdout err => stderr.append(err).append("\n") // stderr )) - - Files.delete(tempFilePath) + } finally { + Files.delete(tempFilePath) + } val err = stderr.toString() val out = stdout.toString() From 91bdc82ad215f5a9893bb78540c043d57e1cd4ca Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 10 May 2024 10:14:23 -0700 Subject: [PATCH 03/20] Coverage: Add a manual test for DF supporting Spark expressions directly --- doc/spark_builtin_expr_df_coverage.txt | 419 ------------------ .../comet/CometExpressionCoverageSuite.scala | 192 ++++---- 2 files changed, 88 insertions(+), 523 deletions(-) delete mode 100644 doc/spark_builtin_expr_df_coverage.txt diff --git a/doc/spark_builtin_expr_df_coverage.txt b/doc/spark_builtin_expr_df_coverage.txt deleted file mode 100644 index 6f401f0b5..000000000 --- a/doc/spark_builtin_expr_df_coverage.txt +++ /dev/null @@ -1,419 +0,0 @@ -+---------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|name |details | -+---------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|! |{FAILED, [{SELECT ! true;, std_err: SQL error: ParserError("Expected an expression:, found: !")\n}]} | -|% |{PASSED, [{SELECT 2 % 1.8;, OK}]} | -|& |{PASSED, [{SELECT 3 & 5;, OK}]} | -|* |{PASSED, [{SELECT 2 * 3;, OK}]} | -|+ |{PASSED, [{SELECT 1 + 2;, OK}]} | -|- |{PASSED, [{SELECT 2 - 1;, OK}]} | -|/ |{PASSED, [{SELECT 3 / 2;, OK}]} | -|< |{PASSED, [{SELECT 1 < 2;, OK}]} | -|<= |{PASSED, [{SELECT 2 <= 2;, OK}]} | -|<=> |{FAILED, [{SELECT 2 <=> 2;, std_err: This feature is not implemented: Unsupported SQL binary operator Spaceship\n}]} | -|= |{PASSED, [{SELECT 2 = 2;, OK}]} | -|== |{PASSED, [{SELECT 2 == 2;, OK}]} | -|> |{PASSED, [{SELECT 2 > 1;, OK}]} | -|>= |{PASSED, [{SELECT 2 >= 1;, OK}]} | -|^ |{PASSED, [{SELECT 3 ^ 5;, OK}]} | -|abs |{PASSED, [{SELECT abs(-1);, OK}]} | -|acos |{PASSED, [{SELECT acos(1);, OK}]} | -|acosh |{PASSED, [{SELECT acosh(1);, OK}]} | -|add_months |{FAILED, [{SELECT add_months('2016-08-31', 1);, std_err: Error during planning: Invalid function 'add_months'.\nDid you mean 'radians'?\n}]} | -|aes_decrypt |{FAILED, [{SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333');, std_err: Error during planning: Invalid function 'aes_decrypt'.\nDid you mean 'list_except'?\n}]} | -|aes_encrypt |{FAILED, [{SELECT hex(aes_encrypt('Spark', '0000111122223333'));, std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n}]} | -|aggregate |{FAILED, [{SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|and |{PASSED, [{SELECT true and true;, OK}]} | -|any |{FAILED, [{SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'any'.\nDid you mean 'abs'?\n}]} | -|any_value |{FAILED, [{SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'any_value'.\nDid you mean 'LAST_VALUE'?\n}]} | -|approx_count_distinct |{FAILED, [{SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'approx_count_distinct'.\nDid you mean 'APPROX_DISTINCT'?\n}]} | -|approx_percentile |{FAILED, [{SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n}]} | -|array |{FAILED, [{SELECT array(1, 2, 3);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_agg |{PASSED, [{SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col);, OK}]} | -|array_append |{FAILED, [{SELECT array_append(array('b', 'd', 'c', 'a'), 'd');, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | -|array_compact |{FAILED, [{SELECT array_compact(array(1, 2, 3, null));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_contains |{FAILED, [{SELECT array_contains(array(1, 2, 3), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_distinct |{FAILED, [{SELECT array_distinct(array(1, 2, 3, null, 3));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_except |{FAILED, [{SELECT array_except(array(1, 2, 3), array(1, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_insert |{FAILED, [{SELECT array_insert(array(1, 2, 3, 4), 5, 5);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_intersect |{FAILED, [{SELECT array_intersect(array(1, 2, 3), array(1, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_join |{FAILED, [{SELECT array_join(array('hello', 'world'), ' ');, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'hello'")\n}]} | -|array_max |{FAILED, [{SELECT array_max(array(1, 20, null, 3));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_min |{FAILED, [{SELECT array_min(array(1, 20, null, 3));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_position |{FAILED, [{SELECT array_position(array(3, 2, 1), 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 3")\n}]} | -|array_remove |{FAILED, [{SELECT array_remove(array(1, 2, 3, null, 3), 3);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|array_repeat |{PASSED, [{SELECT array_repeat('123', 2);, OK}]} | -|array_size |{FAILED, [{SELECT array_size(array('b', 'd', 'c', 'a'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | -|array_sort |{FAILED, [{SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 5")\n}]} | -|array_union |{FAILED, [{SELECT array_union(array(1, 2, 3), array(1, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|arrays_overlap |{FAILED, [{SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|arrays_zip |{FAILED, [{SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|ascii |{PASSED, [{SELECT ascii('222');, OK}]} | -|asin |{PASSED, [{SELECT asin(0);, OK}]} | -|asinh |{PASSED, [{SELECT asinh(0);, OK}]} | -|assert_true |{FAILED, [{SELECT assert_true(0 < 1);, std_err: Error during planning: Invalid function 'assert_true'.\nDid you mean 'date_trunc'?\n}]} | -|atan |{PASSED, [{SELECT atan(0);, OK}]} | -|atan2 |{PASSED, [{SELECT atan2(0, 0);, OK}]} | -|atanh |{PASSED, [{SELECT atanh(0);, OK}]} | -|avg |{PASSED, [{SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | -|base64 |{FAILED, [{SELECT base64('Spark SQL');, std_err: Error during planning: Invalid function 'base64'.\nDid you mean 'asin'?\n}]} | -|bigint |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|bin |{FAILED, [{SELECT bin(13);, std_err: Error during planning: Invalid function 'bin'.\nDid you mean 'sin'?\n}]} | -|binary |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|bit_and |{PASSED, [{SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col);, OK}]} | -|bit_count |{FAILED, [{SELECT bit_count(0);, std_err: Error during planning: Invalid function 'bit_count'.\nDid you mean 'COUNT'?\n}]} | -|bit_get |{FAILED, [{SELECT bit_get(11, 0);, std_err: Error during planning: Invalid function 'bit_get'.\nDid you mean 'BIT_AND'?\n}]} | -|bit_length |{PASSED, [{SELECT bit_length('Spark SQL');, OK}]} | -|bit_or |{PASSED, [{SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col);, OK}]} | -|bit_xor |{PASSED, [{SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col);, OK}]} | -|bool_and |{PASSED, [{SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col);, OK}]} | -|bool_or |{PASSED, [{SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col);, OK}]} | -|boolean |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|bround |{FAILED, [{SELECT bround(2.5, 0);, std_err: Error during planning: Invalid function 'bround'.\nDid you mean 'round'?\n}]} | -|btrim |{PASSED, [{SELECT btrim(' SparkSQL ');, OK}]} | -|cardinality |{FAILED, [{SELECT cardinality(array('b', 'd', 'c', 'a'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | -|cast |{PASSED, [{SELECT cast('10' as int);, OK}]} | -|cbrt |{PASSED, [{SELECT cbrt(27.0);, OK}]} | -|ceil |{PASSED, [{SELECT ceil(-0.1);, OK}]} | -|ceiling |{FAILED, [{SELECT ceiling(-0.1);, std_err: Error during planning: Invalid function 'ceiling'.\nDid you mean 'ceil'?\n}]} | -|char |{FAILED, [{SELECT char(65);, std_err: Error during planning: Invalid function 'char'.\nDid you mean 'chr'?\n}]} | -|char_length |{PASSED, [{SELECT char_length('Spark SQL ');, OK}]} | -|character_length |{PASSED, [{SELECT character_length('Spark SQL ');, OK}]} | -|chr |{PASSED, [{SELECT chr(65);, OK}]} | -|coalesce |{PASSED, [{SELECT coalesce(NULL, 1, NULL);, OK}]} | -|collect_list |{FAILED, [{SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'collect_list'.\nDid you mean 'make_list'?\n}]} | -|collect_set |{FAILED, [{SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'collect_set'.\nDid you mean 'coalesce'?\n}]} | -|concat |{PASSED, [{SELECT concat('Spark', 'SQL');, OK}]} | -|concat_ws |{PASSED, [{SELECT concat_ws(' ', 'Spark', 'SQL');, OK}]} | -|contains |{FAILED, [{SELECT contains('Spark SQL', 'Spark');, std_err: Error during planning: Invalid function 'contains'.\nDid you mean 'concat'?\n}]} | -|conv |{FAILED, [{SELECT conv('100', 2, 10);, std_err: Error during planning: Invalid function 'conv'.\nDid you mean 'cos'?\n}]} | -|convert_timezone |{FAILED, [{SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00');, std_err: SQL error: ParserError("Expected ), found: '2021-12-06 00:00:00'")\n}]} | -|corr |{PASSED, [{SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2);, OK}]} | -|cos |{PASSED, [{SELECT cos(0);, OK}]} | -|cosh |{PASSED, [{SELECT cosh(0);, OK}]} | -|cot |{PASSED, [{SELECT cot(1);, OK}]} | -|count |{PASSED, [{SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col);, OK}]} | -|count_if |{FAILED, [{SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'count_if'.\nDid you mean 'COUNT'?\n}]} | -|count_min_sketch |{FAILED, [{SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected ), found: d")\n}]} | -|covar_pop |{PASSED, [{SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);, OK}]} | -|covar_samp |{PASSED, [{SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2);, OK}]} | -|crc32 |{FAILED, [{SELECT crc32('Spark');, std_err: Error during planning: Invalid function 'crc32'.\nDid you mean 'log2'?\n}]} | -|csc |{FAILED, [{SELECT csc(1);, std_err: Error during planning: Invalid function 'csc'.\nDid you mean 'cosh'?\n}]} | -|cume_dist |{PASSED, [{SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | -|curdate |{FAILED, [{SELECT curdate();, std_err: Error during planning: Invalid function 'curdate'.\nDid you mean 'to_date'?\n}]} | -|current_catalog |{FAILED, [{SELECT current_catalog();, std_err: SQL error: ParserError("Expected end of statement, found: (")\n}]} | -|current_database |{FAILED, [{SELECT current_database();, std_err: Error during planning: Invalid function 'current_database'.\nDid you mean 'current_date'?\n}]} | -|current_date |{PASSED, [{SELECT current_date();, OK}]} | -|current_schema |{FAILED, [{SELECT current_schema();, std_err: Error during planning: Invalid function 'current_schema'.\nDid you mean 'current_date'?\n}]} | -|current_timestamp |{FAILED, [{SELECT current_timestamp();, std_err: Error during planning: Invalid function 'current_timestamp'.\nDid you mean 'current_time'?\n}]} | -|current_timezone |{FAILED, [{SELECT current_timezone();, std_err: Error during planning: Invalid function 'current_timezone'.\nDid you mean 'current_time'?\n}]} | -|current_user |{FAILED, [{SELECT current_user();, std_err: SQL error: ParserError("Expected end of statement, found: (")\n}]} | -|date |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|date_add |{FAILED, [{SELECT date_add('2016-07-30', 1);, std_err: Error during planning: Invalid function 'date_add'.\nDid you mean 'datepart'?\n}]} | -|date_diff |{FAILED, [{SELECT date_diff('2009-07-31', '2009-07-30');, std_err: Error during planning: Invalid function 'date_diff'.\nDid you mean 'date_bin'?\n}]} | -|date_format |{PASSED, [{SELECT date_format('2016-04-08', 'y');, OK}]} | -|date_from_unix_date |{FAILED, [{SELECT date_from_unix_date(1);, std_err: Error during planning: Invalid function 'date_from_unix_date'.\nDid you mean 'from_unixtime'?\n}]} | -|date_part |{PASSED, [{SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456');, OK}]} | -|date_sub |{FAILED, [{SELECT date_sub('2016-07-30', 1);, std_err: Error during planning: Invalid function 'date_sub'.\nDid you mean 'date_bin'?\n}]} | -|date_trunc |{PASSED, [{SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359');, OK}]} | -|dateadd |{FAILED, [{SELECT dateadd('2016-07-30', 1);, std_err: Error during planning: Invalid function 'dateadd'.\nDid you mean 'datepart'?\n}]} | -|datediff |{FAILED, [{SELECT datediff('2009-07-31', '2009-07-30');, std_err: Error during planning: Invalid function 'datediff'.\nDid you mean 'datepart'?\n}]} | -|datepart |{PASSED, [{SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456');, OK}]} | -|day |{FAILED, [{SELECT day('2009-07-30');, std_err: Error during planning: Invalid function 'day'.\nDid you mean 'today'?\n}]} | -|dayofmonth |{FAILED, [{SELECT dayofmonth('2009-07-30');, std_err: Error during planning: Invalid function 'dayofmonth'.\nDid you mean 'date_part'?\n}]} | -|dayofweek |{FAILED, [{SELECT dayofweek('2009-07-30');, std_err: Error during planning: Invalid function 'dayofweek'.\nDid you mean 'lower'?\n}]} | -|dayofyear |{FAILED, [{SELECT dayofyear('2016-04-09');, std_err: Error during planning: Invalid function 'dayofyear'.\nDid you mean 'date_part'?\n}]} | -|decimal |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|decode |{FAILED, [{SELECT decode(encode('abc', 'utf-8'), 'utf-8');, std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n}]} | -|degrees |{PASSED, [{SELECT degrees(3.141592653589793);, OK}]} | -|dense_rank |{FAILED, [{SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'DENSE_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tDENSE_RANK()\n}]} | -|div |{FAILED, [{SELECT 3 div 2;, std_err: SQL error: ParserError("No infix parser for token Word(Word { value: \"div\", quote_style: None, keyword: DIV })")\n}]} | -|double |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|e |{FAILED, [{SELECT e();, std_err: Error during planning: Invalid function 'e'.\nDid you mean 'ln'?\n}]} | -|element_at |{FAILED, [{SELECT element_at(array(1, 2, 3), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|elt |{FAILED, [{SELECT elt(1, 'scala', 'java');, std_err: Error during planning: Invalid function 'elt'.\nDid you mean 'left'?\n}]} | -|encode |{FAILED, [{SELECT encode('abc', 'utf-8');, std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n}]} | -|endswith |{FAILED, [{SELECT endswith('Spark SQL', 'SQL');, std_err: Error during planning: Invalid function 'endswith'.\nDid you mean 'ends_with'?\n}]} | -|equal_null |{FAILED, [{SELECT equal_null(3, 3);, std_err: Error during planning: Invalid function 'equal_null'.\nDid you mean 'ifnull'?\n}]} | -|every |{FAILED, [{SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'every'.\nDid you mean 'reverse'?\n}]} | -|exists |{FAILED, [{SELECT exists(array(1, 2, 3), x -> x % 2 == 0);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n}]} | -|exp |{PASSED, [{SELECT exp(0);, OK}]} | -|explode |{FAILED, [{SELECT explode(array(10, 20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | -|explode_outer |{FAILED, [{SELECT explode_outer(array(10, 20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | -|expm1 |{FAILED, [{SELECT expm1(0);, std_err: Error during planning: Invalid function 'expm1'.\nDid you mean 'exp'?\n}]} | -|extract |{PASSED, [{SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456');, OK}]} | -|factorial |{PASSED, [{SELECT factorial(5);, OK}]} | -|filter |{FAILED, [{SELECT filter(array(1, 2, 3), x -> x % 2 == 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|find_in_set |{PASSED, [{SELECT find_in_set('ab','abc,b,ab,c,def');, OK}]} | -|first |{FAILED, [{SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'first'.\nDid you mean 'digest'?\n}]} | -|first_value |{PASSED, [{SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col);, OK}]} | -|flatten |{FAILED, [{SELECT flatten(array(array(1, 2), array(3, 4)));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n}]} | -|float |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|floor |{PASSED, [{SELECT floor(-0.1);, OK}]} | -|forall |{FAILED, [{SELECT forall(array(1, 2, 3), x -> x % 2 == 0);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|format_number |{FAILED, [{SELECT format_number(12332.123456, 4);, std_err: Error during planning: Invalid function 'format_number'.\nDid you mean 'FIRST_VALUE'?\n}]} | -|format_string |{FAILED, [{SELECT format_string("Hello World %d %s", 100, "days");, std_err: Error during planning: Invalid function 'format_string'.\nDid you mean 'list_to_string'?\n}]} | -|from_csv |{FAILED, [{SELECT from_csv('1, 0.8', 'a INT, b DOUBLE');, std_err: Error during planning: Invalid function 'from_csv'.\nDid you mean 'arrow_cast'?\n}]} | -|from_json |{FAILED, [{SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE');, std_err: Error during planning: Invalid function 'from_json'.\nDid you mean 'array_join'?\n}]} | -|from_unixtime |{FAILED, [{SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss');, std_err: Error during planning: No function matches the given name and argument types 'from_unixtime(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tfrom_unixtime(Int64)\n}]} | -|from_utc_timestamp |{FAILED, [{SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul');, std_err: Error during planning: Invalid function 'from_utc_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | -|get |{FAILED, [{SELECT get(array(1, 2, 3), 0);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|get_json_object |{FAILED, [{SELECT get_json_object('{"a":"b"}', '$.a');, std_err: Error during planning: Invalid function 'get_json_object'.\nDid you mean 'get_field'?\n}]} | -|getbit |{FAILED, [{SELECT getbit(11, 0);, std_err: Error during planning: Invalid function 'getbit'.\nDid you mean 'repeat'?\n}]} | -|greatest |{FAILED, [{SELECT greatest(10, 9, 2, 4, 3);, std_err: Error during planning: Invalid function 'greatest'.\nDid you mean 'repeat'?\n}]} | -|grouping |{FAILED, [{SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name);, [MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#79, spark_grouping_id#78L], [age#75, name#79]\n+- Expand [[age#75, name#76, name#77, 0], [age#75, name#76, null, 1]], [age#75, name#76, name#79, spark_grouping_id#78L]\n +- Project [age#75, name#76, name#76 AS name#77]\n +- SubqueryAlias people\n +- LocalRelation [age#75, name#76]\n}]} | -|grouping_id |{FAILED, [{SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);, [MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#245, height#246, spark_grouping_id#244L], [age#239, name#245, height#246]\n+- Expand [[age#239, name#240, height#241, name#242, height#243, 0], [age#239, name#240, height#241, name#242, null, 1], [age#239, name#240, height#241, null, height#243, 2], [age#239, name#240, height#241, null, null, 3]], [age#239, name#240, height#241, name#245, height#246, spark_grouping_id#244L]\n +- Project [age#239, name#240, height#241, name#240 AS name#242, height#241 AS height#243]\n +- SubqueryAlias people\n +- LocalRelation [age#239, name#240, height#241]\n}]}| -|hash |{FAILED, [{SELECT hash('Spark', array(123), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n}]} | -|hex |{FAILED, [{SELECT hex(17);, std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n}]} | -|histogram_numeric |{FAILED, [{SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'histogram_numeric'.\nDid you mean 'list_remove_n'?\n}]} | -|hour |{FAILED, [{SELECT hour('2009-07-30 12:58:59');, std_err: Error during planning: Invalid function 'hour'.\nDid you mean 'CORR'?\n}]} | -|hypot |{FAILED, [{SELECT hypot(3, 4);, std_err: Error during planning: Invalid function 'hypot'.\nDid you mean 'cot'?\n}]} | -|if |{FAILED, [{SELECT if(1 < 2, 'a', 'b');, std_err: Error during planning: Invalid function 'if'.\nDid you mean 'pi'?\n}]} | -|ifnull |{FAILED, [{SELECT ifnull(NULL, array('2'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n}]} | -|ilike |{FAILED, [{SELECT ilike('Spark', '_Park');, std_err: Error during planning: Invalid function 'ilike'.\nDid you mean 'sinh'?\n}]} | -|in |{PASSED, [{SELECT 1 in(1, 2, 3);, OK}]} | -|initcap |{PASSED, [{SELECT initcap('sPark sql');, OK}]} | -|inline |{FAILED, [{SELECT inline(array(struct(1, 'a'), struct(2, 'b')));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n}]} | -|inline_outer |{FAILED, [{SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b')));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n}]} | -|input_file_block_length |{FAILED, [{SELECT input_file_block_length();, std_err: Error during planning: Invalid function 'input_file_block_length'.\nDid you mean 'octet_length'?\n}]} | -|input_file_block_start |{FAILED, [{SELECT input_file_block_start();, std_err: Error during planning: Invalid function 'input_file_block_start'.\nDid you mean 'list_replace_all'?\n}]} | -|input_file_name |{FAILED, [{SELECT input_file_name();, std_err: Error during planning: Invalid function 'input_file_name'.\nDid you mean 'list_replace'?\n}]} | -|instr |{PASSED, [{SELECT instr('SparkSQL', 'SQL');, OK}]} | -|int |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|isnan |{PASSED, [{SELECT isnan(cast('NaN' as double));, OK}]} | -|isnotnull |{FAILED, [{SELECT isnotnull(1);, std_err: Error during planning: Invalid function 'isnotnull'.\nDid you mean 'ifnull'?\n}]} | -|isnull |{FAILED, [{SELECT isnull(1);, std_err: Error during planning: Invalid function 'isnull'.\nDid you mean 'ifnull'?\n}]} | -|java_method |{FAILED, [{SELECT java_method('java.util.UUID', 'randomUUID');, std_err: Error during planning: Invalid function 'java_method'.\nDid you mean 'make_date'?\n}]} | -|json_array_length |{FAILED, [{SELECT json_array_length('[1,2,3,4]');, std_err: Error during planning: Invalid function 'json_array_length'.\nDid you mean 'array_length'?\n}]} | -|json_object_keys |{FAILED, [{SELECT json_object_keys('{}');, std_err: Error during planning: Invalid function 'json_object_keys'.\nDid you mean 'concat_ws'?\n}]} | -|json_tuple |{FAILED, [{SELECT json_tuple('{"a":1, "b":2}', 'a', 'b');, std_err: Error during planning: Invalid function 'json_tuple'.\nDid you mean 'to_date'?\n}]} | -|kurtosis |{FAILED, [{SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'kurtosis'.\nDid you mean 'rtrim'?\n}]} | -|lag |{PASSED, [{SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | -|last |{FAILED, [{SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'last'.\nDid you mean 'left'?\n}]} | -|last_day |{FAILED, [{SELECT last_day('2009-01-12');, std_err: Error during planning: Invalid function 'last_day'.\nDid you mean 'list_cat'?\n}]} | -|last_value |{PASSED, [{SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col);, OK}]} | -|lcase |{FAILED, [{SELECT lcase('SparkSql');, std_err: Error during planning: Invalid function 'lcase'.\nDid you mean 'lpad'?\n}]} | -|lead |{PASSED, [{SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | -|least |{FAILED, [{SELECT least(10, 9, 2, 4, 3);, std_err: Error during planning: Invalid function 'least'.\nDid you mean 'left'?\n}]} | -|left |{PASSED, [{SELECT left('Spark SQL', 3);, OK}]} | -|len |{FAILED, [{SELECT len('Spark SQL ');, std_err: Error during planning: Invalid function 'len'.\nDid you mean 'ln'?\n}]} | -|length |{PASSED, [{SELECT length('Spark SQL ');, OK}]} | -|levenshtein |{PASSED, [{SELECT levenshtein('kitten', 'sitting');, OK}]} | -|like |{FAILED, [{SELECT like('Spark', '_park');, std_err: Error during planning: Invalid function 'like'.\nDid you mean 'pi'?\n}]} | -|ln |{PASSED, [{SELECT ln(1);, OK}]} | -|localtimestamp |{FAILED, [{SELECT localtimestamp();, std_err: Error during planning: Invalid function 'localtimestamp'.\nDid you mean 'to_timestamp'?\n}]} | -|locate |{FAILED, [{SELECT locate('bar', 'foobarbar');, std_err: Error during planning: Invalid function 'locate'.\nDid you mean 'concat'?\n}]} | -|log |{PASSED, [{SELECT log(10, 100);, OK}]} | -|log10 |{PASSED, [{SELECT log10(10);, OK}]} | -|log1p |{FAILED, [{SELECT log1p(0);, std_err: Error during planning: Invalid function 'log1p'.\nDid you mean 'log10'?\n}]} | -|log2 |{PASSED, [{SELECT log2(2);, OK}]} | -|lower |{PASSED, [{SELECT lower('SparkSql');, OK}]} | -|lpad |{PASSED, [{SELECT lpad('hi', 5, '??');, OK}]} | -|ltrim |{PASSED, [{SELECT ltrim(' SparkSQL ');, OK}]} | -|make_date |{PASSED, [{SELECT make_date(2013, 7, 15);, OK}]} | -|make_dt_interval |{FAILED, [{SELECT make_dt_interval(1, 12, 30, 01.001001);, std_err: Error during planning: Invalid function 'make_dt_interval'.\nDid you mean 'make_date'?\n}]} | -|make_interval |{FAILED, [{SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001);, std_err: Error during planning: Invalid function 'make_interval'.\nDid you mean 'make_array'?\n}]} | -|make_timestamp |{FAILED, [{SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887);, std_err: Error during planning: Invalid function 'make_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | -|make_timestamp_ltz |{FAILED, [{SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887);, std_err: Error during planning: Invalid function 'make_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n}]} | -|make_timestamp_ntz |{FAILED, [{SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887);, std_err: Error during planning: Invalid function 'make_timestamp_ntz'.\nDid you mean 'to_timestamp_nanos'?\n}]} | -|make_ym_interval |{FAILED, [{SELECT make_ym_interval(1, 2);, std_err: Error during planning: Invalid function 'make_ym_interval'.\nDid you mean 'make_date'?\n}]} | -|map |{FAILED, [{SELECT map(1.0, '2', 3.0, '4');, std_err: Error during planning: Invalid function 'map'.\nDid you mean 'MAX'?\n}]} | -|map_concat |{FAILED, [{SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c'));, std_err: Error during planning: Invalid function 'map_concat'.\nDid you mean 'list_concat'?\n}]} | -|map_contains_key |{FAILED, [{SELECT map_contains_key(map(1, 'a', 2, 'b'), 1);, std_err: Error during planning: Invalid function 'map_contains_key'.\nDid you mean 'list_contains'?\n}]} | -|map_entries |{FAILED, [{SELECT map_entries(map(1, 'a', 2, 'b'));, std_err: Error during planning: Invalid function 'map_entries'.\nDid you mean 'make_date'?\n}]} | -|map_filter |{FAILED, [{SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v);, std_err: Error during planning: Invalid function 'map_filter'.\nDid you mean 'make_date'?\n}]} | -|map_from_arrays |{FAILED, [{SELECT map_from_arrays(array(1.0, 3.0), array('2', '4'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1.0")\n}]} | -|map_from_entries |{FAILED, [{SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b')));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n}]} | -|map_keys |{FAILED, [{SELECT map_keys(map(1, 'a', 2, 'b'));, std_err: Error during planning: Invalid function 'map_keys'.\nDid you mean 'make_date'?\n}]} | -|map_values |{FAILED, [{SELECT map_values(map(1, 'a', 2, 'b'));, std_err: Error during planning: Invalid function 'map_values'.\nDid you mean 'LAST_VALUE'?\n}]} | -|map_zip_with |{FAILED, [{SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2));, std_err: Error during planning: Invalid function 'map_zip_with'.\nDid you mean 'ends_with'?\n}]} | -|mask |{FAILED, [{SELECT mask('abcd-EFGH-8765-4321');, std_err: Error during planning: Invalid function 'mask'.\nDid you mean 'MAX'?\n}]} | -|max |{PASSED, [{SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col);, OK}]} | -|max_by |{FAILED, [{SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'max_by'.\nDid you mean 'MAX'?\n}]} | -|md5 |{PASSED, [{SELECT md5('Spark');, OK}]} | -|mean |{PASSED, [{SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | -|median |{PASSED, [{SELECT median(col) FROM VALUES (0), (10) AS tab(col);, OK}]} | -|min |{PASSED, [{SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col);, OK}]} | -|min_by |{FAILED, [{SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'min_by'.\nDid you mean 'MIN'?\n}]} | -|minute |{FAILED, [{SELECT minute('2009-07-30 12:58:59');, std_err: Error during planning: Invalid function 'minute'.\nDid you mean 'instr'?\n}]} | -|mod |{PASSED, [{SELECT 2 % 1.8;, OK}]} | -|mode |{FAILED, [{SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'mode'.\nDid you mean 'md5'?\n}]} | -|monotonically_increasing_id|{FAILED, [{SELECT monotonically_increasing_id();, std_err: Error during planning: Invalid function 'monotonically_increasing_id'.\nDid you mean 'array_prepend'?\n}]} | -|month |{FAILED, [{SELECT month('2016-07-30');, std_err: Error during planning: Invalid function 'month'.\nDid you mean 'sinh'?\n}]} | -|months_between |{FAILED, [{SELECT months_between('1997-02-28 10:30:00', '1996-10-30');, std_err: Error during planning: Invalid function 'months_between'.\nDid you mean 'NTH_VALUE'?\n}]} | -|named_struct |{FAILED, [{SELECT named_struct("a", 1, "b", 2, "c", 3);, std_err: Schema error: No field named a.\n}]} | -|nanvl |{PASSED, [{SELECT nanvl(cast('NaN' as double), 123);, OK}]} | -|negative |{FAILED, [{SELECT negative(1);, std_err: Error during planning: Invalid function 'negative'.\nDid you mean 'nanvl'?\n}]} | -|next_day |{FAILED, [{SELECT next_day('2015-01-14', 'TU');, std_err: Error during planning: Invalid function 'next_day'.\nDid you mean 'today'?\n}]} | -|not |{PASSED, [{SELECT not true;, OK}]} | -|now |{PASSED, [{SELECT now();, OK}]} | -|nth_value |{PASSED, [{SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | -|ntile |{PASSED, [{SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | -|nullif |{PASSED, [{SELECT nullif(2, 2);, OK}]} | -|nvl |{FAILED, [{SELECT nvl(NULL, array('2'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n}]} | -|nvl2 |{PASSED, [{SELECT nvl2(NULL, 2, 1);, OK}]} | -|octet_length |{PASSED, [{SELECT octet_length('Spark SQL');, OK}]} | -|or |{PASSED, [{SELECT true or false;, OK}]} | -|overlay |{PASSED, [{SELECT overlay('Spark SQL' PLACING '_' FROM 6);, OK}]} | -|parse_url |{FAILED, [{SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST');, std_err: Error during planning: Invalid function 'parse_url'.\nDid you mean 'date_part'?\n}]} | -|percent_rank |{FAILED, [{SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'PERCENT_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tPERCENT_RANK()\n}]} | -|percentile |{FAILED, [{SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'percentile'.\nDid you mean 'ceil'?\n}]} | -|percentile_approx |{FAILED, [{SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n}]} | -|pi |{PASSED, [{SELECT pi();, OK}]} | -|pmod |{FAILED, [{SELECT pmod(10, 3);, std_err: Error during planning: Invalid function 'pmod'.\nDid you mean 'pow'?\n}]} | -|posexplode |{FAILED, [{SELECT posexplode(array(10,20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | -|posexplode_outer |{FAILED, [{SELECT posexplode_outer(array(10,20));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n}]} | -|position |{FAILED, [{SELECT position('bar', 'foobarbar');, std_err: SQL error: ParserError("Position function must include IN keyword")\n}]} | -|positive |{FAILED, [{SELECT positive(1);, std_err: Error during planning: Invalid function 'positive'.\nDid you mean 'position'?\n}]} | -|pow |{PASSED, [{SELECT pow(2, 3);, OK}]} | -|power |{PASSED, [{SELECT power(2, 3);, OK}]} | -|printf |{FAILED, [{SELECT printf("Hello World %d %s", 100, "days");, std_err: Error during planning: Invalid function 'printf'.\nDid you mean 'instr'?\n}]} | -|quarter |{FAILED, [{SELECT quarter('2016-08-31');, std_err: Error during planning: Invalid function 'quarter'.\nDid you mean 'upper'?\n}]} | -|radians |{PASSED, [{SELECT radians(180);, OK}]} | -|raise_error |{FAILED, [{SELECT raise_error('custom error message');, std_err: Error during planning: Invalid function 'raise_error'.\nDid you mean 'iszero'?\n}]} | -|rand |{FAILED, [{SELECT rand();, std_err: Error during planning: Invalid function 'rand'.\nDid you mean 'rpad'?\n}]} | -|randn |{FAILED, [{SELECT randn();, std_err: Error during planning: Invalid function 'randn'.\nDid you mean 'random'?\n}]} | -|random |{PASSED, [{SELECT random();, OK}]} | -|rank |{FAILED, [{SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tRANK()\n}]} | -|reduce |{FAILED, [{SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|reflect |{FAILED, [{SELECT reflect('java.util.UUID', 'randomUUID');, std_err: Error during planning: Invalid function 'reflect'.\nDid you mean 'repeat'?\n}]} | -|regexp |{FAILED, [{SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');, std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n}]} | -|regexp_count |{FAILED, [{SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en');, std_err: Error during planning: Invalid function 'regexp_count'.\nDid you mean 'REGR_COUNT'?\n}]} | -|regexp_extract |{FAILED, [{SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1);, std_err: Error during planning: Invalid function 'regexp_extract'.\nDid you mean 'regexp_match'?\n}]} | -|regexp_extract_all |{FAILED, [{SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1);, std_err: Error during planning: Invalid function 'regexp_extract_all'.\nDid you mean 'regexp_replace'?\n}]} | -|regexp_instr |{FAILED, [{SELECT regexp_instr('user@spark.apache.org', '@[^.]*');, std_err: Error during planning: Invalid function 'regexp_instr'.\nDid you mean 'regexp_match'?\n}]} | -|regexp_like |{FAILED, [{SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');, std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n}]} | -|regexp_replace |{PASSED, [{SELECT regexp_replace('100-200', '(\\d+)', 'num');, OK}]} | -|regexp_substr |{FAILED, [{SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en');, std_err: Error during planning: Invalid function 'regexp_substr'.\nDid you mean 'regexp_like'?\n}]} | -|regr_avgx |{PASSED, [{SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | -|regr_avgy |{PASSED, [{SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | -|regr_count |{PASSED, [{SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | -|regr_intercept |{PASSED, [{SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x);, OK}]} | -|regr_r2 |{PASSED, [{SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | -|regr_slope |{PASSED, [{SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x);, OK}]} | -|regr_sxx |{PASSED, [{SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | -|regr_sxy |{PASSED, [{SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | -|regr_syy |{PASSED, [{SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x);, OK}]} | -|repeat |{PASSED, [{SELECT repeat('123', 2);, OK}]} | -|replace |{PASSED, [{SELECT replace('ABCabc', 'abc', 'DEF');, OK}]} | -|reverse |{PASSED, [{SELECT reverse('Spark SQL');, OK}]} | -|right |{PASSED, [{SELECT right('Spark SQL', 3);, OK}]} | -|rint |{FAILED, [{SELECT rint(12.3456);, std_err: Error during planning: Invalid function 'rint'.\nDid you mean 'right'?\n}]} | -|rlike |{FAILED, [{SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*');, std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n}]} | -|round |{PASSED, [{SELECT round(2.5, 0);, OK}]} | -|row_number |{PASSED, [{SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b);, OK}]} | -|rpad |{PASSED, [{SELECT rpad('hi', 5, '??');, OK}]} | -|rtrim |{PASSED, [{SELECT rtrim(' SparkSQL ');, OK}]} | -|schema_of_csv |{FAILED, [{SELECT schema_of_csv('1,abc');, std_err: Error during planning: Invalid function 'schema_of_csv'.\nDid you mean 'concat_ws'?\n}]} | -|schema_of_json |{FAILED, [{SELECT schema_of_json('[{"col":0}]');, std_err: Error during planning: Invalid function 'schema_of_json'.\nDid you mean 'concat_ws'?\n}]} | -|sec |{FAILED, [{SELECT sec(0);, std_err: Error during planning: Invalid function 'sec'.\nDid you mean 'sin'?\n}]} | -|second |{FAILED, [{SELECT second('2009-07-30 12:58:59');, std_err: Error during planning: Invalid function 'second'.\nDid you mean 'decode'?\n}]} | -|sentences |{FAILED, [{SELECT sentences('Hi there! Good morning.');, std_err: Error during planning: Invalid function 'sentences'.\nDid you mean 'degrees'?\n}]} | -|sequence |{FAILED, [{SELECT sequence(1, 5);, std_err: Error during planning: Invalid function 'sequence'.\nDid you mean 'reverse'?\n}]} | -|sha |{FAILED, [{SELECT sha('Spark');, std_err: Error during planning: Invalid function 'sha'.\nDid you mean 'chr'?\n}]} | -|sha1 |{FAILED, [{SELECT sha1('Spark');, std_err: Error during planning: Invalid function 'sha1'.\nDid you mean 'sha512'?\n}]} | -|sha2 |{FAILED, [{SELECT sha2('Spark', 256);, std_err: Error during planning: Invalid function 'sha2'.\nDid you mean 'sha512'?\n}]} | -|shiftleft |{FAILED, [{SELECT shiftleft(2, 1);, std_err: Error during planning: Invalid function 'shiftleft'.\nDid you mean 'left'?\n}]} | -|shiftright |{FAILED, [{SELECT shiftright(4, 1);, std_err: Error during planning: Invalid function 'shiftright'.\nDid you mean 'right'?\n}]} | -|shiftrightunsigned |{FAILED, [{SELECT shiftrightunsigned(4, 1);, std_err: Error during planning: Invalid function 'shiftrightunsigned'.\nDid you mean 'substring_index'?\n}]} | -|shuffle |{FAILED, [{SELECT shuffle(array(1, 20, 3, 5));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|sign |{FAILED, [{SELECT sign(40);, std_err: Error during planning: Invalid function 'sign'.\nDid you mean 'sin'?\n}]} | -|signum |{PASSED, [{SELECT signum(40);, OK}]} | -|sin |{PASSED, [{SELECT sin(0);, OK}]} | -|sinh |{PASSED, [{SELECT sinh(0);, OK}]} | -|size |{FAILED, [{SELECT size(array('b', 'd', 'c', 'a'));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | -|skewness |{FAILED, [{SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'skewness'.\nDid you mean 'degrees'?\n}]} | -|slice |{FAILED, [{SELECT slice(array(1, 2, 3, 4), 2, 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|smallint |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|some |{FAILED, [{SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'some'.\nDid you mean 'SUM'?\n}]} | -|sort_array |{FAILED, [{SELECT sort_array(array('b', 'd', null, 'c', 'a'), true);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n}]} | -|soundex |{FAILED, [{SELECT soundex('Miller');, std_err: Error during planning: Invalid function 'soundex'.\nDid you mean 'round'?\n}]} | -|space |{FAILED, [{SELECT concat(space(2), '1');, std_err: Error during planning: Invalid function 'space'.\nDid you mean 'lpad'?\n}]} | -|spark_partition_id |{FAILED, [{SELECT spark_partition_id();, std_err: Error during planning: Invalid function 'spark_partition_id'.\nDid you mean 'list_position'?\n}]} | -|split |{FAILED, [{SELECT split('oneAtwoBthreeC', '[ABC]');, std_err: Error during planning: Invalid function 'split'.\nDid you mean 'pi'?\n}]} | -|split_part |{PASSED, [{SELECT split_part('11.12.13', '.', 3);, OK}]} | -|sqrt |{PASSED, [{SELECT sqrt(4);, OK}]} | -|stack |{FAILED, [{SELECT stack(2, 1, 2, 3);, std_err: Error during planning: Invalid function 'stack'.\nDid you mean 'tan'?\n}]} | -|startswith |{FAILED, [{SELECT startswith('Spark SQL', 'Spark');, std_err: Error during planning: Invalid function 'startswith'.\nDid you mean 'starts_with'?\n}]} | -|std |{FAILED, [{SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'std'.\nDid you mean 'sin'?\n}]} | -|stddev |{PASSED, [{SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | -|stddev_pop |{PASSED, [{SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | -|stddev_samp |{PASSED, [{SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | -|str_to_map |{FAILED, [{SELECT str_to_map('a:1,b:2,c:3', ',', ':');, std_err: Error during planning: Invalid function 'str_to_map'.\nDid you mean 'date_format'?\n}]} | -|string |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|struct |{PASSED, [{SELECT struct(1, 2, 3);, OK}]} | -|substr |{PASSED, [{SELECT substr('Spark SQL', 5);, OK}]} | -|substring |{FAILED, [{SELECT substring('Spark SQL', 5);, std_err: This feature is not implemented: Unsupported ast node in sqltorel: Substring { expr: Value(SingleQuotedString("Spark SQL")), substring_from: Some(Value(Number("5", false))), substring_for: None, special: true }\n}]} | -|substring_index |{PASSED, [{SELECT substring_index('www.apache.org', '.', 2);, OK}]} | -|sum |{PASSED, [{SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col);, OK}]} | -|tan |{PASSED, [{SELECT tan(0);, OK}]} | -|tanh |{PASSED, [{SELECT tanh(0);, OK}]} | -|timestamp |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|timestamp_micros |{FAILED, [{SELECT timestamp_micros(1230219000123123);, std_err: Error during planning: Invalid function 'timestamp_micros'.\nDid you mean 'to_timestamp_micros'?\n}]} | -|timestamp_millis |{FAILED, [{SELECT timestamp_millis(1230219000123);, std_err: Error during planning: Invalid function 'timestamp_millis'.\nDid you mean 'to_timestamp_millis'?\n}]} | -|timestamp_seconds |{FAILED, [{SELECT timestamp_seconds(1230219000);, std_err: Error during planning: Invalid function 'timestamp_seconds'.\nDid you mean 'to_timestamp_seconds'?\n}]} | -|tinyint |{SKIPPED, [{, No examples found in spark.sessionState.functionRegistry}]} | -|to_binary |{FAILED, [{SELECT to_binary('abc', 'utf-8');, std_err: Error during planning: Invalid function 'to_binary'.\nDid you mean 'to_char'?\n}]} | -|to_char |{FAILED, [{SELECT to_char(454, '999');, std_err: Error during planning: No function matches the given name and argument types 'to_char(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tto_char(Date32, Utf8)\n\tto_char(Date64, Utf8)\n\tto_char(Time32(Millisecond), Utf8)\n\tto_char(Time32(Second), Utf8)\n\tto_char(Time64(Microsecond), Utf8)\n\tto_char(Time64(Nanosecond), Utf8)\n\tto_char(Timestamp(Second, None), Utf8)\n\tto_char(Timestamp(Second, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Millisecond, None), Utf8)\n\tto_char(Timestamp(Millisecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Microsecond, None), Utf8)\n\tto_char(Timestamp(Microsecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Nanosecond, None), Utf8)\n\tto_char(Timestamp(Nanosecond, Some("+TZ")), Utf8)\n\tto_char(Duration(Second), Utf8)\n\tto_char(Duration(Millisecond), Utf8)\n\tto_char(Duration(Microsecond), Utf8)\n\tto_char(Duration(Nanosecond), Utf8)\n}]} | -|to_csv |{FAILED, [{SELECT to_csv(named_struct('a', 1, 'b', 2));, std_err: Error during planning: Invalid function 'to_csv'.\nDid you mean 'to_char'?\n}]} | -|to_date |{PASSED, [{SELECT to_date('2009-07-30 04:17:52');, OK}]} | -|to_json |{FAILED, [{SELECT to_json(named_struct('a', 1, 'b', 2));, std_err: Error during planning: Invalid function 'to_json'.\nDid you mean 'to_date'?\n}]} | -|to_number |{FAILED, [{SELECT to_number('454', '999');, std_err: Error during planning: Invalid function 'to_number'.\nDid you mean 'to_date'?\n}]} | -|to_timestamp |{PASSED, [{SELECT to_timestamp('2016-12-31 00:12:00');, OK}]} | -|to_timestamp_ltz |{FAILED, [{SELECT to_timestamp_ltz('2016-12-31 00:12:00');, std_err: Error during planning: Invalid function 'to_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n}]} | -|to_timestamp_ntz |{FAILED, [{SELECT to_timestamp_ntz('2016-12-31 00:12:00');, std_err: Error during planning: Invalid function 'to_timestamp_ntz'.\nDid you mean 'to_timestamp_nanos'?\n}]} | -|to_unix_timestamp |{FAILED, [{SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd');, std_err: Error during planning: Invalid function 'to_unix_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | -|to_utc_timestamp |{FAILED, [{SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul');, std_err: Error during planning: Invalid function 'to_utc_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | -|transform |{FAILED, [{SELECT transform(array(1, 2, 3), x -> x + 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|transform_keys |{FAILED, [{SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|transform_values |{FAILED, [{SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|translate |{PASSED, [{SELECT translate('AaBbCc', 'abc', '123');, OK}]} | -|trim |{PASSED, [{SELECT trim(' SparkSQL ');, OK}]} | -|trunc |{FAILED, [{SELECT trunc('2019-08-04', 'week');, std_err: Error during planning: No function matches the given name and argument types 'trunc(Utf8, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\ttrunc(Float32, Int64)\n\ttrunc(Float64, Int64)\n\ttrunc(Float64)\n\ttrunc(Float32)\n}]} | -|try_add |{FAILED, [{SELECT try_add(1, 2);, std_err: Error during planning: Invalid function 'try_add'.\nDid you mean 'rpad'?\n}]} | -|try_avg |{FAILED, [{SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'try_avg'.\nDid you mean 'AVG'?\n}]} | -|try_divide |{FAILED, [{SELECT try_divide(3, 2);, std_err: Error during planning: Invalid function 'try_divide'.\nDid you mean 'to_date'?\n}]} | -|try_element_at |{FAILED, [{SELECT try_element_at(array(1, 2, 3), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|try_multiply |{FAILED, [{SELECT try_multiply(2, 3);, std_err: Error during planning: Invalid function 'try_multiply'.\nDid you mean 'trunc'?\n}]} | -|try_subtract |{FAILED, [{SELECT try_subtract(2, 1);, std_err: Error during planning: Invalid function 'try_subtract'.\nDid you mean 'array_extract'?\n}]} | -|try_sum |{FAILED, [{SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'try_sum'.\nDid you mean 'trim'?\n}]} | -|try_to_binary |{FAILED, [{SELECT try_to_binary('abc', 'utf-8');, std_err: Error during planning: Invalid function 'try_to_binary'.\nDid you mean 'date_bin'?\n}]} | -|try_to_number |{FAILED, [{SELECT try_to_number('454', '999');, std_err: Error during planning: Invalid function 'try_to_number'.\nDid you mean 'array_to_string'?\n}]} | -|try_to_timestamp |{FAILED, [{SELECT try_to_timestamp('2016-12-31 00:12:00');, std_err: Error during planning: Invalid function 'try_to_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | -|typeof |{FAILED, [{SELECT typeof(1);, std_err: Error during planning: Invalid function 'typeof'.\nDid you mean 'repeat'?\n}]} | -|ucase |{FAILED, [{SELECT ucase('SparkSql');, std_err: Error during planning: Invalid function 'ucase'.\nDid you mean 'acos'?\n}]} | -|unbase64 |{FAILED, [{SELECT unbase64('U3BhcmsgU1FM');, std_err: Error during planning: Invalid function 'unbase64'.\nDid you mean 'asin'?\n}]} | -|unhex |{FAILED, [{SELECT decode(unhex('537061726B2053514C'), 'UTF-8');, std_err: Error during planning: Invalid function 'unhex'.\nDid you mean 'upper'?\n}]} | -|unix_date |{FAILED, [{SELECT unix_date(DATE("1970-01-02"));, std_err: Error during planning: Invalid function 'unix_date'.\nDid you mean 'to_date'?\n}]} | -|unix_micros |{FAILED, [{SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z'));, std_err: Error during planning: Invalid function 'unix_micros'.\nDid you mean 'iszero'?\n}]} | -|unix_millis |{FAILED, [{SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z'));, std_err: Error during planning: Invalid function 'unix_millis'.\nDid you mean 'nullif'?\n}]} | -|unix_seconds |{FAILED, [{SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'));, std_err: Error during planning: Invalid function 'unix_seconds'.\nDid you mean 'encode'?\n}]} | -|unix_timestamp |{FAILED, [{SELECT unix_timestamp();, std_err: Error during planning: Invalid function 'unix_timestamp'.\nDid you mean 'to_timestamp'?\n}]} | -|upper |{PASSED, [{SELECT upper('SparkSql');, OK}]} | -|url_decode |{FAILED, [{SELECT url_decode('https%3A%2F%2Fspark.apache.org');, std_err: Error during planning: Invalid function 'url_decode'.\nDid you mean 'decode'?\n}]} | -|url_encode |{FAILED, [{SELECT url_encode('https://spark.apache.org');, std_err: Error during planning: Invalid function 'url_encode'.\nDid you mean 'encode'?\n}]} | -|user |{FAILED, [{SELECT user();, std_err: SQL error: ParserError("Expected end of statement, found: (")\n}]} | -|uuid |{PASSED, [{SELECT uuid();, OK}]} | -|var_pop |{PASSED, [{SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | -|var_samp |{PASSED, [{SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col);, OK}]} | -|variance |{FAILED, [{SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col);, std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'variance'.\nDid you mean 'range'?\n}]} | -|version |{FAILED, [{SELECT version();, std_err: Error during planning: Invalid function 'version'.\nDid you mean 'asin'?\n}]} | -|weekday |{FAILED, [{SELECT weekday('2009-07-30');, std_err: Error during planning: Invalid function 'weekday'.\nDid you mean 'overlay'?\n}]} | -|weekofyear |{FAILED, [{SELECT weekofyear('2008-02-20');, std_err: Error during planning: Invalid function 'weekofyear'.\nDid you mean 'to_char'?\n}]} | -|when |{PASSED, [{SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END;, OK}]} | -|width_bucket |{FAILED, [{SELECT width_bucket(5.3, 0.2, 10.6, 5);, std_err: Error during planning: Invalid function 'width_bucket'.\nDid you mean 'list_except'?\n}]} | -|xpath |{FAILED, [{SELECT xpath('b1b2b3c1c2','a/b/text()');, std_err: Error during planning: Invalid function 'xpath'.\nDid you mean 'rpad'?\n}]} | -|xpath_boolean |{FAILED, [{SELECT xpath_boolean('1','a/b');, std_err: Error during planning: Invalid function 'xpath_boolean'.\nDid you mean 'date_format'?\n}]} | -|xpath_double |{FAILED, [{SELECT xpath_double('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_double'.\nDid you mean 'date_trunc'?\n}]} | -|xpath_float |{FAILED, [{SELECT xpath_float('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_float'.\nDid you mean 'date_format'?\n}]} | -|xpath_int |{FAILED, [{SELECT xpath_int('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_int'.\nDid you mean 'date_bin'?\n}]} | -|xpath_long |{FAILED, [{SELECT xpath_long('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_long'.\nDid you mean 'date_bin'?\n}]} | -|xpath_number |{FAILED, [{SELECT xpath_number('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_number'.\nDid you mean 'date_bin'?\n}]} | -|xpath_short |{FAILED, [{SELECT xpath_short('12', 'sum(a/b)');, std_err: Error during planning: Invalid function 'xpath_short'.\nDid you mean 'list_sort'?\n}]} | -|xpath_string |{FAILED, [{SELECT xpath_string('bcc','a/c');, std_err: Error during planning: Invalid function 'xpath_string'.\nDid you mean 'array_to_string'?\n}]} | -|xxhash64 |{FAILED, [{SELECT xxhash64('Spark', array(123), 2);, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n}]} | -|year |{FAILED, [{SELECT year('2016-07-30');, std_err: Error during planning: Invalid function 'year'.\nDid you mean 'VAR'?\n}]} | -|zip_with |{FAILED, [{SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x));, std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n}]} | -|| |{PASSED, [{SELECT 3 | 5;, OK}]} | -|~ |{FAILED, [{SELECT ~ 0;, std_err: SQL error: ParserError("Expected an expression:, found: ~")\n}]} | -+---------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 45921883f..f9ec4bdad 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -25,27 +25,28 @@ import java.nio.file.{Files, Paths} import scala.collection.mutable import scala.sys.process._ -import org.scalatest.Ignore import org.scalatest.exceptions.TestFailedException import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.comet.CoverageResultStatus.CoverageResultStatus + /** * Manual test to calculate Spark builtin expressions coverage support by the Comet * - * The test will update files doc/spark_builtin_expr_coverage.txt, - * doc/spark_builtin_expr_coverage_agg.txt + * The test will update files docs/spark_builtin_expr_coverage.txt, + * docs/spark_builtin_expr_coverage_agg.txt */ -@Ignore +//@Ignore class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ - private val rawCoverageFilePath = "doc/spark_builtin_expr_coverage.txt" - private val aggCoverageFilePath = "doc/spark_builtin_expr_coverage_agg.txt" - private val rawCoverageFileDatafusionPath = "doc/spark_builtin_expr_df_coverage.txt" + private val projectDocFolder = "docs" + private val rawCoverageFilePath = s"$projectDocFolder/spark_builtin_expr_coverage.txt" + private val DATAFUSIONCLI_PATH_ENV_VAR = "DATAFUSIONCLI_PATH" private val queryPattern = """(?i)SELECT (.+?);""".r private val valuesPattern = """(?i)FROM VALUES(.+?);""".r @@ -72,6 +73,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH * doc/spark_builtin_expr_coverage_agg.txt */ test("Test Spark builtin expressions coverage") { + testNames val builtinExamplesMap = getExamples() // key - function name @@ -80,6 +82,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH builtinExamplesMap.foreach { case (funcName, q :: _) => + var dfMessage: Option[String] = None val queryResult = try { // Example with predefined values @@ -91,8 +94,13 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH val values = valuesPattern.findFirstMatchIn(q).map(_.group(0)) (select, values) match { case (Some(s), Some(v)) => - testSingleLineQuery(s"select * $v", s"$s tbl") + withTempDir { dir => + val path = new Path(dir.toURI.toString).toUri.getPath + spark.sql(s"select * $v").repartition(1).write.mode("overwrite").parquet(path) + dfMessage = runDatafusionCli(s"""$s '$path/*.parquet'""") + } + testSingleLineQuery(s"select * $v", s"$s tbl") case _ => sys.error("Cannot parse properly") } @@ -104,145 +112,114 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH // // ConstantFolding is a operator optimization rule in Catalyst that replaces expressions // that can be statically evaluated with their equivalent literal values. + dfMessage = runDatafusionCli(q) testSingleLineQuery( "select 'dummy' x", s"${q.dropRight(1)}, x from tbl", excludedOptimizerRules = Some("org.apache.spark.sql.catalyst.optimizer.ConstantFolding")) } - CoverageResult(CoverageResultStatus.Passed.toString, Seq((q, "OK"))) + CoverageResult( + q, + CoverageResultStatus.Passed, + CoverageResultDetails( + cometMessage = "OK", + datafusionMessage = dfMessage.getOrElse("OK"))) + } catch { case e: TestFailedException - if e.message.getOrElse("").contains("Expected only Comet native operators") => - CoverageResult(CoverageResultStatus.Failed.toString, Seq((q, "Unsupported"))) + if e.getMessage.contains("Expected only Comet native operators") => + CoverageResult( + q, + CoverageResultStatus.Failed, + CoverageResultDetails( + cometMessage = + "Unsupported: Expected only Comet native operators but found Spark fallback", + datafusionMessage = dfMessage.getOrElse(""))) + case e if e.getMessage.contains("CometNativeException") => CoverageResult( - CoverageResultStatus.Failed.toString, - Seq((q, "Failed on native side"))) - case _ => + q, + CoverageResultStatus.Failed, + CoverageResultDetails( + cometMessage = "Failed on native side: found CometNativeException", + datafusionMessage = dfMessage.getOrElse(""))) + + case e => CoverageResult( - CoverageResultStatus.Failed.toString, - Seq((q, "Failed on something else. Check query manually"))) + q, + CoverageResultStatus.Failed, + CoverageResultDetails( + cometMessage = e.getMessage, + datafusionMessage = dfMessage.getOrElse(""))) } resultsMap.put(funcName, queryResult) + // Function with no examples case (funcName, List()) => resultsMap.put( funcName, CoverageResult( - CoverageResultStatus.Skipped.toString, - Seq(("", "No examples found in spark.sessionState.functionRegistry")))) + "", + CoverageResultStatus.Skipped, + CoverageResultDetails( + cometMessage = "No examples found in spark.sessionState.functionRegistry", + datafusionMessage = ""))) } // TODO: convert results into HTML or .md file resultsMap.toSeq.toDF("name", "details").createOrReplaceTempView("t") - val str_agg = showString( +// val str_agg = showString( +// spark.sql( +// "select result, d._2 as details, count(1) cnt from (select name, t.details.result, explode_outer(t.details.details) as d from t) group by 1, 2 order by 1"), +// 1000, +// 0) +// Files.write(Paths.get(aggCoverageFilePath), str_agg.getBytes(StandardCharsets.UTF_8)) + + val str = showString( spark.sql( - "select result, d._2 as details, count(1) cnt from (select name, t.details.result, explode_outer(t.details.details) as d from t) group by 1, 2 order by 1"), + "select name, details.query, details.result, details.details.cometMessage, details.details.datafusionMessage from t order by 1"), 1000, 0) - Files.write(Paths.get(aggCoverageFilePath), str_agg.getBytes(StandardCharsets.UTF_8)) - - val str = showString(spark.sql("select * from t order by 1"), 1000, 0) Files.write(Paths.get(rawCoverageFilePath), str.getBytes(StandardCharsets.UTF_8)) } - /** - * Manual test to calculate Spark builtin expressions coverage support by the Datafusion - * directly - * - * The test will update files doc/spark_builtin_expr_df_coverage.txt, - * - * Requires to set DATAFUSIONCLI_PATH env variable to valid path to datafusion-cli - */ - test("Test Spark builtin expressions coverage by Datafusion directly") { - val builtinExamplesMap = getExamples() - - // key - function name - // value - list of result shows if function supported by Datafusion - val resultsMap = new mutable.HashMap[String, CoverageResult]() - - builtinExamplesMap.foreach { - case (funcName, q :: _) => - val queryResult = - try { - // Example with predefined values - // e.g. SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col) - if (q.toLowerCase.contains(" from values")) { - val select = selectPattern.findFirstMatchIn(q).map(_.group(0)) - val values = valuesPattern.findFirstMatchIn(q).map(_.group(0)) - (select, values) match { - case (Some(s), Some(v)) => - withTempDir { dir => - val path = new Path(dir.toURI.toString).toUri.getPath - spark.sql(s"select * $v").repartition(1).write.mode("overwrite").parquet(path) - runDatafusionCli(s"""$s '$path/*.parquet'""") - } - - case _ => - sys.error("Cannot parse properly") - } - } else { - // Process the simple example like `SELECT cos(0);` - runDatafusionCli(q) - } - CoverageResult(CoverageResultStatus.Passed.toString, Seq((q, "OK"))) - } catch { - case e: Throwable => - CoverageResult(CoverageResultStatus.Failed.toString, Seq((q, e.getMessage))) - } - resultsMap.put(funcName, queryResult) - - case (funcName, List()) => - resultsMap.put( - funcName, - CoverageResult( - CoverageResultStatus.Skipped.toString, - Seq(("", "No examples found in spark.sessionState.functionRegistry")))) - } - - resultsMap.toSeq.toDF("name", "details").createOrReplaceTempView("t") - val str = showString(spark.sql("select * from t order by 1"), 1000, 0) - Files.write(Paths.get(rawCoverageFileDatafusionPath), str.getBytes(StandardCharsets.UTF_8)) - } - - private def runDatafusionCli(sql: String) = { + // Returns error, None means successful execution + private def runDatafusionCli(sql: String): Option[String] = { val datafusionCliPath = sys.env.getOrElse( - "DATAFUSIONCLI_PATH", - sys.error("DATAFUSIONCLI_PATH env variable not set")) + DATAFUSIONCLI_PATH_ENV_VAR, + return Some(s"$DATAFUSIONCLI_PATH_ENV_VAR env variable not set")) val tempFilePath = Files.createTempFile("temp-", ".sql") - try { - Files.write(tempFilePath, sql.getBytes) - - val command = s"""$datafusionCliPath/datafusion-cli -f $tempFilePath""" - val stdout = new StringBuilder val stderr = new StringBuilder - - command.!( - ProcessLogger( - out => stdout.append(out).append("\n"), // stdout - err => stderr.append(err).append("\n") // stderr - )) - } finally { + try { + Files.write(tempFilePath, sql.getBytes) + + val command = s"""$datafusionCliPath/datafusion-cli -f $tempFilePath""" + command.!( + ProcessLogger( + out => stdout.append(out).append("\n"), // stdout + err => stderr.append(err).append("\n") // stderr + )) + } finally { Files.delete(tempFilePath) - } + } - val err = stderr.toString() - val out = stdout.toString() + val err = stderr.toString + val out = stdout.toString if (err.nonEmpty) - sys.error(s"std_err: $err") + return Some(s"std_err: $err") if (out.toLowerCase.contains("error")) - sys.error(s"std_out: $out") + return Some(s"std_out: $out") + + None } } -case class CoverageResult(result: String, details: Seq[(String, String)]) - object CoverageResultStatus extends Enumeration { type CoverageResultStatus = Value @@ -250,3 +227,10 @@ object CoverageResultStatus extends Enumeration { val Passed: Value = Value("PASSED") val Skipped: Value = Value("SKIPPED") } + +case class CoverageResult( + query: String, + result: CoverageResultStatus, + details: CoverageResultDetails) + +case class CoverageResultDetails(cometMessage: String, datafusionMessage: String) From 2c97b5317c0f1d90416d034c439e2ab8e0e8d886 Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 10 May 2024 10:19:35 -0700 Subject: [PATCH 04/20] Coverage: Add a manual test for DF supporting Spark expressions directly --- .../org/apache/comet/CometExpressionCoverageSuite.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index f9ec4bdad..d0f89575a 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -21,17 +21,14 @@ package org.apache.comet import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} - import scala.collection.mutable import scala.sys.process._ - import org.scalatest.exceptions.TestFailedException - import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper - import org.apache.comet.CoverageResultStatus.CoverageResultStatus +import org.scalatest.Ignore /** * Manual test to calculate Spark builtin expressions coverage support by the Comet @@ -39,7 +36,7 @@ import org.apache.comet.CoverageResultStatus.CoverageResultStatus * The test will update files docs/spark_builtin_expr_coverage.txt, * docs/spark_builtin_expr_coverage_agg.txt */ -//@Ignore +@Ignore class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ From 5cac234b2d7503d0a5a313ec785a33cf587f0a96 Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 10 May 2024 10:19:51 -0700 Subject: [PATCH 05/20] Coverage: Add a manual test for DF supporting Spark expressions directly --- .../scala/org/apache/comet/CometExpressionCoverageSuite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index d0f89575a..30b7a93ec 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -33,8 +33,7 @@ import org.scalatest.Ignore /** * Manual test to calculate Spark builtin expressions coverage support by the Comet * - * The test will update files docs/spark_builtin_expr_coverage.txt, - * docs/spark_builtin_expr_coverage_agg.txt + * The test will update files docs/spark_builtin_expr_coverage.txt */ @Ignore class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanHelper { From 41e196eac0205e37b74eb190e52cd1a7fba57c91 Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 10 May 2024 11:09:33 -0700 Subject: [PATCH 06/20] fmt --- .../org/apache/comet/CometExpressionCoverageSuite.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 30b7a93ec..9407926cd 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -21,14 +21,18 @@ package org.apache.comet import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} + import scala.collection.mutable import scala.sys.process._ + +import org.scalatest.Ignore import org.scalatest.exceptions.TestFailedException + import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + import org.apache.comet.CoverageResultStatus.CoverageResultStatus -import org.scalatest.Ignore /** * Manual test to calculate Spark builtin expressions coverage support by the Comet From 29648a2c8e93897db5dfcb1ad7fbaec2177630d9 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 20 May 2024 16:22:10 -0700 Subject: [PATCH 07/20] fmt --- .../org/apache/comet/CometExpressionCoverageSuite.scala | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 9407926cd..5e176b195 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -169,12 +169,6 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH // TODO: convert results into HTML or .md file resultsMap.toSeq.toDF("name", "details").createOrReplaceTempView("t") -// val str_agg = showString( -// spark.sql( -// "select result, d._2 as details, count(1) cnt from (select name, t.details.result, explode_outer(t.details.details) as d from t) group by 1, 2 order by 1"), -// 1000, -// 0) -// Files.write(Paths.get(aggCoverageFilePath), str_agg.getBytes(StandardCharsets.UTF_8)) val str = showString( spark.sql( @@ -184,7 +178,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH Files.write(Paths.get(rawCoverageFilePath), str.getBytes(StandardCharsets.UTF_8)) } - // Returns error, None means successful execution + // Returns execution error, None means successful execution private def runDatafusionCli(sql: String): Option[String] = { val datafusionCliPath = sys.env.getOrElse( From b4ac6e52a4c5a63d476df56b2cc153cb13b5683e Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 20 May 2024 16:28:24 -0700 Subject: [PATCH 08/20] fmt --- .../scala/org/apache/comet/CometExpressionCoverageSuite.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 5e176b195..2bf30c42b 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -39,7 +39,7 @@ import org.apache.comet.CoverageResultStatus.CoverageResultStatus * * The test will update files docs/spark_builtin_expr_coverage.txt */ -@Ignore +//@Ignore class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ @@ -73,7 +73,6 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH * doc/spark_builtin_expr_coverage_agg.txt */ test("Test Spark builtin expressions coverage") { - testNames val builtinExamplesMap = getExamples() // key - function name From 9cd18b1f2365a5a5be2e7f2f60429ebee23dca30 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 20 May 2024 16:30:39 -0700 Subject: [PATCH 09/20] fmt --- .../scala/org/apache/comet/CometExpressionCoverageSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 2bf30c42b..26c9c8fec 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -39,7 +39,7 @@ import org.apache.comet.CoverageResultStatus.CoverageResultStatus * * The test will update files docs/spark_builtin_expr_coverage.txt */ -//@Ignore +@Ignore class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanHelper { import testImplicits._ From e305f3642e6f02e12954da4328798ccbaf511a37 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 20 May 2024 18:33:34 -0700 Subject: [PATCH 10/20] minor: Generate supported spark builtin expression list into user readable format --- docs/source/user-guide/overview.md | 2 +- docs/spark_builtin_expr_coverage.txt | 838 +++++++++--------- docs/spark_expressions_support.md | 477 ++++++++++ .../comet/CometExpressionCoverageSuite.scala | 123 ++- 4 files changed, 1004 insertions(+), 436 deletions(-) create mode 100644 docs/spark_expressions_support.md diff --git a/docs/source/user-guide/overview.md b/docs/source/user-guide/overview.md index ff73176d8..e5a7be80c 100644 --- a/docs/source/user-guide/overview.md +++ b/docs/source/user-guide/overview.md @@ -29,7 +29,7 @@ Comet aims to support: - a native Parquet implementation, including both reader and writer - full implementation of Spark operators, including Filter/Project/Aggregation/Join/Exchange etc. -- full implementation of Spark built-in expressions +- [full implementation](../../../docs/spark_expressions_support.md) of Spark built-in expressions. - a UDF framework for users to migrate their existing UDF to native ## Architecture diff --git a/docs/spark_builtin_expr_coverage.txt b/docs/spark_builtin_expr_coverage.txt index 7486e6380..d30e9c095 100644 --- a/docs/spark_builtin_expr_coverage.txt +++ b/docs/spark_builtin_expr_coverage.txt @@ -1,419 +1,419 @@ -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|name |query |result |cometMessage |datafusionMessage | -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -|! |SELECT ! true; |PASSED |OK |std_err: SQL error: ParserError("Expected an expression:, found: !")\n | -|% |SELECT 2 % 1.8; |PASSED |OK |OK | -|& |SELECT 3 & 5; |PASSED |OK |OK | -|* |SELECT 2 * 3; |PASSED |OK |OK | -|+ |SELECT 1 + 2; |PASSED |OK |OK | -|- |SELECT 2 - 1; |PASSED |OK |OK | -|/ |SELECT 3 / 2; |PASSED |OK |OK | -|< |SELECT 1 < 2; |PASSED |OK |OK | -|<= |SELECT 2 <= 2; |PASSED |OK |OK | -|<=> |SELECT 2 <=> 2; |PASSED |OK |std_err: This feature is not implemented: Unsupported SQL binary operator Spaceship\n | -|= |SELECT 2 = 2; |PASSED |OK |OK | -|== |SELECT 2 == 2; |PASSED |OK |OK | -|> |SELECT 2 > 1; |PASSED |OK |OK | -|>= |SELECT 2 >= 1; |PASSED |OK |OK | -|^ |SELECT 3 ^ 5; |PASSED |OK |OK | -|abs |SELECT abs(-1); |PASSED |OK |OK | -|acos |SELECT acos(1); |PASSED |OK |OK | -|acosh |SELECT acosh(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|add_months |SELECT add_months('2016-08-31', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'add_months'.\nDid you mean 'acos'?\n | -|aes_decrypt |SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'aes_decrypt'.\nDid you mean 'list_except'?\n | -|aes_encrypt |SELECT hex(aes_encrypt('Spark', '0000111122223333')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n | -|aggregate |SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|and |SELECT true and true; |PASSED |OK |OK | -|any |SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'any'.\nDid you mean 'abs'?\n | -|any_value |SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'any_value'.\nDid you mean 'LAST_VALUE'?\n | -|approx_count_distinct |SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'approx_count_distinct'.\nDid you mean 'APPROX_DISTINCT'?\n | -|approx_percentile |SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n | -|array |SELECT array(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_agg |SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|array_append |SELECT array_append(array('b', 'd', 'c', 'a'), 'd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | -|array_compact |SELECT array_compact(array(1, 2, 3, null)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_contains |SELECT array_contains(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_distinct |SELECT array_distinct(array(1, 2, 3, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_except |SELECT array_except(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_insert |SELECT array_insert(array(1, 2, 3, 4), 5, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_intersect |SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_join |SELECT array_join(array('hello', 'world'), ' '); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'hello'")\n | -|array_max |SELECT array_max(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_min |SELECT array_min(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_position |SELECT array_position(array(3, 2, 1), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 3")\n | -|array_remove |SELECT array_remove(array(1, 2, 3, null, 3), 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|array_repeat |SELECT array_repeat('123', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|array_size |SELECT array_size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | -|array_sort |SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 5")\n | -|array_union |SELECT array_union(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|arrays_overlap |SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|arrays_zip |SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|ascii |SELECT ascii('222'); |PASSED |OK |OK | -|asin |SELECT asin(0); |PASSED |OK |OK | -|asinh |SELECT asinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|assert_true |SELECT assert_true(0 < 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'assert_true'.\nDid you mean 'date_trunc'?\n | -|atan |SELECT atan(0); |PASSED |OK |OK | -|atan2 |SELECT atan2(0, 0); |PASSED |OK |OK | -|atanh |SELECT atanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|avg |SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|base64 |SELECT base64('Spark SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'base64'.\nDid you mean 'asinh'?\n | -|bigint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bin |SELECT bin(13); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bin'.\nDid you mean 'sin'?\n | -|binary | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bit_and |SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|bit_count |SELECT bit_count(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bit_count'.\nDid you mean 'COUNT'?\n | -|bit_get |SELECT bit_get(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bit_get'.\nDid you mean 'BIT_AND'?\n | -|bit_length |SELECT bit_length('Spark SQL'); |PASSED |OK |OK | -|bit_or |SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|bit_xor |SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|bool_and |SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|bool_or |SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|boolean | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bround |SELECT bround(2.5, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bround'.\nDid you mean 'round'?\n | -|btrim |SELECT btrim(' SparkSQL '); |PASSED |OK |OK | -|cardinality |SELECT cardinality(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | -|cast |SELECT cast('10' as int); |PASSED |OK |OK | -|cbrt |SELECT cbrt(27.0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|ceil |SELECT ceil(-0.1); |PASSED |OK |OK | -|ceiling |SELECT ceiling(-0.1); |PASSED |OK |std_err: Error during planning: Invalid function 'ceiling'.\nDid you mean 'ceil'?\n | -|char |SELECT char(65); |PASSED |OK |std_err: Error during planning: Invalid function 'char'.\nDid you mean 'chr'?\n | -|char_length |SELECT char_length('Spark SQL '); |PASSED |OK |OK | -|character_length |SELECT character_length('Spark SQL '); |PASSED |OK |OK | -|chr |SELECT chr(65); |PASSED |OK |OK | -|coalesce |SELECT coalesce(NULL, 1, NULL); |PASSED |OK |OK | -|collect_list |SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'collect_list'.\nDid you mean 'make_list'?\n | -|collect_set |SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'collect_set'.\nDid you mean 'coalesce'?\n | -|concat |SELECT concat('Spark', 'SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|concat_ws |SELECT concat_ws(' ', 'Spark', 'SQL'); |PASSED |OK |OK | -|contains |SELECT contains('Spark SQL', 'Spark'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'contains'.\nDid you mean 'concat_ws'?\n | -|conv |SELECT conv('100', 2, 10); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'conv'.\nDid you mean 'cot'?\n | -|convert_timezone |SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected ), found: '2021-12-06 00:00:00'")\n | -|corr |SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|cos |SELECT cos(0); |PASSED |OK |OK | -|cosh |SELECT cosh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|cot |SELECT cot(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|count |SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|count_if |SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'count_if'.\nDid you mean 'COUNT'?\n | -|count_min_sketch |SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected ), found: d")\n | -|covar_pop |SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|covar_samp |SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|crc32 |SELECT crc32('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'crc32'.\nDid you mean 'ascii'?\n | -|csc |SELECT csc(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'csc'.\nDid you mean 'chr'?\n | -|cume_dist |SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|curdate |SELECT curdate(); |PASSED |OK |std_err: Error during planning: Invalid function 'curdate'.\nDid you mean 'to_date'?\n | -|current_catalog |SELECT current_catalog(); |PASSED |OK |std_err: SQL error: ParserError("Expected end of statement, found: (")\n | -|current_database |SELECT current_database(); |PASSED |OK |std_err: Error during planning: Invalid function 'current_database'.\nDid you mean 'current_date'?\n | -|current_date |SELECT current_date(); |PASSED |OK |OK | -|current_schema |SELECT current_schema(); |PASSED |OK |std_err: Error during planning: Invalid function 'current_schema'.\nDid you mean 'current_time'?\n | -|current_timestamp |SELECT current_timestamp(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [current_timestamp() AS current_timestamp()#3031, x#3015]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#3015])\n +- Relation [x#3015] parquet\n\n== Analyzed Logical Plan ==\ncurrent_timestamp(): timestamp, x: string\nProject [current_timestamp() AS current_timestamp()#3031, x#3015]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#3015])\n +- Relation [x#3015] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-10 10:13:21.77322 AS current_timestamp()#3031, x#3015]\n+- Relation [x#3015] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [current_timestamp()#3031, x#3015], [2024-05-10 10:13:21.77322 AS current_timestamp()#3031, x#3015]\n +- CometScan parquet [x#3015] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-62..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-10 10:13:21.749547,dummy] [2024-05-10 10:13:21.77322,dummy]\n \n |std_err: Error during planning: Invalid function 'current_timestamp'.\nDid you mean 'current_time'?\n | -|current_timezone |SELECT current_timezone(); |PASSED |OK |std_err: Error during planning: Invalid function 'current_timezone'.\nDid you mean 'current_time'?\n | -|current_user |SELECT current_user(); |PASSED |OK |std_err: SQL error: ParserError("Expected end of statement, found: (")\n | -|date | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|date_add |SELECT date_add('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_add'.\nDid you mean 'date_bin'?\n | -|date_diff |SELECT date_diff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_diff'.\nDid you mean 'date_bin'?\n | -|date_format |SELECT date_format('2016-04-08', 'y'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|date_from_unix_date |SELECT date_from_unix_date(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_from_unix_date'.\nDid you mean 'from_unixtime'?\n | -|date_part |SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |OK | -|date_sub |SELECT date_sub('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_sub'.\nDid you mean 'date_bin'?\n | -|date_trunc |SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359'); |FAILED |Failed on native side: found CometNativeException | | -|dateadd |SELECT dateadd('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dateadd'.\nDid you mean 'datepart'?\n | -|datediff |SELECT datediff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'datediff'.\nDid you mean 'datepart'?\n | -|datepart |SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |OK | -|day |SELECT day('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'day'.\nDid you mean 'today'?\n | -|dayofmonth |SELECT dayofmonth('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dayofmonth'.\nDid you mean 'datepart'?\n | -|dayofweek |SELECT dayofweek('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dayofweek'.\nDid you mean 'degrees'?\n | -|dayofyear |SELECT dayofyear('2016-04-09'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dayofyear'.\nDid you mean 'to_char'?\n | -|decimal | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|decode |SELECT decode(encode('abc', 'utf-8'), 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n | -|degrees |SELECT degrees(3.141592653589793); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|dense_rank |SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'DENSE_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tDENSE_RANK()\n | -|div |SELECT 3 div 2; |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("No infix parser for token Word(Word { value: \"div\", quote_style: None, keyword: DIV })")\n | -|double | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|e |SELECT e(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'e'.\nDid you mean 'exp'?\n | -|element_at |SELECT element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|elt |SELECT elt(1, 'scala', 'java'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'elt'.\nDid you mean 'ln'?\n | -|encode |SELECT encode('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n | -|endswith |SELECT endswith('Spark SQL', 'SQL'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'endswith'.\nDid you mean 'ends_with'?\n | -|equal_null |SELECT equal_null(3, 3); |PASSED |OK |std_err: Error during planning: Invalid function 'equal_null'.\nDid you mean 'ifnull'?\n | -|every |SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'every'.\nDid you mean 'reverse'?\n | -|exists |SELECT exists(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n | -|exp |SELECT exp(0); |PASSED |OK |OK | -|explode |SELECT explode(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | -|explode_outer |SELECT explode_outer(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | -|expm1 |SELECT expm1(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'expm1'.\nDid you mean 'exp'?\n | -|extract |SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |OK | -|factorial |SELECT factorial(5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|filter |SELECT filter(array(1, 2, 3), x -> x % 2 == 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|find_in_set |SELECT find_in_set('ab','abc,b,ab,c,def'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|first |SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'first'.\nDid you mean 'right'?\n | -|first_value |SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|flatten |SELECT flatten(array(array(1, 2), array(3, 4))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n | -|float | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|floor |SELECT floor(-0.1); |PASSED |OK |OK | -|forall |SELECT forall(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|format_number |SELECT format_number(12332.123456, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'format_number'.\nDid you mean 'FIRST_VALUE'?\n | -|format_string |SELECT format_string("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'format_string'.\nDid you mean 'array_to_string'?\n | -|from_csv |SELECT from_csv('1, 0.8', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'from_csv'.\nDid you mean 'arrow_cast'?\n | -|from_json |SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'from_json'.\nDid you mean 'floor'?\n | -|from_unixtime |SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: No function matches the given name and argument types 'from_unixtime(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tfrom_unixtime(Int64)\n | -|from_utc_timestamp |SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'from_utc_timestamp'.\nDid you mean 'to_timestamp'?\n | -|get |SELECT get(array(1, 2, 3), 0); |PASSED |OK |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|get_json_object |SELECT get_json_object('{"a":"b"}', '$.a'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'get_json_object'.\nDid you mean 'list_pop_back'?\n | -|getbit |SELECT getbit(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'getbit'.\nDid you mean 'ceil'?\n | -|greatest |SELECT greatest(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'greatest'.\nDid you mean 'repeat'?\n | -|grouping |SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name); |FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#2277, spark_grouping_id#2276L], [age#2273, name#2277]\n+- Expand [[age#2273, name#2274, name#2275, 0], [age#2273, name#2274, null, 1]], [age#2273, name#2274, name#2277, spark_grouping_id#2276L]\n +- Project [age#2273, name#2274, name#2274 AS name#2275]\n +- SubqueryAlias people\n +- LocalRelation [age#2273, name#2274]\n | | -|grouping_id |SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);|FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#7432, height#7433, spark_grouping_id#7431L], [age#7426, name#7432, height#7433]\n+- Expand [[age#7426, name#7427, height#7428, name#7429, height#7430, 0], [age#7426, name#7427, height#7428, name#7429, null, 1], [age#7426, name#7427, height#7428, null, height#7430, 2], [age#7426, name#7427, height#7428, null, null, 3]], [age#7426, name#7427, height#7428, name#7432, height#7433, spark_grouping_id#7431L]\n +- Project [age#7426, name#7427, height#7428, name#7427 AS name#7429, height#7428 AS height#7430]\n +- SubqueryAlias people\n +- LocalRelation [age#7426, name#7427, height#7428]\n | | -|hash |SELECT hash('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n | -|hex |SELECT hex(17); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n | -|histogram_numeric |SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'histogram_numeric'.\nDid you mean 'list_remove_n'?\n | -|hour |SELECT hour('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'hour'.\nDid you mean 'CORR'?\n | -|hypot |SELECT hypot(3, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'hypot'.\nDid you mean 'pow'?\n | -|if |SELECT if(1 < 2, 'a', 'b'); |PASSED |OK |std_err: Error during planning: Invalid function 'if'.\nDid you mean 'sin'?\n | -|ifnull |SELECT ifnull(NULL, array('2')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n | -|ilike |SELECT ilike('Spark', '_Park'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'ilike'.\nDid you mean 'lpad'?\n | -|in |SELECT 1 in(1, 2, 3); |PASSED |OK |OK | -|initcap |SELECT initcap('sPark sql'); |PASSED |OK |OK | -|inline |SELECT inline(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n | -|inline_outer |SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n | -|input_file_block_length |SELECT input_file_block_length(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'input_file_block_length'.\nDid you mean 'octet_length'?\n | -|input_file_block_start |SELECT input_file_block_start(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'input_file_block_start'.\nDid you mean 'list_replace_all'?\n | -|input_file_name |SELECT input_file_name(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'input_file_name'.\nDid you mean 'bit_length'?\n | -|instr |SELECT instr('SparkSQL', 'SQL'); |PASSED |OK |OK | -|int | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|isnan |SELECT isnan(cast('NaN' as double)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|isnotnull |SELECT isnotnull(1); |PASSED |OK |std_err: Error during planning: Invalid function 'isnotnull'.\nDid you mean 'ifnull'?\n | -|isnull |SELECT isnull(1); |PASSED |OK |std_err: Error during planning: Invalid function 'isnull'.\nDid you mean 'ifnull'?\n | -|java_method |SELECT java_method('java.util.UUID', 'randomUUID'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'java_method'.\nDid you mean 'make_date'?\n | -|json_array_length |SELECT json_array_length('[1,2,3,4]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'json_array_length'.\nDid you mean 'array_length'?\n | -|json_object_keys |SELECT json_object_keys('{}'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'json_object_keys'.\nDid you mean 'concat_ws'?\n | -|json_tuple |SELECT json_tuple('{"a":1, "b":2}', 'a', 'b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'json_tuple'.\nDid you mean 'strpos'?\n | -|kurtosis |SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'kurtosis'.\nDid you mean 'rtrim'?\n | -|lag |SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|last |SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'last'.\nDid you mean 'left'?\n | -|last_day |SELECT last_day('2009-01-12'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'last_day'.\nDid you mean 'list_cat'?\n | -|last_value |SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|lcase |SELECT lcase('SparkSql'); |PASSED |OK |std_err: Error during planning: Invalid function 'lcase'.\nDid you mean 'acos'?\n | -|lead |SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|least |SELECT least(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'least'.\nDid you mean 'left'?\n | -|left |SELECT left('Spark SQL', 3); |FAILED |Failed on native side: found CometNativeException | | -|len |SELECT len('Spark SQL '); |PASSED |OK |std_err: Error during planning: Invalid function 'len'.\nDid you mean 'ln'?\n | -|length |SELECT length('Spark SQL '); |PASSED |OK |OK | -|levenshtein |SELECT levenshtein('kitten', 'sitting'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|like |SELECT like('Spark', '_park'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'like'.\nDid you mean 'lower'?\n | -|ln |SELECT ln(1); |PASSED |OK |OK | -|localtimestamp |SELECT localtimestamp(); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'localtimestamp'.\nDid you mean 'to_timestamp'?\n | -|locate |SELECT locate('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'locate'.\nDid you mean 'to_date'?\n | -|log |SELECT log(10, 100); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|log10 |SELECT log10(10); |PASSED |OK |OK | -|log1p |SELECT log1p(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'log1p'.\nDid you mean 'log10'?\n | -|log2 |SELECT log2(2); |PASSED |OK |OK | -|lower |SELECT lower('SparkSql'); |PASSED |OK |OK | -|lpad |SELECT lpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|ltrim |SELECT ltrim(' SparkSQL '); |PASSED |OK |OK | -|make_date |SELECT make_date(2013, 7, 15); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|make_dt_interval |SELECT make_dt_interval(1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_dt_interval'.\nDid you mean 'make_date'?\n | -|make_interval |SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_interval'.\nDid you mean 'make_array'?\n | -|make_timestamp |SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_timestamp'.\nDid you mean 'to_timestamp'?\n | -|make_timestamp_ltz |SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n | -|make_timestamp_ntz |SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_timestamp_ntz'.\nDid you mean 'to_timestamp'?\n | -|make_ym_interval |SELECT make_ym_interval(1, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_ym_interval'.\nDid you mean 'array_intersect'?\n | -|map |SELECT map(1.0, '2', 3.0, '4'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map'.\nDid you mean 'MAX'?\n | -|map_concat |SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_concat'.\nDid you mean 'array_concat'?\n | -|map_contains_key |SELECT map_contains_key(map(1, 'a', 2, 'b'), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_contains_key'.\nDid you mean 'array_contains'?\n | -|map_entries |SELECT map_entries(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_entries'.\nDid you mean 'make_array'?\n | -|map_filter |SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_filter'.\nDid you mean 'make_date'?\n | -|map_from_arrays |SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1.0")\n | -|map_from_entries |SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n | -|map_keys |SELECT map_keys(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_keys'.\nDid you mean 'make_list'?\n | -|map_values |SELECT map_values(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_values'.\nDid you mean 'LAST_VALUE'?\n | -|map_zip_with |SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_zip_with'.\nDid you mean 'starts_with'?\n | -|mask |SELECT mask('abcd-EFGH-8765-4321'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'mask'.\nDid you mean 'MAX'?\n | -|max |SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|max_by |SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'max_by'.\nDid you mean 'MAX'?\n | -|md5 |SELECT md5('Spark'); |PASSED |OK |OK | -|mean |SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|median |SELECT median(col) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|min |SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|min_by |SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'min_by'.\nDid you mean 'MIN'?\n | -|minute |SELECT minute('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'minute'.\nDid you mean 'instr'?\n | -|mod |SELECT 2 % 1.8; |PASSED |OK |OK | -|mode |SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'mode'.\nDid you mean 'md5'?\n | -|monotonically_increasing_id|SELECT monotonically_increasing_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'monotonically_increasing_id'.\nDid you mean 'array_intersect'?\n | -|month |SELECT month('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'month'.\nDid you mean 'tanh'?\n | -|months_between |SELECT months_between('1997-02-28 10:30:00', '1996-10-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'months_between'.\nDid you mean 'NTH_VALUE'?\n | -|named_struct |SELECT named_struct("a", 1, "b", 2, "c", 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Schema error: No field named a.\n | -|nanvl |SELECT nanvl(cast('NaN' as double), 123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|negative |SELECT negative(1); |PASSED |OK |std_err: Error during planning: Invalid function 'negative'.\nDid you mean 'nanvl'?\n | -|next_day |SELECT next_day('2015-01-14', 'TU'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'next_day'.\nDid you mean 'today'?\n | -|not |SELECT not true; |PASSED |OK |OK | -|now |SELECT now(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [now() AS now()#4526, x#4510]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#4510])\n +- Relation [x#4510] parquet\n\n== Analyzed Logical Plan ==\nnow(): timestamp, x: string\nProject [now() AS now()#4526, x#4510]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#4510])\n +- Relation [x#4510] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-10 10:13:29.302484 AS now()#4526, x#4510]\n+- Relation [x#4510] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [now()#4526, x#4510], [2024-05-10 10:13:29.302484 AS now()#4526, x#4510]\n +- CometScan parquet [x#4510] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-28..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-10 10:13:29.285019,dummy] [2024-05-10 10:13:29.302484,dummy]\n \n | | -|nth_value |SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|ntile |SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|nullif |SELECT nullif(2, 2); |PASSED |OK |OK | -|nvl |SELECT nvl(NULL, array('2')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n | -|nvl2 |SELECT nvl2(NULL, 2, 1); |PASSED |OK |OK | -|octet_length |SELECT octet_length('Spark SQL'); |PASSED |OK |OK | -|or |SELECT true or false; |PASSED |OK |OK | -|overlay |SELECT overlay('Spark SQL' PLACING '_' FROM 6); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|parse_url |SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'parse_url'.\nDid you mean 'date_part'?\n | -|percent_rank |SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'PERCENT_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tPERCENT_RANK()\n | -|percentile |SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'percentile'.\nDid you mean 'current_time'?\n | -|percentile_approx |SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n | -|pi |SELECT pi(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|pmod |SELECT pmod(10, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'pmod'.\nDid you mean 'pow'?\n | -|posexplode |SELECT posexplode(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | -|posexplode_outer |SELECT posexplode_outer(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | -|position |SELECT position('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Position function must include IN keyword")\n | -|positive |SELECT positive(1); |PASSED |OK |std_err: Error during planning: Invalid function 'positive'.\nDid you mean 'position'?\n | -|pow |SELECT pow(2, 3); |PASSED |OK |OK | -|power |SELECT power(2, 3); |PASSED |OK |OK | -|printf |SELECT printf("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'printf'.\nDid you mean 'asinh'?\n | -|quarter |SELECT quarter('2016-08-31'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'quarter'.\nDid you mean 'flatten'?\n | -|radians |SELECT radians(180); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|raise_error |SELECT raise_error('custom error message'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'raise_error'.\nDid you mean 'make_array'?\n | -|rand |SELECT rand(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'rand'.\nDid you mean 'tan'?\n | -|randn |SELECT randn(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'randn'.\nDid you mean 'random'?\n | -|random |SELECT random(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|rank |SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: No function matches the given name and argument types 'RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tRANK()\n | -|reduce |SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|reflect |SELECT reflect('java.util.UUID', 'randomUUID'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'reflect'.\nDid you mean 'replace'?\n | -|regexp |SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n | -|regexp_count |SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_count'.\nDid you mean 'REGR_COUNT'?\n | -|regexp_extract |SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_extract'.\nDid you mean 'regexp_match'?\n | -|regexp_extract_all |SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_extract_all'.\nDid you mean 'regexp_match'?\n | -|regexp_instr |SELECT regexp_instr('user@spark.apache.org', '@[^.]*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_instr'.\nDid you mean 'regexp_like'?\n | -|regexp_like |SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n | -|regexp_replace |SELECT regexp_replace('100-200', '(\\d+)', 'num'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regexp_substr |SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_substr'.\nDid you mean 'regexp_like'?\n | -|regr_avgx |SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_avgy |SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_count |SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_intercept |SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_r2 |SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_slope |SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_sxx |SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_sxy |SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|regr_syy |SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|repeat |SELECT repeat('123', 2); |PASSED |OK |OK | -|replace |SELECT replace('ABCabc', 'abc', 'DEF'); |PASSED |OK |OK | -|reverse |SELECT reverse('Spark SQL'); |PASSED |OK |OK | -|right |SELECT right('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|rint |SELECT rint(12.3456); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'rint'.\nDid you mean 'sin'?\n | -|rlike |SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n | -|round |SELECT round(2.5, 0); |PASSED |OK |OK | -|row_number |SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|rpad |SELECT rpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|rtrim |SELECT rtrim(' SparkSQL '); |PASSED |OK |OK | -|schema_of_csv |SELECT schema_of_csv('1,abc'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'schema_of_csv'.\nDid you mean 'concat_ws'?\n | -|schema_of_json |SELECT schema_of_json('[{"col":0}]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'schema_of_json'.\nDid you mean 'concat_ws'?\n | -|sec |SELECT sec(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sec'.\nDid you mean 'sin'?\n | -|second |SELECT second('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'second'.\nDid you mean 'decode'?\n | -|sentences |SELECT sentences('Hi there! Good morning.'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sentences'.\nDid you mean 'degrees'?\n | -|sequence |SELECT sequence(1, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sequence'.\nDid you mean 'coalesce'?\n | -|sha |SELECT sha('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sha'.\nDid you mean 'chr'?\n | -|sha1 |SELECT sha1('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sha1'.\nDid you mean 'sha512'?\n | -|sha2 |SELECT sha2('Spark', 256); |PASSED |OK |std_err: Error during planning: Invalid function 'sha2'.\nDid you mean 'sha224'?\n | -|shiftleft |SELECT shiftleft(2, 1); |PASSED |OK |std_err: Error during planning: Invalid function 'shiftleft'.\nDid you mean 'left'?\n | -|shiftright |SELECT shiftright(4, 1); |PASSED |OK |std_err: Error during planning: Invalid function 'shiftright'.\nDid you mean 'right'?\n | -|shiftrightunsigned |SELECT shiftrightunsigned(4, 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'shiftrightunsigned'.\nDid you mean 'list_union'?\n | -|shuffle |SELECT shuffle(array(1, 20, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|sign |SELECT sign(40); |PASSED |OK |std_err: Error during planning: Invalid function 'sign'.\nDid you mean 'sin'?\n | -|signum |SELECT signum(40); |PASSED |OK |OK | -|sin |SELECT sin(0); |PASSED |OK |OK | -|sinh |SELECT sinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|size |SELECT size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | -|skewness |SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'skewness'.\nDid you mean 'degrees'?\n | -|slice |SELECT slice(array(1, 2, 3, 4), 2, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|smallint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|some |SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'some'.\nDid you mean 'SUM'?\n | -|sort_array |SELECT sort_array(array('b', 'd', null, 'c', 'a'), true); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | -|soundex |SELECT soundex('Miller'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'soundex'.\nDid you mean 'round'?\n | -|space |SELECT concat(space(2), '1'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'space'.\nDid you mean 'lpad'?\n | -|spark_partition_id |SELECT spark_partition_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'spark_partition_id'.\nDid you mean 'list_position'?\n | -|split |SELECT split('oneAtwoBthreeC', '[ABC]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'split'.\nDid you mean 'sqrt'?\n | -|split_part |SELECT split_part('11.12.13', '.', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|sqrt |SELECT sqrt(4); |PASSED |OK |OK | -|stack |SELECT stack(2, 1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'stack'.\nDid you mean 'atanh'?\n | -|startswith |SELECT startswith('Spark SQL', 'Spark'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'startswith'.\nDid you mean 'starts_with'?\n | -|std |SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'std'.\nDid you mean 'gcd'?\n | -|stddev |SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|stddev_pop |SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|stddev_samp |SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|str_to_map |SELECT str_to_map('a:1,b:2,c:3', ',', ':'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'str_to_map'.\nDid you mean 'strpos'?\n | -|string | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|struct |SELECT struct(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|substr |SELECT substr('Spark SQL', 5); |FAILED |Failed on native side: found CometNativeException | | -|substring |SELECT substring('Spark SQL', 5); |FAILED |Failed on native side: found CometNativeException |std_err: This feature is not implemented: Unsupported ast node in sqltorel: Substring { expr: Value(SingleQuotedString("Spark SQL")), substring_from: Some(Value(Number("5", false))), substring_for: None, special: true }\n | -|substring_index |SELECT substring_index('www.apache.org', '.', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|sum |SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|tan |SELECT tan(0); |PASSED |OK |OK | -|tanh |SELECT tanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|timestamp | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|timestamp_micros |SELECT timestamp_micros(1230219000123123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'timestamp_micros'.\nDid you mean 'to_timestamp_micros'?\n | -|timestamp_millis |SELECT timestamp_millis(1230219000123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'timestamp_millis'.\nDid you mean 'to_timestamp_millis'?\n | -|timestamp_seconds |SELECT timestamp_seconds(1230219000); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'timestamp_seconds'.\nDid you mean 'to_timestamp_seconds'?\n | -|tinyint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|to_binary |SELECT to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_binary'.\nDid you mean 'to_char'?\n | -|to_char |SELECT to_char(454, '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: No function matches the given name and argument types 'to_char(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tto_char(Date32, Utf8)\n\tto_char(Date64, Utf8)\n\tto_char(Time32(Millisecond), Utf8)\n\tto_char(Time32(Second), Utf8)\n\tto_char(Time64(Microsecond), Utf8)\n\tto_char(Time64(Nanosecond), Utf8)\n\tto_char(Timestamp(Second, None), Utf8)\n\tto_char(Timestamp(Second, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Millisecond, None), Utf8)\n\tto_char(Timestamp(Millisecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Microsecond, None), Utf8)\n\tto_char(Timestamp(Microsecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Nanosecond, None), Utf8)\n\tto_char(Timestamp(Nanosecond, Some("+TZ")), Utf8)\n\tto_char(Duration(Second), Utf8)\n\tto_char(Duration(Millisecond), Utf8)\n\tto_char(Duration(Microsecond), Utf8)\n\tto_char(Duration(Nanosecond), Utf8)\n| -|to_csv |SELECT to_csv(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_csv'.\nDid you mean 'to_hex'?\n | -|to_date |SELECT to_date('2009-07-30 04:17:52'); |PASSED |OK |OK | -|to_json |SELECT to_json(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_json'.\nDid you mean 'to_hex'?\n | -|to_number |SELECT to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_number'.\nDid you mean 'to_char'?\n | -|to_timestamp |SELECT to_timestamp('2016-12-31 00:12:00'); |PASSED |OK |OK | -|to_timestamp_ltz |SELECT to_timestamp_ltz('2016-12-31 00:12:00'); |PASSED |OK |std_err: Error during planning: Invalid function 'to_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n | -|to_timestamp_ntz |SELECT to_timestamp_ntz('2016-12-31 00:12:00'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'to_timestamp_ntz'.\nDid you mean 'to_timestamp_nanos'?\n | -|to_unix_timestamp |SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_unix_timestamp'.\nDid you mean 'to_timestamp'?\n | -|to_utc_timestamp |SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_utc_timestamp'.\nDid you mean 'to_timestamp'?\n | -|transform |SELECT transform(array(1, 2, 3), x -> x + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|transform_keys |SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|transform_values |SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|translate |SELECT translate('AaBbCc', 'abc', '123'); |PASSED |OK |OK | -|trim |SELECT trim(' SparkSQL '); |PASSED |OK |OK | -|trunc |SELECT trunc('2019-08-04', 'week'); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: No function matches the given name and argument types 'trunc(Utf8, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\ttrunc(Float32, Int64)\n\ttrunc(Float64, Int64)\n\ttrunc(Float64)\n\ttrunc(Float32)\n | -|try_add |SELECT try_add(1, 2); |PASSED |OK |std_err: Error during planning: Invalid function 'try_add'.\nDid you mean 'rpad'?\n | -|try_avg |SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'try_avg'.\nDid you mean 'AVG'?\n | -|try_divide |SELECT try_divide(3, 2); |PASSED |OK |std_err: Error during planning: Invalid function 'try_divide'.\nDid you mean 'to_date'?\n | -|try_element_at |SELECT try_element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|try_multiply |SELECT try_multiply(2, 3); |PASSED |OK |std_err: Error during planning: Invalid function 'try_multiply'.\nDid you mean 'array_union'?\n | -|try_subtract |SELECT try_subtract(2, 1); |PASSED |OK |std_err: Error during planning: Invalid function 'try_subtract'.\nDid you mean 'array_extract'?\n | -|try_sum |SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'try_sum'.\nDid you mean 'trim'?\n | -|try_to_binary |SELECT try_to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'try_to_binary'.\nDid you mean 'string_to_array'?\n | -|try_to_number |SELECT try_to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'try_to_number'.\nDid you mean 'array_to_string'?\n | -|try_to_timestamp |SELECT try_to_timestamp('2016-12-31 00:12:00'); |PASSED |OK |std_err: Error during planning: Invalid function 'try_to_timestamp'.\nDid you mean 'to_timestamp'?\n | -|typeof |SELECT typeof(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'typeof'.\nDid you mean 'repeat'?\n | -|ucase |SELECT ucase('SparkSql'); |PASSED |OK |std_err: Error during planning: Invalid function 'ucase'.\nDid you mean 'acos'?\n | -|unbase64 |SELECT unbase64('U3BhcmsgU1FM'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unbase64'.\nDid you mean 'sha256'?\n | -|unhex |SELECT decode(unhex('537061726B2053514C'), 'UTF-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unhex'.\nDid you mean 'upper'?\n | -|unix_date |SELECT unix_date(DATE("1970-01-02")); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_date'.\nDid you mean 'to_date'?\n | -|unix_micros |SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_micros'.\nDid you mean 'initcap'?\n | -|unix_millis |SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_millis'.\nDid you mean 'nullif'?\n | -|unix_seconds |SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_seconds'.\nDid you mean 'decode'?\n | -|unix_timestamp |SELECT unix_timestamp(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_timestamp'.\nDid you mean 'to_timestamp'?\n | -|upper |SELECT upper('SparkSql'); |PASSED |OK |OK | -|url_decode |SELECT url_decode('https%3A%2F%2Fspark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'url_decode'.\nDid you mean 'decode'?\n | -|url_encode |SELECT url_encode('https://spark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'url_encode'.\nDid you mean 'encode'?\n | -|user |SELECT user(); |PASSED |OK |std_err: SQL error: ParserError("Expected end of statement, found: (")\n | -|uuid |SELECT uuid(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|var_pop |SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|var_samp |SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | -|variance |SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v37.1.0\nError: Error during planning: Invalid function 'variance'.\nDid you mean 'range'?\n | -|version |SELECT version(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'version'.\nDid you mean 'sin'?\n | -|weekday |SELECT weekday('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'weekday'.\nDid you mean 'today'?\n | -|weekofyear |SELECT weekofyear('2008-02-20'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'weekofyear'.\nDid you mean 'repeat'?\n | -|when |SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END; |PASSED |OK |OK | -|width_bucket |SELECT width_bucket(5.3, 0.2, 10.6, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'width_bucket'.\nDid you mean 'list_cat'?\n | -|xpath |SELECT xpath('b1b2b3c1c2','a/b/text()'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath'.\nDid you mean 'tanh'?\n | -|xpath_boolean |SELECT xpath_boolean('1','a/b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_boolean'.\nDid you mean 'date_format'?\n | -|xpath_double |SELECT xpath_double('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_double'.\nDid you mean 'date_format'?\n | -|xpath_float |SELECT xpath_float('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_float'.\nDid you mean 'date_format'?\n | -|xpath_int |SELECT xpath_int('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_int'.\nDid you mean 'date_bin'?\n | -|xpath_long |SELECT xpath_long('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_long'.\nDid you mean 'date_bin'?\n | -|xpath_number |SELECT xpath_number('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_number'.\nDid you mean 'date_bin'?\n | -|xpath_short |SELECT xpath_short('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_short'.\nDid you mean 'list_sort'?\n | -|xpath_string |SELECT xpath_string('bcc','a/c'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_string'.\nDid you mean 'date_trunc'?\n | -|xxhash64 |SELECT xxhash64('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n | -|year |SELECT year('2016-07-30'); |PASSED |OK |std_err: Error during planning: Invalid function 'year'.\nDid you mean 'VAR'?\n | -|zip_with |SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | -|| |SELECT 3 | 5; |PASSED |OK |OK | -|~ |SELECT ~ 0; |FAILED |Failed on native side: found CometNativeException |std_err: SQL error: ParserError("Expected an expression:, found: ~")\n | -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ +|name |query |result |cometMessage |datafusionMessage | ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ +|! |SELECT ! true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|% |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|& |SELECT 3 & 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|* |SELECT 2 * 3; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|+ |SELECT 1 + 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|- |SELECT 2 - 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|/ |SELECT 3 / 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|< |SELECT 1 < 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|<= |SELECT 2 <= 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|<=> |SELECT 2 <=> 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|= |SELECT 2 = 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|== |SELECT 2 == 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|> |SELECT 2 > 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|>= |SELECT 2 >= 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|^ |SELECT 3 ^ 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|abs |SELECT abs(-1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|acos |SELECT acos(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|acosh |SELECT acosh(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|add_months |SELECT add_months('2016-08-31', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|aes_decrypt |SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|aes_encrypt |SELECT hex(aes_encrypt('Spark', '0000111122223333')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|aggregate |SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|and |SELECT true and true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|any |SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|any_value |SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|approx_count_distinct |SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|approx_percentile |SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array |SELECT array(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_agg |SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_append |SELECT array_append(array('b', 'd', 'c', 'a'), 'd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_compact |SELECT array_compact(array(1, 2, 3, null)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_contains |SELECT array_contains(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_distinct |SELECT array_distinct(array(1, 2, 3, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_except |SELECT array_except(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_insert |SELECT array_insert(array(1, 2, 3, 4), 5, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_intersect |SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_join |SELECT array_join(array('hello', 'world'), ' '); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_max |SELECT array_max(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_min |SELECT array_min(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_position |SELECT array_position(array(3, 2, 1), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_remove |SELECT array_remove(array(1, 2, 3, null, 3), 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_repeat |SELECT array_repeat('123', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_size |SELECT array_size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_sort |SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_union |SELECT array_union(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|arrays_overlap |SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|arrays_zip |SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ascii |SELECT ascii('222'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|asin |SELECT asin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|asinh |SELECT asinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|assert_true |SELECT assert_true(0 < 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|atan |SELECT atan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|atan2 |SELECT atan2(0, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|atanh |SELECT atanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|avg |SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|base64 |SELECT base64('Spark SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bigint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bin |SELECT bin(13); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|binary | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bit_and |SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bit_count |SELECT bit_count(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bit_get |SELECT bit_get(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bit_length |SELECT bit_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|bit_or |SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bit_xor |SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bool_and |SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bool_or |SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|boolean | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bround |SELECT bround(2.5, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|btrim |SELECT btrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|cardinality |SELECT cardinality(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cast |SELECT cast('10' as int); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|cbrt |SELECT cbrt(27.0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ceil |SELECT ceil(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|ceiling |SELECT ceiling(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|char |SELECT char(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|char_length |SELECT char_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|character_length |SELECT character_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|chr |SELECT chr(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|coalesce |SELECT coalesce(NULL, 1, NULL); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|collect_list |SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|collect_set |SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|concat |SELECT concat('Spark', 'SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|concat_ws |SELECT concat_ws(' ', 'Spark', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|contains |SELECT contains('Spark SQL', 'Spark'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|conv |SELECT conv('100', 2, 10); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|convert_timezone |SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|corr |SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cos |SELECT cos(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|cosh |SELECT cosh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cot |SELECT cot(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|count |SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|count_if |SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|count_min_sketch |SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|covar_pop |SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|covar_samp |SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|crc32 |SELECT crc32('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|csc |SELECT csc(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cume_dist |SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|curdate |SELECT curdate(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_catalog |SELECT current_catalog(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_database |SELECT current_database(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_date |SELECT current_date(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_schema |SELECT current_schema(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_timestamp |SELECT current_timestamp(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [current_timestamp() AS current_timestamp()#6412, x#6396]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#6396])\n +- Relation [x#6396] parquet\n\n== Analyzed Logical Plan ==\ncurrent_timestamp(): timestamp, x: string\nProject [current_timestamp() AS current_timestamp()#6412, x#6396]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#6396])\n +- Relation [x#6396] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-20 18:25:00.607744 AS current_timestamp()#6412, x#6396]\n+- Relation [x#6396] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [current_timestamp()#6412, x#6396], [2024-05-20 18:25:00.607744 AS current_timestamp()#6412, x#6396]\n +- CometScan parquet [x#6396] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-7a..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-20 18:25:00.586056,dummy] [2024-05-20 18:25:00.607744,dummy]\n \n |DATAFUSIONCLI_PATH env variable not set| +|current_timezone |SELECT current_timezone(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_user |SELECT current_user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|date | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|date_add |SELECT date_add('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_diff |SELECT date_diff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_format |SELECT date_format('2016-04-08', 'y'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_from_unix_date |SELECT date_from_unix_date(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_part |SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|date_sub |SELECT date_sub('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_trunc |SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|dateadd |SELECT dateadd('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|datediff |SELECT datediff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|datepart |SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|day |SELECT day('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dayofmonth |SELECT dayofmonth('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dayofweek |SELECT dayofweek('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dayofyear |SELECT dayofyear('2016-04-09'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|decimal | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|decode |SELECT decode(encode('abc', 'utf-8'), 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|degrees |SELECT degrees(3.141592653589793); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dense_rank |SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|div |SELECT 3 div 2; |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|double | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|e |SELECT e(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|element_at |SELECT element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|elt |SELECT elt(1, 'scala', 'java'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|encode |SELECT encode('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|endswith |SELECT endswith('Spark SQL', 'SQL'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|equal_null |SELECT equal_null(3, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|every |SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|exists |SELECT exists(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|exp |SELECT exp(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|explode |SELECT explode(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|explode_outer |SELECT explode_outer(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|expm1 |SELECT expm1(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|extract |SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|factorial |SELECT factorial(5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|filter |SELECT filter(array(1, 2, 3), x -> x % 2 == 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|find_in_set |SELECT find_in_set('ab','abc,b,ab,c,def'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|first |SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|first_value |SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|flatten |SELECT flatten(array(array(1, 2), array(3, 4))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|float | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|floor |SELECT floor(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|forall |SELECT forall(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|format_number |SELECT format_number(12332.123456, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|format_string |SELECT format_string("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_csv |SELECT from_csv('1, 0.8', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_json |SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_unixtime |SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_utc_timestamp |SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|get |SELECT get(array(1, 2, 3), 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|get_json_object |SELECT get_json_object('{"a":"b"}', '$.a'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|getbit |SELECT getbit(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|greatest |SELECT greatest(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|grouping |SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name); |FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#5732, spark_grouping_id#5731L], [age#5728, name#5732]\n+- Expand [[age#5728, name#5729, name#5730, 0], [age#5728, name#5729, null, 1]], [age#5728, name#5729, name#5732, spark_grouping_id#5731L]\n +- Project [age#5728, name#5729, name#5729 AS name#5730]\n +- SubqueryAlias people\n +- LocalRelation [age#5728, name#5729]\n | | +|grouping_id |SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);|FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#6188, height#6189, spark_grouping_id#6187L], [age#6182, name#6188, height#6189]\n+- Expand [[age#6182, name#6183, height#6184, name#6185, height#6186, 0], [age#6182, name#6183, height#6184, name#6185, null, 1], [age#6182, name#6183, height#6184, null, height#6186, 2], [age#6182, name#6183, height#6184, null, null, 3]], [age#6182, name#6183, height#6184, name#6188, height#6189, spark_grouping_id#6187L]\n +- Project [age#6182, name#6183, height#6184, name#6183 AS name#6185, height#6184 AS height#6186]\n +- SubqueryAlias people\n +- LocalRelation [age#6182, name#6183, height#6184]\n | | +|hash |SELECT hash('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|hex |SELECT hex(17); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|histogram_numeric |SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|hour |SELECT hour('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|hypot |SELECT hypot(3, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|if |SELECT if(1 < 2, 'a', 'b'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|ifnull |SELECT ifnull(NULL, array('2')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ilike |SELECT ilike('Spark', '_Park'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|in |SELECT 1 in(1, 2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|initcap |SELECT initcap('sPark sql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|inline |SELECT inline(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|inline_outer |SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|input_file_block_length |SELECT input_file_block_length(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|input_file_block_start |SELECT input_file_block_start(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|input_file_name |SELECT input_file_name(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|instr |SELECT instr('SparkSQL', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|int | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|isnan |SELECT isnan(cast('NaN' as double)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|isnotnull |SELECT isnotnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|isnull |SELECT isnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|java_method |SELECT java_method('java.util.UUID', 'randomUUID'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|json_array_length |SELECT json_array_length('[1,2,3,4]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|json_object_keys |SELECT json_object_keys('{}'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|json_tuple |SELECT json_tuple('{"a":1, "b":2}', 'a', 'b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|kurtosis |SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|lag |SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|last |SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|last_day |SELECT last_day('2009-01-12'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|last_value |SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|lcase |SELECT lcase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|lead |SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|least |SELECT least(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|left |SELECT left('Spark SQL', 3); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|len |SELECT len('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|length |SELECT length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|levenshtein |SELECT levenshtein('kitten', 'sitting'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|like |SELECT like('Spark', '_park'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|ln |SELECT ln(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|localtimestamp |SELECT localtimestamp(); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|locate |SELECT locate('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|log |SELECT log(10, 100); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|log10 |SELECT log10(10); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|log1p |SELECT log1p(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|log2 |SELECT log2(2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|lower |SELECT lower('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|lpad |SELECT lpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ltrim |SELECT ltrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|make_date |SELECT make_date(2013, 7, 15); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_dt_interval |SELECT make_dt_interval(1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_interval |SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_timestamp |SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_timestamp_ltz |SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_timestamp_ntz |SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_ym_interval |SELECT make_ym_interval(1, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map |SELECT map(1.0, '2', 3.0, '4'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_concat |SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_contains_key |SELECT map_contains_key(map(1, 'a', 2, 'b'), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_entries |SELECT map_entries(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_filter |SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_from_arrays |SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_from_entries |SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_keys |SELECT map_keys(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_values |SELECT map_values(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_zip_with |SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|mask |SELECT mask('abcd-EFGH-8765-4321'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|max |SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|max_by |SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|md5 |SELECT md5('Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|mean |SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|median |SELECT median(col) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|min |SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|min_by |SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|minute |SELECT minute('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|mod |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|mode |SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|monotonically_increasing_id|SELECT monotonically_increasing_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|month |SELECT month('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|months_between |SELECT months_between('1997-02-28 10:30:00', '1996-10-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|named_struct |SELECT named_struct("a", 1, "b", 2, "c", 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|nanvl |SELECT nanvl(cast('NaN' as double), 123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|negative |SELECT negative(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|next_day |SELECT next_day('2015-01-14', 'TU'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|not |SELECT not true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|now |SELECT now(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [now() AS now()#1254, x#1238]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#1238])\n +- Relation [x#1238] parquet\n\n== Analyzed Logical Plan ==\nnow(): timestamp, x: string\nProject [now() AS now()#1254, x#1238]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#1238])\n +- Relation [x#1238] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-20 18:24:37.462092 AS now()#1254, x#1238]\n+- Relation [x#1238] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [now()#1254, x#1238], [2024-05-20 18:24:37.462092 AS now()#1254, x#1238]\n +- CometScan parquet [x#1238] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-01..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-20 18:24:37.434216,dummy] [2024-05-20 18:24:37.462092,dummy]\n \n |DATAFUSIONCLI_PATH env variable not set| +|nth_value |SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ntile |SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|nullif |SELECT nullif(2, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|nvl |SELECT nvl(NULL, array('2')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|nvl2 |SELECT nvl2(NULL, 2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|octet_length |SELECT octet_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|or |SELECT true or false; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|overlay |SELECT overlay('Spark SQL' PLACING '_' FROM 6); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|parse_url |SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|percent_rank |SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|percentile |SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|percentile_approx |SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|pi |SELECT pi(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|pmod |SELECT pmod(10, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|posexplode |SELECT posexplode(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|posexplode_outer |SELECT posexplode_outer(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|position |SELECT position('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|positive |SELECT positive(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|pow |SELECT pow(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|power |SELECT power(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|printf |SELECT printf("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|quarter |SELECT quarter('2016-08-31'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|radians |SELECT radians(180); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|raise_error |SELECT raise_error('custom error message'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rand |SELECT rand(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|randn |SELECT randn(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|random |SELECT random(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rank |SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|reduce |SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|reflect |SELECT reflect('java.util.UUID', 'randomUUID'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp |SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_count |SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_extract |SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_extract_all |SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_instr |SELECT regexp_instr('user@spark.apache.org', '@[^.]*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_like |SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_replace |SELECT regexp_replace('100-200', '(\\d+)', 'num'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_substr |SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_avgx |SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_avgy |SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_count |SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_intercept |SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_r2 |SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_slope |SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_sxx |SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_sxy |SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_syy |SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|repeat |SELECT repeat('123', 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|replace |SELECT replace('ABCabc', 'abc', 'DEF'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|reverse |SELECT reverse('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|right |SELECT right('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rint |SELECT rint(12.3456); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rlike |SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|round |SELECT round(2.5, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|row_number |SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rpad |SELECT rpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rtrim |SELECT rtrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|schema_of_csv |SELECT schema_of_csv('1,abc'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|schema_of_json |SELECT schema_of_json('[{"col":0}]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sec |SELECT sec(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|second |SELECT second('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|sentences |SELECT sentences('Hi there! Good morning.'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sequence |SELECT sequence(1, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sha |SELECT sha('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sha1 |SELECT sha1('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sha2 |SELECT sha2('Spark', 256); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|shiftleft |SELECT shiftleft(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|shiftright |SELECT shiftright(4, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|shiftrightunsigned |SELECT shiftrightunsigned(4, 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|shuffle |SELECT shuffle(array(1, 20, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sign |SELECT sign(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|signum |SELECT signum(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|sin |SELECT sin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|sinh |SELECT sinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|size |SELECT size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|skewness |SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|slice |SELECT slice(array(1, 2, 3, 4), 2, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|smallint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|some |SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sort_array |SELECT sort_array(array('b', 'd', null, 'c', 'a'), true); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|soundex |SELECT soundex('Miller'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|space |SELECT concat(space(2), '1'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|spark_partition_id |SELECT spark_partition_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|split |SELECT split('oneAtwoBthreeC', '[ABC]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|split_part |SELECT split_part('11.12.13', '.', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sqrt |SELECT sqrt(4); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|stack |SELECT stack(2, 1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|startswith |SELECT startswith('Spark SQL', 'Spark'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|std |SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|stddev |SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|stddev_pop |SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|stddev_samp |SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|str_to_map |SELECT str_to_map('a:1,b:2,c:3', ',', ':'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|string | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|struct |SELECT struct(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|substr |SELECT substr('Spark SQL', 5); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|substring |SELECT substring('Spark SQL', 5); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|substring_index |SELECT substring_index('www.apache.org', '.', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sum |SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|tan |SELECT tan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|tanh |SELECT tanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|timestamp | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|timestamp_micros |SELECT timestamp_micros(1230219000123123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|timestamp_millis |SELECT timestamp_millis(1230219000123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|timestamp_seconds |SELECT timestamp_seconds(1230219000); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|tinyint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|to_binary |SELECT to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_char |SELECT to_char(454, '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_csv |SELECT to_csv(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_date |SELECT to_date('2009-07-30 04:17:52'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|to_json |SELECT to_json(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_number |SELECT to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_timestamp |SELECT to_timestamp('2016-12-31 00:12:00'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|to_timestamp_ltz |SELECT to_timestamp_ltz('2016-12-31 00:12:00'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|to_timestamp_ntz |SELECT to_timestamp_ntz('2016-12-31 00:12:00'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|to_unix_timestamp |SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_utc_timestamp |SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|transform |SELECT transform(array(1, 2, 3), x -> x + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|transform_keys |SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|transform_values |SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|translate |SELECT translate('AaBbCc', 'abc', '123'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|trim |SELECT trim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|trunc |SELECT trunc('2019-08-04', 'week'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|try_add |SELECT try_add(1, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_avg |SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_divide |SELECT try_divide(3, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_element_at |SELECT try_element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_multiply |SELECT try_multiply(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_subtract |SELECT try_subtract(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_sum |SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_to_binary |SELECT try_to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_to_number |SELECT try_to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_to_timestamp |SELECT try_to_timestamp('2016-12-31 00:12:00'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|typeof |SELECT typeof(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ucase |SELECT ucase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|unbase64 |SELECT unbase64('U3BhcmsgU1FM'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unhex |SELECT decode(unhex('537061726B2053514C'), 'UTF-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_date |SELECT unix_date(DATE("1970-01-02")); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_micros |SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_millis |SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_seconds |SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_timestamp |SELECT unix_timestamp(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|upper |SELECT upper('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|url_decode |SELECT url_decode('https%3A%2F%2Fspark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|url_encode |SELECT url_encode('https://spark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|user |SELECT user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|uuid |SELECT uuid(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|var_pop |SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|var_samp |SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|variance |SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|version |SELECT version(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|weekday |SELECT weekday('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|weekofyear |SELECT weekofyear('2008-02-20'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|when |SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|width_bucket |SELECT width_bucket(5.3, 0.2, 10.6, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath |SELECT xpath('b1b2b3c1c2','a/b/text()'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_boolean |SELECT xpath_boolean('1','a/b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_double |SELECT xpath_double('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_float |SELECT xpath_float('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_int |SELECT xpath_int('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_long |SELECT xpath_long('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_number |SELECT xpath_number('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_short |SELECT xpath_short('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_string |SELECT xpath_string('bcc','a/c'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xxhash64 |SELECT xxhash64('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|year |SELECT year('2016-07-30'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|zip_with |SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|| |SELECT 3 | 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|~ |SELECT ~ 0; |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ diff --git a/docs/spark_expressions_support.md b/docs/spark_expressions_support.md new file mode 100644 index 000000000..c48426669 --- /dev/null +++ b/docs/spark_expressions_support.md @@ -0,0 +1,477 @@ + + +# Supported Spark Expressions + +### agg_funcs + - [ ] any + - [ ] any_value + - [ ] approx_count_distinct + - [ ] approx_percentile + - [ ] array_agg + - [ ] avg + - [ ] bit_and + - [ ] bit_or + - [ ] bit_xor + - [ ] bool_and + - [ ] bool_or + - [ ] collect_list + - [ ] collect_set + - [ ] corr + - [ ] count + - [ ] count_if + - [ ] count_min_sketch + - [ ] covar_pop + - [ ] covar_samp + - [ ] every + - [ ] first + - [ ] first_value + - [ ] grouping + - [ ] grouping_id + - [ ] histogram_numeric + - [ ] kurtosis + - [ ] last + - [ ] last_value + - [ ] max + - [ ] max_by + - [ ] mean + - [ ] median + - [ ] min + - [ ] min_by + - [ ] mode + - [ ] percentile + - [ ] percentile_approx + - [ ] regr_avgx + - [ ] regr_avgy + - [ ] regr_count + - [ ] regr_intercept + - [ ] regr_r2 + - [ ] regr_slope + - [ ] regr_sxx + - [ ] regr_sxy + - [ ] regr_syy + - [ ] skewness + - [ ] some + - [ ] std + - [ ] stddev + - [ ] stddev_pop + - [ ] stddev_samp + - [ ] sum + - [ ] try_avg + - [ ] try_sum + - [ ] var_pop + - [ ] var_samp + - [ ] variance + +### array_funcs + - [ ] array + - [ ] array_append + - [ ] array_compact + - [ ] array_contains + - [ ] array_distinct + - [ ] array_except + - [ ] array_insert + - [ ] array_intersect + - [ ] array_join + - [ ] array_max + - [ ] array_min + - [ ] array_position + - [ ] array_remove + - [ ] array_repeat + - [ ] array_union + - [ ] arrays_overlap + - [ ] arrays_zip + - [ ] flatten + - [x] get + - [ ] sequence + - [ ] shuffle + - [ ] slice + - [ ] sort_array + +### bitwise_funcs + - [x] & + - [x] ^ + - [ ] bit_count + - [ ] bit_get + - [ ] getbit + - [x] shiftright + - [ ] shiftrightunsigned + - [x] | + - [ ] ~ + +### collection_funcs + - [ ] array_size + - [ ] cardinality + - [ ] concat + - [x] reverse + - [ ] size + +### conditional_funcs + - [x] coalesce + - [x] if + - [ ] ifnull + - [ ] nanvl + - [x] nullif + - [ ] nvl + - [x] nvl2 + - [x] when + +### conversion_funcs + - [ ] bigint + - [ ] binary + - [ ] boolean + - [x] cast + - [ ] date + - [ ] decimal + - [ ] double + - [ ] float + - [ ] int + - [ ] smallint + - [ ] string + - [ ] timestamp + - [ ] tinyint + +### csv_funcs + - [ ] from_csv + - [ ] schema_of_csv + - [ ] to_csv + +### datetime_funcs + - [ ] add_months + - [ ] convert_timezone + - [x] curdate + - [x] current_date + - [ ] current_timestamp + - [x] current_timezone + - [ ] date_add + - [ ] date_diff + - [ ] date_format + - [ ] date_from_unix_date + - [x] date_part + - [ ] date_sub + - [ ] date_trunc + - [ ] dateadd + - [ ] datediff + - [x] datepart + - [ ] day + - [ ] dayofmonth + - [ ] dayofweek + - [ ] dayofyear + - [x] extract + - [ ] from_unixtime + - [ ] from_utc_timestamp + - [ ] hour + - [ ] last_day + - [ ] localtimestamp + - [ ] make_date + - [ ] make_dt_interval + - [ ] make_interval + - [ ] make_timestamp + - [ ] make_timestamp_ltz + - [ ] make_timestamp_ntz + - [ ] make_ym_interval + - [ ] minute + - [ ] month + - [ ] months_between + - [ ] next_day + - [ ] now + - [ ] quarter + - [ ] second + - [ ] timestamp_micros + - [ ] timestamp_millis + - [ ] timestamp_seconds + - [x] to_date + - [x] to_timestamp + - [x] to_timestamp_ltz + - [ ] to_timestamp_ntz + - [ ] to_unix_timestamp + - [ ] to_utc_timestamp + - [ ] trunc + - [x] try_to_timestamp + - [ ] unix_date + - [ ] unix_micros + - [ ] unix_millis + - [ ] unix_seconds + - [ ] unix_timestamp + - [ ] weekday + - [ ] weekofyear + - [x] year + +### generator_funcs + - [ ] explode + - [ ] explode_outer + - [ ] inline + - [ ] inline_outer + - [ ] posexplode + - [ ] posexplode_outer + - [ ] stack + +### hash_funcs + - [ ] crc32 + - [ ] hash + - [x] md5 + - [ ] sha + - [ ] sha1 + - [x] sha2 + - [ ] xxhash64 + +### json_funcs + - [ ] from_json + - [ ] get_json_object + - [ ] json_array_length + - [ ] json_object_keys + - [ ] json_tuple + - [ ] schema_of_json + - [ ] to_json + +### lambda_funcs + - [ ] aggregate + - [ ] array_sort + - [ ] exists + - [ ] filter + - [ ] forall + - [ ] map_filter + - [ ] map_zip_with + - [ ] reduce + - [ ] transform + - [ ] transform_keys + - [ ] transform_values + - [ ] zip_with + +### map_funcs + - [ ] element_at + - [ ] map + - [ ] map_concat + - [ ] map_contains_key + - [ ] map_entries + - [ ] map_from_arrays + - [ ] map_from_entries + - [ ] map_keys + - [ ] map_values + - [ ] str_to_map + - [ ] try_element_at + +### math_funcs + - [x] % + - [x] * + - [x] + + - [x] - + - [x] / + - [x] abs + - [x] acos + - [ ] acosh + - [x] asin + - [ ] asinh + - [x] atan + - [x] atan2 + - [ ] atanh + - [ ] bin + - [ ] bround + - [ ] cbrt + - [x] ceil + - [x] ceiling + - [ ] conv + - [x] cos + - [ ] cosh + - [ ] cot + - [ ] csc + - [ ] degrees + - [ ] div + - [ ] e + - [x] exp + - [ ] expm1 + - [ ] factorial + - [x] floor + - [ ] greatest + - [ ] hex + - [ ] hypot + - [ ] least + - [x] ln + - [ ] log + - [x] log10 + - [ ] log1p + - [x] log2 + - [x] mod + - [x] negative + - [ ] pi + - [ ] pmod + - [x] positive + - [x] pow + - [x] power + - [ ] radians + - [ ] rand + - [ ] randn + - [ ] random + - [ ] rint + - [x] round + - [ ] sec + - [x] shiftleft + - [x] sign + - [x] signum + - [x] sin + - [ ] sinh + - [x] sqrt + - [x] tan + - [ ] tanh + - [x] try_add + - [x] try_divide + - [x] try_multiply + - [x] try_subtract + - [ ] unhex + - [ ] width_bucket + +### misc_funcs + - [ ] aes_decrypt + - [ ] aes_encrypt + - [ ] assert_true + - [x] current_catalog + - [x] current_database + - [x] current_schema + - [x] current_user + - [x] equal_null + - [ ] input_file_block_length + - [ ] input_file_block_start + - [ ] input_file_name + - [ ] java_method + - [ ] monotonically_increasing_id + - [ ] raise_error + - [ ] reflect + - [ ] spark_partition_id + - [ ] typeof + - [x] user + - [ ] uuid + - [ ] version + +### predicate_funcs + - [x] ! + - [x] < + - [x] <= + - [x] <=> + - [x] = + - [x] == + - [x] > + - [x] >= + - [x] and + - [ ] ilike + - [x] in + - [ ] isnan + - [x] isnotnull + - [x] isnull + - [ ] like + - [x] not + - [x] or + - [ ] regexp + - [ ] regexp_like + - [ ] rlike + +### string_funcs + - [x] ascii + - [ ] base64 + - [x] bit_length + - [x] btrim + - [x] char + - [x] char_length + - [x] character_length + - [x] chr + - [x] concat_ws + - [ ] contains + - [ ] decode + - [ ] elt + - [ ] encode + - [ ] endswith + - [ ] find_in_set + - [ ] format_number + - [ ] format_string + - [x] initcap + - [x] instr + - [x] lcase + - [ ] left + - [x] len + - [x] length + - [ ] levenshtein + - [ ] locate + - [x] lower + - [ ] lpad + - [x] ltrim + - [ ] mask + - [x] octet_length + - [ ] overlay + - [ ] position + - [ ] printf + - [ ] regexp_count + - [ ] regexp_extract + - [ ] regexp_extract_all + - [ ] regexp_instr + - [ ] regexp_replace + - [ ] regexp_substr + - [x] repeat + - [x] replace + - [ ] right + - [ ] rpad + - [x] rtrim + - [ ] sentences + - [ ] soundex + - [ ] space + - [ ] split + - [ ] split_part + - [ ] startswith + - [ ] substr + - [ ] substring + - [ ] substring_index + - [ ] to_binary + - [ ] to_char + - [ ] to_number + - [x] translate + - [x] trim + - [ ] try_to_binary + - [ ] try_to_number + - [x] ucase + - [ ] unbase64 + - [x] upper + +### struct_funcs + - [ ] named_struct + - [ ] struct + +### url_funcs + - [ ] parse_url + - [ ] url_decode + - [ ] url_encode + +### window_funcs + - [ ] cume_dist + - [ ] dense_rank + - [ ] lag + - [ ] lead + - [ ] nth_value + - [ ] ntile + - [ ] percent_rank + - [ ] rank + - [ ] row_number + +### xml_funcs + - [ ] xpath + - [ ] xpath_boolean + - [ ] xpath_double + - [ ] xpath_float + - [ ] xpath_int + - [ ] xpath_long + - [ ] xpath_number + - [ ] xpath_short + - [ ] xpath_string \ No newline at end of file diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 26c9c8fec..ca9508cb5 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -27,12 +27,14 @@ import scala.sys.process._ import org.scalatest.Ignore import org.scalatest.exceptions.TestFailedException +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import org.apache.hadoop.fs.Path -import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.{CometTestBase, DataFrame} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.functions.collect_list -import org.apache.comet.CoverageResultStatus.CoverageResultStatus +import org.apache.comet.CoverageResultStatus.{CoverageResultStatus, Passed} /** * Manual test to calculate Spark builtin expressions coverage support by the Comet @@ -46,13 +48,15 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH private val projectDocFolder = "docs" private val rawCoverageFilePath = s"$projectDocFolder/spark_builtin_expr_coverage.txt" + private val mdCoverageFilePath = s"$projectDocFolder/spark_expressions_support.md" private val DATAFUSIONCLI_PATH_ENV_VAR = "DATAFUSIONCLI_PATH" - private val queryPattern = """(?i)SELECT (.+?);""".r private val valuesPattern = """(?i)FROM VALUES(.+?);""".r private val selectPattern = """(i?)SELECT(.+?)FROM""".r - def getExamples(): Map[String, List[String]] = + // key - function name + // value - examples + def getExamples(): Map[FunctionInfo, List[String]] = spark.sessionState.functionRegistry .listFunction() .map(spark.sessionState.catalog.lookupFunctionInfo(_)) @@ -62,7 +66,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH !List("window", "session_window", "window_time").contains(f.getName.toLowerCase)) .map(f => { val selectRows = queryPattern.findAllMatchIn(f.getExamples).map(_.group(0)).toList - (f.getName, selectRows.filter(_.nonEmpty)) + (FunctionInfo(f.getName, f.getGroup), selectRows.filter(_.nonEmpty)) }) .toMap @@ -80,7 +84,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH val resultsMap = new mutable.HashMap[String, CoverageResult]() builtinExamplesMap.foreach { - case (funcName, q :: _) => + case (func, q :: _) => var dfMessage: Option[String] = None val queryResult = try { @@ -123,7 +127,8 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH CoverageResultStatus.Passed, CoverageResultDetails( cometMessage = "OK", - datafusionMessage = dfMessage.getOrElse("OK"))) + datafusionMessage = dfMessage.getOrElse("OK")), + group = func.group) } catch { case e: TestFailedException @@ -134,7 +139,8 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH CoverageResultDetails( cometMessage = "Unsupported: Expected only Comet native operators but found Spark fallback", - datafusionMessage = dfMessage.getOrElse(""))) + datafusionMessage = dfMessage.getOrElse("")), + group = func.group) case e if e.getMessage.contains("CometNativeException") => CoverageResult( @@ -142,7 +148,8 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH CoverageResultStatus.Failed, CoverageResultDetails( cometMessage = "Failed on native side: found CometNativeException", - datafusionMessage = dfMessage.getOrElse(""))) + datafusionMessage = dfMessage.getOrElse("")), + group = func.group) case e => CoverageResult( @@ -150,23 +157,24 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH CoverageResultStatus.Failed, CoverageResultDetails( cometMessage = e.getMessage, - datafusionMessage = dfMessage.getOrElse(""))) + datafusionMessage = dfMessage.getOrElse("")), + group = func.group) } - resultsMap.put(funcName, queryResult) + resultsMap.put(func.name, queryResult) // Function with no examples - case (funcName, List()) => + case (func, List()) => resultsMap.put( - funcName, + func.name, CoverageResult( "", CoverageResultStatus.Skipped, CoverageResultDetails( cometMessage = "No examples found in spark.sessionState.functionRegistry", - datafusionMessage = ""))) + datafusionMessage = ""), + group = func.group)) } - // TODO: convert results into HTML or .md file resultsMap.toSeq.toDF("name", "details").createOrReplaceTempView("t") val str = showString( @@ -175,6 +183,86 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH 1000, 0) Files.write(Paths.get(rawCoverageFilePath), str.getBytes(StandardCharsets.UTF_8)) + Files.write( + Paths.get(mdCoverageFilePath), + generateMarkdown(spark.sql("select * from t")).getBytes(StandardCharsets.UTF_8)) + } + + test("Test markdown") { + val map = new scala.collection.mutable.HashMap[String, CoverageResult]() + map.put( + "f1", + CoverageResult("q1", CoverageResultStatus.Passed, CoverageResultDetails("", ""), "group1")) + map.put( + "f2", + CoverageResult( + "q2", + CoverageResultStatus.Failed, + CoverageResultDetails("err", "err"), + "group1")) + map.put( + "f3", + CoverageResult("q3", CoverageResultStatus.Passed, CoverageResultDetails("", ""), "group2")) + map.put( + "f4", + CoverageResult( + "q4", + CoverageResultStatus.Failed, + CoverageResultDetails("err", "err"), + "group2")) + map.put( + "f5", + CoverageResult("q5", CoverageResultStatus.Passed, CoverageResultDetails("", ""), "group3")) + val str = generateMarkdown(map.toSeq.toDF("name", "details")) + str shouldBe s"${getLicenseHeader()}\n# Supported Spark Expressions\n\n### group1\n - [x] f1\n - [ ] f2\n\n### group2\n - [x] f3\n - [ ] f4\n\n### group3\n - [x] f5" + } + + def generateMarkdown(df: DataFrame): String = { + val groupedDF = df + .orderBy("name") + .groupBy("details.group") + .agg(collect_list("name").as("names"), collect_list("details.result").as("statuses")) + .orderBy("group") + val sb = new StringBuilder(s"${getLicenseHeader()}\n# Supported Spark Expressions") + groupedDF.collect().foreach { row => + val groupName = row.getAs[String]("group") + val names = row.getAs[Seq[String]]("names") + val statuses = row.getAs[Seq[String]]("statuses") + + val passedMarks = names + .zip(statuses) + .map(x => + x._2 match { + case s if s == Passed.toString => s" - [x] ${x._1}" + case _ => s" - [ ] ${x._1}" + }) + + sb.append(s"\n\n### $groupName\n" + passedMarks.mkString("\n")) + } + + sb.result() + } + + private def getLicenseHeader(): String = { + """ + |""".stripMargin } // Returns execution error, None means successful execution @@ -224,6 +312,9 @@ object CoverageResultStatus extends Enumeration { case class CoverageResult( query: String, result: CoverageResultStatus, - details: CoverageResultDetails) + details: CoverageResultDetails, + group: String) case class CoverageResultDetails(cometMessage: String, datafusionMessage: String) + +case class FunctionInfo(name: String, group: String) From da1af5908248341329b43125ca44325fe80ca59f Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 24 May 2024 14:28:40 -0700 Subject: [PATCH 11/20] avoid spark skip comet for literals --- docs/spark_builtin_expr_coverage.txt | 836 +++++++++--------- docs/spark_expressions_support.md | 104 ++- .../comet/CometExpressionCoverageSuite.scala | 147 ++- .../apache/comet/CometExpressionSuite.scala | 30 + .../org/apache/spark/sql/CometTestBase.scala | 6 +- 5 files changed, 637 insertions(+), 486 deletions(-) diff --git a/docs/spark_builtin_expr_coverage.txt b/docs/spark_builtin_expr_coverage.txt index d30e9c095..ceb2b0814 100644 --- a/docs/spark_builtin_expr_coverage.txt +++ b/docs/spark_builtin_expr_coverage.txt @@ -1,419 +1,417 @@ -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ -|name |query |result |cometMessage |datafusionMessage | -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ -|! |SELECT ! true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|% |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|& |SELECT 3 & 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|* |SELECT 2 * 3; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|+ |SELECT 1 + 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|- |SELECT 2 - 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|/ |SELECT 3 / 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|< |SELECT 1 < 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|<= |SELECT 2 <= 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|<=> |SELECT 2 <=> 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|= |SELECT 2 = 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|== |SELECT 2 == 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|> |SELECT 2 > 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|>= |SELECT 2 >= 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|^ |SELECT 3 ^ 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|abs |SELECT abs(-1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|acos |SELECT acos(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|acosh |SELECT acosh(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|add_months |SELECT add_months('2016-08-31', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|aes_decrypt |SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|aes_encrypt |SELECT hex(aes_encrypt('Spark', '0000111122223333')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|aggregate |SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|and |SELECT true and true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|any |SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|any_value |SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|approx_count_distinct |SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|approx_percentile |SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array |SELECT array(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_agg |SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_append |SELECT array_append(array('b', 'd', 'c', 'a'), 'd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_compact |SELECT array_compact(array(1, 2, 3, null)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_contains |SELECT array_contains(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_distinct |SELECT array_distinct(array(1, 2, 3, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_except |SELECT array_except(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_insert |SELECT array_insert(array(1, 2, 3, 4), 5, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_intersect |SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_join |SELECT array_join(array('hello', 'world'), ' '); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_max |SELECT array_max(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_min |SELECT array_min(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_position |SELECT array_position(array(3, 2, 1), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_remove |SELECT array_remove(array(1, 2, 3, null, 3), 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_repeat |SELECT array_repeat('123', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_size |SELECT array_size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_sort |SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_union |SELECT array_union(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|arrays_overlap |SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|arrays_zip |SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ascii |SELECT ascii('222'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|asin |SELECT asin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|asinh |SELECT asinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|assert_true |SELECT assert_true(0 < 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|atan |SELECT atan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|atan2 |SELECT atan2(0, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|atanh |SELECT atanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|avg |SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|base64 |SELECT base64('Spark SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bigint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bin |SELECT bin(13); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|binary | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bit_and |SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bit_count |SELECT bit_count(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bit_get |SELECT bit_get(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bit_length |SELECT bit_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|bit_or |SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bit_xor |SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bool_and |SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bool_or |SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|boolean | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bround |SELECT bround(2.5, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|btrim |SELECT btrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|cardinality |SELECT cardinality(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cast |SELECT cast('10' as int); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|cbrt |SELECT cbrt(27.0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ceil |SELECT ceil(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|ceiling |SELECT ceiling(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|char |SELECT char(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|char_length |SELECT char_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|character_length |SELECT character_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|chr |SELECT chr(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|coalesce |SELECT coalesce(NULL, 1, NULL); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|collect_list |SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|collect_set |SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|concat |SELECT concat('Spark', 'SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|concat_ws |SELECT concat_ws(' ', 'Spark', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|contains |SELECT contains('Spark SQL', 'Spark'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|conv |SELECT conv('100', 2, 10); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|convert_timezone |SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|corr |SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cos |SELECT cos(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|cosh |SELECT cosh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cot |SELECT cot(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|count |SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|count_if |SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|count_min_sketch |SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|covar_pop |SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|covar_samp |SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|crc32 |SELECT crc32('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|csc |SELECT csc(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cume_dist |SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|curdate |SELECT curdate(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_catalog |SELECT current_catalog(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_database |SELECT current_database(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_date |SELECT current_date(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_schema |SELECT current_schema(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_timestamp |SELECT current_timestamp(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [current_timestamp() AS current_timestamp()#6412, x#6396]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#6396])\n +- Relation [x#6396] parquet\n\n== Analyzed Logical Plan ==\ncurrent_timestamp(): timestamp, x: string\nProject [current_timestamp() AS current_timestamp()#6412, x#6396]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#6396])\n +- Relation [x#6396] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-20 18:25:00.607744 AS current_timestamp()#6412, x#6396]\n+- Relation [x#6396] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [current_timestamp()#6412, x#6396], [2024-05-20 18:25:00.607744 AS current_timestamp()#6412, x#6396]\n +- CometScan parquet [x#6396] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-7a..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-20 18:25:00.586056,dummy] [2024-05-20 18:25:00.607744,dummy]\n \n |DATAFUSIONCLI_PATH env variable not set| -|current_timezone |SELECT current_timezone(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_user |SELECT current_user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|date | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|date_add |SELECT date_add('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_diff |SELECT date_diff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_format |SELECT date_format('2016-04-08', 'y'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_from_unix_date |SELECT date_from_unix_date(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_part |SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|date_sub |SELECT date_sub('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_trunc |SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|dateadd |SELECT dateadd('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|datediff |SELECT datediff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|datepart |SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|day |SELECT day('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dayofmonth |SELECT dayofmonth('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dayofweek |SELECT dayofweek('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dayofyear |SELECT dayofyear('2016-04-09'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|decimal | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|decode |SELECT decode(encode('abc', 'utf-8'), 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|degrees |SELECT degrees(3.141592653589793); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dense_rank |SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|div |SELECT 3 div 2; |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|double | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|e |SELECT e(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|element_at |SELECT element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|elt |SELECT elt(1, 'scala', 'java'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|encode |SELECT encode('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|endswith |SELECT endswith('Spark SQL', 'SQL'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|equal_null |SELECT equal_null(3, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|every |SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|exists |SELECT exists(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|exp |SELECT exp(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|explode |SELECT explode(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|explode_outer |SELECT explode_outer(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|expm1 |SELECT expm1(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|extract |SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|factorial |SELECT factorial(5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|filter |SELECT filter(array(1, 2, 3), x -> x % 2 == 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|find_in_set |SELECT find_in_set('ab','abc,b,ab,c,def'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|first |SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|first_value |SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|flatten |SELECT flatten(array(array(1, 2), array(3, 4))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|float | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|floor |SELECT floor(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|forall |SELECT forall(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|format_number |SELECT format_number(12332.123456, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|format_string |SELECT format_string("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_csv |SELECT from_csv('1, 0.8', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_json |SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_unixtime |SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_utc_timestamp |SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|get |SELECT get(array(1, 2, 3), 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|get_json_object |SELECT get_json_object('{"a":"b"}', '$.a'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|getbit |SELECT getbit(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|greatest |SELECT greatest(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|grouping |SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name); |FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#5732, spark_grouping_id#5731L], [age#5728, name#5732]\n+- Expand [[age#5728, name#5729, name#5730, 0], [age#5728, name#5729, null, 1]], [age#5728, name#5729, name#5732, spark_grouping_id#5731L]\n +- Project [age#5728, name#5729, name#5729 AS name#5730]\n +- SubqueryAlias people\n +- LocalRelation [age#5728, name#5729]\n | | -|grouping_id |SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);|FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#6188, height#6189, spark_grouping_id#6187L], [age#6182, name#6188, height#6189]\n+- Expand [[age#6182, name#6183, height#6184, name#6185, height#6186, 0], [age#6182, name#6183, height#6184, name#6185, null, 1], [age#6182, name#6183, height#6184, null, height#6186, 2], [age#6182, name#6183, height#6184, null, null, 3]], [age#6182, name#6183, height#6184, name#6188, height#6189, spark_grouping_id#6187L]\n +- Project [age#6182, name#6183, height#6184, name#6183 AS name#6185, height#6184 AS height#6186]\n +- SubqueryAlias people\n +- LocalRelation [age#6182, name#6183, height#6184]\n | | -|hash |SELECT hash('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|hex |SELECT hex(17); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|histogram_numeric |SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|hour |SELECT hour('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|hypot |SELECT hypot(3, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|if |SELECT if(1 < 2, 'a', 'b'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|ifnull |SELECT ifnull(NULL, array('2')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ilike |SELECT ilike('Spark', '_Park'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|in |SELECT 1 in(1, 2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|initcap |SELECT initcap('sPark sql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|inline |SELECT inline(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|inline_outer |SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|input_file_block_length |SELECT input_file_block_length(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|input_file_block_start |SELECT input_file_block_start(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|input_file_name |SELECT input_file_name(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|instr |SELECT instr('SparkSQL', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|int | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|isnan |SELECT isnan(cast('NaN' as double)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|isnotnull |SELECT isnotnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|isnull |SELECT isnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|java_method |SELECT java_method('java.util.UUID', 'randomUUID'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|json_array_length |SELECT json_array_length('[1,2,3,4]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|json_object_keys |SELECT json_object_keys('{}'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|json_tuple |SELECT json_tuple('{"a":1, "b":2}', 'a', 'b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|kurtosis |SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|lag |SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|last |SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|last_day |SELECT last_day('2009-01-12'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|last_value |SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|lcase |SELECT lcase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|lead |SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|least |SELECT least(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|left |SELECT left('Spark SQL', 3); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|len |SELECT len('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|length |SELECT length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|levenshtein |SELECT levenshtein('kitten', 'sitting'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|like |SELECT like('Spark', '_park'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|ln |SELECT ln(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|localtimestamp |SELECT localtimestamp(); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|locate |SELECT locate('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|log |SELECT log(10, 100); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|log10 |SELECT log10(10); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|log1p |SELECT log1p(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|log2 |SELECT log2(2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|lower |SELECT lower('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|lpad |SELECT lpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ltrim |SELECT ltrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|make_date |SELECT make_date(2013, 7, 15); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_dt_interval |SELECT make_dt_interval(1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_interval |SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_timestamp |SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_timestamp_ltz |SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_timestamp_ntz |SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_ym_interval |SELECT make_ym_interval(1, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map |SELECT map(1.0, '2', 3.0, '4'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_concat |SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_contains_key |SELECT map_contains_key(map(1, 'a', 2, 'b'), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_entries |SELECT map_entries(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_filter |SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_from_arrays |SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_from_entries |SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_keys |SELECT map_keys(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_values |SELECT map_values(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_zip_with |SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|mask |SELECT mask('abcd-EFGH-8765-4321'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|max |SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|max_by |SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|md5 |SELECT md5('Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|mean |SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|median |SELECT median(col) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|min |SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|min_by |SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|minute |SELECT minute('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|mod |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|mode |SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|monotonically_increasing_id|SELECT monotonically_increasing_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|month |SELECT month('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|months_between |SELECT months_between('1997-02-28 10:30:00', '1996-10-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|named_struct |SELECT named_struct("a", 1, "b", 2, "c", 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|nanvl |SELECT nanvl(cast('NaN' as double), 123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|negative |SELECT negative(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|next_day |SELECT next_day('2015-01-14', 'TU'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|not |SELECT not true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|now |SELECT now(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [now() AS now()#1254, x#1238]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#1238])\n +- Relation [x#1238] parquet\n\n== Analyzed Logical Plan ==\nnow(): timestamp, x: string\nProject [now() AS now()#1254, x#1238]\n+- SubqueryAlias tbl\n +- View (`tbl`, [x#1238])\n +- Relation [x#1238] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-20 18:24:37.462092 AS now()#1254, x#1238]\n+- Relation [x#1238] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [now()#1254, x#1238], [2024-05-20 18:24:37.462092 AS now()#1254, x#1238]\n +- CometScan parquet [x#1238] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-01..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-20 18:24:37.434216,dummy] [2024-05-20 18:24:37.462092,dummy]\n \n |DATAFUSIONCLI_PATH env variable not set| -|nth_value |SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ntile |SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|nullif |SELECT nullif(2, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|nvl |SELECT nvl(NULL, array('2')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|nvl2 |SELECT nvl2(NULL, 2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|octet_length |SELECT octet_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|or |SELECT true or false; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|overlay |SELECT overlay('Spark SQL' PLACING '_' FROM 6); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|parse_url |SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|percent_rank |SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|percentile |SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|percentile_approx |SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|pi |SELECT pi(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|pmod |SELECT pmod(10, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|posexplode |SELECT posexplode(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|posexplode_outer |SELECT posexplode_outer(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|position |SELECT position('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|positive |SELECT positive(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|pow |SELECT pow(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|power |SELECT power(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|printf |SELECT printf("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|quarter |SELECT quarter('2016-08-31'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|radians |SELECT radians(180); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|raise_error |SELECT raise_error('custom error message'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rand |SELECT rand(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|randn |SELECT randn(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|random |SELECT random(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rank |SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|reduce |SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|reflect |SELECT reflect('java.util.UUID', 'randomUUID'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp |SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_count |SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_extract |SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_extract_all |SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_instr |SELECT regexp_instr('user@spark.apache.org', '@[^.]*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_like |SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_replace |SELECT regexp_replace('100-200', '(\\d+)', 'num'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_substr |SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_avgx |SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_avgy |SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_count |SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_intercept |SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_r2 |SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_slope |SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_sxx |SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_sxy |SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_syy |SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|repeat |SELECT repeat('123', 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|replace |SELECT replace('ABCabc', 'abc', 'DEF'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|reverse |SELECT reverse('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|right |SELECT right('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rint |SELECT rint(12.3456); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rlike |SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|round |SELECT round(2.5, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|row_number |SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rpad |SELECT rpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rtrim |SELECT rtrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|schema_of_csv |SELECT schema_of_csv('1,abc'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|schema_of_json |SELECT schema_of_json('[{"col":0}]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sec |SELECT sec(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|second |SELECT second('2009-07-30 12:58:59'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|sentences |SELECT sentences('Hi there! Good morning.'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sequence |SELECT sequence(1, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sha |SELECT sha('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sha1 |SELECT sha1('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sha2 |SELECT sha2('Spark', 256); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|shiftleft |SELECT shiftleft(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|shiftright |SELECT shiftright(4, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|shiftrightunsigned |SELECT shiftrightunsigned(4, 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|shuffle |SELECT shuffle(array(1, 20, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sign |SELECT sign(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|signum |SELECT signum(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|sin |SELECT sin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|sinh |SELECT sinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|size |SELECT size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|skewness |SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|slice |SELECT slice(array(1, 2, 3, 4), 2, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|smallint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|some |SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sort_array |SELECT sort_array(array('b', 'd', null, 'c', 'a'), true); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|soundex |SELECT soundex('Miller'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|space |SELECT concat(space(2), '1'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|spark_partition_id |SELECT spark_partition_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|split |SELECT split('oneAtwoBthreeC', '[ABC]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|split_part |SELECT split_part('11.12.13', '.', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sqrt |SELECT sqrt(4); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|stack |SELECT stack(2, 1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|startswith |SELECT startswith('Spark SQL', 'Spark'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|std |SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|stddev |SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|stddev_pop |SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|stddev_samp |SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|str_to_map |SELECT str_to_map('a:1,b:2,c:3', ',', ':'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|string | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|struct |SELECT struct(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|substr |SELECT substr('Spark SQL', 5); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|substring |SELECT substring('Spark SQL', 5); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|substring_index |SELECT substring_index('www.apache.org', '.', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sum |SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|tan |SELECT tan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|tanh |SELECT tanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|timestamp | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|timestamp_micros |SELECT timestamp_micros(1230219000123123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|timestamp_millis |SELECT timestamp_millis(1230219000123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|timestamp_seconds |SELECT timestamp_seconds(1230219000); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|tinyint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|to_binary |SELECT to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_char |SELECT to_char(454, '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_csv |SELECT to_csv(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_date |SELECT to_date('2009-07-30 04:17:52'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|to_json |SELECT to_json(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_number |SELECT to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_timestamp |SELECT to_timestamp('2016-12-31 00:12:00'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|to_timestamp_ltz |SELECT to_timestamp_ltz('2016-12-31 00:12:00'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|to_timestamp_ntz |SELECT to_timestamp_ntz('2016-12-31 00:12:00'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|to_unix_timestamp |SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_utc_timestamp |SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|transform |SELECT transform(array(1, 2, 3), x -> x + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|transform_keys |SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|transform_values |SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|translate |SELECT translate('AaBbCc', 'abc', '123'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|trim |SELECT trim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|trunc |SELECT trunc('2019-08-04', 'week'); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|try_add |SELECT try_add(1, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_avg |SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_divide |SELECT try_divide(3, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_element_at |SELECT try_element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_multiply |SELECT try_multiply(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_subtract |SELECT try_subtract(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_sum |SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_to_binary |SELECT try_to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_to_number |SELECT try_to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_to_timestamp |SELECT try_to_timestamp('2016-12-31 00:12:00'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|typeof |SELECT typeof(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ucase |SELECT ucase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|unbase64 |SELECT unbase64('U3BhcmsgU1FM'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unhex |SELECT decode(unhex('537061726B2053514C'), 'UTF-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_date |SELECT unix_date(DATE("1970-01-02")); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_micros |SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_millis |SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_seconds |SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_timestamp |SELECT unix_timestamp(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|upper |SELECT upper('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|url_decode |SELECT url_decode('https%3A%2F%2Fspark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|url_encode |SELECT url_encode('https://spark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|user |SELECT user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|uuid |SELECT uuid(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|var_pop |SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|var_samp |SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|variance |SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|version |SELECT version(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|weekday |SELECT weekday('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|weekofyear |SELECT weekofyear('2008-02-20'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|when |SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|width_bucket |SELECT width_bucket(5.3, 0.2, 10.6, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath |SELECT xpath('b1b2b3c1c2','a/b/text()'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_boolean |SELECT xpath_boolean('1','a/b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_double |SELECT xpath_double('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_float |SELECT xpath_float('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_int |SELECT xpath_int('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_long |SELECT xpath_long('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_number |SELECT xpath_number('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_short |SELECT xpath_short('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_string |SELECT xpath_string('bcc','a/c'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xxhash64 |SELECT xxhash64('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|year |SELECT year('2016-07-30'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|zip_with |SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|| |SELECT 3 | 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|~ |SELECT ~ 0; |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ +|name |query |result |cometMessage |datafusionMessage | ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ +|! |SELECT ! true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|% |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|& |SELECT 3 & 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|* |SELECT 2 * 3; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|+ |SELECT 1 + 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|- |SELECT 2 - 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|/ |SELECT 3 / 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|< |SELECT 1 < 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|<= |SELECT 2 <= 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|<=> |SELECT 2 <=> 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|= |SELECT 2 = 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|== |SELECT 2 == 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|> |SELECT 2 > 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|>= |SELECT 2 >= 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|^ |SELECT 3 ^ 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|abs |SELECT abs(-1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|acos |SELECT acos(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|acosh |SELECT acosh(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|add_months |SELECT add_months('2016-08-31', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|aes_decrypt |SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|aes_encrypt |SELECT hex(aes_encrypt('Spark', '0000111122223333')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|aggregate |SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|and |SELECT true and true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|any |SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|any_value |SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|approx_count_distinct |SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|approx_percentile |SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array |SELECT array(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_agg |SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_append |SELECT array_append(array('b', 'd', 'c', 'a'), 'd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_compact |SELECT array_compact(array(1, 2, 3, null)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_contains |SELECT array_contains(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_distinct |SELECT array_distinct(array(1, 2, 3, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_except |SELECT array_except(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_insert |SELECT array_insert(array(1, 2, 3, 4), 5, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_intersect |SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_join |SELECT array_join(array('hello', 'world'), ' '); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_max |SELECT array_max(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_min |SELECT array_min(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_position |SELECT array_position(array(3, 2, 1), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_remove |SELECT array_remove(array(1, 2, 3, null, 3), 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_repeat |SELECT array_repeat('123', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_size |SELECT array_size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_sort |SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|array_union |SELECT array_union(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|arrays_overlap |SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|arrays_zip |SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ascii |SELECT ascii('222'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|asin |SELECT asin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|asinh |SELECT asinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|assert_true |SELECT assert_true(0 < 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|atan |SELECT atan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|atan2 |SELECT atan2(0, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|atanh |SELECT atanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|avg |SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|base64 |SELECT base64('Spark SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bigint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bin |SELECT bin(13); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|binary | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bit_and |SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|bit_count |SELECT bit_count(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bit_get |SELECT bit_get(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|bit_length |SELECT bit_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|bit_or |SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|bit_xor |SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|bool_and |SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|bool_or |SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|boolean | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bround |SELECT bround(2.5, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|btrim |SELECT btrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|cardinality |SELECT cardinality(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cast |SELECT cast('10' as int); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cbrt |SELECT cbrt(27.0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ceil |SELECT ceil(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|ceiling |SELECT ceiling(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|char |SELECT char(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|char_length |SELECT char_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|character_length |SELECT character_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|chr |SELECT chr(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|coalesce |SELECT coalesce(NULL, 1, NULL); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|collect_list |SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|collect_set |SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|concat |SELECT concat('Spark', 'SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|concat_ws |SELECT concat_ws(' ', 'Spark', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|contains |SELECT contains('Spark SQL', 'Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|conv |SELECT conv('100', 2, 10); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|convert_timezone |SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|corr |SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cos |SELECT cos(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|cosh |SELECT cosh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cot |SELECT cot(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|count |SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|count_if |SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|count_min_sketch |SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|covar_pop |SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|covar_samp |SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|crc32 |SELECT crc32('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|csc |SELECT csc(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|cume_dist |SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|curdate |SELECT curdate(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_catalog |SELECT current_catalog(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_database |SELECT current_database(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_date |SELECT current_date(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_schema |SELECT current_schema(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_timestamp |SELECT current_timestamp(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [current_timestamp() AS current_timestamp()#8386]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#8375])\n +- Relation [1#8375] parquet\n\n== Analyzed Logical Plan ==\ncurrent_timestamp(): timestamp\nProject [current_timestamp() AS current_timestamp()#8386]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#8375])\n +- Relation [1#8375] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-24 14:26:05.062063 AS current_timestamp()#8386]\n+- Relation [1#8375] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [current_timestamp()#8386], [2024-05-24 14:26:05.062063 AS current_timestamp()#8386]\n +- CometScan parquet [] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-e5..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-24 14:26:05.040693] [2024-05-24 14:26:05.062063]\n \n |DATAFUSIONCLI_PATH env variable not set| +|current_timezone |SELECT current_timezone(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|current_user |SELECT current_user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|date | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|date_add |SELECT date_add('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_diff |SELECT date_diff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_format |SELECT date_format('2016-04-08', 'y'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_from_unix_date |SELECT date_from_unix_date(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_part |SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|date_sub |SELECT date_sub('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|date_trunc |SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dateadd |SELECT dateadd('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|datediff |SELECT datediff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|datepart |SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|day |SELECT day('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dayofmonth |SELECT dayofmonth('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dayofweek |SELECT dayofweek('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dayofyear |SELECT dayofyear('2016-04-09'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|decimal | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|decode |SELECT decode(encode('abc', 'utf-8'), 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|degrees |SELECT degrees(3.141592653589793); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|dense_rank |SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|div |SELECT 3 div 2; |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|double | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|e |SELECT e(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|element_at |SELECT element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|elt |SELECT elt(1, 'scala', 'java'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|encode |SELECT encode('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|endswith |SELECT endswith('Spark SQL', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|equal_null |SELECT equal_null(3, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|every |SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|exists |SELECT exists(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|exp |SELECT exp(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|explode |SELECT explode(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|explode_outer |SELECT explode_outer(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|expm1 |SELECT expm1(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|extract |SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|factorial |SELECT factorial(5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|filter |SELECT filter(array(1, 2, 3), x -> x % 2 == 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|find_in_set |SELECT find_in_set('ab','abc,b,ab,c,def'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|first |SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|first_value |SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|flatten |SELECT flatten(array(array(1, 2), array(3, 4))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|float | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|floor |SELECT floor(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|forall |SELECT forall(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|format_number |SELECT format_number(12332.123456, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|format_string |SELECT format_string("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_csv |SELECT from_csv('1, 0.8', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_json |SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_unixtime |SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|from_utc_timestamp |SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|get |SELECT get(array(1, 2, 3), 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|get_json_object |SELECT get_json_object('{"a":"b"}', '$.a'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|getbit |SELECT getbit(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|greatest |SELECT greatest(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|grouping |SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name); |FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#7445, spark_grouping_id#7444L], [age#7441, name#7445]\n+- Expand [[age#7441, name#7442, name#7443, 0], [age#7441, name#7442, null, 1]], [age#7441, name#7442, name#7445, spark_grouping_id#7444L]\n +- Project [age#7441, name#7442, name#7442 AS name#7443]\n +- SubqueryAlias people\n +- LocalRelation [age#7441, name#7442]\n | | +|grouping_id |SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);|FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#8183, height#8184, spark_grouping_id#8182L], [age#8177, name#8183, height#8184]\n+- Expand [[age#8177, name#8178, height#8179, name#8180, height#8181, 0], [age#8177, name#8178, height#8179, name#8180, null, 1], [age#8177, name#8178, height#8179, null, height#8181, 2], [age#8177, name#8178, height#8179, null, null, 3]], [age#8177, name#8178, height#8179, name#8183, height#8184, spark_grouping_id#8182L]\n +- Project [age#8177, name#8178, height#8179, name#8178 AS name#8180, height#8179 AS height#8181]\n +- SubqueryAlias people\n +- LocalRelation [age#8177, name#8178, height#8179]\n | | +|hash |SELECT hash('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|hex |SELECT hex(17); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|histogram_numeric |SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|hour |SELECT hour('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|hypot |SELECT hypot(3, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|if |SELECT if(1 < 2, 'a', 'b'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|ifnull |SELECT ifnull(NULL, array('2')); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|ilike |SELECT ilike('Spark', '_Park'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|in |SELECT 1 in(1, 2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|initcap |SELECT initcap('sPark sql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|inline |SELECT inline(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|inline_outer |SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|input_file_block_length |SELECT input_file_block_length(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|input_file_block_start |SELECT input_file_block_start(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|input_file_name |SELECT input_file_name(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|instr |SELECT instr('SparkSQL', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|int | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|isnan |SELECT isnan(cast('NaN' as double)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|isnotnull |SELECT isnotnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|isnull |SELECT isnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|json_array_length |SELECT json_array_length('[1,2,3,4]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|json_object_keys |SELECT json_object_keys('{}'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|json_tuple |SELECT json_tuple('{"a":1, "b":2}', 'a', 'b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|kurtosis |SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|lag |SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|last |SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|last_day |SELECT last_day('2009-01-12'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|last_value |SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|lcase |SELECT lcase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|lead |SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|least |SELECT least(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|left |SELECT left('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|len |SELECT len('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|length |SELECT length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|levenshtein |SELECT levenshtein('kitten', 'sitting'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|like |SELECT like('Spark', '_park'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|ln |SELECT ln(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|localtimestamp |SELECT localtimestamp(); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| +|locate |SELECT locate('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|log |SELECT log(10, 100); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|log10 |SELECT log10(10); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|log1p |SELECT log1p(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|log2 |SELECT log2(2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|lower |SELECT lower('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|lpad |SELECT lpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ltrim |SELECT ltrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|make_date |SELECT make_date(2013, 7, 15); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_dt_interval |SELECT make_dt_interval(1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_interval |SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_timestamp |SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_timestamp_ltz |SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_timestamp_ntz |SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|make_ym_interval |SELECT make_ym_interval(1, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map |SELECT map(1.0, '2', 3.0, '4'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_concat |SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_contains_key |SELECT map_contains_key(map(1, 'a', 2, 'b'), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_entries |SELECT map_entries(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_filter |SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_from_arrays |SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_from_entries |SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_keys |SELECT map_keys(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_values |SELECT map_values(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|map_zip_with |SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|mask |SELECT mask('abcd-EFGH-8765-4321'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|max |SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|max_by |SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|md5 |SELECT md5('Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|mean |SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|median |SELECT median(col) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|min |SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|min_by |SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|minute |SELECT minute('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|mod |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|mode |SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|monotonically_increasing_id|SELECT monotonically_increasing_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|month |SELECT month('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|months_between |SELECT months_between('1997-02-28 10:30:00', '1996-10-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|named_struct |SELECT named_struct("a", 1, "b", 2, "c", 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|nanvl |SELECT nanvl(cast('NaN' as double), 123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|negative |SELECT negative(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|next_day |SELECT next_day('2015-01-14', 'TU'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|not |SELECT not true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|now |SELECT now(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [now() AS now()#1510]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#1499])\n +- Relation [1#1499] parquet\n\n== Analyzed Logical Plan ==\nnow(): timestamp\nProject [now() AS now()#1510]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#1499])\n +- Relation [1#1499] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-24 14:25:36.919361 AS now()#1510]\n+- Relation [1#1499] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [now()#1510], [2024-05-24 14:25:36.919361 AS now()#1510]\n +- CometScan parquet [] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-f0..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-24 14:25:36.887745] [2024-05-24 14:25:36.919361]\n \n |DATAFUSIONCLI_PATH env variable not set| +|nth_value |SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ntile |SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|nullif |SELECT nullif(2, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|nvl |SELECT nvl(NULL, array('2')); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|nvl2 |SELECT nvl2(NULL, 2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|octet_length |SELECT octet_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|or |SELECT true or false; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|overlay |SELECT overlay('Spark SQL' PLACING '_' FROM 6); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|parse_url |SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|percent_rank |SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|percentile |SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|percentile_approx |SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|pi |SELECT pi(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|pmod |SELECT pmod(10, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|posexplode |SELECT posexplode(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|posexplode_outer |SELECT posexplode_outer(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|position |SELECT position('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|positive |SELECT positive(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|pow |SELECT pow(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|power |SELECT power(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|printf |SELECT printf("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|quarter |SELECT quarter('2016-08-31'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|radians |SELECT radians(180); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|raise_error |SELECT raise_error('custom error message'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rand |SELECT rand(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|randn |SELECT randn(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|random |SELECT random(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rank |SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|reduce |SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp |SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_count |SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_extract |SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_extract_all |SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_instr |SELECT regexp_instr('user@spark.apache.org', '@[^.]*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_like |SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_replace |SELECT regexp_replace('100-200', '(\\d+)', 'num'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regexp_substr |SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_avgx |SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|regr_avgy |SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|regr_count |SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|regr_intercept |SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_r2 |SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_slope |SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_sxx |SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_sxy |SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|regr_syy |SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|repeat |SELECT repeat('123', 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|replace |SELECT replace('ABCabc', 'abc', 'DEF'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|reverse |SELECT reverse('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|right |SELECT right('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rint |SELECT rint(12.3456); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rlike |SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|round |SELECT round(2.5, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|row_number |SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rpad |SELECT rpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|rtrim |SELECT rtrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|schema_of_csv |SELECT schema_of_csv('1,abc'); |FAILED |[DATATYPE_MISMATCH.NON_FOLDABLE_INPUT] Cannot resolve "schema_of_csv(c0)" due to data type mismatch: the input csv should be a foldable "STRING" expression; however, got "c0".; line 1 pos 7;\n'Project [unresolvedalias(schema_of_csv(c0#6515), None)]\n+- SubqueryAlias tbl\n +- View (`tbl`, [c0#6515])\n +- Relation [c0#6515] parquet\n |DATAFUSIONCLI_PATH env variable not set| +|schema_of_json |SELECT schema_of_json('[{"col":0}]'); |FAILED |[DATATYPE_MISMATCH.NON_FOLDABLE_INPUT] Cannot resolve "schema_of_json(c0)" due to data type mismatch: the input json should be a foldable "STRING" expression; however, got "c0".; line 1 pos 7;\n'Project [unresolvedalias(schema_of_json(c0#7685), None)]\n+- SubqueryAlias tbl\n +- View (`tbl`, [c0#7685])\n +- Relation [c0#7685] parquet\n |DATAFUSIONCLI_PATH env variable not set| +|sec |SELECT sec(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|second |SELECT second('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sentences |SELECT sentences('Hi there! Good morning.'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sequence |SELECT sequence(1, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sha |SELECT sha('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sha1 |SELECT sha1('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sha2 |SELECT sha2('Spark', 256); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|shiftleft |SELECT shiftleft(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|shiftright |SELECT shiftright(4, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|shiftrightunsigned |SELECT shiftrightunsigned(4, 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|shuffle |SELECT shuffle(array(1, 20, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sign |SELECT sign(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|signum |SELECT signum(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|sin |SELECT sin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|sinh |SELECT sinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|size |SELECT size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|skewness |SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|slice |SELECT slice(array(1, 2, 3, 4), 2, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|smallint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|some |SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|sort_array |SELECT sort_array(array('b', 'd', null, 'c', 'a'), true); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|soundex |SELECT soundex('Miller'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|space |SELECT concat(space(2), '1'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|spark_partition_id |SELECT spark_partition_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|split |SELECT split('oneAtwoBthreeC', '[ABC]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|split_part |SELECT split_part('11.12.13', '.', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sqrt |SELECT sqrt(4); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|stack |SELECT stack(2, 1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|startswith |SELECT startswith('Spark SQL', 'Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|std |SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|stddev |SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|stddev_pop |SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|stddev_samp |SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|str_to_map |SELECT str_to_map('a:1,b:2,c:3', ',', ':'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|string | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|struct |SELECT struct(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|substr |SELECT substr('Spark SQL', 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|substring |SELECT substring('Spark SQL', 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|substring_index |SELECT substring_index('www.apache.org', '.', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|sum |SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|tan |SELECT tan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|tanh |SELECT tanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|timestamp | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|timestamp_micros |SELECT timestamp_micros(1230219000123123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|timestamp_millis |SELECT timestamp_millis(1230219000123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|timestamp_seconds |SELECT timestamp_seconds(1230219000); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|tinyint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|to_binary |SELECT to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_char |SELECT to_char(454, '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_csv |SELECT to_csv(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_date |SELECT to_date('2009-07-30 04:17:52'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_json |SELECT to_json(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_number |SELECT to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_timestamp |SELECT to_timestamp('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_timestamp_ltz |SELECT to_timestamp_ltz('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_timestamp_ntz |SELECT to_timestamp_ntz('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_unix_timestamp |SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|to_utc_timestamp |SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|transform |SELECT transform(array(1, 2, 3), x -> x + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|transform_keys |SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|transform_values |SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|translate |SELECT translate('AaBbCc', 'abc', '123'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|trim |SELECT trim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|trunc |SELECT trunc('2019-08-04', 'week'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_add |SELECT try_add(1, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_avg |SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_divide |SELECT try_divide(3, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_element_at |SELECT try_element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_multiply |SELECT try_multiply(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_subtract |SELECT try_subtract(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|try_sum |SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_to_binary |SELECT try_to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_to_number |SELECT try_to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|try_to_timestamp |SELECT try_to_timestamp('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|typeof |SELECT typeof(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|ucase |SELECT ucase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|unbase64 |SELECT unbase64('U3BhcmsgU1FM'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unhex |SELECT decode(unhex('537061726B2053514C'), 'UTF-8'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|unix_date |SELECT unix_date(DATE("1970-01-02")); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_micros |SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_millis |SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_seconds |SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|unix_timestamp |SELECT unix_timestamp(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|upper |SELECT upper('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|url_decode |SELECT url_decode('https%3A%2F%2Fspark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|url_encode |SELECT url_encode('https://spark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|user |SELECT user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|uuid |SELECT uuid(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|var_pop |SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|var_samp |SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|variance |SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|version |SELECT version(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|weekday |SELECT weekday('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|weekofyear |SELECT weekofyear('2008-02-20'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|when |SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END; |FAILED |\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'then'.(line 1, pos 18)\n\n== SQL ==\nselect case a > b then c else d end from tbl\n------------------^^^\n |DATAFUSIONCLI_PATH env variable not set| +|width_bucket |SELECT width_bucket(5.3, 0.2, 10.6, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath |SELECT xpath('b1b2b3c1c2','a/b/text()'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_boolean |SELECT xpath_boolean('1','a/b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_double |SELECT xpath_double('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_float |SELECT xpath_float('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_int |SELECT xpath_int('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_long |SELECT xpath_long('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_number |SELECT xpath_number('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_short |SELECT xpath_short('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xpath_string |SELECT xpath_string('bcc','a/c'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|xxhash64 |SELECT xxhash64('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|year |SELECT year('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| +|zip_with |SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); |FAILED |[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `zip_with` requires 3 parameters but the actual number is 2. Please, refer to 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix.; line 1 pos 7 |DATAFUSIONCLI_PATH env variable not set| +|| |SELECT 3 | 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| +|~ |SELECT ~ 0; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ diff --git a/docs/spark_expressions_support.md b/docs/spark_expressions_support.md index c48426669..ecfe9b8a0 100644 --- a/docs/spark_expressions_support.md +++ b/docs/spark_expressions_support.md @@ -20,46 +20,46 @@ # Supported Spark Expressions ### agg_funcs - - [ ] any - - [ ] any_value + - [x] any + - [x] any_value - [ ] approx_count_distinct - [ ] approx_percentile - [ ] array_agg - - [ ] avg - - [ ] bit_and - - [ ] bit_or - - [ ] bit_xor - - [ ] bool_and - - [ ] bool_or + - [x] avg + - [x] bit_and + - [x] bit_or + - [x] bit_xor + - [x] bool_and + - [x] bool_or - [ ] collect_list - [ ] collect_set - [ ] corr - - [ ] count - - [ ] count_if + - [x] count + - [x] count_if - [ ] count_min_sketch - - [ ] covar_pop - - [ ] covar_samp - - [ ] every - - [ ] first - - [ ] first_value + - [x] covar_pop + - [x] covar_samp + - [x] every + - [x] first + - [x] first_value - [ ] grouping - [ ] grouping_id - [ ] histogram_numeric - [ ] kurtosis - - [ ] last - - [ ] last_value - - [ ] max + - [x] last + - [x] last_value + - [x] max - [ ] max_by - - [ ] mean + - [x] mean - [ ] median - - [ ] min + - [x] min - [ ] min_by - [ ] mode - [ ] percentile - [ ] percentile_approx - - [ ] regr_avgx - - [ ] regr_avgy - - [ ] regr_count + - [x] regr_avgx + - [x] regr_avgy + - [x] regr_count - [ ] regr_intercept - [ ] regr_r2 - [ ] regr_slope @@ -67,17 +67,17 @@ - [ ] regr_sxy - [ ] regr_syy - [ ] skewness - - [ ] some - - [ ] std - - [ ] stddev - - [ ] stddev_pop - - [ ] stddev_samp - - [ ] sum + - [x] some + - [x] std + - [x] stddev + - [x] stddev_pop + - [x] stddev_samp + - [x] sum - [ ] try_avg - [ ] try_sum - - [ ] var_pop - - [ ] var_samp - - [ ] variance + - [x] var_pop + - [x] var_samp + - [x] variance ### array_funcs - [ ] array @@ -98,7 +98,7 @@ - [ ] arrays_overlap - [ ] arrays_zip - [ ] flatten - - [x] get + - [ ] get - [ ] sequence - [ ] shuffle - [ ] slice @@ -113,7 +113,7 @@ - [x] shiftright - [ ] shiftrightunsigned - [x] | - - [ ] ~ + - [x] ~ ### collection_funcs - [ ] array_size @@ -125,18 +125,18 @@ ### conditional_funcs - [x] coalesce - [x] if - - [ ] ifnull + - [x] ifnull - [ ] nanvl - [x] nullif - - [ ] nvl + - [x] nvl - [x] nvl2 - - [x] when + - [ ] when ### conversion_funcs - [ ] bigint - [ ] binary - [ ] boolean - - [x] cast + - [ ] cast - [ ] date - [ ] decimal - [ ] double @@ -196,14 +196,14 @@ - [ ] timestamp_micros - [ ] timestamp_millis - [ ] timestamp_seconds - - [x] to_date - - [x] to_timestamp - - [x] to_timestamp_ltz + - [ ] to_date + - [ ] to_timestamp + - [ ] to_timestamp_ltz - [ ] to_timestamp_ntz - [ ] to_unix_timestamp - [ ] to_utc_timestamp - [ ] trunc - - [x] try_to_timestamp + - [ ] try_to_timestamp - [ ] unix_date - [ ] unix_micros - [ ] unix_millis @@ -211,7 +211,7 @@ - [ ] unix_timestamp - [ ] weekday - [ ] weekofyear - - [x] year + - [ ] year ### generator_funcs - [ ] explode @@ -228,7 +228,7 @@ - [x] md5 - [ ] sha - [ ] sha1 - - [x] sha2 + - [ ] sha2 - [ ] xxhash64 ### json_funcs @@ -333,7 +333,7 @@ - [x] try_divide - [x] try_multiply - [x] try_subtract - - [ ] unhex + - [x] unhex - [ ] width_bucket ### misc_funcs @@ -348,10 +348,8 @@ - [ ] input_file_block_length - [ ] input_file_block_start - [ ] input_file_name - - [ ] java_method - [ ] monotonically_increasing_id - [ ] raise_error - - [ ] reflect - [ ] spark_partition_id - [ ] typeof - [x] user @@ -368,12 +366,12 @@ - [x] > - [x] >= - [x] and - - [ ] ilike + - [x] ilike - [x] in - [ ] isnan - [x] isnotnull - [x] isnull - - [ ] like + - [x] like - [x] not - [x] or - [ ] regexp @@ -390,11 +388,11 @@ - [x] character_length - [x] chr - [x] concat_ws - - [ ] contains + - [x] contains - [ ] decode - [ ] elt - [ ] encode - - [ ] endswith + - [x] endswith - [ ] find_in_set - [ ] format_number - [ ] format_string @@ -427,10 +425,10 @@ - [x] rtrim - [ ] sentences - [ ] soundex - - [ ] space + - [x] space - [ ] split - [ ] split_part - - [ ] startswith + - [x] startswith - [ ] substr - [ ] substring - [ ] substring_index diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index ca9508cb5..e38a49a09 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -31,6 +31,9 @@ import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import org.apache.hadoop.fs.Path import org.apache.spark.sql.{CometTestBase, DataFrame} +import org.apache.spark.sql.catalyst.analysis.UnresolvedFunction +import org.apache.spark.sql.catalyst.expressions.LambdaFunction +import org.apache.spark.sql.catalyst.plans.logical.Project import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.collect_list @@ -54,6 +57,70 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH private val valuesPattern = """(?i)FROM VALUES(.+?);""".r private val selectPattern = """(i?)SELECT(.+?)FROM""".r + // exclude funcs Comet has no plans to support streaming in near future + // like spark streaming functions, java calls + private val outofRoadmapFuncs = + List("window", "session_window", "window_time", "java_method", "reflect") + private val sqlConf = Seq( + "spark.comet.exec.shuffle.enabled" -> "true", + "spark.sql.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding", + "spark.sql.adaptive.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") + + // Tests to run manually as its syntax is different from usual or nested + val manualTests: Map[String, (String, String)] = Map( + "!" -> ("select true a", "select ! true from tbl"), + "%" -> ("select 1 a, 2 b", "select a + b from tbl"), + "&" -> ("select 1 a, 2 b", "select a & b from tbl"), + "*" -> ("select 1 a, 2 b", "select a * b from tbl"), + "+" -> ("select 1 a, 2 b", "select a + b from tbl"), + "-" -> ("select 1 a, 2 b", "select a - b from tbl"), + "/" -> ("select 1 a, 2 b", "select a / b from tbl"), + "<" -> ("select 1 a, 2 b", "select a < b from tbl"), + "<=" -> ("select 1 a, 2 b", "select a <= b from tbl"), + "<=>" -> ("select 1 a, 2 b", "select a <=> b from tbl"), + "=" -> ("select 1 a, 2 b", "select a = b from tbl"), + "==" -> ("select 1 a, 2 b", "select a == b from tbl"), + ">" -> ("select 1 a, 2 b", "select a > b from tbl"), + ">=" -> ("select 1 a, 2 b", "select a >= b from tbl"), + "^" -> ("select 1 a, 2 b", "select a ^ b from tbl"), + "|" -> ("select 1 a, 2 b", "select a | b from tbl"), + "stack" -> ("select 1 a, 2 b", "select stack(1, a, b) from tbl"), + "~" -> ("select 1 a", "select ~ a from tbl"), + "unhex" -> ("select '537061726B2053514C' a", "select unhex(a) from tbl"), + "when" -> ("select 1 a, 2 b, 3 c, 4 d", "select case a > b then c else d end from tbl"), + "transform_values" -> ("select array(1, 2, 3) a", "select transform_values(map_from_arrays(a, a), (k, v) -> v + 1) from tbl"), + "transform_keys" -> ("select array(1, 2, 3) a", "select transform_keys(map_from_arrays(a, a), (k, v) -> v + 1) from tbl"), + "transform" -> ("select array(1, 2, 3) a", "select transform(a, (k, v) -> v + 1) from tbl"), + "reduce" -> ("select array(1, 2, 3) a", "select reduce(a, 0, (acc, x) -> acc + x) from tbl"), + "struct" -> ("select 1 a, 2 b", "select struct(a, b) from tbl"), + "space" -> ("select 1 a", "select space(a) from tbl"), + "sort_array" -> ("select array('b', 'd', null, 'c', 'a') a", "select sort_array(a) from tbl"), + "or" -> ("select true a, false b", "select a or b from tbl"), + "overlay" -> ("select 'Spark SQL' a", "select overlay(a PLACING '_' FROM 6) from tbl"), + "nvl" -> ("select 1 a, cast(null as int) b", "select nvl(b, a) from tbl"), + "nvl2" -> ("select 1 a, cast(null as int) b, cast(null as int) c", "select nvl2(c, b, a) from tbl"), + "coalesce" -> ("select 1 a, cast(null as int) b, cast(null as int) c", "select coalesce(c, b, a) from tbl"), + "and" -> ("select true a, false b", "select a and b from tbl"), + "not" -> ("select true a", "select not a from tbl"), + "named_struct" -> ("select 1 a", "select named_struct('a', a) from tbl"), + "mod" -> ("select 1 a, 1 b", "select mod(b, a) from tbl"), + "div" -> ("select 1 a, 1 b", "select div(b, a) from tbl"), + "map_zip_with" -> ("select map(1, 'a', 2, 'b') a, map(1, 'x', 2, 'y') b", "SELECT map_zip_with(a, b, (k, v1, v2) -> concat(v1, v2)) from tbl"), + "map_filter" -> ("select map(1, 0, 2, 2, 3, -1) a", "SELECT map_filter(a, (k, v) -> k > v) from tbl"), + "in" -> ("select 1 a", "SELECT a in ('1', '2', '3') from tbl"), + "ifnull" -> ("select 1 a, cast(null as int) b", "SELECT ifnull(b, a) from tbl"), + "from_json" -> ("select '{\"a\":1, \"b\":0.8}' a", "SELECT from_json(a, 'a INT, b DOUBLE') from tbl"), + "from_csv" -> ("select '1, 0.8' a", "SELECT from_csv(a, 'a INT, b DOUBLE') from tbl"), + "forall" -> ("select array(1, 2, 3) a", "SELECT forall(a, x -> x % 2 == 0) from tbl"), + "filter" -> ("select array(1, 2, 3) a", "SELECT filter(a, x -> x % 2 == 1) from tbl"), + "exists" -> ("select array(1, 2, 3) a", "SELECT filter(a, x -> x % 2 == 0) from tbl"), + "aggregate" -> ("select array(1, 2, 3) a", "SELECT aggregate(a, 0, (acc, x) -> acc + x) from tbl"), + "extract" -> ("select TIMESTAMP '2019-08-12 01:00:00.123456' a", "SELECT extract(YEAR FROM a) from tbl"), + "datepart" -> ("select TIMESTAMP '2019-08-12 01:00:00.123456' a", "SELECT datepart('YEAR', a) from tbl"), + "date_part" -> ("select TIMESTAMP '2019-08-12 01:00:00.123456' a", "SELECT date_part('YEAR', a) from tbl"), + "cast" -> ("select '10' a", "SELECT cast(a as int) from tbl"), + "aes_encrypt" -> ("select 'Spark' a, '0000111122223333' b", "SELECT aes_encrypt(a, b) from tbl")) + // key - function name // value - examples def getExamples(): Map[FunctionInfo, List[String]] = @@ -61,9 +128,8 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH .listFunction() .map(spark.sessionState.catalog.lookupFunctionInfo(_)) .filter(_.getSource.toLowerCase == "built-in") - // exclude spark streaming functions, Comet has no plans to support streaming in near future - .filter(f => - !List("window", "session_window", "window_time").contains(f.getName.toLowerCase)) + // .filter(_.getName == "curdate") + .filter(f => !outofRoadmapFuncs.contains(f.getName.toLowerCase)) .map(f => { val selectRows = queryPattern.findAllMatchIn(f.getExamples).map(_.group(0)).toList (FunctionInfo(f.getName, f.getGroup), selectRows.filter(_.nonEmpty)) @@ -103,7 +169,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH dfMessage = runDatafusionCli(s"""$s '$path/*.parquet'""") } - testSingleLineQuery(s"select * $v", s"$s tbl") + testSingleLineQuery(s"select * $v", s"$s tbl", sqlConf = sqlConf) case _ => sys.error("Cannot parse properly") } @@ -116,12 +182,55 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH // ConstantFolding is a operator optimization rule in Catalyst that replaces expressions // that can be statically evaluated with their equivalent literal values. dfMessage = runDatafusionCli(q) - testSingleLineQuery( - "select 'dummy' x", - s"${q.dropRight(1)}, x from tbl", - excludedOptimizerRules = - Some("org.apache.spark.sql.catalyst.optimizer.ConstantFolding")) + + manualTests.get(func.name) match { + // the test is manual query + case Some(test) => testSingleLineQuery(test._1, test._2, sqlConf = sqlConf) + case None => + // extract function arguments as a sql text + // example: + // cos(0) -> 0 + // explode_outer(array(10, 20)) -> array(10, 20) + val args = getSqlFunctionArguments(q.dropRight(1)) + val (aliased, aliases) = + if (Seq( + "bround", + "rlike", + "round", + "to_binary", + "to_char", + "to_number", + "try_to_binary", + "try_to_number", + "xpath", + "xpath_boolean", + "xpath_double", + "xpath_double", + "xpath_float", + "xpath_int", + "xpath_long", + "xpath_number", + "xpath_short", + "xpath_string").contains(func.name.toLowerCase)) { + // c0 column, c1 foldable literal(cannot be from column) + ( + Seq(s"${args.head} as c0").mkString(","), + Seq(s"c0, ${args(1)}").mkString(",")) + } else { + ( + args.zipWithIndex.map(x => s"${x._1} as c${x._2}").mkString(","), + args.zipWithIndex.map(x => s"c${x._2}").mkString(",")) + } + + val select = s"select ${func.name}($aliases)" + + testSingleLineQuery( + s"select ${if (aliased.nonEmpty) aliased else 1}", + s"$select from tbl", + sqlConf = sqlConf) + } } + CoverageResult( q, CoverageResultStatus.Passed, @@ -129,7 +238,6 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH cometMessage = "OK", datafusionMessage = dfMessage.getOrElse("OK")), group = func.group) - } catch { case e: TestFailedException if e.getMessage.contains("Expected only Comet native operators") => @@ -217,6 +325,25 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH str shouldBe s"${getLicenseHeader()}\n# Supported Spark Expressions\n\n### group1\n - [x] f1\n - [ ] f2\n\n### group2\n - [x] f3\n - [ ] f4\n\n### group3\n - [x] f5" } + test("get sql function arguments") { + // getSqlFunctionArguments("SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'))") shouldBe Seq("TIMESTAMP('1970-01-01 00:00:01Z')") + // getSqlFunctionArguments("SELECT decode(unhex('537061726B2053514C'), 'UTF-8')") shouldBe Seq("unhex('537061726B2053514C')", "'UTF-8'") + // getSqlFunctionArguments("SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456')") shouldBe Seq("'YEAR'", "TIMESTAMP '2019-08-12 01:00:00.123456'") + // getSqlFunctionArguments("SELECT exists(array(1, 2, 3), x -> x % 2 == 0)") shouldBe Seq("array(1, 2, 3)") + getSqlFunctionArguments("select to_char(454, '999')") shouldBe Seq("array(1, 2, 3)") + } + + def getSqlFunctionArguments(sql: String): Seq[String] = { + val plan = spark.sessionState.sqlParser.parsePlan(sql) + plan match { + case Project(projectList, _) => + // unwrap projection to get first expression arguments + // assuming first expression is Unresolved function + val projection = projectList.head.children.head.asInstanceOf[UnresolvedFunction].arguments + projection.filter(!_.isInstanceOf[LambdaFunction]).map(_.sql) + } + } + def generateMarkdown(df: DataFrame): String = { val groupedDF = df .orderBy("name") diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 98a2bad02..8a341b061 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1445,6 +1445,36 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("sttdev") { + Seq(false, true).foreach { dictionary => + withSQLConf( + "parquet.enable.dictionary" -> dictionary.toString, + "spark.comet.exec.shuffle.enabled" -> "true") { + val table = "test" + withTable(table) { + sql(s"create table $table(col int) using parquet") + sql(s"insert into $table VALUES (1), (2), (3)") + checkSparkAnswerAndOperator(s"SELECT stddev_pop(col) FROM $table") + } + } + } + } + + test("to_date") { + Seq(false, true).foreach { dictionary => + withSQLConf( + "parquet.enable.dictionary" -> dictionary.toString, + "spark.comet.exec.shuffle.enabled" -> "true") { + val table = "test" + withTable(table) { + sql(s"create table $table(col string) using parquet") + sql(s"insert into $table VALUES ('2009-07-30 04:17:52')") + checkSparkAnswerAndOperator(s"SELECT to_date(col) FROM $table") + } + } + } + } + test("hash functions") { Seq(true, false).foreach { dictionary => withSQLConf( diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index 0530d764c..a632d51ef 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -788,7 +788,7 @@ abstract class CometTestBase testQuery: String, testName: String = "test", tableName: String = "tbl", - excludedOptimizerRules: Option[String] = None): Unit = { + sqlConf: Seq[(String, String)] = Seq.empty): Unit = { withTempDir { dir => val path = new Path(dir.toURI.toString, testName).toUri.toString @@ -804,9 +804,7 @@ abstract class CometTestBase spark.createDataFrame(data, schema).repartition(1).write.parquet(path) readParquetFile(path, Some(schema)) { df => df.createOrReplaceTempView(tableName) } - withSQLConf( - "spark.sql.optimizer.excludedRules" -> excludedOptimizerRules.getOrElse(""), - "spark.sql.adaptive.optimizer.excludedRules" -> excludedOptimizerRules.getOrElse("")) { + withSQLConf(sqlConf: _*) { checkSparkAnswerAndOperator(sql(testQuery)) } } From 59e8e78c486db93dc9e3172d81ab58ef3a5d18fe Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 24 May 2024 14:31:10 -0700 Subject: [PATCH 12/20] rm not related change --- .../apache/comet/CometExpressionSuite.scala | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 8a341b061..98a2bad02 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1445,36 +1445,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("sttdev") { - Seq(false, true).foreach { dictionary => - withSQLConf( - "parquet.enable.dictionary" -> dictionary.toString, - "spark.comet.exec.shuffle.enabled" -> "true") { - val table = "test" - withTable(table) { - sql(s"create table $table(col int) using parquet") - sql(s"insert into $table VALUES (1), (2), (3)") - checkSparkAnswerAndOperator(s"SELECT stddev_pop(col) FROM $table") - } - } - } - } - - test("to_date") { - Seq(false, true).foreach { dictionary => - withSQLConf( - "parquet.enable.dictionary" -> dictionary.toString, - "spark.comet.exec.shuffle.enabled" -> "true") { - val table = "test" - withTable(table) { - sql(s"create table $table(col string) using parquet") - sql(s"insert into $table VALUES ('2009-07-30 04:17:52')") - checkSparkAnswerAndOperator(s"SELECT to_date(col) FROM $table") - } - } - } - } - test("hash functions") { Seq(true, false).foreach { dictionary => withSQLConf( From ce0d82e6651ab0c98052a17f3f4494a02b006c6c Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 27 May 2024 19:08:24 -0700 Subject: [PATCH 13/20] uncomment tests --- .../org/apache/comet/CometExpressionCoverageSuite.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index e38a49a09..2352b2c96 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -326,10 +326,10 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH } test("get sql function arguments") { - // getSqlFunctionArguments("SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'))") shouldBe Seq("TIMESTAMP('1970-01-01 00:00:01Z')") - // getSqlFunctionArguments("SELECT decode(unhex('537061726B2053514C'), 'UTF-8')") shouldBe Seq("unhex('537061726B2053514C')", "'UTF-8'") - // getSqlFunctionArguments("SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456')") shouldBe Seq("'YEAR'", "TIMESTAMP '2019-08-12 01:00:00.123456'") - // getSqlFunctionArguments("SELECT exists(array(1, 2, 3), x -> x % 2 == 0)") shouldBe Seq("array(1, 2, 3)") + getSqlFunctionArguments("SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'))") shouldBe Seq("TIMESTAMP('1970-01-01 00:00:01Z')") + getSqlFunctionArguments("SELECT decode(unhex('537061726B2053514C'), 'UTF-8')") shouldBe Seq("unhex('537061726B2053514C')", "'UTF-8'") + getSqlFunctionArguments("SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456')") shouldBe Seq("'YEAR'", "TIMESTAMP '2019-08-12 01:00:00.123456'") + getSqlFunctionArguments("SELECT exists(array(1, 2, 3), x -> x % 2 == 0)") shouldBe Seq("array(1, 2, 3)") getSqlFunctionArguments("select to_char(454, '999')") shouldBe Seq("array(1, 2, 3)") } From a014192efcdf53ffc90114aaccc28c4ca6e0ee4d Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 27 May 2024 19:08:55 -0700 Subject: [PATCH 14/20] uncomment tests --- .../scala/org/apache/comet/CometExpressionCoverageSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 2352b2c96..cc3ad47a6 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -69,7 +69,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH // Tests to run manually as its syntax is different from usual or nested val manualTests: Map[String, (String, String)] = Map( "!" -> ("select true a", "select ! true from tbl"), - "%" -> ("select 1 a, 2 b", "select a + b from tbl"), + "%" -> ("select 1 a, 2 b", "select a % b from tbl"), "&" -> ("select 1 a, 2 b", "select a & b from tbl"), "*" -> ("select 1 a, 2 b", "select a * b from tbl"), "+" -> ("select 1 a, 2 b", "select a + b from tbl"), From aa051a62e38a76422929fac78a79b4c82273efdc Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 27 May 2024 19:30:32 -0700 Subject: [PATCH 15/20] fmt --- .../comet/CometExpressionCoverageSuite.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index cc3ad47a6..83016d9a0 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -326,10 +326,18 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH } test("get sql function arguments") { - getSqlFunctionArguments("SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'))") shouldBe Seq("TIMESTAMP('1970-01-01 00:00:01Z')") - getSqlFunctionArguments("SELECT decode(unhex('537061726B2053514C'), 'UTF-8')") shouldBe Seq("unhex('537061726B2053514C')", "'UTF-8'") - getSqlFunctionArguments("SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456')") shouldBe Seq("'YEAR'", "TIMESTAMP '2019-08-12 01:00:00.123456'") - getSqlFunctionArguments("SELECT exists(array(1, 2, 3), x -> x % 2 == 0)") shouldBe Seq("array(1, 2, 3)") + getSqlFunctionArguments( + "SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z'))") shouldBe Seq( + "TIMESTAMP('1970-01-01 00:00:01Z')") + getSqlFunctionArguments("SELECT decode(unhex('537061726B2053514C'), 'UTF-8')") shouldBe Seq( + "unhex('537061726B2053514C')", + "'UTF-8'") + getSqlFunctionArguments( + "SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456')") shouldBe Seq( + "'YEAR'", + "TIMESTAMP '2019-08-12 01:00:00.123456'") + getSqlFunctionArguments("SELECT exists(array(1, 2, 3), x -> x % 2 == 0)") shouldBe Seq( + "array(1, 2, 3)") getSqlFunctionArguments("select to_char(454, '999')") shouldBe Seq("array(1, 2, 3)") } From c82f2712358230384b8cdacf22694038f90c89d7 Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 28 May 2024 08:06:56 -0700 Subject: [PATCH 16/20] fmt --- .../scala/org/apache/comet/CometExpressionCoverageSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 83016d9a0..3b8e3d82e 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -338,7 +338,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH "TIMESTAMP '2019-08-12 01:00:00.123456'") getSqlFunctionArguments("SELECT exists(array(1, 2, 3), x -> x % 2 == 0)") shouldBe Seq( "array(1, 2, 3)") - getSqlFunctionArguments("select to_char(454, '999')") shouldBe Seq("array(1, 2, 3)") + getSqlFunctionArguments("select to_char(454, '999')") shouldBe Seq("454", "'999'") } def getSqlFunctionArguments(sql: String): Seq[String] = { From 42ba0786a367c7493fa6c1d6a79826255651ed13 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 30 May 2024 16:57:05 -0700 Subject: [PATCH 17/20] fix some tests --- docs/spark_builtin_expr_coverage.txt | 834 +++++++++--------- docs/spark_expressions_support.md | 4 +- .../apache/comet/CometExpressionSuite.scala | 40 + 3 files changed, 459 insertions(+), 419 deletions(-) diff --git a/docs/spark_builtin_expr_coverage.txt b/docs/spark_builtin_expr_coverage.txt index ceb2b0814..8e71cb8d2 100644 --- a/docs/spark_builtin_expr_coverage.txt +++ b/docs/spark_builtin_expr_coverage.txt @@ -1,417 +1,417 @@ -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ -|name |query |result |cometMessage |datafusionMessage | -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ -|! |SELECT ! true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|% |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|& |SELECT 3 & 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|* |SELECT 2 * 3; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|+ |SELECT 1 + 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|- |SELECT 2 - 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|/ |SELECT 3 / 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|< |SELECT 1 < 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|<= |SELECT 2 <= 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|<=> |SELECT 2 <=> 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|= |SELECT 2 = 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|== |SELECT 2 == 2; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|> |SELECT 2 > 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|>= |SELECT 2 >= 1; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|^ |SELECT 3 ^ 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|abs |SELECT abs(-1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|acos |SELECT acos(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|acosh |SELECT acosh(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|add_months |SELECT add_months('2016-08-31', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|aes_decrypt |SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|aes_encrypt |SELECT hex(aes_encrypt('Spark', '0000111122223333')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|aggregate |SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|and |SELECT true and true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|any |SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|any_value |SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|approx_count_distinct |SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|approx_percentile |SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array |SELECT array(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_agg |SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_append |SELECT array_append(array('b', 'd', 'c', 'a'), 'd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_compact |SELECT array_compact(array(1, 2, 3, null)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_contains |SELECT array_contains(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_distinct |SELECT array_distinct(array(1, 2, 3, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_except |SELECT array_except(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_insert |SELECT array_insert(array(1, 2, 3, 4), 5, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_intersect |SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_join |SELECT array_join(array('hello', 'world'), ' '); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_max |SELECT array_max(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_min |SELECT array_min(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_position |SELECT array_position(array(3, 2, 1), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_remove |SELECT array_remove(array(1, 2, 3, null, 3), 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_repeat |SELECT array_repeat('123', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_size |SELECT array_size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_sort |SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|array_union |SELECT array_union(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|arrays_overlap |SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|arrays_zip |SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ascii |SELECT ascii('222'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|asin |SELECT asin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|asinh |SELECT asinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|assert_true |SELECT assert_true(0 < 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|atan |SELECT atan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|atan2 |SELECT atan2(0, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|atanh |SELECT atanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|avg |SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|base64 |SELECT base64('Spark SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bigint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bin |SELECT bin(13); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|binary | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bit_and |SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|bit_count |SELECT bit_count(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bit_get |SELECT bit_get(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|bit_length |SELECT bit_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|bit_or |SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|bit_xor |SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|bool_and |SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|bool_or |SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|boolean | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|bround |SELECT bround(2.5, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|btrim |SELECT btrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|cardinality |SELECT cardinality(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cast |SELECT cast('10' as int); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cbrt |SELECT cbrt(27.0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ceil |SELECT ceil(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|ceiling |SELECT ceiling(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|char |SELECT char(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|char_length |SELECT char_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|character_length |SELECT character_length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|chr |SELECT chr(65); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|coalesce |SELECT coalesce(NULL, 1, NULL); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|collect_list |SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|collect_set |SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|concat |SELECT concat('Spark', 'SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|concat_ws |SELECT concat_ws(' ', 'Spark', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|contains |SELECT contains('Spark SQL', 'Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|conv |SELECT conv('100', 2, 10); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|convert_timezone |SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|corr |SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cos |SELECT cos(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|cosh |SELECT cosh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cot |SELECT cot(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|count |SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|count_if |SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|count_min_sketch |SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|covar_pop |SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|covar_samp |SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|crc32 |SELECT crc32('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|csc |SELECT csc(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|cume_dist |SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|curdate |SELECT curdate(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_catalog |SELECT current_catalog(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_database |SELECT current_database(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_date |SELECT current_date(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_schema |SELECT current_schema(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_timestamp |SELECT current_timestamp(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [current_timestamp() AS current_timestamp()#8386]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#8375])\n +- Relation [1#8375] parquet\n\n== Analyzed Logical Plan ==\ncurrent_timestamp(): timestamp\nProject [current_timestamp() AS current_timestamp()#8386]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#8375])\n +- Relation [1#8375] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-24 14:26:05.062063 AS current_timestamp()#8386]\n+- Relation [1#8375] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [current_timestamp()#8386], [2024-05-24 14:26:05.062063 AS current_timestamp()#8386]\n +- CometScan parquet [] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-e5..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-24 14:26:05.040693] [2024-05-24 14:26:05.062063]\n \n |DATAFUSIONCLI_PATH env variable not set| -|current_timezone |SELECT current_timezone(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|current_user |SELECT current_user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|date | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|date_add |SELECT date_add('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_diff |SELECT date_diff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_format |SELECT date_format('2016-04-08', 'y'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_from_unix_date |SELECT date_from_unix_date(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_part |SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|date_sub |SELECT date_sub('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|date_trunc |SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dateadd |SELECT dateadd('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|datediff |SELECT datediff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|datepart |SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|day |SELECT day('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dayofmonth |SELECT dayofmonth('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dayofweek |SELECT dayofweek('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dayofyear |SELECT dayofyear('2016-04-09'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|decimal | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|decode |SELECT decode(encode('abc', 'utf-8'), 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|degrees |SELECT degrees(3.141592653589793); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|dense_rank |SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|div |SELECT 3 div 2; |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|double | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|e |SELECT e(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|element_at |SELECT element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|elt |SELECT elt(1, 'scala', 'java'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|encode |SELECT encode('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|endswith |SELECT endswith('Spark SQL', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|equal_null |SELECT equal_null(3, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|every |SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|exists |SELECT exists(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|exp |SELECT exp(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|explode |SELECT explode(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|explode_outer |SELECT explode_outer(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|expm1 |SELECT expm1(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|extract |SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|factorial |SELECT factorial(5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|filter |SELECT filter(array(1, 2, 3), x -> x % 2 == 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|find_in_set |SELECT find_in_set('ab','abc,b,ab,c,def'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|first |SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|first_value |SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|flatten |SELECT flatten(array(array(1, 2), array(3, 4))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|float | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|floor |SELECT floor(-0.1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|forall |SELECT forall(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|format_number |SELECT format_number(12332.123456, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|format_string |SELECT format_string("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_csv |SELECT from_csv('1, 0.8', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_json |SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_unixtime |SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|from_utc_timestamp |SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|get |SELECT get(array(1, 2, 3), 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|get_json_object |SELECT get_json_object('{"a":"b"}', '$.a'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|getbit |SELECT getbit(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|greatest |SELECT greatest(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|grouping |SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name); |FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#7445, spark_grouping_id#7444L], [age#7441, name#7445]\n+- Expand [[age#7441, name#7442, name#7443, 0], [age#7441, name#7442, null, 1]], [age#7441, name#7442, name#7445, spark_grouping_id#7444L]\n +- Project [age#7441, name#7442, name#7442 AS name#7443]\n +- SubqueryAlias people\n +- LocalRelation [age#7441, name#7442]\n | | -|grouping_id |SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);|FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#8183, height#8184, spark_grouping_id#8182L], [age#8177, name#8183, height#8184]\n+- Expand [[age#8177, name#8178, height#8179, name#8180, height#8181, 0], [age#8177, name#8178, height#8179, name#8180, null, 1], [age#8177, name#8178, height#8179, null, height#8181, 2], [age#8177, name#8178, height#8179, null, null, 3]], [age#8177, name#8178, height#8179, name#8183, height#8184, spark_grouping_id#8182L]\n +- Project [age#8177, name#8178, height#8179, name#8178 AS name#8180, height#8179 AS height#8181]\n +- SubqueryAlias people\n +- LocalRelation [age#8177, name#8178, height#8179]\n | | -|hash |SELECT hash('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|hex |SELECT hex(17); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|histogram_numeric |SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|hour |SELECT hour('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|hypot |SELECT hypot(3, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|if |SELECT if(1 < 2, 'a', 'b'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|ifnull |SELECT ifnull(NULL, array('2')); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|ilike |SELECT ilike('Spark', '_Park'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|in |SELECT 1 in(1, 2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|initcap |SELECT initcap('sPark sql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|inline |SELECT inline(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|inline_outer |SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|input_file_block_length |SELECT input_file_block_length(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|input_file_block_start |SELECT input_file_block_start(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|input_file_name |SELECT input_file_name(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|instr |SELECT instr('SparkSQL', 'SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|int | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|isnan |SELECT isnan(cast('NaN' as double)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|isnotnull |SELECT isnotnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|isnull |SELECT isnull(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|json_array_length |SELECT json_array_length('[1,2,3,4]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|json_object_keys |SELECT json_object_keys('{}'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|json_tuple |SELECT json_tuple('{"a":1, "b":2}', 'a', 'b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|kurtosis |SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|lag |SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|last |SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|last_day |SELECT last_day('2009-01-12'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|last_value |SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|lcase |SELECT lcase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|lead |SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|least |SELECT least(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|left |SELECT left('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|len |SELECT len('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|length |SELECT length('Spark SQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|levenshtein |SELECT levenshtein('kitten', 'sitting'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|like |SELECT like('Spark', '_park'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|ln |SELECT ln(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|localtimestamp |SELECT localtimestamp(); |FAILED |Failed on native side: found CometNativeException |DATAFUSIONCLI_PATH env variable not set| -|locate |SELECT locate('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|log |SELECT log(10, 100); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|log10 |SELECT log10(10); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|log1p |SELECT log1p(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|log2 |SELECT log2(2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|lower |SELECT lower('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|lpad |SELECT lpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ltrim |SELECT ltrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|make_date |SELECT make_date(2013, 7, 15); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_dt_interval |SELECT make_dt_interval(1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_interval |SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_timestamp |SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_timestamp_ltz |SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_timestamp_ntz |SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|make_ym_interval |SELECT make_ym_interval(1, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map |SELECT map(1.0, '2', 3.0, '4'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_concat |SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_contains_key |SELECT map_contains_key(map(1, 'a', 2, 'b'), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_entries |SELECT map_entries(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_filter |SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_from_arrays |SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_from_entries |SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_keys |SELECT map_keys(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_values |SELECT map_values(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|map_zip_with |SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|mask |SELECT mask('abcd-EFGH-8765-4321'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|max |SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|max_by |SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|md5 |SELECT md5('Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|mean |SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|median |SELECT median(col) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|min |SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|min_by |SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|minute |SELECT minute('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|mod |SELECT 2 % 1.8; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|mode |SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|monotonically_increasing_id|SELECT monotonically_increasing_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|month |SELECT month('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|months_between |SELECT months_between('1997-02-28 10:30:00', '1996-10-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|named_struct |SELECT named_struct("a", 1, "b", 2, "c", 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|nanvl |SELECT nanvl(cast('NaN' as double), 123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|negative |SELECT negative(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|next_day |SELECT next_day('2015-01-14', 'TU'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|not |SELECT not true; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|now |SELECT now(); |FAILED |\nResults do not match for query:\nTimezone: sun.util.calendar.ZoneInfo[id="America/Los_Angeles",offset=-28800000,dstSavings=3600000,useDaylight=true,transitions=185,lastRule=java.util.SimpleTimeZone[id=America/Los_Angeles,offset=-28800000,dstSavings=3600000,useDaylight=true,startYear=0,startMode=3,startMonth=2,startDay=8,startDayOfWeek=1,startTime=7200000,startTimeMode=0,endMode=3,endMonth=10,endDay=1,endDayOfWeek=1,endTime=7200000,endTimeMode=0]]\nTimezone Env: \n\n== Parsed Logical Plan ==\nProject [now() AS now()#1510]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#1499])\n +- Relation [1#1499] parquet\n\n== Analyzed Logical Plan ==\nnow(): timestamp\nProject [now() AS now()#1510]\n+- SubqueryAlias tbl\n +- View (`tbl`, [1#1499])\n +- Relation [1#1499] parquet\n\n== Optimized Logical Plan ==\nProject [2024-05-24 14:25:36.919361 AS now()#1510]\n+- Relation [1#1499] parquet\n\n== Physical Plan ==\n*(1) ColumnarToRow\n+- CometProject [now()#1510], [2024-05-24 14:25:36.919361 AS now()#1510]\n +- CometScan parquet [] Batched: true, DataFilters: [], Format: CometParquet, Location: InMemoryFileIndex(1 paths)[file:/private/var/folders/6f/_s1vnnd55zgfkx7zlwnrnv0h0000gn/T/spark-f0..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<>\n\n== Results ==\n\n== Results ==\n!== Correct Answer - 1 == == Spark Answer - 1 ==\n struct struct\n![2024-05-24 14:25:36.887745] [2024-05-24 14:25:36.919361]\n \n |DATAFUSIONCLI_PATH env variable not set| -|nth_value |SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ntile |SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|nullif |SELECT nullif(2, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|nvl |SELECT nvl(NULL, array('2')); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|nvl2 |SELECT nvl2(NULL, 2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|octet_length |SELECT octet_length('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|or |SELECT true or false; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|overlay |SELECT overlay('Spark SQL' PLACING '_' FROM 6); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|parse_url |SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|percent_rank |SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|percentile |SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|percentile_approx |SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|pi |SELECT pi(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|pmod |SELECT pmod(10, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|posexplode |SELECT posexplode(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|posexplode_outer |SELECT posexplode_outer(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|position |SELECT position('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|positive |SELECT positive(1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|pow |SELECT pow(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|power |SELECT power(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|printf |SELECT printf("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|quarter |SELECT quarter('2016-08-31'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|radians |SELECT radians(180); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|raise_error |SELECT raise_error('custom error message'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rand |SELECT rand(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|randn |SELECT randn(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|random |SELECT random(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rank |SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|reduce |SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp |SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_count |SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_extract |SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_extract_all |SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_instr |SELECT regexp_instr('user@spark.apache.org', '@[^.]*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_like |SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_replace |SELECT regexp_replace('100-200', '(\\d+)', 'num'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regexp_substr |SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_avgx |SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|regr_avgy |SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|regr_count |SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|regr_intercept |SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_r2 |SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_slope |SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_sxx |SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_sxy |SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|regr_syy |SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|repeat |SELECT repeat('123', 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|replace |SELECT replace('ABCabc', 'abc', 'DEF'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|reverse |SELECT reverse('Spark SQL'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|right |SELECT right('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rint |SELECT rint(12.3456); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rlike |SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|round |SELECT round(2.5, 0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|row_number |SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rpad |SELECT rpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|rtrim |SELECT rtrim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|schema_of_csv |SELECT schema_of_csv('1,abc'); |FAILED |[DATATYPE_MISMATCH.NON_FOLDABLE_INPUT] Cannot resolve "schema_of_csv(c0)" due to data type mismatch: the input csv should be a foldable "STRING" expression; however, got "c0".; line 1 pos 7;\n'Project [unresolvedalias(schema_of_csv(c0#6515), None)]\n+- SubqueryAlias tbl\n +- View (`tbl`, [c0#6515])\n +- Relation [c0#6515] parquet\n |DATAFUSIONCLI_PATH env variable not set| -|schema_of_json |SELECT schema_of_json('[{"col":0}]'); |FAILED |[DATATYPE_MISMATCH.NON_FOLDABLE_INPUT] Cannot resolve "schema_of_json(c0)" due to data type mismatch: the input json should be a foldable "STRING" expression; however, got "c0".; line 1 pos 7;\n'Project [unresolvedalias(schema_of_json(c0#7685), None)]\n+- SubqueryAlias tbl\n +- View (`tbl`, [c0#7685])\n +- Relation [c0#7685] parquet\n |DATAFUSIONCLI_PATH env variable not set| -|sec |SELECT sec(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|second |SELECT second('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sentences |SELECT sentences('Hi there! Good morning.'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sequence |SELECT sequence(1, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sha |SELECT sha('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sha1 |SELECT sha1('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sha2 |SELECT sha2('Spark', 256); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|shiftleft |SELECT shiftleft(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|shiftright |SELECT shiftright(4, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|shiftrightunsigned |SELECT shiftrightunsigned(4, 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|shuffle |SELECT shuffle(array(1, 20, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sign |SELECT sign(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|signum |SELECT signum(40); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|sin |SELECT sin(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|sinh |SELECT sinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|size |SELECT size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|skewness |SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|slice |SELECT slice(array(1, 2, 3, 4), 2, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|smallint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|some |SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|sort_array |SELECT sort_array(array('b', 'd', null, 'c', 'a'), true); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|soundex |SELECT soundex('Miller'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|space |SELECT concat(space(2), '1'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|spark_partition_id |SELECT spark_partition_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|split |SELECT split('oneAtwoBthreeC', '[ABC]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|split_part |SELECT split_part('11.12.13', '.', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sqrt |SELECT sqrt(4); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|stack |SELECT stack(2, 1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|startswith |SELECT startswith('Spark SQL', 'Spark'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|std |SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|stddev |SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|stddev_pop |SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|stddev_samp |SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|str_to_map |SELECT str_to_map('a:1,b:2,c:3', ',', ':'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|string | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|struct |SELECT struct(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|substr |SELECT substr('Spark SQL', 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|substring |SELECT substring('Spark SQL', 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|substring_index |SELECT substring_index('www.apache.org', '.', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|sum |SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|tan |SELECT tan(0); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|tanh |SELECT tanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|timestamp | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|timestamp_micros |SELECT timestamp_micros(1230219000123123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|timestamp_millis |SELECT timestamp_millis(1230219000123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|timestamp_seconds |SELECT timestamp_seconds(1230219000); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|tinyint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | -|to_binary |SELECT to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_char |SELECT to_char(454, '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_csv |SELECT to_csv(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_date |SELECT to_date('2009-07-30 04:17:52'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_json |SELECT to_json(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_number |SELECT to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_timestamp |SELECT to_timestamp('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_timestamp_ltz |SELECT to_timestamp_ltz('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_timestamp_ntz |SELECT to_timestamp_ntz('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_unix_timestamp |SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|to_utc_timestamp |SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|transform |SELECT transform(array(1, 2, 3), x -> x + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|transform_keys |SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|transform_values |SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|translate |SELECT translate('AaBbCc', 'abc', '123'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|trim |SELECT trim(' SparkSQL '); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|trunc |SELECT trunc('2019-08-04', 'week'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_add |SELECT try_add(1, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_avg |SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_divide |SELECT try_divide(3, 2); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_element_at |SELECT try_element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_multiply |SELECT try_multiply(2, 3); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_subtract |SELECT try_subtract(2, 1); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|try_sum |SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_to_binary |SELECT try_to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_to_number |SELECT try_to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|try_to_timestamp |SELECT try_to_timestamp('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|typeof |SELECT typeof(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|ucase |SELECT ucase('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|unbase64 |SELECT unbase64('U3BhcmsgU1FM'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unhex |SELECT decode(unhex('537061726B2053514C'), 'UTF-8'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|unix_date |SELECT unix_date(DATE("1970-01-02")); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_micros |SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_millis |SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_seconds |SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|unix_timestamp |SELECT unix_timestamp(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|upper |SELECT upper('SparkSql'); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|url_decode |SELECT url_decode('https%3A%2F%2Fspark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|url_encode |SELECT url_encode('https://spark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|user |SELECT user(); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|uuid |SELECT uuid(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|var_pop |SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|var_samp |SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|variance |SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|version |SELECT version(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|weekday |SELECT weekday('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|weekofyear |SELECT weekofyear('2008-02-20'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|when |SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END; |FAILED |\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'then'.(line 1, pos 18)\n\n== SQL ==\nselect case a > b then c else d end from tbl\n------------------^^^\n |DATAFUSIONCLI_PATH env variable not set| -|width_bucket |SELECT width_bucket(5.3, 0.2, 10.6, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath |SELECT xpath('b1b2b3c1c2','a/b/text()'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_boolean |SELECT xpath_boolean('1','a/b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_double |SELECT xpath_double('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_float |SELECT xpath_float('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_int |SELECT xpath_int('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_long |SELECT xpath_long('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_number |SELECT xpath_number('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_short |SELECT xpath_short('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xpath_string |SELECT xpath_string('bcc','a/c'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|xxhash64 |SELECT xxhash64('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|year |SELECT year('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |DATAFUSIONCLI_PATH env variable not set| -|zip_with |SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); |FAILED |[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `zip_with` requires 3 parameters but the actual number is 2. Please, refer to 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix.; line 1 pos 7 |DATAFUSIONCLI_PATH env variable not set| -|| |SELECT 3 | 5; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -|~ |SELECT ~ 0; |PASSED |OK |DATAFUSIONCLI_PATH env variable not set| -+---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------+ ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|name |query |result |cometMessage |datafusionMessage | ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +|! |SELECT ! true; |PASSED |OK |std_err: SQL error: ParserError("Expected an expression:, found: !")\n | +|% |SELECT 2 % 1.8; |PASSED |OK |OK | +|& |SELECT 3 & 5; |PASSED |OK |OK | +|* |SELECT 2 * 3; |PASSED |OK |OK | +|+ |SELECT 1 + 2; |PASSED |OK |OK | +|- |SELECT 2 - 1; |PASSED |OK |OK | +|/ |SELECT 3 / 2; |PASSED |OK |OK | +|< |SELECT 1 < 2; |PASSED |OK |OK | +|<= |SELECT 2 <= 2; |PASSED |OK |OK | +|<=> |SELECT 2 <=> 2; |PASSED |OK |std_err: This feature is not implemented: Unsupported SQL binary operator Spaceship\n | +|= |SELECT 2 = 2; |PASSED |OK |OK | +|== |SELECT 2 == 2; |PASSED |OK |OK | +|> |SELECT 2 > 1; |PASSED |OK |OK | +|>= |SELECT 2 >= 1; |PASSED |OK |OK | +|^ |SELECT 3 ^ 5; |PASSED |OK |OK | +|abs |SELECT abs(-1); |PASSED |OK |OK | +|acos |SELECT acos(1); |PASSED |OK |OK | +|acosh |SELECT acosh(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|add_months |SELECT add_months('2016-08-31', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'add_months'.\nDid you mean 'radians'?\n | +|aes_decrypt |SELECT aes_decrypt(unhex('83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94'), '0000111122223333'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'aes_decrypt'.\nDid you mean 'list_except'?\n | +|aes_encrypt |SELECT hex(aes_encrypt('Spark', '0000111122223333')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n | +|aggregate |SELECT aggregate(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|and |SELECT true and true; |PASSED |OK |OK | +|any |SELECT any(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'any'.\nDid you mean 'ln'?\n | +|any_value |SELECT any_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'any_value'.\nDid you mean 'LAST_VALUE'?\n | +|approx_count_distinct |SELECT approx_count_distinct(col1) FROM VALUES (1), (1), (2), (2), (3) tab(col1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'approx_count_distinct'.\nDid you mean 'APPROX_DISTINCT'?\n | +|approx_percentile |SELECT approx_percentile(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n | +|array |SELECT array(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_agg |SELECT array_agg(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|array_append |SELECT array_append(array('b', 'd', 'c', 'a'), 'd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | +|array_compact |SELECT array_compact(array(1, 2, 3, null)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_contains |SELECT array_contains(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_distinct |SELECT array_distinct(array(1, 2, 3, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_except |SELECT array_except(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_insert |SELECT array_insert(array(1, 2, 3, 4), 5, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_intersect |SELECT array_intersect(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_join |SELECT array_join(array('hello', 'world'), ' '); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'hello'")\n | +|array_max |SELECT array_max(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_min |SELECT array_min(array(1, 20, null, 3)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_position |SELECT array_position(array(3, 2, 1), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 3")\n | +|array_remove |SELECT array_remove(array(1, 2, 3, null, 3), 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|array_repeat |SELECT array_repeat('123', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|array_size |SELECT array_size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | +|array_sort |SELECT array_sort(array(5, 6, 1), (left, right) -> case when left < right then -1 when left > right then 1 else 0 end); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 5")\n | +|array_union |SELECT array_union(array(1, 2, 3), array(1, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|arrays_overlap |SELECT arrays_overlap(array(1, 2, 3), array(3, 4, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|arrays_zip |SELECT arrays_zip(array(1, 2, 3), array(2, 3, 4)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|ascii |SELECT ascii('222'); |PASSED |OK |OK | +|asin |SELECT asin(0); |PASSED |OK |OK | +|asinh |SELECT asinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|assert_true |SELECT assert_true(0 < 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'assert_true'.\nDid you mean 'date_trunc'?\n | +|atan |SELECT atan(0); |PASSED |OK |OK | +|atan2 |SELECT atan2(0, 0); |PASSED |OK |OK | +|atanh |SELECT atanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|avg |SELECT avg(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |OK | +|base64 |SELECT base64('Spark SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'base64'.\nDid you mean 'asinh'?\n | +|bigint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bin |SELECT bin(13); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bin'.\nDid you mean 'sin'?\n | +|binary | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bit_and |SELECT bit_and(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |OK | +|bit_count |SELECT bit_count(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bit_count'.\nDid you mean 'COUNT'?\n | +|bit_get |SELECT bit_get(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bit_get'.\nDid you mean 'BIT_AND'?\n | +|bit_length |SELECT bit_length('Spark SQL'); |PASSED |OK |OK | +|bit_or |SELECT bit_or(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |OK | +|bit_xor |SELECT bit_xor(col) FROM VALUES (3), (5) AS tab(col); |PASSED |OK |OK | +|bool_and |SELECT bool_and(col) FROM VALUES (true), (true), (true) AS tab(col); |PASSED |OK |OK | +|bool_or |SELECT bool_or(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |OK | +|boolean | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|bround |SELECT bround(2.5, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'bround'.\nDid you mean 'round'?\n | +|btrim |SELECT btrim(' SparkSQL '); |PASSED |OK |OK | +|cardinality |SELECT cardinality(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | +|cast |SELECT cast('10' as int); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|cbrt |SELECT cbrt(27.0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|ceil |SELECT ceil(-0.1); |PASSED |OK |OK | +|ceiling |SELECT ceiling(-0.1); |PASSED |OK |std_err: Error during planning: Invalid function 'ceiling'.\nDid you mean 'ceil'?\n | +|char |SELECT char(65); |PASSED |OK |std_err: Error during planning: Invalid function 'char'.\nDid you mean 'chr'?\n | +|char_length |SELECT char_length('Spark SQL '); |PASSED |OK |OK | +|character_length |SELECT character_length('Spark SQL '); |PASSED |OK |OK | +|chr |SELECT chr(65); |PASSED |OK |OK | +|coalesce |SELECT coalesce(NULL, 1, NULL); |PASSED |OK |OK | +|collect_list |SELECT collect_list(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'collect_list'.\nDid you mean 'make_list'?\n | +|collect_set |SELECT collect_set(col) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'collect_set'.\nDid you mean 'coalesce'?\n | +|concat |SELECT concat('Spark', 'SQL'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|concat_ws |SELECT concat_ws(' ', 'Spark', 'SQL'); |PASSED |OK |OK | +|contains |SELECT contains('Spark SQL', 'Spark'); |PASSED |OK |std_err: Error during planning: Invalid function 'contains'.\nDid you mean 'concat'?\n | +|conv |SELECT conv('100', 2, 10); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'conv'.\nDid you mean 'cos'?\n | +|convert_timezone |SELECT convert_timezone('Europe/Brussels', 'America/Los_Angeles', timestamp_ntz'2021-12-06 00:00:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected ), found: '2021-12-06 00:00:00'")\n | +|corr |SELECT corr(c1, c2) FROM VALUES (3, 2), (3, 3), (6, 4) as tab(c1, c2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|cos |SELECT cos(0); |PASSED |OK |OK | +|cosh |SELECT cosh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|cot |SELECT cot(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|count |SELECT count(*) FROM VALUES (NULL), (5), (5), (20) AS tab(col); |PASSED |OK |OK | +|count_if |SELECT count_if(col % 2 = 0) FROM VALUES (NULL), (0), (1), (2), (3) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'count_if'.\nDid you mean 'COUNT'?\n | +|count_min_sketch |SELECT hex(count_min_sketch(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: SQL error: ParserError("Expected ), found: d")\n | +|covar_pop |SELECT covar_pop(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |PASSED |OK |OK | +|covar_samp |SELECT covar_samp(c1, c2) FROM VALUES (1,1), (2,2), (3,3) AS tab(c1, c2); |PASSED |OK |OK | +|crc32 |SELECT crc32('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'crc32'.\nDid you mean 'nvl2'?\n | +|csc |SELECT csc(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'csc'.\nDid you mean 'cot'?\n | +|cume_dist |SELECT a, b, cume_dist() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|curdate |SELECT curdate(); |PASSED |OK |std_err: Error during planning: Invalid function 'curdate'.\nDid you mean 'to_date'?\n | +|current_catalog |SELECT current_catalog(); |PASSED |OK |std_err: SQL error: ParserError("Expected end of statement, found: (")\n | +|current_database |SELECT current_database(); |PASSED |OK |std_err: Error during planning: Invalid function 'current_database'.\nDid you mean 'current_date'?\n | +|current_date |SELECT current_date(); |PASSED |OK |OK | +|current_schema |SELECT current_schema(); |PASSED |OK |std_err: Error during planning: Invalid function 'current_schema'.\nDid you mean 'current_time'?\n | +|current_timestamp |SELECT current_timestamp(); |FAILED |Unsupported: Results do not match for query |std_err: Error during planning: Invalid function 'current_timestamp'.\nDid you mean 'current_time'?\n | +|current_timezone |SELECT current_timezone(); |PASSED |OK |std_err: Error during planning: Invalid function 'current_timezone'.\nDid you mean 'current_time'?\n | +|current_user |SELECT current_user(); |PASSED |OK |std_err: SQL error: ParserError("Expected end of statement, found: (")\n | +|date | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|date_add |SELECT date_add('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_add'.\nDid you mean 'date_bin'?\n | +|date_diff |SELECT date_diff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_diff'.\nDid you mean 'date_bin'?\n | +|date_format |SELECT date_format('2016-04-08', 'y'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|date_from_unix_date |SELECT date_from_unix_date(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_from_unix_date'.\nDid you mean 'from_unixtime'?\n | +|date_part |SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |OK | +|date_sub |SELECT date_sub('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'date_sub'.\nDid you mean 'date_bin'?\n | +|date_trunc |SELECT date_trunc('YEAR', '2015-03-05T09:32:05.359'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|dateadd |SELECT dateadd('2016-07-30', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dateadd'.\nDid you mean 'datepart'?\n | +|datediff |SELECT datediff('2009-07-31', '2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'datediff'.\nDid you mean 'date_bin'?\n | +|datepart |SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |OK | +|day |SELECT day('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'day'.\nDid you mean 'tan'?\n | +|dayofmonth |SELECT dayofmonth('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dayofmonth'.\nDid you mean 'date_part'?\n | +|dayofweek |SELECT dayofweek('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dayofweek'.\nDid you mean 'lower'?\n | +|dayofyear |SELECT dayofyear('2016-04-09'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'dayofyear'.\nDid you mean 'to_char'?\n | +|decimal | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|decode |SELECT decode(encode('abc', 'utf-8'), 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n | +|degrees |SELECT degrees(3.141592653589793); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|dense_rank |SELECT a, b, dense_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: No function matches the given name and argument types 'DENSE_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tDENSE_RANK()\n | +|div |SELECT 3 div 2; |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("No infix parser for token Word(Word { value: \"div\", quote_style: None, keyword: DIV })")\n | +|double | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|e |SELECT e(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'e'.\nDid you mean 'ln'?\n | +|element_at |SELECT element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|elt |SELECT elt(1, 'scala', 'java'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'elt'.\nDid you mean 'exp'?\n | +|encode |SELECT encode('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: There is no built-in encoding named 'utf-8', currently supported encodings are: base64, hex\n | +|endswith |SELECT endswith('Spark SQL', 'SQL'); |PASSED |OK |std_err: Error during planning: Invalid function 'endswith'.\nDid you mean 'ends_with'?\n | +|equal_null |SELECT equal_null(3, 3); |PASSED |OK |std_err: Error during planning: Invalid function 'equal_null'.\nDid you mean 'ifnull'?\n | +|every |SELECT every(col) FROM VALUES (true), (true), (true) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'every'.\nDid you mean 'overlay'?\n | +|exists |SELECT exists(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n | +|exp |SELECT exp(0); |PASSED |OK |OK | +|explode |SELECT explode(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | +|explode_outer |SELECT explode_outer(array(10, 20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | +|expm1 |SELECT expm1(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'expm1'.\nDid you mean 'exp'?\n | +|extract |SELECT extract(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'); |PASSED |OK |OK | +|factorial |SELECT factorial(5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|filter |SELECT filter(array(1, 2, 3), x -> x % 2 == 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|find_in_set |SELECT find_in_set('ab','abc,b,ab,c,def'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|first |SELECT first(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'first'.\nDid you mean 'sqrt'?\n | +|first_value |SELECT first_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |OK | +|flatten |SELECT flatten(array(array(1, 2), array(3, 4))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: array")\n | +|float | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|floor |SELECT floor(-0.1); |PASSED |OK |OK | +|forall |SELECT forall(array(1, 2, 3), x -> x % 2 == 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|format_number |SELECT format_number(12332.123456, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'format_number'.\nDid you mean 'FIRST_VALUE'?\n | +|format_string |SELECT format_string("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'format_string'.\nDid you mean 'array_to_string'?\n | +|from_csv |SELECT from_csv('1, 0.8', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'from_csv'.\nDid you mean 'arrow_cast'?\n | +|from_json |SELECT from_json('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'from_json'.\nDid you mean 'floor'?\n | +|from_unixtime |SELECT from_unixtime(0, 'yyyy-MM-dd HH:mm:ss'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Error during planning: [data_types_with_scalar_udf] Coercion from [Int64, Utf8] to the signature Uniform(1, [Int64]) failed. and No function matches the given name and argument types 'from_unixtime(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tfrom_unixtime(Int64)\n | +|from_utc_timestamp |SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'from_utc_timestamp'.\nDid you mean 'to_timestamp'?\n | +|get |SELECT get(array(1, 2, 3), 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|get_json_object |SELECT get_json_object('{"a":"b"}', '$.a'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'get_json_object'.\nDid you mean 'list_intersect'?\n | +|getbit |SELECT getbit(11, 0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'getbit'.\nDid you mean 'ceil'?\n | +|greatest |SELECT greatest(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'greatest'.\nDid you mean 'repeat'?\n | +|grouping |SELECT name, grouping(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name); |FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#7445, spark_grouping_id#7444L], [age#7441, name#7445]\n+- Expand [[age#7441, name#7442, name#7443, 0], [age#7441, name#7442, null, 1]], [age#7441, name#7442, name#7445, spark_grouping_id#7444L]\n +- Project [age#7441, name#7442, name#7442 AS name#7443]\n +- SubqueryAlias people\n +- LocalRelation [age#7441, name#7442]\n | | +|grouping_id |SELECT name, grouping_id(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height);|FAILED |[MISSING_AGGREGATION] The non-aggregating expression "age" is based on columns which are not participating in the GROUP BY clause.\nAdd the columns or the expression to the GROUP BY, aggregate the expression, or use "any_value(age)" if you do not care which of the values within a group is returned.;\nAggregate [name#8183, height#8184, spark_grouping_id#8182L], [age#8177, name#8183, height#8184]\n+- Expand [[age#8177, name#8178, height#8179, name#8180, height#8181, 0], [age#8177, name#8178, height#8179, name#8180, null, 1], [age#8177, name#8178, height#8179, null, height#8181, 2], [age#8177, name#8178, height#8179, null, null, 3]], [age#8177, name#8178, height#8179, name#8183, height#8184, spark_grouping_id#8182L]\n +- Project [age#8177, name#8178, height#8179, name#8178 AS name#8180, height#8179 AS height#8181]\n +- SubqueryAlias people\n +- LocalRelation [age#8177, name#8178, height#8179]\n| | +|hash |SELECT hash('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n | +|hex |SELECT hex(17); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'hex'.\nDid you mean 'exp'?\n | +|histogram_numeric |SELECT histogram_numeric(col, 5) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'histogram_numeric'.\nDid you mean 'list_reverse'?\n | +|hour |SELECT hour('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'hour'.\nDid you mean 'CORR'?\n | +|hypot |SELECT hypot(3, 4); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'hypot'.\nDid you mean 'pow'?\n | +|if |SELECT if(1 < 2, 'a', 'b'); |PASSED |OK |std_err: Error during planning: Invalid function 'if'.\nDid you mean 'sin'?\n | +|ifnull |SELECT ifnull(NULL, array('2')); |PASSED |OK |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n | +|ilike |SELECT ilike('Spark', '_Park'); |PASSED |OK |std_err: Error during planning: Invalid function 'ilike'.\nDid you mean 'ceil'?\n | +|in |SELECT 1 in(1, 2, 3); |PASSED |OK |OK | +|initcap |SELECT initcap('sPark sql'); |PASSED |OK |OK | +|inline |SELECT inline(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n | +|inline_outer |SELECT inline_outer(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n | +|input_file_block_length |SELECT input_file_block_length(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'input_file_block_length'.\nDid you mean 'octet_length'?\n | +|input_file_block_start |SELECT input_file_block_start(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'input_file_block_start'.\nDid you mean 'list_replace_all'?\n | +|input_file_name |SELECT input_file_name(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'input_file_name'.\nDid you mean 'to_timestamp'?\n | +|instr |SELECT instr('SparkSQL', 'SQL'); |PASSED |OK |OK | +|int | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|isnan |SELECT isnan(cast('NaN' as double)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|isnotnull |SELECT isnotnull(1); |PASSED |OK |std_err: Error during planning: Invalid function 'isnotnull'.\nDid you mean 'ifnull'?\n | +|isnull |SELECT isnull(1); |PASSED |OK |std_err: Error during planning: Invalid function 'isnull'.\nDid you mean 'ifnull'?\n | +|json_array_length |SELECT json_array_length('[1,2,3,4]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'json_array_length'.\nDid you mean 'array_length'?\n | +|json_object_keys |SELECT json_object_keys('{}'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'json_object_keys'.\nDid you mean 'concat_ws'?\n | +|json_tuple |SELECT json_tuple('{"a":1, "b":2}', 'a', 'b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'json_tuple'.\nDid you mean 'ifnull'?\n | +|kurtosis |SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'kurtosis'.\nDid you mean 'rtrim'?\n | +|lag |SELECT a, b, lag(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|last |SELECT last(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'last'.\nDid you mean 'left'?\n | +|last_day |SELECT last_day('2009-01-12'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'last_day'.\nDid you mean 'list_cat'?\n | +|last_value |SELECT last_value(col) FROM VALUES (10), (5), (20) AS tab(col); |PASSED |OK |OK | +|lcase |SELECT lcase('SparkSql'); |PASSED |OK |std_err: Error during planning: Invalid function 'lcase'.\nDid you mean 'cos'?\n | +|lead |SELECT a, b, lead(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|least |SELECT least(10, 9, 2, 4, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'least'.\nDid you mean 'left'?\n | +|left |SELECT left('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|len |SELECT len('Spark SQL '); |PASSED |OK |std_err: Error during planning: Invalid function 'len'.\nDid you mean 'ln'?\n | +|length |SELECT length('Spark SQL '); |PASSED |OK |OK | +|levenshtein |SELECT levenshtein('kitten', 'sitting'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|like |SELECT like('Spark', '_park'); |PASSED |OK |std_err: Error during planning: Invalid function 'like'.\nDid you mean 'lpad'?\n | +|ln |SELECT ln(1); |PASSED |OK |OK | +|localtimestamp |SELECT localtimestamp(); |FAILED |Failed on native side: found CometNativeException |std_err: Error during planning: Invalid function 'localtimestamp'.\nDid you mean 'to_timestamp'?\n | +|locate |SELECT locate('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'locate'.\nDid you mean 'concat'?\n | +|log |SELECT log(10, 100); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|log10 |SELECT log10(10); |PASSED |OK |OK | +|log1p |SELECT log1p(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'log1p'.\nDid you mean 'log10'?\n | +|log2 |SELECT log2(2); |PASSED |OK |OK | +|lower |SELECT lower('SparkSql'); |PASSED |OK |OK | +|lpad |SELECT lpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|ltrim |SELECT ltrim(' SparkSQL '); |PASSED |OK |OK | +|make_date |SELECT make_date(2013, 7, 15); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|make_dt_interval |SELECT make_dt_interval(1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_dt_interval'.\nDid you mean 'make_date'?\n | +|make_interval |SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_interval'.\nDid you mean 'make_date'?\n | +|make_timestamp |SELECT make_timestamp(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_timestamp'.\nDid you mean 'to_timestamp'?\n | +|make_timestamp_ltz |SELECT make_timestamp_ltz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n | +|make_timestamp_ntz |SELECT make_timestamp_ntz(2014, 12, 28, 6, 30, 45.887); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_timestamp_ntz'.\nDid you mean 'to_timestamp'?\n | +|make_ym_interval |SELECT make_ym_interval(1, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'make_ym_interval'.\nDid you mean 'make_date'?\n | +|map |SELECT map(1.0, '2', 3.0, '4'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map'.\nDid you mean 'MAX'?\n | +|map_concat |SELECT map_concat(map(1, 'a', 2, 'b'), map(3, 'c')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_concat'.\nDid you mean 'array_concat'?\n | +|map_contains_key |SELECT map_contains_key(map(1, 'a', 2, 'b'), 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_contains_key'.\nDid you mean 'array_contains'?\n | +|map_entries |SELECT map_entries(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_entries'.\nDid you mean 'make_list'?\n | +|map_filter |SELECT map_filter(map(1, 0, 2, 2, 3, -1), (k, v) -> k > v); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_filter'.\nDid you mean 'make_date'?\n | +|map_from_arrays |SELECT map_from_arrays(array(1.0, 3.0), array('2', '4')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1.0")\n | +|map_from_entries |SELECT map_from_entries(array(struct(1, 'a'), struct(2, 'b'))); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: struct")\n | +|map_keys |SELECT map_keys(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_keys'.\nDid you mean 'to_hex'?\n | +|map_values |SELECT map_values(map(1, 'a', 2, 'b')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_values'.\nDid you mean 'LAST_VALUE'?\n | +|map_zip_with |SELECT map_zip_with(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'map_zip_with'.\nDid you mean 'starts_with'?\n | +|mask |SELECT mask('abcd-EFGH-8765-4321'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'mask'.\nDid you mean 'MAX'?\n | +|max |SELECT max(col) FROM VALUES (10), (50), (20) AS tab(col); |PASSED |OK |OK | +|max_by |SELECT max_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'max_by'.\nDid you mean 'MAX'?\n | +|md5 |SELECT md5('Spark'); |PASSED |OK |OK | +|mean |SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |OK | +|median |SELECT median(col) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|min |SELECT min(col) FROM VALUES (10), (-1), (20) AS tab(col); |PASSED |OK |OK | +|min_by |SELECT min_by(x, y) FROM VALUES (('a', 10)), (('b', 50)), (('c', 20)) AS tab(x, y); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'min_by'.\nDid you mean 'MIN'?\n | +|minute |SELECT minute('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'minute'.\nDid you mean 'instr'?\n | +|mod |SELECT 2 % 1.8; |PASSED |OK |OK | +|mode |SELECT mode(col) FROM VALUES (0), (10), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'mode'.\nDid you mean 'md5'?\n | +|monotonically_increasing_id|SELECT monotonically_increasing_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'monotonically_increasing_id'.\nDid you mean 'array_intersect'?\n | +|month |SELECT month('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'month'.\nDid you mean 'cosh'?\n | +|months_between |SELECT months_between('1997-02-28 10:30:00', '1996-10-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'months_between'.\nDid you mean 'NTH_VALUE'?\n | +|named_struct |SELECT named_struct("a", 1, "b", 2, "c", 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Schema error: No field named a.\n | +|nanvl |SELECT nanvl(cast('NaN' as double), 123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|negative |SELECT negative(1); |PASSED |OK |std_err: Error during planning: Invalid function 'negative'.\nDid you mean 'nanvl'?\n | +|next_day |SELECT next_day('2015-01-14', 'TU'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'next_day'.\nDid you mean 'today'?\n | +|not |SELECT not true; |PASSED |OK |OK | +|now |SELECT now(); |FAILED |Unsupported: Results do not match for query | | +|nth_value |SELECT a, b, nth_value(b, 2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|ntile |SELECT a, b, ntile(2) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|nullif |SELECT nullif(2, 2); |PASSED |OK |OK | +|nvl |SELECT nvl(NULL, array('2')); |PASSED |OK |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: '2'")\n | +|nvl2 |SELECT nvl2(NULL, 2, 1); |PASSED |OK |OK | +|octet_length |SELECT octet_length('Spark SQL'); |PASSED |OK |OK | +|or |SELECT true or false; |PASSED |OK |OK | +|overlay |SELECT overlay('Spark SQL' PLACING '_' FROM 6); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|parse_url |SELECT parse_url('http://spark.apache.org/path?query=1', 'HOST'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'parse_url'.\nDid you mean 'power'?\n | +|percent_rank |SELECT a, b, percent_rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: No function matches the given name and argument types 'PERCENT_RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tPERCENT_RANK()\n | +|percentile |SELECT percentile(col, 0.3) FROM VALUES (0), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'percentile'.\nDid you mean 'ceil'?\n | +|percentile_approx |SELECT percentile_approx(col, array(0.5, 0.4, 0.1), 100) FROM VALUES (0), (1), (2), (10) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 0.5")\n | +|pi |SELECT pi(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|pmod |SELECT pmod(10, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'pmod'.\nDid you mean 'pow'?\n | +|posexplode |SELECT posexplode(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | +|posexplode_outer |SELECT posexplode_outer(array(10,20)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 10")\n | +|position |SELECT position('bar', 'foobarbar'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Position function must include IN keyword")\n | +|positive |SELECT positive(1); |PASSED |OK |std_err: Error during planning: Invalid function 'positive'.\nDid you mean 'position'?\n | +|pow |SELECT pow(2, 3); |PASSED |OK |OK | +|power |SELECT power(2, 3); |PASSED |OK |OK | +|printf |SELECT printf("Hello World %d %s", 100, "days"); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'printf'.\nDid you mean 'instr'?\n | +|quarter |SELECT quarter('2016-08-31'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'quarter'.\nDid you mean 'flatten'?\n | +|radians |SELECT radians(180); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|raise_error |SELECT raise_error('custom error message'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'raise_error'.\nDid you mean 'make_array'?\n | +|rand |SELECT rand(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'rand'.\nDid you mean 'rpad'?\n | +|randn |SELECT randn(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'randn'.\nDid you mean 'range'?\n | +|random |SELECT random(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|rank |SELECT a, b, rank(b) OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: No function matches the given name and argument types 'RANK(Int32)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tRANK()\n | +|reduce |SELECT reduce(array(1, 2, 3), 0, (acc, x) -> acc + x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|regexp |SELECT regexp('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n | +|regexp_count |SELECT regexp_count('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_count'.\nDid you mean 'REGR_COUNT'?\n | +|regexp_extract |SELECT regexp_extract('100-200', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_extract'.\nDid you mean 'regexp_match'?\n | +|regexp_extract_all |SELECT regexp_extract_all('100-200, 300-400', '(\\d+)-(\\d+)', 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_extract_all'.\nDid you mean 'regexp_replace'?\n | +|regexp_instr |SELECT regexp_instr('user@spark.apache.org', '@[^.]*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_instr'.\nDid you mean 'regexp_like'?\n | +|regexp_like |SELECT regexp_like('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n | +|regexp_replace |SELECT regexp_replace('100-200', '(\\d+)', 'num'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|regexp_substr |SELECT regexp_substr('Steven Jones and Stephen Smith are the best players', 'Ste(v|ph)en'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'regexp_substr'.\nDid you mean 'regexp_match'?\n | +|regr_avgx |SELECT regr_avgx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |OK | +|regr_avgy |SELECT regr_avgy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |OK | +|regr_count |SELECT regr_count(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |PASSED |OK |OK | +|regr_intercept |SELECT regr_intercept(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|regr_r2 |SELECT regr_r2(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|regr_slope |SELECT regr_slope(y, x) FROM VALUES (1,1), (2,2), (3,3) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|regr_sxx |SELECT regr_sxx(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|regr_sxy |SELECT regr_sxy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|regr_syy |SELECT regr_syy(y, x) FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|repeat |SELECT repeat('123', 2); |PASSED |OK |OK | +|replace |SELECT replace('ABCabc', 'abc', 'DEF'); |PASSED |OK |OK | +|reverse |SELECT reverse('Spark SQL'); |PASSED |OK |OK | +|right |SELECT right('Spark SQL', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|rint |SELECT rint(12.3456); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'rint'.\nDid you mean 'sin'?\n | +|rlike |SELECT rlike('%SystemDrive%\Users\John', '%SystemDrive%\\Users.*'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: TokenizerError("unsupported escape char: '\\U'")\n | +|round |SELECT round(2.5, 0); |PASSED |OK |OK | +|row_number |SELECT a, b, row_number() OVER (PARTITION BY a ORDER BY b) FROM VALUES ('A1', 2), ('A1', 1), ('A2', 3), ('A1', 1) tab(a, b); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|rpad |SELECT rpad('hi', 5, '??'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|rtrim |SELECT rtrim(' SparkSQL '); |PASSED |OK |OK | +|schema_of_csv |SELECT schema_of_csv('1,abc'); |FAILED |[DATATYPE_MISMATCH.NON_FOLDABLE_INPUT] Cannot resolve "schema_of_csv(c0)" due to data type mismatch: the input csv should be a foldable "STRING" expression; however, got "c0".; line 1 pos 7;\n'Project [unresolvedalias(schema_of_csv(c0#6515), None)]\n+- SubqueryAlias tbl\n +- View (`tbl`, [c0#6515])\n +- Relation [c0#6515] parquet\n |std_err: Error during planning: Invalid function 'schema_of_csv'.\nDid you mean 'concat_ws'?\n | +|schema_of_json |SELECT schema_of_json('[{"col":0}]'); |FAILED |[DATATYPE_MISMATCH.NON_FOLDABLE_INPUT] Cannot resolve "schema_of_json(c0)" due to data type mismatch: the input json should be a foldable "STRING" expression; however, got "c0".; line 1 pos 7;\n'Project [unresolvedalias(schema_of_json(c0#7685), None)]\n+- SubqueryAlias tbl\n +- View (`tbl`, [c0#7685])\n +- Relation [c0#7685] parquet\n |std_err: Error during planning: Invalid function 'schema_of_json'.\nDid you mean 'concat_ws'?\n | +|sec |SELECT sec(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sec'.\nDid you mean 'sin'?\n | +|second |SELECT second('2009-07-30 12:58:59'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'second'.\nDid you mean 'decode'?\n | +|sentences |SELECT sentences('Hi there! Good morning.'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sentences'.\nDid you mean 'degrees'?\n | +|sequence |SELECT sequence(1, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sequence'.\nDid you mean 'replace'?\n | +|sha |SELECT sha('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sha'.\nDid you mean 'chr'?\n | +|sha1 |SELECT sha1('Spark'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sha1'.\nDid you mean 'sha512'?\n | +|sha2 |SELECT sha2('Spark', 256); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'sha2'.\nDid you mean 'sha224'?\n | +|shiftleft |SELECT shiftleft(2, 1); |PASSED |OK |std_err: Error during planning: Invalid function 'shiftleft'.\nDid you mean 'left'?\n | +|shiftright |SELECT shiftright(4, 1); |PASSED |OK |std_err: Error during planning: Invalid function 'shiftright'.\nDid you mean 'right'?\n | +|shiftrightunsigned |SELECT shiftrightunsigned(4, 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'shiftrightunsigned'.\nDid you mean 'list_union'?\n | +|shuffle |SELECT shuffle(array(1, 20, 3, 5)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|sign |SELECT sign(40); |PASSED |OK |std_err: Error during planning: Invalid function 'sign'.\nDid you mean 'sin'?\n | +|signum |SELECT signum(40); |PASSED |OK |OK | +|sin |SELECT sin(0); |PASSED |OK |OK | +|sinh |SELECT sinh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|size |SELECT size(array('b', 'd', 'c', 'a')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | +|skewness |SELECT skewness(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'skewness'.\nDid you mean 'degrees'?\n | +|slice |SELECT slice(array(1, 2, 3, 4), 2, 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|smallint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|some |SELECT some(col) FROM VALUES (true), (false), (false) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'some'.\nDid you mean 'SUM'?\n | +|sort_array |SELECT sort_array(array('b', 'd', null, 'c', 'a'), true); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 'b'")\n | +|soundex |SELECT soundex('Miller'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'soundex'.\nDid you mean 'round'?\n | +|space |SELECT concat(space(2), '1'); |PASSED |OK |std_err: Error during planning: Invalid function 'space'.\nDid you mean 'rpad'?\n | +|spark_partition_id |SELECT spark_partition_id(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'spark_partition_id'.\nDid you mean 'array_position'?\n | +|split |SELECT split('oneAtwoBthreeC', '[ABC]'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'split'.\nDid you mean 'sqrt'?\n | +|split_part |SELECT split_part('11.12.13', '.', 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|sqrt |SELECT sqrt(4); |PASSED |OK |OK | +|stack |SELECT stack(2, 1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'stack'.\nDid you mean 'atan'?\n | +|startswith |SELECT startswith('Spark SQL', 'Spark'); |PASSED |OK |std_err: Error during planning: Invalid function 'startswith'.\nDid you mean 'starts_with'?\n | +|std |SELECT std(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'std'.\nDid you mean 'sin'?\n | +|stddev |SELECT stddev(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |OK | +|stddev_pop |SELECT stddev_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |OK | +|stddev_samp |SELECT stddev_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |OK | +|str_to_map |SELECT str_to_map('a:1,b:2,c:3', ',', ':'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'str_to_map'.\nDid you mean 'date_format'?\n | +|string | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|struct |SELECT struct(1, 2, 3); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|substr |SELECT substr('Spark SQL', 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|substring |SELECT substring('Spark SQL', 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|substring_index |SELECT substring_index('www.apache.org', '.', 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|sum |SELECT sum(col) FROM VALUES (5), (10), (15) AS tab(col); |PASSED |OK |OK | +|tan |SELECT tan(0); |PASSED |OK |OK | +|tanh |SELECT tanh(0); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|timestamp | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|timestamp_micros |SELECT timestamp_micros(1230219000123123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'timestamp_micros'.\nDid you mean 'to_timestamp_micros'?\n | +|timestamp_millis |SELECT timestamp_millis(1230219000123); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'timestamp_millis'.\nDid you mean 'to_timestamp_millis'?\n | +|timestamp_seconds |SELECT timestamp_seconds(1230219000); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'timestamp_seconds'.\nDid you mean 'to_timestamp_seconds'?\n | +|tinyint | |SKIPPED|No examples found in spark.sessionState.functionRegistry | | +|to_binary |SELECT to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_binary'.\nDid you mean 'to_char'?\n | +|to_char |SELECT to_char(454, '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Error during planning: [data_types_with_scalar_udf] Coercion from [Int64, Utf8] to the signature OneOf([Exact([Date32, Utf8]), Exact([Date64, Utf8]), Exact([Time32(Millisecond), Utf8]), Exact([Time32(Second), Utf8]), Exact([Time64(Microsecond), Utf8]), Exact([Time64(Nanosecond), Utf8]), Exact([Timestamp(Second, None), Utf8]), Exact([Timestamp(Second, Some("+TZ")), Utf8]), Exact([Timestamp(Millisecond, None), Utf8]), Exact([Timestamp(Millisecond, Some("+TZ")), Utf8]), Exact([Timestamp(Microsecond, None), Utf8]), Exact([Timestamp(Microsecond, Some("+TZ")), Utf8]), Exact([Timestamp(Nanosecond, None), Utf8]), Exact([Timestamp(Nanosecond, Some("+TZ")), Utf8]), Exact([Duration(Second), Utf8]), Exact([Duration(Millisecond), Utf8]), Exact([Duration(Microsecond), Utf8]), Exact([Duration(Nanosecond), Utf8])]) failed. and No function matches the given name and argument types 'to_char(Int64, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\tto_char(Date32, Utf8)\n\tto_char(Date64, Utf8)\n\tto_char(Time32(Millisecond), Utf8)\n\tto_char(Time32(Second), Utf8)\n\tto_char(Time64(Microsecond), Utf8)\n\tto_char(Time64(Nanosecond), Utf8)\n\tto_char(Timestamp(Second, None), Utf8)\n\tto_char(Timestamp(Second, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Millisecond, None), Utf8)\n\tto_char(Timestamp(Millisecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Microsecond, None), Utf8)\n\tto_char(Timestamp(Microsecond, Some("+TZ")), Utf8)\n\tto_char(Timestamp(Nanosecond, None), Utf8)\n\tto_char(Timestamp(Nanosecond, Some("+TZ")), Utf8)\n\tto_char(Duration(Second), Utf8)\n\tto_char(Duration(Millisecond), Utf8)\n\tto_char(Duration(Microsecond), Utf8)\n\tto_char(Duration(Nanosecond), Utf8)\n| +|to_csv |SELECT to_csv(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_csv'.\nDid you mean 'to_char'?\n | +|to_date |SELECT to_date('2009-07-30 04:17:52'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|to_json |SELECT to_json(named_struct('a', 1, 'b', 2)); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_json'.\nDid you mean 'to_char'?\n | +|to_number |SELECT to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_number'.\nDid you mean 'to_date'?\n | +|to_timestamp |SELECT to_timestamp('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|to_timestamp_ltz |SELECT to_timestamp_ltz('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_timestamp_ltz'.\nDid you mean 'to_timestamp'?\n | +|to_timestamp_ntz |SELECT to_timestamp_ntz('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_timestamp_ntz'.\nDid you mean 'to_timestamp'?\n | +|to_unix_timestamp |SELECT to_unix_timestamp('2016-04-08', 'yyyy-MM-dd'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_unix_timestamp'.\nDid you mean 'to_timestamp'?\n | +|to_utc_timestamp |SELECT to_utc_timestamp('2016-08-31', 'Asia/Seoul'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'to_utc_timestamp'.\nDid you mean 'to_timestamp'?\n | +|transform |SELECT transform(array(1, 2, 3), x -> x + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|transform_keys |SELECT transform_keys(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|transform_values |SELECT transform_values(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|translate |SELECT translate('AaBbCc', 'abc', '123'); |PASSED |OK |OK | +|trim |SELECT trim(' SparkSQL '); |PASSED |OK |OK | +|trunc |SELECT trunc('2019-08-04', 'week'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Error during planning: [data_types_with_scalar_udf] Coercion from [Utf8, Utf8] to the signature OneOf([Exact([Float32, Int64]), Exact([Float64, Int64]), Exact([Float64]), Exact([Float32])]) failed. and No function matches the given name and argument types 'trunc(Utf8, Utf8)'. You might need to add explicit type casts.\n\tCandidate functions:\n\ttrunc(Float32, Int64)\n\ttrunc(Float64, Int64)\n\ttrunc(Float64)\n\ttrunc(Float32)\n | +|try_add |SELECT try_add(1, 2); |FAILED |Unsupported: Results do not match for query |std_err: Error during planning: Invalid function 'try_add'.\nDid you mean 'rpad'?\n | +|try_avg |SELECT try_avg(col) FROM VALUES (1), (2), (3) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'try_avg'.\nDid you mean 'AVG'?\n | +|try_divide |SELECT try_divide(3, 2); |PASSED |OK |std_err: Error during planning: Invalid function 'try_divide'.\nDid you mean 'to_date'?\n | +|try_element_at |SELECT try_element_at(array(1, 2, 3), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|try_multiply |SELECT try_multiply(2, 3); |FAILED |Unsupported: Results do not match for query |std_err: Error during planning: Invalid function 'try_multiply'.\nDid you mean 'to_date'?\n | +|try_subtract |SELECT try_subtract(2, 1); |PASSED |OK |std_err: Error during planning: Invalid function 'try_subtract'.\nDid you mean 'array_extract'?\n | +|try_sum |SELECT try_sum(col) FROM VALUES (5), (10), (15) AS tab(col); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'try_sum'.\nDid you mean 'trim'?\n | +|try_to_binary |SELECT try_to_binary('abc', 'utf-8'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'try_to_binary'.\nDid you mean 'date_bin'?\n | +|try_to_number |SELECT try_to_number('454', '999'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'try_to_number'.\nDid you mean 'to_char'?\n | +|try_to_timestamp |SELECT try_to_timestamp('2016-12-31 00:12:00'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'try_to_timestamp'.\nDid you mean 'to_timestamp'?\n | +|typeof |SELECT typeof(1); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'typeof'.\nDid you mean 'repeat'?\n | +|ucase |SELECT ucase('SparkSql'); |PASSED |OK |std_err: Error during planning: Invalid function 'ucase'.\nDid you mean 'cos'?\n | +|unbase64 |SELECT unbase64('U3BhcmsgU1FM'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unbase64'.\nDid you mean 'sha384'?\n | +|unhex |SELECT decode(unhex('537061726B2053514C'), 'UTF-8'); |PASSED |OK |std_err: Error during planning: Invalid function 'unhex'.\nDid you mean 'upper'?\n | +|unix_date |SELECT unix_date(DATE("1970-01-02")); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_date'.\nDid you mean 'to_date'?\n | +|unix_micros |SELECT unix_micros(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_micros'.\nDid you mean 'acos'?\n | +|unix_millis |SELECT unix_millis(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_millis'.\nDid you mean 'nullif'?\n | +|unix_seconds |SELECT unix_seconds(TIMESTAMP('1970-01-01 00:00:01Z')); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_seconds'.\nDid you mean 'decode'?\n | +|unix_timestamp |SELECT unix_timestamp(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'unix_timestamp'.\nDid you mean 'to_timestamp'?\n | +|upper |SELECT upper('SparkSql'); |PASSED |OK |OK | +|url_decode |SELECT url_decode('https%3A%2F%2Fspark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'url_decode'.\nDid you mean 'decode'?\n | +|url_encode |SELECT url_encode('https://spark.apache.org'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'url_encode'.\nDid you mean 'encode'?\n | +|user |SELECT user(); |PASSED |OK |std_err: SQL error: ParserError("Expected end of statement, found: (")\n | +|uuid |SELECT uuid(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback | | +|var_pop |SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |OK | +|var_samp |SELECT var_samp(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |OK | +|variance |SELECT variance(col) FROM VALUES (1), (2), (3) AS tab(col); |PASSED |OK |std_out: DataFusion CLI v38.0.0\nError: Error during planning: Invalid function 'variance'.\nDid you mean 'range'?\n | +|version |SELECT version(); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'version'.\nDid you mean 'asin'?\n | +|weekday |SELECT weekday('2009-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'weekday'.\nDid you mean 'overlay'?\n | +|weekofyear |SELECT weekofyear('2008-02-20'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'weekofyear'.\nDid you mean 'power'?\n | +|when |SELECT CASE WHEN 1 > 0 THEN 1 WHEN 2 > 0 THEN 2.0 ELSE 1.2 END; |FAILED |\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'then'.(line 1, pos 18)\n\n== SQL ==\nselect case a > b then c else d end from tbl\n------------------^^^\n | | +|width_bucket |SELECT width_bucket(5.3, 0.2, 10.6, 5); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'width_bucket'.\nDid you mean 'list_cat'?\n | +|xpath |SELECT xpath('b1b2b3c1c2','a/b/text()'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath'.\nDid you mean 'atanh'?\n | +|xpath_boolean |SELECT xpath_boolean('1','a/b'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_boolean'.\nDid you mean 'date_bin'?\n | +|xpath_double |SELECT xpath_double('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_double'.\nDid you mean 'date_format'?\n | +|xpath_float |SELECT xpath_float('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_float'.\nDid you mean 'date_format'?\n | +|xpath_int |SELECT xpath_int('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_int'.\nDid you mean 'date_bin'?\n | +|xpath_long |SELECT xpath_long('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_long'.\nDid you mean 'date_bin'?\n | +|xpath_number |SELECT xpath_number('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_number'.\nDid you mean 'date_bin'?\n | +|xpath_short |SELECT xpath_short('12', 'sum(a/b)'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_short'.\nDid you mean 'list_sort'?\n | +|xpath_string |SELECT xpath_string('bcc','a/c'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'xpath_string'.\nDid you mean 'date_trunc'?\n | +|xxhash64 |SELECT xxhash64('Spark', array(123), 2); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 123")\n | +|year |SELECT year('2016-07-30'); |FAILED |Unsupported: Expected only Comet native operators but found Spark fallback |std_err: Error during planning: Invalid function 'year'.\nDid you mean 'VAR'?\n | +|zip_with |SELECT zip_with(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); |FAILED |[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `zip_with` requires 3 parameters but the actual number is 2. Please, refer to 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix.; line 1 pos 7 |std_err: SQL error: ParserError("Expected SELECT, VALUES, or a subquery in the query body, found: 1")\n | +|| |SELECT 3 | 5; |PASSED |OK |OK | +|~ |SELECT ~ 0; |PASSED |OK |std_err: SQL error: ParserError("Expected an expression:, found: ~")\n | ++---------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------+-------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/spark_expressions_support.md b/docs/spark_expressions_support.md index ecfe9b8a0..8fb975862 100644 --- a/docs/spark_expressions_support.md +++ b/docs/spark_expressions_support.md @@ -329,9 +329,9 @@ - [x] sqrt - [x] tan - [ ] tanh - - [x] try_add + - [ ] try_add - [x] try_divide - - [x] try_multiply + - [ ] try_multiply - [x] try_subtract - [x] unhex - [ ] width_bucket diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 98a2bad02..46bf5b91f 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1445,6 +1445,46 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("try_multiply") { + Seq(false, true).foreach { dictionary => + withSQLConf( + "parquet.enable.dictionary" -> dictionary.toString, + "spark.comet.exec.shuffle.enabled" -> "true", + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENFORCE_MODE_ENABLED.key -> "true", + CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true", + ) { + val table = "test" + withTable(table) { + sql(s"create table $table(a int, b int) using parquet") + sql(s"insert into $table VALUES (-2147483647, 1)") + checkSparkAnswerAndOperator(s"SELECT try_subtract(a, b) FROM $table") + } + } + } + } + + test("regr_avgx") { + Seq(false, true).foreach { dictionary => + withSQLConf( + "parquet.enable.dictionary" -> dictionary.toString, + "spark.comet.exec.shuffle.enabled" -> "true", + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_ENFORCE_MODE_ENABLED.key -> "true", + CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true", + ) { + val table = "test" + withTable(table) { + sql(s"create table $table(a int, b int) using parquet") + sql(s"insert into $table VALUES (1, 2), (2, 2), (2, 3), (2, 4)") + checkSparkAnswerAndOperator(s"SELECT regr_avgx(a, b) FROM $table") + } + } + } + } + test("hash functions") { Seq(true, false).foreach { dictionary => withSQLConf( From 848e106f569d931004e51aeb94cc63bf8f39789d Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 30 May 2024 16:58:20 -0700 Subject: [PATCH 18/20] fix some tests --- .../comet/CometExpressionCoverageSuite.scala | 15 ++++++- .../apache/comet/CometExpressionSuite.scala | 40 ------------------- 2 files changed, 14 insertions(+), 41 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 3b8e3d82e..01550849f 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -84,10 +84,14 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH ">=" -> ("select 1 a, 2 b", "select a >= b from tbl"), "^" -> ("select 1 a, 2 b", "select a ^ b from tbl"), "|" -> ("select 1 a, 2 b", "select a | b from tbl"), + "try_multiply" -> ("select 2000000 a, 30000000 b", "select try_multiply(a, b) from tbl"), + "try_add" -> ("select 2147483647 a, 1 b", "select try_add(a, b) from tbl"), + "try_subtract" -> ("select cast(-2147483647 as int) a, cast(1 as int) b", "select try_subtract(a, b) from tbl"), "stack" -> ("select 1 a, 2 b", "select stack(1, a, b) from tbl"), "~" -> ("select 1 a", "select ~ a from tbl"), "unhex" -> ("select '537061726B2053514C' a", "select unhex(a) from tbl"), "when" -> ("select 1 a, 2 b, 3 c, 4 d", "select case a > b then c else d end from tbl"), + "case" -> ("select 1 a, 2 b, 3 c, 4 d", "select case a when 1 then c else d end from tbl"), "transform_values" -> ("select array(1, 2, 3) a", "select transform_values(map_from_arrays(a, a), (k, v) -> v + 1) from tbl"), "transform_keys" -> ("select array(1, 2, 3) a", "select transform_keys(map_from_arrays(a, a), (k, v) -> v + 1) from tbl"), "transform" -> ("select array(1, 2, 3) a", "select transform(a, (k, v) -> v + 1) from tbl"), @@ -128,7 +132,6 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH .listFunction() .map(spark.sessionState.catalog.lookupFunctionInfo(_)) .filter(_.getSource.toLowerCase == "built-in") - // .filter(_.getName == "curdate") .filter(f => !outofRoadmapFuncs.contains(f.getName.toLowerCase)) .map(f => { val selectRows = queryPattern.findAllMatchIn(f.getExamples).map(_.group(0)).toList @@ -250,6 +253,16 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH datafusionMessage = dfMessage.getOrElse("")), group = func.group) + case e: TestFailedException + if e.getMessage.contains("Results do not match for query") => + CoverageResult( + q, + CoverageResultStatus.Failed, + CoverageResultDetails( + cometMessage = "Unsupported: Results do not match for query", + datafusionMessage = dfMessage.getOrElse("")), + group = func.group) + case e if e.getMessage.contains("CometNativeException") => CoverageResult( q, diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index 46bf5b91f..98a2bad02 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -1445,46 +1445,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("try_multiply") { - Seq(false, true).foreach { dictionary => - withSQLConf( - "parquet.enable.dictionary" -> dictionary.toString, - "spark.comet.exec.shuffle.enabled" -> "true", - CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_SHUFFLE_ENFORCE_MODE_ENABLED.key -> "true", - CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true", - ) { - val table = "test" - withTable(table) { - sql(s"create table $table(a int, b int) using parquet") - sql(s"insert into $table VALUES (-2147483647, 1)") - checkSparkAnswerAndOperator(s"SELECT try_subtract(a, b) FROM $table") - } - } - } - } - - test("regr_avgx") { - Seq(false, true).foreach { dictionary => - withSQLConf( - "parquet.enable.dictionary" -> dictionary.toString, - "spark.comet.exec.shuffle.enabled" -> "true", - CometConf.COMET_ENABLED.key -> "true", - CometConf.COMET_EXEC_ENABLED.key -> "true", - CometConf.COMET_SHUFFLE_ENFORCE_MODE_ENABLED.key -> "true", - CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true", - ) { - val table = "test" - withTable(table) { - sql(s"create table $table(a int, b int) using parquet") - sql(s"insert into $table VALUES (1, 2), (2, 2), (2, 3), (2, 4)") - checkSparkAnswerAndOperator(s"SELECT regr_avgx(a, b) FROM $table") - } - } - } - } - test("hash functions") { Seq(true, false).foreach { dictionary => withSQLConf( From ec885d6ee1833ac5ac97207ecc308e2483ea1aed Mon Sep 17 00:00:00 2001 From: Oleks V Date: Tue, 4 Jun 2024 14:49:03 -0700 Subject: [PATCH 19/20] Update docs/source/user-guide/overview.md Co-authored-by: Andy Grove --- docs/source/user-guide/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/overview.md b/docs/source/user-guide/overview.md index e5a7be80c..b5425d774 100644 --- a/docs/source/user-guide/overview.md +++ b/docs/source/user-guide/overview.md @@ -29,7 +29,7 @@ Comet aims to support: - a native Parquet implementation, including both reader and writer - full implementation of Spark operators, including Filter/Project/Aggregation/Join/Exchange etc. -- [full implementation](../../../docs/spark_expressions_support.md) of Spark built-in expressions. +- full implementation of Spark built-in expressions. - a UDF framework for users to migrate their existing UDF to native ## Architecture From b5817f470de55a38e397be32171a15e37813d561 Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 4 Jun 2024 17:26:35 -0700 Subject: [PATCH 20/20] rm some out of roadmap funcs --- .../comet/CometExpressionCoverageSuite.scala | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala index 01550849f..55d45a21b 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionCoverageSuite.scala @@ -59,14 +59,26 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH // exclude funcs Comet has no plans to support streaming in near future // like spark streaming functions, java calls - private val outofRoadmapFuncs = - List("window", "session_window", "window_time", "java_method", "reflect") + private val outOfRoadmapFuncs = + List( + "window", + "session_window", + "window_time", + "java_method", + "reflect", + "current_catalog", + "current_user", + "current_schema", + "current_database") + // Spark Comet configuration to run the tests private val sqlConf = Seq( "spark.comet.exec.shuffle.enabled" -> "true", "spark.sql.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding", "spark.sql.adaptive.optimizer.excludedRules" -> "org.apache.spark.sql.catalyst.optimizer.ConstantFolding") // Tests to run manually as its syntax is different from usual or nested + // This can be simplified once comet supports MemoryScan, now Comet triggers from the FileScan + // If MemoryScan supported we can just run Spark examples as is val manualTests: Map[String, (String, String)] = Map( "!" -> ("select true a", "select ! true from tbl"), "%" -> ("select 1 a, 2 b", "select a % b from tbl"), @@ -132,7 +144,7 @@ class CometExpressionCoverageSuite extends CometTestBase with AdaptiveSparkPlanH .listFunction() .map(spark.sessionState.catalog.lookupFunctionInfo(_)) .filter(_.getSource.toLowerCase == "built-in") - .filter(f => !outofRoadmapFuncs.contains(f.getName.toLowerCase)) + .filter(f => !outOfRoadmapFuncs.contains(f.getName.toLowerCase)) .map(f => { val selectRows = queryPattern.findAllMatchIn(f.getExamples).map(_.group(0)).toList (FunctionInfo(f.getName, f.getGroup), selectRows.filter(_.nonEmpty))