feat(spark): map EqualNullSafe fn to is_not_distinct_from (#306)

substrait-io · Oct 16, 2024 · 357cc01 · 357cc01
1 parent d5452ad
commit 357cc01
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/spark/src/main/scala/io/substrait/spark/expression/FunctionMappings.scala b/spark/src/main/scala/io/substrait/spark/expression/FunctionMappings.scala
@@ -47,6 +47,7 @@ class FunctionMappings {
     s[GreaterThan]("gt"),
     s[GreaterThanOrEqual]("gte"),
     s[EqualTo]("equal"),
+    s[EqualNullSafe]("is_not_distinct_from"),
     // s[BitwiseXor]("xor"),
     s[IsNull]("is_null"),
     s[IsNotNull]("is_not_null"),

diff --git a/spark/src/test/scala/io/substrait/spark/TPCDSPlan.scala b/spark/src/test/scala/io/substrait/spark/TPCDSPlan.scala
@@ -33,14 +33,14 @@ class TPCDSPlan extends TPCDSBase with SubstraitPlanTestBase {
 
   // "q9" failed in spark 3.3
   val successfulSQL: Set[String] = Set("q1", "q3", "q4", "q7",
-    "q11", "q13", "q15", "q16", "q18", "q19",
+    "q11", "q13", "q14b", "q15", "q16", "q18", "q19",
     "q21", "q22", "q23a", "q23b", "q25", "q26", "q28", "q29",
-    "q30", "q31", "q32", "q33", "q37",
+    "q30", "q31", "q32", "q33", "q37", "q38",
     "q41", "q42", "q43", "q46", "q48",
     "q50", "q52", "q54", "q55", "q56", "q58", "q59",
     "q60", "q61", "q62", "q65", "q66", "q68", "q69",
     "q71", "q73", "q76", "q79",
-    "q81", "q82", "q85", "q88",
+    "q81", "q82", "q85", "q87", "q88",
     "q90", "q91", "q92", "q93", "q94", "q95", "q96", "q97", "q99")
 
   tpcdsQueries.foreach {