From ce1cfe973ee916f6e8de6373b7b4a48f99bcd776 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 24 Aug 2026 10:22:00 -0700 Subject: [PATCH 01/19] feat: support Map for `CreateArray` literal --- .../org/apache/comet/serde/literals.scala | 96 ++++++++++++++++++- .../expressions/array/create_array.sql | 87 +++++++++++++++++ 2 files changed, 179 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index bac3631babe..6cb7a05122d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -22,9 +22,10 @@ package org.apache.comet.serde.literals import java.lang import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.expressions.{Attribute, Literal} -import org.apache.spark.sql.catalyst.util.ArrayData -import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DateType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, NullType, ShortType, StringType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateArray, CreateMap, CreateNamedStruct, Expression, KnownNullable, Literal} +import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} +import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, NullType, ShortType, StringType, StructType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String import com.google.protobuf.ByteString @@ -32,7 +33,7 @@ import com.google.protobuf.ByteString import org.apache.comet.CometSparkSessionExtensions.withFallbackReason import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.{CometExpressionSerde, Compatible, ExprOuterClass, LiteralOuterClass, SupportLevel, Unsupported} -import org.apache.comet.serde.QueryPlanSerde.{isTimeType, serializeDataType, supportedDataType} +import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, isTimeType, serializeDataType, supportedDataType} import org.apache.comet.serde.Types.ListLiteral object CometLiteral extends CometExpressionSerde[Literal] with Logging { @@ -55,6 +56,9 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { .elementType .isInstanceOf[ArrayType])))) { Compatible(None) + } else if (canExpandComplexLiteral(expr)) { + // Rebuilt as a Create[Array|Map|NamedStruct] tree in `convert`. + Compatible(None) } else { expr.dataType match { case _: DayTimeIntervalType => Compatible(None) @@ -70,6 +74,12 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { val dataType = expr.dataType val value = expr.value + if (canExpandComplexLiteral(expr)) { + return exprToProtoInternal(expandComplexLiteral(value, dataType), inputs, binding).orElse { + withFallbackReason(expr, s"Unsupported data type $dataType") + None + } + } val exprBuilder = LiteralOuterClass.Literal.newBuilder() if (value == null) { @@ -215,4 +225,82 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { } listLiteralBuilder } + + /** + * True when a non-null Literal of this type is not encodable in the native `Literal` proto and + * `convert` should try to rebuild it from primitive-typed Literals. The native proto today + * carries scalars and nested `ListLiteral`s (arrays of arrays / arrays of scalars). It does not + * carry Map or Struct values, so any Literal whose type contains a `MapType` or a `StructType` + * needs to be expanded before serialization. + */ + private def needsExpansion(dataType: DataType): Boolean = dataType match { + case _: MapType | _: StructType => true + case ArrayType(et, _) => needsExpansion(et) + case _ => false + } + + /** + * True when the Literal is a non-null complex value that we can rebuild from primitive Literals + * via [[expandComplexLiteral]]. Empty top-level containers are excluded because a synthesized + * `Create[Array|Map]` with no children cannot recover the original element type. + */ + private def canExpandComplexLiteral(expr: Literal): Boolean = { + val value = expr.value + if (value == null || !needsExpansion(expr.dataType)) return false + expr.dataType match { + case _: ArrayType => value.asInstanceOf[ArrayData].numElements() > 0 + case _: MapType => value.asInstanceOf[MapData].numElements() > 0 + case _: StructType => true + case _ => false + } + } + + /** + * Rebuild a folded complex Literal as an equivalent tree of `CreateArray` / `CreateMap` / + * `CreateNamedStruct` over primitive-typed Literals. Callers must gate on + * [[canExpandComplexLiteral]] so `dataType` is one of the three complex cases and top-level + * containers are non-empty. + */ + private def expandComplexLiteral(value: Any, dataType: DataType): Expression = + dataType match { + case ArrayType(et, _) => + val arr = value.asInstanceOf[ArrayData] + val elems = (0 until arr.numElements()).map { i => + if (arr.isNullAt(i)) Literal(null, et) + else asNullable(Literal(arr.get(i, et), et)) + } + CreateArray(elems, useStringTypeWhenEmpty = false) + case MapType(kt, vt, _) => + val mapData = value.asInstanceOf[MapData] + val keys = mapData.keyArray() + val vals = mapData.valueArray() + val children = (0 until keys.numElements()).flatMap { i => + val k = if (keys.isNullAt(i)) Literal(null, kt) else Literal(keys.get(i, kt), kt) + val v = + if (vals.isNullAt(i)) Literal(null, vt) + else asNullable(Literal(vals.get(i, vt), vt)) + Seq(k, v) + } + CreateMap(children, useStringTypeWhenEmpty = false) + case StructType(fields) => + val row = value.asInstanceOf[InternalRow] + val children = fields.zipWithIndex.flatMap { case (f, i) => + val v = + if (row.isNullAt(i)) Literal(null, f.dataType) + else asNullable(Literal(row.get(i, f.dataType), f.dataType)) + Seq(Literal(f.name), v) + } + CreateNamedStruct(children.toSeq) + case other => + throw new IllegalStateException(s"expandComplexLiteral called on $other") + } + + /** + * Wrap a non-null expression in `KnownNullable` so a surrounding `Create*` sees every sibling + * as nullable. DataFusion `make_array` asserts strict Arrow-type equality across siblings and + * would panic if a folded literal produced a non-nullable child next to a nullable one. + * `CometKnownNullable` forwards the child on the wire, so runtime is unaffected. + */ + private def asNullable(expr: Expression): Expression = + if (expr.nullable) expr else KnownNullable(expr) } diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql index 5d59df4577e..34fcda4a4ab 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -15,6 +15,8 @@ -- specific language governing permissions and limitations -- under the License. +-- ConfigMatrix: parquet.enable.dictionary=false,true + statement CREATE TABLE test_create_array(a int, b int, c int) USING parquet @@ -26,3 +28,88 @@ SELECT array(a, b, c) FROM test_create_array query SELECT array(1, 2, 3, NULL) + +-- Boundary integer values including int min/max. +query +SELECT array(2147483647, -2147483648, 0, NULL) + +-- Table for column-based coverage of map/struct/nested-array children. +statement +CREATE TABLE test_create_array_complex(k int, v int, s string, arr array) USING parquet + +statement +INSERT INTO test_create_array_complex VALUES + (1, 10, 'a', array(1, 2, 3)), + (2, 20, NULL, array(4, NULL, 6)), + (NULL, NULL, 'c', array()), + (NULL, NULL, NULL, NULL) + +-- Array of nested arrays. +query +SELECT array(array(1, 2), array(3, NULL)) + +query +SELECT array(array(a, b), array(b, c)) FROM test_create_array + +-- Array of maps built from primitive int values. Under Spark constant folding this collapses +-- each `map(...)` to a MapType Literal; CometLiteral expands it back to a CreateMap of int +-- literals so the outer `array(...)` runs natively via `make_array`. +query +SELECT array(map(1, 10), map(2, 20)) + +-- Array of maps whose values are arrays. This is the exact fallback shape reported in +-- https://github.com/apache/datafusion-comet — `MapType(IntegerType, ArrayType(IntegerType, false), true)`. +query +SELECT array(map(1, array(1, 2, 3)), map(2, array(4, 5, 6))) + +-- Array of maps with a NULL value inside (map value expansion must handle NULLs). +query +SELECT array(map('x', CAST(NULL AS INT), 'y', 2), map('z', 3)) + +-- Array containing a folded NULL map literal alongside a non-null map. +query +SELECT array(CAST(NULL AS MAP), map(1, 2)) + +-- Deeply nested folded complex literal: array of struct of map of array of struct. +query +SELECT array(named_struct('m', map(1, array(named_struct('id', 1, 's', 'x'))))) + +-- Array of maps with string keys. +query +SELECT array(map('x', 1, 'y', 2), map('z', 3)) + +-- Single-element array of map. +query +SELECT array(map(1, 2)) + +-- Array of arrays of maps (recursive expansion of ArrayType(ArrayType(MapType)) literal). +query +SELECT array(array(map(1, 'a')), array(map(2, 'b'))) + +-- Array of structs. Under constant folding each named_struct(...) folds to a StructType +-- Literal; expansion rebuilds it as CreateNamedStruct of primitive literals. +query +SELECT array(named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 'b', 'y')) + +-- Column-based array of maps. With constant folding disabled by the SQL-file harness this +-- flows through CometCreateMap's codegen dispatch, not through Literal expansion, but it +-- must remain natively supported. Filter out NULL keys because Spark forbids them. +query +SELECT array(map(k, v)) FROM test_create_array_complex WHERE k IS NOT NULL + +-- Column-based array of structs. +query +SELECT array(named_struct('k', k, 's', s)) FROM test_create_array_complex + +-- Column-based array of nested arrays. +query +SELECT array(arr, array(k, v)) FROM test_create_array_complex + +-- Array of maps whose value is a nested array column. +query +SELECT array(map(k, arr)) FROM test_create_array_complex WHERE k IS NOT NULL + +-- Empty complex ArrayType literal takes the pre-expansion makeListLiteral path (empty +-- ListLiteral) so its element type survives even without any children. +query +SELECT CAST(array() AS ARRAY>) From f121109ef94abf027c1f29c8ddc9750dd6fb9e55 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 24 Aug 2026 11:48:20 -0700 Subject: [PATCH 02/19] feat: support Map for `CreateArray` literal --- .../org/apache/comet/serde/literals.scala | 74 ++++++++++++------- .../expressions/array/create_array.sql | 20 +---- .../comet/CometArrayExpressionSuite.scala | 57 ++++++++++++++ 3 files changed, 110 insertions(+), 41 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index 6cb7a05122d..83490b6689f 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -22,10 +22,9 @@ package org.apache.comet.serde.literals import java.lang import org.apache.spark.internal.Logging -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateArray, CreateMap, CreateNamedStruct, Expression, KnownNullable, Literal} +import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateArray, CreateMap, Expression, KnownNullable, Literal} import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} -import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, NullType, ShortType, StringType, StructType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, NullType, ShortType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String import com.google.protobuf.ByteString @@ -230,11 +229,21 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * True when a non-null Literal of this type is not encodable in the native `Literal` proto and * `convert` should try to rebuild it from primitive-typed Literals. The native proto today * carries scalars and nested `ListLiteral`s (arrays of arrays / arrays of scalars). It does not - * carry Map or Struct values, so any Literal whose type contains a `MapType` or a `StructType` - * needs to be expanded before serialization. + * carry Map values, so any Literal whose type contains a `MapType` needs to be expanded before + * serialization. + * + * `StructType` (at any nesting depth) is deliberately excluded. Native `CometCreateNamedStruct` + * (`spark/src/main/scala/org/apache/comet/serde/structs.scala`) uses `values_to_arrays` in + * `native/spark-expr/src/struct_funcs/create_named_struct.rs`, which returns a 1-row + * `StructArray` whenever all children are scalar values. That collides with the row count of + * the surrounding batch and fails `make_array`'s length check. Field nullability is likewise + * inferred from the concrete child expressions, and `CometKnownNullable` + * (`spark/src/main/scala/org/apache/comet/serde/contraintExpressions.scala:99`) drops the tag + * on the wire, so wrapping a non-null child in `KnownNullable` does not carry across. Fall back + * to Spark for those shapes. */ private def needsExpansion(dataType: DataType): Boolean = dataType match { - case _: MapType | _: StructType => true + case _: MapType => true case ArrayType(et, _) => needsExpansion(et) case _ => false } @@ -242,24 +251,48 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { /** * True when the Literal is a non-null complex value that we can rebuild from primitive Literals * via [[expandComplexLiteral]]. Empty top-level containers are excluded because a synthesized - * `Create[Array|Map]` with no children cannot recover the original element type. + * `Create[Array|Map]` with no children cannot recover the original element type. A folded + * `MapData` with duplicate keys is also excluded: Spark's `CreateMap.eval` + * (`sql/catalyst/.../complexTypeCreator.scala:250`) feeds every entry through + * `ArrayBasedMapBuilder`, which throws under the default `MAP_KEY_DEDUP_POLICY=EXCEPTION` + * (`sql/catalyst/.../util/ArrayBasedMapBuilder.scala`). Rebuilding a folded literal that came + * from `from_json` or a similar source would then throw where the original literal had executed + * cleanly. */ private def canExpandComplexLiteral(expr: Literal): Boolean = { - val value = expr.value - if (value == null || !needsExpansion(expr.dataType)) return false + if (expr.value == null) return false expr.dataType match { - case _: ArrayType => value.asInstanceOf[ArrayData].numElements() > 0 - case _: MapType => value.asInstanceOf[MapData].numElements() > 0 - case _: StructType => true + case at: ArrayType if needsExpansion(at) => + expr.value.asInstanceOf[ArrayData].numElements() > 0 + case MapType(kt, _, _) => + val mapData = expr.value.asInstanceOf[MapData] + mapData.numElements() > 0 && !hasDuplicateMapKeys(mapData.keyArray(), kt) case _ => false } } /** - * Rebuild a folded complex Literal as an equivalent tree of `CreateArray` / `CreateMap` / - * `CreateNamedStruct` over primitive-typed Literals. Callers must gate on - * [[canExpandComplexLiteral]] so `dataType` is one of the three complex cases and top-level - * containers are non-empty. + * True when the folded map's key array contains any duplicate values. Uses Catalyst internal + * equality (`UTF8String`, `Decimal`, `ArrayData`, `InternalRow`, and boxed primitives all + * implement `equals` / `hashCode`), matching `ArrayBasedMapBuilder`'s own dedup semantics. + */ + private def hasDuplicateMapKeys(keys: ArrayData, keyType: DataType): Boolean = { + val n = keys.numElements() + if (n < 2) return false + val seen = new java.util.HashSet[Any](n) + var i = 0 + while (i < n) { + val k = if (keys.isNullAt(i)) null else keys.get(i, keyType) + if (!seen.add(k)) return true + i += 1 + } + false + } + + /** + * Rebuild a folded complex Literal as an equivalent tree of `CreateArray` / `CreateMap` over + * primitive-typed Literals. Callers must gate on [[canExpandComplexLiteral]] so `dataType` is + * one of the two complex cases and top-level containers are non-empty and have unique keys. */ private def expandComplexLiteral(value: Any, dataType: DataType): Expression = dataType match { @@ -282,15 +315,6 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { Seq(k, v) } CreateMap(children, useStringTypeWhenEmpty = false) - case StructType(fields) => - val row = value.asInstanceOf[InternalRow] - val children = fields.zipWithIndex.flatMap { case (f, i) => - val v = - if (row.isNullAt(i)) Literal(null, f.dataType) - else asNullable(Literal(row.get(i, f.dataType), f.dataType)) - Seq(Literal(f.name), v) - } - CreateNamedStruct(children.toSeq) case other => throw new IllegalStateException(s"expandComplexLiteral called on $other") } diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql index 34fcda4a4ab..ffcdc2d2620 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -57,23 +57,10 @@ SELECT array(array(a, b), array(b, c)) FROM test_create_array query SELECT array(map(1, 10), map(2, 20)) --- Array of maps whose values are arrays. This is the exact fallback shape reported in --- https://github.com/apache/datafusion-comet — `MapType(IntegerType, ArrayType(IntegerType, false), true)`. +-- Array of maps whose values are arrays: `MapType(IntegerType, ArrayType(IntegerType, false), true)`. query SELECT array(map(1, array(1, 2, 3)), map(2, array(4, 5, 6))) --- Array of maps with a NULL value inside (map value expansion must handle NULLs). -query -SELECT array(map('x', CAST(NULL AS INT), 'y', 2), map('z', 3)) - --- Array containing a folded NULL map literal alongside a non-null map. -query -SELECT array(CAST(NULL AS MAP), map(1, 2)) - --- Deeply nested folded complex literal: array of struct of map of array of struct. -query -SELECT array(named_struct('m', map(1, array(named_struct('id', 1, 's', 'x'))))) - -- Array of maps with string keys. query SELECT array(map('x', 1, 'y', 2), map('z', 3)) @@ -86,8 +73,9 @@ SELECT array(map(1, 2)) query SELECT array(array(map(1, 'a')), array(map(2, 'b'))) --- Array of structs. Under constant folding each named_struct(...) folds to a StructType --- Literal; expansion rebuilds it as CreateNamedStruct of primitive literals. +-- Array of structs. The SQL-file harness disables ConstantFolding, so `named_struct(...)` +-- stays as `CreateNamedStruct` (not a folded Literal) and this exercises the constructor +-- path, not `CometLiteral` expansion. query SELECT array(named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 'b', 'y')) diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 05a6e8e650d..249076ee9fd 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1173,6 +1173,63 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + // Constant folding is enabled by default here, so each `map(...)` collapses to a MapType + // Literal and the outer `array(...)` reaches `CometCreateArray` with folded-Literal children. + // `CometLiteral` rebuilds each folded map as an equivalent `CreateMap` of primitive literals. + test("array of folded map literals with array values (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + val df = + spark.table("tbl").selectExpr("array(map(1, array(1, 2, 3)), map(2, array(4, 5, 6)))") + checkSparkAnswerAndOperator(df) + } + } + + test("array of folded map literals (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator(spark.table("tbl").selectExpr("array(map(1, 10), map(2, 20))")) + } + } + + // P2 regression: standalone folded struct literal reaches native `CreateNamedStruct` with all + // scalar children. `values_to_arrays` returns a 1-row `StructArray` regardless of batch size, + // which fails length checks. `CometLiteral` excludes StructType from expansion so the projection + // falls back to Spark rather than producing a truncated result. + test("folded struct literal in multirow projection falls back") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + // Spark's `named_struct(...)` folds to a single struct Literal. Comet must not run + // `CreateNamedStruct` with all-scalar children against a 3-row batch. + val df = spark.table("tbl").selectExpr("_1 AS id", "named_struct('a', 1) AS s") + checkSparkAnswerAndFallbackReason(df, "Unsupported data type StructType") + } + } + + // P2 regression: `array(named_struct(...), named_struct(...))` with divergent field + // nullabilities folds to a single ArrayType(StructType) Literal. `KnownNullable` is dropped on + // the wire, so recursive expansion would produce sibling struct arrays with divergent per-field + // nullabilities and `make_array` would panic. Excluding StructType keeps this on Spark. + test("folded array of structs with divergent field nullability falls back (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + val df = spark + .table("tbl") + .selectExpr("array(named_struct('a', CAST(NULL AS INT)), named_struct('a', 1)) AS arr") + checkSparkAnswerAndFallbackReason(df, "Unsupported data type ArrayType") + } + } + + // P2 regression: `from_json` produces a MapData whose keys may be duplicated. Rebuilding via + // `CreateMap` invokes `ArrayBasedMapBuilder`, which throws under the default + // `MAP_KEY_DEDUP_POLICY=EXCEPTION` policy. `CometLiteral` inspects the folded map's key array + // and declines expansion when duplicates are present, letting Spark evaluate the projection. + test("folded map literal with duplicate keys falls back (multirow)") { + assume(isSpark35Plus) + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + val df = spark + .table("tbl") + .selectExpr("_1 AS id", "from_json('{\"a\":1,\"a\":2}', 'MAP') AS m") + checkSparkAnswerAndFallbackReason(df, "Unsupported data type MapType") + } + } + // Local table scan carries non-null array child fields (an in-memory Seq encodes // containsNull=false) into native kernels that promise nullable elements. ConvertToLocalRelation // must be disabled or the optimizer folds the expression at plan time and nothing runs natively. From 31f28b2e0ea5341716f2824be3b7a26bc8c82acb Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 24 Aug 2026 13:41:14 -0700 Subject: [PATCH 03/19] feat: support Map for `CreateArray` literal --- .../org/apache/comet/serde/literals.scala | 166 +++++++++--------- .../expressions/array/create_array.sql | 20 +-- .../comet/CometArrayExpressionSuite.scala | 70 +++++--- 3 files changed, 132 insertions(+), 124 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index 83490b6689f..c3e60b1dd6d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -32,7 +32,7 @@ import com.google.protobuf.ByteString import org.apache.comet.CometSparkSessionExtensions.withFallbackReason import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.{CometExpressionSerde, Compatible, ExprOuterClass, LiteralOuterClass, SupportLevel, Unsupported} -import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, isTimeType, serializeDataType, supportedDataType} +import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, hasNonDefaultStringCollation, isTimeType, serializeDataType, supportedDataType} import org.apache.comet.serde.Types.ListLiteral object CometLiteral extends CometExpressionSerde[Literal] with Logging { @@ -55,8 +55,8 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { .elementType .isInstanceOf[ArrayType])))) { Compatible(None) - } else if (canExpandComplexLiteral(expr)) { - // Rebuilt as a Create[Array|Map|NamedStruct] tree in `convert`. + } else if (expandComplexLiteral(expr).isDefined) { + // Rebuilt as a `CreateArray` / `CreateMap` tree in `convert`. Compatible(None) } else { expr.dataType match { @@ -73,8 +73,9 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { val dataType = expr.dataType val value = expr.value - if (canExpandComplexLiteral(expr)) { - return exprToProtoInternal(expandComplexLiteral(value, dataType), inputs, binding).orElse { + val expanded = expandComplexLiteral(expr) + if (expanded.isDefined) { + return exprToProtoInternal(expanded.get, inputs, binding).orElse { withFallbackReason(expr, s"Unsupported data type $dataType") None } @@ -226,21 +227,58 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { } /** - * True when a non-null Literal of this type is not encodable in the native `Literal` proto and - * `convert` should try to rebuild it from primitive-typed Literals. The native proto today - * carries scalars and nested `ListLiteral`s (arrays of arrays / arrays of scalars). It does not - * carry Map values, so any Literal whose type contains a `MapType` needs to be expanded before - * serialization. + * Rebuild a folded complex Literal as an equivalent tree of `CreateArray` / `CreateMap` over + * primitive-typed Literals, or `None` when the shape cannot be rebuilt. The native `Literal` + * proto carries scalars and nested `ListLiteral`s but no map values, so a Literal whose type + * contains a `MapType` has to be expanded before serialization. Teaching the proto to transport + * maps directly would remove the need for this rewrite and for the declines below: + * https://github.com/apache/datafusion-comet/issues/1937 * - * `StructType` (at any nesting depth) is deliberately excluded. Native `CometCreateNamedStruct` - * (`spark/src/main/scala/org/apache/comet/serde/structs.scala`) uses `values_to_arrays` in - * `native/spark-expr/src/struct_funcs/create_named_struct.rs`, which returns a 1-row - * `StructArray` whenever all children are scalar values. That collides with the row count of - * the surrounding batch and fails `make_array`'s length check. Field nullability is likewise - * inferred from the concrete child expressions, and `CometKnownNullable` - * (`spark/src/main/scala/org/apache/comet/serde/contraintExpressions.scala:99`) drops the tag - * on the wire, so wrapping a non-null child in `KnownNullable` does not carry across. Fall back - * to Spark for those shapes. + * Declined shapes: + * - Null values and empty top-level containers: a synthesized `Create*` with no children + * cannot recover the original element type. Empty `ArrayType` literals still serialize via + * `makeListLiteral`, which keeps the type. + * - `StructType` at any depth. Native `CreateNamedStruct` builds a 1-row `StructArray` + * whenever all of its children are scalars (`values_to_arrays`), which collides with the + * surrounding batch's row count. Its proto message also carries no type, so Spark's + * declared field nullability cannot survive the wire either way. + * - Map key types whose Spark equality semantics native lookup cannot honor, see + * [[hasUnsafeMapKeyType]]. + * - Folded maps with duplicate keys. The rebuilt `CreateMap` evaluates through + * `ArrayBasedMapBuilder`, which throws under `MAP_KEY_DEDUP_POLICY=EXCEPTION`, where the + * original literal had already folded cleanly. + */ + private def expandComplexLiteral(expr: Literal): Option[Expression] = { + if (expr.value == null) return None + expr.dataType match { + case ArrayType(et, _) if needsExpansion(et) => + val arr = expr.value.asInstanceOf[ArrayData] + if (arr.numElements() == 0) { + None + } else { + val elements = (0 until arr.numElements()).map(i => asNullable(literalAt(arr, i, et))) + Some(CreateArray(elements, useStringTypeWhenEmpty = false)) + } + case MapType(kt, vt, _) => + val mapData = expr.value.asInstanceOf[MapData] + val keys = mapData.keyArray() + if (mapData.numElements() == 0 || hasUnsafeMapKeyType(kt) || + hasDuplicateMapKeys(keys, kt)) { + None + } else { + val values = mapData.valueArray() + val children = (0 until keys.numElements()).flatMap(i => + Seq(literalAt(keys, i, kt), asNullable(literalAt(values, i, vt)))) + Some(CreateMap(children, useStringTypeWhenEmpty = false)) + } + case _ => None + } + } + + /** + * True when a Literal of this type has to be expanded rather than serialized, because the + * native `Literal` proto carries no map values. Walks array nesting only: a `StructType` is not + * expandable at all (see [[expandComplexLiteral]]), so the walk stops there. */ private def needsExpansion(dataType: DataType): Boolean = dataType match { case _: MapType => true @@ -248,81 +286,41 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { case _ => false } + /** Element `i` of `arr`, or `null` for a null slot. */ + private def valueAt(arr: ArrayData, i: Int, dt: DataType): Any = + if (arr.isNullAt(i)) null else arr.get(i, dt) + + /** Element `i` of `arr` as a Literal of type `dt`. */ + private def literalAt(arr: ArrayData, i: Int, dt: DataType): Literal = + Literal(valueAt(arr, i, dt), dt) + /** - * True when the Literal is a non-null complex value that we can rebuild from primitive Literals - * via [[expandComplexLiteral]]. Empty top-level containers are excluded because a synthesized - * `Create[Array|Map]` with no children cannot recover the original element type. A folded - * `MapData` with duplicate keys is also excluded: Spark's `CreateMap.eval` - * (`sql/catalyst/.../complexTypeCreator.scala:250`) feeds every entry through - * `ArrayBasedMapBuilder`, which throws under the default `MAP_KEY_DEDUP_POLICY=EXCEPTION` - * (`sql/catalyst/.../util/ArrayBasedMapBuilder.scala`). Rebuilding a folded literal that came - * from `from_json` or a similar source would then throw where the original literal had executed - * cleanly. + * True when the map key type has Spark equality semantics that native map lookup (Arrow + * bytewise comparison) cannot honor, in which case declining expansion keeps the projection on + * Spark. `NormalizeFloatingNumbers` makes Spark treat `+0.0` and `-0.0` as the same key and + * canonicalises NaN, and a non-default collation compares under rules native applies as + * `UTF8_BINARY`. Both are checked at every nesting level of the key type. */ - private def canExpandComplexLiteral(expr: Literal): Boolean = { - if (expr.value == null) return false - expr.dataType match { - case at: ArrayType if needsExpansion(at) => - expr.value.asInstanceOf[ArrayData].numElements() > 0 - case MapType(kt, _, _) => - val mapData = expr.value.asInstanceOf[MapData] - mapData.numElements() > 0 && !hasDuplicateMapKeys(mapData.keyArray(), kt) - case _ => false - } - } + private def hasUnsafeMapKeyType(kt: DataType): Boolean = + hasNonDefaultStringCollation(kt) || + SupportLevel.containsType(kt, classOf[FloatType], classOf[DoubleType]) /** - * True when the folded map's key array contains any duplicate values. Uses Catalyst internal - * equality (`UTF8String`, `Decimal`, `ArrayData`, `InternalRow`, and boxed primitives all - * implement `equals` / `hashCode`), matching `ArrayBasedMapBuilder`'s own dedup semantics. + * True when the folded map's key array holds duplicates under Catalyst internal equality + * (`UTF8String`, `Decimal`, `ArrayData`, `InternalRow` and boxed primitives all implement + * `equals` / `hashCode`), which is what `ArrayBasedMapBuilder` uses for the key types reachable + * here. */ private def hasDuplicateMapKeys(keys: ArrayData, keyType: DataType): Boolean = { val n = keys.numElements() - if (n < 2) return false - val seen = new java.util.HashSet[Any](n) - var i = 0 - while (i < n) { - val k = if (keys.isNullAt(i)) null else keys.get(i, keyType) - if (!seen.add(k)) return true - i += 1 - } - false + n > 1 && (0 until n).map(i => valueAt(keys, i, keyType)).distinct.size < n } /** - * Rebuild a folded complex Literal as an equivalent tree of `CreateArray` / `CreateMap` over - * primitive-typed Literals. Callers must gate on [[canExpandComplexLiteral]] so `dataType` is - * one of the two complex cases and top-level containers are non-empty and have unique keys. - */ - private def expandComplexLiteral(value: Any, dataType: DataType): Expression = - dataType match { - case ArrayType(et, _) => - val arr = value.asInstanceOf[ArrayData] - val elems = (0 until arr.numElements()).map { i => - if (arr.isNullAt(i)) Literal(null, et) - else asNullable(Literal(arr.get(i, et), et)) - } - CreateArray(elems, useStringTypeWhenEmpty = false) - case MapType(kt, vt, _) => - val mapData = value.asInstanceOf[MapData] - val keys = mapData.keyArray() - val vals = mapData.valueArray() - val children = (0 until keys.numElements()).flatMap { i => - val k = if (keys.isNullAt(i)) Literal(null, kt) else Literal(keys.get(i, kt), kt) - val v = - if (vals.isNullAt(i)) Literal(null, vt) - else asNullable(Literal(vals.get(i, vt), vt)) - Seq(k, v) - } - CreateMap(children, useStringTypeWhenEmpty = false) - case other => - throw new IllegalStateException(s"expandComplexLiteral called on $other") - } - - /** - * Wrap a non-null expression in `KnownNullable` so a surrounding `Create*` sees every sibling - * as nullable. DataFusion `make_array` asserts strict Arrow-type equality across siblings and - * would panic if a folded literal produced a non-nullable child next to a nullable one. + * Wrap a non-null expression in `KnownNullable` so the surrounding `Create*` derives a nullable + * element / value type. DataFusion `make_array` asserts strict Arrow-type equality across + * siblings and would panic on a non-nullable child next to a nullable one. Same root cause as + * the mismatched-type decline in `CometCreateArray` (apache/datafusion#22366). * `CometKnownNullable` forwards the child on the wire, so runtime is unaffected. */ private def asNullable(expr: Expression): Expression = diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql index ffcdc2d2620..42fa2309dfb 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -51,9 +51,10 @@ SELECT array(array(1, 2), array(3, NULL)) query SELECT array(array(a, b), array(b, c)) FROM test_create_array --- Array of maps built from primitive int values. Under Spark constant folding this collapses --- each `map(...)` to a MapType Literal; CometLiteral expands it back to a CreateMap of int --- literals so the outer `array(...)` runs natively via `make_array`. +-- Array of maps built from primitive int values. The SQL-file harness always excludes +-- ConstantFolding, so every `map(...)` below stays a `CreateMap` and reaches native +-- `make_array` through the constructor path. Folded-literal expansion in `CometLiteral` is +-- covered by `CometArrayExpressionSuite` instead, where folding is left enabled. query SELECT array(map(1, 10), map(2, 20)) @@ -69,19 +70,15 @@ SELECT array(map('x', 1, 'y', 2), map('z', 3)) query SELECT array(map(1, 2)) --- Array of arrays of maps (recursive expansion of ArrayType(ArrayType(MapType)) literal). +-- Array of arrays of maps: `ArrayType(ArrayType(MapType(IntegerType, StringType)))`. query SELECT array(array(map(1, 'a')), array(map(2, 'b'))) --- Array of structs. The SQL-file harness disables ConstantFolding, so `named_struct(...)` --- stays as `CreateNamedStruct` (not a folded Literal) and this exercises the constructor --- path, not `CometLiteral` expansion. +-- Array of structs. query SELECT array(named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 'b', 'y')) --- Column-based array of maps. With constant folding disabled by the SQL-file harness this --- flows through CometCreateMap's codegen dispatch, not through Literal expansion, but it --- must remain natively supported. Filter out NULL keys because Spark forbids them. +-- Column-based array of maps. Filter out NULL keys because Spark forbids them. query SELECT array(map(k, v)) FROM test_create_array_complex WHERE k IS NOT NULL @@ -97,7 +94,6 @@ SELECT array(arr, array(k, v)) FROM test_create_array_complex query SELECT array(map(k, arr)) FROM test_create_array_complex WHERE k IS NOT NULL --- Empty complex ArrayType literal takes the pre-expansion makeListLiteral path (empty --- ListLiteral) so its element type survives even without any children. +-- Empty array cast to a nested array type: the element type has to survive with no children. query SELECT CAST(array() AS ARRAY>) diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 249076ee9fd..9eb2003a50a 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1178,55 +1178,69 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp // `CometLiteral` rebuilds each folded map as an equivalent `CreateMap` of primitive literals. test("array of folded map literals with array values (multirow)") { withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { - val df = - spark.table("tbl").selectExpr("array(map(1, array(1, 2, 3)), map(2, array(4, 5, 6)))") - checkSparkAnswerAndOperator(df) + checkSparkAnswerAndOperator( + "SELECT array(map(1, array(1, 2, 3)), map(2, array(4, 5, 6))) FROM tbl") } } test("array of folded map literals (multirow)") { withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndOperator(spark.table("tbl").selectExpr("array(map(1, 10), map(2, 20))")) + checkSparkAnswerAndOperator("SELECT array(map(1, 10), map(2, 20)) FROM tbl") } } - // P2 regression: standalone folded struct literal reaches native `CreateNamedStruct` with all - // scalar children. `values_to_arrays` returns a 1-row `StructArray` regardless of batch size, - // which fails length checks. `CometLiteral` excludes StructType from expansion so the projection - // falls back to Spark rather than producing a truncated result. + // A folded struct literal would reach native `CreateNamedStruct` with all-scalar children, + // which builds a 1-row `StructArray` regardless of batch size. `CometLiteral` excludes + // StructType from expansion so the projection falls back instead of truncating the result. test("folded struct literal in multirow projection falls back") { withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { - // Spark's `named_struct(...)` folds to a single struct Literal. Comet must not run - // `CreateNamedStruct` with all-scalar children against a 3-row batch. - val df = spark.table("tbl").selectExpr("_1 AS id", "named_struct('a', 1) AS s") - checkSparkAnswerAndFallbackReason(df, "Unsupported data type StructType") + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, named_struct('a', 1) AS s FROM tbl", + "Unsupported data type StructType") } } - // P2 regression: `array(named_struct(...), named_struct(...))` with divergent field - // nullabilities folds to a single ArrayType(StructType) Literal. `KnownNullable` is dropped on - // the wire, so recursive expansion would produce sibling struct arrays with divergent per-field - // nullabilities and `make_array` would panic. Excluding StructType keeps this on Spark. + // Folds to one ArrayType(StructType) Literal whose structs disagree on field nullability. + // `KnownNullable` is dropped on the wire, so expansion could not make the sibling struct + // arrays agree and `make_array` would panic. Excluding StructType keeps this on Spark. test("folded array of structs with divergent field nullability falls back (multirow)") { withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { - val df = spark - .table("tbl") - .selectExpr("array(named_struct('a', CAST(NULL AS INT)), named_struct('a', 1)) AS arr") - checkSparkAnswerAndFallbackReason(df, "Unsupported data type ArrayType") + checkSparkAnswerAndFallbackReason( + "SELECT array(named_struct('a', CAST(NULL AS INT)), named_struct('a', 1)) AS arr FROM tbl", + "Unsupported data type ArrayType") } } - // P2 regression: `from_json` produces a MapData whose keys may be duplicated. Rebuilding via - // `CreateMap` invokes `ArrayBasedMapBuilder`, which throws under the default - // `MAP_KEY_DEDUP_POLICY=EXCEPTION` policy. `CometLiteral` inspects the folded map's key array - // and declines expansion when duplicates are present, letting Spark evaluate the projection. + // `from_json` can fold to a MapData with duplicate keys. Rebuilding it as a `CreateMap` would + // run `ArrayBasedMapBuilder`, which throws under `MAP_KEY_DEDUP_POLICY=EXCEPTION`, so + // `CometLiteral` declines expansion and Spark evaluates the projection. test("folded map literal with duplicate keys falls back (multirow)") { assume(isSpark35Plus) withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { - val df = spark - .table("tbl") - .selectExpr("_1 AS id", "from_json('{\"a\":1,\"a\":2}', 'MAP') AS m") - checkSparkAnswerAndFallbackReason(df, "Unsupported data type MapType") + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, from_json('{\"a\":1,\"a\":2}', 'MAP') AS m FROM tbl", + "Unsupported data type MapType") + } + } + + // Spark treats `+0.0` and `-0.0` as the same map key; native lookup compares bytes. + test("folded map literal with double keys falls back to Spark for +/-0.0 equality") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, element_at(map(CAST(0 AS DOUBLE), 7), " + + "CAST(concat('-', CAST(_1 - 1 AS STRING), '.0') AS DOUBLE)) AS v FROM tbl", + "Unsupported data type MapType") + } + } + + // Spark compares the key under its declared collation; native compares under UTF8_BINARY. + test("folded map literal with collated string keys falls back") { + assume(isSpark40Plus) + withParquetTable(Seq(("a1", 0)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT element_at(map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7), " + + "CAST(_1 AS STRING COLLATE UTF8_LCASE)) AS v FROM tbl", + "Unsupported data type MapType") } } From 97544ebe6e60ef558ebb5dc0f6b34aa7ec1ea242 Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 25 Aug 2026 10:25:06 -0700 Subject: [PATCH 04/19] feat: support Map for `CreateArray` literal --- native/core/src/execution/planner.rs | 41 +++++-- .../scala/org/apache/comet/serde/arrays.scala | 36 +++--- .../org/apache/comet/serde/literals.scala | 109 ++++++++++------- .../scala/org/apache/comet/serde/maps.scala | 63 +++++++++- .../expressions/array/create_array.sql | 13 +++ .../expressions/map/element_at_map.sql | 24 ++++ .../sql-tests/expressions/map/map_entries.sql | 13 +++ .../comet/CometArrayExpressionSuite.scala | 110 ++++++++++++++---- .../comet/CometMapExpressionSuite.scala | 78 +++++++++++++ 9 files changed, 400 insertions(+), 87 deletions(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index d109627e825..25667a96027 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -2897,13 +2897,13 @@ impl PhysicalPlanner { } AggExprStruct::CollectSet(expr) => { let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?; - let child = Self::coerce_collect_child_nullability(child, &schema)?; + let child = Self::coerce_child_fields_nullable(child, &schema)?; let func = AggregateUDF::new_from_impl(SparkCollectSet::new()); Self::create_aggr_func_expr("collect_set", schema, vec![child], func) } AggExprStruct::CollectList(expr) => { let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?; - let child = Self::coerce_collect_child_nullability(child, &schema)?; + let child = Self::coerce_child_fields_nullable(child, &schema)?; let func = AggregateUDF::new_from_impl(SparkCollectList::new()); Self::create_aggr_func_expr("collect_list", schema, vec![child], func) } @@ -3384,6 +3384,16 @@ impl PhysicalPlanner { .collect::, _>>()?; let fun_name = &expr.func; + // `map_entries` reuses the input map's entries array but declares the entry `value` field + // nullable, so widen the argument first or Arrow rejects the child type. See + // `coerce_child_fields_nullable`. + let args = if fun_name == "map_entries" { + args.into_iter() + .map(|arg| Self::coerce_child_fields_nullable(arg, &input_schema)) + .collect::, ExecutionError>>()? + } else { + args + }; let input_expr_types = args .iter() .map(|x| x.data_type(input_schema.as_ref())) @@ -3516,15 +3526,24 @@ impl PhysicalPlanner { Ok(scalar_expr) } - /// `collect_list` / `collect_set` build their result list with all element fields marked - /// nullable, regardless of the input's nullability. However `SparkCollectList::return_type` - /// (and `SparkCollectSet`) derive the element type directly from the child, preserving any - /// non-nullable nested field. When the child is a nested type with a non-nullable inner field - /// (e.g. a struct field), the declared aggregate output disagrees with the array the - /// accumulator actually produces, and DataFusion's grouped `AggregateExec` fails validating - /// its output batch ("column types must match schema types"). Cast the child to the - /// all-nullable variant of its type so the declared and produced types stay consistent. - fn coerce_collect_child_nullability( + /// Casts `child` to the all-nullable variant of its own type, or returns it unchanged when it + /// already is. Used where a kernel's declared output type marks every nested field nullable + /// but its implementation reuses the child's arrays, so a non-nullable inner field makes the + /// declared and produced types disagree: + /// + /// - `collect_list` / `collect_set` build their result list with all element fields marked + /// nullable, while `SparkCollectList::return_type` (and `SparkCollectSet`) derive the + /// element type directly from the child. With a non-nullable nested field (e.g. a struct + /// field), DataFusion's grouped `AggregateExec` fails validating its output batch + /// ("column types must match schema types"). + /// - `map_entries` declares its list element as `Struct(key non-null, value nullable)` but + /// reuses the input map's own entries array as the list values, so `ListArray::new` + /// rejects a map whose entry `value` field is non-nullable. Every Spark `map(...)` + /// constructor over non-null values produces exactly that type. + /// + /// `make_all_fields_nullable` keeps an Arrow map's key field non-nullable, so widening a map + /// stays a legal map type. + fn coerce_child_fields_nullable( child: Arc, schema: &SchemaRef, ) -> Result, ExecutionError> { diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 748b1cee231..a730f5c47a2 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -474,11 +474,12 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { // `MutableArrayData::with_capacities` and panics on a mismatch. Spark's CreateArray is more // permissive: its type coercion compares element types with `sameType`, which ignores // nullability, so children that share a surface type but differ only in nested field - // nullability get no unifying cast. DataFusion tolerates container nullability differences - // (an `ArrayType.containsNull` / `MapType.valueContainsNull` mismatch is coerced), but NOT a - // struct field's nullability -- `array(struct(a not null), struct(a nullable))` panics inside - // `make_array_inner`. Decline only those cases (i.e. children that still differ after - // normalizing container nullability) so Spark's evaluator handles them. + // nullability get no unifying cast. DataFusion coerces an `ArrayType.containsNull` mismatch + // away before that assert, but NOT a struct field's or a `MapType.valueContainsNull` + // mismatch, so `array(struct(a not null), struct(a nullable))` and + // `array(map('x', CAST(NULL AS INT)), map('z', 3))` both panic inside `make_array_inner`. + // Decline only those cases (i.e. children that still differ after normalizing the container + // nullability DataFusion does coerce) so Spark's evaluator handles them. // // TODO: remove this decline once apache/datafusion#22366 lands; the upstream fix widens the // element type via nullability-OR-merge and casts each child before MutableArrayData. @@ -502,20 +503,25 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { } /** - * Rewrites a type so that container nullability (`ArrayType.containsNull`, - * `MapType.valueContainsNull`) is forced to `true` everywhere, while struct field nullability - * is left intact. Two CreateArray children whose types differ ONLY in container nullability are - * tolerated by DataFusion's `make_array` (coerced), so they normalize equal here; a difference - * in a struct field's nullability survives normalization and triggers the decline above. + * Rewrites a type so that `ArrayType.containsNull` is forced to `true` everywhere, while struct + * field nullability and `MapType.valueContainsNull` are left intact. Two CreateArray children + * whose types differ ONLY in `containsNull` are tolerated by DataFusion's `make_array`, because + * `type_union_resolution_coercion` routes `List` arguments through `list_coercion`, which + * merges the element field's nullability and lets Comet's planner insert the unifying cast. Its + * fallback chain has no `map_coercion` arm in DataFusion 54.1, so two `Map` arguments that + * differ in value nullability fail to unify, no cast is inserted, and `MutableArrayData` + * panics. Keep `valueContainsNull` significant here so those children are declined instead. The + * same is true of a struct field's nullability, which `coerce_struct_by_*` would merge but + * which `MutableArrayData` still rejects for the array's own element type. */ private def normalizeContainerNullability(dt: DataType): DataType = dt match { case ArrayType(elementType, _) => ArrayType(normalizeContainerNullability(elementType), containsNull = true) - case MapType(keyType, valueType, _) => + case MapType(keyType, valueType, valueContainsNull) => MapType( normalizeContainerNullability(keyType), normalizeContainerNullability(valueType), - valueContainsNull = true) + valueContainsNull) case StructType(fields) => StructType(fields.map(f => f.copy(dataType = normalizeContainerNullability(f.dataType)))) case other => other @@ -593,7 +599,11 @@ object CometElementAt extends CometExpressionSerde[ElementAt] { override def getSupportLevel(expr: ElementAt): SupportLevel = { expr.left.dataType match { case _: ArrayType => Compatible() - case _: MapType => Compatible() + case MapType(keyType, _, _) => + MapKeySupport.unsupportedKeyTypeReason(keyType) match { + case Some(reason) => Unsupported(Some(reason)) + case None => Compatible() + } case _ => Unsupported(Some("Input must be an array or map")) } } diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index c3e60b1dd6d..e11b79d9fe5 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -23,7 +23,7 @@ import java.lang import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateArray, CreateMap, Expression, KnownNullable, Literal} -import org.apache.spark.sql.catalyst.util.{ArrayData, MapData} +import org.apache.spark.sql.catalyst.util.{ArrayData, MapData, TypeUtils} import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, NullType, ShortType, StringType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String @@ -32,7 +32,7 @@ import com.google.protobuf.ByteString import org.apache.comet.CometSparkSessionExtensions.withFallbackReason import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.{CometExpressionSerde, Compatible, ExprOuterClass, LiteralOuterClass, SupportLevel, Unsupported} -import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, hasNonDefaultStringCollation, isTimeType, serializeDataType, supportedDataType} +import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, isTimeType, serializeDataType, supportedDataType} import org.apache.comet.serde.Types.ListLiteral object CometLiteral extends CometExpressionSerde[Literal] with Logging { @@ -234,41 +234,49 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * maps directly would remove the need for this rewrite and for the declines below: * https://github.com/apache/datafusion-comet/issues/1937 * + * The rebuilt tree is the tree Spark itself had before `ConstantFolding` collapsed it, down to + * every container's declared nullability (see [[withNullability]]). That equivalence is the + * safety property this rewrite rests on: whatever the rebuilt expression does natively is what + * the same query already does with `ConstantFolding` disabled, so expansion cannot introduce a + * folding-only behaviour difference. Shapes the native map kernels cannot handle are therefore + * declined by their own serdes rather than here, see [[MapKeySupport]] and [[CometMapEntries]]. + * * Declined shapes: * - Null values and empty top-level containers: a synthesized `Create*` with no children * cannot recover the original element type. Empty `ArrayType` literals still serialize via * `makeListLiteral`, which keeps the type. - * - `StructType` at any depth. Native `CreateNamedStruct` builds a 1-row `StructArray` - * whenever all of its children are scalars (`values_to_arrays`), which collides with the - * surrounding batch's row count. Its proto message also carries no type, so Spark's - * declared field nullability cannot survive the wire either way. - * - Map key types whose Spark equality semantics native lookup cannot honor, see - * [[hasUnsafeMapKeyType]]. - * - Folded maps with duplicate keys. The rebuilt `CreateMap` evaluates through - * `ArrayBasedMapBuilder`, which throws under `MAP_KEY_DEDUP_POLICY=EXCEPTION`, where the - * original literal had already folded cleanly. + * - Arrays whose elements are structs, because [[needsExpansion]] does not walk into a + * `StructType`. Native `CreateNamedStruct` builds a 1-row `StructArray` whenever all of its + * children are scalars (`values_to_arrays`), which collides with the surrounding batch's + * row count, and its proto message carries no type, so Spark's declared field nullability + * cannot survive the wire either way. A struct that is only a map value is safe: + * `CometCreateMap` hands the whole rebuilt `CreateMap` to the JVM codegen dispatcher, so + * Spark's own code builds the struct. + * - Folded maps with duplicate keys, see [[hasDuplicateMapKeys]]. */ private def expandComplexLiteral(expr: Literal): Option[Expression] = { if (expr.value == null) return None expr.dataType match { - case ArrayType(et, _) if needsExpansion(et) => + case ArrayType(et, containsNull) if needsExpansion(et) => val arr = expr.value.asInstanceOf[ArrayData] if (arr.numElements() == 0) { None } else { - val elements = (0 until arr.numElements()).map(i => asNullable(literalAt(arr, i, et))) + val elements = (0 until arr.numElements()) + .map(i => withNullability(literalAt(arr, i, et), containsNull)) Some(CreateArray(elements, useStringTypeWhenEmpty = false)) } - case MapType(kt, vt, _) => + case MapType(kt, vt, valueContainsNull) => val mapData = expr.value.asInstanceOf[MapData] val keys = mapData.keyArray() - if (mapData.numElements() == 0 || hasUnsafeMapKeyType(kt) || - hasDuplicateMapKeys(keys, kt)) { + if (mapData.numElements() == 0 || hasDuplicateMapKeys(keys, kt)) { None } else { val values = mapData.valueArray() val children = (0 until keys.numElements()).flatMap(i => - Seq(literalAt(keys, i, kt), asNullable(literalAt(values, i, vt)))) + Seq( + literalAt(keys, i, kt), + withNullability(literalAt(values, i, vt), valueContainsNull))) Some(CreateMap(children, useStringTypeWhenEmpty = false)) } case _ => None @@ -277,8 +285,8 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { /** * True when a Literal of this type has to be expanded rather than serialized, because the - * native `Literal` proto carries no map values. Walks array nesting only: a `StructType` is not - * expandable at all (see [[expandComplexLiteral]]), so the walk stops there. + * native `Literal` proto carries no map values. Walks array nesting only: an array of structs + * is not expandable (see [[expandComplexLiteral]]), so the walk stops at a `StructType`. */ private def needsExpansion(dataType: DataType): Boolean = dataType match { case _: MapType => true @@ -295,34 +303,51 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { Literal(valueAt(arr, i, dt), dt) /** - * True when the map key type has Spark equality semantics that native map lookup (Arrow - * bytewise comparison) cannot honor, in which case declining expansion keeps the projection on - * Spark. `NormalizeFloatingNumbers` makes Spark treat `+0.0` and `-0.0` as the same key and - * canonicalises NaN, and a non-default collation compares under rules native applies as - * `UTF8_BINARY`. Both are checked at every nesting level of the key type. - */ - private def hasUnsafeMapKeyType(kt: DataType): Boolean = - hasNonDefaultStringCollation(kt) || - SupportLevel.containsType(kt, classOf[FloatType], classOf[DoubleType]) - - /** - * True when the folded map's key array holds duplicates under Catalyst internal equality - * (`UTF8String`, `Decimal`, `ArrayData`, `InternalRow` and boxed primitives all implement - * `equals` / `hashCode`), which is what `ArrayBasedMapBuilder` uses for the key types reachable - * here. + * True when the folded map's key array holds duplicates under Spark's own key equality, in + * which case rebuilding the value as a `CreateMap` would change semantics the folded `MapData` + * had already settled: `ArrayBasedMapBuilder` throws `DUPLICATED_MAP_KEY` under + * `MAP_KEY_DEDUP_POLICY=EXCEPTION` and silently drops the earlier entry under `LAST_WIN`, where + * the original literal (from `from_json`, or a cast of one) kept both entries. + * + * `ArrayBasedMapBuilder` hashes most atomic keys but compares `BinaryType`, collated + * `StringType` and complex keys through `TypeUtils.getInterpretedOrdering`, and it normalizes + * `-0.0` and `NaN` before either. The interpreted ordering is the one comparison that agrees + * with all of those: `Array[Byte]` keys compare by content rather than by identity, a collated + * `StringType` compares under its collation, and `nanSafeCompareDoubles` already treats `-0.0` + * and `+0.0`, and any two `NaN`s, as equal. A null key cannot occur in a map Spark built, so + * treat one as a duplicate and keep the projection on Spark instead of asking the ordering to + * compare it. */ private def hasDuplicateMapKeys(keys: ArrayData, keyType: DataType): Boolean = { val n = keys.numElements() - n > 1 && (0 until n).map(i => valueAt(keys, i, keyType)).distinct.size < n + if (n < 2) { + false + } else if ((0 until n).exists(keys.isNullAt)) { + true + } else { + val ordering = TypeUtils.getInterpretedOrdering(keyType) + val sorted = (0 until n).map(i => keys.get(i, keyType)).sorted(ordering) + sorted.sliding(2).exists(pair => ordering.compare(pair.head, pair.last) == 0) + } } /** - * Wrap a non-null expression in `KnownNullable` so the surrounding `Create*` derives a nullable - * element / value type. DataFusion `make_array` asserts strict Arrow-type equality across - * siblings and would panic on a non-nullable child next to a nullable one. Same root cause as - * the mismatched-type decline in `CometCreateArray` (apache/datafusion#22366). - * `CometKnownNullable` forwards the child on the wire, so runtime is unaffected. + * Wrap `expr` in `KnownNullable` when the container it came out of declares its elements or + * values nullable, so the rebuilt `Create*` reports exactly the literal's declared + * `ArrayType.containsNull` / `MapType.valueContainsNull`. Both directions matter: + * - when the flag is true, a null slot yields a nullable `Literal` while a populated slot + * does not, and DataFusion `make_array` asserts strict Arrow-type equality across siblings + * (apache/datafusion#22366), so all children have to agree. + * - when the flag is false, widening it would make the rebuilt map disagree with a non-folded + * `CreateMap` sibling whose type Spark's own coercion had already accepted, which + * `CometCreateArray` no longer normalizes away because DataFusion 54.1 cannot coerce a + * `MapType.valueContainsNull` mismatch. + * + * A container that declares itself non-nullable never holds a null slot: `CreateArray` and + * `CreateMap` only report `false` when every child is non-nullable, and Spark's other sources + * of a complex type (a `CAST` target, a `from_json` schema, `MapType.apply`) default the flag + * to `true`. `CometKnownNullable` forwards the child on the wire, so runtime is unaffected. */ - private def asNullable(expr: Expression): Expression = - if (expr.nullable) expr else KnownNullable(expr) + private def withNullability(expr: Expression, nullable: Boolean): Expression = + if (nullable && !expr.nullable) KnownNullable(expr) else expr } diff --git a/spark/src/main/scala/org/apache/comet/serde/maps.scala b/spark/src/main/scala/org/apache/comet/serde/maps.scala index c889284129f..1195f3aaeb4 100644 --- a/spark/src/main/scala/org/apache/comet/serde/maps.scala +++ b/spark/src/main/scala/org/apache/comet/serde/maps.scala @@ -23,9 +23,61 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -import org.apache.comet.serde.QueryPlanSerde.{createBinaryExpr, exprToProtoInternal, scalarFunctionExprToProto} +import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.serde.QueryPlanSerde.{createBinaryExpr, exprToProtoInternal, hasNonDefaultStringCollation, scalarFunctionExprToProto} import org.apache.comet.shims.CometTypeShim +/** + * Shared gate for the native map kernels that compare a lookup key against a map's stored keys + * (`map_extract`, reached from both `GetMapValue` and `ElementAt`). + */ +private[serde] object MapKeySupport { + + val floatingPointReason: String = + "Spark normalizes floating-point map keys, so `-0.0` matches a `+0.0` key and all `NaN`s " + + "match each other; Comet's native map lookup compares the raw Arrow values." + + val collationReason: String = + "Comet's native map lookup compares string keys as `UTF8_BINARY` and cannot honour a " + + "non-default collation." + + val complexKeyReason: String = + "Comet's native `map_extract` casts the lookup key to the map's exact Arrow key type, so a " + + "complex key type whose components are declared non-nullable rejects a lookup key that " + + "holds a `NULL` inside one of them." + + /** + * The reason Comet's native map lookup cannot reproduce Spark's key equality for `keyType`, or + * `None` when it can. Spark finds a key with `TypeUtils.getInterpretedOrdering` over the keys + * `ArrayBasedMapBuilder` stored, having first normalized them, while native `map_extract` + * compares the Arrow values as they are: + * - `ArrayBasedMapBuilder` rewrites a `-0.0` key to `+0.0` and canonicalises `NaN`, and + * `nanSafeCompareDoubles` treats `-0.0` and `+0.0` as equal, so Spark answers a `-0.0` + * lookup from a `+0.0` key where native finds nothing. + * - a non-default collation compares under rules native applies as `UTF8_BINARY`. + * - for a complex key type, `map_extract`'s `coerce_types` returns the map's exact key field + * type, so Comet's planner casts the lookup key to it. When the key type declares a nested + * component non-nullable, a `NULL` inside the lookup key aborts that cast with + * `Non-nullable field of ListArray "item" cannot contain nulls` rather than missing the + * lookup, which is what Spark does. + * + * `BinaryType` keys need no decline: Arrow compares them by content, as Spark's ordering does. + * The floating-point and collation checks walk every nesting level of the key type, so they + * keep holding if native `map_extract` ever gains complex-key support. + */ + def unsupportedKeyTypeReason(keyType: DataType): Option[String] = { + if (SupportLevel.containsType(keyType, classOf[FloatType], classOf[DoubleType])) { + Some(floatingPointReason) + } else if (hasNonDefaultStringCollation(keyType)) { + Some(collationReason) + } else if (isComplexType(keyType)) { + Some(complexKeyReason) + } else { + None + } + } +} + object CometMapKeys extends CometExpressionSerde[MapKeys] { override def convert( @@ -64,6 +116,15 @@ object CometMapValues extends CometExpressionSerde[MapValues] { object CometMapExtract extends CometExpressionSerde[GetMapValue] { + override def getSupportLevel(expr: GetMapValue): SupportLevel = expr.child.dataType match { + case MapType(keyType, _, _) => + MapKeySupport.unsupportedKeyTypeReason(keyType) match { + case Some(reason) => Unsupported(Some(reason)) + case None => Compatible() + } + case _ => Compatible() + } + override def convert( expr: GetMapValue, inputs: Seq[Attribute], diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql index 42fa2309dfb..5dfda61fabd 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -66,6 +66,19 @@ SELECT array(map(1, array(1, 2, 3)), map(2, array(4, 5, 6))) query SELECT array(map('x', 1, 'y', 2), map('z', 3)) +-- Array of maps whose children disagree on `valueContainsNull`. Spark's CreateArray coercion +-- compares element types with `sameType`, which ignores nullability, so it inserts no unifying +-- cast. DataFusion 54.1 cannot unify two Map arguments either: the fallback chain in +-- `type_union_resolution_coercion` has no `map_coercion` arm, so `make_array` receives the two +-- differently typed map arrays and `MutableArrayData` panics. `CometCreateArray` therefore keeps +-- `MapType.valueContainsNull` significant when it normalizes container nullability and declines. +query expect_fallback(CreateArray children have mismatched data types) +SELECT array(map('x', CAST(NULL AS INT), 'y', 2), map('z', 3)) + +-- Both children agree that the value is nullable, so this stays native. +query +SELECT array(map('x', CAST(NULL AS INT)), map('z', CAST(NULL AS INT))) + -- Single-element array of map. query SELECT array(map(1, 2)) diff --git a/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql b/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql index 5a838e260d4..5bb322b7f32 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql @@ -51,3 +51,27 @@ SELECT element_at(mi, CAST(1 AS BIGINT)), element_at(mi, CAST(2 AS SMALLINT)) FR -- literal map arguments query SELECT element_at(map('a', 1, 'b', 2), 'a'), element_at(map('a', 1, 'b', 2), 'missing'), element_at(map('a', 1, 'b', 2), NULL) + +-- Map key types whose Spark equality Comet's native `map_extract` cannot reproduce fall back to +-- Spark. These stay on the constructor path here because the SQL harness excludes +-- `ConstantFolding`; `CometMapExpressionSuite` covers the folded-literal form of each. + +-- Spark stores `-0.0` map keys as `+0.0` and compares with `nanSafeCompareDoubles`, so a `-0.0` +-- lookup finds the `+0.0` key. Native lookup compares the raw Arrow values. +query expect_fallback(Spark normalizes floating-point map keys) +SELECT element_at(map(CAST(0 AS DOUBLE), 7), CAST(-0.0 AS DOUBLE)) + +query expect_fallback(Spark normalizes floating-point map keys) +SELECT element_at(map(CAST(0 AS FLOAT), 7), CAST(-0.0 AS FLOAT)) + +-- A complex key type: `map_extract` casts the lookup key to the map's exact Arrow key type, so a +-- NULL inside the lookup key would abort the cast instead of missing the lookup. +query expect_fallback(casts the lookup key to the map's exact Arrow key type) +SELECT element_at(map(array(1), 7), array(CAST(NULL AS INT))) + +query expect_fallback(casts the lookup key to the map's exact Arrow key type) +SELECT element_at(map(named_struct('a', 1), 7), named_struct('a', 1)) + +-- `BinaryType` keys need no decline: Arrow compares them by content, as Spark's ordering does. +query +SELECT element_at(map(CAST('a' AS BINARY), 1, CAST('b' AS BINARY), 2), CAST('b' AS BINARY)) diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql index 91720cc2ab1..1b1cd26eccf 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql @@ -23,3 +23,16 @@ INSERT INTO test_map_entries VALUES (map('a', 1, 'b', 2)), (map()), (NULL) query spark_answer_only SELECT map_entries(m) FROM test_map_entries + +-- `MapType(_, _, valueContainsNull = false)`, which every `map(...)` over non-null values +-- produces. DataFusion's `map_entries` declares the entry `value` field nullable but reuses the +-- input map's entries array, so the planner widens the argument before the call. +query +SELECT map_entries(map(1, 2, 3, 4)) + +query +SELECT map_entries(map('a', array(1, 2))) + +-- A map nested in a map value keeps `valueContainsNull = false` through the extract. +query +SELECT map_entries(element_at(map(1, map(1, 2)), 1)) diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 9eb2003a50a..cd3e40b8f6d 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1189,6 +1189,65 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + // The folded `map(1, 2)` and the dynamic `map(2, coalesce(...))` both declare + // `MapType(IntegerType, IntegerType, valueContainsNull = false)`, so `CometCreateArray` admits + // the pair. Rebuilding the literal has to report that exact type: widening the rebuilt map's + // value to nullable would leave the sibling behind and `make_array` would panic, because + // DataFusion 54.1 cannot coerce a `MapType.valueContainsNull` mismatch away. + test("folded map literal keeps non-nullable values next to a dynamic map sibling (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, array(map(1, 2), map(2, coalesce(_1, 0))) AS arr FROM tbl") + } + } + + // The whole `array(...)` folds to one `ArrayType(MapType(IntegerType, IntegerType, true))` + // literal, so every rebuilt element must report the unified nullable value type. + test("folded array of maps with a NULL value (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT array(map(1, CAST(NULL AS INT)), map(2, 3)) AS arr FROM tbl") + } + } + + // A NULL element sits next to a populated one inside a single folded + // `ArrayType(MapType(IntegerType, IntegerType, false), containsNull = true)` literal. The null + // slot serializes as a typed null literal carrying the declared map type, so the rebuilt sibling + // has to report that same type for `make_array` to accept the pair. + test("folded array mixing a map literal and a NULL element (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator("SELECT array(map(1, 2), NULL) AS arr FROM tbl") + checkSparkAnswerAndOperator("SELECT array(NULL, map(1, 2)) AS arr FROM tbl") + } + } + + // A map value that is itself a map or an array of maps is built by Spark's own generated code + // inside `CometCreateMap`, so it keeps its declared nullability all the way to the consumer. + test("folded map literals with nested map values (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator("SELECT _1 AS id, map(1, map(1, 2)) AS m FROM tbl") + checkSparkAnswerAndOperator("SELECT _1 AS id, map(1, array(map(2, 3))) AS m FROM tbl") + checkSparkAnswerAndOperator( + "SELECT _1 AS id, map(1, named_struct('a', 1, 'b', 'x')) AS m FROM tbl") + } + } + + // An empty container has no children to recover the element type from, and a NULL literal + // serializes as a typed null without expansion, so only the empty cases fall back. + test("folded empty and NULL complex literals") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, CAST(map() AS MAP) AS m FROM tbl", + "Unsupported data type MapType") + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, CAST(array() AS ARRAY>) AS a FROM tbl", + "Unsupported data type ArrayType") + checkSparkAnswerAndOperator("SELECT _1 AS id, CAST(NULL AS MAP) AS m FROM tbl") + checkSparkAnswerAndOperator( + "SELECT _1 AS id, CAST(NULL AS ARRAY>) AS a FROM tbl") + } + } + // A folded struct literal would reach native `CreateNamedStruct` with all-scalar children, // which builds a 1-row `StructArray` regardless of batch size. `CometLiteral` excludes // StructType from expansion so the projection falls back instead of truncating the result. @@ -1212,35 +1271,46 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } // `from_json` can fold to a MapData with duplicate keys. Rebuilding it as a `CreateMap` would - // run `ArrayBasedMapBuilder`, which throws under `MAP_KEY_DEDUP_POLICY=EXCEPTION`, so - // `CometLiteral` declines expansion and Spark evaluates the projection. + // run `ArrayBasedMapBuilder`, which throws under `MAP_KEY_DEDUP_POLICY=EXCEPTION` and silently + // drops the earlier entry under `LAST_WIN`, so `CometLiteral` declines expansion under either + // policy and Spark evaluates the projection. test("folded map literal with duplicate keys falls back (multirow)") { assume(isSpark35Plus) withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndFallbackReason( - "SELECT _1 AS id, from_json('{\"a\":1,\"a\":2}', 'MAP') AS m FROM tbl", - "Unsupported data type MapType") + Seq("EXCEPTION", "LAST_WIN").foreach { policy => + withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> policy) { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, from_json('{\"a\":1,\"a\":2}', 'MAP') AS m FROM tbl", + "Unsupported data type MapType") + } + } } } - // Spark treats `+0.0` and `-0.0` as the same map key; native lookup compares bytes. - test("folded map literal with double keys falls back to Spark for +/-0.0 equality") { - withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndFallbackReason( - "SELECT _1 AS id, element_at(map(CAST(0 AS DOUBLE), 7), " + - "CAST(concat('-', CAST(_1 - 1 AS STRING), '.0') AS DOUBLE)) AS v FROM tbl", - "Unsupported data type MapType") + // Spark's map cast preserves both entries, so the folded value still holds duplicate keys after + // the key type becomes BinaryType. `Array[Byte]` has no value-based `equals`, so the duplicate + // check has to compare keys the way `ArrayBasedMapBuilder` does, through + // `TypeUtils.getInterpretedOrdering`. + test("folded map literal with duplicate binary keys falls back (multirow)") { + assume(isSpark35Plus) + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + Seq("EXCEPTION", "LAST_WIN").foreach { policy => + withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> policy) { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, CAST(from_json('{\"a\":1,\"a\":2}', 'MAP') " + + "AS MAP) AS m FROM tbl", + "Unsupported data type MapType") + } + } } } - // Spark compares the key under its declared collation; native compares under UTF8_BINARY. - test("folded map literal with collated string keys falls back") { - assume(isSpark40Plus) - withParquetTable(Seq(("a1", 0)), "tbl") { - checkSparkAnswerAndFallbackReason( - "SELECT element_at(map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7), " + - "CAST(_1 AS STRING COLLATE UTF8_LCASE)) AS v FROM tbl", - "Unsupported data type MapType") + // Distinct binary keys are fine: Arrow compares them by content, as Spark's ordering does. + test("folded map literal with distinct binary keys (multirow)") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, element_at(map(CAST('1' AS BINARY), 10, CAST('2' AS BINARY), 20), " + + "CAST(CAST(_1 AS STRING) AS BINARY)) AS v FROM tbl") } } diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 31c36a07c72..ad3769c1a41 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -272,4 +272,82 @@ class CometMapExpressionSuite extends CometTestBase { } } + // A map that reaches `map_entries` through an expression rather than a scan keeps + // `valueContainsNull = false`, which every `map(...)` over non-null values produces. DataFusion's + // `map_entries` declares the entry `value` field nullable but reuses the input map's entries + // array, so the planner widens the argument before the call. Both queries below depend on a + // column, so `ConstantFolding` cannot collapse the whole `map_entries(...)` call into an + // `ArrayType(StructType)` literal that `CometLiteral` would decline. `map_entries.sql` covers the + // all-constant shapes, where the harness excludes `ConstantFolding`. + test("map_entries on a non-nullable-value map expression (multirow)") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator("SELECT _1 AS id, map_entries(map(_1, 2)) AS e FROM tbl") + checkSparkAnswerAndOperator( + "SELECT _1 AS id, map_entries(element_at(map(1, map(1, 2)), _1)) AS e FROM tbl") + } + } + + // Spark stores a `-0.0` map key as `+0.0` and finds it with `nanSafeCompareDoubles`, which + // treats `-0.0` and `+0.0` as equal. Native `map_extract` compares the raw Arrow values, so + // `CometMapExtract` / `CometElementAt` decline the lookup. The map here folds to a literal, so + // the decline has to come from the lookup rather than from `CometLiteral`, which cannot see how + // a map nested in a map value will be consumed. + test("map lookup with floating-point keys falls back") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + val negZero = "CAST(concat('-', CAST(_1 - 1 AS STRING), '.0') AS DOUBLE)" + checkSparkAnswerAndFallbackReason( + s"SELECT _1 AS id, element_at(map(CAST(0 AS DOUBLE), 7), $negZero) AS v FROM tbl", + "Spark normalizes floating-point map keys") + checkSparkAnswerAndFallbackReason( + s"SELECT _1 AS id, map(CAST(0 AS DOUBLE), 7)[$negZero] AS v FROM tbl", + "Spark normalizes floating-point map keys") + // The outer key type is INT, so only inspecting it would admit the lookup; the inner map is + // handed to the JVM dispatcher whole and never revisits `CometLiteral`. + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, element_at(element_at(map(1, map(CAST(0 AS DOUBLE), 7)), _1), " + + s"$negZero) AS v FROM tbl", + "Spark normalizes floating-point map keys") + } + } + + // Spark compares the key under its declared collation; native compares under UTF8_BINARY. + test("map lookup with collated string keys falls back") { + assume(isSpark40Plus) + withParquetTable(Seq(("a1", 0)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT element_at(map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7), " + + "CAST(_1 AS STRING COLLATE UTF8_LCASE)) AS v FROM tbl", + "cannot honour a non-default collation") + checkSparkAnswerAndFallbackReason( + "SELECT element_at(element_at(map(1, map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7)), 1), " + + "CAST(_1 AS STRING COLLATE UTF8_LCASE)) AS v FROM tbl", + "cannot honour a non-default collation") + } + } + + // `map_extract`'s `coerce_types` returns the map's exact Arrow key field type, so Comet casts the + // lookup key to it. A NULL inside the lookup key then aborts the cast with `Non-nullable field of + // ListArray "item" cannot contain nulls` instead of missing the lookup, which is what Spark does. + test("map lookup with complex keys falls back") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, element_at(map(array(1), 7), " + + "array(IF(_1 = 2, CAST(NULL AS INT), _1))) AS v FROM tbl", + "casts the lookup key to the map's exact Arrow key type") + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, element_at(map(named_struct('a', 1), 7), " + + "named_struct('a', _1)) AS v FROM tbl", + "casts the lookup key to the map's exact Arrow key type") + } + } + + // `BinaryType` keys need no decline: Arrow compares them by content, as Spark's ordering does. + test("map lookup with binary keys stays native") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, element_at(map(CAST('1' AS BINARY), 10, CAST('2' AS BINARY), 20), " + + "CAST(CAST(_1 AS STRING) AS BINARY)) AS v FROM tbl") + } + } + } From fecf175ab8f7ad434fa53a255851d4df90349728 Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 25 Aug 2026 12:52:27 -0700 Subject: [PATCH 05/19] feat: support Map for `CreateArray` literal --- .../scala/org/apache/comet/serde/arrays.scala | 11 +-- .../org/apache/comet/serde/literals.scala | 55 +++++++++------ .../scala/org/apache/comet/serde/maps.scala | 46 ++++++------- .../map/element_at_map_collation.sql | 35 ++++++++++ .../expressions/map/get_map_value.sql | 16 +++++ .../sql-tests/expressions/map/map_entries.sql | 2 +- .../comet/CometMapExpressionSuite.scala | 68 +++++-------------- 7 files changed, 130 insertions(+), 103 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/map/element_at_map_collation.sql diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index a730f5c47a2..502ee85e8cf 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -513,6 +513,11 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { * panics. Keep `valueContainsNull` significant here so those children are declined instead. The * same is true of a struct field's nullability, which `coerce_struct_by_*` would merge but * which `MutableArrayData` still rejects for the array's own element type. + * + * TODO: DataFusion 55.0.0 adds a `map_coercion` arm to `type_union_resolution_coercion` + * (apache/datafusion#23521), which coerces a `MapType.valueContainsNull` mismatch away. When + * Comet upgrades onto it, stop keeping `valueContainsNull` significant here or this decline + * stays correct but grows over-conservative, falling maps back that DataFusion can now unify. */ private def normalizeContainerNullability(dt: DataType): DataType = dt match { case ArrayType(elementType, _) => @@ -599,11 +604,7 @@ object CometElementAt extends CometExpressionSerde[ElementAt] { override def getSupportLevel(expr: ElementAt): SupportLevel = { expr.left.dataType match { case _: ArrayType => Compatible() - case MapType(keyType, _, _) => - MapKeySupport.unsupportedKeyTypeReason(keyType) match { - case Some(reason) => Unsupported(Some(reason)) - case None => Compatible() - } + case MapType(keyType, _, _) => MapKeySupport.keySupport(keyType) case _ => Unsupported(Some("Input must be an array or map")) } } diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index e11b79d9fe5..19bd20c8086 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -55,7 +55,7 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { .elementType .isInstanceOf[ArrayType])))) { Compatible(None) - } else if (expandComplexLiteral(expr).isDefined) { + } else if (canExpandComplexLiteral(expr)) { // Rebuilt as a `CreateArray` / `CreateMap` tree in `convert`. Compatible(None) } else { @@ -239,7 +239,9 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * safety property this rewrite rests on: whatever the rebuilt expression does natively is what * the same query already does with `ConstantFolding` disabled, so expansion cannot introduce a * folding-only behaviour difference. Shapes the native map kernels cannot handle are therefore - * declined by their own serdes rather than here, see [[MapKeySupport]] and [[CometMapEntries]]. + * declined by their consumers rather than here: a map lookup with an unsupported key type by + * [[MapKeySupport]], and a non-nullable-value map into `map_entries` by the native planner + * widening its argument. * * Declined shapes: * - Null values and empty top-level containers: a synthesized `Create*` with no children @@ -255,34 +257,45 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * - Folded maps with duplicate keys, see [[hasDuplicateMapKeys]]. */ private def expandComplexLiteral(expr: Literal): Option[Expression] = { - if (expr.value == null) return None + if (!canExpandComplexLiteral(expr)) return None expr.dataType match { - case ArrayType(et, containsNull) if needsExpansion(et) => + case ArrayType(et, containsNull) => val arr = expr.value.asInstanceOf[ArrayData] - if (arr.numElements() == 0) { - None - } else { - val elements = (0 until arr.numElements()) - .map(i => withNullability(literalAt(arr, i, et), containsNull)) - Some(CreateArray(elements, useStringTypeWhenEmpty = false)) - } + val elements = (0 until arr.numElements()) + .map(i => withNullability(literalAt(arr, i, et), containsNull)) + Some(CreateArray(elements, useStringTypeWhenEmpty = false)) case MapType(kt, vt, valueContainsNull) => val mapData = expr.value.asInstanceOf[MapData] val keys = mapData.keyArray() - if (mapData.numElements() == 0 || hasDuplicateMapKeys(keys, kt)) { - None - } else { - val values = mapData.valueArray() - val children = (0 until keys.numElements()).flatMap(i => - Seq( - literalAt(keys, i, kt), - withNullability(literalAt(values, i, vt), valueContainsNull))) - Some(CreateMap(children, useStringTypeWhenEmpty = false)) - } + val values = mapData.valueArray() + val children = (0 until keys.numElements()).flatMap(i => + Seq( + literalAt(keys, i, kt), + withNullability(literalAt(values, i, vt), valueContainsNull))) + Some(CreateMap(children, useStringTypeWhenEmpty = false)) case _ => None } } + /** + * True when [[expandComplexLiteral]] can rebuild this folded literal, tested without allocating + * the rebuilt tree so `getSupportLevel` can probe cheaply (`convert` runs the build only once, + * on the value it keeps). Mirrors the shapes [[expandComplexLiteral]] declines: a non-null + * value, a non-empty top-level container, an array whose elements need expansion, and a map + * with no duplicate keys. + */ + private def canExpandComplexLiteral(expr: Literal): Boolean = { + if (expr.value == null) return false + expr.dataType match { + case ArrayType(et, _) if needsExpansion(et) => + expr.value.asInstanceOf[ArrayData].numElements() > 0 + case MapType(kt, _, _) => + val mapData = expr.value.asInstanceOf[MapData] + mapData.numElements() > 0 && !hasDuplicateMapKeys(mapData.keyArray(), kt) + case _ => false + } + } + /** * True when a Literal of this type has to be expanded rather than serialized, because the * native `Literal` proto carries no map values. Walks array nesting only: an array of structs diff --git a/spark/src/main/scala/org/apache/comet/serde/maps.scala b/spark/src/main/scala/org/apache/comet/serde/maps.scala index 1195f3aaeb4..51fa428b543 100644 --- a/spark/src/main/scala/org/apache/comet/serde/maps.scala +++ b/spark/src/main/scala/org/apache/comet/serde/maps.scala @@ -33,47 +33,47 @@ import org.apache.comet.shims.CometTypeShim */ private[serde] object MapKeySupport { - val floatingPointReason: String = + private val floatingPointReason: String = "Spark normalizes floating-point map keys, so `-0.0` matches a `+0.0` key and all `NaN`s " + "match each other; Comet's native map lookup compares the raw Arrow values." - val collationReason: String = + private val collationReason: String = "Comet's native map lookup compares string keys as `UTF8_BINARY` and cannot honour a " + "non-default collation." - val complexKeyReason: String = - "Comet's native `map_extract` casts the lookup key to the map's exact Arrow key type, so a " + - "complex key type whose components are declared non-nullable rejects a lookup key that " + - "holds a `NULL` inside one of them." + private val complexKeyReason: String = + "Comet's native `map_extract` casts the lookup key to the map's exact Arrow key type, which " + + "cannot reproduce Spark's equality for a complex key type (for example a `NULL` inside the " + + "lookup key aborts the cast against a non-nullable nested component)." /** - * The reason Comet's native map lookup cannot reproduce Spark's key equality for `keyType`, or - * `None` when it can. Spark finds a key with `TypeUtils.getInterpretedOrdering` over the keys - * `ArrayBasedMapBuilder` stored, having first normalized them, while native `map_extract` - * compares the Arrow values as they are: + * The `SupportLevel` for a map-consuming expression whose stored-key type is `keyType`. Spark + * finds a key with `TypeUtils.getInterpretedOrdering` over the keys `ArrayBasedMapBuilder` + * stored, having first normalized them, while native `map_extract` compares the Arrow values as + * they are, so decline the key types where those disagree: * - `ArrayBasedMapBuilder` rewrites a `-0.0` key to `+0.0` and canonicalises `NaN`, and * `nanSafeCompareDoubles` treats `-0.0` and `+0.0` as equal, so Spark answers a `-0.0` * lookup from a `+0.0` key where native finds nothing. * - a non-default collation compares under rules native applies as `UTF8_BINARY`. * - for a complex key type, `map_extract`'s `coerce_types` returns the map's exact key field - * type, so Comet's planner casts the lookup key to it. When the key type declares a nested - * component non-nullable, a `NULL` inside the lookup key aborts that cast with + * type, so Comet's planner casts the lookup key to it. That cannot reproduce Spark's + * interpreted-ordering equality, and a `NULL` inside the lookup key aborts the cast with * `Non-nullable field of ListArray "item" cannot contain nulls` rather than missing the - * lookup, which is what Spark does. + * lookup. Declined for every complex key type, since which of these trips is a runtime + * property of the lookup. * * `BinaryType` keys need no decline: Arrow compares them by content, as Spark's ordering does. - * The floating-point and collation checks walk every nesting level of the key type, so they - * keep holding if native `map_extract` ever gains complex-key support. + * The floating-point and collation checks walk every nesting level of the key type. */ - def unsupportedKeyTypeReason(keyType: DataType): Option[String] = { + def keySupport(keyType: DataType): SupportLevel = { if (SupportLevel.containsType(keyType, classOf[FloatType], classOf[DoubleType])) { - Some(floatingPointReason) + Unsupported(Some(floatingPointReason)) } else if (hasNonDefaultStringCollation(keyType)) { - Some(collationReason) + Unsupported(Some(collationReason)) } else if (isComplexType(keyType)) { - Some(complexKeyReason) + Unsupported(Some(complexKeyReason)) } else { - None + Compatible() } } } @@ -117,11 +117,7 @@ object CometMapValues extends CometExpressionSerde[MapValues] { object CometMapExtract extends CometExpressionSerde[GetMapValue] { override def getSupportLevel(expr: GetMapValue): SupportLevel = expr.child.dataType match { - case MapType(keyType, _, _) => - MapKeySupport.unsupportedKeyTypeReason(keyType) match { - case Some(reason) => Unsupported(Some(reason)) - case None => Compatible() - } + case MapType(keyType, _, _) => MapKeySupport.keySupport(keyType) case _ => Compatible() } diff --git a/spark/src/test/resources/sql-tests/expressions/map/element_at_map_collation.sql b/spark/src/test/resources/sql-tests/expressions/map/element_at_map_collation.sql new file mode 100644 index 00000000000..763e1b31f12 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/element_at_map_collation.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- MinSparkVersion: 4.0 + +-- Spark 4.0+ supports string collations. Spark compares a map key under its declared collation, +-- so a `UTF8_LCASE` map matches `A1` against a dynamic `a1` lookup and returns `7`. Comet's native +-- `map_extract` compares string keys as `UTF8_BINARY`, so `CometElementAt` declines the lookup and +-- Spark evaluates the projection. The `element_at` form is used rather than `m[key]` because +-- Spark's `SimplifyExtractValueOps` rewrites `map(...)[key]` over a literal map into a `CASE` +-- before it can reach the native map lookup. + +statement +CREATE TABLE test_element_at_collation(k string) USING parquet + +statement +INSERT INTO test_element_at_collation VALUES ('a1'), ('A1'), ('zz'), (NULL) + +query expect_fallback(cannot honour a non-default collation) +SELECT element_at(map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7), CAST(k AS STRING COLLATE UTF8_LCASE)) +FROM test_element_at_collation diff --git a/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql index 4b497177cee..2c6c23846eb 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/get_map_value.sql @@ -30,3 +30,19 @@ SELECT m['x'], m['missing'] FROM test_map -- literal arguments query spark_answer_only SELECT map('a', 1, 'b', 2)['a'], map('a', 1, 'b', 2)['missing'], map('a', 1, 'b', 2)[NULL] + +-- Map key types whose Spark equality Comet's native `map_extract` cannot reproduce fall back to +-- Spark. `x[key]` routes through `GetMapValue` / `CometMapExtract` (the `element_at` form in +-- `element_at_map.sql` exercises the same guard on `CometElementAt`). A `map` column +-- is used rather than a `map(...)` literal because Spark's `SimplifyExtractValueOps` rewrites +-- `map(...)[key]` over a literal map into a `CASE` before it can reach the native lookup. Spark +-- stores a `-0.0` key as `+0.0` and finds it with `nanSafeCompareDoubles`, so it returns `7` for a +-- `-0.0` lookup; native lookup compares the raw Arrow values. +statement +CREATE TABLE test_map_double(m map) USING parquet + +statement +INSERT INTO test_map_double VALUES (map(CAST(0 AS DOUBLE), 7)), (map(CAST(1 AS DOUBLE), 8)), (NULL) + +query expect_fallback(Spark normalizes floating-point map keys) +SELECT m[CAST(-0.0 AS DOUBLE)] FROM test_map_double diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql index 1b1cd26eccf..46e4f7c111a 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql @@ -21,7 +21,7 @@ CREATE TABLE test_map_entries(m map) USING parquet statement INSERT INTO test_map_entries VALUES (map('a', 1, 'b', 2)), (map()), (NULL) -query spark_answer_only +query SELECT map_entries(m) FROM test_map_entries -- `MapType(_, _, valueContainsNull = false)`, which every `map(...)` over non-null values diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index ad3769c1a41..d91dd671b82 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -275,34 +275,28 @@ class CometMapExpressionSuite extends CometTestBase { // A map that reaches `map_entries` through an expression rather than a scan keeps // `valueContainsNull = false`, which every `map(...)` over non-null values produces. DataFusion's // `map_entries` declares the entry `value` field nullable but reuses the input map's entries - // array, so the planner widens the argument before the call. Both queries below depend on a - // column, so `ConstantFolding` cannot collapse the whole `map_entries(...)` call into an - // `ArrayType(StructType)` literal that `CometLiteral` would decline. `map_entries.sql` covers the - // all-constant shapes, where the harness excludes `ConstantFolding`. - test("map_entries on a non-nullable-value map expression (multirow)") { + // array, so the planner widens the argument before the call. Here the inner `map(1, map(1, 2))` + // folds to a literal that `CometLiteral` rebuilds, and the outer `element_at` yields a + // non-nullable-value map straight into native `map_entries`. This exercises the widening on the + // default folding-on path; `map_entries.sql` covers the constructor path, where the harness + // excludes `ConstantFolding`. + test("map_entries on a folded non-nullable-value map (multirow)") { withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndOperator("SELECT _1 AS id, map_entries(map(_1, 2)) AS e FROM tbl") checkSparkAnswerAndOperator( "SELECT _1 AS id, map_entries(element_at(map(1, map(1, 2)), _1)) AS e FROM tbl") } } - // Spark stores a `-0.0` map key as `+0.0` and finds it with `nanSafeCompareDoubles`, which - // treats `-0.0` and `+0.0` as equal. Native `map_extract` compares the raw Arrow values, so - // `CometMapExtract` / `CometElementAt` decline the lookup. The map here folds to a literal, so - // the decline has to come from the lookup rather than from `CometLiteral`, which cannot see how - // a map nested in a map value will be consumed. - test("map lookup with floating-point keys falls back") { + // Finding E: a map nested inside a map value is handed to the JVM dispatcher whole, so its double + // keys never revisit `CometLiteral`. The guard has to live on the lookup instead. The outer key + // type is INT, so inspecting only the outermost map would admit the query; the fallback comes + // from the inner `element_at`, whose child is `MapType(DoubleType, IntegerType)`. Constant folding + // is on here, which is the only configuration where the inner map becomes such a literal, so this + // regression cannot be expressed in a SQL fixture (the harness disables folding). The direct + // single-level double-key lookups live in `element_at_map.sql` / `get_map_value.sql`. + test("nested map lookup with floating-point keys falls back") { withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { val negZero = "CAST(concat('-', CAST(_1 - 1 AS STRING), '.0') AS DOUBLE)" - checkSparkAnswerAndFallbackReason( - s"SELECT _1 AS id, element_at(map(CAST(0 AS DOUBLE), 7), $negZero) AS v FROM tbl", - "Spark normalizes floating-point map keys") - checkSparkAnswerAndFallbackReason( - s"SELECT _1 AS id, map(CAST(0 AS DOUBLE), 7)[$negZero] AS v FROM tbl", - "Spark normalizes floating-point map keys") - // The outer key type is INT, so only inspecting it would admit the lookup; the inner map is - // handed to the JVM dispatcher whole and never revisits `CometLiteral`. checkSparkAnswerAndFallbackReason( "SELECT _1 AS id, element_at(element_at(map(1, map(CAST(0 AS DOUBLE), 7)), _1), " + s"$negZero) AS v FROM tbl", @@ -310,14 +304,11 @@ class CometMapExpressionSuite extends CometTestBase { } } - // Spark compares the key under its declared collation; native compares under UTF8_BINARY. - test("map lookup with collated string keys falls back") { + // Finding E for a collated inner key. Same folding-on-only bypass as the double-key case above; + // the direct single-level collated lookup lives in `element_at_map_collation.sql`. + test("nested map lookup with collated string keys falls back") { assume(isSpark40Plus) withParquetTable(Seq(("a1", 0)), "tbl") { - checkSparkAnswerAndFallbackReason( - "SELECT element_at(map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7), " + - "CAST(_1 AS STRING COLLATE UTF8_LCASE)) AS v FROM tbl", - "cannot honour a non-default collation") checkSparkAnswerAndFallbackReason( "SELECT element_at(element_at(map(1, map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7)), 1), " + "CAST(_1 AS STRING COLLATE UTF8_LCASE)) AS v FROM tbl", @@ -325,29 +316,4 @@ class CometMapExpressionSuite extends CometTestBase { } } - // `map_extract`'s `coerce_types` returns the map's exact Arrow key field type, so Comet casts the - // lookup key to it. A NULL inside the lookup key then aborts the cast with `Non-nullable field of - // ListArray "item" cannot contain nulls` instead of missing the lookup, which is what Spark does. - test("map lookup with complex keys falls back") { - withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndFallbackReason( - "SELECT _1 AS id, element_at(map(array(1), 7), " + - "array(IF(_1 = 2, CAST(NULL AS INT), _1))) AS v FROM tbl", - "casts the lookup key to the map's exact Arrow key type") - checkSparkAnswerAndFallbackReason( - "SELECT _1 AS id, element_at(map(named_struct('a', 1), 7), " + - "named_struct('a', _1)) AS v FROM tbl", - "casts the lookup key to the map's exact Arrow key type") - } - } - - // `BinaryType` keys need no decline: Arrow compares them by content, as Spark's ordering does. - test("map lookup with binary keys stays native") { - withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndOperator( - "SELECT _1 AS id, element_at(map(CAST('1' AS BINARY), 10, CAST('2' AS BINARY), 20), " + - "CAST(CAST(_1 AS STRING) AS BINARY)) AS v FROM tbl") - } - } - } From 541f4e59c1d5b4953248662dcd041eb94948984c Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 25 Aug 2026 18:59:18 -0700 Subject: [PATCH 06/19] feat: support Map for `CreateArray` literal --- native/core/src/execution/planner.rs | 93 +++++++++---- .../scala/org/apache/comet/serde/arrays.scala | 55 ++++---- .../org/apache/comet/serde/literals.scala | 128 +++++++++++------- .../expressions/array/create_array.sql | 19 ++- .../sql-tests/expressions/map/map_entries.sql | 16 +++ .../comet/CometArrayExpressionSuite.scala | 13 ++ .../comet/CometMapExpressionSuite.scala | 58 ++++++++ 7 files changed, 273 insertions(+), 109 deletions(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 25667a96027..74f0d395f6b 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -48,7 +48,9 @@ use crate::execution::{ }; use crate::jvm_bridge::{jni_call, JVMClasses}; use arrow::compute::CastOptions; -use arrow::datatypes::{DataType, Field, FieldRef, Schema, TimeUnit, DECIMAL128_MAX_PRECISION}; +use arrow::datatypes::{ + DataType, Field, FieldRef, Fields, Schema, TimeUnit, DECIMAL128_MAX_PRECISION, +}; use arrow::ffi_stream::FFI_ArrowArrayStream; use datafusion::functions_aggregate::bit_and_or_xor::{bit_and_udaf, bit_or_udaf, bit_xor_udaf}; use datafusion::functions_aggregate::count::count_udaf; @@ -216,6 +218,39 @@ fn make_all_fields_nullable(data_type: &DataType) -> DataType { } } +/// Return a copy of a `Map` type with only the outer entries `value` field marked nullable, keeping +/// the key field non-nullable and every nested key/value type byte-for-byte unchanged. Any non-`Map` +/// type is returned unchanged. +/// +/// This is the shallow counterpart of `make_all_fields_nullable`, used for `map_entries`. +/// `map_entries` reuses the input map's entries array as its output list values but declares that +/// element's `value` field nullable (`Struct(key non-null, value nullable)`), deriving both nested +/// types verbatim from the input. Arrow's `GenericListArray::try_new` compares the declared element +/// field's type against the reused values array's type in full, so the ONLY field that can mismatch +/// is the entries `value` field's own `nullable` flag; widening just that field is sufficient. +/// Recursing into nested types (as `make_all_fields_nullable` does) would additionally flip, say, a +/// nested `Map`'s `valueContainsNull`, which then diverges from the Spark-serialized `return_type` +/// and makes a downstream `make_array` see unequal map types and panic. +fn widen_map_entry_value_nullable(data_type: &DataType) -> DataType { + match data_type { + DataType::Map(entries, sorted) => match entries.data_type() { + DataType::Struct(kv) if kv.len() == 2 && !kv[1].is_nullable() => { + let new_value = Arc::new(kv[1].as_ref().clone().with_nullable(true)); + let new_kv: Fields = vec![Arc::clone(&kv[0]), new_value].into(); + let new_entries = Arc::new( + entries + .as_ref() + .clone() + .with_data_type(DataType::Struct(new_kv)), + ); + DataType::Map(new_entries, *sorted) + } + _ => data_type.clone(), + }, + other => other.clone(), + } +} + /// If `expr` evaluates to `Timestamp(_, Some(_))` against `schema`, wrap it in a /// metadata-only cast to `Timestamp(_, None)`. This is required because /// DataFusion's `SortMergeJoinExec` comparator only supports timezone-less @@ -3385,11 +3420,14 @@ impl PhysicalPlanner { let fun_name = &expr.func; // `map_entries` reuses the input map's entries array but declares the entry `value` field - // nullable, so widen the argument first or Arrow rejects the child type. See - // `coerce_child_fields_nullable`. + // nullable, so widen ONLY that outer field first or Arrow rejects the child type. Widening + // nested types would corrupt a nested map's `valueContainsNull` and break a downstream + // consumer. See `widen_map_entry_value_nullable`. let args = if fun_name == "map_entries" { args.into_iter() - .map(|arg| Self::coerce_child_fields_nullable(arg, &input_schema)) + .map(|arg| { + Self::coerce_child_to(arg, &input_schema, widen_map_entry_value_nullable) + }) .collect::, ExecutionError>>()? } else { args @@ -3526,36 +3564,41 @@ impl PhysicalPlanner { Ok(scalar_expr) } - /// Casts `child` to the all-nullable variant of its own type, or returns it unchanged when it - /// already is. Used where a kernel's declared output type marks every nested field nullable - /// but its implementation reuses the child's arrays, so a non-nullable inner field makes the - /// declared and produced types disagree: - /// - /// - `collect_list` / `collect_set` build their result list with all element fields marked - /// nullable, while `SparkCollectList::return_type` (and `SparkCollectSet`) derive the - /// element type directly from the child. With a non-nullable nested field (e.g. a struct - /// field), DataFusion's grouped `AggregateExec` fails validating its output batch - /// ("column types must match schema types"). - /// - `map_entries` declares its list element as `Struct(key non-null, value nullable)` but - /// reuses the input map's own entries array as the list values, so `ListArray::new` - /// rejects a map whose entry `value` field is non-nullable. Every Spark `map(...)` - /// constructor over non-null values produces exactly that type. - /// - /// `make_all_fields_nullable` keeps an Arrow map's key field non-nullable, so widening a map - /// stays a legal map type. - fn coerce_child_fields_nullable( + /// Casts `child` so its type matches `widen(child_type)`, wrapping it in a `CastExpr` only when + /// the widened type differs. Shared skeleton for `coerce_child_fields_nullable` and the + /// `map_entries` argument widening; each caller supplies the widening its consumer needs. + fn coerce_child_to( child: Arc, schema: &SchemaRef, + widen: impl Fn(&DataType) -> DataType, ) -> Result, ExecutionError> { let child_type = child.data_type(schema.as_ref())?; - let nullable_type = make_all_fields_nullable(&child_type); - if child_type.equals_datatype(&nullable_type) { + let widened = widen(&child_type); + if child_type.equals_datatype(&widened) { Ok(child) } else { - Ok(Arc::new(CastExpr::new(child, nullable_type, None))) + Ok(Arc::new(CastExpr::new(child, widened, None))) } } + /// Casts `child` to the all-nullable variant of its own type, or returns it unchanged when it + /// already is. Used by `collect_list` / `collect_set`, which build their result list with all + /// element fields marked nullable while `SparkCollectList::return_type` (and `SparkCollectSet`) + /// derive the element type directly from the child. With a non-nullable nested field (e.g. a + /// struct field), DataFusion's grouped `AggregateExec` fails validating its output batch + /// ("column types must match schema types"). + /// + /// `make_all_fields_nullable` keeps an Arrow map's key field non-nullable, so widening a map + /// stays a legal map type. `map_entries` needs only its outer entry `value` field widened, not + /// the deep widening this performs; it passes `widen_map_entry_value_nullable` to + /// `coerce_child_to` instead. + fn coerce_child_fields_nullable( + child: Arc, + schema: &SchemaRef, + ) -> Result, ExecutionError> { + Self::coerce_child_to(child, schema, make_all_fields_nullable) + } + fn create_aggr_func_expr( name: &str, schema: SchemaRef, diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 502ee85e8cf..50adf158765 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -473,16 +473,19 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { // DataFusion's `make_array` asserts strict element-type equality in // `MutableArrayData::with_capacities` and panics on a mismatch. Spark's CreateArray is more // permissive: its type coercion compares element types with `sameType`, which ignores - // nullability, so children that share a surface type but differ only in nested field - // nullability get no unifying cast. DataFusion coerces an `ArrayType.containsNull` mismatch - // away before that assert, but NOT a struct field's or a `MapType.valueContainsNull` - // mismatch, so `array(struct(a not null), struct(a nullable))` and - // `array(map('x', CAST(NULL AS INT)), map('z', 3))` both panic inside `make_array_inner`. - // Decline only those cases (i.e. children that still differ after normalizing the container - // nullability DataFusion does coerce) so Spark's evaluator handles them. + // nullability, so children that share a surface type but differ only in nested nullability get + // no unifying cast, and Spark reports a merged element type (nullability-OR across children) + // as the array's own type. Reconcile that here by casting every child to the merged element + // type, so `make_array` sees identical Arrow types. Comet's cast widens container nullability + // (`ArrayType.containsNull`, `MapType.valueContainsNull`, and either nested) via the recursive + // `cast_map_to_map` / array casts, which is what lets `array(map('x', CAST(NULL AS INT)), + // map('z', 3))` and `array(map(1, array(1)), map(2, array(col)))` run natively rather than + // falling back. // - // TODO: remove this decline once apache/datafusion#22366 lands; the upstream fix widens the - // element type via nullability-OR-merge and casts each child before MutableArrayData. + // A struct FIELD's own nullability is the one difference kept significant (see + // [[normalizeContainerNullability]]): `array(struct(a not null), struct(a nullable))` is + // declined so Spark evaluates it, matching the coverage the `make_array` struct-field panic + // drove. val normalizedTypes = children.map(c => normalizeContainerNullability(c.dataType)) if (normalizedTypes.distinct.size > 1) { withFallbackReason( @@ -492,7 +495,13 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { return None } - val childExprs = children.map(exprToProtoInternal(_, inputs, binding)) + // Cast each child that is not already the merged element type to it. The element data types + // are identical, so the cast only widens nullability metadata and never changes values. + val elementType = expr.dataType.asInstanceOf[ArrayType].elementType + val childExprs = children.map { c => + val unified = if (c.dataType == elementType) c else Cast(c, elementType) + exprToProtoInternal(unified, inputs, binding) + } if (childExprs.forall(_.isDefined)) { scalarFunctionExprToProto("make_array", childExprs: _*) @@ -503,30 +512,22 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { } /** - * Rewrites a type so that `ArrayType.containsNull` is forced to `true` everywhere, while struct - * field nullability and `MapType.valueContainsNull` are left intact. Two CreateArray children - * whose types differ ONLY in `containsNull` are tolerated by DataFusion's `make_array`, because - * `type_union_resolution_coercion` routes `List` arguments through `list_coercion`, which - * merges the element field's nullability and lets Comet's planner insert the unifying cast. Its - * fallback chain has no `map_coercion` arm in DataFusion 54.1, so two `Map` arguments that - * differ in value nullability fail to unify, no cast is inserted, and `MutableArrayData` - * panics. Keep `valueContainsNull` significant here so those children are declined instead. The - * same is true of a struct field's nullability, which `coerce_struct_by_*` would merge but - * which `MutableArrayData` still rejects for the array's own element type. - * - * TODO: DataFusion 55.0.0 adds a `map_coercion` arm to `type_union_resolution_coercion` - * (apache/datafusion#23521), which coerces a `MapType.valueContainsNull` mismatch away. When - * Comet upgrades onto it, stop keeping `valueContainsNull` significant here or this decline - * stays correct but grows over-conservative, falling maps back that DataFusion can now unify. + * Rewrites a type so that container nullability Comet can cast away (`ArrayType.containsNull` + * and `MapType.valueContainsNull`, at every nesting level) is forced to `true`, while each + * struct FIELD's own nullability is left intact. Two CreateArray children whose types differ + * only in container nullability normalize equal, so [[convert]] unifies them by casting each to + * the merged element type before `make_array` (DataFusion 54.1 only coerces `List` nullability + * on its own, not `Map`; the explicit cast covers both). Children that differ in a struct + * field's own nullability normalize distinct and are declined instead, so Spark evaluates them. */ private def normalizeContainerNullability(dt: DataType): DataType = dt match { case ArrayType(elementType, _) => ArrayType(normalizeContainerNullability(elementType), containsNull = true) - case MapType(keyType, valueType, valueContainsNull) => + case MapType(keyType, valueType, _) => MapType( normalizeContainerNullability(keyType), normalizeContainerNullability(valueType), - valueContainsNull) + valueContainsNull = true) case StructType(fields) => StructType(fields.map(f => f.copy(dataType = normalizeContainerNullability(f.dataType)))) case other => other diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index 19bd20c8086..889c6ff5e31 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -24,14 +24,14 @@ import java.lang import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateArray, CreateMap, Expression, KnownNullable, Literal} import org.apache.spark.sql.catalyst.util.{ArrayData, MapData, TypeUtils} -import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, NullType, ShortType, StringType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, CalendarIntervalType, DataType, DateType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, MapType, NullType, ShortType, StringType, StructType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.UTF8String import com.google.protobuf.ByteString import org.apache.comet.CometSparkSessionExtensions.withFallbackReason import org.apache.comet.DataTypeSupport.isComplexType -import org.apache.comet.serde.{CometExpressionSerde, Compatible, ExprOuterClass, LiteralOuterClass, SupportLevel, Unsupported} +import org.apache.comet.serde.{CometExpressionSerde, Compatible, ExprOuterClass, LiteralOuterClass, MapKeySupport, SupportLevel, Unsupported} import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, isTimeType, serializeDataType, supportedDataType} import org.apache.comet.serde.Types.ListLiteral @@ -55,7 +55,7 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { .elementType .isInstanceOf[ArrayType])))) { Compatible(None) - } else if (canExpandComplexLiteral(expr)) { + } else if (expandComplexLiteral(expr).isDefined) { // Rebuilt as a `CreateArray` / `CreateMap` tree in `convert`. Compatible(None) } else { @@ -228,25 +228,37 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { /** * Rebuild a folded complex Literal as an equivalent tree of `CreateArray` / `CreateMap` over - * primitive-typed Literals, or `None` when the shape cannot be rebuilt. The native `Literal` - * proto carries scalars and nested `ListLiteral`s but no map values, so a Literal whose type - * contains a `MapType` has to be expanded before serialization. Teaching the proto to transport - * maps directly would remove the need for this rewrite and for the declines below: + * primitive-typed Literals, or `None` when the shape cannot be rebuilt. `getSupportLevel` + * probes with this same function and reports `Compatible` when it returns a `Some`, so + * admission and expansion cannot diverge. The native `Literal` proto carries scalars and nested + * `ListLiteral`s but no map values, so a Literal whose type contains a `MapType` has to be + * expanded before serialization. Teaching the proto to transport maps directly would remove the + * need for this rewrite and for the declines below: * https://github.com/apache/datafusion-comet/issues/1937 * * The rebuilt tree is the tree Spark itself had before `ConstantFolding` collapsed it, down to * every container's declared nullability (see [[withNullability]]). That equivalence is the * safety property this rewrite rests on: whatever the rebuilt expression does natively is what * the same query already does with `ConstantFolding` disabled, so expansion cannot introduce a - * folding-only behaviour difference. Shapes the native map kernels cannot handle are therefore - * declined by their consumers rather than here: a map lookup with an unsupported key type by - * [[MapKeySupport]], and a non-nullable-value map into `map_entries` by the native planner - * widening its argument. + * folding-only behaviour difference. A non-nullable-value map into `map_entries` is handled by + * the native planner widening its argument rather than declined here. + * + * Map key semantics, however, are declined here at admission (see [[mapKeyTypesExpandable]]) + * rather than left to consumers. A map lookup gates itself on [[MapKeySupport]], but not every + * native map consumer does: `map_contains_key` lowers to `array_contains(map_keys(...), key)`, + * and neither kernel checks the key type. A map that is only the value of another map is also + * opaque once rebuilt, because `CometCreateMap` hands the whole `CreateMap` to the JVM codegen + * dispatcher, so a nested unsupported key type never revisits this serde. Declining any folded + * literal that (recursively) contains an unsupported or non-orderable map key type keeps all of + * those paths on Spark. * * Declined shapes: * - Null values and empty top-level containers: a synthesized `Create*` with no children * cannot recover the original element type. Empty `ArrayType` literals still serialize via * `makeListLiteral`, which keeps the type. + * - Any map key type native map kernels cannot reproduce Spark equality for (floating-point, + * collated string, complex), or whose interpreted ordering is undefined + * (`CalendarIntervalType`), at any nesting level. See [[mapKeyTypesExpandable]]. * - Arrays whose elements are structs, because [[needsExpansion]] does not walk into a * `StructType`. Native `CreateNamedStruct` builds a 1-row `StructArray` whenever all of its * children are scalars (`values_to_arrays`), which collides with the surrounding batch's @@ -257,43 +269,61 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * - Folded maps with duplicate keys, see [[hasDuplicateMapKeys]]. */ private def expandComplexLiteral(expr: Literal): Option[Expression] = { - if (!canExpandComplexLiteral(expr)) return None + if (expr.value == null || !mapKeyTypesExpandable(expr.dataType)) return None expr.dataType match { - case ArrayType(et, containsNull) => + case ArrayType(et, containsNull) if needsExpansion(et) => val arr = expr.value.asInstanceOf[ArrayData] - val elements = (0 until arr.numElements()) - .map(i => withNullability(literalAt(arr, i, et), containsNull)) - Some(CreateArray(elements, useStringTypeWhenEmpty = false)) + if (arr.numElements() == 0) { + None + } else { + val elements = (0 until arr.numElements()) + .map(i => withNullability(literalAt(arr, i, et), containsNull)) + Some(CreateArray(elements, useStringTypeWhenEmpty = false)) + } case MapType(kt, vt, valueContainsNull) => val mapData = expr.value.asInstanceOf[MapData] val keys = mapData.keyArray() - val values = mapData.valueArray() - val children = (0 until keys.numElements()).flatMap(i => - Seq( - literalAt(keys, i, kt), - withNullability(literalAt(values, i, vt), valueContainsNull))) - Some(CreateMap(children, useStringTypeWhenEmpty = false)) + if (mapData.numElements() == 0 || hasDuplicateMapKeys(keys, kt)) { + None + } else { + val values = mapData.valueArray() + val children = (0 until keys.numElements()).flatMap(i => + Seq( + literalAt(keys, i, kt), + withNullability(literalAt(values, i, vt), valueContainsNull))) + Some(CreateMap(children, useStringTypeWhenEmpty = false)) + } case _ => None } } /** - * True when [[expandComplexLiteral]] can rebuild this folded literal, tested without allocating - * the rebuilt tree so `getSupportLevel` can probe cheaply (`convert` runs the build only once, - * on the value it keeps). Mirrors the shapes [[expandComplexLiteral]] declines: a non-null - * value, a non-empty top-level container, an array whose elements need expansion, and a map - * with no duplicate keys. + * True when every map key type reachable inside `dataType` can be rebuilt and consumed + * natively. A folded map is rebuilt as a `CreateMap`; once native it may reach a map consumer + * that has no key-type gate of its own -- `map_contains_key` lowers to + * `array_contains(map_keys(...), key)`, and neither `map_keys` nor `array_contains` consults + * [[MapKeySupport]]. A map that is only the value of another map is handed to the JVM + * dispatcher whole by `CometCreateMap` and never revisits this serde, so a nested unsupported + * key type would otherwise slip through. Mirror the [[MapKeySupport]] gate here, walking into + * map values, array elements, and struct fields, so an unsupported key type (floating-point, + * collated string, complex) declines expansion instead of reaching a native kernel whose key + * equality disagrees with Spark. + * + * Also decline a key type whose interpreted ordering is undefined -- one that recursively + * contains a `CalendarIntervalType`, whose `PhysicalCalendarIntervalType` ordering throws "does + * not support ordered operations". `ArrayBasedMapBuilder` dedups such keys by hash equality and + * never asks for an ordering, but [[hasDuplicateMapKeys]] compares keys through that ordering, + * so gating here keeps it from being called on a type it cannot order. Such literals fell back + * before this rewrite too, so declining them matches the prior behaviour. */ - private def canExpandComplexLiteral(expr: Literal): Boolean = { - if (expr.value == null) return false - expr.dataType match { - case ArrayType(et, _) if needsExpansion(et) => - expr.value.asInstanceOf[ArrayData].numElements() > 0 - case MapType(kt, _, _) => - val mapData = expr.value.asInstanceOf[MapData] - mapData.numElements() > 0 && !hasDuplicateMapKeys(mapData.keyArray(), kt) - case _ => false - } + private def mapKeyTypesExpandable(dataType: DataType): Boolean = dataType match { + case MapType(kt, vt, _) => + MapKeySupport.keySupport(kt).isInstanceOf[Compatible] && + !SupportLevel.containsType(kt, classOf[CalendarIntervalType]) && + mapKeyTypesExpandable(vt) + case ArrayType(et, _) => mapKeyTypesExpandable(et) + case StructType(fields) => fields.forall(f => mapKeyTypesExpandable(f.dataType)) + case _ => true } /** @@ -307,13 +337,9 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { case _ => false } - /** Element `i` of `arr`, or `null` for a null slot. */ - private def valueAt(arr: ArrayData, i: Int, dt: DataType): Any = - if (arr.isNullAt(i)) null else arr.get(i, dt) - - /** Element `i` of `arr` as a Literal of type `dt`. */ + /** Element `i` of `arr` as a Literal of type `dt`, or a null Literal for a null slot. */ private def literalAt(arr: ArrayData, i: Int, dt: DataType): Literal = - Literal(valueAt(arr, i, dt), dt) + Literal(if (arr.isNullAt(i)) null else arr.get(i, dt), dt) /** * True when the folded map's key array holds duplicates under Spark's own key equality, in @@ -322,14 +348,14 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * `MAP_KEY_DEDUP_POLICY=EXCEPTION` and silently drops the earlier entry under `LAST_WIN`, where * the original literal (from `from_json`, or a cast of one) kept both entries. * - * `ArrayBasedMapBuilder` hashes most atomic keys but compares `BinaryType`, collated - * `StringType` and complex keys through `TypeUtils.getInterpretedOrdering`, and it normalizes - * `-0.0` and `NaN` before either. The interpreted ordering is the one comparison that agrees - * with all of those: `Array[Byte]` keys compare by content rather than by identity, a collated - * `StringType` compares under its collation, and `nanSafeCompareDoubles` already treats `-0.0` - * and `+0.0`, and any two `NaN`s, as equal. A null key cannot occur in a map Spark built, so - * treat one as a duplicate and keep the projection on Spark instead of asking the ordering to - * compare it. + * Only reached for key types [[mapKeyTypesExpandable]] admits, i.e. atomic non-floating-point, + * default-collation `StringType`, and `BinaryType` -- floating-point, collated, complex, and + * `CalendarIntervalType` keys are already declined before this runs. Every admitted key type is + * orderable, so `TypeUtils.getInterpretedOrdering` (which `ArrayBasedMapBuilder` itself uses + * for `BinaryType`) is safe and matches Spark's equality: `Array[Byte]` keys compare by content + * rather than by identity, and the atomic orderings agree with hash equality. A null key cannot + * occur in a map Spark built, so treat one as a duplicate and keep the projection on Spark + * instead of asking the ordering to compare it. */ private def hasDuplicateMapKeys(keys: ArrayData, keyType: DataType): Boolean = { val n = keys.numElements() diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql index 5dfda61fabd..320d4985a25 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -68,17 +68,24 @@ SELECT array(map('x', 1, 'y', 2), map('z', 3)) -- Array of maps whose children disagree on `valueContainsNull`. Spark's CreateArray coercion -- compares element types with `sameType`, which ignores nullability, so it inserts no unifying --- cast. DataFusion 54.1 cannot unify two Map arguments either: the fallback chain in --- `type_union_resolution_coercion` has no `map_coercion` arm, so `make_array` receives the two --- differently typed map arrays and `MutableArrayData` panics. `CometCreateArray` therefore keeps --- `MapType.valueContainsNull` significant when it normalizes container nullability and declines. -query expect_fallback(CreateArray children have mismatched data types) +-- cast; it reports the nullability-merged map type as the array's element type. `make_array` +-- needs identical Arrow types, so `CometCreateArray` casts each child to that merged type +-- (`cast_map_to_map` widens `valueContainsNull`) and runs natively. +query SELECT array(map('x', CAST(NULL AS INT), 'y', 2), map('z', 3)) --- Both children agree that the value is nullable, so this stays native. +-- Both children agree that the value is nullable, so this needs no cast. query SELECT array(map('x', CAST(NULL AS INT)), map('z', CAST(NULL AS INT))) +-- Array of maps whose values are arrays that disagree only on the array's `containsNull`: +-- `map(1, array(1))` has value `ArrayType(IntegerType, containsNull = false)` while +-- `map(2, array(a))` has `ArrayType(IntegerType, true)`. The difference is container nullability +-- nested inside the map value, which Comet's map cast widens, so `CometCreateArray` casts each +-- child to the merged element type and runs natively. +query +SELECT array(map(1, array(1)), map(2, array(a))) FROM test_create_array + -- Single-element array of map. query SELECT array(map(1, 2)) diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql index 46e4f7c111a..a6c145e9f3c 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql @@ -36,3 +36,19 @@ SELECT map_entries(map('a', array(1, 2))) -- A map nested in a map value keeps `valueContainsNull = false` through the extract. query SELECT map_entries(element_at(map(1, map(1, 2)), 1)) + +-- The `map_entries` argument is widened so its entry `value` field is nullable, but ONLY that outer +-- field: the nested `map(1, 2)` value must keep `valueContainsNull = false`. Extracting it with +-- `[0].value` and pairing it with the `map(2, coalesce(id, 0))` sibling would otherwise hit +-- `make_array` with unequal map types (one widened to `valueContainsNull = true`) and panic. +statement +CREATE TABLE test_map_entries_nested(id int) USING parquet + +statement +INSERT INTO test_map_entries_nested VALUES (1), (2), (3) + +query +SELECT array( + map_entries(map(1, IF(id = 1, map(1, 2), NULL)))[0].value, + map(2, coalesce(id, 0))) AS a +FROM test_map_entries_nested diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index cd3e40b8f6d..c72d94866c8 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1183,6 +1183,19 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + // A folded `map(1, array(1))` (value `ArrayType(IntegerType, containsNull = false)`) sits beside a + // dynamic `map(2, array(_1))` (value `ArrayType(IntegerType, true)`). Both maps still declare + // `valueContainsNull = false`, so only the nested array's `containsNull` differs. `CometCreateArray` + // casts each child to the nullability-merged element type (`cast_map_to_map` widens the nested + // array), so `make_array` sees identical Arrow types and this runs natively. + test( + "folded map with non-null nested array beside a dynamic map sibling runs natively (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT array(map(1, array(1)), map(2, array(_1))) AS a FROM tbl") + } + } + test("array of folded map literals (multirow)") { withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { checkSparkAnswerAndOperator("SELECT array(map(1, 10), map(2, 20)) FROM tbl") diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index d91dd671b82..c26c352915f 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -316,4 +316,62 @@ class CometMapExpressionSuite extends CometTestBase { } } + // A folded map with `CalendarIntervalType` keys reaches `CometLiteral`. `ArrayBasedMapBuilder` + // dedups such keys by hash equality, but the folded-literal duplicate-key check needs an + // interpreted ordering and `PhysicalCalendarIntervalType` has none ("does not support ordered + // operations"). Expansion must decline these keys and keep the projection on Spark rather than + // crash planning by asking for that ordering; such literals fell back before this rewrite too. + test("folded map literal with calendar-interval keys falls back (multirow)") { + withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, map(make_interval(1), 1, make_interval(2), 2) AS m FROM tbl", + "Unsupported data type MapType") + } + } + + // `map_entries` widens its argument so the entry `value` field is nullable, but it must widen ONLY + // that outer field. The outer `map(1, IF(...))` has `valueContainsNull = true`, and its value is a + // folded `map(1, 2)` with `valueContainsNull = false`. Widening the nested map too would extract a + // `valueContainsNull = true` map that disagrees with the dynamic `map(2, coalesce(...))` sibling, + // and native `make_array` would panic; the shallow widen keeps the nested type intact. + test("map_entries widening keeps nested map value nullability (multirow)") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, array(" + + "map_entries(map(1, IF(_1 = 1, map(1, 2), NULL)))[0].value, " + + "map(2, coalesce(_1, 0))) AS a FROM tbl") + } + } + + // `map_contains_key` lowers to `array_contains(map_keys(...), key)`; neither `map_keys` nor + // `array_contains` gates the key type, so a folded map with floating-point keys would reach them + // and compare `-0.0` against a normalized `+0.0` key bytewise, unlike Spark. The double-keyed map + // is nested inside an INT-keyed outer map, so the lookup guard on the outer `element_at` does not + // see it; expansion has to inspect nested map key types and decline. Spark returns `7, NULL, NULL`. + test("map_contains_key over nested floating-point map keys falls back (multirow)") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + val negZero = "CAST(concat('-', CAST(_1 - 1 AS STRING), '.0') AS DOUBLE)" + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, map_contains_key(" + + s"element_at(map(1, map(CAST(0 AS DOUBLE), 7)), _1), $negZero) AS present FROM tbl", + "Unsupported data type MapType") + } + } + + // The collated counterpart of the double-key `map_contains_key` case above: a nested + // `UTF8_LCASE`-keyed map would reach `array_contains` with bytewise comparison. Expansion declines + // the folded literal so the case-insensitive lookup stays on Spark. The outer lookup key is the + // dynamic `_1` so the inner map survives as a literal (a constant key would let Spark fold + // `map_keys` into a collated-string array literal instead, which never reaches this guard). + test("map_contains_key over nested collated map keys falls back (multirow)") { + assume(isSpark40Plus) + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, map_contains_key(" + + "element_at(map(1, map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7)), _1), " + + "CAST('a1' AS STRING COLLATE UTF8_LCASE)) AS present FROM tbl", + "Unsupported data type MapType") + } + } + } From ba011bbf1538f3cf1e6fc22d3ea189e81ae0a13f Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 26 Aug 2026 10:01:02 -0700 Subject: [PATCH 07/19] feat: support Map for `CreateArray` literal --- .../expressions/map/element_at_map.sql | 5 ++ .../comet/CometMapExpressionSuite.scala | 54 +++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql b/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql index 5bb322b7f32..ded57f490e2 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql @@ -64,6 +64,11 @@ SELECT element_at(map(CAST(0 AS DOUBLE), 7), CAST(-0.0 AS DOUBLE)) query expect_fallback(Spark normalizes floating-point map keys) SELECT element_at(map(CAST(0 AS FLOAT), 7), CAST(-0.0 AS FLOAT)) +-- The floating-point decline walks every nesting level of the key type, so an array-of-double key +-- falls back for the same reason. +query expect_fallback(Spark normalizes floating-point map keys) +SELECT element_at(map(array(CAST(0 AS DOUBLE)), 7), array(CAST(-0.0 AS DOUBLE))) + -- A complex key type: `map_extract` casts the lookup key to the map's exact Arrow key type, so a -- NULL inside the lookup key would abort the cast instead of missing the lookup. query expect_fallback(casts the lookup key to the map's exact Arrow key type) diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index c26c352915f..74e88231c54 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -374,4 +374,58 @@ class CometMapExpressionSuite extends CometTestBase { } } + // Direct single-level folded map with floating-point keys: `map(CAST(0 AS DOUBLE), 7)` folds to a + // literal and `element_at` with a dynamic `-0.0` lookup must match Spark's `+0.0`-normalized key. + // Native `map_extract` compares raw Arrow values, so `MapKeySupport` declines it at `element_at`. + // Spark returns 7, NULL, NULL. (`element_at_map.sql` covers the folding-off constructor form.) + test("folded map literal with floating-point keys in element_at falls back (multirow)") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + val lookup = "CAST(concat('-', CAST(_1 - 1 AS STRING), '.0') AS DOUBLE)" + checkSparkAnswerAndFallbackReason( + s"SELECT _1 AS id, element_at(map(CAST(0 AS DOUBLE), 7), $lookup) AS v FROM tbl", + "Spark normalizes floating-point map keys") + } + } + + // Direct single-level folded map with collated string keys. Native lookup is bytewise, so a + // case-insensitive `a1` lookup against a stored `A1` cannot match; `MapKeySupport` declines it. + test("folded map literal with collated string keys in element_at falls back (multirow)") { + assume(isSpark40Plus) + withParquetTable(Seq(("a1", 0)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT element_at(map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7), " + + "CAST(_1 AS STRING COLLATE UTF8_LCASE)) AS v FROM tbl", + "cannot honour a non-default collation") + } + } + + // Folded map with a complex (array) key. Spark permits a dynamic lookup array containing a NULL + // element; native `map_extract` casts the lookup to the key's exact Arrow type and cannot + // reproduce Spark's equality, so `MapKeySupport` declines every complex key type. Spark returns + // 7, NULL, NULL. (`element_at_map.sql` covers the constructor form with a non-null lookup.) + test("folded map literal with complex array key in element_at falls back (multirow)") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, element_at(map(array(1), 7), " + + "array(IF(_1 = 2, CAST(NULL AS INT), _1))) AS v FROM tbl", + "casts the lookup key to the map's exact Arrow key type") + } + } + + // A folded nested INT-keyed map is admitted natively (all key-type guards pass), so the outer + // `element_at` runs as native `map_extract`. With ANSI disabled, a remainder-by-zero lookup key + // evaluates to NULL instead of throwing, so `map_extract(NULL_map, NULL)` returns NULL and the + // result matches Spark's 7, NULL, NULL. Under ANSI, native scalar functions evaluate the key + // eagerly and throw where Spark short-circuits after the NULL inner map -- a pre-existing eager + // evaluation difference in native `ElementAt`, not specific to folded literals. + test("folded nested map lookup with per-row key evaluation (multirow, non-ansi)") { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, element_at(element_at(map(1, map(0, 7)), _1), _1 % (_1 - 2)) AS v " + + "FROM tbl") + } + } + } + } From b64837f9057813d0791ab53b896388f9f4b433d8 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 26 Aug 2026 16:29:10 -0700 Subject: [PATCH 08/19] feat: support Map for `CreateArray` literal --- native/core/src/execution/planner.rs | 6 +- .../comet/codegen/CometBatchKernel.java | 8 +- .../scala/org/apache/comet/serde/arrays.scala | 63 +++++---------- .../org/apache/comet/serde/literals.scala | 77 ++++++++++--------- .../expressions/array/create_array.sql | 15 ++++ .../expressions/map/element_at_map.sql | 16 ++++ .../sql-tests/expressions/map/map_entries.sql | 11 +++ .../comet/CometArrayExpressionSuite.scala | 48 ++---------- .../comet/CometMapExpressionSuite.scala | 22 ++++++ 9 files changed, 142 insertions(+), 124 deletions(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 74f0d395f6b..3536073daa0 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -3419,10 +3419,8 @@ impl PhysicalPlanner { .collect::, _>>()?; let fun_name = &expr.func; - // `map_entries` reuses the input map's entries array but declares the entry `value` field - // nullable, so widen ONLY that outer field first or Arrow rejects the child type. Widening - // nested types would corrupt a nested map's `valueContainsNull` and break a downstream - // consumer. See `widen_map_entry_value_nullable`. + // `map_entries` needs its argument's entry `value` field widened to nullable first (only + // that outer field). See `widen_map_entry_value_nullable`. let args = if fun_name == "map_entries" { args.into_iter() .map(|arg| { diff --git a/spark/src/main/java/org/apache/comet/codegen/CometBatchKernel.java b/spark/src/main/java/org/apache/comet/codegen/CometBatchKernel.java index a515cbe32d6..a7bdfe93cf1 100644 --- a/spark/src/main/java/org/apache/comet/codegen/CometBatchKernel.java +++ b/spark/src/main/java/org/apache/comet/codegen/CometBatchKernel.java @@ -31,7 +31,13 @@ */ public abstract class CometBatchKernel extends CometInternalRow { - protected final Object[] references; + // `public` (not `protected`) so that the nested helper classes Spark's codegen emits when it + // splits a large expression (e.g. a folded map rebuilt as a big `CreateMap`) can read it. Those + // helpers are separate classes in the generated package, not subclasses of this one, so a + // `protected` field would raise `IllegalAccessError` at runtime under cross-package protected + // access rules. Spark's own generated classes avoid this by declaring `references` on the + // generated class itself; Comet inherits it here instead, so it must be public. + public final Object[] references; protected CometBatchKernel(Object[] references) { this.references = references; diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 50adf158765..8800d7512e0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -472,32 +472,15 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { // DataFusion's `make_array` asserts strict element-type equality in // `MutableArrayData::with_capacities` and panics on a mismatch. Spark's CreateArray is more - // permissive: its type coercion compares element types with `sameType`, which ignores - // nullability, so children that share a surface type but differ only in nested nullability get - // no unifying cast, and Spark reports a merged element type (nullability-OR across children) - // as the array's own type. Reconcile that here by casting every child to the merged element - // type, so `make_array` sees identical Arrow types. Comet's cast widens container nullability - // (`ArrayType.containsNull`, `MapType.valueContainsNull`, and either nested) via the recursive - // `cast_map_to_map` / array casts, which is what lets `array(map('x', CAST(NULL AS INT)), - // map('z', 3))` and `array(map(1, array(1)), map(2, array(col)))` run natively rather than - // falling back. - // - // A struct FIELD's own nullability is the one difference kept significant (see - // [[normalizeContainerNullability]]): `array(struct(a not null), struct(a nullable))` is - // declined so Spark evaluates it, matching the coverage the `make_array` struct-field panic - // drove. - val normalizedTypes = children.map(c => normalizeContainerNullability(c.dataType)) - if (normalizedTypes.distinct.size > 1) { - withFallbackReason( - expr, - "CreateArray children have mismatched data types: " + - children.map(_.dataType).distinct.mkString(", ")) - return None - } - - // Cast each child that is not already the merged element type to it. The element data types - // are identical, so the cast only widens nullability metadata and never changes values. - val elementType = expr.dataType.asInstanceOf[ArrayType].elementType + // permissive: its coercion compares element types with `sameType` (nullability ignored), so + // children that share a surface type but differ in nullability reach here as distinct types. + // Comet's native runtime types are also frequently MORE nullable than Spark's Catalyst types + // (`map_entries` forces the entry `value` field nullable, list elements are nullable, ...), so + // casting to Spark's declared element type does not reliably unify them. Cast every child to a + // deeply-nullable element type instead (every array/map/struct field nullable at all nesting + // levels; the cast only widens metadata and never changes values), so `make_array` always sees + // identical Arrow types. A child whose cast is unsupported declines below. + val elementType = deepNullable(expr.dataType.asInstanceOf[ArrayType].elementType) val childExprs = children.map { c => val unified = if (c.dataType == elementType) c else Cast(c, elementType) exprToProtoInternal(unified, inputs, binding) @@ -512,24 +495,20 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { } /** - * Rewrites a type so that container nullability Comet can cast away (`ArrayType.containsNull` - * and `MapType.valueContainsNull`, at every nesting level) is forced to `true`, while each - * struct FIELD's own nullability is left intact. Two CreateArray children whose types differ - * only in container nullability normalize equal, so [[convert]] unifies them by casting each to - * the merged element type before `make_array` (DataFusion 54.1 only coerces `List` nullability - * on its own, not `Map`; the explicit cast covers both). Children that differ in a struct - * field's own nullability normalize distinct and are declined instead, so Spark evaluates them. + * A copy of `dt` with every array `containsNull`, map `valueContainsNull`, and struct field + * nullability forced to `true` at every nesting level (map key fields stay non-null per Arrow's + * map invariant). This is Spark's `DataType.asNullable`, re-derived here because that method is + * `private[spark]` and unreachable from this package. Used as the common cast target for + * `CometCreateArray` so that children whose native runtime types are more nullable than Spark's + * Catalyst types (e.g. a `map_entries` entry struct, whose `value` field Comet forces nullable) + * still unify for `make_array`. */ - private def normalizeContainerNullability(dt: DataType): DataType = dt match { - case ArrayType(elementType, _) => - ArrayType(normalizeContainerNullability(elementType), containsNull = true) - case MapType(keyType, valueType, _) => - MapType( - normalizeContainerNullability(keyType), - normalizeContainerNullability(valueType), - valueContainsNull = true) + private def deepNullable(dt: DataType): DataType = dt match { + case ArrayType(et, _) => ArrayType(deepNullable(et), containsNull = true) + case MapType(kt, vt, _) => + MapType(deepNullable(kt), deepNullable(vt), valueContainsNull = true) case StructType(fields) => - StructType(fields.map(f => f.copy(dataType = normalizeContainerNullability(f.dataType)))) + StructType(fields.map(f => f.copy(dataType = deepNullable(f.dataType), nullable = true))) case other => other } } diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index 889c6ff5e31..d50a0af5d60 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -55,7 +55,7 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { .elementType .isInstanceOf[ArrayType])))) { Compatible(None) - } else if (expandComplexLiteral(expr).isDefined) { + } else if (canExpandComplexLiteral(expr)) { // Rebuilt as a `CreateArray` / `CreateMap` tree in `convert`. Compatible(None) } else { @@ -228,13 +228,13 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { /** * Rebuild a folded complex Literal as an equivalent tree of `CreateArray` / `CreateMap` over - * primitive-typed Literals, or `None` when the shape cannot be rebuilt. `getSupportLevel` - * probes with this same function and reports `Compatible` when it returns a `Some`, so - * admission and expansion cannot diverge. The native `Literal` proto carries scalars and nested - * `ListLiteral`s but no map values, so a Literal whose type contains a `MapType` has to be - * expanded before serialization. Teaching the proto to transport maps directly would remove the - * need for this rewrite and for the declines below: - * https://github.com/apache/datafusion-comet/issues/1937 + * primitive-typed Literals, or `None` when [[canExpandComplexLiteral]] rejects the shape. + * `getSupportLevel` probes cheaply with [[canExpandComplexLiteral]], which shares this method's + * admission checks, so the two cannot diverge and only `convert` materializes the tree. The + * native `Literal` proto carries scalars and nested `ListLiteral`s but no map values, so a + * Literal whose type contains a `MapType` has to be expanded before serialization. Teaching the + * proto to transport maps directly would remove the need for this rewrite and for the declines + * below: https://github.com/apache/datafusion-comet/issues/1937 * * The rebuilt tree is the tree Spark itself had before `ConstantFolding` collapsed it, down to * every container's declared nullability (see [[withNullability]]). That equivalence is the @@ -243,14 +243,8 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * folding-only behaviour difference. A non-nullable-value map into `map_entries` is handled by * the native planner widening its argument rather than declined here. * - * Map key semantics, however, are declined here at admission (see [[mapKeyTypesExpandable]]) - * rather than left to consumers. A map lookup gates itself on [[MapKeySupport]], but not every - * native map consumer does: `map_contains_key` lowers to `array_contains(map_keys(...), key)`, - * and neither kernel checks the key type. A map that is only the value of another map is also - * opaque once rebuilt, because `CometCreateMap` hands the whole `CreateMap` to the JVM codegen - * dispatcher, so a nested unsupported key type never revisits this serde. Declining any folded - * literal that (recursively) contains an unsupported or non-orderable map key type keeps all of - * those paths on Spark. + * Map key semantics are declined here at admission rather than left to consumers, because not + * every native map consumer gates on [[MapKeySupport]]; see [[mapKeyTypesExpandable]] for why. * * Declined shapes: * - Null values and empty top-level containers: a synthesized `Create*` with no children @@ -269,34 +263,47 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { * - Folded maps with duplicate keys, see [[hasDuplicateMapKeys]]. */ private def expandComplexLiteral(expr: Literal): Option[Expression] = { - if (expr.value == null || !mapKeyTypesExpandable(expr.dataType)) return None + if (!canExpandComplexLiteral(expr)) return None expr.dataType match { - case ArrayType(et, containsNull) if needsExpansion(et) => + case ArrayType(et, containsNull) => val arr = expr.value.asInstanceOf[ArrayData] - if (arr.numElements() == 0) { - None - } else { - val elements = (0 until arr.numElements()) - .map(i => withNullability(literalAt(arr, i, et), containsNull)) - Some(CreateArray(elements, useStringTypeWhenEmpty = false)) - } + val elements = (0 until arr.numElements()) + .map(i => withNullability(literalAt(arr, i, et), containsNull)) + Some(CreateArray(elements, useStringTypeWhenEmpty = false)) case MapType(kt, vt, valueContainsNull) => val mapData = expr.value.asInstanceOf[MapData] val keys = mapData.keyArray() - if (mapData.numElements() == 0 || hasDuplicateMapKeys(keys, kt)) { - None - } else { - val values = mapData.valueArray() - val children = (0 until keys.numElements()).flatMap(i => - Seq( - literalAt(keys, i, kt), - withNullability(literalAt(values, i, vt), valueContainsNull))) - Some(CreateMap(children, useStringTypeWhenEmpty = false)) - } + val values = mapData.valueArray() + val children = (0 until keys.numElements()).flatMap(i => + Seq( + literalAt(keys, i, kt), + withNullability(literalAt(values, i, vt), valueContainsNull))) + Some(CreateMap(children, useStringTypeWhenEmpty = false)) case _ => None } } + /** + * Cheap admission test that mirrors [[expandComplexLiteral]] without materializing the rebuilt + * `Create*` tree, so `getSupportLevel` can probe a large folded literal without allocating the + * N `Literal`s `convert` would immediately rebuild. Declines a null value or empty top-level + * container (no children to recover the element type), a folded map with duplicate keys (see + * [[hasDuplicateMapKeys]]), any unsupported or non-orderable map key type at any nesting level + * (see [[mapKeyTypesExpandable]]), and an array of structs ([[needsExpansion]] stops at a + * `StructType`). [[expandComplexLiteral]] gates on this, so the two cannot diverge. + */ + private def canExpandComplexLiteral(expr: Literal): Boolean = { + if (expr.value == null || !mapKeyTypesExpandable(expr.dataType)) return false + expr.dataType match { + case ArrayType(et, _) if needsExpansion(et) => + expr.value.asInstanceOf[ArrayData].numElements() > 0 + case MapType(kt, _, _) => + val mapData = expr.value.asInstanceOf[MapData] + mapData.numElements() > 0 && !hasDuplicateMapKeys(mapData.keyArray(), kt) + case _ => false + } + } + /** * True when every map key type reachable inside `dataType` can be rebuilt and consumed * natively. A folded map is rebuilt as a `CreateMap`; once native it may reach a map consumer diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql index 320d4985a25..0acc1f25de3 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -98,6 +98,21 @@ SELECT array(array(map(1, 'a')), array(map(2, 'b'))) query SELECT array(named_struct('a', 1, 'b', 'x'), named_struct('a', 2, 'b', 'y')) +-- Array of structs differing only in a field's nullability: the first `ct` is a non-null literal, +-- the second is a nullable CASE. Spark compares element types with `sameType` (nullability ignored) +-- so it keeps distinct StructTypes; `CometCreateArray` casts each child to the merged struct type +-- (widening `ct` to nullable) before `make_array`, so this runs natively. +query +SELECT array( + named_struct('id', a, 'ct', 'x'), + named_struct('id', a, 'ct', CASE WHEN a = 0 THEN 'y' END)) FROM test_create_array + +-- Same nested-field-nullability divergence wrapped in a map value. +query +SELECT array( + map('k', named_struct('id', a, 'ct', 'x')), + map('k', named_struct('id', a, 'ct', CASE WHEN a = 0 THEN 'y' END))) FROM test_create_array + -- Column-based array of maps. Filter out NULL keys because Spark forbids them. query SELECT array(map(k, v)) FROM test_create_array_complex WHERE k IS NOT NULL diff --git a/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql b/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql index ded57f490e2..b9e505336e2 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/element_at_map.sql @@ -80,3 +80,19 @@ SELECT element_at(map(named_struct('a', 1), 7), named_struct('a', 1)) -- `BinaryType` keys need no decline: Arrow compares them by content, as Spark's ordering does. query SELECT element_at(map(CAST('a' AS BINARY), 1, CAST('b' AS BINARY), 2), CAST('b' AS BINARY)) + +-- Nested INT-keyed map: the inner `element_at` returns NULL for ids not in the outer map (2, 3), +-- and the outer `element_at` looks it up with a per-row key `id % (id - 2)`. This harness runs with +-- ANSI disabled, so the remainder-by-zero at id = 2 evaluates to NULL rather than throwing, and +-- `element_at(NULL_map, NULL)` returns NULL -- matching Spark's 7, NULL, NULL natively. (Under ANSI, +-- native scalar functions evaluate the key eagerly and throw where Spark short-circuits after the +-- NULL inner map; that is a pre-existing eager-evaluation difference in native `ElementAt`.) +statement +CREATE TABLE test_element_at_nested(id int) USING parquet + +statement +INSERT INTO test_element_at_nested VALUES (1), (2), (3) + +query +SELECT id, element_at(element_at(map(1, map(0, 7)), id), id % (id - 2)) AS v +FROM test_element_at_nested diff --git a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql index a6c145e9f3c..2d172f436f8 100644 --- a/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql +++ b/spark/src/test/resources/sql-tests/expressions/map/map_entries.sql @@ -52,3 +52,14 @@ SELECT array( map_entries(map(1, IF(id = 1, map(1, 2), NULL)))[0].value, map(2, coalesce(id, 0))) AS a FROM test_map_entries_nested + +-- `map_entries` declares its entry `value` field nullable, so the extracted entry struct has a +-- nullable `value` while the sibling `named_struct('key', 1, 'value', id IS NOT NULL)` has a +-- non-nullable one. The two struct children differ only in that field's nullability, so +-- `CometCreateArray` casts each to the merged struct type before `make_array` and this runs +-- natively rather than panicking on the struct-field mismatch. +query +SELECT array( + map_entries(element_at(map(1, map(1, true)), id))[0], + named_struct('key', 1, 'value', id IS NOT NULL)) AS a +FROM test_map_entries_nested diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index c72d94866c8..40505562300 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1134,48 +1134,12 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } } - test("array of structs with nullability-divergent children") { - // Spark's type coercion compares element types with `sameType`, which ignores nullability, - // so two struct children that differ ONLY in a nested field's nullability get no unifying - // cast -- CreateArray keeps children of different StructTypes. DataFusion's make_array asserts - // strict element-type equality (down to nested nullability) and panics on the mismatch. Comet - // must decline this CreateArray so Spark's evaluator handles it. - withParquetTable((0 until 5).map(i => (i, i.toLong)), "tbl") { - val df = spark - .table("tbl") - .select( - array( - // ct is NOT NULL (literal) - struct(col("_1").as("id"), lit("a").as("ct")), - // ct is NULLABLE (when without otherwise) -- same type, different nullability - struct(col("_1").as("id"), when(col("_1") === 0, lit("b")).as("ct"))).as("arr")) - checkSparkAnswerAndFallbackReason(df, "CreateArray children have mismatched data types") - } - } - - test("array of maps with nullability-divergent struct values") { - // Same nested-nullability divergence as the struct case, but wrapped in a MapType value so we - // exercise normalizeContainerNullability's MapType branch: the two map children share a surface - // type and differ only in a nested struct field's nullability, so they survive container - // (`MapType.valueContainsNull`) normalization as distinct types and CreateArray must still - // decline -- DataFusion's make_array would otherwise panic on the struct-field mismatch. - withParquetTable((0 until 5).map(i => (i, i.toLong)), "tbl") { - val df = spark - .table("tbl") - .select( - array( - // map value struct has ct NOT NULL (literal) - map(lit("k"), struct(col("_1").as("id"), lit("a").as("ct"))), - // map value struct has ct NULLABLE -- same type, different nested nullability - map(lit("k"), struct(col("_1").as("id"), when(col("_1") === 0, lit("b")).as("ct")))) - .as("arr")) - checkSparkAnswerAndFallbackReason(df, "CreateArray children have mismatched data types") - } - } - - // Constant folding is enabled by default here, so each `map(...)` collapses to a MapType - // Literal and the outer `array(...)` reaches `CometCreateArray` with folded-Literal children. - // `CometLiteral` rebuilds each folded map as an equivalent `CreateMap` of primitive literals. + // The tests below deliberately live in this suite, not a `sql-tests` fixture: constant folding is + // enabled by default here, so each `map(...)` collapses to a MapType Literal and the outer + // `array(...)` reaches `CometCreateArray` with folded-Literal children, which `CometLiteral` + // rebuilds as an equivalent `CreateMap` of primitive literals -- the folded-literal expansion path + // under review. `CometSqlFileTestSuite` force-disables `ConstantFolding`, so an equivalent SQL + // fixture would only exercise the constructor path (which `create_array.sql` already covers). test("array of folded map literals with array values (multirow)") { withParquetTable((0 until 3).map(i => (i, i.toLong)), "tbl") { checkSparkAnswerAndOperator( diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 74e88231c54..7311ef38d14 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -272,6 +272,15 @@ class CometMapExpressionSuite extends CometTestBase { } } + // ============================================================================================== + // Folded complex-literal tests. These live in this suite, not a `sql-tests` fixture, on purpose: + // they need `ConstantFolding` ON so `map(...)` / `array(...)` collapses to a `Literal` that + // `CometLiteral` then rebuilds -- the folded-literal expansion path under review. The SQL harness + // (`CometSqlFileTestSuite`) force-disables `ConstantFolding`, so an equivalent fixture would only + // exercise the constructor path. The `*.sql` files cover that constructor path where the same + // guard fires regardless of folding. + // ============================================================================================== + // A map that reaches `map_entries` through an expression rather than a scan keeps // `valueContainsNull = false`, which every `map(...)` over non-null values produces. DataFusion's // `map_entries` declares the entry `value` field nullable but reuses the input map's entries @@ -428,4 +437,17 @@ class CometMapExpressionSuite extends CometTestBase { } } + // A large folded map (`map_from_arrays(sequence(...), sequence(...))` collapses to a many-entry + // MapType literal) rebuilds as a big `CreateMap` whose generated code Spark's codegen splits into + // nested helper classes. Those helpers read `CometBatchKernel.references`, which must be public + // (not protected) or the split code raises `IllegalAccessError` at runtime. 100k entries is large + // enough to force the split. + test("large folded map literal in element_at runs natively (multirow)") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, element_at(" + + "map_from_arrays(sequence(1, 100000), sequence(1, 100000)), _1) AS v FROM tbl") + } + } + } From a3aa0136a3de0d1d593cffec329979503e91fff8 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 27 Aug 2026 09:32:53 -0700 Subject: [PATCH 09/19] feat: support Map for `CreateArray` literal --- .../src/struct_funcs/create_named_struct.rs | 24 +++- .../scala/org/apache/comet/serde/arrays.scala | 131 ++++++++++++++---- .../org/apache/comet/serde/literals.scala | 2 +- .../expressions/array/array_contains.sql | 33 +++++ .../expressions/array/array_insert.sql | 17 +++ .../expressions/array/create_array.sql | 8 ++ .../expressions/map/element_at_map_ansi.sql | 36 +++++ 7 files changed, 218 insertions(+), 33 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/map/element_at_map_ansi.sql diff --git a/native/spark-expr/src/struct_funcs/create_named_struct.rs b/native/spark-expr/src/struct_funcs/create_named_struct.rs index 1a63cb1cc95..90e9684d651 100644 --- a/native/spark-expr/src/struct_funcs/create_named_struct.rs +++ b/native/spark-expr/src/struct_funcs/create_named_struct.rs @@ -18,7 +18,7 @@ use arrow::array::StructArray; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; -use datafusion::common::Result as DataFusionResult; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::{ @@ -71,13 +71,25 @@ impl PhysicalExpr for CreateNamedStruct { .iter() .map(|expr| expr.evaluate(batch)) .collect::>>()?; + // When every field value is a scalar (e.g. an all-literal `named_struct` that reaches native + // as a `CreateNamedStruct` because constant folding is disabled), return a scalar struct + // rather than a length-1 `StructArray`. A downstream consumer such as `make_array` then + // broadcasts the constant struct to the batch row count instead of failing with a + // mixed-length error when a sibling child is a full-length column. This matches what a + // constant-folded struct literal produces, and `GetStructField::evaluate` already handles a + // scalar struct input. + let all_scalar = + !values.is_empty() && values.iter().all(|v| matches!(v, ColumnarValue::Scalar(_))); let arrays = ColumnarValue::values_to_arrays(&values)?; let fields = self.fields(&batch.schema())?; - Ok(ColumnarValue::Array(Arc::new(StructArray::new( - fields.into(), - arrays, - None, - )))) + let struct_array = StructArray::new(fields.into(), arrays, None); + if all_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::Struct(Arc::new( + struct_array, + )))) + } else { + Ok(ColumnarValue::Array(Arc::new(struct_array))) + } } fn children(&self) -> Vec<&Arc> { diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 8800d7512e0..487185aa839 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -29,6 +29,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.withFallbackReason +import org.apache.comet.DataTypeSupport.isComplexType import org.apache.comet.serde.QueryPlanSerde._ import org.apache.comet.shims.{CometExprShim, CometTypeShim} @@ -100,7 +101,27 @@ object CometArrayAppend extends CometExpressionSerde[ArrayAppend] { } } -object CometArrayContains extends CometExpressionSerde[ArrayContains] { +object CometArrayContains + extends CometExpressionSerde[ArrayContains] + with CodegenDispatchFallback { + + private val floatingPointReason: String = + "Spark compares array elements with ordering.equiv, so -0.0 matches +0.0 and all NaNs match " + + "each other; Comet's native array_contains compares the raw Arrow values bitwise" + + override def getIncompatibleReasons(): Seq[String] = Seq(floatingPointReason) + + override def getSupportLevel(expr: ArrayContains): SupportLevel = expr.left.dataType match { + // Native array_contains compares floating-point elements bitwise, disagreeing with Spark for + // -0.0/+0.0 and NaN. Report Incompatible (not Unsupported) for float/double element types (at + // any nesting level) so the expression routes through the JVM codegen dispatcher (Spark's own + // doGenCode) and stays native + Spark-exact under the default config, while non-float arrays + // keep the fast native kernel. Under allowIncompatible=true the native kernel is used as before. + case ArrayType(elementType, _) + if SupportLevel.containsType(elementType, classOf[FloatType], classOf[DoubleType]) => + Incompatible(Some(floatingPointReason)) + case _ => Compatible() + } override def convert( expr: ArrayContains, @@ -384,7 +405,7 @@ object CometArrayJoin } } -object CometArrayInsert extends CometExpressionSerde[ArrayInsert] { +object CometArrayInsert extends CometExpressionSerde[ArrayInsert] with ArraysBase { override def getSupportLevel(expr: ArrayInsert): SupportLevel = Compatible() @@ -392,9 +413,31 @@ object CometArrayInsert extends CometExpressionSerde[ArrayInsert] { expr: ArrayInsert, inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { - val srcExprProto = exprToProtoInternal(expr.children.head, inputs, binding) + val srcArray = expr.children.head + val item = expr.children(2) + val srcElementType = srcArray.dataType.asInstanceOf[ArrayType].elementType + + // Native ArrayInsert requires the item's Arrow type to equal the source array's element type + // exactly, including nested nullability. For complex element types the two sides can disagree: + // a `CreateArray` source is widened to a deeply-nullable element type (see CometCreateArray), + // while a standalone item (e.g. `map(2, coalesce(id, 0))`) keeps Spark's Catalyst nullability. + // Cast BOTH sides in lockstep to the same deeply-nullable element type so their Arrow types + // match; Spark's `ArrayInsert.dataType` is `first.dataType.asNullable`, so this also matches the + // declared output element type. Casting only widens metadata and never changes values. Primitive + // element types are byte-identical on both sides, so the gate leaves them untouched. + val (srcChild, itemChild) = if (isComplexType(srcElementType)) { + val elementType = deepNullable(srcElementType) + val arrayType = ArrayType(elementType, containsNull = true) + val widenedSrc = if (srcArray.dataType == arrayType) srcArray else Cast(srcArray, arrayType) + val widenedItem = if (item.dataType == elementType) item else Cast(item, elementType) + (widenedSrc, widenedItem) + } else { + (srcArray, item) + } + + val srcExprProto = exprToProtoInternal(srcChild, inputs, binding) val posExprProto = exprToProtoInternal(expr.children(1), inputs, binding) - val itemExprProto = exprToProtoInternal(expr.children(2), inputs, binding) + val itemExprProto = exprToProtoInternal(itemChild, inputs, binding) val legacyNegativeIndex = SQLConf.get.getConfString("spark.sql.legacy.negativeIndexInArrayInsert").toBoolean if (srcExprProto.isDefined && posExprProto.isDefined && itemExprProto.isDefined) { @@ -455,7 +498,7 @@ object CometArrayUnion extends CometExpressionSerde[ArrayUnion] { } } -object CometCreateArray extends CometExpressionSerde[CreateArray] { +object CometCreateArray extends CometExpressionSerde[CreateArray] with ArraysBase { override def convert( expr: CreateArray, inputs: Seq[Attribute], @@ -493,24 +536,6 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] { None } } - - /** - * A copy of `dt` with every array `containsNull`, map `valueContainsNull`, and struct field - * nullability forced to `true` at every nesting level (map key fields stay non-null per Arrow's - * map invariant). This is Spark's `DataType.asNullable`, re-derived here because that method is - * `private[spark]` and unreachable from this package. Used as the common cast target for - * `CometCreateArray` so that children whose native runtime types are more nullable than Spark's - * Catalyst types (e.g. a `map_entries` entry struct, whose `value` field Comet forces nullable) - * still unify for `make_array`. - */ - private def deepNullable(dt: DataType): DataType = dt match { - case ArrayType(et, _) => ArrayType(deepNullable(et), containsNull = true) - case MapType(kt, vt, _) => - MapType(deepNullable(kt), deepNullable(vt), valueContainsNull = true) - case StructType(fields) => - StructType(fields.map(f => f.copy(dataType = deepNullable(f.dataType), nullable = true))) - case other => other - } } object CometGetArrayItem extends CometExpressionSerde[GetArrayItem] { @@ -596,10 +621,9 @@ object CometElementAt extends CometExpressionSerde[ElementAt] { val childExpr = exprToProtoInternal(expr.left, inputs, binding) val ordinalExpr = exprToProtoInternal(expr.right, inputs, binding) - expr.left.dataType match { + val baseExpr = expr.left.dataType match { case _: MapType => - val mapExtractExpr = scalarFunctionExprToProto("map_extract", childExpr, ordinalExpr) - mapExtractExpr + scalarFunctionExprToProto("map_extract", childExpr, ordinalExpr) case _ => val defaultExpr = expr.defaultValueOutOfBound.flatMap(exprToProtoInternal(_, inputs, binding)) @@ -625,6 +649,42 @@ object CometElementAt extends CometExpressionSerde[ElementAt] { None } } + + // Spark's ElementAt is a BinaryExpression: for a NULL map/array it returns NULL WITHOUT + // evaluating the key/index child. Native scalar functions evaluate the key eagerly over the + // whole batch, so under ANSI (failOnError) a throwing key (e.g. a divide-by-zero) fires even on + // rows whose map/array is NULL, where Spark short-circuits. When the left can actually be NULL, + // guard the lookup with CASE WHEN left IS NOT NULL THEN ELSE null so the key is only + // evaluated on the selected rows (DataFusion's CaseExpr filters the batch before the THEN + // branch), reproducing the short-circuit. Mirrors the CASE-WHEN idiom in CometArrayAppend / + // CometSize; the ELSE null literal carries the result type, as in CometArraysZip. + if (expr.failOnError && expr.left.nullable) { + val isNotNullExpr = createUnaryExpr( + expr, + expr.left, + inputs, + binding, + (builder, unaryExpr) => builder.setIsNotNull(unaryExpr)) + val nullLiteralProto = exprToProto(Literal(null, expr.dataType), Seq.empty) + for { + base <- baseExpr + notNull <- isNotNullExpr + nullLit <- nullLiteralProto + } yield { + val caseWhenExpr = ExprOuterClass.CaseWhen + .newBuilder() + .addWhen(notNull) + .addThen(base) + .setElseExpr(nullLit) + .build() + ExprOuterClass.Expr + .newBuilder() + .setCaseWhen(caseWhenExpr) + .build() + } + } else { + baseExpr + } } } @@ -841,6 +901,25 @@ trait ArraysBase { .collectFirst { case dt if !isTypeSupported(dt) => dt } .map(dt => Unsupported(Some(s"data type not supported: $dt"))) .getOrElse(Compatible()) + + /** + * A copy of `dt` with every array `containsNull`, map `valueContainsNull`, and struct field + * nullability forced to `true` at every nesting level (map key fields stay non-null per Arrow's + * map invariant). This is Spark's `DataType.asNullable`, re-derived here because that method is + * `private[spark]` and unreachable from this package. Used as a common cast target so that + * children whose native runtime types are more nullable than Spark's Catalyst types (e.g. a + * `map_entries` entry struct, whose `value` field Comet forces nullable) still unify for + * `make_array` (`CometCreateArray`) and for the strict source-element/item type check in native + * `ArrayInsert` (`CometArrayInsert`). + */ + def deepNullable(dt: DataType): DataType = dt match { + case ArrayType(et, _) => ArrayType(deepNullable(et), containsNull = true) + case MapType(kt, vt, _) => + MapType(deepNullable(kt), deepNullable(vt), valueContainsNull = true) + case StructType(fields) => + StructType(fields.map(f => f.copy(dataType = deepNullable(f.dataType), nullable = true))) + case other => other + } } object CometArrayTransform extends CometCodegenDispatch[ArrayTransform] diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index d50a0af5d60..a951ae28577 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -373,7 +373,7 @@ object CometLiteral extends CometExpressionSerde[Literal] with Logging { } else { val ordering = TypeUtils.getInterpretedOrdering(keyType) val sorted = (0 until n).map(i => keys.get(i, keyType)).sorted(ordering) - sorted.sliding(2).exists(pair => ordering.compare(pair.head, pair.last) == 0) + (1 until sorted.length).exists(i => ordering.compare(sorted(i - 1), sorted(i)) == 0) } } diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql index d00c5f55706..3c28a976367 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql @@ -56,3 +56,36 @@ SELECT array_contains(cast(NULL as array), cast(NULL as int)) -- NULL array with column value query SELECT array_contains(cast(NULL as array), val) FROM test_array_contains + +-- ============================================================ +-- Floating-point elements: Spark compares with ordering.equiv, so -0.0 == +0.0 and NaN == NaN. +-- Native array_contains compares raw Arrow values bitwise, so float/double element types route +-- through the JVM codegen dispatcher (Spark's own doGenCode) to stay native and Spark-exact. +-- ============================================================ + +statement +CREATE TABLE test_array_contains_fp(arr array, val double) USING parquet + +statement +INSERT INTO test_array_contains_fp VALUES + (array(0.0D, 1.0D), -0.0D), + (array(-0.0D, 2.0D), 0.0D), + (array(cast('NaN' as double), 3.0D), cast('NaN' as double)), + (array(1.0D, 2.0D), 5.0D), + (array(1.0D, NULL), cast(NULL as double)), + (array(1.0D, NULL), 4.0D), + (NULL, 1.0D) +query +SELECT array_contains(arr, val) FROM test_array_contains_fp + +-- literal forms: -0.0 vs +0.0 and NaN vs NaN, for DOUBLE and FLOAT +query +SELECT array_contains(array(-0.0D, 1.0D), 0.0D), + array_contains(array(0.0D, 1.0D), -0.0D), + array_contains(array(cast('NaN' as double), 1.0D), cast('NaN' as double)), + array_contains(array(cast('-0.0' as float), 1.0F), cast('0.0' as float)), + array_contains(array(cast('NaN' as float), 1.0F), cast('NaN' as float)) + +-- nested float element: the element type is array, so the float decline must recurse +query +SELECT array_contains(array(array(0.0D), array(1.0D)), array(-0.0D)) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql b/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql index c00dd82fff7..56df1fcccbc 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql @@ -280,3 +280,20 @@ SELECT array_insert(array(CAST(1 AS TINYINT), CAST(2 AS TINYINT)), 2, CAST(3 AS query SELECT array_insert(array(CAST(1.1 AS FLOAT), CAST(2.2 AS FLOAT)), 2, CAST(3.3 AS FLOAT)) + +-- ============================================================ +-- Array of maps (complex element type) +-- The source array's map value is widened to nullable by CometCreateArray, while the standalone +-- item map keeps value non-null because coalesce(id, 0) is non-null. Native ArrayInsert requires +-- the item Arrow type to equal the source element type exactly, so the serde casts both sides to a +-- common deeply-nullable element type. id includes a NULL row (coalesce yields 0). +-- ============================================================ + +statement +CREATE TABLE test_array_insert_map(id int) USING parquet + +statement +INSERT INTO test_array_insert_map VALUES (1), (2), (NULL) + +query +SELECT array_insert(array(map(1, coalesce(id, 0))), 2, map(2, coalesce(id, 0))) FROM test_array_insert_map diff --git a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql index 0acc1f25de3..33e37b6765f 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/create_array.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/create_array.sql @@ -129,6 +129,14 @@ SELECT array(arr, array(k, v)) FROM test_create_array_complex query SELECT array(map(k, arr)) FROM test_create_array_complex WHERE k IS NOT NULL +-- Array combining an all-literal struct child with a column-referencing struct child. The SQL +-- harness always excludes ConstantFolding, so the literal `named_struct('a', 1)` stays a +-- `CreateNamedStruct` whose native evaluation is a constant (row-count-independent) struct. It must +-- broadcast to the batch row count to sit alongside the column-based `named_struct('a', a)` inside +-- `make_array`, which otherwise rejects the length-1 vs length-N mismatch. +query +SELECT array(named_struct('a', 1), named_struct('a', a)) FROM test_create_array + -- Empty array cast to a nested array type: the element type has to survive with no children. query SELECT CAST(array() AS ARRAY>) diff --git a/spark/src/test/resources/sql-tests/expressions/map/element_at_map_ansi.sql b/spark/src/test/resources/sql-tests/expressions/map/element_at_map_ansi.sql new file mode 100644 index 00000000000..1d1bececfbe --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/map/element_at_map_ansi.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ANSI-mode element_at over maps. The non-ANSI forms live in element_at_map.sql. + +-- Config: spark.sql.ansi.enabled=true + +statement +CREATE TABLE test_element_at_nested_ansi(id int) USING parquet + +statement +INSERT INTO test_element_at_nested_ansi VALUES (1), (2), (3) + +-- Nested INT-keyed map lookup with a per-row key. The inner element_at returns NULL for ids not in +-- the outer map (2, 3); the outer element_at looks it up with `id % (id - 2)`, which is 2 % 0 at +-- id = 2. Spark's ElementAt is a BinaryExpression that short-circuits the NULL inner map and never +-- evaluates the throwing key, so under ANSI it returns 7, NULL, NULL. CometElementAt reproduces the +-- short-circuit natively with a CASE WHEN IS NOT NULL guard, so the key runs only where the +-- map is non-null and the query runs with native acceleration. +query +SELECT id, element_at(element_at(map(1, map(0, 7)), id), id % (id - 2)) AS v +FROM test_element_at_nested_ansi From 4fe95db56c9939100b9a78825b5ecd86c1beb2a8 Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 27 Aug 2026 10:55:53 -0700 Subject: [PATCH 10/19] feat: support Map for `CreateArray` literal --- .../src/main/scala/org/apache/comet/serde/arrays.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 487185aa839..ddfff750120 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -116,7 +116,8 @@ object CometArrayContains // -0.0/+0.0 and NaN. Report Incompatible (not Unsupported) for float/double element types (at // any nesting level) so the expression routes through the JVM codegen dispatcher (Spark's own // doGenCode) and stays native + Spark-exact under the default config, while non-float arrays - // keep the fast native kernel. Under allowIncompatible=true the native kernel is used as before. + // keep the fast native kernel. Under allowIncompatible=true the native kernel is used + // as before. case ArrayType(elementType, _) if SupportLevel.containsType(elementType, classOf[FloatType], classOf[DoubleType]) => Incompatible(Some(floatingPointReason)) @@ -422,9 +423,10 @@ object CometArrayInsert extends CometExpressionSerde[ArrayInsert] with ArraysBas // a `CreateArray` source is widened to a deeply-nullable element type (see CometCreateArray), // while a standalone item (e.g. `map(2, coalesce(id, 0))`) keeps Spark's Catalyst nullability. // Cast BOTH sides in lockstep to the same deeply-nullable element type so their Arrow types - // match; Spark's `ArrayInsert.dataType` is `first.dataType.asNullable`, so this also matches the - // declared output element type. Casting only widens metadata and never changes values. Primitive - // element types are byte-identical on both sides, so the gate leaves them untouched. + // match; Spark's `ArrayInsert.dataType` is `first.dataType.asNullable`, so this also + // matches the declared output element type. Casting only widens metadata and never + // changes values. Primitive element types are byte-identical on both sides, so the gate + // leaves them untouched. val (srcChild, itemChild) = if (isComplexType(srcElementType)) { val elementType = deepNullable(srcElementType) val arrayType = ArrayType(elementType, containsNull = true) From a9b24c3563f36114043d78b7eed69c5c2ab6dbfe Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 27 Aug 2026 11:02:20 -0700 Subject: [PATCH 11/19] feat: support Map for `CreateArray` literal --- .../org/apache/comet/DataTypeSupport.scala | 15 +++++++++++++ .../scala/org/apache/comet/serde/arrays.scala | 21 +------------------ 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala index 019b4d629b9..8594d9558dc 100644 --- a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala +++ b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala @@ -80,6 +80,21 @@ object DataTypeSupport { case _ => false } + /** + * `dt` with every array/map/struct nullability flag forced to `true` at all nesting levels (map + * key fields stay non-null per Arrow's map invariant). Re-derives Spark's `private[spark]` + * `DataType.asNullable`, used as a common cast target to unify types whose Comet runtime + * nullability exceeds Spark's Catalyst nullability. + */ + def deepNullable(dt: DataType): DataType = dt match { + case ArrayType(et, _) => ArrayType(deepNullable(et), containsNull = true) + case MapType(kt, vt, _) => + MapType(deepNullable(kt), deepNullable(vt), valueContainsNull = true) + case StructType(fields) => + StructType(fields.map(f => f.copy(dataType = deepNullable(f.dataType), nullable = true))) + case other => other + } + def hasTemporalType(t: DataType): Boolean = t match { case DataTypes.DateType | DataTypes.TimestampType | DataTypes.TimestampNTZType => true diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index ddfff750120..5d8e33412f7 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.withFallbackReason -import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.DataTypeSupport.{deepNullable, isComplexType} import org.apache.comet.serde.QueryPlanSerde._ import org.apache.comet.shims.{CometExprShim, CometTypeShim} @@ -903,25 +903,6 @@ trait ArraysBase { .collectFirst { case dt if !isTypeSupported(dt) => dt } .map(dt => Unsupported(Some(s"data type not supported: $dt"))) .getOrElse(Compatible()) - - /** - * A copy of `dt` with every array `containsNull`, map `valueContainsNull`, and struct field - * nullability forced to `true` at every nesting level (map key fields stay non-null per Arrow's - * map invariant). This is Spark's `DataType.asNullable`, re-derived here because that method is - * `private[spark]` and unreachable from this package. Used as a common cast target so that - * children whose native runtime types are more nullable than Spark's Catalyst types (e.g. a - * `map_entries` entry struct, whose `value` field Comet forces nullable) still unify for - * `make_array` (`CometCreateArray`) and for the strict source-element/item type check in native - * `ArrayInsert` (`CometArrayInsert`). - */ - def deepNullable(dt: DataType): DataType = dt match { - case ArrayType(et, _) => ArrayType(deepNullable(et), containsNull = true) - case MapType(kt, vt, _) => - MapType(deepNullable(kt), deepNullable(vt), valueContainsNull = true) - case StructType(fields) => - StructType(fields.map(f => f.copy(dataType = deepNullable(f.dataType), nullable = true))) - case other => other - } } object CometArrayTransform extends CometCodegenDispatch[ArrayTransform] From a99951a657dbb80f3dab90e816452f3ad9cd4e8a Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 27 Aug 2026 13:53:15 -0700 Subject: [PATCH 12/19] feat: support Map for `CreateArray` literal --- .../sql-tests/expressions/array/array_contains.sql | 1 + .../sql-tests/expressions/array/element_at.sql | 13 +++++++++++-- .../sql-tests/expressions/array/try_element_at.sql | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql index 3c28a976367..e0fb8709b93 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_contains.sql @@ -75,6 +75,7 @@ INSERT INTO test_array_contains_fp VALUES (array(1.0D, NULL), cast(NULL as double)), (array(1.0D, NULL), 4.0D), (NULL, 1.0D) + query SELECT array_contains(arr, val) FROM test_array_contains_fp diff --git a/spark/src/test/resources/sql-tests/expressions/array/element_at.sql b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql index 438a20cd593..fb0a28f16dc 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/element_at.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql @@ -21,9 +21,18 @@ CREATE TABLE test_element_at(arr array) USING parquet statement INSERT INTO test_element_at VALUES (array(1, 2, 3)), (array(10)), (NULL) -query spark_answer_only +query SELECT element_at(arr, 1), element_at(arr, -1) FROM test_element_at -- literal arguments -query ignore(Spark codegen bug with literal element_at when constant folding is disabled) +query SELECT element_at(array(1, 2, 3), 1), element_at(array(1, 2, 3), -1) + +statement +CREATE TABLE t(id int) USING parquet + +statement +INSERT INTO t VALUES (1), (2), (3), (-1), (NULL) + +query +SELECT element_at(element_at(array(CAST(NULL AS ARRAY),array(monotonically_increasing_id(), CAST(NULL AS BIGINT))),id),id - 1) FROM t diff --git a/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql b/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql index 357de991f66..e9becb24744 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql @@ -43,7 +43,7 @@ query SELECT try_element_at(CAST(NULL AS ARRAY), 1) -- literal array arguments: same codegen bug as element_at with literal arrays -query ignore(Spark codegen bug with literal element_at when constant folding is disabled) +query SELECT try_element_at(array(10, 20, 30), 1), try_element_at(array(10, 20, 30), 99) -- map input From 1f63556a115dcb0d70c7df90c0e62e34b237838b Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 27 Aug 2026 17:27:15 -0700 Subject: [PATCH 13/19] feat: support Map for `CreateArray` literal --- .../resources/sql-tests/expressions/array/element_at.sql | 7 +++---- .../sql-tests/expressions/array/try_element_at.sql | 9 +++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/spark/src/test/resources/sql-tests/expressions/array/element_at.sql b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql index fb0a28f16dc..f32e453a182 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/element_at.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/element_at.sql @@ -24,15 +24,14 @@ INSERT INTO test_element_at VALUES (array(1, 2, 3)), (array(10)), (NULL) query SELECT element_at(arr, 1), element_at(arr, -1) FROM test_element_at --- literal arguments -query -SELECT element_at(array(1, 2, 3), 1), element_at(array(1, 2, 3), -1) - statement CREATE TABLE t(id int) USING parquet statement INSERT INTO t VALUES (1), (2), (3), (-1), (NULL) +query +SELECT element_at(array(1, 2, 3), id), element_at(array(1, 2, 3), -id) FROM t + query SELECT element_at(element_at(array(CAST(NULL AS ARRAY),array(monotonically_increasing_id(), CAST(NULL AS BIGINT))),id),id - 1) FROM t diff --git a/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql b/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql index e9becb24744..05200a254d0 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/try_element_at.sql @@ -42,9 +42,14 @@ SELECT try_element_at(arr, 100) FROM test_try_element_at query SELECT try_element_at(CAST(NULL AS ARRAY), 1) --- literal array arguments: same codegen bug as element_at with literal arrays +statement +CREATE TABLE t(id int) USING parquet + +statement +INSERT INTO t VALUES (1), (3), (-1), (99), (NULL) + query -SELECT try_element_at(array(10, 20, 30), 1), try_element_at(array(10, 20, 30), 99) +SELECT try_element_at(array(10, 20, 30), id) FROM t -- map input query From 3d35f16e30c0a6fba9bb33bb45ff0016cefe1034 Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 28 Aug 2026 12:52:21 -0700 Subject: [PATCH 14/19] feat: support Map for `CreateArray` literal --- .../apache/comet/serde/QueryPlanSerde.scala | 23 +++++++++++---- .../scala/org/apache/comet/serde/arrays.scala | 9 +++++- .../comet/CometMapExpressionSuite.scala | 28 +++++++++++++------ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 6802dfaa646..0efda0ee73a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -531,6 +531,22 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { } } + /** + * Attach a fresh `expr_id` and, when the expression's origin carries one, its `QueryContext` to + * `protoExpr`. Native ANSI errors resolve their context by the `expr_id` of the `Expr` that + * throws (see `register_query_context` and `ListExtract` in the native planner), so the metadata + * has to sit on that `Expr`. The generic serde path applies this to the top-level `Expr` it + * returns; a serde that nests a throwing expression inside a wrapper (e.g. `CometElementAt`'s + * CASE-WHEN NULL guard) must also call it on the inner `Expr`, or that expression's error renders + * without Spark's `== SQL ... ==` query context. + */ + private[serde] def attachExprIdAndContext(expr: Expression, protoExpr: Expr): Expr = { + val builder = protoExpr.toBuilder + builder.setExprId(nextExprId()) + extractQueryContext(expr).foreach(builder.setQueryContext) + builder.build() + } + def supportedDataType(dt: DataType, allowComplex: Boolean = false): Boolean = dt match { case _: ByteType | _: ShortType | _: IntegerType | _: LongType | _: FloatType | _: DoubleType | _: StringType | _: BinaryType | _: TimestampType | _: TimestampNTZType | @@ -1004,12 +1020,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { withNativeExpr(expr, CometExplainInfo.exprDisplayName(expr)) } // Attach QueryContext and expr_id to the expression - val builder = protoExpr.toBuilder - builder.setExprId(nextExprId()) - extractQueryContext(expr).foreach { ctx => - builder.setQueryContext(ctx) - } - builder.build() + attachExprIdAndContext(expr, protoExpr) } } diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 5d8e33412f7..63860c629cc 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -673,10 +673,17 @@ object CometElementAt extends CometExpressionSerde[ElementAt] { notNull <- isNotNullExpr nullLit <- nullLiteralProto } yield { + // The generic serde path attaches this ElementAt's expr_id and QueryContext to the CASE we + // return here, but the lookup that actually throws under ANSI (ListExtract, for the array + // case) is nested inside the THEN branch. Attach them to that inner Expr too, so a native + // INVALID_ARRAY_INDEX_IN_ELEMENT_AT / INVALID_INDEX_OF_ZERO error still renders Spark's + // `== SQL ... ==` query context. (map_extract ignores expr_id, so the map case is + // unaffected and never throws an index error anyway.) + val guardedBase = QueryPlanSerde.attachExprIdAndContext(expr, base) val caseWhenExpr = ExprOuterClass.CaseWhen .newBuilder() .addWhen(notNull) - .addThen(base) + .addThen(guardedBase) .setElseExpr(nullLit) .build() ExprOuterClass.Expr diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 7311ef38d14..f3fba58c605 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -23,6 +23,7 @@ import scala.util.Random import org.apache.hadoop.fs.Path import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.catalyst.expressions.ArrayContains import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.BinaryType @@ -352,18 +353,27 @@ class CometMapExpressionSuite extends CometTestBase { } } - // `map_contains_key` lowers to `array_contains(map_keys(...), key)`; neither `map_keys` nor - // `array_contains` gates the key type, so a folded map with floating-point keys would reach them - // and compare `-0.0` against a normalized `+0.0` key bytewise, unlike Spark. The double-keyed map - // is nested inside an INT-keyed outer map, so the lookup guard on the outer `element_at` does not - // see it; expansion has to inspect nested map key types and decline. Spark returns `7, NULL, NULL`. + // `map_contains_key` lowers to `array_contains(map_keys(...), key)`. `CometArrayContains` reports + // Incompatible for floating-point element types, because native `array_contains` compares + // -0.0/+0.0 and NaN bitwise unlike Spark. So under the default config it codegen-dispatches to + // Spark and the query stays native with Spark-exact results and no fallback. Forcing the native + // kernel with `ArrayContains.allowIncompatible=true` makes it serialize its children instead, so + // the folded double-keyed map literal reaches the literal-expansion path. That map is nested + // inside the INT-keyed outer map, past the outer `element_at` key guard which only sees the INT + // key. The nested-key walk in the expansion (`mapKeyTypesExpandable`) is what has to decline the + // floating-point keys. Otherwise a rebuilt `CreateMap` would reach the native kernel whose key + // equality disagrees with Spark. Spark returns `7, NULL, NULL`. test("map_contains_key over nested floating-point map keys falls back (multirow)") { withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { val negZero = "CAST(concat('-', CAST(_1 - 1 AS STRING), '.0') AS DOUBLE)" - checkSparkAnswerAndFallbackReason( - "SELECT _1 AS id, map_contains_key(" + - s"element_at(map(1, map(CAST(0 AS DOUBLE), 7)), _1), $negZero) AS present FROM tbl", - "Unsupported data type MapType") + // allowIncompatible flips array_contains from codegen dispatch to the native kernel, so the + // nested-map literal is actually serialized and the expansion guard runs. + withSQLConf(CometConf.getExprAllowIncompatConfigKey(classOf[ArrayContains]) -> "true") { + checkSparkAnswerAndFallbackReason( + "SELECT _1 AS id, map_contains_key(" + + s"element_at(map(1, map(CAST(0 AS DOUBLE), 7)), _1), $negZero) AS present FROM tbl", + "Unsupported data type MapType") + } } } From 67fbae59727d108dda1a875e5f00c33a545855e4 Mon Sep 17 00:00:00 2001 From: comphead Date: Fri, 28 Aug 2026 13:38:15 -0700 Subject: [PATCH 15/19] feat: support Map for `CreateArray` literal --- .../scala/org/apache/comet/serde/QueryPlanSerde.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 0efda0ee73a..1268c66957d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -534,11 +534,11 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { /** * Attach a fresh `expr_id` and, when the expression's origin carries one, its `QueryContext` to * `protoExpr`. Native ANSI errors resolve their context by the `expr_id` of the `Expr` that - * throws (see `register_query_context` and `ListExtract` in the native planner), so the metadata - * has to sit on that `Expr`. The generic serde path applies this to the top-level `Expr` it - * returns; a serde that nests a throwing expression inside a wrapper (e.g. `CometElementAt`'s - * CASE-WHEN NULL guard) must also call it on the inner `Expr`, or that expression's error renders - * without Spark's `== SQL ... ==` query context. + * throws (see `register_query_context` and `ListExtract` in the native planner), so the + * metadata has to sit on that `Expr`. The generic serde path applies this to the top-level + * `Expr` it returns; a serde that nests a throwing expression inside a wrapper (e.g. + * `CometElementAt`'s CASE-WHEN NULL guard) must also call it on the inner `Expr`, or that + * expression's error renders without Spark's `== SQL ... ==` query context. */ private[serde] def attachExprIdAndContext(expr: Expression, protoExpr: Expr): Expr = { val builder = protoExpr.toBuilder From f6057825cd6644cea51a3f775529d60338b6ec57 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Sat, 29 Aug 2026 16:10:31 +0000 Subject: [PATCH 16/19] fix: preserve lazy map and array lookup evaluation Evaluate the collection once and evaluate the key only for non-null rows. This covers element_at, bracket lookup, and try_element_at in both ANSI modes without duplicating stateful operands through a CASE expression. Keep the existing map lookup and type coercion, preserve scalar broadcasting, and mask null-map rows. Add native and Spark regressions for throwing keys, stateful operands, null inputs, and batch shape. --- .../src/execution/expressions/map_extract.rs | 448 ++++++++++++++++++ native/core/src/execution/expressions/mod.rs | 1 + native/core/src/execution/planner.rs | 10 +- .../src/array_funcs/list_extract.rs | 206 +++++++- .../apache/comet/serde/QueryPlanSerde.scala | 5 +- .../scala/org/apache/comet/serde/arrays.scala | 44 +- .../comet/CometArrayExpressionSuite.scala | 35 ++ .../comet/CometMapExpressionSuite.scala | 57 ++- 8 files changed, 739 insertions(+), 67 deletions(-) create mode 100644 native/core/src/execution/expressions/map_extract.rs diff --git a/native/core/src/execution/expressions/map_extract.rs b/native/core/src/execution/expressions/map_extract.rs new file mode 100644 index 00000000000..35beeebce50 --- /dev/null +++ b/native/core/src/execution/expressions/map_extract.rs @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use arrow::array::{new_null_array, BooleanArray, RecordBatch}; +use arrow::compute::{not, nullif}; +use arrow::datatypes::{FieldRef, Schema}; +use datafusion::common::config::ConfigOptions; +use datafusion::common::{internal_err, Result}; +use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs}; +use datafusion::physical_expr::{PhysicalExpr, ScalarFunctionExpr}; + +/// Preserve Spark's left-to-right null short-circuit while using DataFusion's map lookup. +/// The planner still wraps the returned list in `ListExtract` to produce a single value. +#[derive(Debug)] +pub struct MapExtractExpr { + function: ScalarFunctionExpr, + config_options: Arc, +} + +impl MapExtractExpr { + pub fn try_new(function: ScalarFunctionExpr) -> Result { + if function.args().len() != 2 { + return internal_err!("MapExtractExpr expects exactly two arguments"); + } + let config_options = Arc::new(function.config_options().clone()); + Ok(Self { + function, + config_options, + }) + } +} + +impl Display for MapExtractExpr { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.function, f) + } +} + +impl PartialEq for MapExtractExpr { + fn eq(&self, other: &Self) -> bool { + self.function.eq(&other.function) + } +} + +impl Eq for MapExtractExpr {} + +impl Hash for MapExtractExpr { + fn hash(&self, state: &mut H) { + self.function.hash(state); + } +} + +impl PhysicalExpr for MapExtractExpr { + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(self, f) + } + + fn return_field(&self, input_schema: &Schema) -> Result { + self.function.return_field(input_schema) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + let num_rows = batch.num_rows(); + if num_rows == 0 { + return Ok(ColumnarValue::Array(new_null_array( + self.function.return_type(), + 0, + ))); + } + + let args = self.function.args(); + let map = args[0].evaluate(batch)?; + let all_null = match &map { + ColumnarValue::Scalar(map) => map.is_null(), + ColumnarValue::Array(map) => map.null_count() == num_rows, + }; + if all_null { + return Ok(ColumnarValue::Array(new_null_array( + self.function.return_type(), + num_rows, + ))); + } + + let selection = match &map { + ColumnarValue::Array(map) => map + .nulls() + .filter(|nulls| nulls.null_count() > 0) + .map(|nulls| BooleanArray::new(nulls.inner().clone(), None)), + ColumnarValue::Scalar(_) => None, + }; + let key = match &selection { + Some(selection) => args[1].evaluate_selection(batch, selection)?, + None => args[1].evaluate(batch)?, + }; + let arg_fields = args + .iter() + .map(|arg| arg.return_field(batch.schema_ref())) + .collect::>>()?; + let result = self.function.fun().invoke_with_args(ScalarFunctionArgs { + args: vec![map, key], + arg_fields, + number_rows: num_rows, + return_field: self.function.return_field(batch.schema_ref())?, + config_options: Arc::clone(&self.config_options), + })?; + + // DataFusion's map_extract does not consult the map's row validity. In particular, + // a scalar key stays scalar after evaluate_selection, and a null map may retain + // nonempty entry offsets. Mask those rows even if the UDF found a matching entry. + match selection { + Some(selection) => { + let result = result.into_array(num_rows)?; + Ok(ColumnarValue::Array(nullif( + result.as_ref(), + ¬(&selection)?, + )?)) + } + None => Ok(result), + } + } + + fn children(&self) -> Vec<&Arc> { + self.function.children() + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + let function = ScalarFunctionExpr::new( + self.function.name(), + Arc::new(self.function.fun().clone()), + children, + self.function.return_field(&Schema::empty())?, + Arc::clone(&self.config_options), + ); + Ok(Arc::new(Self::try_new(function)?)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use arrow::array::{ + Array, ArrayRef, BooleanBuilder, Int32Array, Int32Builder, Int64Array, Int64Builder, + ListArray, MapBuilder, + }; + use arrow::datatypes::{DataType, Int64Type}; + use datafusion::common::ScalarValue; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{BinaryExpr, Column, Literal}; + use datafusion_comet_spark_expr::monotonically_increasing_id::MonotonicallyIncreasingId; + use datafusion_comet_spark_expr::{create_query_context_map, ListExtract}; + use datafusion_functions_nested::map_extract::map_extract_udf; + + #[derive(Debug)] + struct CountedExpr { + child: Arc, + calls: Arc, + } + + impl Display for CountedExpr { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.child, f) + } + } + + impl PartialEq for CountedExpr { + fn eq(&self, other: &Self) -> bool { + self.child.eq(&other.child) + } + } + + impl Eq for CountedExpr {} + + impl Hash for CountedExpr { + fn hash(&self, state: &mut H) { + self.child.hash(state); + } + } + + impl PhysicalExpr for CountedExpr { + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(self, f) + } + + fn return_field(&self, input_schema: &Schema) -> Result { + self.child.return_field(input_schema) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + self.child.evaluate(batch) + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.child] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(Self { + child: Arc::clone(&children[0]), + calls: Arc::clone(&self.calls), + })) + } + } + + fn map_batch(base: i32, first_valid: bool) -> Result { + let mut builder = MapBuilder::new(None, Int64Builder::new(), Int32Builder::new()); + for valid in [first_valid, true] { + // Keep entries even in the null row, as Arrow permits. + builder.keys().append_value(0); + builder.values().append_value(base); + builder.keys().append_value(1); + builder.values().append_value(base + 1); + builder.append(valid)?; + } + RecordBatch::try_from_iter(vec![("map", Arc::new(builder.finish()) as ArrayRef)]) + .map_err(Into::into) + } + + fn map_expr( + map: Arc, + key: Arc, + schema: &Schema, + ) -> Result> { + Ok(Arc::new(MapExtractExpr::try_new( + ScalarFunctionExpr::try_new( + map_extract_udf(), + vec![map, key], + schema, + Arc::new(ConfigOptions::default()), + )?, + )?)) + } + + fn single_value(map: Arc) -> ListExtract { + ListExtract::new( + map, + Arc::new(Literal::new(ScalarValue::Int32(Some(1)))), + None, + true, + false, + None, + create_query_context_map(), + ) + } + + #[test] + fn test_map_extract_evaluates_map_once_and_advances_key_only_for_valid_maps() -> Result<()> { + let first = map_batch(10, false)?; + let calls = Arc::new(AtomicUsize::new(0)); + let lookup = map_expr( + Arc::new(CountedExpr { + child: Arc::new(Column::new("map", 0)), + calls: Arc::clone(&calls), + }), + Arc::new(MonotonicallyIncreasingId::from_offset(0)), + first.schema_ref(), + )?; + let expr = single_value(Arc::clone(&lookup)); + for (batch, expected) in [(first, 10), (map_batch(20, false)?, 21)] { + lookup.evaluate(&batch.slice(0, 1))?; + let before = calls.load(Ordering::Relaxed); + lookup.evaluate(&batch.slice(0, 0))?; + assert_eq!(calls.load(Ordering::Relaxed), before); + let result = expr.evaluate(&batch)?.into_array(2)?; + assert_eq!(calls.load(Ordering::Relaxed), before + 1); + assert_eq!( + result.to_data(), + Int32Array::from(vec![None, Some(expected)]).to_data() + ); + } + Ok(()) + } + + #[test] + fn test_map_extract_skips_throwing_key_for_null_map() -> Result<()> { + let maps = map_batch(10, false)?; + let index_list = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(0)]), + Some(vec![Some(0)]), + ]); + let batch = RecordBatch::try_from_iter(vec![ + ("map", Arc::clone(maps.column(0))), + ("index_arr", Arc::new(index_list) as ArrayRef), + ("index", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ])?; + let key = Arc::new(ListExtract::new( + Arc::new(Column::new("index_arr", 1)), + Arc::new(Column::new("index", 2)), + None, + true, + false, + None, + create_query_context_map(), + )); + assert!(key.evaluate(&batch).is_err()); + let expr = single_value(map_expr( + Arc::new(Column::new("map", 0)), + key, + batch.schema_ref(), + )?); + let result = expr.evaluate(&batch)?.into_array(2)?; + assert_eq!( + result.to_data(), + Int32Array::from(vec![None, Some(10)]).to_data() + ); + + let valid_maps = map_batch(10, true)?; + let valid_batch = RecordBatch::try_new( + batch.schema(), + vec![ + Arc::clone(valid_maps.column(0)), + Arc::clone(batch.column(1)), + Arc::clone(batch.column(2)), + ], + )?; + assert!(expr.evaluate(&valid_batch).is_err()); + Ok(()) + } + + #[test] + fn test_map_extract_skips_key_for_null_and_empty_input() -> Result<()> { + let batch = map_batch(10, false)?.slice(0, 1); + let map_type = batch.column(0).data_type(); + let maps: Vec> = vec![ + Arc::new(Column::new("map", 0)), + Arc::new(Literal::new(ScalarValue::try_new_null(map_type)?)), + ]; + for map in maps { + let calls = Arc::new(AtomicUsize::new(0)); + let key = Arc::new(CountedExpr { + child: Arc::new(BinaryExpr::new( + Arc::new(Literal::new(ScalarValue::Int64(Some(1)))), + Operator::Divide, + Arc::new(Literal::new(ScalarValue::Int64(Some(0)))), + )), + calls: Arc::clone(&calls), + }); + let expr = map_expr(map, key, batch.schema_ref())?; + for input in [&batch, &batch.slice(0, 0)] { + let result = expr.evaluate(input)?.into_array(input.num_rows())?; + assert!(matches!(result.data_type(), DataType::List(_))); + assert_eq!(result.null_count(), input.num_rows()); + } + assert_eq!(calls.load(Ordering::Relaxed), 0); + } + Ok(()) + } + + #[test] + fn test_map_extract_masks_null_map_entries_with_scalar_key() -> Result<()> { + let batch = map_batch(10, false)?; + let key: Arc = Arc::new(Literal::new(ScalarValue::Int64(Some(0)))); + let expr = single_value(map_expr( + Arc::new(Column::new("map", 0)), + Arc::clone(&key), + batch.schema_ref(), + )?); + let result = expr.evaluate(&batch)?.into_array(2)?; + assert_eq!( + result.to_data(), + Int32Array::from(vec![None, Some(10)]).to_data() + ); + + let scalar_map = ScalarValue::try_from_array(batch.column(0), 1)?; + let lookup = map_expr( + Arc::new(Literal::new(scalar_map.clone())), + key, + batch.schema_ref(), + )?; + assert!(matches!(lookup.evaluate(&batch)?, ColumnarValue::Scalar(_))); + let expr = single_value(lookup); + let result = expr.evaluate(&batch)?.into_array(2)?; + assert_eq!(result.to_data(), Int32Array::from(vec![10, 10]).to_data()); + + let keys = RecordBatch::try_from_iter(vec![( + "key", + Arc::new(Int64Array::from(vec![Some(0), Some(5), None])) as ArrayRef, + )])?; + let expr = single_value(map_expr( + Arc::new(Literal::new(scalar_map)), + Arc::new(Column::new("key", 0)), + keys.schema_ref(), + )?); + let result = expr.evaluate(&keys)?.into_array(3)?; + assert_eq!( + result.to_data(), + Int32Array::from(vec![Some(10), None, None]).to_data() + ); + Ok(()) + } + + #[test] + fn test_map_extract_handles_boolean_scalar_keys_on_null_maps() -> Result<()> { + let mut builder = MapBuilder::new(None, BooleanBuilder::new(), Int32Builder::new()); + for valid in [false, true] { + builder.keys().append_value(false); + builder.values().append_value(10); + builder.keys().append_value(true); + builder.values().append_value(20); + builder.append(valid)?; + } + let batch = + RecordBatch::try_from_iter(vec![("map", Arc::new(builder.finish()) as ArrayRef)])?; + for (key, expected) in [ + (Some(true), Some(20)), + (Some(false), Some(10)), + (None, None), + ] { + let expr = single_value(map_expr( + Arc::new(Column::new("map", 0)), + Arc::new(Literal::new(ScalarValue::Boolean(key))), + batch.schema_ref(), + )?); + let result = expr.evaluate(&batch)?.into_array(2)?; + assert_eq!( + result.to_data(), + Int32Array::from(vec![None, expected]).to_data() + ); + } + Ok(()) + } +} diff --git a/native/core/src/execution/expressions/mod.rs b/native/core/src/execution/expressions/mod.rs index 4c25109af6a..c9639f612da 100644 --- a/native/core/src/execution/expressions/mod.rs +++ b/native/core/src/execution/expressions/mod.rs @@ -23,6 +23,7 @@ pub mod comparison; pub mod list_empty_to_null; pub mod list_positions; pub mod logical; +pub mod map_extract; pub mod nullcheck; pub mod partition; pub mod random; diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 3536073daa0..431b27c5010 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -36,6 +36,7 @@ use crate::execution::operators::IcebergScanExec; use crate::execution::{ expressions::list_empty_to_null::ListEmptyToNullExpr, expressions::list_positions::ListPositionsExpr, + expressions::map_extract::MapExtractExpr, expressions::subquery::Subquery, operators::{ ExecutionError, ExpandExec, ParquetCompression, ParquetWriterExec, SampleExec, ScanExec, @@ -3528,13 +3529,18 @@ impl PhysicalPlanner { }) .collect::>(); - let scalar_expr: Arc = Arc::new(ScalarFunctionExpr::new( + let scalar_expr = ScalarFunctionExpr::new( fun_name, fun_expr, args.to_vec(), Arc::new(Field::new(fun_name, data_type.clone(), true)), Arc::new(ConfigOptions::default()), - )); + ); + let scalar_expr: Arc = if fun_name == "map_extract" { + Arc::new(MapExtractExpr::try_new(scalar_expr)?) + } else { + Arc::new(scalar_expr) + }; // DF53 changed some UDFs (e.g. md5) to return StringViewArray at execution // time (apache/datafusion#20045). Comet does not yet support view types, so diff --git a/native/spark-expr/src/array_funcs/list_extract.rs b/native/spark-expr/src/array_funcs/list_extract.rs index d68784ca70a..39a66380aab 100644 --- a/native/spark-expr/src/array_funcs/list_extract.rs +++ b/native/spark-expr/src/array_funcs/list_extract.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{Array, GenericListArray, Int32Array, OffsetSizeTrait}; +use arrow::array::{ + new_null_array, Array, BooleanArray, GenericListArray, Int32Array, OffsetSizeTrait, +}; use arrow::datatypes::{DataType, FieldRef, Schema}; use arrow::{array::MutableArrayData, datatypes::ArrowNativeType, record_batch::RecordBatch}; use datafusion::common::{ @@ -135,9 +137,29 @@ impl PhysicalExpr for ListExtract { } fn evaluate(&self, batch: &RecordBatch) -> DataFusionResult { - let child_value = self.child.evaluate(batch)?.into_array(batch.num_rows())?; - let ordinal_value = self.ordinal.evaluate(batch)?.into_array(batch.num_rows())?; let element_type = self.data_type(&batch.schema())?; + let num_rows = batch.num_rows(); + if num_rows == 0 { + return Ok(ColumnarValue::Array(new_null_array(&element_type, 0))); + } + + // Spark evaluates the array once, then evaluates the index only for non-null arrays. + // Keep that order in every mode: an index expression can throw even without ANSI, and + // evaluating stateful children on a different set of rows changes their results. + let child_value = self.child.evaluate(batch)?.into_array(num_rows)?; + if child_value.null_count() == num_rows { + return Ok(ColumnarValue::Array(new_null_array( + &element_type, + num_rows, + ))); + } + let ordinal_value = if child_value.null_count() == 0 { + self.ordinal.evaluate(batch)? + } else { + let selection = BooleanArray::new(child_value.nulls().unwrap().inner().clone(), None); + self.ordinal.evaluate_selection(batch, &selection)? + } + .into_array(num_rows)?; let default_value = self .default_value @@ -334,12 +356,186 @@ impl Display for ListExtract { #[cfg(test)] mod test { use super::*; - use arrow::array::{Array, Int32Array, ListArray}; + use arrow::array::{Array, ArrayRef, Int32Array, LargeListArray, ListArray}; use arrow::datatypes::{Field, Int32Type}; use datafusion::common::{Result, ScalarValue}; - use datafusion::physical_expr::expressions::Column; + use datafusion::logical_expr::Operator; + use datafusion::physical_expr::expressions::{BinaryExpr, CastExpr, Column, Literal}; use datafusion::physical_plan::ColumnarValue; + use crate::nondetermenistic_funcs::monotonically_increasing_id::MonotonicallyIncreasingId; + + #[test] + fn test_list_extract_skips_throwing_index_for_null_array() -> Result<()> { + for one_based in [false, true] { + for fail_on_error in [false, true] { + let list = ListArray::from_iter_primitive::(vec![ + None, + Some(vec![Some(99)]), + ]); + let index_list = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(i32::from(one_based))]), + Some(vec![Some(i32::from(one_based))]), + ]); + let batch = RecordBatch::try_from_iter(vec![ + ("arr", Arc::new(list) as ArrayRef), + ("index_arr", Arc::new(index_list) as ArrayRef), + ("index", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ])?; + // Index zero throws even without ANSI, but belongs to the null outer array. + let ordinal = Arc::new(ListExtract::new( + Arc::new(Column::new("index_arr", 1)), + Arc::new(Column::new("index", 2)), + None, + true, + false, + None, + crate::create_query_context_map(), + )); + assert!(ordinal.evaluate(&batch).is_err()); + let expr = ListExtract::new( + Arc::new(Column::new("arr", 0)), + ordinal, + None, + one_based, + fail_on_error, + None, + crate::create_query_context_map(), + ); + let result = expr.evaluate(&batch)?.into_array(2)?; + assert_eq!( + result.to_data(), + Int32Array::from(vec![None, Some(99)]).to_data() + ); + } + } + Ok(()) + } + + #[test] + fn test_list_extract_skips_index_for_null_and_empty_input() -> Result<()> { + let lists: Vec = vec![ + Arc::new(ListArray::from_iter_primitive::(vec![ + None::>>, + None, + ])), + Arc::new(LargeListArray::from_iter_primitive::( + vec![None::>>, None], + )), + ]; + for list in lists { + let batch = RecordBatch::try_from_iter(vec![("arr", Arc::clone(&list))])?; + let throwing_index: Arc = Arc::new(BinaryExpr::new( + Arc::new(Literal::new(ScalarValue::Int32(Some(1)))), + Operator::Divide, + Arc::new(Literal::new(ScalarValue::Int32(Some(0)))), + )); + assert!(throwing_index.evaluate(&batch).is_err()); + let children: Vec> = vec![ + Arc::new(Column::new("arr", 0)), + Arc::new(Literal::new(ScalarValue::try_new_null(list.data_type())?)), + ]; + for child in children { + for fail_on_error in [false, true] { + let expr = ListExtract::new( + Arc::clone(&child), + Arc::clone(&throwing_index), + None, + true, + fail_on_error, + None, + crate::create_query_context_map(), + ); + for input in [&batch, &batch.slice(0, 0)] { + let result = expr.evaluate(input)?.into_array(input.num_rows())?; + assert_eq!(result.data_type(), &DataType::Int32); + assert_eq!(result.null_count(), input.num_rows()); + } + } + } + } + Ok(()) + } + + #[test] + fn test_list_extract_advances_stateful_index_only_for_non_null_arrays() -> Result<()> { + for fail_on_error in [false, true] { + let ordinal = Arc::new(CastExpr::new( + Arc::new(MonotonicallyIncreasingId::from_offset(0)), + DataType::Int32, + None, + )); + let expr = ListExtract::new( + Arc::new(Column::new("arr", 0)), + ordinal, + None, + false, + fail_on_error, + None, + crate::create_query_context_map(), + ); + for (values, expected) in [([10, 11], 10), ([20, 21], 21)] { + let list = ListArray::from_iter_primitive::(vec![ + None, + Some(values.map(Some).to_vec()), + ]); + let batch = RecordBatch::try_from_iter(vec![("arr", Arc::new(list) as ArrayRef)])?; + // Neither an all-null batch nor an empty batch may advance the index counter. + expr.evaluate(&batch.slice(0, 1))?; + expr.evaluate(&batch.slice(0, 0))?; + let result = expr.evaluate(&batch)?.into_array(2)?; + assert_eq!( + result.to_data(), + Int32Array::from(vec![None, Some(expected)]).to_data() + ); + } + } + Ok(()) + } + + #[test] + fn test_list_extract_short_circuits_sliced_large_list() -> Result<()> { + let list = LargeListArray::from_iter_primitive::(vec![ + Some(vec![Some(99)]), + None, + Some(vec![Some(10)]), + None, + Some(vec![Some(20)]), + ]); + let batch = RecordBatch::try_from_iter(vec![ + ("arr", Arc::new(list) as ArrayRef), + ( + "denominator", + Arc::new(Int32Array::from(vec![1, 0, 1, 0, 1])) as ArrayRef, + ), + ])? + .slice(1, 4); + assert_eq!(batch.column(0).nulls().unwrap().inner().offset(), 1); + let ordinal: Arc = Arc::new(BinaryExpr::new( + Arc::new(Literal::new(ScalarValue::Int32(Some(1)))), + Operator::Divide, + Arc::new(Column::new("denominator", 1)), + )); + assert!(ordinal.evaluate(&batch).is_err()); + for fail_on_error in [false, true] { + let expr = ListExtract::new( + Arc::new(Column::new("arr", 0)), + Arc::clone(&ordinal), + None, + true, + fail_on_error, + None, + crate::create_query_context_map(), + ); + let result = expr.evaluate(&batch)?.into_array(4)?; + assert_eq!( + result.to_data(), + Int32Array::from(vec![None, Some(10), None, Some(20)]).to_data() + ); + } + Ok(()) + } + #[test] fn test_nullable_when_array_is_nullable() -> Result<()> { // Regression test for SPARK-55747: GetArrayItem over a nullable array (e.g. the diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 1268c66957d..b5c2162d687 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -535,10 +535,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { * Attach a fresh `expr_id` and, when the expression's origin carries one, its `QueryContext` to * `protoExpr`. Native ANSI errors resolve their context by the `expr_id` of the `Expr` that * throws (see `register_query_context` and `ListExtract` in the native planner), so the - * metadata has to sit on that `Expr`. The generic serde path applies this to the top-level - * `Expr` it returns; a serde that nests a throwing expression inside a wrapper (e.g. - * `CometElementAt`'s CASE-WHEN NULL guard) must also call it on the inner `Expr`, or that - * expression's error renders without Spark's `== SQL ... ==` query context. + * metadata has to sit on that `Expr`. */ private[serde] def attachExprIdAndContext(expr: Expression, protoExpr: Expr): Expr = { val builder = protoExpr.toBuilder diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 63860c629cc..047ae53f73a 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -623,7 +623,7 @@ object CometElementAt extends CometExpressionSerde[ElementAt] { val childExpr = exprToProtoInternal(expr.left, inputs, binding) val ordinalExpr = exprToProtoInternal(expr.right, inputs, binding) - val baseExpr = expr.left.dataType match { + expr.left.dataType match { case _: MapType => scalarFunctionExprToProto("map_extract", childExpr, ordinalExpr) case _ => @@ -652,48 +652,6 @@ object CometElementAt extends CometExpressionSerde[ElementAt] { } } - // Spark's ElementAt is a BinaryExpression: for a NULL map/array it returns NULL WITHOUT - // evaluating the key/index child. Native scalar functions evaluate the key eagerly over the - // whole batch, so under ANSI (failOnError) a throwing key (e.g. a divide-by-zero) fires even on - // rows whose map/array is NULL, where Spark short-circuits. When the left can actually be NULL, - // guard the lookup with CASE WHEN left IS NOT NULL THEN ELSE null so the key is only - // evaluated on the selected rows (DataFusion's CaseExpr filters the batch before the THEN - // branch), reproducing the short-circuit. Mirrors the CASE-WHEN idiom in CometArrayAppend / - // CometSize; the ELSE null literal carries the result type, as in CometArraysZip. - if (expr.failOnError && expr.left.nullable) { - val isNotNullExpr = createUnaryExpr( - expr, - expr.left, - inputs, - binding, - (builder, unaryExpr) => builder.setIsNotNull(unaryExpr)) - val nullLiteralProto = exprToProto(Literal(null, expr.dataType), Seq.empty) - for { - base <- baseExpr - notNull <- isNotNullExpr - nullLit <- nullLiteralProto - } yield { - // The generic serde path attaches this ElementAt's expr_id and QueryContext to the CASE we - // return here, but the lookup that actually throws under ANSI (ListExtract, for the array - // case) is nested inside the THEN branch. Attach them to that inner Expr too, so a native - // INVALID_ARRAY_INDEX_IN_ELEMENT_AT / INVALID_INDEX_OF_ZERO error still renders Spark's - // `== SQL ... ==` query context. (map_extract ignores expr_id, so the map case is - // unaffected and never throws an index error anyway.) - val guardedBase = QueryPlanSerde.attachExprIdAndContext(expr, base) - val caseWhenExpr = ExprOuterClass.CaseWhen - .newBuilder() - .addWhen(notNull) - .addThen(guardedBase) - .setElseExpr(nullLit) - .build() - ExprOuterClass.Expr - .newBuilder() - .setCaseWhen(caseWhenExpr) - .build() - } - } else { - baseExpr - } } } diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 40505562300..2e7b3f856c7 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1134,6 +1134,41 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + test("element_at evaluates stateful operands only on the required rows") { + import testImplicits._ + + // Both rows must share a partition and batch: a duplicated left operand would give its + // independent THEN-side counter only the second row, producing 0 instead of Spark's 1. + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + CometConf.COMET_BATCH_SIZE.key -> "8192") { + withTempPath { dir => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + Seq(1, 2).toDF("id").coalesce(1).write.parquet(dir.toString) + } + withTempView("stateful_lookup") { + val input = spark.read.parquet(dir.toString) + assert(input.rdd.getNumPartitions == 1) + assert(input.collect().map(_.getInt(0)).toSeq == Seq(1, 2)) + input.createOrReplaceTempView("stateful_lookup") + Seq(false, true).foreach { ansi => + withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) { + checkSparkAnswerAndOperator( + "SELECT id, element_at(element_at(" + + "array(CAST(NULL AS ARRAY), " + + "array(monotonically_increasing_id(), CAST(NULL AS BIGINT))), id), " + + "id - 1) AS v FROM stateful_lookup") + // A stateful index must advance only for the non-null array row. + checkSparkAnswerAndOperator("SELECT id, element_at(IF(id = 1, NULL, array(7)), " + + "CAST(monotonically_increasing_id() AS INT) + 1) AS v FROM stateful_lookup") + } + } + } + } + } + } + // The tests below deliberately live in this suite, not a `sql-tests` fixture: constant folding is // enabled by default here, so each `map(...)` collapses to a MapType Literal and the outer // `array(...)` reaches `CometCreateArray` with folded-Literal children, which `CometLiteral` diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index f3fba58c605..423b8179acb 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.BinaryType -import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus +import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} import org.apache.comet.testing.{DataGenOptions, ParquetGenerator, SchemaGenOptions} class CometMapExpressionSuite extends CometTestBase { @@ -431,18 +431,49 @@ class CometMapExpressionSuite extends CometTestBase { } } - // A folded nested INT-keyed map is admitted natively (all key-type guards pass), so the outer - // `element_at` runs as native `map_extract`. With ANSI disabled, a remainder-by-zero lookup key - // evaluates to NULL instead of throwing, so `map_extract(NULL_map, NULL)` returns NULL and the - // result matches Spark's 7, NULL, NULL. Under ANSI, native scalar functions evaluate the key - // eagerly and throw where Spark short-circuits after the NULL inner map -- a pre-existing eager - // evaluation difference in native `ElementAt`, not specific to folded literals. - test("folded nested map lookup with per-row key evaluation (multirow, non-ansi)") { - withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { - withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndOperator( - "SELECT _1 AS id, element_at(element_at(map(1, map(0, 7)), _1), _1 % (_1 - 2)) AS v " + - "FROM tbl") + // Keep folding enabled so the inner map takes the new literal expansion path. A null map must + // skip the key even in non-ANSI mode: an element_at index of zero throws in both modes. + for (ansi <- Seq(false, true)) { + test(s"folded nested map lookups skip throwing keys for null maps (ansi=$ansi)") { + withSQLConf( + SQLConf.ANSI_ENABLED.key -> ansi.toString, + "spark.sql.optimizer.excludedRules" -> "") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + val map = "element_at(map(1, map(0, 7)), _1)" + val key = if (ansi) "_1 % (_1 - 2)" else "element_at(array(0), _1 - 2)" + val lookups = Seq(s"element_at($map, $key)", s"($map)[$key]") ++ + (if (isSpark35Plus) Seq(s"try_element_at($map, $key)") else Seq.empty) + lookups.foreach { lookup => + checkSparkAnswerAndOperator(s"SELECT _1 AS id, $lookup AS v FROM tbl") + } + } + } + } + } + + test("map lookup evaluates a stateful key only for non-null maps") { + import testImplicits._ + + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + CometConf.COMET_BATCH_SIZE.key -> "8192") { + withTempPath { dir => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + Seq(1, 2).toDF("id").coalesce(1).write.parquet(dir.toString) + } + withTempView("stateful_map_lookup") { + val input = spark.read.parquet(dir.toString) + assert(input.rdd.getNumPartitions == 1) + assert(input.collect().map(_.getInt(0)).toSeq == Seq(1, 2)) + input.createOrReplaceTempView("stateful_map_lookup") + Seq(false, true).foreach { ansi => + withSQLConf(SQLConf.ANSI_ENABLED.key -> ansi.toString) { + checkSparkAnswerAndOperator("SELECT id, element_at(IF(id = 1, NULL, map(0, 7)), " + + "CAST(monotonically_increasing_id() AS INT)) AS v FROM stateful_map_lookup") + } + } + } } } } From 6ce68f78626e02a28733d1a71ac1261ab8d62ee3 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Sat, 29 Aug 2026 16:10:48 +0000 Subject: [PATCH 17/19] fix: preserve collations in array membership comparisons Dispatch collated array_contains operands through Spark codegen, including nested arrays and structs. Check collations before floating-point opt-in so allowIncompatible cannot select a kernel that loses string collation. Cover folded integer-key maps with collated values, case-sensitive controls, nulls, nested values, and both compatibility settings. Update collated-key membership coverage to expect the safe codegen path. --- .../scala/org/apache/comet/serde/arrays.scala | 31 ++++--- .../comet/CometMapExpressionSuite.scala | 84 ++++++++++++++++--- 2 files changed, 92 insertions(+), 23 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 047ae53f73a..bb9604613a4 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -111,17 +111,26 @@ object CometArrayContains override def getIncompatibleReasons(): Seq[String] = Seq(floatingPointReason) - override def getSupportLevel(expr: ArrayContains): SupportLevel = expr.left.dataType match { - // Native array_contains compares floating-point elements bitwise, disagreeing with Spark for - // -0.0/+0.0 and NaN. Report Incompatible (not Unsupported) for float/double element types (at - // any nesting level) so the expression routes through the JVM codegen dispatcher (Spark's own - // doGenCode) and stays native + Spark-exact under the default config, while non-float arrays - // keep the fast native kernel. Under allowIncompatible=true the native kernel is used - // as before. - case ArrayType(elementType, _) - if SupportLevel.containsType(elementType, classOf[FloatType], classOf[DoubleType]) => - Incompatible(Some(floatingPointReason)) - case _ => Compatible() + override def getUnsupportedReasons(): Seq[String] = + Seq(ComparisonUtils.nonDefaultCollationDocReason) + + override def getSupportLevel(expr: ArrayContains): SupportLevel = { + // Match the other comparison predicates: native equality cannot honour collations, including + // strings inside nested arrays or structs. Dispatch the whole expression to Spark's codegen + // even with allowIncompatible=true. Check this before floating-point types so a mixed struct + // cannot opt into bytewise string comparison through the floating-point compatibility flag. + if (ComparisonUtils.hasCollatedOperand(expr.left, expr.right)) { + Unsupported(Some(ComparisonUtils.nonDefaultCollationReason("array_contains"))) + } else { + expr.left.dataType match { + // Native array_contains compares floating-point elements bitwise, disagreeing with Spark + // for -0.0/+0.0 and NaN. Keep the existing native opt-in for these element types. + case ArrayType(elementType, _) + if SupportLevel.containsType(elementType, classOf[FloatType], classOf[DoubleType]) => + Incompatible(Some(floatingPointReason)) + case _ => Compatible() + } + } } override def convert( diff --git a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala index 423b8179acb..2ec8139802e 100644 --- a/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometMapExpressionSuite.scala @@ -377,19 +377,79 @@ class CometMapExpressionSuite extends CometTestBase { } } - // The collated counterpart of the double-key `map_contains_key` case above: a nested - // `UTF8_LCASE`-keyed map would reach `array_contains` with bytewise comparison. Expansion declines - // the folded literal so the case-insensitive lookup stays on Spark. The outer lookup key is the - // dynamic `_1` so the inner map survives as a literal (a constant key would let Spark fold - // `map_keys` into a collated-string array literal instead, which never reaches this guard). - test("map_contains_key over nested collated map keys falls back (multirow)") { + // `map_contains_key` lowers to `array_contains(map_keys(...), key)`. Collated membership must + // dispatch the whole expression to Spark's codegen, including the folded map literal, even when + // the native floating-point opt-in is enabled. The dynamic outer key keeps the lookup unfolded. + test("map_contains_key over nested collated map keys uses codegen (multirow)") { assume(isSpark40Plus) - withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { - checkSparkAnswerAndFallbackReason( - "SELECT _1 AS id, map_contains_key(" + - "element_at(map(1, map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7)), _1), " + - "CAST('a1' AS STRING COLLATE UTF8_LCASE)) AS present FROM tbl", - "Unsupported data type MapType") + withSQLConf(CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { + withParquetTable((1 until 4).map(i => (i, i.toLong)), "tbl") { + Seq(false, true).foreach { allowIncompatible => + withSQLConf( + CometConf.getExprAllowIncompatConfigKey(classOf[ArrayContains]) -> + allowIncompatible.toString) { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, map_contains_key(" + + "element_at(map(1, map(CAST('A1' AS STRING COLLATE UTF8_LCASE), 7)), _1), " + + "CAST('a1' AS STRING COLLATE UTF8_LCASE)) AS present FROM tbl") + } + } + } + } + } + + // These maps have only integer keys, so the map-key guards cannot protect their collated values. + // Keep normal constant folding enabled: the map folds to a literal but its dynamic outer lookup + // leaves map_values and array_contains to execute. Plain strings retain case-sensitive equality. + test("array_contains over folded map string values honors collation (multirow)") { + assume(isSpark40Plus) + withSQLConf( + "spark.sql.optimizer.excludedRules" -> "", + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { + withParquetTable(Seq((1, "a1"), (1, "A1"), (1, "missing"), (2, "a1"), (1, null)), "tbl") { + for { + stringType <- Seq("STRING", "STRING COLLATE UTF8_LCASE") + allowIncompatible <- Seq(false, true) + } { + withSQLConf( + CometConf.getExprAllowIncompatConfigKey(classOf[ArrayContains]) -> + allowIncompatible.toString) { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, _2 AS lookup, array_contains(map_values(element_at(" + + s"map(1, map(1, CAST('A1' AS $stringType))), _1)), " + + s"CAST(_2 AS $stringType)) AS present FROM tbl") + } + } + } + } + } + + // The collation check must recurse into arrays and structs. The DOUBLE field also verifies that + // a mixed struct cannot bypass collation handling via ArrayContains.allowIncompatible=true. + test("array_contains over folded map values preserves nested collations (multirow)") { + assume(isSpark40Plus) + withSQLConf( + "spark.sql.optimizer.excludedRules" -> "", + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { + withParquetTable(Seq((1, "a1"), (1, "A1"), (1, "missing"), (2, "a1"), (1, null)), "tbl") { + val value = "CAST('A1' AS STRING COLLATE UTF8_LCASE)" + val lookup = "CAST(_2 AS STRING COLLATE UTF8_LCASE)" + val nestedValues = Seq( + s"array($value)" -> s"array($lookup)", + s"named_struct('s', $value, 'd', 0D)" -> s"named_struct('s', $lookup, 'd', 0D)") + for { + (nestedValue, nestedLookup) <- nestedValues + allowIncompatible <- Seq(false, true) + } { + withSQLConf( + CometConf.getExprAllowIncompatConfigKey(classOf[ArrayContains]) -> + allowIncompatible.toString) { + checkSparkAnswerAndOperator( + "SELECT _1 AS id, _2 AS lookup, array_contains(map_values(element_at(" + + s"map(1, map(1, $nestedValue)), _1)), $nestedLookup) AS present FROM tbl") + } + } + } } } From 732e7acfc051a98ca4222208ebc68dbd21451193 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Sat, 29 Aug 2026 16:11:07 +0000 Subject: [PATCH 18/19] fix: preserve constructed arrays' nested result types Keep deeply nullable children while constructing an array, then restore its declared nested element type. This prevents metadata widening from breaking downstream slice and conditional expressions. Cover folded and dynamic map arrays and conditional branches on all-true, all-false, and mixed batches. Retain ArrayInsert's separate result-type normalization and refresh its comments. --- .../scala/org/apache/comet/serde/arrays.scala | 35 +++++++++++++------ .../expressions/array/array_insert.sql | 7 ++-- .../comet/CometArrayExpressionSuite.scala | 34 ++++++++++++++++++ 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index bb9604613a4..769eda013d3 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -427,15 +427,10 @@ object CometArrayInsert extends CometExpressionSerde[ArrayInsert] with ArraysBas val item = expr.children(2) val srcElementType = srcArray.dataType.asInstanceOf[ArrayType].elementType - // Native ArrayInsert requires the item's Arrow type to equal the source array's element type - // exactly, including nested nullability. For complex element types the two sides can disagree: - // a `CreateArray` source is widened to a deeply-nullable element type (see CometCreateArray), - // while a standalone item (e.g. `map(2, coalesce(id, 0))`) keeps Spark's Catalyst nullability. - // Cast BOTH sides in lockstep to the same deeply-nullable element type so their Arrow types - // match; Spark's `ArrayInsert.dataType` is `first.dataType.asNullable`, so this also - // matches the declared output element type. Casting only widens metadata and never - // changes values. Primitive element types are byte-identical on both sides, so the gate - // leaves them untouched. + // Spark declares ArrayInsert's result as first.dataType.asNullable, including nested fields. + // Cast the source and item to that common element type so both the insertion kernel and its + // consumers see the declared output type. Widening nullability changes metadata, not values; + // primitive element types do not need a cast. val (srcChild, itemChild) = if (isComplexType(srcElementType)) { val elementType = deepNullable(srcElementType) val arrayType = ArrayType(elementType, containsNull = true) @@ -510,6 +505,8 @@ object CometArrayUnion extends CometExpressionSerde[ArrayUnion] { } object CometCreateArray extends CometExpressionSerde[CreateArray] with ArraysBase { + import org.apache.comet.expressions.{CometCast, CometEvalMode} + override def convert( expr: CreateArray, inputs: Seq[Attribute], @@ -534,14 +531,30 @@ object CometCreateArray extends CometExpressionSerde[CreateArray] with ArraysBas // deeply-nullable element type instead (every array/map/struct field nullable at all nesting // levels; the cast only widens metadata and never changes values), so `make_array` always sees // identical Arrow types. A child whose cast is unsupported declines below. - val elementType = deepNullable(expr.dataType.asInstanceOf[ArrayType].elementType) + val declaredElementType = expr.dataType.asInstanceOf[ArrayType].elementType + val elementType = deepNullable(declaredElementType) val childExprs = children.map { c => val unified = if (c.dataType == elementType) c else Cast(c, elementType) exprToProtoInternal(unified, inputs, binding) } if (childExprs.forall(_.isDefined)) { - scalarFunctionExprToProto("make_array", childExprs: _*) + val arrayExpr = scalarFunctionExprToProto("make_array", childExprs: _*) + if (elementType == declaredElementType) { + arrayExpr + } else { + // Deep nullability is only needed while constructing the array. Restore Spark's declared + // nested element type so consumers such as Slice and conditional branches see consistent + // Arrow types. Keep the outer containsNull=true convention used by native list functions. + arrayExpr.flatMap { child => + CometCast.castToProto( + expr, + None, + ArrayType(declaredElementType, containsNull = true), + child, + CometEvalMode.LEGACY) + } + } } else { withFallbackReason(expr, "unsupported arguments for CreateArray") None diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql b/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql index 56df1fcccbc..e576c69ca00 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_insert.sql @@ -283,10 +283,9 @@ SELECT array_insert(array(CAST(1.1 AS FLOAT), CAST(2.2 AS FLOAT)), 2, CAST(3.3 A -- ============================================================ -- Array of maps (complex element type) --- The source array's map value is widened to nullable by CometCreateArray, while the standalone --- item map keeps value non-null because coalesce(id, 0) is non-null. Native ArrayInsert requires --- the item Arrow type to equal the source element type exactly, so the serde casts both sides to a --- common deeply-nullable element type. id includes a NULL row (coalesce yields 0). +-- Both input maps have non-null values, while ArrayInsert declares a deeply-nullable result type. +-- The serde casts both operands to that common element type so the insertion kernel and downstream +-- consumers agree on the output schema. id includes a NULL row (coalesce yields 0). -- ============================================================ statement diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 2e7b3f856c7..9974154e0d1 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1213,6 +1213,40 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + test("constructed arrays retain nested map types for downstream expressions") { + import testImplicits._ + + withSQLConf( + "spark.sql.optimizer.excludedRules" -> "", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.USE_V1_SOURCE_LIST.key -> "parquet", + CometConf.COMET_BATCH_SIZE.key -> "8192") { + withTempPath { dir => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + Seq(1, 2, 3).toDF("id").coalesce(1).write.parquet(dir.toString) + } + withTempView("nested_array_types") { + spark.read.parquet(dir.toString).createOrReplaceTempView("nested_array_types") + val constructed = "array(map(1, coalesce(id, 0)))" + val repeated = "array_repeat(map(2, coalesce(id, 0)), 1)" + checkSparkAnswerAndOperator( + s"SELECT id, slice($constructed, 1, 1) FROM nested_array_types") + checkSparkAnswerAndOperator( + "SELECT id, slice(array(map(1, 2), map(2, coalesce(id, 0))), 1, 2) " + + "FROM nested_array_types") + // All-false batches use a different native conditional path than mixed batches. + for { + condition <- Seq("id < 0", "id > 0", "id = 1") + (whenTrue, whenFalse) <- Seq(constructed -> repeated, repeated -> constructed) + } { + checkSparkAnswerAndOperator( + s"SELECT id, IF($condition, $whenTrue, $whenFalse) FROM nested_array_types") + } + } + } + } + } + // The whole `array(...)` folds to one `ArrayType(MapType(IntegerType, IntegerType, true))` // literal, so every rebuilt element must report the unified nullable value type. test("folded array of maps with a NULL value (multirow)") { From 47a7c5f3d887e99e7b3ed0ef48743cb5af73ebe1 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Sat, 29 Aug 2026 17:33:12 +0000 Subject: [PATCH 19/19] test: keep the lazy array lookup fixture nullable on older Spark Include a NULL array element so Spark 3.4, 3.5, and 4.0 retain the nullable result of element_at for a NULL array in ANSI mode. The stateful index still distinguishes lazy evaluation: eager evaluation selects the NULL second element rather than 7. --- .../scala/org/apache/comet/CometArrayExpressionSuite.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala index 9974154e0d1..e6400075643 100644 --- a/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometArrayExpressionSuite.scala @@ -1160,8 +1160,10 @@ class CometArrayExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelp "array(monotonically_increasing_id(), CAST(NULL AS BIGINT))), id), " + "id - 1) AS v FROM stateful_lookup") // A stateful index must advance only for the non-null array row. - checkSparkAnswerAndOperator("SELECT id, element_at(IF(id = 1, NULL, array(7)), " + - "CAST(monotonically_increasing_id() AS INT) + 1) AS v FROM stateful_lookup") + // A null element keeps Spark's ANSI result nullable on older versions. + checkSparkAnswerAndOperator( + "SELECT id, element_at(IF(id = 1, NULL, array(7, CAST(NULL AS INT))), " + + "CAST(monotonically_increasing_id() AS INT) + 1) AS v FROM stateful_lookup") } } }