From ee6dbf7e070d60321e3efcd6f5f04341bab4d0c1 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 11 Jun 2024 11:41:05 +0200 Subject: [PATCH 1/3] GH-41884: [Python] Fix RecordBatchReader.cast to support casting to equal schema for all types --- python/pyarrow/src/arrow/python/ipc.cc | 3 ++- python/pyarrow/tests/test_ipc.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/src/arrow/python/ipc.cc b/python/pyarrow/src/arrow/python/ipc.cc index 0ed152242425..f739dfa373da 100644 --- a/python/pyarrow/src/arrow/python/ipc.cc +++ b/python/pyarrow/src/arrow/python/ipc.cc @@ -79,7 +79,8 @@ Status CastingRecordBatchReader::Init(std::shared_ptr parent, // Ensure all columns can be cast before succeeding for (int i = 0; i < num_fields; i++) { - if (!compute::CanCast(*src->field(i)->type(), *schema->field(i)->type())) { + if ((!((src->field(i)->type()->Equals(schema->field(i)->type())))) && + (!compute::CanCast(*src->field(i)->type(), *schema->field(i)->type()))) { return Status::TypeError("Field ", i, " cannot be cast from ", src->field(i)->type()->ToString(), " to ", schema->field(i)->type()->ToString()); diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py index d38f45b5feff..af7a0c5331a1 100644 --- a/python/pyarrow/tests/test_ipc.py +++ b/python/pyarrow/tests/test_ipc.py @@ -16,6 +16,7 @@ # under the License. from collections import UserList +import datetime import io import pathlib import pytest @@ -1272,6 +1273,15 @@ def test_record_batch_reader_cast(): with pytest.raises(pa.lib.ArrowTypeError, match='Field 0 cannot be cast'): reader.cast(pa.schema([pa.field('a', pa.list_(pa.int32()))])) + # Cast to same type should always work also for date32 + # (https://github.com/apache/arrow/issues/41884) + schema_src = pa.schema([pa.field('a', pa.date32())]) + arr = pa.array([datetime.date(2024, 6, 11)], type=pa.date32()) + data = [pa.record_batch([arr], names=['a']), pa.record_batch([arr], names=['a'])] + table_src = pa.Table.from_batches(data) + reader = pa.RecordBatchReader.from_batches(schema_src, data) + assert reader.cast(schema_src).read_all() == table_src + def test_record_batch_reader_cast_nulls(): schema_src = pa.schema([pa.field('a', pa.int64())]) From b1278d21f2aed5484935415d3680abc47aed8651 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 13 Jun 2024 09:01:47 +0200 Subject: [PATCH 2/3] Update python/pyarrow/src/arrow/python/ipc.cc Co-authored-by: Felipe Oliveira Carvalho --- python/pyarrow/src/arrow/python/ipc.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/src/arrow/python/ipc.cc b/python/pyarrow/src/arrow/python/ipc.cc index f739dfa373da..e18c13e06c05 100644 --- a/python/pyarrow/src/arrow/python/ipc.cc +++ b/python/pyarrow/src/arrow/python/ipc.cc @@ -79,8 +79,9 @@ Status CastingRecordBatchReader::Init(std::shared_ptr parent, // Ensure all columns can be cast before succeeding for (int i = 0; i < num_fields; i++) { - if ((!((src->field(i)->type()->Equals(schema->field(i)->type())))) && - (!compute::CanCast(*src->field(i)->type(), *schema->field(i)->type()))) { + auto& src_type = src->field(i)->type(); + auto& schema_type = schema->field(i)->type(); + if (!src_type->Equals(schema_type) && !compute::CanCast(*src_type, *schema_type)) { return Status::TypeError("Field ", i, " cannot be cast from ", src->field(i)->type()->ToString(), " to ", schema->field(i)->type()->ToString()); From 6c649581dfadf726d8609bb22cea3ff131ae2e12 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Thu, 13 Jun 2024 09:19:35 +0200 Subject: [PATCH 3/3] update comment --- python/pyarrow/tests/test_ipc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/tests/test_ipc.py b/python/pyarrow/tests/test_ipc.py index af7a0c5331a1..d8eb6e926e4c 100644 --- a/python/pyarrow/tests/test_ipc.py +++ b/python/pyarrow/tests/test_ipc.py @@ -1273,7 +1273,7 @@ def test_record_batch_reader_cast(): with pytest.raises(pa.lib.ArrowTypeError, match='Field 0 cannot be cast'): reader.cast(pa.schema([pa.field('a', pa.list_(pa.int32()))])) - # Cast to same type should always work also for date32 + # Cast to same type should always work (also for types without a T->T cast function) # (https://github.com/apache/arrow/issues/41884) schema_src = pa.schema([pa.field('a', pa.date32())]) arr = pa.array([datetime.date(2024, 6, 11)], type=pa.date32())