Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2194,6 +2194,17 @@ def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array:
elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}:
return values.cast(target_type)
raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}")
elif field.field_type == TimestamptzNanoType():
# Writers label UTC differently (e.g. "+00:00"); relabel so files can be concatenated
if (
pa.types.is_timestamp(target_type)
and target_type.tz == "UTC"
and pa.types.is_timestamp(values.type)
and values.type.unit == "ns"
and (values.type.tz in UTC_ALIASES or values.type.tz is None)
):
return values.cast(target_type)
raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}")
elif isinstance(field.field_type, (IntegerType, LongType)):
# Cast smaller integer types to target type for cross-platform compatibility
# Only allow widening conversions (smaller bit width to larger)
Expand Down
29 changes: 29 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@
StructType,
TimestampNanoType,
TimestampType,
TimestamptzNanoType,
TimestamptzType,
TimeType,
UUIDType,
Expand Down Expand Up @@ -3936,6 +3937,34 @@ def test__to_requested_schema_timestamptz_to_timestamp_projection() -> None:
assert expected.equals(actual_result)


def test__to_requested_schema_timestamptz_ns_utc_alias_concatenates() -> None:
table_schema = Schema(
NestedField(1, "id", LongType(), required=False),
NestedField(2, "ts_field", TimestamptzNanoType(), required=False),
)

# file written before the column was added
old_file_schema = Schema(NestedField(1, "id", LongType(), required=False))
old_batch = pa.record_batch([pa.array([1], type=pa.int64())], names=["id"])

# file written by another writer that labels UTC as "+00:00"
new_batch = pa.record_batch(
[
pa.array([2], type=pa.int64()),
pa.array([1755172800123456789], type=pa.timestamp("ns", tz="+00:00")),
],
names=["id", "ts_field"],
)

old_result = _to_requested_schema(table_schema, old_file_schema, old_batch)
new_result = _to_requested_schema(table_schema, table_schema, new_batch)

assert new_result.schema.field("ts_field").type == pa.timestamp("ns", tz="UTC")
combined = pa.concat_tables([pa.Table.from_batches([old_result]), pa.Table.from_batches([new_result])])
assert combined.column("ts_field").to_pylist()[0] is None
assert combined.column("ts_field").cast(pa.int64()).to_pylist()[1] == 1755172800123456789


def test__to_requested_schema_timestamptz_to_timestamp_write_rejects() -> None:
"""Test that the write path (default) rejects timestamptz to timestamp casting.

Expand Down
Loading