diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..ec6b0f9485 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -2194,6 +2194,17 @@ def _cast_if_needed(self, field: NestedField, values: pa.Array) -> pa.Array: elif target_type.unit == "us" and values.type.unit in {"s", "ms", "us"}: return values.cast(target_type) raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}") + elif field.field_type == TimestamptzNanoType(): + # Writers label UTC differently (e.g. "+00:00"); relabel so files can be concatenated + if ( + pa.types.is_timestamp(target_type) + and target_type.tz == "UTC" + and pa.types.is_timestamp(values.type) + and values.type.unit == "ns" + and (values.type.tz in UTC_ALIASES or values.type.tz is None) + ): + return values.cast(target_type) + raise ValueError(f"Unsupported schema projection from {values.type} to {target_type}") elif isinstance(field.field_type, (IntegerType, LongType)): # Cast smaller integer types to target type for cross-platform compatibility # Only allow widening conversions (smaller bit width to larger) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 439405d121..c39f3053d1 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -137,6 +137,7 @@ StructType, TimestampNanoType, TimestampType, + TimestamptzNanoType, TimestamptzType, TimeType, UUIDType, @@ -3936,6 +3937,34 @@ def test__to_requested_schema_timestamptz_to_timestamp_projection() -> None: assert expected.equals(actual_result) +def test__to_requested_schema_timestamptz_ns_utc_alias_concatenates() -> None: + table_schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "ts_field", TimestamptzNanoType(), required=False), + ) + + # file written before the column was added + old_file_schema = Schema(NestedField(1, "id", LongType(), required=False)) + old_batch = pa.record_batch([pa.array([1], type=pa.int64())], names=["id"]) + + # file written by another writer that labels UTC as "+00:00" + new_batch = pa.record_batch( + [ + pa.array([2], type=pa.int64()), + pa.array([1755172800123456789], type=pa.timestamp("ns", tz="+00:00")), + ], + names=["id", "ts_field"], + ) + + old_result = _to_requested_schema(table_schema, old_file_schema, old_batch) + new_result = _to_requested_schema(table_schema, table_schema, new_batch) + + assert new_result.schema.field("ts_field").type == pa.timestamp("ns", tz="UTC") + combined = pa.concat_tables([pa.Table.from_batches([old_result]), pa.Table.from_batches([new_result])]) + assert combined.column("ts_field").to_pylist()[0] is None + assert combined.column("ts_field").cast(pa.int64()).to_pylist()[1] == 1755172800123456789 + + def test__to_requested_schema_timestamptz_to_timestamp_write_rejects() -> None: """Test that the write path (default) rejects timestamptz to timestamp casting.