diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 45c8ffd196..279aa06b6c 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -825,6 +825,8 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | if isinstance(field_type, (IntegerType, LongType, DecimalType)): if isinstance(pred, BoundLiteralPredicate): + if isinstance(field_type, (IntegerType, LongType)): + return _truncate_integer_strict(name, pred, self.transform(field_type), field_type, self._width) return _truncate_number_strict(name, pred, self.transform(field_type)) elif isinstance(pred, BoundNotIn): return _set_apply_transform(name, pred, self.transform(field_type)) @@ -864,7 +866,7 @@ def transform(self, source: IcebergType) -> Callable[[S | None], S | None]: if isinstance(source, (IntegerType, LongType)): def truncate_func(v: Any) -> Any: - return v - v % self._width + return _wrap_integer(v - v % self._width, source) elif isinstance(source, (StringType, BinaryType)): @@ -1030,6 +1032,50 @@ def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Arr return lambda arr: pa.nulls(len(arr), type=arr.type) +def _integer_bounds(source: IcebergType) -> tuple[int, int]: + """Return the minimum value and the size of the range of an integer type.""" + if isinstance(source, IntegerType): + return -(2**31), 2**32 + return -(2**63), 2**64 + + +def _wrap_integer(value: int, source: IcebergType) -> int: + """Wrap an integer into the range of the source type, like Java's two's complement arithmetic.""" + minimum, modulus = _integer_bounds(source) + return (value - minimum) % modulus + minimum + + +def _truncate_integer_strict( + name: str, + pred: BoundLiteralPredicate, + transform: Callable[[Any | None], Any | None], + source: IcebergType, + width: int, +) -> UnboundPredicate | None: + """Project a predicate strictly on a truncate partition of an integer column. + + Values from the minimum of the source type truncate to a partition value at the top of the + type's range, which does not sort with the other partition values. No single predicate on the + partition value can account for that, so those ranges have no strict projection. + """ + minimum, _ = _integer_bounds(source) + floor_mod = minimum % width + if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): + return _truncate_number_strict(name, pred, transform) + + first_block_end = minimum - floor_mod + width + if isinstance(pred, BoundLessThan): + if pred.literal.value < first_block_end: + return None + elif isinstance(pred, BoundLessThanOrEqual): + if pred.literal.increment().value < first_block_end: + return None + elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)): + return None + + return _truncate_number_strict(name, pred, transform) + + def _truncate_number( name: str, pred: BoundLiteralPredicate, transform: Callable[[Any | None], Any | None] ) -> UnboundPredicate | None: diff --git a/tests/expressions/test_projection.py b/tests/expressions/test_projection.py index 4d0c2c1346..304ef21fef 100644 --- a/tests/expressions/test_projection.py +++ b/tests/expressions/test_projection.py @@ -46,6 +46,7 @@ ) from pyiceberg.types import ( DateType, + IntegerType, LongType, NestedField, StringType, @@ -270,6 +271,21 @@ def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: assert expected[index] == expr, predicate +def test_int_truncate_projection_near_minimum(schema: Schema) -> None: + spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "id_trunc")) + project = inclusive_projection(schema, spec) + + assert project(EqualTo("id", -(2**63))) == EqualTo("id_trunc", 9223372036854775806) + + +def test_integer_truncate_projection_near_minimum() -> None: + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "x_trunc")) + project = inclusive_projection(schema, spec) + + assert project(EqualTo("x", -(2**31))) == EqualTo("x_trunc", 2147483646) + + def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec) -> None: predicates = [ NotNull("data"), diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9c418e50a..7582c91a15 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -71,6 +71,7 @@ EqualTo, GreaterThan, IsNull, + LessThan, Not, Or, ) @@ -3663,6 +3664,47 @@ def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None: assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter +def test_scan_filters_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=True), + NestedField(id=2, name="x", field_type=IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_truncate_minimum", schema=schema, partition_spec=spec) + table.append( + pa.Table.from_pylist( + [{"id": 0, "x": -(2**31)}, {"id": 1, "x": -(2**31) + 100}, {"id": 2, "x": 5}], schema=schema.as_arrow() + ) + ) + + assert table.scan(row_filter=EqualTo("x", -(2**31))).to_arrow()["id"].to_pylist() == [0] + + +def test_count_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=True), + NestedField(id=2, name="x", field_type=IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc")) + catalog.create_namespace("default") + table = catalog.create_table( + "default.test_count_truncate_minimum", + schema=schema, + partition_spec=spec, + properties={"write.metadata.metrics.default": "counts"}, + ) + table.append(pa.Table.from_pylist([{"id": 0, "x": -(2**31)}, {"id": 1, "x": 5}], schema=schema.as_arrow())) + + less_than = table.scan(row_filter=LessThan("x", -(2**31) + 100)) + assert less_than.to_arrow()["id"].to_pylist() == [0] + assert less_than.count() == 1 + + greater_than = table.scan(row_filter=GreaterThan("x", 0)) + assert greater_than.to_arrow()["id"].to_pylist() == [1] + assert greater_than.count() == 1 + + def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="id", field_type=LongType(), required=False), diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 6ab7b52c4c..e8129ad882 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -465,6 +465,11 @@ def test_truncate_integer(type_var: PrimitiveType, input_var: int, expected: int assert trunc.transform(type_var)(input_var) == expected +def test_truncate_integer_minimum_wraps() -> None: + assert TruncateTransform(10).transform(IntegerType())(-(2**31)) == 2147483646 + assert TruncateTransform(10).transform(LongType())(-(2**63)) == 9223372036854775806 + + @pytest.mark.parametrize( "input_var,expected", [ @@ -1504,8 +1509,8 @@ def test_truncate_strict_integer_lower_bound(bound_reference_int: BoundReference _assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "100") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "100") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "100") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1522,8 +1527,8 @@ def test_truncate_strict_integer_upper_bound(bound_reference_int: BoundReference _assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "90") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "90") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "90") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse) @@ -1538,8 +1543,8 @@ def test_truncate_strict_long_lower_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "100") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "100") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "100") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1556,8 +1561,8 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "90") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "90") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "90") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1568,6 +1573,50 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundIn(term=bound_reference_long, literals={value_dec, value, value_inc}), transform, NotIn) +def test_truncate_strict_integer_minimum(bound_reference_int: BoundReference) -> None: + minimum = -(2**31) + first_block_end = minimum - minimum % 1000 + 1000 + transform = TruncateTransform(1000) + + assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_int, literal=literal(minimum))) is None + assert ( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_int, literal=literal(minimum + 1))) + is None + ) + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=literal(0))) is None + assert ( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=literal(0))) is None + ) + _assert_projection_strict( + BoundLessThan(term=bound_reference_int, literal=literal(first_block_end)), + transform, + LessThan, + str(first_block_end), + ) + + +def test_truncate_strict_long_minimum(bound_reference_long: BoundReference) -> None: + minimum = -(2**63) + first_block_end = minimum - minimum % 1000 + 1000 + transform = TruncateTransform(1000) + + assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_long, literal=literal(minimum))) is None + assert ( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_long, literal=literal(minimum + 1))) + is None + ) + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=literal(0))) is None + assert ( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=literal(0))) is None + ) + _assert_projection_strict( + BoundLessThan(term=bound_reference_long, literal=literal(first_block_end)), + transform, + LessThan, + str(first_block_end), + ) + + def test_truncate_strict_decimal_lower_bound(bound_reference_decimal: BoundReference) -> None: dec = DecimalType(9, 2) value = literal("100.00").to(dec)