From 911b1e7958e0025eb5d1113c88d548258ac495cc Mon Sep 17 00:00:00 2001 From: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:18:35 +0530 Subject: [PATCH] Leave intermediate snapshots out of the snapshot log A commit that sets main more than once, such as an overwrite, added every snapshot to the snapshot log, so time travel could resolve to a snapshot that was never the committed state of the table. Keep only the last one, like Java does. Co-Authored-By: Claude Opus 5.5 --- pyiceberg/table/update/__init__.py | 16 +++++++++++ tests/catalog/test_catalog_behaviors.py | 17 ++++++++++++ tests/table/test_init.py | 37 +++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/pyiceberg/table/update/__init__.py b/pyiceberg/table/update/__init__.py index 62d0bbd724..a57d74d6d8 100644 --- a/pyiceberg/table/update/__init__.py +++ b/pyiceberg/table/update/__init__.py @@ -292,6 +292,17 @@ def is_added_sort_order(self, sort_order_id: int) -> bool: update.sort_order.order_id == sort_order_id for update in self._updates if isinstance(update, AddSortOrderUpdate) ) + def intermediate_snapshot_ids(self, current_snapshot_id: int | None) -> set[int]: + """Return the added snapshots that were set as the current snapshot, but replaced by a later update.""" + return { + update.snapshot_id + for update in self._updates + if isinstance(update, SetSnapshotRefUpdate) + and update.ref_name == MAIN_BRANCH + and update.snapshot_id != current_snapshot_id + and self.is_added_snapshot(update.snapshot_id) + } + def has_changes(self) -> bool: return len(self._updates) > 0 @@ -755,6 +766,11 @@ def update_table_metadata( for update in updates: new_metadata = _apply_table_update(update, new_metadata, context) + # Intermediate snapshots are never committed as the current snapshot, so they are not part of the history + if intermediate_snapshot_ids := context.intermediate_snapshot_ids(new_metadata.current_snapshot_id): + snapshot_log = [entry for entry in new_metadata.snapshot_log if entry.snapshot_id not in intermediate_snapshot_ids] + new_metadata = new_metadata.model_copy(update={"snapshot_log": snapshot_log}) + # last_updated_ms must not precede the base's, which becomes the latest metadata-log entry if context.has_changes(): if metadata_location: diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index e4f834b016..9b1de00883 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1491,6 +1491,23 @@ def test_overwrite_record_batch_reader(catalog: Catalog) -> None: assert len(tbl.scan().to_arrow()) == total_rows +def test_overwrite_adds_one_entry_to_history(catalog: Catalog) -> None: + catalog.create_namespace("default") + table = catalog.create_table("default.overwrite_history", schema=pa.schema([("x", pa.int64())])) + table.append(pa.table({"x": [1]})) + table.overwrite(pa.table({"x": [2]})) + + append, delete, overwrite = table.snapshots() + assert [snapshot.summary.operation for snapshot in (append, delete, overwrite) if snapshot.summary] == [ + Operation.APPEND, + Operation.DELETE, + Operation.APPEND, + ] + # The delete and the append are committed together, so the table was never empty + for tbl in (table, catalog.load_table("default.overwrite_history")): + assert [entry.snapshot_id for entry in tbl.history()] == [append.snapshot_id, overwrite.snapshot_id] + + def test_append_record_batch_reader_to_partitioned_table_raises(catalog: Catalog) -> None: catalog.create_namespace("default") identifier = f"default.append_record_batch_reader_partitioned_{catalog.name}" diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 547ca94fc9..4c636dc505 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -1275,6 +1275,43 @@ def test_update_metadata_add_snapshot(table_v2: Table) -> None: assert new_metadata.last_updated_ms == new_snapshot.timestamp_ms +def test_update_metadata_drops_intermediate_snapshots_from_snapshot_log(table_v2: Table) -> None: + metadata = table_v2.metadata + timestamp_ms = metadata.last_updated_ms + 1_000 + + updates: list[TableUpdate] = [] + for offset, snapshot_id in enumerate((25, 26)): + updates += [ + AddSnapshotUpdate( + snapshot=Snapshot( + snapshot_id=snapshot_id, + parent_snapshot_id=metadata.current_snapshot_id, + sequence_number=metadata.last_sequence_number + 1 + offset, + timestamp_ms=timestamp_ms + offset, + manifest_list="s3:/a/b/c.avro", + summary=Summary(Operation.APPEND), + schema_id=1, + ) + ), + SetSnapshotRefUpdate(ref_name=MAIN_BRANCH, type="branch", snapshot_id=snapshot_id), + ] + + # Snapshot 25 is replaced by 26 in the same commit, so it was never the current snapshot + table_v2.metadata = update_table_metadata(metadata, tuple(updates)) + assert table_v2.history() == metadata.snapshot_log + [SnapshotLogEntry(snapshot_id=26, timestamp_ms=timestamp_ms + 1)] + assert table_v2.snapshot_by_id(25) is not None + assert table_v2.snapshot_as_of_timestamp(timestamp_ms) == table_v2.snapshot_by_id(metadata.snapshot_log[-1].snapshot_id) + assert table_v2.snapshot_as_of_timestamp(timestamp_ms + 1) == table_v2.snapshot_by_id(26) + + # Committed separately, both snapshots have been current + assert update_table_metadata( + update_table_metadata(metadata, tuple(updates[:2])), tuple(updates[2:]) + ).snapshot_log == metadata.snapshot_log + [ + SnapshotLogEntry(snapshot_id=25, timestamp_ms=timestamp_ms), + SnapshotLogEntry(snapshot_id=26, timestamp_ms=timestamp_ms + 1), + ] + + def test_update_metadata_set_ref_snapshot(table_v2: Table) -> None: update, _ = table_v2.transaction()._set_ref_snapshot( snapshot_id=3051729675574597004,