Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions pyiceberg/table/update/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,17 @@ def is_added_sort_order(self, sort_order_id: int) -> bool:
update.sort_order.order_id == sort_order_id for update in self._updates if isinstance(update, AddSortOrderUpdate)
)

def intermediate_snapshot_ids(self, current_snapshot_id: int | None) -> set[int]:
"""Return the added snapshots that were set as the current snapshot, but replaced by a later update."""
return {
update.snapshot_id
for update in self._updates
if isinstance(update, SetSnapshotRefUpdate)
and update.ref_name == MAIN_BRANCH
and update.snapshot_id != current_snapshot_id
and self.is_added_snapshot(update.snapshot_id)
}

def has_changes(self) -> bool:
return len(self._updates) > 0

Expand Down Expand Up @@ -755,6 +766,11 @@ def update_table_metadata(
for update in updates:
new_metadata = _apply_table_update(update, new_metadata, context)

# Intermediate snapshots are never committed as the current snapshot, so they are not part of the history
if intermediate_snapshot_ids := context.intermediate_snapshot_ids(new_metadata.current_snapshot_id):
snapshot_log = [entry for entry in new_metadata.snapshot_log if entry.snapshot_id not in intermediate_snapshot_ids]
new_metadata = new_metadata.model_copy(update={"snapshot_log": snapshot_log})

# last_updated_ms must not precede the base's, which becomes the latest metadata-log entry
if context.has_changes():
if metadata_location:
Expand Down
17 changes: 17 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1491,6 +1491,23 @@ def test_overwrite_record_batch_reader(catalog: Catalog) -> None:
assert len(tbl.scan().to_arrow()) == total_rows


def test_overwrite_adds_one_entry_to_history(catalog: Catalog) -> None:
catalog.create_namespace("default")
table = catalog.create_table("default.overwrite_history", schema=pa.schema([("x", pa.int64())]))
table.append(pa.table({"x": [1]}))
table.overwrite(pa.table({"x": [2]}))

append, delete, overwrite = table.snapshots()
assert [snapshot.summary.operation for snapshot in (append, delete, overwrite) if snapshot.summary] == [
Operation.APPEND,
Operation.DELETE,
Operation.APPEND,
]
# The delete and the append are committed together, so the table was never empty
for tbl in (table, catalog.load_table("default.overwrite_history")):
assert [entry.snapshot_id for entry in tbl.history()] == [append.snapshot_id, overwrite.snapshot_id]


def test_append_record_batch_reader_to_partitioned_table_raises(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.append_record_batch_reader_partitioned_{catalog.name}"
Expand Down
37 changes: 37 additions & 0 deletions tests/table/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -1275,6 +1275,43 @@ def test_update_metadata_add_snapshot(table_v2: Table) -> None:
assert new_metadata.last_updated_ms == new_snapshot.timestamp_ms


def test_update_metadata_drops_intermediate_snapshots_from_snapshot_log(table_v2: Table) -> None:
metadata = table_v2.metadata
timestamp_ms = metadata.last_updated_ms + 1_000

updates: list[TableUpdate] = []
for offset, snapshot_id in enumerate((25, 26)):
updates += [
AddSnapshotUpdate(
snapshot=Snapshot(
snapshot_id=snapshot_id,
parent_snapshot_id=metadata.current_snapshot_id,
sequence_number=metadata.last_sequence_number + 1 + offset,
timestamp_ms=timestamp_ms + offset,
manifest_list="s3:/a/b/c.avro",
summary=Summary(Operation.APPEND),
schema_id=1,
)
),
SetSnapshotRefUpdate(ref_name=MAIN_BRANCH, type="branch", snapshot_id=snapshot_id),
]

# Snapshot 25 is replaced by 26 in the same commit, so it was never the current snapshot
table_v2.metadata = update_table_metadata(metadata, tuple(updates))
assert table_v2.history() == metadata.snapshot_log + [SnapshotLogEntry(snapshot_id=26, timestamp_ms=timestamp_ms + 1)]
assert table_v2.snapshot_by_id(25) is not None
assert table_v2.snapshot_as_of_timestamp(timestamp_ms) == table_v2.snapshot_by_id(metadata.snapshot_log[-1].snapshot_id)
assert table_v2.snapshot_as_of_timestamp(timestamp_ms + 1) == table_v2.snapshot_by_id(26)

# Committed separately, both snapshots have been current
assert update_table_metadata(
update_table_metadata(metadata, tuple(updates[:2])), tuple(updates[2:])
).snapshot_log == metadata.snapshot_log + [
SnapshotLogEntry(snapshot_id=25, timestamp_ms=timestamp_ms),
SnapshotLogEntry(snapshot_id=26, timestamp_ms=timestamp_ms + 1),
]


def test_update_metadata_set_ref_snapshot(table_v2: Table) -> None:
update, _ = table_v2.transaction()._set_ref_snapshot(
snapshot_id=3051729675574597004,
Expand Down
Loading