Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions pyiceberg/catalog/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
union,
update,
)
from sqlalchemy.engine import make_url
from sqlalchemy.exc import IntegrityError, NoResultFound, OperationalError, ProgrammingError
from sqlalchemy.orm import (
DeclarativeBase,
Expand Down Expand Up @@ -139,8 +140,16 @@ def __init__(self, name: str, engine: Engine | None = None, **properties: str):
echo_str = str(self.properties.get("echo", DEFAULT_ECHO_VALUE)).lower()
echo = strtobool(echo_str) if echo_str != "debug" else "debug"
pool_pre_ping = strtobool(self.properties.get("pool_pre_ping", DEFAULT_POOL_PRE_PING_VALUE))

self.engine = create_engine(uri_prop, echo=echo, pool_pre_ping=pool_pre_ping)
url = make_url(uri_prop)
connect_args: dict[str, bool] = {}
if (
url.get_backend_name() == "sqlite"
and url.get_driver_name() == "pysqlite"
and "check_same_thread" not in url.query
):
connect_args["check_same_thread"] = False

self.engine = create_engine(url, echo=echo, pool_pre_ping=pool_pre_ping, connect_args=connect_args)

self._init_catalog()

Expand Down
133 changes: 131 additions & 2 deletions tests/catalog/test_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,19 @@
# specific language governing permissions and limitations
# under the License.

import gc
import logging
import sqlite3
import warnings
from collections.abc import Generator
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import cast

import pytest
from sqlalchemy import Engine, create_engine, event, inspect, text
from sqlalchemy.exc import ArgumentError
from sqlalchemy import Engine, Pool, create_engine, event, inspect, text
from sqlalchemy.exc import ArgumentError, OperationalError
from sqlalchemy.pool import QueuePool, SingletonThreadPool

from pyiceberg.catalog import load_catalog
from pyiceberg.catalog.sql import (
Expand Down Expand Up @@ -288,6 +294,129 @@ def test_sql_catalog_multiple_close_calls(self, catalog_sqlite: SqlCatalog) -> N
# Second close should not raise an exception
catalog_sqlite.close()

def test_sql_catalog_cross_thread_sqlite_close(self, warehouse: Path, caplog: pytest.LogCaptureFixture) -> None:
"""Closing a SQLite SqlCatalog on a different thread than the one that created succeeds."""
gc.collect()

def create_in_worker() -> SqlCatalog:
return SqlCatalog("test", uri="sqlite:///:memory:", warehouse=f"file://{warehouse}")

with ThreadPoolExecutor(max_workers=1) as executor:
catalog = executor.submit(create_in_worker).result()

with warnings.catch_warnings(record=True) as caught_warnings:
warnings.simplefilter("always", ResourceWarning)
with caplog.at_level(logging.ERROR, logger="sqlalchemy.pool"):
catalog.close()

pool_errors = [
(record.exc_info[0], str(record.exc_info[1]))
for record in caplog.records
if record.exc_info and record.exc_info[0] is not None and record.exc_info[1] is not None
]
caplog.clear()
del catalog
gc.collect()

assert pool_errors == []
assert [str(w.message) for w in caught_warnings] == []


@pytest.mark.parametrize("uri", ["sqlite:///:memory:", "sqlite+pysqlite:///:memory:"])
def test_sql_catalog_sqlite_memory_preserves_thread_isolation(
warehouse: Path, caplog: pytest.LogCaptureFixture, uri: str
) -> None:
"""check_same_thread=False keeps SingletonThreadPool isolation, and only affects close and garbage collection."""
gc.collect()
catalog = SqlCatalog("test", uri=uri, warehouse=f"file://{warehouse}")
assert isinstance(catalog.engine.pool, SingletonThreadPool)

catalog.create_namespace("ns")
assert catalog.list_namespaces() == [("ns",)]

# worker thread cannot see the main thread DB
with ThreadPoolExecutor(max_workers=1) as executor:
with pytest.raises(OperationalError, match="no such table"):
executor.submit(catalog.list_namespaces).result()

# closing the main thread connection also closes the worker thread's connection
with warnings.catch_warnings(record=True) as caught_warnings:

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note: these checks, which ensure that cross-thread close() and garbage collection complete successfully, are the same as those performed in the previous test (test_sql_catalog_cross_thread_sqlite_close). Depending on what the policy is on repetition and what a single test should cover, I can delete the previous test / delete the close/garbage collection code in this test / etc.

warnings.simplefilter("always", ResourceWarning)
with caplog.at_level(logging.ERROR, logger="sqlalchemy.pool"):
catalog.close()

pool_errors = [
(record.exc_info[0], str(record.exc_info[1]))
for record in caplog.records
if record.exc_info and record.exc_info[0] is not None and record.exc_info[1] is not None
]
caplog.clear()
del catalog
gc.collect()

assert pool_errors == []
assert [str(w.message) for w in caught_warnings] == []


def test_sql_catalog_sqlite_explicit_check_same_thread_enforced(warehouse: Path, caplog: pytest.LogCaptureFixture) -> None:
"""Explicit check_same_thread=true enforces sqlite3 same-thread checks at runtime during close()."""
with ThreadPoolExecutor(max_workers=1) as executor:

def create_in_worker() -> tuple[SqlCatalog, sqlite3.Connection]:
cat = SqlCatalog("test", uri="sqlite:///:memory:?check_same_thread=true", warehouse=f"file://{warehouse}")
with cat.engine.connect() as conn:
dbapi_conn = cast(sqlite3.Connection, conn.connection.dbapi_connection)
return cat, dbapi_conn

strict_catalog, worker_conn = executor.submit(create_in_worker).result()
try:
with caplog.at_level(logging.ERROR, logger="sqlalchemy.pool"):
strict_catalog.close()
pool_errors = [record.exc_info[0] for record in caplog.records if record.exc_info and record.exc_info[0] is not None]
caplog.clear()
assert pool_errors == [sqlite3.ProgrammingError]
finally:
executor.submit(worker_conn.close).result()


@pytest.mark.parametrize(
("uri_template", "expected_pool_type", "expected_cparams"),
[
("sqlite:///:memory:?check_same_thread=true", SingletonThreadPool, {"check_same_thread": True}),
(
"sqlite:///{warehouse}/sql-catalog-explicit-thread?check_same_thread=true",
QueuePool,
{"check_same_thread": True},
),
(
"sqlite:///:memory:?timeout=15&cached_statements=50",
SingletonThreadPool,
{"timeout": 15.0, "cached_statements": 50, "check_same_thread": False},
),
],
)
def test_sql_catalog_sqlite_preserves_explicit_query_params(
warehouse: Path,
uri_template: str,
expected_pool_type: type[Pool],
expected_cparams: dict[str, object],
) -> None:
"""Explicit SQLite URI query parameters (including check_same_thread=true) must not be overridden."""
uri = uri_template.format(warehouse=warehouse.as_posix())
catalog = SqlCatalog(name="test", uri=uri, warehouse=f"file://{warehouse}")
with catalog:
assert isinstance(catalog.engine.pool, expected_pool_type)
catalog.engine.dispose()
seen_cparams: list[dict[str, object]] = []

@event.listens_for(catalog.engine, "do_connect")
def capture_cparams(_dialect: object, _conn_rec: object, _cargs: object, cparams: dict[str, object]) -> None:
seen_cparams.append(dict(cparams))

with catalog.engine.connect():
pass
assert seen_cparams == [expected_cparams]


def get_columns(engine: Engine) -> set[str]:
return {c["name"] for c in inspect(engine).get_columns("iceberg_tables")}
Expand Down
Loading