Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions src/everos/core/persistence/lancedb/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,11 +224,12 @@ def _remove_empty_index_dirs(
retry clears. ``LanceError(IO): Execution error: Spill has sent an error`` is
DataFusion's sort / merge spill to the OS temp dir failing mid-query; lancedb
raises it as a bare ``RuntimeError``. Seen only on the Windows soak box under
nine concurrent clients (187 / 180 / 34 times over three runs), never on an
nine concurrent clients (187 / 180 / 34 / 42 times over four runs), never on an
idle box, and the same row projected fine on the next attempt — yet the worker
filed every one as unrecoverable, so ~200 md files per run needed a manual
``cascade fix``. Match the exact phrase: a generic IO error (disk full, file
gone) must stay permanent."""
``cascade fix``. The traceback frames put it in ``merge_insert`` (the write
path, under :meth:`_locked`); the read path is covered as well. Match the
exact phrase: a generic IO error (disk full, file gone) must stay permanent."""


def _is_transient_execution_error(exc: BaseException) -> bool:
Expand Down Expand Up @@ -390,6 +391,22 @@ async def _locked(self, budget: float, op: str) -> AsyncIterator[None]:
f"{op} on table {self.table_name!r} exceeded its "
f"{budget:g}s write-lock deadline"
) from exc
except RuntimeError as exc:
# The soak's spill failures came out of ``merge_insert`` (this
# path), not the reads: worker -> upsert -> execute_merge_insert.
# The write is idempotent by id, so a retry is the right answer.
if not _is_transient_execution_error(exc):
raise
logger.warning(
"lancedb_transient_execution_error",
table=self.table_name,
op=op,
error=str(exc)[:200],
)
raise VectorStoreBusyError(
f"{op} on table {self.table_name!r} hit a transient lance "
f"execution error: {exc}"
) from exc
else:
held = time.monotonic() - (acquired_at or started)
if held >= _SLOW_HOLD_LOG_SECONDS:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,13 @@ async def test_spill_failure_inside_a_read_becomes_a_busy_error() -> None:
raise RuntimeError(_SPILL)


async def test_spill_failure_inside_a_write_becomes_a_busy_error() -> None:
"""The soak's spill failures came out of merge_insert (the write path)."""
with pytest.raises(VectorStoreBusyError, match="transient lance execution"):
async with _Repo()._locked(1.0, "upsert"):
raise RuntimeError(_SPILL)


async def test_other_runtime_errors_still_propagate_unchanged() -> None:
with pytest.raises(RuntimeError, match="No space left"):
async with _Repo()._deadline(1.0, "find_where"):
Expand Down
Loading