diff --git a/src/everos/core/persistence/lancedb/repository.py b/src/everos/core/persistence/lancedb/repository.py index 6454c032d..3dab3c123 100644 --- a/src/everos/core/persistence/lancedb/repository.py +++ b/src/everos/core/persistence/lancedb/repository.py @@ -224,11 +224,12 @@ def _remove_empty_index_dirs( retry clears. ``LanceError(IO): Execution error: Spill has sent an error`` is DataFusion's sort / merge spill to the OS temp dir failing mid-query; lancedb raises it as a bare ``RuntimeError``. Seen only on the Windows soak box under -nine concurrent clients (187 / 180 / 34 times over three runs), never on an +nine concurrent clients (187 / 180 / 34 / 42 times over four runs), never on an idle box, and the same row projected fine on the next attempt — yet the worker filed every one as unrecoverable, so ~200 md files per run needed a manual -``cascade fix``. Match the exact phrase: a generic IO error (disk full, file -gone) must stay permanent.""" +``cascade fix``. The traceback frames put it in ``merge_insert`` (the write +path, under :meth:`_locked`); the read path is covered as well. Match the +exact phrase: a generic IO error (disk full, file gone) must stay permanent.""" def _is_transient_execution_error(exc: BaseException) -> bool: @@ -390,6 +391,22 @@ async def _locked(self, budget: float, op: str) -> AsyncIterator[None]: f"{op} on table {self.table_name!r} exceeded its " f"{budget:g}s write-lock deadline" ) from exc + except RuntimeError as exc: + # The soak's spill failures came out of ``merge_insert`` (this + # path), not the reads: worker -> upsert -> execute_merge_insert. + # The write is idempotent by id, so a retry is the right answer. + if not _is_transient_execution_error(exc): + raise + logger.warning( + "lancedb_transient_execution_error", + table=self.table_name, + op=op, + error=str(exc)[:200], + ) + raise VectorStoreBusyError( + f"{op} on table {self.table_name!r} hit a transient lance " + f"execution error: {exc}" + ) from exc else: held = time.monotonic() - (acquired_at or started) if held >= _SLOW_HOLD_LOG_SECONDS: diff --git a/tests/unit/test_core/test_persistence/test_lancedb/test_transient_execution_errors.py b/tests/unit/test_core/test_persistence/test_lancedb/test_transient_execution_errors.py index 800241968..018929c80 100644 --- a/tests/unit/test_core/test_persistence/test_lancedb/test_transient_execution_errors.py +++ b/tests/unit/test_core/test_persistence/test_lancedb/test_transient_execution_errors.py @@ -41,6 +41,13 @@ async def test_spill_failure_inside_a_read_becomes_a_busy_error() -> None: raise RuntimeError(_SPILL) +async def test_spill_failure_inside_a_write_becomes_a_busy_error() -> None: + """The soak's spill failures came out of merge_insert (the write path).""" + with pytest.raises(VectorStoreBusyError, match="transient lance execution"): + async with _Repo()._locked(1.0, "upsert"): + raise RuntimeError(_SPILL) + + async def test_other_runtime_errors_still_propagate_unchanged() -> None: with pytest.raises(RuntimeError, match="No space left"): async with _Repo()._deadline(1.0, "find_where"):