From b0c0c62a2f01ab218c22f4a0264124f616e3abdf Mon Sep 17 00:00:00 2001 From: Alec Rosenbaum Date: Thu, 13 Aug 2026 16:58:04 +0000 Subject: [PATCH 1/4] Prevent swallowed timeouts from wedging sync workers --- tasktiger/timeouts.py | 11 ++++++++++- tests/test_timeouts.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/test_timeouts.py diff --git a/tasktiger/timeouts.py b/tasktiger/timeouts.py index af63e41c..e976fa9d 100644 --- a/tasktiger/timeouts.py +++ b/tasktiger/timeouts.py @@ -1,4 +1,5 @@ import signal +import sys from types import TracebackType from typing import Any, Literal, Optional, Type @@ -43,7 +44,15 @@ def cancel_death_penalty(self) -> None: class UnixSignalDeathPenalty(BaseDeathPenalty): + def __init__(self, timeout: float) -> None: + super().__init__(timeout) + self.retries = 0 + def handle_death_penalty(self, signum: int, frame: Any) -> None: + if self.retries >= 3: + sys.exit(1) + + self.retries += 1 raise JobTimeoutException( "Job exceeded maximum timeout value (%d seconds)." % self._timeout ) @@ -54,7 +63,7 @@ def setup_death_penalty(self) -> None: seconds). """ signal.signal(signal.SIGALRM, self.handle_death_penalty) - signal.setitimer(signal.ITIMER_REAL, self._timeout) + signal.setitimer(signal.ITIMER_REAL, self._timeout, 5) def cancel_death_penalty(self) -> None: """Removes the death penalty alarm and puts back the system into diff --git a/tests/test_timeouts.py b/tests/test_timeouts.py new file mode 100644 index 00000000..1b95568e --- /dev/null +++ b/tests/test_timeouts.py @@ -0,0 +1,42 @@ +import signal +from unittest.mock import Mock + +import pytest + +from tasktiger.exceptions import JobTimeoutException +from tasktiger.timeouts import UnixSignalDeathPenalty + + +def test_unix_signal_death_penalty_retries_before_exiting() -> None: + death_penalty = UnixSignalDeathPenalty(timeout=10) + + # Raise the regular timeout on the initial delivery and the first two + # interval retries. + for expected_retries in range(1, 4): + with pytest.raises(JobTimeoutException): + death_penalty.handle_death_penalty(signal.SIGALRM, None) + assert death_penalty.retries == expected_retries + + # If all three deliveries failed to unwind the job, force the process to + # exit when the interval timer delivers the third retry. + with pytest.raises(SystemExit) as exc_info: + death_penalty.handle_death_penalty(signal.SIGALRM, None) + + assert exc_info.value.code == 1 + + +def test_unix_signal_death_penalty_uses_interval_timer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + signal_mock = Mock() + setitimer_mock = Mock() + monkeypatch.setattr(signal, "signal", signal_mock) + monkeypatch.setattr(signal, "setitimer", setitimer_mock) + + death_penalty = UnixSignalDeathPenalty(timeout=10) + death_penalty.setup_death_penalty() + + signal_mock.assert_called_once_with( + signal.SIGALRM, death_penalty.handle_death_penalty + ) + setitimer_mock.assert_called_once_with(signal.ITIMER_REAL, 10, 5) From 44ceab6e50a89f638f9878c12b7c10b1fc798cca Mon Sep 17 00:00:00 2001 From: Alec Rosenbaum Date: Thu, 13 Aug 2026 17:08:26 +0000 Subject: [PATCH 2/4] Drop timeout unit tests --- tests/test_timeouts.py | 42 ------------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 tests/test_timeouts.py diff --git a/tests/test_timeouts.py b/tests/test_timeouts.py deleted file mode 100644 index 1b95568e..00000000 --- a/tests/test_timeouts.py +++ /dev/null @@ -1,42 +0,0 @@ -import signal -from unittest.mock import Mock - -import pytest - -from tasktiger.exceptions import JobTimeoutException -from tasktiger.timeouts import UnixSignalDeathPenalty - - -def test_unix_signal_death_penalty_retries_before_exiting() -> None: - death_penalty = UnixSignalDeathPenalty(timeout=10) - - # Raise the regular timeout on the initial delivery and the first two - # interval retries. - for expected_retries in range(1, 4): - with pytest.raises(JobTimeoutException): - death_penalty.handle_death_penalty(signal.SIGALRM, None) - assert death_penalty.retries == expected_retries - - # If all three deliveries failed to unwind the job, force the process to - # exit when the interval timer delivers the third retry. - with pytest.raises(SystemExit) as exc_info: - death_penalty.handle_death_penalty(signal.SIGALRM, None) - - assert exc_info.value.code == 1 - - -def test_unix_signal_death_penalty_uses_interval_timer( - monkeypatch: pytest.MonkeyPatch, -) -> None: - signal_mock = Mock() - setitimer_mock = Mock() - monkeypatch.setattr(signal, "signal", signal_mock) - monkeypatch.setattr(signal, "setitimer", setitimer_mock) - - death_penalty = UnixSignalDeathPenalty(timeout=10) - death_penalty.setup_death_penalty() - - signal_mock.assert_called_once_with( - signal.SIGALRM, death_penalty.handle_death_penalty - ) - setitimer_mock.assert_called_once_with(signal.ITIMER_REAL, 10, 5) From ee0c21a2e931dd7b9d322a1d4567c8f7704b4168 Mon Sep 17 00:00:00 2001 From: Alec Rosenbaum Date: Mon, 17 Aug 2026 18:55:40 +0000 Subject: [PATCH 3/4] Force exit after repeated timeout delivery --- tasktiger/timeouts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasktiger/timeouts.py b/tasktiger/timeouts.py index e976fa9d..a5debc24 100644 --- a/tasktiger/timeouts.py +++ b/tasktiger/timeouts.py @@ -1,5 +1,5 @@ +import os import signal -import sys from types import TracebackType from typing import Any, Literal, Optional, Type @@ -50,7 +50,7 @@ def __init__(self, timeout: float) -> None: def handle_death_penalty(self, signum: int, frame: Any) -> None: if self.retries >= 3: - sys.exit(1) + os._exit(1) self.retries += 1 raise JobTimeoutException( From 14544fadee684534779220d4d158ee7d08cfdc9f Mon Sep 17 00:00:00 2001 From: Alec Rosenbaum Date: Tue, 18 Aug 2026 15:45:50 +0000 Subject: [PATCH 4/4] Log before forcing timeout exit --- tasktiger/timeouts.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tasktiger/timeouts.py b/tasktiger/timeouts.py index a5debc24..40cf037b 100644 --- a/tasktiger/timeouts.py +++ b/tasktiger/timeouts.py @@ -50,7 +50,15 @@ def __init__(self, timeout: float) -> None: def handle_death_penalty(self, signum: int, frame: Any) -> None: if self.retries >= 3: - os._exit(1) + # Avoid logging locks because this runs in a signal handler. + try: + os.write( + 2, + b"Job timeout did not stop task after 3 attempts; " + b"exiting worker.\n", + ) + finally: + os._exit(1) self.retries += 1 raise JobTimeoutException(