From bf1c98baba7431ee6a60d5972c5759699473acad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Thu, 10 Jul 2025 12:48:10 +0600 Subject: [PATCH 001/169] Revert "Remove dependency on `pycurl` (#9526)" (#9620) This reverts commit 9bf05461dc8de9cb88f4279799e90e1dc0688196. --- requirements/extras/sqs.txt | 2 ++ requirements/test-ci-default.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements/extras/sqs.txt b/requirements/extras/sqs.txt index a7be017ff2f..78ba57ff78c 100644 --- a/requirements/extras/sqs.txt +++ b/requirements/extras/sqs.txt @@ -1,3 +1,5 @@ boto3>=1.26.143 +pycurl>=7.43.0.5,<7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version < "3.9" +pycurl>=7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version >= "3.9" urllib3>=1.26.16 kombu[sqs]>=5.5.0 diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt index e689866e245..78994fa8e45 100644 --- a/requirements/test-ci-default.txt +++ b/requirements/test-ci-default.txt @@ -21,4 +21,5 @@ git+https://github.com/celery/kombu.git # SQS dependencies other than boto -urllib3>=1.26.16 +pycurl>=7.43.0.5,<7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version < "3.9" +pycurl>=7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version >= "3.9" From b19cdbb707504af8c8d4f51bab2102d285329516 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Fri, 1 Aug 2025 02:40:02 +0300 Subject: [PATCH 002/169] Add Blacksmith Docker layer caching to all Docker builds (#9840) --- .github/workflows/docker.yml | 20 ++++++++++++++++++++ .github/workflows/python-package.yml | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f04a34cc2c..a3609aa3eba 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,6 +30,10 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 + - name: Setup Blacksmith Docker caching + uses: useblacksmith/build-push-action@v1 + with: + setup-only: true - name: Build Docker container run: make docker-build @@ -38,6 +42,10 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + - name: Setup Blacksmith Docker caching + uses: useblacksmith/build-push-action@v1 + with: + setup-only: true - name: "Build smoke tests container: dev" run: docker build -f t/smoke/workers/docker/dev . @@ -46,6 +54,10 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + - name: Setup Blacksmith Docker caching + uses: useblacksmith/build-push-action@v1 + with: + setup-only: true - name: "Build smoke tests container: latest" run: docker build -f t/smoke/workers/docker/pypi . @@ -54,6 +66,10 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + - name: Setup Blacksmith Docker caching + uses: useblacksmith/build-push-action@v1 + with: + setup-only: true - name: "Build smoke tests container: pypi" run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="5" . @@ -62,5 +78,9 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + - name: Setup Blacksmith Docker caching + uses: useblacksmith/build-push-action@v1 + with: + setup-only: true - name: "Build smoke tests container: legacy" run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="4" . diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 473f9b64e35..fbb15b23490 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -177,6 +177,10 @@ jobs: sudo sysctl -w vm.overcommit_memory=1 - uses: actions/checkout@v4 + - name: Setup Blacksmith Docker caching + uses: useblacksmith/build-push-action@v1 + with: + setup-only: true - name: Set up Python ${{ matrix.python-version }} uses: useblacksmith/setup-python@v6 with: From 04337f8bdf127279d5f62a46f08bb690f730b4b2 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Fri, 1 Aug 2025 03:20:04 +0300 Subject: [PATCH 003/169] Bump Kombu to v5.6.0b1 (#9839) --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index fc85b911128..7e4b1ea24bd 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,5 +1,5 @@ billiard>=4.2.1,<5.0 -kombu>=5.5.2,<5.6 +kombu>=5.6.0b1,<5.7 vine>=5.1.0,<6.0 click>=8.1.2,<9.0 click-didyoumean>=0.3.0 From 4841c99b27d0ac720805553b3e05be36c0a4f652 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 3 Aug 2025 03:09:14 +0300 Subject: [PATCH 004/169] Disable pytest-xdist for smoke tests and increase retries (CI ONLY) (#9842) --- .github/workflows/python-package.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fbb15b23490..69547c1f5b8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -195,8 +195,8 @@ jobs: - name: Run tox for "${{ matrix.python-version }}-smoke-${{ matrix.test-case }}" uses: nick-fields/retry@v3 with: - timeout_minutes: 30 - max_attempts: 2 - retry_wait_seconds: 0 + timeout_minutes: 20 + max_attempts: 5 + retry_wait_seconds: 60 command: | - tox --verbose --verbose -e "${{ matrix.python-version }}-smoke" -- -n auto -k ${{ matrix.test-case }} + tox --verbose --verbose -e "${{ matrix.python-version }}-smoke" -- -k ${{ matrix.test-case }} From 46443dc86df23fc2c0aacadbb98ce14160fc58ec Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 3 Aug 2025 23:02:47 +0300 Subject: [PATCH 005/169] Fix Python 3.13 compatibility in events dumper (#9826) Replace deprecated datetime.utcfromtimestamp() with datetime.fromtimestamp() using timezone.utc. The deprecated method was removed in Python 3.12+. Also fix test timezone handling to create proper UTC timestamps and update assertions to expect timezone-aware datetime format. Fixes failing tests: - test_on_event_task_received - test_on_event_non_task --- celery/events/dumper.py | 4 ++-- t/unit/events/test_dumper.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/celery/events/dumper.py b/celery/events/dumper.py index 24c7b3e9421..08ee12027ca 100644 --- a/celery/events/dumper.py +++ b/celery/events/dumper.py @@ -4,7 +4,7 @@ as they happen. Think of it like a `tcpdump` for Celery events. """ import sys -from datetime import datetime +from datetime import datetime, timezone from celery.app import app_or_default from celery.utils.functional import LRUCache @@ -48,7 +48,7 @@ def say(self, msg): pass def on_event(self, ev): - timestamp = datetime.utcfromtimestamp(ev.pop('timestamp')) + timestamp = datetime.fromtimestamp(ev.pop('timestamp'), timezone.utc) type = ev.pop('type').lower() hostname = ev.pop('hostname') if type.startswith('task-'): diff --git a/t/unit/events/test_dumper.py b/t/unit/events/test_dumper.py index e6f8a577e99..eb259db49d3 100644 --- a/t/unit/events/test_dumper.py +++ b/t/unit/events/test_dumper.py @@ -1,5 +1,5 @@ import io -from datetime import datetime +from datetime import datetime, timezone from celery.events import dumper @@ -39,7 +39,7 @@ def test_on_event_task_received(): buf = io.StringIO() d = dumper.Dumper(out=buf) event = { - 'timestamp': datetime(2024, 1, 1, 12, 0, 0).timestamp(), + 'timestamp': datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc).timestamp(), 'type': 'task-received', 'hostname': 'worker1', 'uuid': 'abc', @@ -49,7 +49,7 @@ def test_on_event_task_received(): } d.on_event(event.copy()) output = buf.getvalue() - assert 'worker1 [2024-01-01 12:00:00]' in output + assert 'worker1 [2024-01-01 12:00:00+00:00]' in output assert 'task received' in output assert 'mytask(abc) args=(1,) kwargs={}' in output @@ -58,13 +58,13 @@ def test_on_event_non_task(): buf = io.StringIO() d = dumper.Dumper(out=buf) event = { - 'timestamp': datetime(2024, 1, 1, 12, 0, 0).timestamp(), + 'timestamp': datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc).timestamp(), 'type': 'worker-online', 'hostname': 'worker1', 'foo': 'bar', } d.on_event(event.copy()) output = buf.getvalue() - assert 'worker1 [2024-01-01 12:00:00]' in output + assert 'worker1 [2024-01-01 12:00:00+00:00]' in output assert 'started' in output assert 'foo=bar' in output From a8ec7fafe15200fd84ab41b6289bf169338cc6d9 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Mon, 4 Aug 2025 06:31:33 +0300 Subject: [PATCH 006/169] Dockerfile Build Optimizations (#9733) * Dockerfile Build Optimizations * Update docker/Dockerfile * Review Fixes --------- Co-authored-by: Asif Saif Uddin --- Makefile | 2 +- docker/Dockerfile | 172 +++++++++++++++++++++++++--------------------- 2 files changed, 96 insertions(+), 78 deletions(-) diff --git a/Makefile b/Makefile index d28ac57dcf7..6e2eb420942 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ authorcheck: .PHONY: docker-build docker-build: - @docker compose -f docker/docker-compose.yml build + @DOCKER_BUILDKIT=1 docker compose -f docker/docker-compose.yml build .PHONY: docker-lint docker-lint: diff --git a/docker/Dockerfile b/docker/Dockerfile index 479613ac51f..36817c1d1cc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim -ENV PYTHONUNBUFFERED 1 -ENV PYTHONIOENCODING UTF-8 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 ARG DEBIAN_FRONTEND=noninteractive @@ -37,9 +37,10 @@ RUN apt-get update && apt-get install -y build-essential \ # Setup variables. Even though changing these may cause unnecessary invalidation of # unrelated elements, grouping them together makes the Dockerfile read better. -ENV PROVISIONING /provisioning +ENV PROVISIONING=/provisioning ENV PIP_NO_CACHE_DIR=off ENV PYTHONDONTWRITEBYTECODE=1 +ENV PIP_PREFER_BINARY=1 ARG CELERY_USER=developer @@ -47,7 +48,7 @@ ARG CELERY_USER=developer # Check for mandatory build arguments RUN : "${CELERY_USER:?CELERY_USER build argument needs to be set and non-empty.}" -ENV HOME /home/$CELERY_USER +ENV HOME=/home/$CELERY_USER ENV PATH="$HOME/.pyenv/bin:$PATH" # Copy and run setup scripts @@ -65,13 +66,13 @@ USER $CELERY_USER RUN curl https://pyenv.run | bash # Install required Python versions -RUN pyenv install 3.13 -RUN pyenv install 3.12 -RUN pyenv install 3.11 -RUN pyenv install 3.10 -RUN pyenv install 3.9 -RUN pyenv install 3.8 -RUN pyenv install pypy3.10 +RUN pyenv install 3.13 && \ + pyenv install 3.12 && \ + pyenv install 3.11 && \ + pyenv install 3.10 && \ + pyenv install 3.9 && \ + pyenv install 3.8 && \ + pyenv install pypy3.10 # Set global Python versions @@ -86,7 +87,8 @@ RUN chmod gu+x /entrypoint # Define the local pyenvs RUN pyenv local 3.13 3.12 3.11 3.10 3.9 3.8 pypy3.10 -RUN pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \ +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.12 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.11 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.10 -m pip install --upgrade pip setuptools wheel && \ @@ -94,18 +96,76 @@ RUN pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.8 -m pip install --upgrade pip setuptools wheel && \ pyenv exec pypy3.10 -m pip install --upgrade pip setuptools wheel -COPY --chown=1000:1000 . $HOME/celery +# Install requirements first to leverage Docker layer caching +# Split into separate RUN commands to reduce memory pressure and improve layer caching +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.13 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test-pypy3.txt \ + -r requirements/test.txt + +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.12 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test-pypy3.txt \ + -r requirements/test.txt + +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.11 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test-pypy3.txt \ + -r requirements/test.txt + +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.10 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test-pypy3.txt \ + -r requirements/test.txt -RUN pyenv exec python3.13 -m pip install -e $HOME/celery && \ - pyenv exec python3.12 -m pip install -e $HOME/celery && \ - pyenv exec python3.11 -m pip install -e $HOME/celery && \ - pyenv exec python3.10 -m pip install -e $HOME/celery && \ - pyenv exec python3.9 -m pip install -e $HOME/celery && \ - pyenv exec python3.8 -m pip install -e $HOME/celery && \ - pyenv exec pypy3.10 -m pip install -e $HOME/celery +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.9 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test-pypy3.txt \ + -r requirements/test.txt + +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.8 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test-pypy3.txt \ + -r requirements/test.txt -# Setup one celery environment for basic development use -RUN pyenv exec python3.13 -m pip install -r requirements/default.txt \ +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec pypy3.10 -m pip install -r requirements/default.txt \ -r requirements/dev.txt \ -r requirements/docs.txt \ -r requirements/pkgutils.txt \ @@ -113,61 +173,19 @@ RUN pyenv exec python3.13 -m pip install -r requirements/default.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.12 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.11 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.10 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.9 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.8 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec pypy3.10 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt + -r requirements/test.txt + +COPY --chown=1000:1000 . $HOME/celery + +# Install celery in editable mode (dependencies already installed above) +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.13 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.12 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.11 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.10 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.9 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.8 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec pypy3.10 -m pip install --no-deps -e $HOME/celery WORKDIR $HOME/celery From 6dcecbe52da8717c015203f5e0f6b8d684b6ccc9 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Tue, 5 Aug 2025 13:15:11 +0300 Subject: [PATCH 007/169] Migrated from useblacksmith/build-push-action@v1 to useblacksmith/setup-docker-builder@v1 in the CI (#9846) --- .github/workflows/docker.yml | 30 ++++++++++------------------ .github/workflows/python-package.yml | 6 ++---- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a3609aa3eba..a6cd26fbcd7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,10 +30,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 - - name: Setup Blacksmith Docker caching - uses: useblacksmith/build-push-action@v1 - with: - setup-only: true + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 - name: Build Docker container run: make docker-build @@ -42,10 +40,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - name: Setup Blacksmith Docker caching - uses: useblacksmith/build-push-action@v1 - with: - setup-only: true + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: dev" run: docker build -f t/smoke/workers/docker/dev . @@ -54,10 +50,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - name: Setup Blacksmith Docker caching - uses: useblacksmith/build-push-action@v1 - with: - setup-only: true + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: latest" run: docker build -f t/smoke/workers/docker/pypi . @@ -66,10 +60,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - name: Setup Blacksmith Docker caching - uses: useblacksmith/build-push-action@v1 - with: - setup-only: true + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: pypi" run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="5" . @@ -78,9 +70,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - name: Setup Blacksmith Docker caching - uses: useblacksmith/build-push-action@v1 - with: - setup-only: true + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: legacy" run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="4" . diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 69547c1f5b8..44a215a5efb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -177,10 +177,8 @@ jobs: sudo sysctl -w vm.overcommit_memory=1 - uses: actions/checkout@v4 - - name: Setup Blacksmith Docker caching - uses: useblacksmith/build-push-action@v1 - with: - setup-only: true + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 - name: Set up Python ${{ matrix.python-version }} uses: useblacksmith/setup-python@v6 with: From c2b4ad1b6c3a3ce601fd2c6dd5ce5cad084a10ce Mon Sep 17 00:00:00 2001 From: YuppY Date: Fri, 8 Aug 2025 23:14:16 +0500 Subject: [PATCH 008/169] Remove incorrect example pytest-celery is a plugin with a different API, this page is about celery.contrib.pytest plugin. --- docs/userguide/testing.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/userguide/testing.rst b/docs/userguide/testing.rst index 5b2a5761818..1a7f353830c 100644 --- a/docs/userguide/testing.rst +++ b/docs/userguide/testing.rst @@ -121,10 +121,9 @@ use in your integration (or unit) test suites. Enabling -------- -Celery initially ships the plugin in a disabled state, to enable it you can either: +Celery initially ships the plugin in a disabled state. To enable it, you can either: * ``pip install celery[pytest]`` - * ``pip install pytest-celery`` * or add an environment variable ``PYTEST_PLUGINS=celery.contrib.pytest`` * or add ``pytest_plugins = ("celery.contrib.pytest", )`` to your root conftest.py From da4a80dc449301fde4355153b47af8c42caed37c Mon Sep 17 00:00:00 2001 From: Dan LaManna Date: Sun, 10 Aug 2025 03:19:25 -0400 Subject: [PATCH 009/169] Revert "Use Django DB max age connection setting" (#9824) * Revert "Use Django DB max age connection setting" This reverts commit f0c9b40bd4aa7228afa20f589e50f2e4225d804e. This reverts PR #6134 and stops using the close_if_unusable_or_obsolete API since there are edge cases where it's unable to detect if a connection if actually unusable. This is most obvious when Celery interrupts a query in progress via a time limit handler. Django has marked this issue as wontfix (https://code.djangoproject.com/ticket/30646). Since this is effectively an optimization for Celery that can't be reliably used, Celery ought to close the connection after each task instead of trying to manage connections in a way similar to how the Django application does. * Ensure django fixup never calls close_if_unusable_or_obsolete This API can fail to close unusable connections in certain scenarios, namely database failovers and ungraceful terminations (e.g. signal handler for time limit exceeded tasks). This makes close_if_unusable_or_obsolete adequate for HTTP request lifecycle management but inappropriate for use within celery workers. See also: https://code.djangoproject.com/ticket/30646 https://forum.djangoproject.com/t/close-if-unusable-or-obsolete-fails-to-close-unusable-connections/41900 * Add test for close_cache --- celery/fixups/django.py | 9 +++----- t/unit/fixups/test_django.py | 41 +++++++++++++++--------------------- 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index b35499493a6..960077704e4 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -168,7 +168,7 @@ def on_worker_process_init(self, **kwargs: Any) -> None: self._maybe_close_db_fd(c) # use the _ version to avoid DB_REUSE preventing the conn.close() call - self._close_database(force=True) + self._close_database() self.close_cache() def _maybe_close_db_fd(self, c: "BaseDatabaseWrapper") -> None: @@ -197,13 +197,10 @@ def close_database(self, **kwargs: Any) -> None: self._close_database() self._db_recycles += 1 - def _close_database(self, force: bool = False) -> None: + def _close_database(self) -> None: for conn in self._db.connections.all(): try: - if force: - conn.close() - else: - conn.close_if_unusable_or_obsolete() + conn.close() except self.interface_errors: pass except self.DatabaseError as exc: diff --git a/t/unit/fixups/test_django.py b/t/unit/fixups/test_django.py index c09ba61642c..0d6ab1d83b3 100644 --- a/t/unit/fixups/test_django.py +++ b/t/unit/fixups/test_django.py @@ -196,7 +196,7 @@ def test_on_worker_process_init(self, patching): f.on_worker_process_init() mcf.assert_called_with(conns[1].connection) f.close_cache.assert_called_with() - f._close_database.assert_called_with(force=True) + f._close_database.assert_called_with() f.validate_models = Mock(name='validate_models') patching.setenv('FORKED_BY_MULTIPROCESSING', '1') @@ -264,38 +264,31 @@ def test__close_database(self): f._db.connections = Mock() # ConnectionHandler f._db.connections.all.side_effect = lambda: conns - f._close_database(force=True) + f._close_database() conns[0].close.assert_called_with() - conns[0].close_if_unusable_or_obsolete.assert_not_called() conns[1].close.assert_called_with() - conns[1].close_if_unusable_or_obsolete.assert_not_called() conns[2].close.assert_called_with() - conns[2].close_if_unusable_or_obsolete.assert_not_called() - - for conn in conns: - conn.reset_mock() - - f._close_database() - conns[0].close.assert_not_called() - conns[0].close_if_unusable_or_obsolete.assert_called_with() - conns[1].close.assert_not_called() - conns[1].close_if_unusable_or_obsolete.assert_called_with() - conns[2].close.assert_not_called() - conns[2].close_if_unusable_or_obsolete.assert_called_with() conns[1].close.side_effect = KeyError( 'omg') - f._close_database() - with pytest.raises(KeyError): - f._close_database(force=True) - - conns[1].close.side_effect = None - conns[1].close_if_unusable_or_obsolete.side_effect = KeyError( - 'omg') - f._close_database(force=True) with pytest.raises(KeyError): f._close_database() + def test_close_database_always_closes_connections(self): + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + f._db.connections.all = Mock(return_value=[conn]) + f.close_database() + conn.close.assert_called_once_with() + # close_if_unusable_or_obsolete is not safe to call in all conditions, so avoid using + # it to optimize connection handling. + conn.close_if_unusable_or_obsolete.assert_not_called() + + def test_close_cache_raises_error(self): + with self.fixup_context(self.app) as (f, _, _): + f._cache.close_caches.side_effect = AttributeError + f.close_cache() + def test_close_cache(self): with self.fixup_context(self.app) as (f, _, _): f.close_cache() From 7adc9e6afc132c5ced1678fb7b8ed09a8a68f07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?So=C3=B3s=20Tam=C3=A1s?= <39013301+tsoos99dev@users.noreply.github.com> Date: Mon, 11 Aug 2025 09:28:54 +0200 Subject: [PATCH 010/169] Fix pending_result memory leak (#9806) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add call to remove_pending_result, to counter add_pending_result in then. * Add unittest for checking if remove_pending_result is called after a call to forget. --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/result.py | 2 ++ t/smoke/tests/test_canvas.py | 12 ++++++++++++ t/unit/tasks/test_result.py | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/celery/result.py b/celery/result.py index 75512c5aadb..66a9e20aab8 100644 --- a/celery/result.py +++ b/celery/result.py @@ -137,6 +137,8 @@ def forget(self): self._cache = None if self.parent: self.parent.forget() + + self.backend.remove_pending_result(self) self.backend.forget(self.id) def revoke(self, connection=None, terminate=False, signal=None, diff --git a/t/smoke/tests/test_canvas.py b/t/smoke/tests/test_canvas.py index e0886d56e49..b6c69e76397 100644 --- a/t/smoke/tests/test_canvas.py +++ b/t/smoke/tests/test_canvas.py @@ -179,3 +179,15 @@ def test_chord_error_propagation_with_different_body_types( # The chord should fail with the expected exception from the failing task with pytest.raises(ExpectedException): result.get(timeout=RESULT_TIMEOUT) + + +class test_complex_workflow: + def test_pending_tasks_released_on_forget(self, celery_setup: CeleryTestSetup): + sig = add.si(1, 1) | group( + add.s(1) | group(add.si(1, 1), add.si(2, 2)) | add.si(2, 2), + add.s(1) | group(add.si(1, 1), add.si(2, 2)) | add.si(2, 2) + ) | add.si(1, 1) + res = sig.apply_async(queue=celery_setup.worker.worker_queue) + assert not all(len(mapping) == 0 for mapping in res.backend._pending_results) + res.forget() + assert all(len(mapping) == 0 for mapping in res.backend._pending_results) diff --git a/t/unit/tasks/test_result.py b/t/unit/tasks/test_result.py index 062c0695427..d5aaa481926 100644 --- a/t/unit/tasks/test_result.py +++ b/t/unit/tasks/test_result.py @@ -449,6 +449,20 @@ def test_date_done(self, utc_datetime_mock, timezone, date): result = Backend(app=self.app)._get_result_meta(None, states.SUCCESS, None, None) assert result.get('date_done') == date + def test_forget_remove_pending_result(self): + with patch('celery.result.AsyncResult.backend') as backend: + result = self.app.AsyncResult(self.task1['id']) + result.backend = backend + result_clone = copy.copy(result) + result.forget() + backend.remove_pending_result.assert_called_once_with( + result_clone + ) + + result = self.app.AsyncResult(self.task1['id']) + result.backend = None + del result + class test_ResultSet: From f4e2cf8138bcf8cb272d76216169001fd29566ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Tue, 12 Aug 2025 13:00:11 +0600 Subject: [PATCH 011/169] Update python-package.yml (#9856) --- .github/workflows/python-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 44a215a5efb..09f046aed55 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -52,7 +52,7 @@ jobs: if: startsWith(matrix.os, 'blacksmith-4vcpu-ubuntu') run: | sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: useblacksmith/setup-python@v6 with: @@ -117,7 +117,7 @@ jobs: run: | sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} uses: useblacksmith/setup-python@v6 with: @@ -176,7 +176,7 @@ jobs: sudo apt-get install -y procps # Install procps to enable sysctl sudo sysctl -w vm.overcommit_memory=1 - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Set up Python ${{ matrix.python-version }} From e906aae8d3e2956ff4f64047e29a1f58610a18fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 13:07:01 +0600 Subject: [PATCH 012/169] Bump actions/checkout from 4 to 5 (#9857) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/docker.yml | 10 +++++----- .github/workflows/linter.yml | 2 +- .github/workflows/semgrep.yml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 72078f37760..c4372c0848b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a6cd26fbcd7..d91264cf842 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,7 +29,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2204 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build Docker container @@ -39,7 +39,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2204 timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: dev" @@ -49,7 +49,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2204 timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: latest" @@ -59,7 +59,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2204 timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: pypi" @@ -69,7 +69,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2204 timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: legacy" diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 98a05f2b3a4..6f22274e9b7 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -8,7 +8,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 9078d214ff2..c33b7514c85 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -21,5 +21,5 @@ jobs: container: image: returntocorp/semgrep steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - run: semgrep ci From 33eb14852310996b1909c8388cd319809d6c8626 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Tue, 12 Aug 2025 14:22:22 +0300 Subject: [PATCH 013/169] Bump Kombu to v5.6.0b2 (#9858) --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 7e4b1ea24bd..015541462aa 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,5 +1,5 @@ billiard>=4.2.1,<5.0 -kombu>=5.6.0b1,<5.7 +kombu>=5.6.0b2,<5.7 vine>=5.1.0,<6.0 click>=8.1.2,<9.0 click-didyoumean>=0.3.0 From 777d92f9ba74080e0f2a2b4ed546f5883073aff6 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Tue, 26 Aug 2025 04:00:58 +0300 Subject: [PATCH 014/169] Refactor integration and smoke tests CI (#9855) * Disabled tests test_multiprocess_producer and test_multithread_producer * Refactor integration tests CI * Disable test_quorum_queue_qos_cluster_simulation.py * Reduce integration tests timeout from 30m -> 20m and increase attempts from 2 -> 3 (fail/retry faster) * Increase max attempts from 3 -> 5 with 1m break between each retry * TMP Dont wait for unit tests * Changed retry settings * Revert "Add xfail test for RabbitMQ quorum queue global QoS race condition (#9770)" This reverts commit 6d8bfd1d1d3031e8c198a834a3a7bcddb7266620. * Remove test_quorum_queue_qos_cluster_simulation.py from CI * Prevent the billiard QueueListener from deadlocking during worker shutdown * Revert "TMP Dont wait for unit tests" This reverts commit 3da612d8a64e6d1282bdd6137bc345b404bf12b5. * Run smoke if integration passed * Changed retry settings * Disable test_groupresult_serialization * timeout 5, attempts 10, instead of 30m x 2 attempts * Split integration test jobs to be test per module * Disabled dep with unit tests (for faster testing) * Run all integration jobs together * Split smoke test jobs * Simplifed python-package.yml * Cleanup * max-parallel: 4 * fixed smoked tests ci * Removed Python 3.10-3.12 from the integration and smoke tests CI * Integration & Smoke run only if unit tests pass * Reduced more python versions for now (integration min/max, smoke-max) * Revert "Prevent the billiard QueueListener from deadlocking during worker shutdown" This reverts commit cebcb2bf1a209a9fc9561400019f91a515d268d3. * Reapply "Add xfail test for RabbitMQ quorum queue global QoS race condition (#9770)" This reverts commit b10a55c69ebd5971945673f6f462b7c10cde7c78. * Added back `test_quorum_queue_qos_cluster_simulation` to the integration tests * max-parallel: 5 * Revert "Disable test_groupresult_serialization" This reverts commit ade8cbd6bc04d63dc21c9496a2e02d98c08dcdc8. --- .github/workflows/integration-tests.yml | 71 +++++++++++++ .github/workflows/python-package.yml | 126 ++++++------------------ .github/workflows/smoke-tests.yml | 57 +++++++++++ t/integration/test_tasks.py | 2 + 4 files changed, 160 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/integration-tests.yml create mode 100644 .github/workflows/smoke-tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000000..9bc35c1e40e --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,71 @@ +name: Integration Tests + +on: + workflow_call: + inputs: + module_name: + description: 'Name of the test module to run (e.g., test_backend.py)' + required: true + type: string + python_versions: + description: 'JSON array of Python versions to test' + required: false + type: string + default: '["3.8", "3.13"]' + tox_environments: + description: 'JSON array of tox environments to test' + required: false + type: string + default: '["redis", "rabbitmq", "rabbitmq_redis"]' + +jobs: + testing-with: + timeout-minutes: 240 + runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(inputs.python_versions) }} + toxenv: ${{ fromJson(inputs.tox_environments) }} + + services: + redis: + image: redis + ports: + - 6379:6379 + env: + REDIS_HOST: localhost + REDIS_PORT: 6379 + rabbitmq: + image: rabbitmq + ports: + - 5672:5672 + env: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + + steps: + - name: Install apt packages + run: | + sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev + + - uses: actions/checkout@v5 + - name: Set up Python ${{ matrix.python-version }} + uses: useblacksmith/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: 'pip' + cache-dependency-path: '**/setup.py' + - name: Install tox + run: python -m pip install --upgrade pip 'tox' tox-gh-actions + - name: > + Run tox for + "${{ matrix.python-version }}-integration-${{ matrix.toxenv }}-${{ inputs.module_name }}" + uses: nick-fields/retry@v3 + with: + timeout_minutes: 15 + max_attempts: 5 + retry_wait_seconds: 0 + command: | + tox --verbose --verbose -e "${{ matrix.python-version }}-integration-${{ matrix.toxenv }}" -- -k ${{ inputs.module_name }} -vv diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 09f046aed55..913d9a1089c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -83,71 +83,37 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} - Integration: - needs: - - Unit - if: needs.Unit.result == 'success' - timeout-minutes: 240 - - runs-on: blacksmith-4vcpu-ubuntu-2404 - strategy: - fail-fast: false - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] - toxenv: ['redis', 'rabbitmq', 'rabbitmq_redis'] - - services: - redis: - image: redis - ports: - - 6379:6379 - env: - REDIS_HOST: localhost - REDIS_PORT: 6379 - rabbitmq: - image: rabbitmq - ports: - - 5672:5672 - env: - RABBITMQ_DEFAULT_USER: guest - RABBITMQ_DEFAULT_PASS: guest - - steps: - - name: Install apt packages - run: | - sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev - - - uses: actions/checkout@v5 - - name: Set up Python ${{ matrix.python-version }} - uses: useblacksmith/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - cache: 'pip' - cache-dependency-path: '**/setup.py' - - name: Install tox - run: python -m pip install --upgrade pip 'tox' tox-gh-actions - - name: > - Run tox for - "${{ matrix.python-version }}-integration-${{ matrix.toxenv }}" - uses: nick-fields/retry@v3 - with: - timeout_minutes: 60 - max_attempts: 2 - retry_wait_seconds: 0 - command: | - tox --verbose --verbose -e "${{ matrix.python-version }}-integration-${{ matrix.toxenv }}" -vv + Integration-tests: + needs: [Unit] + if: needs.Unit.result == 'success' + strategy: + max-parallel: 5 + matrix: + module: [ + 'test_backend.py', + 'test_canvas.py', + 'test_inspect.py', + 'test_loader.py', + 'test_mem_leak_in_exception_handling.py', + 'test_quorum_queue_qos_cluster_simulation.py', + 'test_rabbitmq_chord_unlock_routing.py', + 'test_rabbitmq_default_queue_type_fallback.py', + 'test_security.py', + 'test_serialization.py', + 'test_tasks.py', + 'test_worker.py' + ] + uses: ./.github/workflows/integration-tests.yml + with: + module_name: ${{ matrix.module }} - Smoke: - needs: - - Unit + Smoke-tests: + needs: [Unit] if: needs.Unit.result == 'success' - runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: - fail-fast: false + max-parallel: 5 matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] - test-case: [ + module: [ 'test_broker_failover.py', 'test_worker_failover.py', 'test_native_delayed_delivery.py', @@ -163,38 +129,6 @@ jobs: 'test_thread_safe.py', 'test_worker.py' ] - - steps: - - name: Fetch Docker Images - run: | - docker pull redis:latest - docker pull rabbitmq:latest - - - name: Install apt packages - run: | - sudo apt update - sudo apt-get install -y procps # Install procps to enable sysctl - sudo sysctl -w vm.overcommit_memory=1 - - - uses: actions/checkout@v5 - - name: Setup Docker Builder - uses: useblacksmith/setup-docker-builder@v1 - - name: Set up Python ${{ matrix.python-version }} - uses: useblacksmith/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - allow-prereleases: true - cache: 'pip' - cache-dependency-path: '**/setup.py' - - - name: Install tox - run: python -m pip install --upgrade pip tox tox-gh-actions - - - name: Run tox for "${{ matrix.python-version }}-smoke-${{ matrix.test-case }}" - uses: nick-fields/retry@v3 - with: - timeout_minutes: 20 - max_attempts: 5 - retry_wait_seconds: 60 - command: | - tox --verbose --verbose -e "${{ matrix.python-version }}-smoke" -- -k ${{ matrix.test-case }} + uses: ./.github/workflows/smoke-tests.yml + with: + module_name: ${{ matrix.module }} diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml new file mode 100644 index 00000000000..27b4cff30ec --- /dev/null +++ b/.github/workflows/smoke-tests.yml @@ -0,0 +1,57 @@ +name: Smoke Tests + +on: + workflow_call: + inputs: + module_name: + description: 'Name of the test module to run (e.g., test_broker_failover.py)' + required: true + type: string + python_versions: + description: 'JSON array of Python versions to test' + required: false + type: string + default: '["3.13"]' + +jobs: + testing-with: + runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(inputs.python_versions) }} + + steps: + - name: Fetch Docker Images + run: | + docker pull redis:latest + docker pull rabbitmq:latest + + - name: Install apt packages + run: | + sudo apt update + sudo apt-get install -y procps # Install procps to enable sysctl + sudo sysctl -w vm.overcommit_memory=1 + + - uses: actions/checkout@v5 + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: useblacksmith/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: 'pip' + cache-dependency-path: '**/setup.py' + + - name: Install tox + run: python -m pip install --upgrade pip tox tox-gh-actions + + - name: Run tox for "${{ matrix.python-version }}-smoke-${{ inputs.module_name }}" + uses: nick-fields/retry@v3 + with: + timeout_minutes: 20 + max_attempts: 5 + retry_wait_seconds: 60 + command: | + tox --verbose --verbose -e "${{ matrix.python-version }}-smoke" -- -k ${{ inputs.module_name }} -n auto diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index 1f6a0499018..0dbb7708c53 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -98,6 +98,7 @@ def test_basic_task(self, manager): assert result.successful() is True @flaky + @pytest.mark.skip(reason="Broken test") def test_multiprocess_producer(self, manager): """Testing multiple processes calling tasks.""" set_multiprocessing_start_method() @@ -108,6 +109,7 @@ def test_multiprocess_producer(self, manager): assert list(ret) == list(range(120)) @flaky + @pytest.mark.skip(reason="Broken test") def test_multithread_producer(self, manager): """Testing multiple threads calling tasks.""" set_multiprocessing_start_method() From 31d05ed4d52807422dd0f8ba23345beba4ce28a1 Mon Sep 17 00:00:00 2001 From: Artem Darizhapov <90085271+temaxuck@users.noreply.github.com> Date: Tue, 26 Aug 2025 07:13:33 +0300 Subject: [PATCH 015/169] Fix `AsyncResult.forget()` with couchdb backend method raises `TypeError: a bytes-like object is required, not 'str'` (#9865) * fix: convert key to str in the couchdb backend delete() method * Add tests for backend results * Raise NotImplementedError instead of NotImplemented --- celery/backends/couchdb.py | 1 + t/unit/backends/test_couchdb.py | 98 ++++++++++++++++++++++++++++++++- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/celery/backends/couchdb.py b/celery/backends/couchdb.py index a4b040dab75..9cc7d7881f2 100644 --- a/celery/backends/couchdb.py +++ b/celery/backends/couchdb.py @@ -96,4 +96,5 @@ def mget(self, keys): return [self.get(key) for key in keys] def delete(self, key): + key = bytes_to_str(key) self.connection.delete(key) diff --git a/t/unit/backends/test_couchdb.py b/t/unit/backends/test_couchdb.py index 07497b18cec..bdae58f339a 100644 --- a/t/unit/backends/test_couchdb.py +++ b/t/unit/backends/test_couchdb.py @@ -1,8 +1,10 @@ from unittest.mock import MagicMock, Mock, sentinel +from urllib.parse import urlparse import pytest +from kombu.utils.encoding import str_to_bytes -from celery import states +from celery import states, uuid from celery.app import backends from celery.backends import couchdb as module from celery.backends.couchdb import CouchBackend @@ -115,3 +117,97 @@ def test_backend_params_by_url(self): assert x.username == 'johndoe' assert x.password == 'mysecret' assert x.port == 123 + + +class CouchSessionMock: + """ + Mock for `requests.session` that emulates couchdb storage. + """ + + _store = {} + + def request(self, method, url, stream=False, data=None, params=None, + headers=None, **kw): + tid = urlparse(url).path.split("/")[-1] + + response = Mock() + response.headers = {"content-type": "application/json"} + response.status_code = 200 + response.content = b'' + + if method == "GET": + if tid not in self._store: + return self._not_found_response() + response.content = self._store.get(tid) + elif method == "PUT": + self._store[tid] = data + response.content = str_to_bytes(f'{{"ok":true,"id":"{tid}","rev":"1-revid"}}') + elif method == "HEAD": + if tid not in self._store: + return self._not_found_response() + response.headers.update({"etag": "1-revid"}) + elif method == "DELETE": + if tid not in self._store: + return self._not_found_response() + del self._store[tid] + response.content = str_to_bytes(f'{{"ok":true,"id":"{tid}","rev":"1-revid"}}') + else: + raise NotImplementedError(f"CouchSessionMock.request() does not handle {method} method") + + return response + + def _not_found_response(self): + response = Mock() + response.headers = {"content-type": "application/json"} + response.status_code = 404 + response.content = str_to_bytes('{"error":"not_found","reason":"missing"}') + return response + + +class test_CouchBackend_result: + def setup_method(self): + self.backend = CouchBackend(app=self.app) + resource = pycouchdb.resource.Resource("resource-url", session=CouchSessionMock()) + self.backend._connection = pycouchdb.client.Database(resource, "container") + + def test_get_set_forget(self): + tid = uuid() + self.backend.store_result(tid, "successful-result", states.SUCCESS) + assert self.backend.get_state(tid) == states.SUCCESS + assert self.backend.get_result(tid) == "successful-result" + self.backend.forget(tid) + assert self.backend.get_state(tid) == states.PENDING + + def test_mark_as_started(self): + tid = uuid() + self.backend.mark_as_started(tid) + assert self.backend.get_state(tid) == states.STARTED + + def test_mark_as_revoked(self): + tid = uuid() + self.backend.mark_as_revoked(tid) + assert self.backend.get_state(tid) == states.REVOKED + + def test_mark_as_retry(self): + tid = uuid() + try: + raise KeyError('foo') + except KeyError as exception: + import traceback + trace = '\n'.join(traceback.format_stack()) + self.backend.mark_as_retry(tid, exception, traceback=trace) + assert self.backend.get_state(tid) == states.RETRY + assert isinstance(self.backend.get_result(tid), KeyError) + assert self.backend.get_traceback(tid) == trace + + def test_mark_as_failure(self): + tid = uuid() + try: + raise KeyError('foo') + except KeyError as exception: + import traceback + trace = '\n'.join(traceback.format_stack()) + self.backend.mark_as_failure(tid, exception, traceback=trace) + assert self.backend.get_state(tid) == states.FAILURE + assert isinstance(self.backend.get_result(tid), KeyError) + assert self.backend.get_traceback(tid) == trace From 6506ad3ea48eff4cdea0d541c09bd2709d28ce0b Mon Sep 17 00:00:00 2001 From: Blaise Muhirwa Date: Mon, 25 Aug 2025 21:17:08 -0700 Subject: [PATCH 016/169] improve docs for SQS authentication (#9868) --- .../backends-and-brokers/sqs.rst | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/getting-started/backends-and-brokers/sqs.rst b/docs/getting-started/backends-and-brokers/sqs.rst index 1e67bc2b58b..d391e790ffc 100644 --- a/docs/getting-started/backends-and-brokers/sqs.rst +++ b/docs/getting-started/backends-and-brokers/sqs.rst @@ -168,6 +168,48 @@ setting:: } } +.. warning:: + + **Important:** When using ``predefined_queues``, do NOT use URL-encoded + credentials (``safequote``) for the ``access_key_id`` and ``secret_access_key`` + values. URL encoding should only be applied to credentials in the broker URL. + + Using URL-encoded credentials in ``predefined_queues`` will cause signature + mismatch errors like: "The request signature we calculated does not match + the signature you provided." + +**Correct example combining broker URL and predefined queues:** + +.. code-block:: python + + import os + from kombu.utils.url import safequote + from celery import Celery + + # Raw credentials from environment + AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") + AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") + + # URL-encode ONLY for broker URL + aws_access_key_encoded = safequote(AWS_ACCESS_KEY_ID) + aws_secret_key_encoded = safequote(AWS_SECRET_ACCESS_KEY) + + # Use encoded credentials in broker URL + broker_url = f"sqs://{aws_access_key_encoded}:{aws_secret_key_encoded}@" + + celery_app = Celery("tasks", broker=broker_url) + celery_app.conf.broker_transport_options = { + "region": "us-east-1", + "predefined_queues": { + "my-queue": { + "url": "https://sqs.us-east-1.amazonaws.com/123456/my-queue", + # Use RAW credentials here (NOT encoded) + "access_key_id": AWS_ACCESS_KEY_ID, + "secret_access_key": AWS_SECRET_ACCESS_KEY, + }, + }, + } + When using this option, the visibility timeout should be set in the SQS queue (in AWS) rather than via the :ref:`visibility timeout ` option. From 6208dec2647870da33ec1e53fdb3f5629f32e092 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Tue, 26 Aug 2025 21:18:37 +0300 Subject: [PATCH 017/169] Added `.github/copilot-instructions.md` for GitHub Copilot (#9874) --- .github/copilot-instructions.md | 567 ++++++++++++++++++++++++++++++++ 1 file changed, 567 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000000..bab8f8dcd2e --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,567 @@ +# GitHub Copilot PR Review Guide + +Conservative, question-first review guidance to keep feedback useful, low-noise, and maintainable for a large, long-lived project. + +## Purpose and scope + +- Role: Assist maintainers during PR reviews with concise, question-first feedback that nudges good decisions and documents rationale for posterity. +- Objectives: Surface user-facing behavior changes needing docs; highlight backward-compatibility risks; keep scope focused; encourage consistency and cleanup; optionally suggest tests and local tox usage. +- Principles: Very-high confidence or question-first; bottom-line first; avoid style/lint remarks; avoid prescriptive internal rules unless unambiguous; minimize noise. +- When to ask vs. assert: Ask by default; assert only for obvious issues (e.g., debug leftovers) or when a strict rule clearly applies. +- When to stay silent: Formatting-only changes, comments-only diffs, tests-only edits, or strictly internal refactors with no user-facing impact. + +### What "question-first" means + +- Default to asking when not 90%+ confident; assert only for obvious issues or clear, documented policy. +- Lead with a concise question that contains the bottom-line ask and one-sentence rationale. +- Make it easy to answer: yes/no + suggested next step (e.g., "Should we add versionchanged::?"). +- Avoid prescribing exact code; clarify intent and offer options when needed. +- If confirmed user-facing, follow docs/versioning guidance; if internal-only, prefer consistency and brief rationale. +- One comment per theme; do not repeat after it is addressed. + +## Collaboration contract (Copilot alongside maintainers) + +- Assist maintainers; do not decide. Questions by default; assertions only on clear policy violations or obvious mistakes. +- Never block the review; comments are non-binding prompts for the human reviewer. +- Keep comments atomic and actionable; include the bottom-line ask and, when helpful, a suggested next step. +- Avoid prescriptive code changes unless asked; prefer intent-focused guidance and options. +- Respect repository conventions and CI; skip style/lint feedback that automation enforces. +- Ask once per theme and stop after it's addressed; avoid repetition and noise. + +## Reviewer persona and behavior + +- Prefer question-first comments; assert only with very-high confidence. +- Bottom line first, then brief rationale, then the ask. +- Avoid style/lint remarks (CI handles these). +- Avoid prescriptive internal rules unless policy is unambiguous. +- Keep comments short, technical, specific. + +## Response formatting for Copilot + +- Use standard GitHub Markdown in comments; keep them concise and technical. +- Use fenced code blocks with explicit language where possible: ```diff, ```python, ```sh, ```yaml, ```toml, ```ini, ```rst, or ```text. +- Prefer small unified diffs (```diff) when referencing exact changes; include only the minimal hunk needed. +- Avoid emojis and decorative formatting; focus on clarity and actionability. +- One comment per theme; avoid repetition once addressed. +- When referencing files/lines, include a GitHub permalink to exact lines or ranges (Copy permalink) using commit-SHA anchored URLs, e.g., https://github.com/celery/celery/blob//celery/app/base.py#L820-L860. + +## High-signal focus areas (question-first by default) + +### 1) Backward compatibility risk + +Triggers include: +- Signature/default changes in user-facing APIs (added/removed/renamed params; changed defaults; narrowed/broadened accepted types). +- Return type/shape/order changes (e.g., list -> iterator/generator; tuple -> dict; stable order -> undefined order). +- Exceptions/validation changes (exception type changed; now raises where it previously passed). +- Config/CLI/ENV defaults that alter behavior (e.g., task_acks_late, timeouts, default_queue/default_exchange/default_routing_key, CLI flag defaults). +- Wire/persistence schema changes (task headers/stamping, message/result schema, serialization/content type, visibility-timeout semantics). +- Removing/deprecating public APIs without a documented deprecation window, alias, or compatibility layer. + +What to look for (detectors): +- Param removed/renamed or default flipped in a public signature (or apply_async/send_task options). +- Return type/shape/order changed in code, docstrings, or tests (yield vs list; mapping vs tuple). +- Exception types changed in raise paths or surfaced in tests/docs. +- Defaults changed in celery/app/defaults.py or via config/CLI/ENV resolution. +- Changes to headers/stamps/message body/result schema or serialization in amqp/backend paths. +- Public symbol/behavior removal with no deprecation entry. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This appears to be a user-facing behavior change (X -> Y), which could break existing users because . Is this intended?" + - If yes: Could we add migration guidance in the PR description and docs (versionchanged::), and consider a compat/deprecation path (e.g., alias/flag) through vX.Y? + - If no: Would reverting to the previous behavior and adding a regression test make sense, or alternatively guarding this behind a feature flag until we can provide a proper deprecation path?" + +Examples: +- Case A: Config default change (task_acks_late) + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- acks_late=Option(False, type='bool'), ++ acks_late=Option(True, type='bool'), + ``` + + - Why it matches: Flipping this default changes when tasks are acknowledged; can impact delivery semantics, retries, and failure handling for users not explicitly setting it. + - Example comment: "I see task_acks_late default changed False -> True; this could change delivery/retry semantics for users relying on the current default. Is this intended? If yes, could we add migration guidance and a versionchanged:: entry, and consider a transition plan (e.g., keep False unless explicitly opted in) through vX.Y? If not, should we revert and add a regression test?" + +- Case B: Return type change (list -> iterator) + - Diff (illustrative): + + ```diff + --- a/celery/app/builtins.py + +++ b/celery/app/builtins.py + @@ +- return [task(item) for item in it] ++ return (task(item) for item in it) + ``` + + - Why it matches: Changing to a generator would break callers that rely on len(), indexing, multiple passes, or list operations. + - Example comment: "I see the return type changed from list to iterator; this can break callers relying on len() or multiple passes. Is this intended? If yes, could we document (versionchanged::), add migration notes, and consider returning a list for one release or gating behind an opt-in flag? If not, let's keep returning a list and add a test to prevent regressions." + +- Case C: Exception type change (TypeError -> ValueError) on argument checking + - Diff (illustrative): + + ```diff + --- a/celery/some_module.py + +++ b/celery/some_module.py + @@ +- raise TypeError("bad arguments") ++ raise ValueError("bad arguments") + ``` + + - Why it matches: Changing the raised exception type breaks existing handlers and test expectations that catch TypeError. + - Example comment: "I see the raised exception changed TypeError -> ValueError; this can break existing error handlers/tests. Is this intended? If yes, could we document with versionchanged:: and suggest catching both for a transition period? If not, keep TypeError and add a test ensuring the type stays consistent." + +- Case D: Routing defaults change that silently reroutes tasks + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- default_queue=Option('celery'), ++ default_queue=Option('celery_v2'), + ``` + + - Why it matches: Changing default_queue (or introducing a non-None default in a call path) can reroute tasks for users who did not specify queue explicitly. + - Example comment: "I see default_queue changed 'celery' -> 'celery_v2'; this may silently reroute tasks for users not specifying queue. Is this intended? If yes, please add migration guidance and a versionchanged:: entry, and consider keeping a compat alias or opt-in flag through vX.Y. If not, revert and add a regression test verifying routing is unchanged when queue is omitted." + +### 2) Documentation versioning (strict but question-first) + +Triggers include: +- New/removed/renamed configuration setting or environment variable. +- Changed default of a documented setting. +- Behavior change in a documented feature (signals, CLI flags, return values, error behavior). +- Added/removed/renamed parameter in a documented API that users call directly. + +What to look for (detectors): +- Defaults changed in celery/app/defaults.py or docs without corresponding docs/whatsnew updates. +- Missing Sphinx directives (versionchanged::/versionadded::) in relevant docs when behavior/settings change. +- Public signatures changed (method/function params) without doc updates or deprecation notes. +- CLI help/defaults changed without docs alignment. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This appears to be a user-facing change (X -> Y). Is this intended? + - If yes: Should we add docs updates (versionchanged::/versionadded::) and a short migration note? + - If no: Should we revert or adjust the docs/code so they remain consistent until we can introduce a documented change?" + +Examples: +- Case A: Changed default of a documented setting (task_time_limit) + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- task_time_limit=Option(300, type='int'), ++ task_time_limit=Option(600, type='int'), + ``` + + - Why it matches: The default is documented and affects runtime behavior; changing it impacts users who relied on the previous default. + - Example comment: "I see task_time_limit default changed 300 -> 600; is this intended? If yes, should we add versionchanged:: in the docs and a brief migration note? If not, should we revert or defer behind a release note with guidance?" + +- Case B: New setting introduced (CELERY_FOO) + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ ++ foo=Option(False, type='bool'), # new + ``` + + - Why it matches: New documented configuration requires docs (usage, default, examples) and possibly a whatsnew entry. + - Example comment: "A new setting (celery.foo) is introduced. Should we add docs (reference + usage) and a versionadded:: note?" + +- Case C: Public API parameter renamed + - Diff (illustrative): + + ```diff + --- a/celery/app/task.py + +++ b/celery/app/task.py + @@ +- def apply_async(self, args=None, kwargs=None, routing_key=None, **options): ++ def apply_async(self, args=None, kwargs=None, route_key=None, **options): + ``` + + - Why it matches: Renamed parameter breaks user code and docs; requires docs changes and possibly a deprecation alias. + - Example comment: "apply_async param routing_key -> route_key is user-facing. Is this intended? If yes, can we add docs updates (versionchanged::) and consider an alias/deprecation path? If not, should we keep routing_key and add a regression test?" + +### 3) Scope and coherence + +Triggers include: +- Mixed concerns in a single PR (refactor/move/rename + behavior change). +- Large formatting sweep bundled with functional changes. +- Multiple unrelated features or modules changed together. + +What to look for (detectors): +- File renames/moves and non-trivial logic changes in the same PR. +- Many formatting-only hunks (whitespace/quotes/import order) mixed with logic edits. +- Multiple features or modules modified without a unifying rationale. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This PR appears to mix refactor/moves with functional changes. Would splitting the concerns improve focus and reviewability? + - If yes: Could we split into (A) refactor-only and (B) behavior change, or at least separate commits? + - If no: Could we provide a brief rationale and ensure commit messages clearly separate concerns?" + +Examples: +- Case A: Move + behavior change in the same change + - Diff (illustrative): + + ```diff + --- a/celery/old_module.py + +++ b/celery/new_module.py + @@ +- def handle(msg): +- return process(msg) ++ def handle(msg): ++ if msg.priority > 5: ++ return fast_path(msg) ++ return process(msg) + ``` + + - Why it matches: Relocation plus logic change complicates review and rollback. + - Example comment: "This includes both move and behavior change. Could we split the move (no-op) and the logic change into separate commits/PRs?" + +- Case B: Formatting sweep + logic change + - Diff (illustrative): + + ```diff + --- a/celery/module.py + +++ b/celery/module.py + @@ +- def f(x,y): return x+y ++ def f(x, y): ++ return x + y ++ ++ def g(x): ++ return x * 2 # new behavior + ``` + + - Why it matches: Formatting noise hides behavior changes. + - Example comment: "There is a formatting sweep plus a new function. Could we isolate logic changes so the diff is high-signal?" + +- Case C: Unrelated rename grouped with feature + - Diff (illustrative): + + ```diff + --- a/celery/feature.py + +++ b/celery/feature.py + @@ +- def add_user(u): ++ def create_user(u): # rename + ... + --- a/celery/other.py + +++ b/celery/other.py + @@ ++ def implement_new_queue(): ++ ... + ``` + + - Why it matches: Unrelated rename grouped with new feature reduces clarity. + - Example comment: "Can we separate the rename from the new feature so history and review stay focused?" + +### 4) Debug/development leftovers + +Triggers include: +- `print`, `pdb`/`breakpoint()`, commented-out blocks, temporary tracing/logging. +- Accidental debug helpers left in code (timers, counters). + +What to look for (detectors): +- `import pdb`, `pdb.set_trace()`, `breakpoint()`; new `print()` statements. +- `logger.debug(...)` with TODO/temporary text; excessive logging added. +- Large commented-out blocks or dead code left behind. +- Unused variables added for debugging only. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This looks like debug/temporary code. Can we remove it before merge? + - If yes: Please drop these lines (or guard behind a verbose flag). + - If no: Could you share why it’s needed and add a comment/guard to ensure it won’t leak in production?" + +Examples: +- Case A: Interactive debugger left in + - Diff (illustrative): + + ```diff + --- a/celery/worker.py + +++ b/celery/worker.py + @@ ++ import pdb ++ pdb.set_trace() + ``` + + - Why it matches: Debugger halts execution in production. + - Example comment: "Debugger calls found; can we remove them before merge?" + +- Case B: Temporary print/log statements + - Diff (illustrative): + + ```diff + --- a/celery/module.py + +++ b/celery/module.py + @@ +- result = compute(x) ++ result = compute(x) ++ print("DEBUG:", result) + ``` + + - Why it matches: Adds noisy output; not suitable for production. + - Example comment: "Temporary prints detected; could we remove or convert to a guarded debug log?" + +- Case C: Commented-out block + - Diff (illustrative): + + ```diff + --- a/celery/module.py + +++ b/celery/module.py + @@ ++ # old approach ++ # data = fetch_old() ++ # process_old(data) + ``` + + - Why it matches: Dead code should be removed for clarity and git history provides recovery. + - Example comment: "Large commented block detected; can we remove it and rely on git history if needed?" + +### 5) "Cover the other ends" for fixes + +Triggers include: +- Fix applied in one place while similar call sites/patterns remain elsewhere. +- Fix made in a wrapper/entry-point but not in the underlying helper used elsewhere. + +What to look for (detectors): +- Duplicate/similar functions that share the same bug but were not updated. +- Shared helpers where only one call path was fixed. +- Tests cover only the changed path but not sibling paths. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This fix updates one call site, but similar sites seem to exist (A/B). Were those reviewed? + - If yes: Could we update them in this PR or in a follow-up with references? + - If no: Would you like pointers on where similar patterns live (grep/symbol refs)?" + +Examples: +- Case A: Fix applied to one module; another equivalent module remains unchanged + - Diff (illustrative): + + ```diff + --- a/celery/foo.py + +++ b/celery/foo.py + @@ +- result = do_work(x) ++ result = do_work(x, safe=True) + ``` + + - Why it matches: bar.py uses the same pattern and likely needs the same safety flag. + - Example comment: "foo.py updated to pass safe=True; bar.py appears to call do_work similarly without the flag. Should we update bar.py too or open a follow-up?" + +- Case B: Wrapper fixed, helper not fixed + - Diff (illustrative): + + ```diff + --- a/celery/api.py + +++ b/celery/api.py + @@ +- def submit(task): +- return _publish(task) ++ def submit(task): ++ return _publish(task, retry=True) + ``` + + - Why it matches: Other entry points call _publish directly and still miss retry=True. + - Example comment: "submit() now passes retry=True, but direct _publish callers won't. Should we fix those call sites or update _publish's default?" + +### 6) Consistency and organization (not lint/style) + +Triggers include: +- New code diverges from nearby structural patterns (module layout, naming, docstrings, imports organization). +- Logger usage/structure differs from the rest of the module. +- Module/API structure inconsistent with sibling modules. + +What to look for (detectors): +- Different naming conventions (CamelCase vs snake_case) near similar code. +- Docstring style/sections differ from adjacent functions/classes. +- Logger names/patterns inconsistent with module-level practice. +- Module splitting/placement differs from sibling feature modules without rationale. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This code diverges from nearby patterns (X). Was that intentional? + - If yes: Could we add a brief rationale in the PR description and consider documenting the new pattern? + - If no: Should we align with the surrounding approach for consistency?" + +Examples: +- Case A: Naming deviates from local convention + - Diff (illustrative): + + ```diff + --- a/celery/jobs.py + +++ b/celery/jobs.py + @@ +- def CreateTask(payload): ++ def create_task(payload): + ... + ``` + + - Why it matches: Local code uses snake_case; CamelCase function name is inconsistent. + - Example comment: "Local convention is snake_case; should we rename to create_task for consistency?" + +- Case B: Logger name/prefix inconsistent + - Diff (illustrative): + + ```diff + --- a/celery/worker.py + +++ b/celery/worker.py + @@ +- log = logging.getLogger("celery.worker") ++ log = logging.getLogger("celery.custom") + ``` + + - Why it matches: Module logger naming differs from the standard. + - Example comment: "Module loggers typically use 'celery.worker'; should we align the logger name here?" + +- Case C: Module layout divergence + - Diff (illustrative): + + ```diff + --- a/celery/feature/__init__.py + +++ b/celery/feature/__init__.py + @@ ++ from .impl import Feature # new public import + ``` + + - Why it matches: New public import/path differs from sibling modules. + - Example comment: "Exposing Feature at package root differs from siblings; was that intentional, or should we keep imports local?" + +### 7) Tests and local workflow (optional nudges) + +Triggers include: +- Behavior change, bug fix, or CI failures without corresponding tests/updates. + +What to look for (detectors): +- Code changes that alter behavior with no new/updated tests. +- API/signature changes with tests still asserting old behavior. +- Failing CI areas that need local reproduction guidance. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "Since behavior changes here, could we add/update a focused unit test that fails before and passes after? + - If yes: A small unit test should suffice; consider narrowing with -k. + - If no: Could you share rationale (e.g., covered by integration/smoke), and note how to reproduce locally?" + +Suggested commands: +- `tox -e lint` +- `tox -e 3.13-unit` +- `tox -e 3.13-integration-rabbitmq_redis` (ensure local RabbitMQ and Redis containers are running) +- `tox -e 3.13-smoke -- -n auto` +- Narrow scope: `tox -e 3.13-unit -- -k ` + +Examples: +- Case A: Bug fix without a regression test + - Diff (illustrative): + + ```diff + --- a/celery/utils.py + +++ b/celery/utils.py + @@ +- return retry(task) ++ return retry(task, backoff=True) + ``` + + - Why it matches: Behavior changed; add a unit test asserting backoff path. + - Example comment: "New backoff behavior added; can we add a unit test that fails before and passes after this change?" + +- Case B: API/signature changed; tests not updated + - Diff (illustrative): + + ```diff + --- a/celery/app/task.py + +++ b/celery/app/task.py + @@ +- def apply_async(self, args=None, kwargs=None, routing_key=None, **options): ++ def apply_async(self, args=None, kwargs=None, route_key=None, **options): + ``` + + - Why it matches: Tests/callers may still pass routing_key. + - Example comment: "apply_async param rename detected; can we update tests and add a note in the PR description on migration?" + +- Case C: Provide local reproduction guidance for CI failures + - Example comment: "CI failures indicate tests in module X. To iterate locally: + - `tox -e 3.13-unit -- -k ` + - If integration-related: `tox -e 3.13-integration-rabbitmq_redis` (ensure services run) + - For smoke: `tox -e 3.13-smoke -- -n auto`" + +### 8) Ecosystem awareness (non-prescriptive) + +Triggers include: +- Changes to internal components or cross-project boundaries (kombu/amqp, backends, transports). +- Acknowledge/visibility-timeout semantics modified; stamped headers or message schema altered. +- Serialization/content-type defaults changed; transport-specific behavior altered. + +What to look for (detectors): +- Edits to amqp producer/consumer internals; ack/requeue/visibility logic. +- Changes to stamped_headers handling or task message headers/body schema. +- Defaults that affect interop (content_type/serializer, queue types, exchange kinds). + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This touches internal messaging/interop semantics and may affect the ecosystem. Could you share the rationale and cross-component considerations? + - If yes: Could we add focused tests (publish/consume round-trip) and a brief docs/whatsnew note? + - If no: Should we revert or gate behind a feature flag until we coordinate across components?" + +Examples: +- Case A: Stamped headers behavior changed + - Diff (illustrative): + + ```diff + --- a/celery/app/base.py + +++ b/celery/app/base.py + @@ +- stamped_headers = options.pop('stamped_headers', []) ++ stamped_headers = options.pop('stamped_headers', ['trace_id']) + ``` + + - Why it matches: Default stamped headers alter on-the-wire metadata; other tools may not expect it. + - Example comment: "Default stamped_headers now include 'trace_id'; is this intended? If yes, can we add tests/docs and note interop impact? If not, should we keep [] and document opt-in?" + +- Case B: Ack/visibility semantics tweaked + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- acks_on_failure_or_timeout=Option(True, type='bool'), ++ acks_on_failure_or_timeout=Option(False, type='bool'), + ``` + + - Why it matches: Changes worker/broker interaction; can affect redelivery and failure semantics. + - Example comment: "acks_on_failure_or_timeout True -> False affects redelivery; is this intended? If yes, could we add tests and a docs note? If not, revert and add a regression test?" + +- Case C: Serialization/content-type default changed + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- serializer=Option('json'), ++ serializer=Option('yaml'), + ``` + + - Why it matches: Affects compatibility with consumers/producers; security considerations for yaml. + - Example comment: "Serializer default json -> yaml changes interop/security profile. Is this intended? If yes, please document risks and add tests; if not, keep json." + +## What to avoid commenting on + +- Style/formatting/line length (lint/CI already enforce repo standards). +- Dependency management specifics. +- Over-specific internal patterns unless explicitly documented policy. +- Repeating the same point after it has been addressed. + +## Noise control (without hard caps) + +- Group related questions into one concise comment per theme when possible. +- Ask once per issue; don't repeat after the contributor responds/updates. +- Skip commentary on pure formatting, comment-only diffs, tests-only edits, or private helper refactors with no user-facing impact. + +## PR title and description (nice-to-have) + +- If title/description don't reflect the change, suggest a concise rewrite that helps future "What's New" compilation - helpful, never blocking. From 3703beb4589b4d590b84a1b08a8d6558149b6f4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A4m?= Date: Sat, 30 Aug 2025 09:09:01 +0200 Subject: [PATCH 018/169] misc: credits (#9877) --- docs/history/changelog-4.3.rst | 4 ---- docs/history/changelog-4.4.rst | 4 ---- docs/history/whatsnew-4.3.rst | 1 - 3 files changed, 9 deletions(-) diff --git a/docs/history/changelog-4.3.rst b/docs/history/changelog-4.3.rst index 0502c1de09e..ad3f6d9e2a6 100644 --- a/docs/history/changelog-4.3.rst +++ b/docs/history/changelog-4.3.rst @@ -339,8 +339,6 @@ Documentation Fixes by: we introduced this new configuration option to specify the accepted content from the backend. - Contributed by **Benjamin Pereto** - - **Canvas**: Fixed error callback processing for class based tasks. Contributed by **Victor Mireyev** @@ -368,8 +366,6 @@ Documentation Fixes by: We now depend on cryptography instead of pyOpenSSL for this serializer. - Contributed by **Benjamin Pereto** - - **Command Line**: :program:`celery report` now reports kernel version along with other platform details. diff --git a/docs/history/changelog-4.4.rst b/docs/history/changelog-4.4.rst index e6a851676cd..4ed3c79a2ac 100644 --- a/docs/history/changelog-4.4.rst +++ b/docs/history/changelog-4.4.rst @@ -556,8 +556,6 @@ Documentation Fixes by: we introduced this new configuration option to specify the accepted content from the backend. - Contributed by **Benjamin Pereto** - - **Canvas**: Fixed error callback processing for class based tasks. Contributed by **Victor Mireyev** @@ -585,8 +583,6 @@ Documentation Fixes by: We now depend on cryptography instead of pyOpenSSL for this serializer. - Contributed by **Benjamin Pereto** - - **Command Line**: :program:`celery report` now reports kernel version along with other platform details. diff --git a/docs/history/whatsnew-4.3.rst b/docs/history/whatsnew-4.3.rst index 230d751c5f6..27de377998d 100644 --- a/docs/history/whatsnew-4.3.rst +++ b/docs/history/whatsnew-4.3.rst @@ -92,7 +92,6 @@ Artem Vasilyev Asif Saif Uddin (Auvi) aviadatsnyk Axel Haustant -Benjamin Pereto Bojan Jovanovic Brett Jackson Brett Randall From 166f705adcae57a43423b0ae7286ab828b55b244 Mon Sep 17 00:00:00 2001 From: Michele Ghirardelli <50736672+ghirailghiro@users.noreply.github.com> Date: Sun, 31 Aug 2025 07:00:21 +0200 Subject: [PATCH 019/169] Choose queue type and exchange type when creating missing queues (fix #9671) (#9815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add two new settings that apply when Celery autogenerates a queue (`task_create_missing_queues=True`): *`task_create_missing_queue_type` *`task_create_missing_queue_exchange_type` Backwards compatibility: default behaviour (classic queue + direct exchange) is unchanged. Closes #9671 * Update celery/app/amqp.py * feat: add configurable durable/exclusive options for control and event queues - Added `event_queue_durable` and `event_queue_exclusive` settings. - Added `control_exchange_durable` and `control_exchange_exclusive` settings. - Updated `EventReceiver` and `Control` to support these options. - Prevented invalid config: both options cannot be True at the same time. - Added related tests and updated documentation accordingly. This commit fix Issue (#9759) on the celery part * fix(control): raise ImproperlyConfigured if both control_queue_durable and control_queue_exclusive are True Prevent misconfiguration by raising ImproperlyConfigured in Control when both control_queue_durable and control_queue_exclusive options are enabled. * Update docs/userguide/configuration.rst * Update docs/userguide/configuration.rst * Update celery/events/receiver.py * Update celery/app/amqp.py * Update celery/app/amqp.py * Update receiver.py fixing cfg to self.app.conf * Update docs/userguide/configuration.rst * Update configuration.rst with version * fix redundant routing_key call * Fix breaking tests --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/amqp.py | 52 +++++++++++++--- celery/app/control.py | 10 ++- celery/app/defaults.py | 6 ++ celery/events/receiver.py | 18 +++++- docs/userguide/configuration.rst | 101 +++++++++++++++++++++++++++++++ docs/userguide/monitoring.rst | 21 +++++++ t/unit/app/test_amqp.py | 25 ++++++++ t/unit/app/test_control.py | 18 +++++- t/unit/events/test_events.py | 34 +++++++++++ 9 files changed, 273 insertions(+), 12 deletions(-) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 8dcec363053..6caedc5c5c6 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -46,6 +46,13 @@ class Queues(dict): create_missing (bool): By default any unknown queues will be added automatically, but if this flag is disabled the occurrence of unknown queues in `wanted` will raise :exc:`KeyError`. + create_missing_queue_type (str): Type of queue to create for missing queues. + Must be either 'classic' (default) or 'quorum'. If set to 'quorum', + the broker will declare new queues using the quorum type. + create_missing_queue_exchange_type (str): Type of exchange to use + when creating missing queues. If not set, the default exchange type + will be used. If set, the exchange type will be set to this value + when creating missing queues. max_priority (int): Default x-max-priority for queues with none set. """ @@ -53,14 +60,19 @@ class Queues(dict): #: The rest of the queues are then used for routing only. _consume_from = None - def __init__(self, queues=None, default_exchange=None, - create_missing=True, autoexchange=None, - max_priority=None, default_routing_key=None): + def __init__( + self, queues=None, default_exchange=None, + create_missing=True, create_missing_queue_type=None, + create_missing_queue_exchange_type=None, autoexchange=None, + max_priority=None, default_routing_key=None, + ): super().__init__() self.aliases = WeakValueDictionary() self.default_exchange = default_exchange self.default_routing_key = default_routing_key self.create_missing = create_missing + self.create_missing_queue_type = create_missing_queue_type + self.create_missing_queue_exchange_type = create_missing_queue_exchange_type self.autoexchange = Exchange if autoexchange is None else autoexchange self.max_priority = max_priority if queues is not None and not isinstance(queues, Mapping): @@ -181,7 +193,21 @@ def deselect(self, exclude): self._consume_from.pop(queue, None) def new_missing(self, name): - return Queue(name, self.autoexchange(name), name) + queue_arguments = None + if self.create_missing_queue_type and self.create_missing_queue_type != "classic": + if self.create_missing_queue_type not in ("classic", "quorum"): + raise ValueError( + f"Invalid queue type '{self.create_missing_queue_type}'. " + "Valid types are 'classic' and 'quorum'." + ) + queue_arguments = {"x-queue-type": self.create_missing_queue_type} + + if self.create_missing_queue_exchange_type: + exchange = Exchange(name, self.create_missing_queue_exchange_type) + else: + exchange = self.autoexchange(name) + + return Queue(name, exchange, name, queue_arguments=queue_arguments) @property def consume_from(self): @@ -238,14 +264,18 @@ def create_task_message(self): def send_task_message(self): return self._create_task_sender() - def Queues(self, queues, create_missing=None, - autoexchange=None, max_priority=None): + def Queues(self, queues, create_missing=None, create_missing_queue_type=None, + create_missing_queue_exchange_type=None, autoexchange=None, max_priority=None): # Create new :class:`Queues` instance, using queue defaults # from the current configuration. conf = self.app.conf default_routing_key = conf.task_default_routing_key if create_missing is None: create_missing = conf.task_create_missing_queues + if create_missing_queue_type is None: + create_missing_queue_type = conf.task_create_missing_queue_type + if create_missing_queue_exchange_type is None: + create_missing_queue_exchange_type = conf.task_create_missing_queue_exchange_type if max_priority is None: max_priority = conf.task_queue_max_priority if not queues and conf.task_default_queue: @@ -259,8 +289,14 @@ def Queues(self, queues, create_missing=None, autoexchange = (self.autoexchange if autoexchange is None else autoexchange) return self.queues_cls( - queues, self.default_exchange, create_missing, - autoexchange, max_priority, default_routing_key, + queues, + default_exchange=self.default_exchange, + create_missing=create_missing, + create_missing_queue_type=create_missing_queue_type, + create_missing_queue_exchange_type=create_missing_queue_exchange_type, + autoexchange=autoexchange, + max_priority=max_priority, + default_routing_key=default_routing_key, ) def Router(self, queues=None, create_missing=None): diff --git a/celery/app/control.py b/celery/app/control.py index 603d930a542..00db75d6ddf 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -20,7 +20,7 @@ from kombu.utils.functional import lazy from kombu.utils.objects import cached_property -from celery.exceptions import DuplicateNodenameWarning +from celery.exceptions import DuplicateNodenameWarning, ImproperlyConfigured from celery.utils.log import get_logger from celery.utils.text import pluralize @@ -428,6 +428,12 @@ class Control: def __init__(self, app=None): self.app = app + if (app.conf.control_queue_durable and + app.conf.control_queue_exclusive): + raise ImproperlyConfigured( + "control_queue_durable and control_queue_exclusive cannot both be True " + "(exclusive queues are automatically deleted and cannot be durable).", + ) self.mailbox = self.Mailbox( app.conf.control_exchange, type='fanout', @@ -437,6 +443,8 @@ def __init__(self, app=None): queue_ttl=app.conf.control_queue_ttl, reply_queue_ttl=app.conf.control_queue_ttl, queue_expires=app.conf.control_queue_expires, + queue_exclusive=app.conf.control_queue_exclusive, + queue_durable=app.conf.control_queue_durable, reply_queue_expires=app.conf.control_queue_expires, ) register_after_fork(self, _after_fork_cleanup_control) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index f8e2511fd01..bd44d8bfbbc 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -150,6 +150,8 @@ def __repr__(self): control=Namespace( queue_ttl=Option(300.0, type='float'), queue_expires=Option(10.0, type='float'), + queue_exclusive=Option(False, type='bool'), + queue_durable=Option(False, type='bool'), exchange=Option('celery', type='string'), ), couchbase=Namespace( @@ -179,6 +181,8 @@ def __repr__(self): queue_expires=Option(60.0, type='float'), queue_ttl=Option(5.0, type='float'), queue_prefix=Option('celeryev'), + queue_exclusive=Option(False, type='bool'), + queue_durable=Option(False, type='bool'), serializer=Option('json'), exchange=Option('celeryev', type='string'), ), @@ -260,6 +264,8 @@ def __repr__(self): annotations=Option(type='any'), compression=Option(type='string', old={'celery_message_compression'}), create_missing_queues=Option(True, type='bool'), + create_missing_queue_type=Option('classic', type='string'), + create_missing_queue_exchange_type=Option(None, type='string'), inherit_parent_priority=Option(False, type='bool'), default_delivery_mode=Option(2, type='string'), default_queue=Option('celery'), diff --git a/celery/events/receiver.py b/celery/events/receiver.py index 14871073322..bda50a10083 100644 --- a/celery/events/receiver.py +++ b/celery/events/receiver.py @@ -8,6 +8,7 @@ from celery import uuid from celery.app import app_or_default +from celery.exceptions import ImproperlyConfigured from celery.utils.time import adjust_timestamp from .event import get_exchange @@ -34,7 +35,9 @@ class EventReceiver(ConsumerMixin): def __init__(self, channel, handlers=None, routing_key='#', node_id=None, app=None, queue_prefix=None, - accept=None, queue_ttl=None, queue_expires=None): + accept=None, queue_ttl=None, queue_expires=None, + queue_exclusive=None, + queue_durable=None): self.app = app_or_default(app or self.app) self.channel = maybe_channel(channel) self.handlers = {} if handlers is None else handlers @@ -48,11 +51,22 @@ def __init__(self, channel, handlers=None, routing_key='#', queue_ttl = self.app.conf.event_queue_ttl if queue_expires is None: queue_expires = self.app.conf.event_queue_expires + if queue_exclusive is None: + queue_exclusive = self.app.conf.event_queue_exclusive + if queue_durable is None: + queue_durable = self.app.conf.event_queue_durable + if queue_exclusive and queue_durable: + raise ImproperlyConfigured( + 'Queue cannot be both exclusive and durable, ' + 'choose one or the other.' + ) self.queue = Queue( '.'.join([self.queue_prefix, self.node_id]), exchange=self.exchange, routing_key=self.routing_key, - auto_delete=True, durable=False, + auto_delete=not queue_durable, + durable=queue_durable, + exclusive=queue_exclusive, message_ttl=queue_ttl, expires=queue_expires, ) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 26b4d64db71..0de8eba8a57 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -103,6 +103,8 @@ have been moved into a new ``task_`` prefix. ``CELERY_MONGODB_BACKEND_SETTINGS`` :setting:`mongodb_backend_settings` ``CELERY_EVENT_QUEUE_EXPIRES`` :setting:`event_queue_expires` ``CELERY_EVENT_QUEUE_TTL`` :setting:`event_queue_ttl` +``CELERY_EVENT_QUEUE_DURABLE`` :setting:`event_queue_durable` +``CELERY_EVENT_QUEUE_EXCLUSIVE`` :setting:`event_queue_exclusive` ``CELERY_EVENT_QUEUE_PREFIX`` :setting:`event_queue_prefix` ``CELERY_EVENT_SERIALIZER`` :setting:`event_serializer` ``CELERY_REDIS_DB`` :setting:`redis_db` @@ -134,6 +136,8 @@ have been moved into a new ``task_`` prefix. ``CELERY_ANNOTATIONS`` :setting:`task_annotations` ``CELERY_COMPRESSION`` :setting:`task_compression` ``CELERY_CREATE_MISSING_QUEUES`` :setting:`task_create_missing_queues` +``CELERY_CREATE_MISSING_QUEUE_TYPE`` :setting:`task_create_missing_queue_type` +``CELERY_CREATE_MISSING_QUEUE_EXCHANGE_TYPE`` :setting:`task_create_missing_queue_exchange_type` ``CELERY_DEFAULT_DELIVERY_MODE`` :setting:`task_default_delivery_mode` ``CELERY_DEFAULT_EXCHANGE`` :setting:`task_default_exchange` ``CELERY_DEFAULT_EXCHANGE_TYPE`` :setting:`task_default_exchange_type` @@ -2619,6 +2623,51 @@ If enabled (default), any queues specified that aren't defined in :setting:`task_queues` will be automatically created. See :ref:`routing-automatic`. +.. setting:: task_create_missing_queue_type + +``task_create_missing_queue_type`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionadded:: 5.6 + +Default: ``"classic"`` + +When Celery needs to declare a queue that doesn’t exist (i.e., when +``task_create_missing_queues`` is enabled), this setting defines what type +of RabbitMQ queue to create. + +- ``"classic"`` (default): declares a standard classic queue. +- ``"quorum"``: declares a RabbitMQ quorum queue (adds ``x-queue-type: quorum``). + +.. setting:: task_create_missing_queue_exchange_type + +``task_create_missing_queue_exchange_type`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionadded:: 5.6 + +Default: ``None`` + +If this option is None or the empty string (the default), Celery leaves the +exchange exactly as returned by your :attr:`app.amqp.Queues.autoexchange` +hook. + +You can set this to a specific exchange type, such as ``"direct"``, ``"topic"``, or +``"fanout"``, to create the missing queue with that exchange type. + +.. tip:: + +Combine this setting with task_create_missing_queue_type = "quorum" +to create quorum queues bound to a topic exchange, for example:: + + app.conf.task_create_missing_queues=True + app.conf.task_create_missing_queue_type="quorum" + app.conf.task_create_missing_queue_exchange_type="topic" + +.. note:: + +Like the queue-type setting above, this option does not affect queues +that you define explicitly in :setting:`task_queues`; it applies only to +queues created implicitly at runtime. + .. setting:: task_default_queue ``task_default_queue`` @@ -3410,6 +3459,33 @@ Default: 60.0 seconds. Expiry time in seconds (int/float) for when after a monitor clients event queue will be deleted (``x-expires``). +.. setting:: event_queue_durable + +``event_queue_durable`` +~~~~~~~~~~~~~~~~~~~~~~~~ +:transports supported: ``amqp`` +.. versionadded:: 5.6 + +Default: ``False`` + +If enabled, the event receiver's queue will be marked as *durable*, meaning it will survive broker restarts. + +.. setting:: event_queue_exclusive + +``event_queue_exclusive`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ +:transports supported: ``amqp`` +.. versionadded:: 5.6 + +Default: ``False`` + +If enabled, the event queue will be *exclusive* to the current connection and automatically deleted when the connection closes. + +.. warning:: + + You **cannot** set both ``event_queue_durable`` and ``event_queue_exclusive`` to ``True`` at the same time. + Celery will raise an :exc:`ImproperlyConfigured` error if both are set. + .. setting:: event_queue_prefix ``event_queue_prefix`` @@ -3566,6 +3642,31 @@ Name of the control command exchange. .. _conf-logging: +.. setting:: control_queue_durable + +``control_queue_durable`` +------------------------- + +- **Default:** ``False`` +- **Type:** ``bool`` + +If set to ``True``, the control exchange and queue will be durable — they will survive broker restarts. + +.. setting:: control_queue_exclusive + +``control_queue_exclusive`` +--------------------------- + +- **Default:** ``False`` +- **Type:** ``bool`` + +If set to ``True``, the control queue will be exclusive to a single connection. This is generally not recommended in distributed environments. + +.. warning:: + + Setting both ``control_queue_durable`` and ``control_queue_exclusive`` to ``True`` is not supported and will raise an error. + + Logging ------- diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index b542633ec9d..66cb6f00871 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -814,3 +814,24 @@ worker-offline :signature: ``worker-offline(hostname, timestamp, freq, sw_ident, sw_ver, sw_sys)`` The worker has disconnected from the broker. + +Mailbox Configuration (Advanced) +-------------------------------- + +Celery uses `kombu.pidbox.Mailbox` internally to send control and broadcast commands +to workers. + +.. versionadded:: Kombu 5.6.0 + +Advanced users can configure the behavior of this mailbox by customizing how it is created. +The following parameters are now supported by `Mailbox`: + +- ``durable`` (default: ``False``): If set to ``True``, the control exchanges will survive broker restarts. +- ``exclusive`` (default: ``False``): If set to ``True``, the exchanges will be usable by only one connection. + +.. warning:: + + Setting both ``durable=True`` and ``exclusive=True`` is not permitted and will + raise an error, as these two options are mutually incompatible in AMQP. + +See :setting:`event_queue_durable` and :setting:`event_queue_exclusive` for advanced configuration. diff --git a/t/unit/app/test_amqp.py b/t/unit/app/test_amqp.py index 4b46148d144..db15c343a99 100644 --- a/t/unit/app/test_amqp.py +++ b/t/unit/app/test_amqp.py @@ -134,6 +134,15 @@ def test_with_max_priority(self, queues_kwargs, qname, q, expected): queues.add(q) assert queues[qname].queue_arguments == expected + def test_missing_queue_quorum(self): + queues = Queues(create_missing_queue_type="quorum", + create_missing_queue_exchange_type="topic") + + q = queues.new_missing("spontaneous") + assert q.name == "spontaneous" + assert q.queue_arguments == {"x-queue-type": "quorum"} + assert q.exchange.type == "topic" + class test_default_queues: @@ -360,6 +369,22 @@ def update_conf_runtime_for_tasks_queues(self): router = self.app.amqp.router assert router != router_was + def test_create_missing_queue_type_from_conf(self): + self.app.conf.task_create_missing_queue_type = "quorum" + self.app.conf.task_create_missing_queue_exchange_type = "topic" + self.app.amqp.__dict__.pop("queues", None) + q = self.app.amqp.queues["auto"] + assert q.queue_arguments == {"x-queue-type": "quorum"} + assert q.exchange.type == "topic" + + def test_create_missing_queue_type_explicit_param(self): + qmap = self.app.amqp.Queues({}, create_missing=True, + create_missing_queue_type="quorum", + create_missing_queue_exchange_type="topic") + q = qmap["auto"] + assert q.queue_arguments == {"x-queue-type": "quorum"} + assert q.exchange.type == "topic" + class test_as_task_v2(test_AMQP_Base): diff --git a/t/unit/app/test_control.py b/t/unit/app/test_control.py index 0908491a9ee..4916880a431 100644 --- a/t/unit/app/test_control.py +++ b/t/unit/app/test_control.py @@ -4,7 +4,7 @@ from celery import uuid from celery.app import control -from celery.exceptions import DuplicateNodenameWarning +from celery.exceptions import DuplicateNodenameWarning, ImproperlyConfigured from celery.utils.collections import LimitedSet @@ -291,6 +291,7 @@ def test_time_limit__with_destination(self): self.mytask.name, soft=10, hard=20, destination='a@q.com', limit=99, ) + self.assert_control_called_with_args( 'time_limit', destination='a@q.com', @@ -564,3 +565,18 @@ def test_control_exchange__setting(self): self.app.conf.control_exchange = 'test_exchange' c = control.Control(self.app) assert c.mailbox.namespace == 'test_exchange' + + def test_control_mailbox_queue_options(self): + self.app.conf.control_queue_durable = True + self.app.conf.control_queue_exclusive = False + + c = control.Control(self.app) + assert c.mailbox.queue_durable is True + assert c.mailbox.queue_exclusive is False + + def test_control_mailbox_invalid_combination(self): + self.app.conf.control_queue_durable = True + self.app.conf.control_queue_exclusive = True + + with pytest.raises(ImproperlyConfigured): + control.Control(self.app) diff --git a/t/unit/events/test_events.py b/t/unit/events/test_events.py index 21fcc5003f1..ae2c4e4930c 100644 --- a/t/unit/events/test_events.py +++ b/t/unit/events/test_events.py @@ -5,6 +5,7 @@ from celery.events import Event from celery.events.receiver import CLIENT_CLOCK_SKEW +from celery.exceptions import ImproperlyConfigured class MockProducer: @@ -327,6 +328,39 @@ def handler(event): channel.close() connection.close() + def test_event_queue_exclusive(self): + self.app.conf.update( + event_queue_exclusive=True, + event_queue_durable=False + ) + + ev_recv = self.app.events.Receiver(Mock(name='connection')) + q = ev_recv.queue + + assert q.exclusive is True + assert q.durable is False + assert q.auto_delete is True + + def test_event_queue_durable_and_validation(self): + self.app.conf.update( + event_queue_exclusive=False, + event_queue_durable=True + ) + ev_recv = self.app.events.Receiver(Mock(name='connection')) + q = ev_recv.queue + + assert q.durable is True + assert q.exclusive is False + assert q.auto_delete is False + + self.app.conf.update( + event_queue_exclusive=True, + event_queue_durable=True + ) + + with pytest.raises(ImproperlyConfigured): + self.app.events.Receiver(Mock(name='connection')) + def test_State(app): state = app.events.State() From 6804ea8615afcdbc95ed68e95b47ae623080fa2b Mon Sep 17 00:00:00 2001 From: Linus Phan <13613724+linusphan@users.noreply.github.com> Date: Sat, 30 Aug 2025 23:16:02 -0700 Subject: [PATCH 020/169] fix: prevent celery from hanging due to spawned greenlet errors in greenlet drainers (#9371) * propagate event drainer errors to prevent infinite loop and require manual restart Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> * remove typing Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * add tests * add tests and refactor implementation Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> * remove test code and add pydoc for clarity Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> * raise error in greenlet to ensure it exits, and add more test coverage Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * calls `teardown_thread` when using `schedule_thread` in tests Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * use wait() instead of while loop for clarity in teardown_thread for test_EventletDrainer Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * fix lint Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * Update celery/backends/asynchronous.py * Update celery/backends/asynchronous.py * Update celery/backends/asynchronous.py * Update celery/backends/asynchronous.py * Address race condition concern when setting and reading exception state Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * Revise docstring * Fix bare except clause in test teardown_thread method * Revert test change Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * Improve naming and docstring/comment clarity Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Linus Phan <13613724+linusphan@users.noreply.github.com> * Update celery/backends/asynchronous.py * Update celery/backends/asynchronous.py * Update celery/backends/asynchronous.py * Add logging import to asynchronous backend --------- Co-authored-by: Jack <57678801+mothershipper@users.noreply.github.com> Co-authored-by: Asif Saif Uddin --- .gitignore | 1 + celery/backends/asynchronous.py | 105 +++++++++++++++++++-------- celery/backends/redis.py | 4 +- t/unit/backends/test_asynchronous.py | 54 +++++++++++++- 4 files changed, 128 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 677430265ab..f70de56dce0 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ coverage.xml test.db pip-wheel-metadata/ .python-version +.tool-versions .vscode/ integration-tests-config.json [0-9]* diff --git a/celery/backends/asynchronous.py b/celery/backends/asynchronous.py index cedae5013a8..a5e0e5d4036 100644 --- a/celery/backends/asynchronous.py +++ b/celery/backends/asynchronous.py @@ -1,4 +1,6 @@ """Async I/O backend support utilities.""" + +import logging import socket import threading import time @@ -13,11 +15,34 @@ from celery.exceptions import TimeoutError from celery.utils.threads import THREAD_TIMEOUT_MAX +E_CELERY_RESTART_REQUIRED = "Celery must be restarted because a shutdown signal was detected." + __all__ = ( 'AsyncBackendMixin', 'BaseResultConsumer', 'Drainer', 'register_drainer', ) + +class EventletAdaptedEvent: + """ + An adapted eventlet event, designed to match the API of `threading.Event` and + `gevent.event.Event`. + """ + + def __init__(self): + import eventlet + self.evt = eventlet.Event() + + def is_set(self): + return self.evt.ready() + + def set(self): + return self.evt.send() + + def wait(self, timeout=None): + return self.evt.wait(timeout) + + drainers = {} @@ -62,46 +87,57 @@ def drain_events_until(self, p, timeout=None, interval=1, on_interval=None, wait def wait_for(self, p, wait, timeout=None): wait(timeout=timeout) + def _event(self): + return threading.Event() + class greenletDrainer(Drainer): spawn = None + _exc = None _g = None _drain_complete_event = None # event, sended (and recreated) after every drain_events iteration - def _create_drain_complete_event(self): - """create new self._drain_complete_event object""" - pass - def _send_drain_complete_event(self): - """raise self._drain_complete_event for wakeup .wait_for""" - pass + self._drain_complete_event.set() + self._drain_complete_event = self._event() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._started = threading.Event() - self._stopped = threading.Event() - self._shutdown = threading.Event() - self._create_drain_complete_event() + + self._started = self._event() + self._stopped = self._event() + self._shutdown = self._event() + self._drain_complete_event = self._event() def run(self): self._started.set() - while not self._stopped.is_set(): + + try: + while not self._stopped.is_set(): + try: + self.result_consumer.drain_events(timeout=1) + self._send_drain_complete_event() + except socket.timeout: + pass + except Exception as e: + self._exc = e + raise + finally: + self._send_drain_complete_event() try: - self.result_consumer.drain_events(timeout=1) - self._send_drain_complete_event() - self._create_drain_complete_event() - except socket.timeout: - pass - self._shutdown.set() + self._shutdown.set() + except RuntimeError as e: + logging.error(f"Failed to set shutdown event: {e}") def start(self): + self._ensure_not_shut_down() + if not self._started.is_set(): self._g = self.spawn(self.run) self._started.wait() def stop(self): self._stopped.set() - self._send_drain_complete_event() self._shutdown.wait(THREAD_TIMEOUT_MAX) def wait_for(self, p, wait, timeout=None): @@ -109,6 +145,23 @@ def wait_for(self, p, wait, timeout=None): if not p.ready: self._drain_complete_event.wait(timeout=timeout) + self._ensure_not_shut_down() + + def _ensure_not_shut_down(self): + """Currently used to ensure the drainer has not run to completion. + + Raises if the shutdown event has been signaled (either due to an exception + or stop() being called). + + The _shutdown event acts as synchronization to ensure _exc is properly + set before it is read from, avoiding need for locks. + """ + if self._shutdown.is_set(): + if self._exc is not None: + raise self._exc + else: + raise Exception(E_CELERY_RESTART_REQUIRED) + @register_drainer('eventlet') class eventletDrainer(greenletDrainer): @@ -119,12 +172,8 @@ def spawn(self, func): sleep(0) return g - def _create_drain_complete_event(self): - from eventlet.event import Event - self._drain_complete_event = Event() - - def _send_drain_complete_event(self): - self._drain_complete_event.send() + def _event(self): + return EventletAdaptedEvent() @register_drainer('gevent') @@ -136,13 +185,9 @@ def spawn(self, func): gevent.sleep(0) return g - def _create_drain_complete_event(self): + def _event(self): from gevent.event import Event - self._drain_complete_event = Event() - - def _send_drain_complete_event(self): - self._drain_complete_event.set() - self._create_drain_complete_event() + return Event() class AsyncBackendMixin: diff --git a/celery/backends/redis.py b/celery/backends/redis.py index e2597be88fd..7ddba5e5d63 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -129,9 +129,9 @@ def reconnect_on_error(self): except self._connection_errors: try: self._ensure(self._reconnect_pubsub, ()) - except self._connection_errors: + except self._connection_errors as e: logger.critical(E_RETRY_LIMIT_EXCEEDED) - raise + raise RuntimeError(E_RETRY_LIMIT_EXCEEDED) from e def _maybe_cancel_ready_task(self, meta): if meta['status'] in states.READY_STATES: diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index 479fd855838..e5dc27eec62 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -8,7 +8,7 @@ import pytest from vine import promise -from celery.backends.asynchronous import BaseResultConsumer +from celery.backends.asynchronous import E_CELERY_RESTART_REQUIRED, BaseResultConsumer from celery.backends.base import Backend from celery.utils import cached_property @@ -142,11 +142,52 @@ def test_drain_timeout(self): assert on_interval.call_count < 20, 'Should have limited number of calls to on_interval' +class GreenletDrainerTests(DrainerTests): + def test_drain_raises_when_greenlet_already_exited(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=Exception("Test Exception")): + thread = self.schedule_thread(self.drainer.run) + + with pytest.raises(Exception, match="Test Exception"): + p = promise() + + for _ in self.drainer.drain_events_until(p, interval=self.interval): + pass + + self.teardown_thread(thread) + + def test_drain_raises_while_waiting_on_exiting_greenlet(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=Exception("Test Exception")): + with pytest.raises(Exception, match="Test Exception"): + p = promise() + + for _ in self.drainer.drain_events_until(p, interval=self.interval): + pass + + def test_start_raises_if_previous_error_in_run(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=Exception("Test Exception")): + thread = self.schedule_thread(self.drainer.run) + + with pytest.raises(Exception, match="Test Exception"): + self.drainer.start() + + self.teardown_thread(thread) + + def test_start_raises_if_drainer_already_stopped(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=lambda **_: self.sleep(0)): + thread = self.schedule_thread(self.drainer.run) + self.drainer.stop() + + with pytest.raises(Exception, match=E_CELERY_RESTART_REQUIRED): + self.drainer.start() + + self.teardown_thread(thread) + + @pytest.mark.skipif( sys.platform == "win32", reason="hangs forever intermittently on windows" ) -class test_EventletDrainer(DrainerTests): +class test_EventletDrainer(GreenletDrainerTests): @pytest.fixture(autouse=True) def setup_drainer(self): self.drainer = self.get_drainer('eventlet') @@ -171,7 +212,12 @@ def schedule_thread(self, thread): return g def teardown_thread(self, thread): - thread.wait() + try: + # eventlet's wait() propagates any errors on the green thread, unlike + # similar methods in gevent or python's threading library + thread.wait() + except Exception: + pass class test_Drainer(DrainerTests): @@ -196,7 +242,7 @@ def teardown_thread(self, thread): thread.join() -class test_GeventDrainer(DrainerTests): +class test_GeventDrainer(GreenletDrainerTests): @pytest.fixture(autouse=True) def setup_drainer(self): self.drainer = self.get_drainer('gevent') From 246bca18e3c65e6881a4a8fef89dd15e1b506a5f Mon Sep 17 00:00:00 2001 From: Daniel Khodos Date: Sun, 31 Aug 2025 12:15:42 +0300 Subject: [PATCH 021/169] Feature/disable prefetch fixes (#9863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add option to disable prefetch * Fix pre-commit hooks failing * worker: disable-prefetch QoS guard uses autoscale max_concurrency or pool size; add CLI --disable-prefetch test; ensure consumer tests pass * Fix test coverage for worker disable_prefetch handling Add dedicated tests for disable_prefetch flag handling in worker.py to improve test coverage. This addresses the coverage issue identified by Codecov in PR #9863. * Fix test coverage for worker disable_prefetch handling Add comprehensive tests for the worker's disable_prefetch flag handling to improve test coverage. Refactored tests to be more concise, focused, and pass all lint checks. This addresses the coverage issue identified by Codecov in PR #9863. * docs: add missing versionadded annotation for worker_disable_prefetch setting * Update docs/userguide/configuration.rst --------- Co-authored-by: rbehal Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} Co-authored-by: dkhodos_sfemu --- celery/app/defaults.py | 1 + celery/bin/worker.py | 10 +++ celery/worker/consumer/tasks.py | 16 ++++ docs/faq.rst | 4 + docs/userguide/configuration.rst | 28 +++++- docs/userguide/optimizing.rst | 9 +- t/unit/bin/test_worker.py | 76 +++++++++++++++- t/unit/worker/test_autoscale.py | 48 ++++++++++ t/unit/worker/test_consumer.py | 149 +++++++++++++++++++++++++++++++ 9 files changed, 334 insertions(+), 7 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index bd44d8bfbbc..28067f0cdcd 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -343,6 +343,7 @@ def __repr__(self): proc_alive_timeout=Option(4.0, type='float'), prefetch_multiplier=Option(4, type='int'), enable_prefetch_count_reduction=Option(True, type='bool'), + disable_prefetch=Option(False, type='bool'), redirect_stdouts=Option( True, type='bool', old={'celery_redirect_stdouts'}, ), diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 0cc3d6664cc..52f09f3a83d 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -182,6 +182,14 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, help_group="Worker Options", help="Set custom prefetch multiplier value " "for this worker instance.") +@click.option('--disable-prefetch', + is_flag=True, + default=None, + callback=lambda ctx, _, + value: ctx.obj.app.conf.worker_disable_prefetch if value is None else value, + cls=CeleryOption, + help_group="Worker Options", + help="Disable broker prefetching. The worker will only fetch a task when a process slot is available.") @click.option('-c', '--concurrency', type=int, @@ -314,6 +322,8 @@ def worker(ctx, hostname=None, pool_cls=None, app=None, uid=None, gid=None, """ try: app = ctx.obj.app + if 'disable_prefetch' in kwargs and kwargs['disable_prefetch'] is not None: + app.conf.worker_disable_prefetch = kwargs.pop('disable_prefetch') if ctx.args: try: app.config_from_cmdline(ctx.args, namespace='worker') diff --git a/celery/worker/consumer/tasks.py b/celery/worker/consumer/tasks.py index 67cbfc1207f..ae7245b5b37 100644 --- a/celery/worker/consumer/tasks.py +++ b/celery/worker/consumer/tasks.py @@ -48,6 +48,22 @@ def set_prefetch_count(prefetch_count): ) c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) + if c.app.conf.worker_disable_prefetch: + from types import MethodType + + from celery.worker import state + channel_qos = c.task_consumer.channel.qos + original_can_consume = channel_qos.can_consume + + def can_consume(self): + # Prefer autoscaler's max_concurrency if set; otherwise fall back to pool size + limit = getattr(c.controller, "max_concurrency", None) or c.pool.num_processes + if len(state.reserved_requests) >= limit: + return False + return original_can_consume() + + channel_qos.can_consume = MethodType(can_consume, channel_qos) + def stop(self, c): """Stop task consumer.""" if c.task_consumer: diff --git a/docs/faq.rst b/docs/faq.rst index cd5f3aa874d..d0946153565 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -788,6 +788,10 @@ to describe the task prefetching *limit*. There's no actual prefetching involve Disabling the prefetch limits is possible, but that means the worker will consume as many tasks as it can, as fast as possible. +You can use the :option:`--disable-prefetch ` +flag (or set :setting:`worker_disable_prefetch` to ``True``) so that a worker +only fetches a task when one of its processes is free. + A discussion on prefetch limits, and configuration settings for a worker that only reserves one task at a time is found here: :ref:`optimizing-prefetch-limit`. diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 0de8eba8a57..ff0ba40bd7d 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -3180,9 +3180,17 @@ workers, note that the first worker to start will receive four times the number of messages initially. Thus the tasks may not be fairly distributed to the workers. -To disable prefetching, set :setting:`worker_prefetch_multiplier` to 1. -Changing that setting to 0 will allow the worker to keep consuming -as many messages as it wants. +To limit the broker to only deliver one message per process at a time, +set :setting:`worker_prefetch_multiplier` to 1. Changing that setting to 0 +will allow the worker to keep consuming as many messages as it wants. + +If you need to completely disable broker prefetching while still using +early acknowledgments, enable :setting:`worker_disable_prefetch`. +When this option is enabled the worker only fetches a task from the broker +when one of its processes is available. + +You can also enable this via the :option:`--disable-prefetch ` +command line flag. For more on prefetching, read :ref:`optimizing-prefetch-limit` @@ -3190,6 +3198,20 @@ For more on prefetching, read :ref:`optimizing-prefetch-limit` Tasks with ETA/countdown aren't affected by prefetch limits. +.. setting:: worker_disable_prefetch + +``worker_disable_prefetch`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.6 + +Default: ``False``. + +When enabled, a worker will only consume messages from the broker when it +has an available process to execute them. This disables prefetching while +still using early acknowledgments, ensuring that tasks are fairly +distributed between workers. + .. setting:: worker_enable_prefetch_count_reduction ``worker_enable_prefetch_count_reduction`` diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 72ce4dc77cb..42cfdda33ad 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -181,9 +181,12 @@ You can enable this behavior by using the following configuration options: task_acks_late = True worker_prefetch_multiplier = 1 -If you want to disable "prefetching of tasks" without using ack_late (because -your tasks are not idempotent) that's impossible right now and you can join the -discussion here https://github.com/celery/celery/discussions/7106 +If your tasks cannot be acknowledged late you can disable broker +prefetching by enabling :setting:`worker_disable_prefetch`. With this +setting the worker fetches a new task only when an execution slot is +free, preventing tasks from waiting behind long running ones on busy +workers. This can also be set from the command line using +:option:`--disable-prefetch `. Memory Usage ------------ diff --git a/t/unit/bin/test_worker.py b/t/unit/bin/test_worker.py index b63a2a03306..0f219e177b1 100644 --- a/t/unit/bin/test_worker.py +++ b/t/unit/bin/test_worker.py @@ -1,11 +1,12 @@ import os -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest from click.testing import CliRunner from celery.app.log import Logging from celery.bin.celery import celery +from celery.worker.consumer.tasks import Tasks @pytest.fixture(scope='session') @@ -13,6 +14,36 @@ def use_celery_app_trap(): return False +@pytest.fixture +def mock_app(): + app = Mock() + app.conf = Mock() + app.conf.worker_disable_prefetch = False + return app + + +@pytest.fixture +def mock_consumer(mock_app): + consumer = Mock() + consumer.app = mock_app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.connection = Mock() + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + return consumer + + def test_cli(isolated_cli_runner: CliRunner): Logging._setup = True # To avoid hitting the logging sanity checks res = isolated_cli_runner.invoke( @@ -33,3 +64,46 @@ def test_cli_skip_checks(isolated_cli_runner: CliRunner): ) assert res.exit_code == 1, (res, res.stdout) assert os.environ["CELERY_SKIP_CHECKS"] == "true", "should set CELERY_SKIP_CHECKS" + + +def test_cli_disable_prefetch_flag(isolated_cli_runner: CliRunner): + Logging._setup = True + with patch('celery.bin.worker.worker.callback') as worker_callback_mock: + res = isolated_cli_runner.invoke( + celery, + ["-A", "t.unit.bin.proj.app", "worker", "--pool", "solo", "--disable-prefetch"], + catch_exceptions=False, + ) + assert res.exit_code == 0 + _, kwargs = worker_callback_mock.call_args + assert kwargs['disable_prefetch'] is True + + +def test_disable_prefetch_affects_qos_behavior(mock_app, mock_consumer): + mock_app.conf.worker_disable_prefetch = True + original_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + with patch('celery.worker.state.reserved_requests', []): + tasks_instance = Tasks(mock_consumer) + tasks_instance.start(mock_consumer) + assert mock_consumer.task_consumer.channel.qos.can_consume != original_can_consume + modified_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + with patch('celery.worker.state.reserved_requests', list(range(4))): + assert not modified_can_consume() + with patch('celery.worker.state.reserved_requests', list(range(2))): + original_can_consume.return_value = True + assert modified_can_consume() + original_can_consume.return_value = False + assert not modified_can_consume() + + +def test_disable_prefetch_none_preserves_behavior(mock_app, mock_consumer): + mock_app.conf.worker_disable_prefetch = False + kwargs_with_none = {'disable_prefetch': None} + if 'disable_prefetch' in kwargs_with_none and kwargs_with_none['disable_prefetch'] is not None: + mock_app.conf.worker_disable_prefetch = kwargs_with_none.pop('disable_prefetch') + assert mock_app.conf.worker_disable_prefetch is False + assert 'disable_prefetch' in kwargs_with_none + original_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + tasks_instance = Tasks(mock_consumer) + tasks_instance.start(mock_consumer) + assert mock_consumer.task_consumer.channel.qos.can_consume == original_can_consume diff --git a/t/unit/worker/test_autoscale.py b/t/unit/worker/test_autoscale.py index c4a2a75ed73..79eded5d923 100644 --- a/t/unit/worker/test_autoscale.py +++ b/t/unit/worker/test_autoscale.py @@ -236,3 +236,51 @@ def test_no_negative_scale(self, sleepdeprived): assert all(x.min_concurrency <= i <= x.max_concurrency for i in total_num_processes) + + def test_disable_prefetch_respects_max_concurrency(self): + """Test that disable_prefetch respects autoscale max_concurrency setting""" + from celery.worker.consumer.tasks import Tasks + + # Create a mock consumer with autoscale and disable_prefetch enabled + consumer = Mock() + consumer.app = Mock() + consumer.app.conf.worker_disable_prefetch = True + consumer.pool = Mock() + consumer.pool.num_processes = 10 + consumer.controller = Mock() + consumer.controller.max_concurrency = 5 # Lower than pool processes + + # Mock task consumer setup + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + consumer.task_consumer.channel.qos.can_consume = Mock(return_value=True) + + # Mock the connection and other required attributes + consumer.connection = Mock() + consumer.connection.default_channel = Mock() + consumer.initial_prefetch_count = 20 + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock the amqp TaskConsumer + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + # Mock 5 reserved requests (at autoscale limit of 5) + mock_requests = [Mock() for _ in range(5)] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should not be able to consume when at autoscale limit + assert consumer.task_consumer.channel.qos.can_consume() is False + + # Test with 4 reserved requests (under autoscale limit of 5) + mock_requests = [Mock() for _ in range(4)] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should be able to consume when under autoscale limit + assert consumer.task_consumer.channel.qos.can_consume() is True diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index 04d167e3d83..1f54a839a68 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -495,6 +495,155 @@ def test_ensure_connected(self, subtests, broker_connection_retry, broker_connec with pytest.raises(ConnectionError): c.ensure_connected(conn) + def test_disable_prefetch_not_enabled(self): + """Test that disable_prefetch doesn't affect behavior when disabled""" + self.app.conf.worker_disable_prefetch = False + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.default_channel = Mock() + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + tasks_instance.start(consumer) + + # Should not modify can_consume method when disabled + assert consumer.task_consumer.channel.qos.can_consume == original_can_consume + + def test_disable_prefetch_enabled_basic(self): + """Test that disable_prefetch modifies can_consume when enabled""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.default_channel = Mock() + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + with patch('celery.worker.state.reserved_requests', []): + tasks_instance.start(consumer) + + # Should modify can_consume method when enabled + assert callable(consumer.task_consumer.channel.qos.can_consume) + assert consumer.task_consumer.channel.qos.can_consume != original_can_consume + + def test_disable_prefetch_respects_reserved_requests_limit(self): + """Test that disable_prefetch respects reserved requests limit""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.default_channel = Mock() + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + consumer.task_consumer.channel.qos.can_consume = Mock(return_value=True) + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + # Mock 4 reserved requests (at limit of 4) + mock_requests = [Mock(), Mock(), Mock(), Mock()] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should not be able to consume when at limit + assert consumer.task_consumer.channel.qos.can_consume() is False + + def test_disable_prefetch_respects_autoscale_max_concurrency(self): + """Test that disable_prefetch respects autoscale max_concurrency limit""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = 2 # Lower than pool processes + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.default_channel = Mock() + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + consumer.task_consumer.channel.qos.can_consume = Mock(return_value=True) + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + # Mock 2 reserved requests (at autoscale limit of 2) + mock_requests = [Mock(), Mock()] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should not be able to consume when at autoscale limit + assert consumer.task_consumer.channel.qos.can_consume() is False + @pytest.mark.parametrize( "broker_connection_retry_on_startup,is_connection_loss_on_startup", From 6da32827cebaf332d22f906386c47e552ec0e38f Mon Sep 17 00:00:00 2001 From: sandeep kesarwani Date: Sun, 31 Aug 2025 14:57:30 +0530 Subject: [PATCH 022/169] Add worker_eta_task_limit configuration to manage ETA task memory usage (#9853) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add worker_eta_task_limit configuration to manage ETA task memory usage - Introduced `worker_eta_task_limit` to limit the number of ETA/countdown tasks a worker can hold in memory, preventing memory exhaustion. - Updated the task execution strategy to reject new ETA tasks when the limit is reached. - Added documentation for the new configuration option. - Implemented unit tests to validate the behavior of the ETA task limit. * Update docs/userguide/configuration.rst * Implement ETA task limit checks and callbacks in task execution strategy * Add ETA task limit configuration and enforcement in worker strategy - Introduced `worker_eta_task_limit` setting to limit the number of ETA tasks a worker can hold in memory. - Implemented `ETATaskTracker` class to track and enforce the ETA task limit. - Updated `default` strategy to reject new ETA tasks when the limit is reached. - Added unit tests to verify the behavior of the ETA task limit and tracker. * Fix retrieval of worker ETA task limit configuration to handle missing attribute gracefully * Refactor ETA task limit handling in default strategy and update tests for QoS limits * Remove ETA task limit handling from default strategy implementation * Rename eta_task_limit to worker_eta_task_limit and update QoS handling in task consumer * Rename worker_eta_task_limit to eta_task_limit for consistency in configuration * fix: access worker_eta_task_limit as field * docs: update ETA task limit description for clarity --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/defaults.py | 1 + celery/worker/consumer/tasks.py | 9 ++++- celery/worker/strategy.py | 5 ++- docs/userguide/configuration.rst | 19 +++++++++ t/unit/worker/test_consumer.py | 66 ++++++++++++++++++++++++++++++++ t/unit/worker/test_strategy.py | 14 ++++--- 6 files changed, 105 insertions(+), 9 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 28067f0cdcd..77fcfd02196 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -342,6 +342,7 @@ def __repr__(self): pool_restarts=Option(False, type='bool'), proc_alive_timeout=Option(4.0, type='float'), prefetch_multiplier=Option(4, type='int'), + eta_task_limit=Option(None, type='int'), enable_prefetch_count_reduction=Option(True, type='bool'), disable_prefetch=Option(False, type='bool'), redirect_stdouts=Option( diff --git a/celery/worker/consumer/tasks.py b/celery/worker/consumer/tasks.py index ae7245b5b37..ae2029bca42 100644 --- a/celery/worker/consumer/tasks.py +++ b/celery/worker/consumer/tasks.py @@ -46,7 +46,10 @@ def set_prefetch_count(prefetch_count): prefetch_count=prefetch_count, apply_global=qos_global, ) - c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) + eta_task_limit = c.app.conf.worker_eta_task_limit + c.qos = QoS( + set_prefetch_count, c.initial_prefetch_count, max_prefetch=eta_task_limit + ) if c.app.conf.worker_disable_prefetch: from types import MethodType @@ -95,7 +98,9 @@ def qos_global(self, c) -> bool: qos_global = not c.connection.qos_semantics_matches_spec if c.app.conf.worker_detect_quorum_queues: - using_quorum_queues, qname = detect_quorum_queues(c.app, c.connection.transport.driver_type) + using_quorum_queues, _ = detect_quorum_queues( + c.app, c.connection.transport.driver_type + ) if using_quorum_queues: qos_global = False diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 3fe5fa145ca..6a1c6225b48 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -109,7 +109,6 @@ def default(task, app, consumer, hostname = consumer.hostname connection_errors = consumer.connection_errors _does_info = logger.isEnabledFor(logging.INFO) - # task event related # (optimized to avoid calling request.send_event) eventer = consumer.event_dispatcher @@ -125,7 +124,8 @@ def default(task, app, consumer, limit_task = consumer._limit_task limit_post_eta = consumer._limit_post_eta Request = symbol_by_name(task.Request) - Req = create_request_cls(Request, task, consumer.pool, hostname, eventer, app=app) + Req = create_request_cls(Request, task, consumer.pool, hostname, eventer, + app=app) revoked_tasks = consumer.controller.state.revoked @@ -194,6 +194,7 @@ def task_message_handler(message, body, ack, reject, callbacks, consumer.qos.increment_eventually() return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6) + if eta: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req,), priority=6) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index ff0ba40bd7d..fe01b6ecd95 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -174,6 +174,7 @@ have been moved into a new ``task_`` prefix. ``CELERYD_POOL_PUTLOCKS`` :setting:`worker_pool_putlocks` ``CELERYD_POOL_RESTARTS`` :setting:`worker_pool_restarts` ``CELERYD_PREFETCH_MULTIPLIER`` :setting:`worker_prefetch_multiplier` +``CELERYD_ETA_TASK_LIMIT`` :setting:`worker_eta_task_limit` ``CELERYD_ENABLE_PREFETCH_COUNT_REDUCTION``:setting:`worker_enable_prefetch_count_reduction` ``CELERYD_REDIRECT_STDOUTS`` :setting:`worker_redirect_stdouts` ``CELERYD_REDIRECT_STDOUTS_LEVEL`` :setting:`worker_redirect_stdouts_level` @@ -3194,6 +3195,24 @@ command line flag. For more on prefetching, read :ref:`optimizing-prefetch-limit` +.. setting:: worker_eta_task_limit + +``worker_eta_task_limit`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.6 + +Default: No limit (None). + +The maximum number of ETA/countdown tasks that a worker can hold in memory at once. +When this limit is reached, the worker will not receive new tasks from the broker +until some of the existing ETA tasks are executed. + +This setting helps prevent memory exhaustion when a queue contains a large number +of tasks with ETA/countdown values, as these tasks are held in memory until their +execution time. Without this limit, workers may fetch thousands of ETA tasks into +memory, potentially causing out-of-memory issues. + .. note:: Tasks with ETA/countdown aren't affected by prefetch limits. diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index 1f54a839a68..bc21d73697e 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -855,6 +855,72 @@ def test_log_when_qos_is_false(self, caplog): assert record.levelname == "INFO" assert record.msg == "Global QoS is disabled. Prefetch count in now static." + def test_qos_with_worker_eta_task_limit(self): + """Test QoS is instantiated with worker_eta_task_limit as max_prefetch.""" + c = self.c + c.app.conf.worker_eta_task_limit = 100 + c.initial_prefetch_count = 10 + c.task_consumer = Mock() + c.app.amqp.TaskConsumer = Mock(return_value=c.task_consumer) + c.connection.default_channel.basic_qos = Mock() + c.update_strategies = Mock() + c.on_decode_error = Mock() + + tasks = Tasks(c) + + with patch('celery.worker.consumer.tasks.QoS') as mock_qos: + tasks.start(c) + + # Verify QoS was called with max_prefetch set to worker_eta_task_limit + mock_qos.assert_called_once() + args, kwargs = mock_qos.call_args + assert len(args) == 2 # callback and initial_value + assert kwargs.get('max_prefetch') == 100 + + def test_qos_without_worker_eta_task_limit(self): + """Test QoS is instantiated with None max_prefetch when worker_eta_task_limit is None.""" + c = self.c + c.app.conf.worker_eta_task_limit = None + c.initial_prefetch_count = 10 + c.task_consumer = Mock() + c.app.amqp.TaskConsumer = Mock(return_value=c.task_consumer) + c.connection.default_channel.basic_qos = Mock() + c.update_strategies = Mock() + c.on_decode_error = Mock() + + tasks = Tasks(c) + + with patch('celery.worker.consumer.tasks.QoS') as mock_qos: + tasks.start(c) + + # Verify QoS was called with max_prefetch set to None + mock_qos.assert_called_once() + args, kwargs = mock_qos.call_args + assert len(args) == 2 # callback and initial_value + assert kwargs.get('max_prefetch') is None + + def test_qos_with_zero_worker_eta_task_limit(self): + """Test that QoS respects zero as a valid worker_eta_task_limit value.""" + c = self.c + c.app.conf.worker_eta_task_limit = 0 + c.initial_prefetch_count = 10 + c.task_consumer = Mock() + c.app.amqp.TaskConsumer = Mock(return_value=c.task_consumer) + c.connection.default_channel.basic_qos = Mock() + c.update_strategies = Mock() + c.on_decode_error = Mock() + + tasks = Tasks(c) + + with patch('celery.worker.consumer.tasks.QoS') as mock_qos: + tasks.start(c) + + # Verify QoS was called with max_prefetch set to 0 + mock_qos.assert_called_once() + args, kwargs = mock_qos.call_args + assert len(args) == 2 # callback and initial_value + assert kwargs.get('max_prefetch') == 0 + class test_Agent: diff --git a/t/unit/worker/test_strategy.py b/t/unit/worker/test_strategy.py index 30c50b98455..b2b829c4f45 100644 --- a/t/unit/worker/test_strategy.py +++ b/t/unit/worker/test_strategy.py @@ -1,5 +1,4 @@ import logging -from collections import defaultdict from contextlib import contextmanager from unittest.mock import ANY, Mock, patch @@ -99,8 +98,8 @@ def was_limited_with_eta(self): assert not self.was_reserved() called = self.consumer.timer.call_at.called if called: - assert self.consumer.timer.call_at.call_args[0][1] == \ - self.consumer._limit_post_eta + callback = self.consumer.timer.call_at.call_args[0][1] + assert callback == self.consumer._limit_post_eta return called def was_scheduled(self): @@ -128,10 +127,15 @@ def _context(self, sig, reserved = Mock() consumer = Mock() - consumer.task_buckets = defaultdict(lambda: None) + # Create a proper mock for task_buckets that supports __getitem__ + task_buckets_mock = Mock() + task_buckets_mock.__getitem__ = Mock(side_effect=lambda key: None) + consumer.task_buckets = task_buckets_mock if limit: bucket = TokenBucket(rate(limit), capacity=1) - consumer.task_buckets[sig.task] = bucket + task_buckets_mock.__getitem__.side_effect = ( + lambda key: bucket if key == sig.task else None + ) consumer.controller.state.revoked = set() consumer.disable_rate_limits = not rate_limits consumer.event_dispatcher.enabled = events From 95bec6d824363a6bb0ff08eda0da42e0d52c2a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Tue, 9 Sep 2025 18:10:02 +0600 Subject: [PATCH 023/169] Update runner version in Docker workflow (#9884) --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d91264cf842..ea8e5af3203 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,7 +26,7 @@ on: jobs: docker-build: - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - uses: actions/checkout@v5 @@ -36,7 +36,7 @@ jobs: run: make docker-build smoke-tests_dev: - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - uses: actions/checkout@v5 @@ -46,7 +46,7 @@ jobs: run: docker build -f t/smoke/workers/docker/dev . smoke-tests_latest: - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - uses: actions/checkout@v5 @@ -56,7 +56,7 @@ jobs: run: docker build -f t/smoke/workers/docker/pypi . smoke-tests_pypi: - runs-on: blacksmith-4vcpu-ubuntu-2204 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - uses: actions/checkout@v5 From 7c75fa738885315180f0194da04cf5105994fe13 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Mon, 15 Sep 2025 03:39:45 +0300 Subject: [PATCH 024/169] Prepare for (pre) release: v5.6.0b1 (#9890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.5.3 → 5.6.0b1 * Immunity -> Recovery * Added Changelog for v5.6.0b1 --- .bumpversion.cfg | 2 +- Changelog.rst | 64 ++++++++++- README.rst | 2 +- celery/__init__.py | 4 +- docs/history/changelog-5.6.rst | 69 ++++++++++++ docs/history/whatsnew-5.6.rst | 196 +++++++++++++++++++++++++++++++++ docs/includes/introduction.txt | 2 +- 7 files changed, 332 insertions(+), 7 deletions(-) create mode 100644 docs/history/changelog-5.6.rst create mode 100644 docs/history/whatsnew-5.6.rst diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 041bac81d1e..3f1fee8d873 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.5.3 +current_version = 5.6.0b1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/Changelog.rst b/Changelog.rst index 1eba0c056b2..f1cdcd6d237 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,8 +5,68 @@ ================ This document contains change notes for bugfix & new features -in the main branch & 5.5.x series, please see :ref:`whatsnew-5.5` for -an overview of what's new in Celery 5.5. +in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for +an overview of what's new in Celery 5.6. + +.. _version-5.6.0b1: + +5.6.0b1 +======= + +:release-date: 2025-09-15 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- docs: mention of json serializer recursive reference message size blowup (#5000) (#9743) +- docs: typo in canvas.rst (#9744) +- Makes _on_retry return a float as required to be used as errback on retry_over_time (#9741) +- Update canvas.rst doc calculation order for callback (#9758) +- Updated Blacksmith logo (#9763) +- Made the Sponsors logos link to their website (#9764) +- add missing cloudamqp logo (#9767) +- Improve sponsor visibility (#9768) +- fix: (#9773) task_id must not be empty with chain as body of a chord (#9774) +- Update setup.py to fix deprecation warning (#9771) +- Adds integration test for chord_unlock bug when routed to quorum/topic queue (#9766) +- Add xfail test for default queue/exchange fallback ignoring task_default_* settings (#9765) +- Add xfail test for RabbitMQ quorum queue global QoS race condition (#9770) +- fix: (#8786) time out when chord header fails with group body (#9788) +- Fix #9738 : Add root_id and parent_id to .apply() (#9784) +- Replace DelayedDelivery connection creation to use context manger (#9793) +- Fix #9794: Pydantic integration fails with __future__.annotations. (#9795) +- add go and rust implementation in docs (#9800) +- Fix memory leak in exception handling (Issue #8882) (#9799) +- Fix handlers docs (Issue #9787) (#9804) +- Remove importlib_metadata leftovers (#9791) +- Update timeout minutes for smoke tests CI (#9807) +- Revert "Remove dependency on `pycurl`" (#9620) +- Add Blacksmith Docker layer caching to all Docker builds (#9840) +- Bump Kombu to v5.6.0b1 (#9839) +- Disable pytest-xdist for smoke tests and increase retries (CI ONLY) (#9842) +- Fix Python 3.13 compatibility in events dumper (#9826) +- Dockerfile Build Optimizations (#9733) +- Migrated from useblacksmith/build-push-action@v1 to useblacksmith/setup-docker-builder@v1 in the CI (#9846) +- Remove incorrect example (#9854) +- Revert "Use Django DB max age connection setting" (#9824) +- Fix pending_result memory leak (#9806) +- Update python-package.yml (#9856) +- Bump Kombu to v5.6.0b2 (#9858) +- Refactor integration and smoke tests CI (#9855) +- Fix `AsyncResult.forget()` with couchdb backend method raises `TypeError: a bytes-like object is required, not 'str'` (#9865) +- Improve Docs for SQS Authentication (#9868) +- Added `.github/copilot-instructions.md` for GitHub Copilot (#9874) +- misc: credit removal (#9877) +- Choose queue type and exchange type when creating missing queues (fix #9671) (#9815) +- fix: prevent celery from hanging due to spawned greenlet errors in greenlet drainers (#9371) +- Feature/disable prefetch fixes (#9863) +- Add worker_eta_task_limit configuration to manage ETA task memory usage (#9853) +- Update runner version in Docker workflow (#9884) +- Prepare for (pre) release: v5.6.0b1 (#9890) .. _version-5.5.3: diff --git a/README.rst b/README.rst index 8415508638d..7537a56e7dd 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.5.3 (immunity) +:Version: 5.6.0b1 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index d291dec8c80..046a034a0c4 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -15,9 +15,9 @@ # Lazy loading from . import local -SERIES = 'immunity' +SERIES = 'recovery' -__version__ = '5.5.3' +__version__ = '5.6.0b1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst new file mode 100644 index 00000000000..8bbf0e39a1f --- /dev/null +++ b/docs/history/changelog-5.6.rst @@ -0,0 +1,69 @@ +.. _changelog-5.6: + +================ + Change history +================ + +This document contains change notes for bugfix & new features +in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for +an overview of what's new in Celery 5.6. + +.. _version-5.6.0b1: + +5.6.0b1 +======= + +:release-date: 2025-09-15 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- docs: mention of json serializer recursive reference message size blowup (#5000) (#9743) +- docs: typo in canvas.rst (#9744) +- Makes _on_retry return a float as required to be used as errback on retry_over_time (#9741) +- Update canvas.rst doc calculation order for callback (#9758) +- Updated Blacksmith logo (#9763) +- Made the Sponsors logos link to their website (#9764) +- add missing cloudamqp logo (#9767) +- Improve sponsor visibility (#9768) +- fix: (#9773) task_id must not be empty with chain as body of a chord (#9774) +- Update setup.py to fix deprecation warning (#9771) +- Adds integration test for chord_unlock bug when routed to quorum/topic queue (#9766) +- Add xfail test for default queue/exchange fallback ignoring task_default_* settings (#9765) +- Add xfail test for RabbitMQ quorum queue global QoS race condition (#9770) +- fix: (#8786) time out when chord header fails with group body (#9788) +- Fix #9738 : Add root_id and parent_id to .apply() (#9784) +- Replace DelayedDelivery connection creation to use context manger (#9793) +- Fix #9794: Pydantic integration fails with __future__.annotations. (#9795) +- add go and rust implementation in docs (#9800) +- Fix memory leak in exception handling (Issue #8882) (#9799) +- Fix handlers docs (Issue #9787) (#9804) +- Remove importlib_metadata leftovers (#9791) +- Update timeout minutes for smoke tests CI (#9807) +- Revert "Remove dependency on `pycurl`" (#9620) +- Add Blacksmith Docker layer caching to all Docker builds (#9840) +- Bump Kombu to v5.6.0b1 (#9839) +- Disable pytest-xdist for smoke tests and increase retries (CI ONLY) (#9842) +- Fix Python 3.13 compatibility in events dumper (#9826) +- Dockerfile Build Optimizations (#9733) +- Migrated from useblacksmith/build-push-action@v1 to useblacksmith/setup-docker-builder@v1 in the CI (#9846) +- Remove incorrect example (#9854) +- Revert "Use Django DB max age connection setting" (#9824) +- Fix pending_result memory leak (#9806) +- Update python-package.yml (#9856) +- Bump Kombu to v5.6.0b2 (#9858) +- Refactor integration and smoke tests CI (#9855) +- Fix `AsyncResult.forget()` with couchdb backend method raises `TypeError: a bytes-like object is required, not 'str'` (#9865) +- Improve Docs for SQS Authentication (#9868) +- Added `.github/copilot-instructions.md` for GitHub Copilot (#9874) +- misc: credit removal (#9877) +- Choose queue type and exchange type when creating missing queues (fix #9671) (#9815) +- fix: prevent celery from hanging due to spawned greenlet errors in greenlet drainers (#9371) +- Feature/disable prefetch fixes (#9863) +- Add worker_eta_task_limit configuration to manage ETA task memory usage (#9853) +- Update runner version in Docker workflow (#9884) +- Prepare for (pre) release: v5.6.0b1 (#9890) diff --git a/docs/history/whatsnew-5.6.rst b/docs/history/whatsnew-5.6.rst new file mode 100644 index 00000000000..6407231bd62 --- /dev/null +++ b/docs/history/whatsnew-5.6.rst @@ -0,0 +1,196 @@ +.. _whatsnew-5.6: + +========================================= + What's new in Celery 5.6 (Recovery) +========================================= +:Author: Tomer Nosrati (``tomer.nosrati at gmail.com``). + +.. sidebar:: Change history + + What's new documents describe the changes in major versions, + we also have a :ref:`changelog` that lists the changes in bugfix + releases (0.0.x), while older series are archived under the :ref:`history` + section. + +Celery is a simple, flexible, and reliable distributed programming framework +to process vast amounts of messages, while providing operations with +the tools required to maintain a distributed system with python. + +It's a task queue with focus on real-time processing, while also +supporting task scheduling. + +Celery has a large and diverse community of users and contributors, +you should come join us :ref:`on IRC ` +or :ref:`our mailing-list `. + +.. note:: + + Following the problems with Freenode, we migrated our IRC channel to Libera Chat + as most projects did. + You can also join us using `Gitter `_. + + We're sometimes there to answer questions. We welcome you to join. + +To read more about Celery you should go read the :ref:`introduction `. + +While this version is **mostly** backward compatible with previous versions +it's important that you read the following section as this release +is a new major version. + +This version is officially supported on CPython 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13. +and is also supported on PyPy3.10+. + +.. _`website`: https://celery.readthedocs.io + +.. topic:: Table of Contents + + Make sure you read the important notes before upgrading to this version. + +.. contents:: + :local: + :depth: 3 + +Preface +======= + +.. note:: + + **This release contains fixes for many long standing bugs & stability issues. + We encourage our users to upgrade to this release as soon as possible.** + +The 5.6.0 release is a new feature release for Celery. + +Releases in the 5.x series are codenamed after songs of `Jon Hopkins `_. +This release has been codenamed `Recovery `_. + +This is the last version to support Python 3.8. + +*— Tomer Nosrati* + +Long Term Support Policy +------------------------ + +We no longer support Celery 4.x as we don't have the resources to do so. +If you'd like to help us, all contributions are welcome. + +Celery 5.x **is not** an LTS release. We will support it until the release +of Celery 6.x. + +We're in the process of defining our Long Term Support policy. +Watch the next "What's New" document for updates. + +Upgrading from Celery 4.x +========================= + +Step 1: Adjust your command line invocation +------------------------------------------- + +Celery 5.0 introduces a new CLI implementation which isn't completely backwards compatible. + +The global options can no longer be positioned after the sub-command. +Instead, they must be positioned as an option for the `celery` command like so:: + + celery --app path.to.app worker + +If you were using our :ref:`daemonizing` guide to deploy Celery in production, +you should revisit it for updates. + +Step 2: Update your configuration with the new setting names +------------------------------------------------------------ + +If you haven't already updated your configuration when you migrated to Celery 4.0, +please do so now. + +We elected to extend the deprecation period until 6.0 since +we did not loudly warn about using these deprecated settings. + +Please refer to the :ref:`migration guide ` for instructions. + +Step 3: Read the important notes in this document +------------------------------------------------- + +Make sure you are not affected by any of the important upgrade notes +mentioned in the :ref:`following section `. + +You should verify that none of the breaking changes in the CLI +do not affect you. Please refer to :ref:`New Command Line Interface ` for details. + +Step 4: Migrate your code to Python 3 +------------------------------------- + +Celery 5.x only supports Python 3. Therefore, you must ensure your code is +compatible with Python 3. + +If you haven't ported your code to Python 3, you must do so before upgrading. + +You can use tools like `2to3 `_ +and `pyupgrade `_ to assist you with +this effort. + +After the migration is done, run your test suite with Celery 5 to ensure +nothing has been broken. + +Step 5: Upgrade to Celery 5.6 +----------------------------- + +At this point you can upgrade your workers and clients with the new version. + +.. _v560-important: + +Important Notes +=============== + +Supported Python Versions +------------------------- + +The supported Python versions are: + +- CPython 3.8 +- CPython 3.9 +- CPython 3.10 +- CPython 3.11 +- CPython 3.12 +- CPython 3.13 +- PyPy3.10 (``pypy3``) + +Python 3.8 Support +------------------ + +Python 3.8 will reach EOL in October, 2024. + +Minimum Dependencies +-------------------- + +Kombu +~~~~~ + +Starting from Celery v5.6, the minimum required version is Kombu 5.6. + +Redis +~~~~~ + +redis-py 4.5.2 is the new minimum required version. + + +SQLAlchemy +~~~~~~~~~~ + +SQLAlchemy 1.4.x & 2.0.x is now supported in Celery v5.6. + +Billiard +~~~~~~~~ + +Minimum required version is now 4.2.1. + +Django +~~~~~~ + +Minimum django version is bumped to v2.2.28. +Also added --skip-checks flag to bypass django core checks. + +.. _v560-news: + +News +==== + +Will be added as we get closer to the release. diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 4184b38313a..651dfa91ce7 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.5.3 (immunity) +:Version: 5.6.0b1 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From f55bb5f4cbbb59271a1b5ff2c58482a4cbb6d57c Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 18 Sep 2025 13:23:39 +0200 Subject: [PATCH 025/169] GitHub Actions: Test on Python 3.14 release candidate 2 (#9891) * GitHub Actions: Test on Python 3.14 release candidate 2 Python v3.14 -- October 7th * https://www.python.org/download/pre-releases * https://www.python.org/downloads/release/python-3140rc2 * Update default Python versions in integration tests --- .github/workflows/python-package.yml | 8 +++++--- requirements/extras/pydantic.txt | 3 ++- tox.ini | 12 +++++++----- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 913d9a1089c..54bb47f65bd 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,19 +33,21 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.10'] os: ["blacksmith-4vcpu-ubuntu-2404", "windows-latest"] exclude: - python-version: '3.9' os: "windows-latest" - - python-version: 'pypy-3.10' - os: "windows-latest" - python-version: '3.10' os: "windows-latest" - python-version: '3.11' os: "windows-latest" - python-version: '3.13' os: "windows-latest" + - python-version: '3.14' + os: "windows-latest" + - python-version: 'pypy3.10' + os: "windows-latest" steps: - name: Install apt packages diff --git a/requirements/extras/pydantic.txt b/requirements/extras/pydantic.txt index 29ac1fa96c9..99167f63c19 100644 --- a/requirements/extras/pydantic.txt +++ b/requirements/extras/pydantic.txt @@ -1 +1,2 @@ -pydantic>=2.4 +pydantic>=2.4 ; python_version < "3.14" +pydantic>=2.12.0a1 ; python_version >= "3.14" diff --git a/tox.ini b/tox.ini index 2b5fdfcfb57..1ee58b92c92 100644 --- a/tox.ini +++ b/tox.ini @@ -2,9 +2,9 @@ requires = tox-gh-actions envlist = - {3.8,3.9,3.10,3.11,3.12,3.13,pypy3}-unit - {3.8,3.9,3.10,3.11,3.12,3.13,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch,docker} - {3.8,3.9,3.10,3.11,3.12,3.13,pypy3}-smoke + {3.8,3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-unit + {3.8,3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch,docker} + {3.8,3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-smoke flake8 apicheck @@ -20,6 +20,7 @@ python = 3.11: 3.11-unit 3.12: 3.12-unit 3.13: 3.13-unit + 3.14: 3.14-unit pypy-3: pypy3-unit [testenv] @@ -32,8 +33,8 @@ deps= -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/pkgutils.txt - 3.8,3.9,3.10,3.11,3.12,3.13: -r{toxinidir}/requirements/test-ci-default.txt - 3.8,3.9,3.10,3.11,3.12,3.13: -r{toxinidir}/requirements/docs.txt + 3.8,3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/test-ci-default.txt + 3.8,3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/docs.txt pypy3: -r{toxinidir}/requirements/test-ci-default.txt integration: -r{toxinidir}/requirements/test-integration.txt @@ -91,6 +92,7 @@ basepython = 3.11: python3.11 3.12: python3.12 3.13: python3.13 + 3.14: python3.14 pypy3: pypy3 mypy: python3.13 lint,apicheck,linkcheck,configcheck,bandit: python3.13 From c5083fb1435e71271dd1dcd8355795805216a6cb Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 18 Sep 2025 14:16:50 +0200 Subject: [PATCH 026/169] Update pypy to python 3.11 (#9896) A focused subset of: * #9798 --- .github/workflows/python-package.yml | 4 ++-- requirements/extras/gcs.txt | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 54bb47f65bd..8fda862d201 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.10'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.11'] os: ["blacksmith-4vcpu-ubuntu-2404", "windows-latest"] exclude: - python-version: '3.9' @@ -46,7 +46,7 @@ jobs: os: "windows-latest" - python-version: '3.14' os: "windows-latest" - - python-version: 'pypy3.10' + - python-version: 'pypy3.11' os: "windows-latest" steps: diff --git a/requirements/extras/gcs.txt b/requirements/extras/gcs.txt index 7a724e51b15..0b06e78ea7c 100644 --- a/requirements/extras/gcs.txt +++ b/requirements/extras/gcs.txt @@ -1,3 +1,4 @@ google-cloud-storage>=2.10.0 google-cloud-firestore==2.20.1 -grpcio==1.67.0 +grpcio==1.67.0 ; python_version < "3.9" +grpcio==1.75.0 ; python_version >= "3.9" From 893ce604859a736e9b508991847a6ed8d89bc916 Mon Sep 17 00:00:00 2001 From: Md Al Amin Date: Thu, 18 Sep 2025 16:07:13 +0200 Subject: [PATCH 027/169] Feature: Add support credential_provider to Redis Backend (#9879) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: existing unit missing dependencies * feat: add option for redis credential provider * feat: add unit redis credential provider * fix: code style * feat: add docs * chore: add more tests and fix docs * doc: fix casing * chore: improve test coverage * fix: code style * chore: improve test coverage * fix: existing unit missing dependencies * feat: add option for redis credential provider * feat: add unit redis credential provider * fix: code style * feat: add docs * chore: add more tests and fix docs * doc: fix casing * chore: improve test coverage * fix: code style * chore: improve test coverage * Update docs/userguide/configuration.rst * chore: add versionadd annotation to 5.6 * fix: suggestion from co-pilot reviews * Update docs/userguide/configuration.rst --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/redis.py | 37 +++++++++ .../backends-and-brokers/redis.rst | 7 ++ docs/userguide/configuration.rst | 15 ++++ requirements/test.txt | 2 + t/unit/backends/test_redis.py | 79 ++++++++++++++++++- 5 files changed, 139 insertions(+), 1 deletion(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 7ddba5e5d63..89dccd7917a 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -5,9 +5,11 @@ from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED from urllib.parse import unquote +from kombu.utils import symbol_by_name from kombu.utils.functional import retry_over_time from kombu.utils.objects import cached_property from kombu.utils.url import _parse_url, maybe_sanitize_url +from redis import CredentialProvider from celery import states from celery._state import task_join_will_block @@ -230,6 +232,7 @@ def __init__(self, host=None, port=None, db=None, password=None, retry_on_timeout = _get('redis_retry_on_timeout') socket_keepalive = _get('redis_socket_keepalive') health_check_interval = _get('redis_backend_health_check_interval') + credential_provider = _get('redis_backend_credential_provider') self.connparams = { 'host': _get('redis_host') or 'localhost', @@ -254,6 +257,23 @@ def __init__(self, host=None, port=None, db=None, password=None, # support for py-redis<3.4.0. self.connparams['username'] = username + if credential_provider: + # if credential provider passed as string or query param + if isinstance(credential_provider, str): + credential_provider_cls = symbol_by_name(credential_provider) + credential_provider = credential_provider_cls() + + if not isinstance(credential_provider, CredentialProvider): + raise ValueError( + "Credential provider is not an instance of a redis.CredentialProvider or a subclass" + ) + + self.connparams['credential_provider'] = credential_provider + + # drop username and password if credential provider is configured + self.connparams.pop("username", None) + self.connparams.pop("password", None) + if health_check_interval: self.connparams["health_check_interval"] = health_check_interval @@ -350,6 +370,23 @@ def _params_from_url(self, url, defaults): db = db.strip('/') if isinstance(db, str) else db connparams['db'] = int(db) + # credential provider as query string + credential_provider = query.pop("credential_provider", None) + if credential_provider: + if isinstance(credential_provider, str): + credential_provider_cls = symbol_by_name(credential_provider) + credential_provider = credential_provider_cls() + + if not isinstance(credential_provider, CredentialProvider): + raise ValueError( + "Credential provider is not an instance of a redis.CredentialProvider or a subclass" + ) + + connparams['credential_provider'] = credential_provider + # drop username and password if credential provider is configured + connparams.pop("username", None) + connparams.pop("password", None) + for key, value in query.items(): if key in redis.connection.URL_QUERY_ARGUMENT_PARSERS: query[key] = redis.connection.URL_QUERY_ARGUMENT_PARSERS[key]( diff --git a/docs/getting-started/backends-and-brokers/redis.rst b/docs/getting-started/backends-and-brokers/redis.rst index 11d42544ec2..aec1232f3f0 100644 --- a/docs/getting-started/backends-and-brokers/redis.rst +++ b/docs/getting-started/backends-and-brokers/redis.rst @@ -38,6 +38,13 @@ Where the URL is in the format of: all fields after the scheme are optional, and will default to ``localhost`` on port 6379, using database 0. +If redis credential provider should be used, the URL needs to be in the following format: + +.. code-block:: text + + redis://@hostname:port/db_number?credential_provider=mymodule.myfile.myclass + + If a Unix socket connection should be used, the URL needs to be in the format: .. code-block:: text diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index fe01b6ecd95..54ac549bc42 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -114,6 +114,7 @@ have been moved into a new ``task_`` prefix. ``CELERY_REDIS_PASSWORD`` :setting:`redis_password` ``CELERY_REDIS_PORT`` :setting:`redis_port` ``CELERY_REDIS_BACKEND_USE_SSL`` :setting:`redis_backend_use_ssl` +``CELERY_REDIS_BACKEND_CREDENTIAL_PROVIDER`` :setting:`redis_backend_credential_provider` ``CELERY_RESULT_BACKEND`` :setting:`result_backend` ``CELERY_MAX_CACHED_RESULTS`` :setting:`result_cache_max` ``CELERY_MESSAGE_COMPRESSION`` :setting:`result_compression` @@ -763,6 +764,7 @@ Can be one of the following: .. _`AzureBlockBlob`: https://azure.microsoft.com/en-us/services/storage/blobs/ .. _`S3`: https://aws.amazon.com/s3/ .. _`GCS`: https://cloud.google.com/storage/ +.. _`RedisCredentialProvider`: https://redis.readthedocs.io/en/stable/examples/connection_examples.html#Connecting-to-a-redis-instance-with-standard-credential-provider .. setting:: result_backend_always_retry @@ -1344,6 +1346,19 @@ the form of a dictionary. The valid key-value pairs are the same as the ones mentioned in the ``redis`` sub-section under :setting:`broker_use_ssl`. +.. setting:: redis_backend_credential_provider + +.. versionadded:: 5.6 + +``redis_backend_credential_provider`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: Disabled. + +The Redis backend supports credential provider. This value must be set in +the form of a class path string or a class instance. e.g. ``mymodule.myfile.myclass`` +check more details in `RedisCredentialProvider`_ doc. + .. setting:: redis_max_connections ``redis_max_connections`` diff --git a/requirements/test.txt b/requirements/test.txt index 527d975f617..a7b758fbaf8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -18,3 +18,5 @@ pre-commit>=4.0.1; python_version >= '3.9' -r extras/mongodb.txt -r extras/gcs.txt -r extras/pydantic.txt +-r extras/azureblockblob.txt +-r extras/gevent.txt diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 314327ef174..a4af637e869 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -10,9 +10,10 @@ import pytest try: - from redis import exceptions + from redis import CredentialProvider, exceptions except ImportError: exceptions = None + CredentialProvider = None from celery import signature, states, uuid from celery.canvas import Signature @@ -369,6 +370,14 @@ def setup_method(self): self.b = self.Backend(app=self.app) +class MyCredentialProvider(CredentialProvider): + pass + + +class NonCredentialProvider: + pass + + class test_RedisBackend(basetest_RedisBackend): @pytest.mark.usefixtures('depends_on_current_app') def test_reduce(self): @@ -397,6 +406,33 @@ def test_username_password_from_redis_conf(self): assert x.connparams['username'] == 'username' assert x.connparams['password'] == 'password' + def test_credential_provider_from_redis_conf(self): + self.app.conf.redis_backend_credential_provider = "redis.CredentialProvider" + x = self.Backend(app=self.app) + + assert x.connparams + assert 'credential_provider' in x.connparams + assert 'username' not in x.connparams + assert 'password' not in x.connparams + + # with local credential provider + self.app.conf.redis_backend_credential_provider = MyCredentialProvider() + x = self.Backend(app=self.app) + assert x.connparams + assert 'credential_provider' in x.connparams + assert 'username' not in x.connparams + assert 'password' not in x.connparams + + # raise ImportError + self.app.conf.redis_backend_credential_provider = "not_exist.CredentialProvider" + with pytest.raises(ImportError): + self.Backend(app=self.app) + + # raise value Error + self.app.conf.redis_backend_credential_provider = NonCredentialProvider() + with pytest.raises(ValueError): + self.Backend(app=self.app) + def test_url(self): self.app.conf.redis_socket_timeout = 30.0 self.app.conf.redis_socket_connect_timeout = 100.0 @@ -424,6 +460,47 @@ def test_url(self): assert x.connparams['socket_timeout'] == 30.0 assert x.connparams['socket_connect_timeout'] == 100.0 + def test_url_with_credential_provider(self): + self.app.conf.redis_socket_timeout = 30.0 + self.app.conf.redis_socket_connect_timeout = 100.0 + x = self.Backend( + 'redis://:bosco@vandelay.com:123/1?credential_provider=redis.CredentialProvider', app=self.app, + ) + + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['socket_timeout'] == 30.0 + assert x.connparams['socket_connect_timeout'] == 100.0 + assert isinstance(x.connparams['credential_provider'], CredentialProvider) + assert "username" not in x.connparams + assert "password" not in x.connparams + + # without username and password + x = self.Backend( + 'redis://@vandelay.com:123/1?credential_provider=redis.UsernamePasswordCredentialProvider', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert isinstance(x.connparams['credential_provider'], CredentialProvider) + + # raise importError + with pytest.raises(ImportError): + self.Backend( + 'redis://@vandelay.com:123/1?credential_provider=not_exist.CredentialProvider', app=self.app, + ) + + # raise valueError + with pytest.raises(ValueError): + # some non-credential provider class + # not ideal but serve purpose + self.Backend( + 'redis://@vandelay.com:123/1?credential_provider=abc.ABC', app=self.app, + ) + def test_timeouts_in_url_coerced(self): pytest.importorskip('redis') From a5e9ce5e348eb34db19919e72624f10c5ccba3d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Sep 2025 03:44:40 +0300 Subject: [PATCH 028/169] Bump pytest-cov from 6.0.0 to 7.0.0 (#9899) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 6.0.0 to 7.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v6.0.0...v7.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-version: 7.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/test-ci-base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test-ci-base.txt b/requirements/test-ci-base.txt index b5649723471..ec8c7c2a780 100644 --- a/requirements/test-ci-base.txt +++ b/requirements/test-ci-base.txt @@ -1,5 +1,5 @@ pytest-cov==5.0.0; python_version<"3.9" -pytest-cov==6.0.0; python_version>="3.9" +pytest-cov==7.0.0; python_version>="3.9" pytest-github-actions-annotate-failures==0.3.0 -r extras/redis.txt -r extras/sqlalchemy.txt From 989aac0889ad0157fa7db8228c81461d9a3fa75e Mon Sep 17 00:00:00 2001 From: Wout De Nolf Date: Sat, 20 Sep 2025 06:32:03 +0200 Subject: [PATCH 029/169] Celery.timezone: try tzlocal.get_localzone() before using LocalTimezone (#9862) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Celery.timezone: try tzlocal.get_localzone() before using LocalTimezone() * app tests for local timezone --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/utils/time.py | 18 ++++++++++++++---- requirements/default.txt | 1 + setup.cfg | 1 + t/unit/app/test_app.py | 32 +++++++++++++++++++++++++++++--- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/celery/utils/time.py b/celery/utils/time.py index 2376bb3b71d..f7a373bf2ca 100644 --- a/celery/utils/time.py +++ b/celery/utils/time.py @@ -1,6 +1,7 @@ """Utilities related to dates, times, intervals, and timezones.""" from __future__ import annotations +import logging import numbers import os import random @@ -17,6 +18,7 @@ from dateutil.parser import isoparse from kombu.utils.functional import reprcall from kombu.utils.objects import cached_property +from tzlocal import get_localzone from .functional import dictfilter from .text import pluralize @@ -26,6 +28,7 @@ else: from backports.zoneinfo import ZoneInfo +logger = logging.getLogger(__name__) __all__ = ( 'LocalTimezone', 'timezone', 'maybe_timedelta', @@ -117,7 +120,7 @@ def _isdst(self, dt: datetime) -> bool: class _Zone: """Timezone class that provides the timezone for the application. - If `enable_utc` is disabled, LocalTimezone is provided as the timezone provider through local(). + If `enable_utc` is disabled, local system timezone is provided as the timezone provider through local(). Otherwise, this class provides a UTC ZoneInfo instance as the timezone provider for the application. Additionally this class provides a few utility methods for converting datetimes. @@ -158,9 +161,16 @@ def get_timezone(self, zone: str | tzinfo) -> tzinfo: return zone @cached_property - def local(self) -> LocalTimezone: - """Return LocalTimezone instance for the application.""" - return LocalTimezone() + def local(self) -> tzinfo: + """Return the local system timezone for the application.""" + try: + timezone = get_localzone() + except Exception as ex: + timezone = None + logger.warning("Failed to retrieve local timezone (%s): %s", type(ex).__name__, ex) + if timezone is None: + return LocalTimezone() + return timezone @cached_property def utc(self) -> tzinfo: diff --git a/requirements/default.txt b/requirements/default.txt index 015541462aa..7077a678c2b 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -7,3 +7,4 @@ click-repl>=0.2.0 click-plugins>=1.1.1 backports.zoneinfo[tzdata]>=0.2.1; python_version < '3.9' python-dateutil>=2.8.2 +tzlocal diff --git a/setup.cfg b/setup.cfg index a74a438d952..775ec0cc776 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ per-file-ignores = [bdist_rpm] requires = backports.zoneinfo>=0.2.1;python_version<'3.9' tzdata>=2022.7 + tzlocal billiard >=4.1.0,<5.0 kombu >= 5.3.4,<6.0.0 diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index ca2dd2b4bf1..32cc338c336 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -32,13 +32,13 @@ from celery.utils.collections import DictAttribute from celery.utils.objects import Bunch from celery.utils.serialization import pickle -from celery.utils.time import localize, timezone, to_utc +from celery.utils.time import LocalTimezone, localize, timezone, to_utc from t.unit import conftest if sys.version_info >= (3, 9): from zoneinfo import ZoneInfo else: - from backports.zoneinfo import ZoneInfo # noqa + from backports.zoneinfo import ZoneInfo THIS_IS_A_KEY = 'this is a value' @@ -1176,7 +1176,7 @@ def test_signature(self): sig = self.app.signature('foo', (1, 2)) assert sig.app is self.app - def test_timezone__none_set(self): + def test_timezone_none_set(self): self.app.conf.timezone = None self.app.conf.enable_utc = True assert self.app.timezone == timezone.utc @@ -1184,6 +1184,32 @@ def test_timezone__none_set(self): self.app.conf.enable_utc = False assert self.app.timezone == timezone.local + def test_use_local_timezone(self): + self.app.conf.timezone = None + self.app.conf.enable_utc = False + + self._clear_timezone_cache() + try: + assert isinstance(self.app.timezone, ZoneInfo) + finally: + self._clear_timezone_cache() + + @patch("celery.utils.time.get_localzone") + def test_use_local_timezone_failure(self, mock_get_localzone): + mock_get_localzone.side_effect = Exception("Failed to get local timezone") + self.app.conf.timezone = None + self.app.conf.enable_utc = False + + self._clear_timezone_cache() + try: + assert isinstance(self.app.timezone, LocalTimezone) + finally: + self._clear_timezone_cache() + + def _clear_timezone_cache(self): + del self.app.timezone + del timezone.local + def test_uses_utc_timezone(self): self.app.conf.timezone = None self.app.conf.enable_utc = True From 9bea091e4804b792d1ec305871d119132146f0aa Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sat, 20 Sep 2025 19:27:12 +0200 Subject: [PATCH 030/169] Run integration tests on Python 3.14 (#9903) * Run integration tests on Python 3.14 * Fix formatting of docstring in __init__.py * Fix formatting of docstring in __init__.py --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9bc35c1e40e..d12d2f1e168 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,7 +11,7 @@ on: description: 'JSON array of Python versions to test' required: false type: string - default: '["3.8", "3.13"]' + default: '["3.8", "3.14"]' tox_environments: description: 'JSON array of tox environments to test' required: false From c276fed72cc18b5e88077004124eaad0327ac666 Mon Sep 17 00:00:00 2001 From: Vlad Borovtsov Date: Sun, 21 Sep 2025 16:57:23 +0200 Subject: [PATCH 031/169] Fix arithmetic overflow for MSSQL result backend (#9904) * Fix arithmetic overflow for MSSQL result backend * Minor formatting adjustment in `models.py` to satisfy linter * Add MSSQL-specific ID field type test for database models * [pre-commit.ci] auto fixes from pre-commit.com hooks --- celery/backends/database/models.py | 6 ++++-- t/unit/backends/test_database.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/celery/backends/database/models.py b/celery/backends/database/models.py index a5df8f4d341..ddc18747bac 100644 --- a/celery/backends/database/models.py +++ b/celery/backends/database/models.py @@ -10,6 +10,8 @@ __all__ = ('Task', 'TaskExtended', 'TaskSet') +DialectSpecificInteger = sa.Integer().with_variant(sa.BigInteger, 'mssql') + class Task(ResultModelBase): """Task result/status.""" @@ -17,7 +19,7 @@ class Task(ResultModelBase): __tablename__ = 'celery_taskmeta' __table_args__ = {'sqlite_autoincrement': True} - id = sa.Column(sa.Integer, sa.Sequence('task_id_sequence'), + id = sa.Column(DialectSpecificInteger, sa.Sequence('task_id_sequence'), primary_key=True, autoincrement=True) task_id = sa.Column(sa.String(155), unique=True) status = sa.Column(sa.String(50), default=states.PENDING) @@ -80,7 +82,7 @@ class TaskSet(ResultModelBase): __tablename__ = 'celery_tasksetmeta' __table_args__ = {'sqlite_autoincrement': True} - id = sa.Column(sa.Integer, sa.Sequence('taskset_id_sequence'), + id = sa.Column(DialectSpecificInteger, sa.Sequence('taskset_id_sequence'), autoincrement=True, primary_key=True) taskset_id = sa.Column(sa.String(155), unique=True) result = sa.Column(PickleType, nullable=True) diff --git a/t/unit/backends/test_database.py b/t/unit/backends/test_database.py index 328ee0c9c02..2a738731c07 100644 --- a/t/unit/backends/test_database.py +++ b/t/unit/backends/test_database.py @@ -45,6 +45,26 @@ def test_context_raises(self): session.close.assert_called_with() +@skip.if_pypy +class test_ModelsIdFieldTypeVariations: + + def test_for_mssql_dialect(self): + """Test that ID columns use BigInteger for MSSQL and Integer for other dialects.""" + from sqlalchemy import BigInteger, Integer + from sqlalchemy.dialects import mssql, mysql, oracle, postgresql, sqlite + + models = [Task, TaskSet] + id_columns = [m.__table__.columns['id'] for m in models] + + for dialect in [mssql, postgresql, mysql, sqlite, oracle]: + for id_column in id_columns: + compiled_type = id_column.type.dialect_impl(dialect.dialect()) + if dialect == mssql: + assert isinstance(compiled_type, BigInteger) + else: + assert isinstance(compiled_type, Integer) + + @skip.if_pypy class test_DatabaseBackend: From 70087e49888cefe092ac2068ceb37769d8e8dd4b Mon Sep 17 00:00:00 2001 From: Sumanth Kaushik Date: Tue, 23 Sep 2025 02:01:05 -0700 Subject: [PATCH 032/169] Add documentation for task_id param for apply_async --- celery/app/task.py | 7 +++++++ docs/userguide/calling.rst | 2 ++ 2 files changed, 9 insertions(+) diff --git a/celery/app/task.py b/celery/app/task.py index 1688eafd01b..5db58db81b4 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -538,6 +538,13 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, The headers can be used as an overlay for custom labeling using the :ref:`canvas-stamping` feature. + task_id (str): Optional argument to override the default task id. + By default, Celery generates a unique id (UUID4) for every task + submission. You can instead provide your own string identifier. + If supplied, this value will be used as the task’s id instead + of generating one automatically. Be careful to avoid collisions + when overriding task ids. + Returns: celery.result.AsyncResult: Promise of future evaluation. diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 63b8998f77f..d9c29dc536c 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -56,6 +56,8 @@ The API defines a standard set of execution options, as well as three methods: - ``T.apply_async(expires=now + timedelta(days=2))`` expires in 2 days, set using :class:`~datetime.datetime`. + - ``T.apply_async(task_id=f'my_own_task_id')`` + sets the id of the task to my_own_task_id instead of a uuid that is normally generated Example ------- From 8297e70abac8daede32bafc7c9ba1576d1a11020 Mon Sep 17 00:00:00 2001 From: Marcello Dalponte Date: Sun, 28 Sep 2025 18:42:55 +0200 Subject: [PATCH 033/169] Support redis client name (#9900) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for redis client_name * Update docs * Add support for redis client_name * Update docs * Update docs/userguide/configuration.rst --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/task.py | 2 +- celery/backends/redis.py | 1 + docs/userguide/configuration.rst | 12 ++++++++++++ t/unit/backends/test_redis.py | 25 +++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/celery/app/task.py b/celery/app/task.py index 5db58db81b4..3ab54ad623e 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -544,7 +544,7 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, If supplied, this value will be used as the task’s id instead of generating one automatically. Be careful to avoid collisions when overriding task ids. - + Returns: celery.result.AsyncResult: Promise of future evaluation. diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 89dccd7917a..6e0713c0568 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -244,6 +244,7 @@ def __init__(self, host=None, port=None, db=None, password=None, 'retry_on_timeout': retry_on_timeout or False, 'socket_connect_timeout': socket_connect_timeout and float(socket_connect_timeout), + 'client_name': _get('redis_client_name'), } username = _get('redis_username') diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 54ac549bc42..8c256afb609 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1420,6 +1420,18 @@ Default: :const:`False` Socket TCP keepalive to keep connections healthy to the Redis server, used by the redis result backend. +.. setting:: redis_client_name + +``redis_client_name`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.6 + +Default: :const:`None` + +Sets the client name for Redis connections used by the result backend. +This can help identify connections in Redis monitoring tools. + .. _conf-cassandra-result-backend: Cassandra/AstraDB backend settings diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index a4af637e869..3ffa60f4477 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -613,6 +613,31 @@ def test_backend_health_check_interval_not_set(self): assert x.connparams['port'] == 123 assert "health_check_interval" not in x.connparams + def test_backend_redis_client_name(self): + pytest.importorskip('redis') + + self.app.conf.redis_client_name = 'celery-worker' + x = self.Backend( + 'redis://vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['client_name'] == 'celery-worker' + + def test_backend_redis_client_name_not_set(self): + pytest.importorskip('redis') + + x = self.Backend( + 'redis://vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['client_name'] is None + @pytest.mark.parametrize('cert_str', [ "required", "CERT_REQUIRED", From 943dfb869bd23195f169a7e71b6d320bfdbe66c9 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 28 Sep 2025 19:43:56 +0300 Subject: [PATCH 034/169] Bump Kombu to v5.6.0rc1 (#9918) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump Kombu to v5.6.0rc1 * Fixed lint error from `main` --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 7077a678c2b..185b6eddd09 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,5 +1,5 @@ billiard>=4.2.1,<5.0 -kombu>=5.6.0b2,<5.7 +kombu>=5.6.0rc1,<5.7 vine>=5.1.0,<6.0 click>=8.1.2,<9.0 click-didyoumean>=0.3.0 From e7a95502fa81bc47d1bdd7d9d893fce35d254e6b Mon Sep 17 00:00:00 2001 From: Sumanth Kaushik <60442070+sumo1998@users.noreply.github.com> Date: Sun, 28 Sep 2025 10:31:41 -0700 Subject: [PATCH 035/169] Fix broker connection retry attempt counter in the error log (#9911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix retry attempt counter in broker connection error log * Prevent retry count change during failover * Update celery/worker/consumer/consumer.py --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/worker/consumer/consumer.py | 9 ++++++++- t/unit/worker/test_consumer.py | 8 ++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 3e6a66df532..9f843afccf1 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -157,6 +157,10 @@ class Consumer: #: connection attempt. first_connection_attempt = True + #: Counter to track number of conn retry attempts + #: to broker. Will be reset to 0 once successful + broker_connection_retry_attempt = 0 + class Blueprint(bootsteps.Blueprint): """Consumer blueprint.""" @@ -488,9 +492,11 @@ def ensure_connected(self, conn): def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP): if getattr(conn, 'alt', None) and interval == 0: next_step = CONNECTION_FAILOVER + elif interval > 0: + self.broker_connection_retry_attempt += 1 next_step = next_step.format( when=humanize_seconds(interval, 'in', ' '), - retries=int(interval / 2), + retries=self.broker_connection_retry_attempt, max_retries=self.app.conf.broker_connection_max_retries) error(CONNECTION_ERROR, conn.as_uri(), exc, next_step) @@ -532,6 +538,7 @@ def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP): callback=maybe_shutdown, ) self.first_connection_attempt = False + self.broker_connection_retry_attempt = 0 return conn def _flush_events(self): diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index bc21d73697e..c98ae23cb00 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -405,6 +405,8 @@ def test_connect_error_handler_progress(self, error): self.app.conf.broker_connection_max_retries = 3 self.app._connection = _amqp_connection() conn = self.app._connection.return_value + # Placeholder alt connection to satisfy failover condition + conn.alt = [conn] c = self.get_consumer() assert c.connect() errback = conn.ensure_connection.call_args[0][0] @@ -412,8 +414,10 @@ def test_connect_error_handler_progress(self, error): assert error.call_args[0][3] == 'Trying again in 2.00 seconds... (1/3)' errback(Mock(), 4) assert error.call_args[0][3] == 'Trying again in 4.00 seconds... (2/3)' - errback(Mock(), 6) - assert error.call_args[0][3] == 'Trying again in 6.00 seconds... (3/3)' + errback(Mock(), 12) + assert error.call_args[0][3] == 'Trying again in 12.00 seconds... (3/3)' + errback(Mock(), 0) + assert getattr(c, 'broker_connection_retry_attempt', 0) == 3 def test_cancel_long_running_tasks_on_connection_loss(self): c = self.get_consumer() From 8063230bba321ddda3a8e0549f1f4c86ba3b5469 Mon Sep 17 00:00:00 2001 From: Daniel Khodos Date: Tue, 30 Sep 2025 09:13:18 +0300 Subject: [PATCH 036/169] fix: restrict disable-prefetch feature to Redis brokers only (#9919) * fix: restrict disable-prefetch feature to Redis brokers only The disable-prefetch feature was originally implemented for all brokers, but users reported compatibility issues with RabbitMQ. This change restricts the feature to Redis brokers only, which is where most testing was conducted. Changes: - Add broker transport detection in Tasks.start() - Only apply disable-prefetch logic for Redis brokers - Log warning for non-Redis brokers when setting is enabled - Update CLI help text to mention Redis-only support - Update documentation to reflect Redis-only limitation - Add tests for non-Redis broker behavior Fixes compatibility issues with RabbitMQ and other non-Redis brokers while maintaining the feature for Redis users. * fix: resolve linting issues - Break long line in worker.py help text - Remove trailing whitespace in test file * fix: prevent disable-prefetch warning from interfering with existing test The test_log_when_qos_is_false test was failing because our new warning for non-Redis brokers was being logged, causing the test to expect 2 log records instead of 1. Fixed by explicitly setting worker_disable_prefetch to False in this test. --- celery/bin/worker.py | 3 +- celery/worker/consumer/tasks.py | 10 +++++++ docs/faq.rst | 3 +- docs/userguide/configuration.rst | 10 +++++++ docs/userguide/optimizing.rst | 3 +- t/unit/bin/test_worker.py | 18 ++++++++++++ t/unit/worker/test_autoscale.py | 2 ++ t/unit/worker/test_consumer.py | 48 ++++++++++++++++++++++++++++++++ 8 files changed, 94 insertions(+), 3 deletions(-) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 52f09f3a83d..f16dafba94f 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -189,7 +189,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, value: ctx.obj.app.conf.worker_disable_prefetch if value is None else value, cls=CeleryOption, help_group="Worker Options", - help="Disable broker prefetching. The worker will only fetch a task when a process slot is available.") + help="Disable broker prefetching. The worker will only fetch a task when a process slot is available. " + "Only supported with Redis brokers.") @click.option('-c', '--concurrency', type=int, diff --git a/celery/worker/consumer/tasks.py b/celery/worker/consumer/tasks.py index ae2029bca42..b017cf52838 100644 --- a/celery/worker/consumer/tasks.py +++ b/celery/worker/consumer/tasks.py @@ -52,6 +52,16 @@ def set_prefetch_count(prefetch_count): ) if c.app.conf.worker_disable_prefetch: + # Only apply disable-prefetch for Redis brokers + is_redis_broker = c.connection.transport.driver_type == 'redis' + if not is_redis_broker: + logger.warning( + f"worker_disable_prefetch is only supported for Redis brokers. " + f"Current broker transport: {c.connection.transport.driver_type}. " + f"Ignoring disable_prefetch setting." + ) + return + from types import MethodType from celery.worker import state diff --git a/docs/faq.rst b/docs/faq.rst index d0946153565..17e3dd5b338 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -790,7 +790,8 @@ consume as many tasks as it can, as fast as possible. You can use the :option:`--disable-prefetch ` flag (or set :setting:`worker_disable_prefetch` to ``True``) so that a worker -only fetches a task when one of its processes is free. +only fetches a task when one of its processes is free. This feature is currently +only supported when using Redis as the broker. A discussion on prefetch limits, and configuration settings for a worker that only reserves one task at a time is found here: diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 8c256afb609..975172a9cfd 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -3217,6 +3217,10 @@ early acknowledgments, enable :setting:`worker_disable_prefetch`. When this option is enabled the worker only fetches a task from the broker when one of its processes is available. +.. note:: + + This feature is currently only supported when using Redis as the broker. + You can also enable this via the :option:`--disable-prefetch ` command line flag. @@ -3258,6 +3262,12 @@ has an available process to execute them. This disables prefetching while still using early acknowledgments, ensuring that tasks are fairly distributed between workers. +.. note:: + + This feature is currently only supported when using Redis as the broker. + Using this setting with other brokers will result in a warning and the + setting will be ignored. + .. setting:: worker_enable_prefetch_count_reduction ``worker_enable_prefetch_count_reduction`` diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 42cfdda33ad..7cc7b635751 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -186,7 +186,8 @@ prefetching by enabling :setting:`worker_disable_prefetch`. With this setting the worker fetches a new task only when an execution slot is free, preventing tasks from waiting behind long running ones on busy workers. This can also be set from the command line using -:option:`--disable-prefetch `. +:option:`--disable-prefetch `. This feature +is currently only supported when using Redis as the broker. Memory Usage ------------ diff --git a/t/unit/bin/test_worker.py b/t/unit/bin/test_worker.py index 0f219e177b1..baa73385d6c 100644 --- a/t/unit/bin/test_worker.py +++ b/t/unit/bin/test_worker.py @@ -19,6 +19,7 @@ def mock_app(): app = Mock() app.conf = Mock() app.conf.worker_disable_prefetch = False + app.conf.worker_detect_quorum_queues = False return app @@ -37,10 +38,14 @@ def mock_consumer(mock_app): original_can_consume = Mock(return_value=True) consumer.task_consumer.channel.qos.can_consume = original_can_consume consumer.connection = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' # Default to Redis for existing tests + consumer.connection.qos_semantics_matches_spec = True consumer.update_strategies = Mock() consumer.on_decode_error = Mock() consumer.app.amqp = Mock() consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + consumer.app.amqp.queues = {} # Empty dict for quorum queue detection return consumer @@ -107,3 +112,16 @@ def test_disable_prefetch_none_preserves_behavior(mock_app, mock_consumer): tasks_instance = Tasks(mock_consumer) tasks_instance.start(mock_consumer) assert mock_consumer.task_consumer.channel.qos.can_consume == original_can_consume + + +def test_disable_prefetch_ignored_for_non_redis_brokers(mock_app, mock_consumer): + """Test that disable_prefetch is ignored for non-Redis brokers.""" + mock_app.conf.worker_disable_prefetch = True + mock_consumer.connection.transport.driver_type = 'amqp' # RabbitMQ + original_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + + tasks_instance = Tasks(mock_consumer) + tasks_instance.start(mock_consumer) + + # Should not modify can_consume method for non-Redis brokers + assert mock_consumer.task_consumer.channel.qos.can_consume == original_can_consume diff --git a/t/unit/worker/test_autoscale.py b/t/unit/worker/test_autoscale.py index 79eded5d923..c5f459b5ff0 100644 --- a/t/unit/worker/test_autoscale.py +++ b/t/unit/worker/test_autoscale.py @@ -259,6 +259,8 @@ def test_disable_prefetch_respects_max_concurrency(self): # Mock the connection and other required attributes consumer.connection = Mock() consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' consumer.initial_prefetch_count = 20 consumer.update_strategies = Mock() consumer.on_decode_error = Mock() diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index c98ae23cb00..02e77e9e58e 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -514,6 +514,8 @@ def test_disable_prefetch_not_enabled(self): consumer.initial_prefetch_count = 16 consumer.connection = Mock() consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' consumer.update_strategies = Mock() consumer.on_decode_error = Mock() @@ -549,6 +551,8 @@ def test_disable_prefetch_enabled_basic(self): consumer.initial_prefetch_count = 16 consumer.connection = Mock() consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' consumer.update_strategies = Mock() consumer.on_decode_error = Mock() @@ -587,6 +591,8 @@ def test_disable_prefetch_respects_reserved_requests_limit(self): consumer.initial_prefetch_count = 16 consumer.connection = Mock() consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' consumer.update_strategies = Mock() consumer.on_decode_error = Mock() @@ -625,6 +631,8 @@ def test_disable_prefetch_respects_autoscale_max_concurrency(self): consumer.initial_prefetch_count = 16 consumer.connection = Mock() consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' consumer.update_strategies = Mock() consumer.on_decode_error = Mock() @@ -648,6 +656,45 @@ def test_disable_prefetch_respects_autoscale_max_concurrency(self): # Should not be able to consume when at autoscale limit assert consumer.task_consumer.channel.qos.can_consume() is False + def test_disable_prefetch_ignored_for_non_redis_brokers(self): + """Test that disable_prefetch is ignored for non-Redis brokers.""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'amqp' # RabbitMQ + consumer.connection.qos_semantics_matches_spec = True + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + consumer.app.amqp.queues = {} # Empty dict for quorum queue detection + + tasks_instance = Tasks(consumer) + tasks_instance.start(consumer) + + # Should not modify can_consume method for non-Redis brokers + assert consumer.task_consumer.channel.qos.can_consume == original_can_consume + @pytest.mark.parametrize( "broker_connection_retry_on_startup,is_connection_loss_on_startup", @@ -847,6 +894,7 @@ def test_log_when_qos_is_false(self, caplog): c = self.c c.connection.transport.driver_type = 'amqp' c.app.conf.broker_native_delayed_delivery = True + c.app.conf.worker_disable_prefetch = False # Prevent our warning from interfering c.app.amqp.queues = {"celery": Mock(queue_arguments={"x-queue-type": "quorum"})} tasks = Tasks(c) From 88f3cc74d98d75ab0c23d21ca5e9513bd28dff90 Mon Sep 17 00:00:00 2001 From: Simone Pozzoli <108676548+simonepozzoli-pix4d@users.noreply.github.com> Date: Sat, 4 Oct 2025 13:49:01 +0200 Subject: [PATCH 037/169] fix(): preserve group order in replaced signature (#9910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(): preserve group order in replaced signature * fix flake8 * add tests * add other integration test * remove comment * skip test with rpc backend * Update celery/canvas.py * Revert "Update celery/canvas.py" This reverts commit 39385e03393191c438b4df879292161206d52679. The orignal commit introduced a syntax error. --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/canvas.py | 2 +- t/integration/test_canvas.py | 7 +++++++ t/unit/tasks/test_canvas.py | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/celery/canvas.py b/celery/canvas.py index 1ceeacc166d..396eb7d307b 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -1741,7 +1741,7 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, def _apply_tasks(self, tasks, producer=None, app=None, p=None, add_to_parent=None, chord=None, - args=None, kwargs=None, **options): + args=None, kwargs=None, group_index=None, **options): """Run all the tasks in the group. This is used by :meth:`apply_async` to run all the tasks in the group diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index d7b47362440..fd036e4cb10 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -1575,6 +1575,13 @@ def test_group_child_replaced_with_chain_last(self, manager): res_obj = orig_sig.delay() assert res_obj.get(timeout=TIMEOUT) == [42, 1337] + def test_task_replace_with_group_preserves_group_order(self, manager): + if manager.app.conf.result_backend.startswith("rpc"): + raise pytest.skip("RPC result backend does not support replacing with a group") + orig_sig = group([add_to_all.s([2, 1], 1), add_to_all.s([4, 3], 1)] * 10) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [[3, 2], [5, 4]] * 10 + def assert_ids(r, expected_value, expected_root_id, expected_parent_id): root_id, parent_id, value = r.get(timeout=TIMEOUT) diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 1eb088f0c51..40f02e6db8a 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -1243,6 +1243,12 @@ def test_group_prepared(self): assert isinstance(result, AsyncResult) assert group_id is not None + def test_task_replace_with_group_preserves_group_order(self): + self.app.conf.task_always_eager = True + sig = self.replace_with_group.s(1, 2) + res = self.helper_test_get_delay(sig.delay()) + assert res == [3, 2] + class test_chord(CanvasCase): def test__get_app_does_not_exhaust_generator(self): From 7fc848fd04f9820874560f3fd7c395ab83919245 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Oct 2025 01:24:08 +0300 Subject: [PATCH 038/169] Bump github/codeql-action from 3 to 4 (#9928) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c4372c0848b..b3d956d48c9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 From ae853421356e837feedb18d7266f59d610bd5ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Thu, 9 Oct 2025 10:57:30 +0600 Subject: [PATCH 039/169] Remove Python 3.8 from CI workflow (#9930) --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8fda862d201..f070cfe9c21 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -33,7 +33,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.11'] os: ["blacksmith-4vcpu-ubuntu-2404", "windows-latest"] exclude: - python-version: '3.9' From 85b053f36ab848a02a0e42af3f5114d67e2aefb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Thu, 9 Oct 2025 10:58:01 +0600 Subject: [PATCH 040/169] Update default Python versions in integration tests (#9931) --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d12d2f1e168..0c4a0fb85d3 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,7 +11,7 @@ on: description: 'JSON array of Python versions to test' required: false type: string - default: '["3.8", "3.14"]' + default: '["3.9", "3.14"]' tox_environments: description: 'JSON array of tox environments to test' required: false From ae89d92a994ebb6304f997a18446a988bbdc33a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Thu, 9 Oct 2025 11:07:05 +0600 Subject: [PATCH 041/169] Update tox.ini to remove Python 3.8 (#9932) Removed Python 3.8 from the environment list and dependencies. --- tox.ini | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 1ee58b92c92..bc8abdc6abf 100644 --- a/tox.ini +++ b/tox.ini @@ -2,9 +2,9 @@ requires = tox-gh-actions envlist = - {3.8,3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-unit - {3.8,3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch,docker} - {3.8,3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-smoke + {3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-unit + {3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch,docker} + {3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-smoke flake8 apicheck @@ -14,7 +14,6 @@ envlist = [gh-actions] python = - 3.8: 3.8-unit 3.9: 3.9-unit 3.10: 3.10-unit 3.11: 3.11-unit @@ -33,8 +32,8 @@ deps= -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/pkgutils.txt - 3.8,3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/test-ci-default.txt - 3.8,3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/docs.txt + 3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/test-ci-default.txt + 3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/docs.txt pypy3: -r{toxinidir}/requirements/test-ci-default.txt integration: -r{toxinidir}/requirements/test-integration.txt @@ -86,7 +85,6 @@ setenv = azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; basepython = - 3.8: python3.8 3.9: python3.9 3.10: python3.10 3.11: python3.11 From 69e846376217933cbff41c16b5fd443d600e1337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Thu, 9 Oct 2025 11:19:51 +0600 Subject: [PATCH 042/169] Remove Python 3.8 from Dockerfile (#9933) Removed Python 3.8 installation and usage from the Dockerfile. --- docker/Dockerfile | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 36817c1d1cc..d78b61394e8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -71,12 +71,11 @@ RUN pyenv install 3.13 && \ pyenv install 3.11 && \ pyenv install 3.10 && \ pyenv install 3.9 && \ - pyenv install 3.8 && \ pyenv install pypy3.10 # Set global Python versions -RUN pyenv global 3.13 3.12 3.11 3.10 3.9 3.8 pypy3.10 +RUN pyenv global 3.13 3.12 3.11 3.10 3.9 pypy3.10 # Install celery WORKDIR $HOME @@ -85,7 +84,7 @@ COPY --chown=1000:1000 docker/entrypoint /entrypoint RUN chmod gu+x /entrypoint # Define the local pyenvs -RUN pyenv local 3.13 3.12 3.11 3.10 3.9 3.8 pypy3.10 +RUN pyenv local 3.13 3.12 3.11 3.10 3.9 pypy3.10 RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \ @@ -93,7 +92,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.11 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.10 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.9 -m pip install --upgrade pip setuptools wheel && \ - pyenv exec python3.8 -m pip install --upgrade pip setuptools wheel && \ pyenv exec pypy3.10 -m pip install --upgrade pip setuptools wheel # Install requirements first to leverage Docker layer caching @@ -153,17 +151,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-pypy3.txt \ -r requirements/test.txt -RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ - pyenv exec python3.8 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt - RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec pypy3.10 -m pip install -r requirements/default.txt \ -r requirements/dev.txt \ @@ -184,7 +171,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.11 -m pip install --no-deps -e $HOME/celery && \ pyenv exec python3.10 -m pip install --no-deps -e $HOME/celery && \ pyenv exec python3.9 -m pip install --no-deps -e $HOME/celery && \ - pyenv exec python3.8 -m pip install --no-deps -e $HOME/celery && \ pyenv exec pypy3.10 -m pip install --no-deps -e $HOME/celery WORKDIR $HOME/celery From 491ef35b71bef5972a8bae7ed498a3dd982ca3a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Thu, 9 Oct 2025 12:24:51 +0600 Subject: [PATCH 043/169] Update Python version requirement to 3.9 (#9935) --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 8b86975dadd..bbd55cbf0d7 100755 --- a/setup.py +++ b/setup.py @@ -147,7 +147,7 @@ def long_description(): license='BSD-3-Clause', platforms=['any'], install_requires=install_requires(), - python_requires=">=3.8", + python_requires=">=3.9", tests_require=reqs('test.txt'), extras_require=extras_require(), include_package_data=True, @@ -170,7 +170,6 @@ def long_description(): "Framework :: Celery", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From bb9da21450441ca4725d1dcb5a385bf4b72a5e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Sun, 12 Oct 2025 13:12:19 +0600 Subject: [PATCH 044/169] Update pypy version from 3.10 to 3.11 in Dockerfile (#9934) * Update pypy version from 3.10 to 3.11 in Dockerfile * Apply suggestion from @auvipy --- docker/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d78b61394e8..ef5e2f1ae7f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -71,11 +71,11 @@ RUN pyenv install 3.13 && \ pyenv install 3.11 && \ pyenv install 3.10 && \ pyenv install 3.9 && \ - pyenv install pypy3.10 + pyenv install pypy3.11 # Set global Python versions -RUN pyenv global 3.13 3.12 3.11 3.10 3.9 pypy3.10 +RUN pyenv global 3.13 3.12 3.11 3.10 3.9 pypy3.11 # Install celery WORKDIR $HOME @@ -84,7 +84,7 @@ COPY --chown=1000:1000 docker/entrypoint /entrypoint RUN chmod gu+x /entrypoint # Define the local pyenvs -RUN pyenv local 3.13 3.12 3.11 3.10 3.9 pypy3.10 +RUN pyenv local 3.13 3.12 3.11 3.10 3.9 pypy3.11 RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \ @@ -92,7 +92,7 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.11 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.10 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.9 -m pip install --upgrade pip setuptools wheel && \ - pyenv exec pypy3.10 -m pip install --upgrade pip setuptools wheel + pyenv exec pypy3.11 -m pip install --upgrade pip setuptools wheel # Install requirements first to leverage Docker layer caching # Split into separate RUN commands to reduce memory pressure and improve layer caching @@ -152,7 +152,7 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ - pyenv exec pypy3.10 -m pip install -r requirements/default.txt \ + pyenv exec pypy3.11 -m pip install -r requirements/default.txt \ -r requirements/dev.txt \ -r requirements/docs.txt \ -r requirements/pkgutils.txt \ @@ -171,7 +171,7 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.11 -m pip install --no-deps -e $HOME/celery && \ pyenv exec python3.10 -m pip install --no-deps -e $HOME/celery && \ pyenv exec python3.9 -m pip install --no-deps -e $HOME/celery && \ - pyenv exec pypy3.10 -m pip install --no-deps -e $HOME/celery + pyenv exec pypy3.11 -m pip install --no-deps -e $HOME/celery WORKDIR $HOME/celery From 37cd233aaf427278244ee0c0cf2ab9607b49577a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Oct 2025 13:18:42 +0600 Subject: [PATCH 045/169] Bump grpcio from 1.67.0 to 1.75.1 (#9519) * Bump grpcio from 1.67.0 to 1.75.1 Bumps [grpcio](https://github.com/grpc/grpc) from 1.67.0 to 1.70.0. - [Release notes](https://github.com/grpc/grpc/releases) - [Changelog](https://github.com/grpc/grpc/blob/master/doc/grpc_release_schedule.md) - [Commits](https://github.com/grpc/grpc/compare/v1.67.0...v1.70.0) --- updated-dependencies: - dependency-name: grpcio dependency-type: direct:production update-type: version-update:semver-minor --------- Co-authored-by: Asif Saif Uddin --- requirements/extras/gcs.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/extras/gcs.txt b/requirements/extras/gcs.txt index 0b06e78ea7c..ef033edcf03 100644 --- a/requirements/extras/gcs.txt +++ b/requirements/extras/gcs.txt @@ -1,4 +1,4 @@ google-cloud-storage>=2.10.0 google-cloud-firestore==2.20.1 -grpcio==1.67.0 ; python_version < "3.9" -grpcio==1.75.0 ; python_version >= "3.9" +grpcio==1.75.1 + From 09cf353ed2275d3106a66f41469eeff07abbae2d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Oct 2025 13:21:14 +0600 Subject: [PATCH 046/169] Bump isort from 5.13.2 to 6.1.0 (#9922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [isort](https://github.com/PyCQA/isort) from 5.13.2 to 6.1.0. - [Release notes](https://github.com/PyCQA/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/PyCQA/isort/compare/5.13.2...6.1.0) --- updated-dependencies: - dependency-name: isort dependency-version: 6.1.0 dependency-type: direct:development update-type: version-update:semver-major ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index fae13c00951..5855800eadf 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,4 +2,4 @@ git+https://github.com/celery/py-amqp.git git+https://github.com/celery/kombu.git git+https://github.com/celery/billiard.git vine>=5.0.0 -isort==5.13.2 +isort==6.1.0 From 66cb940ec670380b83cdfcaf0d774d9d93b7a782 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Oct 2025 13:35:18 +0600 Subject: [PATCH 047/169] Bump cryptography from 44.0.2 to 46.0.2 (#9923) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [cryptography](https://github.com/pyca/cryptography) from 44.0.2 to 46.0.2. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/44.0.2...46.0.2) --- updated-dependencies: - dependency-name: cryptography dependency-version: 46.0.2 dependency-type: direct:production update-type: version-update:semver-major ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/auth.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/auth.txt b/requirements/extras/auth.txt index e9a03334287..7230737424b 100644 --- a/requirements/extras/auth.txt +++ b/requirements/extras/auth.txt @@ -1 +1 @@ -cryptography==44.0.2 +cryptography==46.0.2 From e17a8f13c7a56aee22cfe9466c9474e79398f4cd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 23:03:14 +0600 Subject: [PATCH 048/169] [pre-commit.ci] pre-commit autoupdate (#9554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.19.1 → v3.21.0](https://github.com/asottile/pyupgrade/compare/v3.19.1...v3.21.0) - [github.com/PyCQA/flake8: 7.1.1 → 7.3.0](https://github.com/PyCQA/flake8/compare/7.1.1...7.3.0) - [github.com/codespell-project/codespell: v2.4.0 → v2.4.1](https://github.com/codespell-project/codespell/compare/v2.4.0...v2.4.1) - [github.com/pre-commit/pre-commit-hooks: v5.0.0 → v6.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v5.0.0...v6.0.0) - [github.com/pycqa/isort: 5.13.2 → 7.0.0](https://github.com/pycqa/isort/compare/5.13.2...7.0.0) - [github.com/pre-commit/mirrors-mypy: v1.14.0 → v1.18.2](https://github.com/pre-commit/mirrors-mypy/compare/v1.14.0...v1.18.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c233a488509..f64c3bab0b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v3.19.1 + rev: v3.21.0 hooks: - id: pyupgrade args: ["--py38-plus"] - repo: https://github.com/PyCQA/flake8 - rev: 7.1.1 + rev: 7.3.0 hooks: - id: flake8 @@ -17,7 +17,7 @@ repos: exclude: ^celery/app/task\.py$|^celery/backends/cache\.py$ - repo: https://github.com/codespell-project/codespell - rev: v2.4.0 + rev: v2.4.1 hooks: - id: codespell # See pyproject.toml for args args: [--toml, pyproject.toml, --write-changes] @@ -25,7 +25,7 @@ repos: - tomli - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-merge-conflict - id: check-toml @@ -34,12 +34,12 @@ repos: - id: mixed-line-ending - repo: https://github.com/pycqa/isort - rev: 5.13.2 + rev: 7.0.0 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.14.0 + rev: v1.18.2 hooks: - id: mypy pass_filenames: false From f7094a8957928e0171a4ac91f24c0ba70bea1f81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 22:10:58 +0000 Subject: [PATCH 049/169] Bump pyperclip from 1.9.0 to 1.11.0 Bumps [pyperclip](https://github.com/asweigart/pyperclip) from 1.9.0 to 1.11.0. - [Changelog](https://github.com/asweigart/pyperclip/blob/master/CHANGES.txt) - [Commits](https://github.com/asweigart/pyperclip/commits) --- updated-dependencies: - dependency-name: pyperclip dependency-version: 1.11.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/pkgutils.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/pkgutils.txt b/requirements/pkgutils.txt index eefe5d34af0..cb8932c8439 100644 --- a/requirements/pkgutils.txt +++ b/requirements/pkgutils.txt @@ -8,4 +8,4 @@ sphinx2rst>=1.0 # Disable cyanide until it's fully updated. # cyanide>=1.0.1 bumpversion==0.6.0 -pyperclip==1.9.0 +pyperclip==1.11.0 From 724c816c937b682225d49911921c5f7c73890cfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:29:20 +0600 Subject: [PATCH 050/169] Update elasticsearch requirement from <=8.17.2 to <=9.1.1 (#9945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the requirements on [elasticsearch](https://github.com/elastic/elasticsearch-py) to permit the latest version. - [Release notes](https://github.com/elastic/elasticsearch-py/releases) - [Commits](https://github.com/elastic/elasticsearch-py/compare/0.4.1...v9.1.1) --- updated-dependencies: - dependency-name: elasticsearch dependency-version: 9.1.1 dependency-type: direct:production ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/elasticsearch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index 58cdcae1836..c4bef07d082 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ -elasticsearch<=8.17.2 +elasticsearch<=9.1.1 elastic-transport<=8.17.1 From 13d9d76052546d666fff6266eba9fe6ca395fd74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:31:06 +0600 Subject: [PATCH 051/169] Bump pytest from 8.3.5 to 8.4.2 (#9944) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.5 to 8.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.5...8.4.2) --- updated-dependencies: - dependency-name: pytest dependency-version: 8.4.2 dependency-type: direct:production update-type: version-update:semver-minor ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index a7b758fbaf8..12fc0481489 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,4 +1,4 @@ -pytest==8.3.5 +pytest==8.4.2 pytest-celery[all]>=1.2.0,<1.3.0 pytest-rerunfailures>=14.0,<15.0; python_version >= "3.8" and python_version < "3.9" pytest-rerunfailures>=15.0; python_version >= "3.9" and python_version < "4.0" From 9c8cc433906fa4e02d87d69765b1516426fb6835 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:32:09 +0600 Subject: [PATCH 052/169] Update elastic-transport requirement from <=8.17.1 to <=9.1.0 (#9943) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the requirements on [elastic-transport](https://github.com/elastic/elastic-transport-python) to permit the latest version. - [Release notes](https://github.com/elastic/elastic-transport-python/releases) - [Changelog](https://github.com/elastic/elastic-transport-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/elastic/elastic-transport-python/compare/0.1.0b0...v9.1.0) --- updated-dependencies: - dependency-name: elastic-transport dependency-version: 9.1.0 dependency-type: direct:production ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/elasticsearch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index c4bef07d082..793ad7d0538 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ elasticsearch<=9.1.1 -elastic-transport<=8.17.1 +elastic-transport<=9.1.0 From d9e2f5e2379f7f1cc4c71a9df2ccea81331d14d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Oct 2025 22:10:38 +0000 Subject: [PATCH 053/169] Bump cryptography from 46.0.2 to 46.0.3 Bumps [cryptography](https://github.com/pyca/cryptography) from 46.0.2 to 46.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/46.0.2...46.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-version: 46.0.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/extras/auth.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/auth.txt b/requirements/extras/auth.txt index 7230737424b..7637ae07bf4 100644 --- a/requirements/extras/auth.txt +++ b/requirements/extras/auth.txt @@ -1 +1 @@ -cryptography==46.0.2 +cryptography==46.0.3 From 987e9d91bebae1b2e39121002d55277c737d3bfc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 22:16:20 +0600 Subject: [PATCH 054/169] Bump pytest-timeout from 2.3.1 to 2.4.0 (#9949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) from 2.3.1 to 2.4.0. - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/2.3.1...2.4.0) --- updated-dependencies: - dependency-name: pytest-timeout dependency-version: 2.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index 12fc0481489..a73b8c0a654 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ pytest-rerunfailures>=14.0,<15.0; python_version >= "3.8" and python_version < " pytest-rerunfailures>=15.0; python_version >= "3.9" and python_version < "4.0" pytest-subtests<0.14.0; python_version < "3.9" pytest-subtests>=0.14.1; python_version >= "3.9" -pytest-timeout==2.3.1 +pytest-timeout==2.4.0 pytest-click==1.1.0 pytest-order==1.3.0 boto3>=1.26.143 From 4df81dd14c9a66ca6ff211ea8e41e7f9f34aab5e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 22:17:19 +0600 Subject: [PATCH 055/169] Bump google-cloud-firestore from 2.20.1 to 2.21.0 (#9948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [google-cloud-firestore](https://github.com/googleapis/python-firestore) from 2.20.1 to 2.21.0. - [Release notes](https://github.com/googleapis/python-firestore/releases) - [Changelog](https://github.com/googleapis/python-firestore/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/python-firestore/compare/v2.20.1...v2.21.0) --- updated-dependencies: - dependency-name: google-cloud-firestore dependency-version: 2.21.0 dependency-type: direct:production update-type: version-update:semver-minor ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/gcs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/gcs.txt b/requirements/extras/gcs.txt index ef033edcf03..5a3f1511c12 100644 --- a/requirements/extras/gcs.txt +++ b/requirements/extras/gcs.txt @@ -1,4 +1,4 @@ google-cloud-storage>=2.10.0 -google-cloud-firestore==2.20.1 +google-cloud-firestore==2.21.0 grpcio==1.75.1 From 508cb7c8ce2fd307bc6932a3729e7adcea786201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Sat, 18 Oct 2025 19:42:22 +0000 Subject: [PATCH 056/169] Flake8 fixes (#9955) * remove nonlocal to fix flake8 * remove nonlocal to fix flake8 --- t/smoke/tests/test_thread_safe.py | 1 - t/unit/tasks/test_stamping.py | 1 - 2 files changed, 2 deletions(-) diff --git a/t/smoke/tests/test_thread_safe.py b/t/smoke/tests/test_thread_safe.py index ceab993e24d..47563e6cbec 100644 --- a/t/smoke/tests/test_thread_safe.py +++ b/t/smoke/tests/test_thread_safe.py @@ -56,7 +56,6 @@ def test_multithread_task_publish( @after_task_publish.connect def after_task_publish_handler(*args, **kwargs): - nonlocal signal_was_called signal_was_called(True) def thread_worker(): diff --git a/t/unit/tasks/test_stamping.py b/t/unit/tasks/test_stamping.py index 1c8da859dd7..2161d314fd3 100644 --- a/t/unit/tasks/test_stamping.py +++ b/t/unit/tasks/test_stamping.py @@ -663,7 +663,6 @@ def test_on_signature_gets_the_signature(self): class CustomStampingVisitor(StampingVisitor): def on_signature(self, actual_sig, **headers) -> dict: - nonlocal expected_sig assert actual_sig == expected_sig return {"header": "value"} From d00d7df3604f2ace17ca3bbe73a7dcf86cb9b840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Sun, 19 Oct 2025 04:56:58 +0000 Subject: [PATCH 057/169] Remove test-pypy3.txt from Dockerfile dependencies (#9939) Removed references to 'test-pypy3.txt' from multiple pip install commands in Dockerfile. --- docker/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ef5e2f1ae7f..5c8bcf50902 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ -r requirements/test.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ @@ -115,7 +114,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ -r requirements/test.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ @@ -126,7 +124,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ -r requirements/test.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ @@ -137,7 +134,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ -r requirements/test.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ @@ -148,7 +144,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ -r requirements/test.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ @@ -159,7 +154,6 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ -r requirements/test.txt COPY --chown=1000:1000 . $HOME/celery From 2958c6c55b88a558da9ca556a8243b919e645abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Sun, 19 Oct 2025 07:50:40 +0000 Subject: [PATCH 058/169] Remove backports.zoneinfo for Python 3.9 compatibility (#9956) Removed backports.zoneinfo dependency for Python 3.9+. --- requirements/default.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index 185b6eddd09..79d0ddba43f 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -5,6 +5,5 @@ click>=8.1.2,<9.0 click-didyoumean>=0.3.0 click-repl>=0.2.0 click-plugins>=1.1.1 -backports.zoneinfo[tzdata]>=0.2.1; python_version < '3.9' python-dateutil>=2.8.2 tzlocal From eaba0b50ff2ad297749186c7add40c1188aaf055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Sun, 19 Oct 2025 07:59:57 +0000 Subject: [PATCH 059/169] Update pytest-cov version for Python compatibility (#9957) --- requirements/test-ci-base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/test-ci-base.txt b/requirements/test-ci-base.txt index ec8c7c2a780..a25e4f7b130 100644 --- a/requirements/test-ci-base.txt +++ b/requirements/test-ci-base.txt @@ -1,4 +1,3 @@ -pytest-cov==5.0.0; python_version<"3.9" pytest-cov==7.0.0; python_version>="3.9" pytest-github-actions-annotate-failures==0.3.0 -r extras/redis.txt From c4e4bab4a62c499a368b0921253149c8f02554ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Sun, 19 Oct 2025 08:06:03 +0000 Subject: [PATCH 060/169] Update pytest-rerunfailures and pre-commit versions (#9958) * Update pytest-rerunfailures and pre-commit versions * Update requirements/test.txt * Apply suggestion from @Copilot --- requirements/test.txt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index a73b8c0a654..dd6850c6480 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,17 +1,14 @@ pytest==8.4.2 pytest-celery[all]>=1.2.0,<1.3.0 -pytest-rerunfailures>=14.0,<15.0; python_version >= "3.8" and python_version < "3.9" pytest-rerunfailures>=15.0; python_version >= "3.9" and python_version < "4.0" -pytest-subtests<0.14.0; python_version < "3.9" -pytest-subtests>=0.14.1; python_version >= "3.9" +pytest-subtests>=0.14.1; python_version >= "3.9" and python_version < "4.0" pytest-timeout==2.4.0 pytest-click==1.1.0 pytest-order==1.3.0 boto3>=1.26.143 moto>=4.1.11,<5.1.0 -# typing extensions +# type checking mypy==1.14.1; platform_python_implementation=="CPython" -pre-commit>=3.5.0,<3.8.0; python_version < '3.9' pre-commit>=4.0.1; python_version >= '3.9' -r extras/yaml.txt -r extras/msgpack.txt From 273186043cb93bcb2ce82886b54bfbcd7ddb3459 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Mon, 20 Oct 2025 11:44:01 +0300 Subject: [PATCH 061/169] Prepare for (pre) release: v5.6.0b2 (#9938) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.6.0b1 → 5.6.0b2 * Added Changelog for v5.6.0b2 * Remove Python 3.8 support from the release docs and update Kombu to v5.6.0rc2 (minimum version) --- .bumpversion.cfg | 2 +- Changelog.rst | 39 ++++++++++++++++++++++++++++++++++ README.rst | 2 +- celery/__init__.py | 2 +- docs/history/changelog-5.6.rst | 39 ++++++++++++++++++++++++++++++++++ docs/history/whatsnew-5.6.rst | 8 +++---- docs/includes/introduction.txt | 2 +- requirements/default.txt | 2 +- 8 files changed, 87 insertions(+), 9 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3f1fee8d873..aeacfc9b407 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.6.0b1 +current_version = 5.6.0b2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/Changelog.rst b/Changelog.rst index f1cdcd6d237..828dc917bab 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,45 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0b2: + +5.6.0b2 +======= + +:release-date: 2025-10-20 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- GitHub Actions: Test on Python 3.14 release candidate 2 (#9891) +- Update pypy to python 3.11 (#9896) +- Feature: Add support credential_provider to Redis Backend (#9879) +- Celery.timezone: try tzlocal.get_localzone() before using LocalTimezone (#9862) +- Run integration tests on Python 3.14 (#9903) +- Fix arithmetic overflow for MSSQL result backend (#9904) +- Add documentation for task_id param for apply_async function (#9906) +- Support redis client name (#9900) +- Bump Kombu to v5.6.0rc1 (#9918) +- Fix broker connection retry attempt counter in the error log (#9911) +- fix: restrict disable-prefetch feature to Redis brokers only (#9919) +- fix(): preserve group order in replaced signature (#9910) +- Remove Python 3.8 from CI workflow (#9930) +- Update default Python versions in integration tests (#9931) +- Update tox.ini to remove Python 3.8 (#9932) +- Remove Python 3.8 from Dockerfile (#9933) +- Update Python version requirement to 3.9 (#9935) +- Update pypy version from 3.10 to 3.11 in Dockerfile (#9934) +- Flake8 fixes (#9955) +- Remove test-pypy3.txt from Dockerfile dependencies (#9939) +- Remove backports.zoneinfo for Python 3.9 compatibility (#9956) +- Update pytest-cov version for Python compatibility (#9957) +- Update pytest-rerunfailures and pre-commit versions (#9958) +- Prepare for (pre) release: v5.6.0b2 (#9938) + .. _version-5.6.0b1: 5.6.0b1 diff --git a/README.rst b/README.rst index 7537a56e7dd..c863b8c4c86 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.6.0b1 (recovery) +:Version: 5.6.0b2 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 046a034a0c4..d100d86c306 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'recovery' -__version__ = '5.6.0b1' +__version__ = '5.6.0b2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst index 8bbf0e39a1f..8f5a28786c7 100644 --- a/docs/history/changelog-5.6.rst +++ b/docs/history/changelog-5.6.rst @@ -8,6 +8,45 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0b2: + +5.6.0b2 +======= + +:release-date: 2025-10-20 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- GitHub Actions: Test on Python 3.14 release candidate 2 (#9891) +- Update pypy to python 3.11 (#9896) +- Feature: Add support credential_provider to Redis Backend (#9879) +- Celery.timezone: try tzlocal.get_localzone() before using LocalTimezone (#9862) +- Run integration tests on Python 3.14 (#9903) +- Fix arithmetic overflow for MSSQL result backend (#9904) +- Add documentation for task_id param for apply_async function (#9906) +- Support redis client name (#9900) +- Bump Kombu to v5.6.0rc1 (#9918) +- Fix broker connection retry attempt counter in the error log (#9911) +- fix: restrict disable-prefetch feature to Redis brokers only (#9919) +- fix(): preserve group order in replaced signature (#9910) +- Remove Python 3.8 from CI workflow (#9930) +- Update default Python versions in integration tests (#9931) +- Update tox.ini to remove Python 3.8 (#9932) +- Remove Python 3.8 from Dockerfile (#9933) +- Update Python version requirement to 3.9 (#9935) +- Update pypy version from 3.10 to 3.11 in Dockerfile (#9934) +- Flake8 fixes (#9955) +- Remove test-pypy3.txt from Dockerfile dependencies (#9939) +- Remove backports.zoneinfo for Python 3.9 compatibility (#9956) +- Update pytest-cov version for Python compatibility (#9957) +- Update pytest-rerunfailures and pre-commit versions (#9958) +- Prepare for (pre) release: v5.6.0b2 (#9938) + .. _version-5.6.0b1: 5.6.0b1 diff --git a/docs/history/whatsnew-5.6.rst b/docs/history/whatsnew-5.6.rst index 6407231bd62..5fe1ee31b7e 100644 --- a/docs/history/whatsnew-5.6.rst +++ b/docs/history/whatsnew-5.6.rst @@ -63,7 +63,8 @@ The 5.6.0 release is a new feature release for Celery. Releases in the 5.x series are codenamed after songs of `Jon Hopkins `_. This release has been codenamed `Recovery `_. -This is the last version to support Python 3.8. +This is the last version to support Python 3.9. +Support for Python 3.8 was removed after v5.6.0b1. *— Tomer Nosrati* @@ -145,7 +146,6 @@ Supported Python Versions The supported Python versions are: -- CPython 3.8 - CPython 3.9 - CPython 3.10 - CPython 3.11 @@ -153,10 +153,10 @@ The supported Python versions are: - CPython 3.13 - PyPy3.10 (``pypy3``) -Python 3.8 Support +Python 3.9 Support ------------------ -Python 3.8 will reach EOL in October, 2024. +Python 3.9 will reach EOL in October, 2025. Minimum Dependencies -------------------- diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 651dfa91ce7..202730eb93a 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.6.0b1 (recovery) +:Version: 5.6.0b2 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/requirements/default.txt b/requirements/default.txt index 79d0ddba43f..8ef6816d28c 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,5 +1,5 @@ billiard>=4.2.1,<5.0 -kombu>=5.6.0rc1,<5.7 +kombu>=5.6.0rc2,<5.7 vine>=5.1.0,<6.0 click>=8.1.2,<9.0 click-didyoumean>=0.3.0 From 1ad3bd45369106e22b6aabad047480119140b8ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20P=C5=99ikryl?= <2625825+petrprikryl@users.noreply.github.com> Date: Mon, 20 Oct 2025 15:13:23 +0200 Subject: [PATCH 062/169] Add support for Django Connection pool (#9953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for Django Connection pool https://docs.djangoproject.com/en/dev/ref/databases/#postgresql-pool * tests for close_pool * close_pool called only if DB pool is enabled in Django settings * conn pool docs --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/fixups/django.py | 5 +++ docs/django/first-steps-with-django.rst | 10 +++++ t/unit/fixups/test_django.py | 52 +++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 960077704e4..80a6e3e84d7 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -1,4 +1,5 @@ """Django-specific customization.""" +import contextlib import os import sys import warnings @@ -201,6 +202,10 @@ def _close_database(self) -> None: for conn in self._db.connections.all(): try: conn.close() + pool_enabled = self._settings.DATABASES.get(conn.alias, {}).get("OPTIONS", {}).get("pool") + if pool_enabled and hasattr(conn, "close_pool"): + with contextlib.suppress(KeyError): + conn.close_pool() except self.interface_errors: pass except self.DatabaseError as exc: diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 8ac28d342e3..e0c962880a9 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -216,6 +216,16 @@ However, if your app :ref:`uses a custom task base class `, you'll need inherit from :class:`~celery.contrib.django.task.DjangoTask` instead of :class:`~celery.app.task.Task` to get this behaviour. +Django Connection pool +---------------------- +From Django 5.1+ there is built-in support for database connection pooling. +If you enable it in Django ``DATABASES`` settings Celery will automatically +handle connection pool closing in worker processes via ``close_pool`` +database backend method as +`sharing connections across processes is not possible. `_ + +You can find more about Connection pool at `Django docs. `_ + Extensions ========== diff --git a/t/unit/fixups/test_django.py b/t/unit/fixups/test_django.py index 0d6ab1d83b3..89a045dfc6a 100644 --- a/t/unit/fixups/test_django.py +++ b/t/unit/fixups/test_django.py @@ -284,6 +284,58 @@ def test_close_database_always_closes_connections(self): # it to optimize connection handling. conn.close_if_unusable_or_obsolete.assert_not_called() + def test_close_database_skip_conn_pool(self): + class Connection: + """Mock connection without `close_pool` method.""" + alias = 'default' + + def close(self): + pass + + with self.fixup_context(self.app) as (f, _, _): + conn = Mock(spec=Connection) + f._db.connections.all = Mock(return_value=[conn]) + f.close_database() + assert not hasattr(conn, "close_pool") + conn.close.assert_called_once_with() + + def test_close_database_suppresses_close_pool_keyerror(self): + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + conn.close_pool = Mock(side_effect=KeyError("pool already closed")) + f._db.connections.all = Mock(return_value=[conn]) + f.close_database() # should not raise + conn.close.assert_called_once_with() + conn.close_pool.assert_called_once_with() + + def test_close_database_conn_pool_based_on_settings(self): + class DJSettings: + DATABASES = {} + + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + conn.alias = "default" + conn.close_pool = Mock() + f._db.connections.all = Mock(return_value=[conn]) + f._settings = DJSettings + + f._settings.DATABASES["default"] = {"OPTIONS": {}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_not_called() + + conn.reset_mock() + f._settings.DATABASES["default"] = {"OPTIONS": {"pool": True}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_called_once_with() + + conn.reset_mock() + f._settings.DATABASES["default"] = {"OPTIONS": {"pool": False}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_not_called() + def test_close_cache_raises_error(self): with self.fixup_context(self.app) as (f, _, _): f._cache.close_caches.side_effect = AttributeError From 2f606424c98e8eadc5bce58a90d99a1b25b7168a Mon Sep 17 00:00:00 2001 From: Samiul Sk Date: Sat, 25 Oct 2025 17:25:28 +0530 Subject: [PATCH 063/169] Pin tblib to ==3.1.0 --- requirements/extras/tblib.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/extras/tblib.txt b/requirements/extras/tblib.txt index 5a837d19198..eebba715eb8 100644 --- a/requirements/extras/tblib.txt +++ b/requirements/extras/tblib.txt @@ -1,2 +1 @@ -tblib>=1.5.0;python_version>='3.8.0' -tblib>=1.3.0;python_version<'3.8.0' +tblib==3.1.0 From 7d501cac24cbbb4046a9752ae2a5c53660be8cd7 Mon Sep 17 00:00:00 2001 From: Isabelle COWAN-BERGMAN Date: Sat, 25 Oct 2025 18:35:41 +0200 Subject: [PATCH 064/169] fix(worker): continue to attempt to bind other queues after a native delayed delivery binding failure has occurred (#9959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix native delayed delivery binding failures * Continue to attempt to bind other queues after a native delayed delivery binding failure has occurred. * Add test for native delayed delivery retries * Native delayed deliveries retries should bypass exception grouping and raise retriable exception immediately. * Update celery/worker/consumer/delayed_delivery.py * Replace ExceptionGroup in DelayedDelivery * Use agronholm's exceptiongroup backport * Update t/unit/worker/test_native_delayed_delivery.py Co-authored-by: Isabelle COWAN-BERGMAN * Update celery/worker/consumer/delayed_delivery.py Co-authored-by: Isabelle COWAN-BERGMAN * Add integration test for native delayed delivery bindings --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- .github/workflows/integration-tests.yml | 3 +- celery/worker/consumer/delayed_delivery.py | 30 ++- docker/docker-compose.yml | 2 +- requirements/default.txt | 1 + .../test_native_delayed_delivery_binding.py | 190 ++++++++++++++++++ t/unit/worker/test_native_delayed_delivery.py | 160 +++++++++++++++ 6 files changed, 381 insertions(+), 5 deletions(-) create mode 100644 t/integration/test_native_delayed_delivery_binding.py diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 0c4a0fb85d3..86a205d5c37 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -37,9 +37,10 @@ jobs: REDIS_HOST: localhost REDIS_PORT: 6379 rabbitmq: - image: rabbitmq + image: rabbitmq:management ports: - 5672:5672 + - 15672:15672 env: RABBITMQ_DEFAULT_USER: guest RABBITMQ_DEFAULT_PASS: guest diff --git a/celery/worker/consumer/delayed_delivery.py b/celery/worker/consumer/delayed_delivery.py index b9d37a12511..43b0a600365 100644 --- a/celery/worker/consumer/delayed_delivery.py +++ b/celery/worker/consumer/delayed_delivery.py @@ -5,6 +5,9 @@ """ from typing import Iterator, List, Optional, Set, Union, ValuesView +# Backport of PEP 654 for Python versions < 3.11 +# In Python 3.11+, exceptiongroup uses the built-in ExceptionGroup +from exceptiongroup import ExceptionGroup from kombu import Connection, Queue from kombu.transport.native_delayed_delivery import (bind_queue_to_native_delayed_delivery_exchange, declare_native_delayed_delivery_exchanges_and_queues) @@ -23,7 +26,7 @@ # Default retry settings RETRY_INTERVAL = 1.0 # seconds between retries MAX_RETRIES = 3 # maximum number of retries - +RETRIED_EXCEPTIONS = (ConnectionRefusedError, OSError) # Valid queue types for delayed delivery VALID_QUEUE_TYPES = {'classic', 'quorum'} @@ -84,7 +87,7 @@ def start(self, c: Consumer) -> None: retry_over_time( self._setup_delayed_delivery, args=(c, broker_url), - catch=(ConnectionRefusedError, OSError), + catch=RETRIED_EXCEPTIONS, errback=self._on_retry, interval_start=RETRY_INTERVAL, max_retries=MAX_RETRIES, @@ -157,6 +160,7 @@ def _bind_queues(self, app: Celery, connection: Connection) -> None: logger.warning("No queues found to bind for delayed delivery") return + exceptions: list[Exception] = [] for queue in queues: try: logger.debug("Binding queue %r to delayed delivery exchange", queue.name) @@ -166,7 +170,27 @@ def _bind_queues(self, app: Celery, connection: Connection) -> None: "Failed to bind queue %r: %s", queue.name, str(e) ) - raise + + # We must re-raise on retried exceptions to ensure they are + # caught with the outer retry_over_time mechanism. + # + # This could be removed if one of: + # * The minimum python version for Celery and Kombu is + # increased to 3.11. Kombu updated to use the `except*` + # clause to catch specific exceptions from an ExceptionGroup. + # * Kombu's retry_over_time utility is updated to use the + # catch utility from agronholm's exceptiongroup backport. + if isinstance(e, RETRIED_EXCEPTIONS): + raise + + exceptions.append(e) + + if exceptions: + raise ExceptionGroup( + ("One or more failures occurred while binding queues to " + "delayed delivery exchanges"), + exceptions, + ) def _on_retry(self, exc: Exception, interval_range: Iterator[float], intervals_count: int) -> float: """Callback for retry attempts. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c31138f1942..05bac847c0c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -25,7 +25,7 @@ services: - azurite rabbit: - image: rabbitmq:latest + image: rabbitmq:management redis: image: redis:latest diff --git a/requirements/default.txt b/requirements/default.txt index 8ef6816d28c..b64d254e01d 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -6,4 +6,5 @@ click-didyoumean>=0.3.0 click-repl>=0.2.0 click-plugins>=1.1.1 python-dateutil>=2.8.2 +exceptiongroup>=1.3.0 tzlocal diff --git a/t/integration/test_native_delayed_delivery_binding.py b/t/integration/test_native_delayed_delivery_binding.py new file mode 100644 index 00000000000..f738810203c --- /dev/null +++ b/t/integration/test_native_delayed_delivery_binding.py @@ -0,0 +1,190 @@ +"""Integration tests for native delayed delivery queue binding. + +Tests that verify queue bindings are created correctly for native delayed +delivery, especially when some queues in task_queues fail to bind. +""" +import os +import uuid +from urllib.parse import quote + +import pytest +import requests +from kombu import Exchange, Queue +from requests.auth import HTTPBasicAuth + +from celery import Celery +from celery.contrib.testing.worker import start_worker + + +def get_rabbitmq_credentials(): + """Get RabbitMQ credentials from environment.""" + user = os.environ.get("RABBITMQ_DEFAULT_USER", "guest") + password = os.environ.get("RABBITMQ_DEFAULT_PASSWORD", "guest") + return user, password + + +def get_rabbitmq_url(): + """Get RabbitMQ broker URL from environment.""" + user, password = get_rabbitmq_credentials() + return os.environ.get( + "TEST_BROKER", f"pyamqp://{user}:{password}@localhost:5672//") + + +def get_management_api_url(): + """Get RabbitMQ Management API base URL.""" + return "http://localhost:15672/api" + + +def get_bindings_for_exchange(exchange_name, vhost='/'): + """Fetch bindings where the given exchange is the source. + + Args: + exchange_name: Name of the exchange + vhost: Virtual host (default '/') + + Returns: + List of binding dictionaries + """ + user, password = get_rabbitmq_credentials() + vhost_encoded = quote(vhost, safe='') + exchange_encoded = quote(exchange_name, safe='') + api_url = ( + f"{get_management_api_url()}/exchanges/{vhost_encoded}/" + f"{exchange_encoded}/bindings/source" + ) + response = requests.get(api_url, auth=HTTPBasicAuth(user, password)) + response.raise_for_status() + return response.json() + + +def get_bindings_for_queue(queue_name, vhost='/'): + """Fetch bindings for a specific queue. + + Args: + queue_name: Name of the queue + vhost: Virtual host (default '/') + + Returns: + List of binding dictionaries + """ + user, password = get_rabbitmq_credentials() + vhost_encoded = quote(vhost, safe='') + queue_encoded = quote(queue_name, safe='') + api_url = ( + f"{get_management_api_url()}/queues/{vhost_encoded}/{queue_encoded}/" + "bindings" + ) + response = requests.get(api_url, auth=HTTPBasicAuth(user, password)) + response.raise_for_status() + return response.json() + + +def create_test_app(unique_id): + """Create Celery app configured for native delayed delivery testing. + + Args: + unique_id: Unique identifier to ensure queue/exchange names don't + conflict + + Returns: + Tuple of (app, exchange_name, queue_a_name, queue_b_name) + """ + broker_url = get_rabbitmq_url() + + # Get Redis backend URL from environment + redis_host = os.environ.get("REDIS_HOST", "localhost") + redis_port = os.environ.get("REDIS_PORT", "6379") + backend_url = os.environ.get( + "TEST_BACKEND", f"redis://{redis_host}:{redis_port}/0") + + app = Celery( + "test_native_delayed_delivery_binding", + broker=broker_url, + backend=backend_url, + ) + + # Configure topic exchange with unique name + exchange_name = f'celery.topic_{unique_id}' + default_exchange = Exchange(exchange_name, type='topic') + + # Define task queues with queue-a first, queue-b second + queue_a_name = f'queue-a_{unique_id}' + queue_b_name = f'queue-b_{unique_id}' + app.conf.task_queues = [ + Queue(queue_a_name, exchange=default_exchange, + routing_key=queue_a_name, + queue_arguments={'x-queue-type': 'quorum'}), + Queue(queue_b_name, exchange=default_exchange, + routing_key=queue_b_name, + queue_arguments={'x-queue-type': 'quorum'}), + ] + + # Recommended setting for using celery with quorum queues + app.conf.broker_transport_options = {"confirm_publish": True} + + # Enable quorum queue detection to disable global QoS + app.conf.worker_detect_quorum_queues = True + + return app, exchange_name, queue_a_name, queue_b_name + + +@pytest.mark.amqp +@pytest.mark.timeout(90) +def test_worker_binds_consumed_queue_despite_earlier_queue_failure(): + """Test that queue binding continues even when earlier queues fail to bind. + + This test reproduces the scenario from + https://github.com/celery/celery/issues/9960 + """ + unique_id = uuid.uuid4().hex + app, exchange_name, queue_a_name, queue_b_name = create_test_app(unique_id) + + # Set default queue to queue-b so the start_worker ping task is received + # by our worker + app.conf.task_default_queue = queue_b_name + + # Start worker that only consumes from queue-b + # queue-a is NOT consumed, so it won't be declared by this worker + with start_worker( + app, + queues=[queue_b_name], + loglevel="INFO", + perform_ping_check=True, + shutdown_timeout=15, + ): + # Check celery_delayed_delivery → exchange bindings + delayed_delivery_bindings = \ + get_bindings_for_exchange('celery_delayed_delivery') + queue_b_delayed_binding = [ + b for b in delayed_delivery_bindings + if b.get('destination') == exchange_name + and b.get('routing_key') == f'#.{queue_b_name}' + ] + assert len(queue_b_delayed_binding) >= 1, ( + f"Expected delayed delivery binding for {queue_b_name!r}, but " + f"got bindings: {delayed_delivery_bindings!r}" + ) + + # Check celery.topic → queue-b bindings + # Should have bindings from the topic exchange to queue-b for both + # immediate and delayed delivery + queue_b_bindings = get_bindings_for_queue(queue_b_name) + topic_to_queue_bindings = [ + b for b in queue_b_bindings + if b.get('source') == exchange_name + ] + topic_to_queue_routing_keys = { + b.get('routing_key') for b in topic_to_queue_bindings + } + + # Check the routing key for immediate delivery + assert queue_b_name in topic_to_queue_routing_keys, ( + f"Expected routing key {queue_b_name!r} in bindings, but got: " + f"{topic_to_queue_bindings!r}" + ) + + # Check the routing key for delayed delivery + assert f"#.{queue_b_name}" in topic_to_queue_routing_keys, ( + f"Expected at least one binding from {exchange_name!r} to " + f"{queue_b_name!r}, but got: {topic_to_queue_bindings!r}" + ) diff --git a/t/unit/worker/test_native_delayed_delivery.py b/t/unit/worker/test_native_delayed_delivery.py index 654d7c15ab7..c5f2adcfc11 100644 --- a/t/unit/worker/test_native_delayed_delivery.py +++ b/t/unit/worker/test_native_delayed_delivery.py @@ -4,6 +4,8 @@ from unittest.mock import MagicMock, Mock, patch import pytest +from amqp import NotFound +from exceptiongroup import ExceptionGroup from kombu import Exchange, Queue from kombu.utils.functional import retry_over_time @@ -306,3 +308,161 @@ def test_setup_bind_error(self, mock_bind, caplog): assert len([r for r in caplog.records if r.levelname == "CRITICAL"]) == 1 assert any("Failed to bind queue" in r.message for r in caplog.records) assert any("Failed to setup delayed delivery for all broker URLs" in r.message for r in caplog.records) + + @patch('celery.worker.consumer.delayed_delivery.bind_queue_to_native_delayed_delivery_exchange') + def test_bind_queues_continues_after_failure(self, mock_bind, caplog): + """ + Test that binding continues for remaining queues after one fails. + """ + consumer_mock = MagicMock() + consumer_mock.app.conf.broker_native_delayed_delivery_queue_type = \ + 'classic' + consumer_mock.app.conf.broker_url = 'amqp://' + + # Create three queues + queue1 = Queue('queue1', exchange=Exchange('exchange1', type='topic')) + queue2 = Queue('queue2', exchange=Exchange('exchange2', type='topic')) + queue3 = Queue('queue3', exchange=Exchange('exchange3', type='topic')) + + consumer_mock.app.amqp.queues = { + 'queue1': queue1, + 'queue2': queue2, + 'queue3': queue3, + } + + # Make the second queue fail to bind + def bind_side_effect(connection, queue): + if queue.name == 'queue2': + raise NotFound( + reply_text="NOT_FOUND - no queue 'queue2' in vhost '/'", + method_name="Queue.bind", + reply_code=404, + ) + + mock_bind.side_effect = bind_side_effect + + delayed_delivery = DelayedDelivery(consumer_mock) + delayed_delivery.start(consumer_mock) + + # Verify that bind was called for all three queues + assert mock_bind.call_count == 3 + + # Verify error was logged for queue2 + error_logs = [r for r in caplog.records if r.levelname == "ERROR"] + expected_msg = \ + "Queue.bind: (404) NOT_FOUND - no queue 'queue2' in vhost '/'" + assert any(expected_msg in r.message for r in error_logs) + + @patch('celery.worker.consumer.delayed_delivery.bind_queue_to_native_delayed_delivery_exchange') + def test_bind_queues_raises_exceptions_on_failures(self, mock_bind): + """Test that Exceptions are raised with all binding failures.""" + consumer_mock = MagicMock() + consumer_mock.app.conf.broker_native_delayed_delivery_queue_type = \ + 'classic' + consumer_mock.app.conf.broker_url = 'amqp://' + + # Create three queues + queue1 = Queue('queue1', exchange=Exchange('exchange1', type='topic')) + queue2 = Queue('queue2', exchange=Exchange('exchange2', type='topic')) + queue3 = Queue('queue3', exchange=Exchange('exchange3', type='topic')) + + consumer_mock.app.amqp.queues = { + 'queue1': queue1, + 'queue2': queue2, + 'queue3': queue3, + } + + # Make queue1 and queue3 fail with different errors + def bind_side_effect(connection, queue): + if queue.name == 'queue1': + raise ValueError("Queue1 binding failed") + elif queue.name == 'queue3': + raise RuntimeError("Queue3 binding failed") + + mock_bind.side_effect = bind_side_effect + + delayed_delivery = DelayedDelivery(consumer_mock) + + # Should raise RuntimeError containing both exceptions messages + with pytest.raises(ExceptionGroup) as exc_info: + delayed_delivery._setup_delayed_delivery(consumer_mock, 'amqp://') + + # Verify the RuntimeError message contains both exceptions + exception_group = exc_info.value + assert str(exception_group) == ( + "One or more failures occurred while binding queues to delayed " + "delivery exchanges (2 sub-exceptions)" + ) + + # Verify the ExceptionGroup contains both exceptions + exceptions = exception_group.exceptions + assert len(exceptions) == 2 + assert any( + isinstance(ex, ValueError) and str(ex) == "Queue1 binding failed" + for ex in exceptions + ) + assert any( + isinstance(ex, RuntimeError) and str(ex) == "Queue3 binding failed" + for ex in exceptions + ) + + # Verify bind was called for all three queues + assert mock_bind.call_count == 3 + + @patch('celery.worker.consumer.delayed_delivery.bind_queue_to_native_delayed_delivery_exchange') + def test_bind_retries_on_retried_exception(self, mock_bind, caplog): + """ + Test that retried exceptions from + bind_queue_to_native_delayed_delivery_exchange trigger the retry + mechanism. + """ + consumer_mock = MagicMock() + consumer_mock.app.conf.broker_native_delayed_delivery_queue_type = \ + 'classic' + consumer_mock.app.conf.broker_url = 'amqp://' + + # Create a queue + queue1 = Queue('queue1', exchange=Exchange('exchange1', type='topic')) + consumer_mock.app.amqp.queues = {'queue1': queue1} + + # Track bind attempts + bind_attempts = [0] + + # Make bind raise a ConnectionRefusedError twice, then succeed + # This simulates a transient connection issue that resolves on retry + def bind_side_effect(connection, queue): + bind_attempts[0] += 1 + if bind_attempts[0] <= 2: + # ConnectionRefusedError is one of the RETRIED_EXCEPTIONS + raise ConnectionRefusedError("Connection refused") + # Succeed on third attempt + + mock_bind.side_effect = bind_side_effect + + delayed_delivery = DelayedDelivery(consumer_mock) + delayed_delivery.start(consumer_mock) + + # Verify bind was attempted multiple times (indicating retries + # occurred) + assert bind_attempts[0] == 3, \ + "Expected 3 bind attempts (2 failures + 1 success), got " + \ + f"{bind_attempts[0]}" + + # Verify retry warnings were logged + warning_logs = [r for r in caplog.records if r.levelname == "WARNING"] + retry_warnings = [ + r for r in warning_logs + if "Retrying delayed delivery setup" in r.message + ] + + # Should have 2 retry warnings (one for each failed attempt) + assert len(retry_warnings) == 2, \ + f"Expected 2 retry warnings, got {len(retry_warnings)}. " + \ + f"All warnings: {[r.message for r in warning_logs]}" + + # Verify the retry messages contain the expected information and + # correct attempt numbers + assert "Connection refused" in retry_warnings[0].message + assert "attempt 1/" in retry_warnings[0].message + assert "Connection refused" in retry_warnings[1].message + assert "attempt 2/" in retry_warnings[1].message From 26bee54d621d98b9322abd9aa2bb14f4b1a59972 Mon Sep 17 00:00:00 2001 From: pavlos kallis Date: Sat, 25 Oct 2025 19:37:43 +0300 Subject: [PATCH 065/169] Handle UnpicklingError in persistent scheduler initialization (#9952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Handle UnpicklingError in persistent scheduler initialization * Update t/unit/app/test_beat.py * Fix incorrect comment referring to dbm.error instead of UnpicklingError * Fix broken tests * Fix lint offense * Dummy commit to trigger pipeline --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/beat.py | 5 +++-- t/unit/app/test_beat.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 86ad837f0d5..93203bf0f89 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -12,6 +12,7 @@ from calendar import timegm from collections import namedtuple from functools import total_ordering +from pickle import UnpicklingError from threading import Event, Thread from billiard import ensure_multiprocessing @@ -569,11 +570,11 @@ def _create_schedule(self): for _ in (1, 2): try: self._store['entries'] - except (KeyError, UnicodeDecodeError, TypeError): + except (KeyError, UnicodeDecodeError, TypeError, UnpicklingError): # new schedule db try: self._store['entries'] = {} - except (KeyError, UnicodeDecodeError, TypeError) + dbm.error as exc: + except (KeyError, UnicodeDecodeError, TypeError, UnpicklingError) + dbm.error as exc: self._store = self._destroy_open_corrupted_schedule(exc) continue else: diff --git a/t/unit/app/test_beat.py b/t/unit/app/test_beat.py index b81a11426e1..55feb9d7c9c 100644 --- a/t/unit/app/test_beat.py +++ b/t/unit/app/test_beat.py @@ -1,5 +1,6 @@ import dbm import errno +import pickle import sys from datetime import datetime, timedelta, timezone from pickle import dumps, loads @@ -708,6 +709,26 @@ def test_create_schedule_corrupted_dbm_error(self): s._create_schedule() s._destroy_open_corrupted_schedule.assert_called_with(expected_error) + def test_create_schedule_corrupted_pickle_error(self): + """ + Test that any UnpicklingError that might happen when opening beat-schedule.db is caught + """ + s = create_persistent_scheduler()[0](app=self.app, + schedule_filename='schedule') + s._store = MagicMock() + s._destroy_open_corrupted_schedule = Mock() + s._destroy_open_corrupted_schedule.return_value = MagicMock() + + # self._store['entries'] = {} will throw a pickle.UnpicklingError + s._store.__getitem__.side_effect = pickle.UnpicklingError("test") + # then, when _create_schedule tries to reset _store['entries'], + # throw another error, specifically pickle.UnpicklingError + expected_error = pickle.UnpicklingError("test") + s._store.__setitem__.side_effect = expected_error + + s._create_schedule() + s._destroy_open_corrupted_schedule.assert_called_with(expected_error) + def test_create_schedule_missing_entries(self): """ Test that if _create_schedule can't find the key "entries" in _store it will recreate it From 2aebae52ebc9b5156faa040fe957fb8ef5421556 Mon Sep 17 00:00:00 2001 From: Kumuthu Edirisinghe <91903331+kumuthu53@users.noreply.github.com> Date: Sun, 26 Oct 2025 17:32:26 +1100 Subject: [PATCH 066/169] Bug Fix: Nested Chords Fail When Using django-celery-results with a Redis Backend (#9950) * ~pass in app when calling GroupResult.restore in RedisBackend > on_chord_part_return * ~updated unit test to align with bug fix to RedisBackend --- celery/backends/redis.py | 2 +- t/unit/backends/test_redis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 6e0713c0568..28e9b723102 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -540,7 +540,7 @@ def on_chord_part_return(self, request, state, result, callback = maybe_signature(request.chord, app=app) total = int(chord_size_bytes) + totaldiff if readycount == total: - header_result = GroupResult.restore(gid) + header_result = GroupResult.restore(gid, app=app) if header_result is not None: # If we manage to restore a `GroupResult`, then it must # have been complex and saved by `apply_chord()` earlier. diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 3ffa60f4477..e2233b01a08 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -1285,7 +1285,7 @@ def test_on_chord_part_return( self.b.client.zrange.assert_not_called() self.b.client.lrange.assert_not_called() # Confirm that the `GroupResult.restore` mock was called - complex_header_result.assert_called_once_with(request.group) + complex_header_result.assert_called_once_with(request.group, app=self.b.app) # Confirm that the callback was called with the `join()`ed group result if supports_native_join: expected_join = mock_result_obj.join_native From 0a455095d170fb49a33dc93916c9e2fa27209a2f Mon Sep 17 00:00:00 2001 From: kidoz Date: Tue, 28 Oct 2025 15:06:46 +0300 Subject: [PATCH 067/169] Add support pymongo 4.12 (#9665) Co-authored-by: Asif Saif Uddin --- celery/backends/mongodb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 1789f6cf0b0..3bdea5ed974 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -20,6 +20,7 @@ from bson.binary import Binary except ImportError: from pymongo.binary import Binary + from pymongo import uri_parser from pymongo.errors import InvalidDocument else: # pragma: no cover Binary = None @@ -73,7 +74,7 @@ def __init__(self, app=None, **kwargs): if self.url: self.url = self._ensure_mongodb_uri_compliance(self.url) - uri_data = pymongo.uri_parser.parse_uri(self.url) + uri_data = uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection hostslist = [ f'{x[0]}:{x[1]}' for x in uri_data['nodelist'] From 47b7b50abf62bd510a82916b3a41c3ddf58f1833 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Tue, 28 Oct 2025 15:33:49 +0000 Subject: [PATCH 068/169] Make tests compatible with pymongo >= 4.14 (#9968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/mongodb/mongo-python-driver/pull/2413 caused some test regressions here. This isn't currently a problem for the upstream test suite since it pins pymongo==4.10.1 via kombu, but we're running into it in Debian where we've already upgraded pymongo for other reasons. kombu already tried to upgrade pymongo but had to revert due to these test regressions (see https://github.com/celery/kombu/pull/2384 and https://github.com/celery/celery/pull/9938). One of the test fixes (relating to `mongodb_backend_settings`) illustrates an incompatibility where I couldn't figure out a reasonable way to avoid passing it through to Celery users, so I added a note to the documentation about it. It may also be worth including a brief mention of it in the release notes. Using the canonical case for the option in question should work with both old and new versions of pymongo. Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/userguide/configuration.rst | 6 ++++ t/unit/backends/test_mongodb.py | 52 ++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 975172a9cfd..ea0e66d9705 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1208,6 +1208,12 @@ This is a dict supporting the following keys: constructor. See the :mod:`pymongo` docs to see a list of arguments supported. +.. note:: + + With pymongo>=4.14, options are case-sensitive when they were previously + case-insensitive. See :class:`~pymongo.mongo_client.MongoClient` to + determine the correct case. + .. _example-mongodb-result-config: Example configuration diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index 9ae340ee149..0c29111654b 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -130,17 +130,21 @@ def test_init_with_settings(self): 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == MONGODB_BACKEND_HOST - assert mb.options == dict( - mb._prepare_client_options(), - replicaset='rs0', - ) + if 'replicaSet' in mb.options: # pragma: no cover # pymongo >= 4.14 + replicaset_option = 'replicaSet' + else: # pragma: no cover # pymongo < 4.14 + replicaset_option = 'replicaset' + assert mb.options == { + **mb._prepare_client_options(), + replicaset_option: 'rs0', + } assert mb.user == CELERY_USER assert mb.password == CELERY_PASSWORD assert mb.database_name == CELERY_DATABASE # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { - 'replicaset': 'rs1', + replicaset_option: 'rs1', 'user': 'backenduser', 'database': 'another_db', 'options': { @@ -149,11 +153,11 @@ def test_init_with_settings(self): } mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == MONGODB_BACKEND_HOST - assert mb.options == dict( - mb._prepare_client_options(), - replicaset='rs1', - socketKeepAlive=True, - ) + assert mb.options == { + **mb._prepare_client_options(), + replicaset_option: 'rs1', + 'socketKeepAlive': True, + } assert mb.user == 'backenduser' assert mb.password == CELERY_PASSWORD assert mb.database_name == 'another_db' @@ -222,11 +226,15 @@ def test_init_mongodb_dnspython2_pymongo4_seedlist(self): with patch('dns.resolver.resolve', side_effect=resolver): mb = self.perform_seedlist_assertions() - assert mb.options == dict( - mb._prepare_client_options(), - replicaset='rs0', - tls=True - ) + if 'replicaSet' in mb.options: # pragma: no cover # pymongo >= 4.14 + replicaset_option = 'replicaSet' + else: # pragma: no cover # pymongo < 4.14 + replicaset_option = 'replicaset' + assert mb.options == { + **mb._prepare_client_options(), + replicaset_option: 'rs0', + 'tls': True, + } def perform_seedlist_assertions(self): mb = MongoBackend(app=self.app, url=MONGODB_SEEDLIST_URI) @@ -299,12 +307,15 @@ def test_get_connection_with_authmechanism(self): mb = MongoBackend(app=self.app, url=uri) mock_Connection.return_value = sentinel.connection connection = mb._get_connection() + if 'authMechanism' in mb.options: # pragma: no cover # pymongo >= 4.14 + authmechanism_option = 'authMechanism' + else: # pragma: no cover # pymongo < 4.14 + authmechanism_option = 'authmechanism' mock_Connection.assert_called_once_with( host=['localhost:27017'], username=CELERY_USER, password=CELERY_PASSWORD, - authmechanism='SCRAM-SHA-256', - **mb._prepare_client_options() + **{**mb._prepare_client_options(), authmechanism_option: 'SCRAM-SHA-256'} ) assert sentinel.connection == connection @@ -319,10 +330,13 @@ def test_get_connection_with_authmechanism_no_username(self): 'SCRAM-SHA-256 requires a username.') with pytest.raises(ConfigurationError): mb._get_connection() + if 'authMechanism' in mb.options: # pragma: no cover # pymongo >= 4.14 + authmechanism_option = 'authMechanism' + else: # pragma: no cover # pymongo < 4.14 + authmechanism_option = 'authmechanism' mock_Connection.assert_called_once_with( host=['localhost:27017'], - authmechanism='SCRAM-SHA-256', - **mb._prepare_client_options() + **{**mb._prepare_client_options(), authmechanism_option: 'SCRAM-SHA-256'} ) @patch('celery.backends.mongodb.MongoBackend._get_connection') From 0c30c182d6dc95a561bb1aae4a39bc3de7a148db Mon Sep 17 00:00:00 2001 From: Mehraz Hossain Rumman <59512321+MehrazRumman@users.noreply.github.com> Date: Tue, 28 Oct 2025 21:36:43 +0600 Subject: [PATCH 069/169] tblib updated from 3.1.0 to 3.2.0 (#9970) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tblib updated from 3.1.0 to 3.2.0 * waiting for redis & RabbitMQ * tests fixed * max_retires set to 1 --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/tblib.txt | 2 +- t/integration/test_tasks.py | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/requirements/extras/tblib.txt b/requirements/extras/tblib.txt index eebba715eb8..d34d459173d 100644 --- a/requirements/extras/tblib.txt +++ b/requirements/extras/tblib.txt @@ -1 +1 @@ -tblib==3.1.0 +tblib==3.2.0 diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index 0dbb7708c53..74e15b5913c 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -438,10 +438,16 @@ def test_retry_with_unpickleable_exception(self, manager): res = job.result assert job.status == 'RETRY' # make sure that it wasn't completed yet - # Check it - assert isinstance(res, UnpickleableExceptionWrapper) - assert res.exc_cls_name == "UnpickleableException" - assert res.exc_args == ("foo",) + # Check it. Accept both the dedicated wrapper and plain Exception + # (some environments may return a stringified Exception instead). + if isinstance(res, UnpickleableExceptionWrapper): + assert res.exc_cls_name == "UnpickleableException" + assert res.exc_args == ("foo",) + else: + # Fallback: ensure the exception string mentions the class and argument + res_str = str(res) + assert "UnpickleableException" in res_str + assert "foo" in res_str job.revoke() @@ -452,12 +458,20 @@ def test_fail_with_unpickleable_exception(self, manager): """ result = fail_unpickleable.delay("foo", "bar") - with pytest.raises(UnpickleableExceptionWrapper) as exc_info: + # Accept either the dedicated wrapper exception or a plain Exception + # whose string contains the class name and args (some backends + # may return a stringified exception). + try: result.get() - - exc_wrapper = exc_info.value - assert exc_wrapper.exc_cls_name == "UnpickleableException" - assert exc_wrapper.exc_args == ("foo",) + pytest.fail("Expected an exception when getting result") + except UnpickleableExceptionWrapper as exc_wrapper: + assert exc_wrapper.exc_cls_name == "UnpickleableException" + assert exc_wrapper.exc_args == ("foo",) + except Exception as exc: + # Fallback: ensure the exception string mentions the class and argument + exc_str = str(exc) + assert "UnpickleableException" in exc_str + assert "foo" in exc_str assert result.status == 'FAILURE' From f3857e3591e5fc39fb5c773575c8485d82657163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Wcis=C5=82o?= <115464873+wcislo-saleor@users.noreply.github.com> Date: Tue, 28 Oct 2025 17:23:26 +0100 Subject: [PATCH 070/169] Fix remaining function typing and docstring (#9971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/utils/time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/utils/time.py b/celery/utils/time.py index f7a373bf2ca..bd9dba1a2e6 100644 --- a/celery/utils/time.py +++ b/celery/utils/time.py @@ -212,7 +212,7 @@ def delta_resolution(dt: datetime, delta: timedelta) -> datetime: def remaining( - start: datetime, ends_in: timedelta, now: Callable | None = None, + start: datetime, ends_in: timedelta, now: datetime | None = None, relative: bool = False) -> timedelta: """Calculate the real remaining time for a start date and a timedelta. @@ -224,7 +224,7 @@ def remaining( relative (bool): If enabled the end time will be calculated using :func:`delta_resolution` (i.e., rounded to the resolution of `ends_in`). - now (Callable): Function returning the current time and date. + now (~datetime.datetime): Current time and date. Defaults to :func:`datetime.now(timezone.utc)`. Returns: From 1f27d71c5f54da828a34d12fe30c22cedc893272 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Oct 2025 22:07:46 +0000 Subject: [PATCH 071/169] Update elasticsearch requirement from <=9.1.1 to <=9.1.2 Updates the requirements on [elasticsearch](https://github.com/elastic/elasticsearch-py) to permit the latest version. - [Release notes](https://github.com/elastic/elasticsearch-py/releases) - [Commits](https://github.com/elastic/elasticsearch-py/compare/0.4.1...v9.1.2) --- updated-dependencies: - dependency-name: elasticsearch dependency-version: 9.1.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements/extras/elasticsearch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index 793ad7d0538..605852ada53 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ -elasticsearch<=9.1.1 +elasticsearch<=9.1.2 elastic-transport<=9.1.0 From f47d5be0b99fc69fb3c910a3ac127c0f52e1ca0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Oct 2025 22:09:12 +0000 Subject: [PATCH 072/169] Bump tblib from 3.2.0 to 3.2.1 Bumps [tblib](https://github.com/ionelmc/python-tblib) from 3.2.0 to 3.2.1. - [Release notes](https://github.com/ionelmc/python-tblib/releases) - [Changelog](https://github.com/ionelmc/python-tblib/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-tblib/compare/v3.2.0...v3.2.1) --- updated-dependencies: - dependency-name: tblib dependency-version: 3.2.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/extras/tblib.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/tblib.txt b/requirements/extras/tblib.txt index d34d459173d..1e1bed11711 100644 --- a/requirements/extras/tblib.txt +++ b/requirements/extras/tblib.txt @@ -1 +1 @@ -tblib==3.2.0 +tblib==3.2.1 From 7aaab22da15d6b07299c9cb79cfd69dba2af09cc Mon Sep 17 00:00:00 2001 From: sagar <129282569+sagar1343@users.noreply.github.com> Date: Sat, 1 Nov 2025 11:27:25 +0530 Subject: [PATCH 073/169] Fix regex pattern in version parsing and remove duplicate entry in __all__ (#9978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix: Escape second dot in version parsing regex pattern The pattern r'(\d+)\.(\d+).(\d+)' incorrectly used unescaped dot which matches any character. Changed to r'(\d+)\.(\d+)\.(\d+)' to correctly match literal dots in version strings. - Fix: Remove duplicate 'gen_task_name' entry in celery/utils/__init__.py __all__ Removed redundant duplicate entry in the __all__ tuple. Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/__init__.py | 2 +- celery/utils/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index d100d86c306..ea4d10060a0 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -42,7 +42,7 @@ # bumpversion can only search for {current_version} # so we have to parse the version here. _temp = re.match( - r'(\d+)\.(\d+).(\d+)(.+)?', __version__).groups() + r'(\d+)\.(\d+)\.(\d+)(.+)?', __version__).groups() VERSION = version_info = version_info_t( int(_temp[0]), int(_temp[1]), int(_temp[2]), _temp[3] or '', '') del _temp diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index e905c247837..0e2e1a33070 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -22,7 +22,6 @@ 'cached_property', 'chunks', 'gen_task_name', - 'gen_task_name', 'gen_unique_id', 'get_cls_by_name', 'get_full_cls_name', From ebc6435aa15872b943bb4aafb4e440a321a15e14 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sat, 1 Nov 2025 18:18:35 +0200 Subject: [PATCH 074/169] Bump Kombu to v5.6.0 and removed <5.7 limit on kombu (#9981) --- requirements/default.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/default.txt b/requirements/default.txt index b64d254e01d..25db1a331f0 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,5 +1,5 @@ billiard>=4.2.1,<5.0 -kombu>=5.6.0rc2,<5.7 +kombu>=5.6.0 vine>=5.1.0,<6.0 click>=8.1.2,<9.0 click-didyoumean>=0.3.0 From 5fd0b3e004ecae004848e0cbe236344e2b2892cc Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 2 Nov 2025 01:36:17 +0200 Subject: [PATCH 075/169] Prepare for (pre) release: v5.6.0rc1 (#9982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.6.0b2 → 5.6.0rc1 * Added Changelog for v5.6.0rc1 --- .bumpversion.cfg | 2 +- Changelog.rst | 27 +++++++++++++++++++++++++++ README.rst | 2 +- celery/__init__.py | 2 +- docs/history/changelog-5.6.rst | 27 +++++++++++++++++++++++++++ docs/includes/introduction.txt | 2 +- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index aeacfc9b407..6a9ca0ddb28 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.6.0b2 +current_version = 5.6.0rc1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/Changelog.rst b/Changelog.rst index 828dc917bab..f73a4307e1e 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,33 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0rc1: + +5.6.0rc1 +======== + +:release-date: 2025-11-02 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Add support for Django Connection pool (#9953) +- Pin tblib to ==3.1.0 (#9967) +- fix(worker): continue to attempt to bind other queues after a native delayed delivery binding failure has occurred (#9959) +- Handle UnpicklingError in persistent scheduler initialization (#9952) +- Bug Fix: Nested Chords Fail When Using django-celery-results with a Redis Backend (#9950) +- Add support pymongo 4.12 (#9665) +- Make tests compatible with pymongo >= 4.14 (#9968) +- tblib updated from 3.1.0 to 3.2.0 (#9970) +- Fix remaining function typing and docstring (#9971) +- Fix regex pattern in version parsing and remove duplicate entry in __all__ (#9978) +- Bump Kombu to v5.6.0 and removed <5.7 limit on kombu (#9981) +- Prepare for (pre) release: v5.6.0rc1 (#9982) + .. _version-5.6.0b2: 5.6.0b2 diff --git a/README.rst b/README.rst index c863b8c4c86..a5f9c1a6355 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.6.0b2 (recovery) +:Version: 5.6.0rc1 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index ea4d10060a0..a56379df6e0 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'recovery' -__version__ = '5.6.0b2' +__version__ = '5.6.0rc1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst index 8f5a28786c7..d16377a276d 100644 --- a/docs/history/changelog-5.6.rst +++ b/docs/history/changelog-5.6.rst @@ -8,6 +8,33 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0rc1: + +5.6.0rc1 +======== + +:release-date: 2025-11-02 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Add support for Django Connection pool (#9953) +- Pin tblib to ==3.1.0 (#9967) +- fix(worker): continue to attempt to bind other queues after a native delayed delivery binding failure has occurred (#9959) +- Handle UnpicklingError in persistent scheduler initialization (#9952) +- Bug Fix: Nested Chords Fail When Using django-celery-results with a Redis Backend (#9950) +- Add support pymongo 4.12 (#9665) +- Make tests compatible with pymongo >= 4.14 (#9968) +- tblib updated from 3.1.0 to 3.2.0 (#9970) +- Fix remaining function typing and docstring (#9971) +- Fix regex pattern in version parsing and remove duplicate entry in __all__ (#9978) +- Bump Kombu to v5.6.0 and removed <5.7 limit on kombu (#9981) +- Prepare for (pre) release: v5.6.0rc1 (#9982) + .. _version-5.6.0b2: 5.6.0b2 diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 202730eb93a..1fc945b4f75 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.6.0b2 (recovery) +:Version: 5.6.0rc1 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 6d7c14ed26cebebbbab4849677311e2a93a18f4c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 21:33:36 +0200 Subject: [PATCH 076/169] [pre-commit.ci] pre-commit autoupdate (#9992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.21.0 → v3.21.1](https://github.com/asottile/pyupgrade/compare/v3.21.0...v3.21.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f64c3bab0b4..65184222194 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v3.21.0 + rev: v3.21.1 hooks: - id: pyupgrade args: ["--py38-plus"] From afb67303ae95c9af499412940046a6c0a3b46a51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Nov 2025 00:48:55 +0200 Subject: [PATCH 077/169] Bump tblib from 3.2.1 to 3.2.2 (#9995) Bumps [tblib](https://github.com/ionelmc/python-tblib) from 3.2.1 to 3.2.2. - [Release notes](https://github.com/ionelmc/python-tblib/releases) - [Changelog](https://github.com/ionelmc/python-tblib/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-tblib/compare/v3.2.1...v3.2.2) --- updated-dependencies: - dependency-name: tblib dependency-version: 3.2.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/extras/tblib.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/tblib.txt b/requirements/extras/tblib.txt index 1e1bed11711..81d957704c6 100644 --- a/requirements/extras/tblib.txt +++ b/requirements/extras/tblib.txt @@ -1 +1 @@ -tblib==3.2.1 +tblib==3.2.2 From 929412e2d4d328e337be3f177ca99fff10ab9bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=9Clgen=20Sar=C4=B1kavak?= Date: Fri, 14 Nov 2025 03:23:26 +0300 Subject: [PATCH 078/169] Remove Python 4.0 version condition for pytest dependencies (#9993) These were added in * https://github.com/celery/celery/commit/f1ddd58647ee24bee4f74c9c4e45812728cfd514 * https://github.com/celery/celery/commit/c4e4bab4a62c499a368b0921253149c8f02554ad but they are not helpful. Decided to clean in https://github.com/celery/celery/pull/9987#discussion_r2510046671 --- requirements/test.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index dd6850c6480..c4493e03440 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,7 +1,7 @@ pytest==8.4.2 pytest-celery[all]>=1.2.0,<1.3.0 -pytest-rerunfailures>=15.0; python_version >= "3.9" and python_version < "4.0" -pytest-subtests>=0.14.1; python_version >= "3.9" and python_version < "4.0" +pytest-rerunfailures>=15.0; python_version >= "3.9" +pytest-subtests>=0.14.1; python_version >= "3.9" pytest-timeout==2.4.0 pytest-click==1.1.0 pytest-order==1.3.0 From 30649dbd41308fc8eef79b2aae179a908eaa7a51 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 14 Nov 2025 12:29:10 +0100 Subject: [PATCH 079/169] Fix log leaking broker credentials (#9997) Sanitize broker URLs in Celery's delayed delivery debug logs by wrapping `connection.as_uri()` with `maybe_sanitize_url()` from Kombu. This prevents user:password credentials from being printed in logs, even at DEBUG level. This improves security and aligns Celery with common logging best practices (no secrets in logs). No functional behavior is changed. --- celery/worker/consumer/delayed_delivery.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/celery/worker/consumer/delayed_delivery.py b/celery/worker/consumer/delayed_delivery.py index 43b0a600365..46996ca2caf 100644 --- a/celery/worker/consumer/delayed_delivery.py +++ b/celery/worker/consumer/delayed_delivery.py @@ -12,6 +12,7 @@ from kombu.transport.native_delayed_delivery import (bind_queue_to_native_delayed_delivery_exchange, declare_native_delayed_delivery_exchanges_and_queues) from kombu.utils.functional import retry_over_time +from kombu.utils.url import maybe_sanitize_url from celery import Celery, bootsteps from celery.utils.log import get_logger @@ -95,7 +96,7 @@ def start(self, c: Consumer) -> None: except Exception as e: logger.warning( "Failed to setup delayed delivery for %r: %s", - broker_url, str(e) + maybe_sanitize_url(broker_url), str(e) ) setup_errors.append((broker_url, e)) @@ -121,7 +122,7 @@ def _setup_delayed_delivery(self, c: Consumer, broker_url: str) -> None: queue_type = c.app.conf.broker_native_delayed_delivery_queue_type logger.debug( "Setting up delayed delivery for broker %r with queue type %r", - broker_url, queue_type + maybe_sanitize_url(broker_url), queue_type ) try: @@ -132,7 +133,7 @@ def _setup_delayed_delivery(self, c: Consumer, broker_url: str) -> None: except Exception as e: logger.warning( "Failed to declare exchanges and queues for %r: %s", - broker_url, str(e) + maybe_sanitize_url(broker_url), str(e) ) raise @@ -141,7 +142,7 @@ def _setup_delayed_delivery(self, c: Consumer, broker_url: str) -> None: except Exception as e: logger.warning( "Failed to bind queues for %r: %s", - broker_url, str(e) + maybe_sanitize_url(broker_url), str(e) ) raise From 63c1910221d9fb8c1931ff62c80c4c37232b2d58 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Mon, 17 Nov 2025 23:40:22 +0200 Subject: [PATCH 080/169] Don't fail task on timeout during cold shutdown (#9678) * Don't fail task on timeout during cold shutdown * Update celery/worker/request.py * Update celery/worker/request.py * Update t/unit/worker/test_request.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update t/unit/worker/test_request.py * Fixed bug where tasks that finished during the soft shutdown were set to RETRY * Update t/unit/worker/test_request.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update t/unit/worker/test_request.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Fixed lint error * Bugfix: Successful tasks that were canceled during soft shutdown will avoid setting the task to RETRY * Renamed cancel_all_unacked_requests -> cancel_active_requests * Added unit tests --------- Co-authored-by: Asif Saif Uddin Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- celery/apps/worker.py | 14 ++++-- celery/worker/consumer/consumer.py | 19 +++++++-- celery/worker/request.py | 49 +++++++++++++-------- t/smoke/tests/test_worker.py | 62 +++++++++++++++++++++++++++ t/unit/worker/test_consumer.py | 32 +++++++++++--- t/unit/worker/test_request.py | 68 ++++++++++++++++++++++++++++++ 6 files changed, 214 insertions(+), 30 deletions(-) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 5558dab8e5f..7286d4b8543 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -350,7 +350,7 @@ def during_soft_shutdown(worker: Worker): install_worker_term_hard_handler(worker, sig='SIGQUIT', callback=on_hard_shutdown) # Cancel all unacked requests and allow the worker to terminate naturally - worker.consumer.cancel_all_unacked_requests() + worker.consumer.cancel_active_requests() # We get here if the worker was in the middle of the soft (cold) shutdown process, # and the matching signal was received. This can typically happen when the worker is @@ -409,11 +409,19 @@ def on_cold_shutdown(worker: Worker): # Initiate soft shutdown process (if enabled and tasks are running) worker.wait_for_soft_shutdown() + # Stop consuming new tasks to prevents requeued messages from being immediately redelivered + if worker.consumer.task_consumer: + worker.consumer.task_consumer.cancel() + # Cancel all unacked requests and allow the worker to terminate naturally - worker.consumer.cancel_all_unacked_requests() + worker.consumer.cancel_active_requests() + + from celery.worker import state + state.should_terminate = True # Stop the pool to allow successful tasks call on_success() - worker.consumer.pool.stop() + if worker.consumer.pool: + worker.consumer.pool.stop() # Allow SIGTERM to be remapped to SIGQUIT to initiate cold shutdown instead of warm shutdown using SIGTERM diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 9f843afccf1..95e73707621 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -31,7 +31,8 @@ from celery.utils.text import truncate from celery.utils.time import humanize_seconds, rate from celery.worker import loops -from celery.worker.state import active_requests, maybe_shutdown, requests, reserved_requests, task_reserved +from celery.worker.state import (active_requests, maybe_shutdown, requests, reserved_requests, successful_requests, + task_reserved) __all__ = ('Consumer', 'Evloop', 'dump_body') @@ -740,9 +741,13 @@ def __repr__(self): self=self, state=self.blueprint.human_state(), ) - def cancel_all_unacked_requests(self): - """Cancel all active requests that either do not require late acknowledgments or, + def cancel_active_requests(self): + """Cancel active requests during shutdown. + + Cancels all active requests that either do not require late acknowledgments or, if they do, have not been acknowledged yet. + + Does not cancel successful tasks, even if they have not been acknowledged yet. """ def should_cancel(request): @@ -752,6 +757,9 @@ def should_cancel(request): if not request.acknowledged: # Task is late acknowledged, but it has not been acknowledged yet, cancel it. + if request.id in successful_requests: + # Unless it was successful, in which case we don't want to cancel it. + return False return True # Task is late acknowledged, but it has already been acknowledged. @@ -761,7 +769,10 @@ def should_cancel(request): if requests_to_cancel: for request in requests_to_cancel: - request.cancel(self.pool) + # For acks_late tasks, don't emit RETRY signal since broker will handle redelivery + # For non-acks_late tasks, emit RETRY signal as usual + emit_retry = not request.task.acks_late + request.cancel(self.pool, emit_retry=emit_retry) class Evloop(bootsteps.StartStopStep): diff --git a/celery/worker/request.py b/celery/worker/request.py index df99b549270..2b975266f68 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -422,29 +422,34 @@ def terminate(self, pool, signal=None): if obj is not None: obj.terminate(signal) - def cancel(self, pool, signal=None): + def cancel(self, pool, signal=None, emit_retry=True): signal = _signals.signum(signal or TERM_SIGNAME) if self.time_start: pool.terminate_job(self.worker_pid, signal) - self._announce_cancelled() + self._announce_cancelled(emit_retry=emit_retry) if self._apply_result is not None: obj = self._apply_result() # is a weakref if obj is not None: obj.terminate(signal) - def _announce_cancelled(self): + def _announce_cancelled(self, emit_retry=True): task_ready(self) self.send_event('task-cancelled') - reason = 'cancelled by Celery' - exc = Retry(message=reason) - self.task.backend.mark_as_retry(self.id, - exc, - request=self._context) - self.task.on_retry(exc, self.id, self.args, self.kwargs, None) + if emit_retry: + reason = 'cancelled by Celery' + exc = Retry(message=reason) + self.task.backend.mark_as_retry(self.id, + exc, + request=self._context) + + self.task.on_retry(exc, self.id, self.args, self.kwargs, None) + self._already_cancelled = True - send_retry(self.task, request=self._context, einfo=None) + + if emit_retry: + send_retry(self.task, request=self._context, einfo=None) def _announce_revoked(self, reason, terminated, signum, expired): task_ready(self) @@ -525,14 +530,16 @@ def on_timeout(self, soft, timeout): timeout, self.name, self.id) else: task_ready(self) - error('Hard time limit (%ss) exceeded for %s[%s]', - timeout, self.name, self.id) - exc = TimeLimitExceeded(timeout) - - self.task.backend.mark_as_failure( - self.id, exc, request=self._context, - store_result=self.store_errors, - ) + # This is a special case where the task timeout handling is done during + # the cold shutdown process. + if not state.should_terminate: + error('Hard time limit (%ss) exceeded for %s[%s]', timeout, self.name, self.id) + exc = TimeLimitExceeded(timeout) + + self.task.backend.mark_as_failure( + self.id, exc, request=self._context, + store_result=self.store_errors, + ) if self.task.acks_late and self.task.acks_on_failure_or_timeout: self.acknowledge() @@ -617,6 +624,12 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): # need to be removed from prefetched local queue self.reject(requeue=False) + # This is a special case where the task failure handling is done during + # the cold shutdown process. + if state.should_terminate: + return_ok = True + send_failed_event = False + # This is a special case where the process would not have had time # to write the result. if not requeue and (is_worker_lost or not return_ok): diff --git a/t/smoke/tests/test_worker.py b/t/smoke/tests/test_worker.py index 2165f4296af..617576d1860 100644 --- a/t/smoke/tests/test_worker.py +++ b/t/smoke/tests/test_worker.py @@ -215,6 +215,68 @@ def test_hard_shutdown_from_soft(self, celery_setup: CeleryTestSetup): assert_container_exited(worker) + def test_task_completes_during_soft_shutdown(self, celery_setup: CeleryTestSetup): + app = celery_setup.app + queue = celery_setup.worker.worker_queue + worker = celery_setup.worker + + task_duration = app.conf.worker_soft_shutdown_timeout // 2 + sig = long_running_task.si(task_duration, verbose=True).set(queue=queue) + res = sig.delay() + + worker.assert_log_exists("Starting long running task") + self.kill_worker(worker, WorkerKill.Method.SIGQUIT) + worker.assert_log_exists( + f"Initiating Soft Shutdown, terminating in {app.conf.worker_soft_shutdown_timeout} seconds" + ) + worker.assert_log_exists("worker: Cold shutdown (MainProcess)") + + assert_container_exited(worker) + assert res.get(RESULT_TIMEOUT) + assert res.state == 'SUCCESS' + + class test_time_limit(SuiteOperations): + @pytest.fixture + def default_worker_app(self, default_worker_app: Celery) -> Celery: + app = default_worker_app + app.conf.worker_soft_shutdown_timeout = 16 + app.conf.task_time_limit = 15 + app.conf.task_soft_time_limit = 10 + return app + + def test_task_completes_during_soft_shutdown_with_time_limit(self, celery_setup: CeleryTestSetup): + app = celery_setup.app + queue = celery_setup.worker.worker_queue + worker = celery_setup.worker + + task_duration = 8 + sig = long_running_task.si(task_duration, verbose=True).set(queue=queue) + res = sig.delay() + + worker.assert_log_exists("Starting long running task") + self.kill_worker(worker, WorkerKill.Method.SIGQUIT) + + worker.assert_log_exists( + f"Initiating Soft Shutdown, terminating in {app.conf.worker_soft_shutdown_timeout} seconds" + ) + + worker.assert_log_exists("Finished long running task") + worker.assert_log_exists(f"long_running_task[{res.id}] succeeded") + + worker.assert_log_does_not_exist( + f"Task handler raised error: TimeLimitExceeded({app.conf.task_time_limit})", + timeout=RESULT_TIMEOUT, + ) + worker.assert_log_does_not_exist( + f"Hard time limit ({app.conf.task_time_limit}s) exceeded for {long_running_task.name}[{res.id}]", + timeout=RESULT_TIMEOUT, + ) + worker.assert_log_exists("worker: Cold shutdown (MainProcess)") + + assert_container_exited(worker) + assert res.get(RESULT_TIMEOUT) + assert res.state == 'SUCCESS' + class test_REMAP_SIGTERM(SuiteOperations): @pytest.fixture def default_worker_env(self, default_worker_env: dict) -> dict: diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index 02e77e9e58e..734efbaef57 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -19,7 +19,7 @@ from celery.worker.consumer.heart import Heart from celery.worker.consumer.mingle import Mingle from celery.worker.consumer.tasks import Tasks -from celery.worker.state import active_requests +from celery.worker.state import active_requests, successful_requests class ConsumerTestCase: @@ -453,7 +453,7 @@ def test_cancel_long_running_tasks_on_connection_loss__warning(self): c.on_connection_error_after_connected(Mock()) @pytest.mark.usefixtures('depends_on_current_app') - def test_cancel_all_unacked_requests(self): + def test_cancel_active_requests(self): c = self.get_consumer() mock_request_acks_late_not_acknowledged = Mock(id='1') @@ -469,14 +469,36 @@ def test_cancel_all_unacked_requests(self): active_requests.add(mock_request_acks_late_acknowledged) active_requests.add(mock_request_acks_early) - c.cancel_all_unacked_requests() + c.cancel_active_requests() - mock_request_acks_late_not_acknowledged.cancel.assert_called_once_with(c.pool) + # acks_late unacknowledged tasks should be cancelled without RETRY + mock_request_acks_late_not_acknowledged.cancel.assert_called_once_with(c.pool, emit_retry=False) + # acks_late acknowledged tasks should NOT be cancelled mock_request_acks_late_acknowledged.cancel.assert_not_called() - mock_request_acks_early.cancel.assert_called_once_with(c.pool) + # Non-acks_late tasks should be cancelled normally (with RETRY) + mock_request_acks_early.cancel.assert_called_once_with(c.pool, emit_retry=True) active_requests.clear() + @pytest.mark.usefixtures('depends_on_current_app') + def test_cancel_active_requests_preserves_successful_tasks(self): + c = self.get_consumer() + + mock_successful_request = Mock(id='successful-task') + mock_successful_request.task.acks_late = True + mock_successful_request.acknowledged = False + + active_requests.add(mock_successful_request) + + successful_requests.add('successful-task') + + try: + c.cancel_active_requests() + mock_successful_request.cancel.assert_not_called() + finally: + active_requests.clear() + successful_requests.clear() + @pytest.mark.parametrize("broker_connection_retry", [True, False]) @pytest.mark.parametrize("broker_connection_retry_on_startup", [None, False]) @pytest.mark.parametrize("first_connection_attempt", [True, False]) diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 172ca5162ac..fb4354942c3 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -575,6 +575,39 @@ def test_cancel__task_reserved(self): pool.terminate_job.assert_not_called() assert job._terminate_on_ack is None + def test_cancel__emit_retry_true(self): + pool = Mock() + signum = signal.SIGTERM + job = self.get_request(self.mytask.s(1, f='x')) + job._apply_result = Mock(name='_apply_result') + job.task.backend.mark_as_retry = Mock(name='mark_as_retry') + job.task.on_retry = Mock(name='on_retry') + with self.assert_signal_called( + task_retry, sender=job.task, request=job._context, + einfo=None): + job.time_start = monotonic() + job.worker_pid = 314 + job.cancel(pool, signal='TERM', emit_retry=True) + pool.terminate_job.assert_called_with(job.worker_pid, signum) + job.task.backend.mark_as_retry.assert_called_once() + job.task.on_retry.assert_called_once() + assert job._already_cancelled is True + + def test_cancel__emit_retry_false(self): + pool = Mock() + signum = signal.SIGTERM + job = self.get_request(self.mytask.s(1, f='x')) + job._apply_result = Mock(name='_apply_result') + job.task.backend.mark_as_retry = Mock(name='mark_as_retry') + job.task.on_retry = Mock(name='on_retry') + job.time_start = monotonic() + job.worker_pid = 314 + job.cancel(pool, signal='TERM', emit_retry=False) + pool.terminate_job.assert_called_with(job.worker_pid, signum) + job.task.backend.mark_as_retry.assert_not_called() + job.task.on_retry.assert_not_called() + assert job._already_cancelled is True + def test_revoked_expires_expired(self): job = self.get_request(self.mytask.s(1, f='x').set( expires=datetime.now(timezone.utc) - timedelta(days=1) @@ -870,6 +903,26 @@ def test_on_failure_task_cancelled(self): job.on_failure(exc_info) assert not job.eventer.send.called + def test_on_failure_should_terminate(self): + from celery.worker import state + original_should_terminate = state.should_terminate + state.should_terminate = True + job = self.xRequest() + job.send_event = Mock(name='send_event') + job.task.backend = Mock(name='backend') + + try: + try: + raise KeyError('foo') + except KeyError: + exc_info = ExceptionInfo() + job.on_failure(exc_info) + + job.send_event.assert_not_called() + job.task.backend.mark_as_failure.assert_not_called() + finally: + state.should_terminate = original_should_terminate + def test_from_message_invalid_kwargs(self): m = self.TaskMessage(self.mytask.name, args=(), kwargs='foo') req = Request(m, app=self.app) @@ -937,6 +990,21 @@ def test_on_soft_timeout(self, patching): job.on_timeout(soft=True, timeout=1336) assert self.mytask.backend.get_status(job.id) == states.PENDING + def test_on_timeout_should_terminate(self, patching): + from celery.worker import state + warn = patching('celery.worker.request.warn') + error = patching('celery.worker.request.error') + + original_should_terminate = state.should_terminate + try: + state.should_terminate = True + job = self.xRequest() + job.on_timeout(None, None) + warn.assert_not_called() + error.assert_not_called() + finally: + state.should_terminate = original_should_terminate + def test_fast_trace_task(self): assert self.app.use_fast_trace_task is False setup_worker_optimizations(self.app) From f32b92f0e481601e9cc9f1212a4feced3f48e1a0 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Tue, 18 Nov 2025 13:59:47 +0200 Subject: [PATCH 081/169] Add Py39-314t to CI (#9999) * Add Py39-314t to CI * Allow unlimited parallel runs --- .github/workflows/integration-tests.yml | 2 +- .github/workflows/python-package.yml | 6 +++--- .github/workflows/smoke-tests.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 86a205d5c37..daa30b0e233 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,7 +11,7 @@ on: description: 'JSON array of Python versions to test' required: false type: string - default: '["3.9", "3.14"]' + default: '["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"]' tox_environments: description: 'JSON array of tox environments to test' required: false diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f070cfe9c21..70351d32bb6 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,9 +42,11 @@ jobs: os: "windows-latest" - python-version: '3.11' os: "windows-latest" + - python-version: '3.12' + os: "windows-latest" - python-version: '3.13' os: "windows-latest" - - python-version: '3.14' + - python-version: '3.14t' os: "windows-latest" - python-version: 'pypy3.11' os: "windows-latest" @@ -89,7 +91,6 @@ jobs: needs: [Unit] if: needs.Unit.result == 'success' strategy: - max-parallel: 5 matrix: module: [ 'test_backend.py', @@ -113,7 +114,6 @@ jobs: needs: [Unit] if: needs.Unit.result == 'success' strategy: - max-parallel: 5 matrix: module: [ 'test_broker_failover.py', diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index 27b4cff30ec..d1ee2639ac1 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -11,7 +11,7 @@ on: description: 'JSON array of Python versions to test' required: false type: string - default: '["3.13"]' + default: '["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"]' jobs: testing-with: From 95d05527c0e1fff5c1a38877ba87592983a7f993 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:59:02 +0200 Subject: [PATCH 082/169] Bump actions/checkout from 5 to 6 (#10003) Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/docker.yml | 10 +++++----- .github/workflows/integration-tests.yml | 2 +- .github/workflows/linter.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/smoke-tests.yml | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b3d956d48c9..3f014e19107 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ea8e5af3203..43fde7bb735 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,7 +29,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Build Docker container @@ -39,7 +39,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: dev" @@ -49,7 +49,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: latest" @@ -59,7 +59,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: pypi" @@ -69,7 +69,7 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2204 timeout-minutes: 10 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: "Build smoke tests container: legacy" diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index daa30b0e233..c3cf3258e0f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -50,7 +50,7 @@ jobs: run: | sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: useblacksmith/setup-python@v6 with: diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 6f22274e9b7..498b950d377 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -8,7 +8,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 70351d32bb6..6235f5cdd82 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -56,7 +56,7 @@ jobs: if: startsWith(matrix.os, 'blacksmith-4vcpu-ubuntu') run: | sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: useblacksmith/setup-python@v6 with: diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index c33b7514c85..42fd5fcb02e 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -21,5 +21,5 @@ jobs: container: image: returntocorp/semgrep steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - run: semgrep ci diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index d1ee2639ac1..b23cc833e54 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -33,7 +33,7 @@ jobs: sudo apt-get install -y procps # Install procps to enable sysctl sudo sysctl -w vm.overcommit_memory=1 - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Setup Docker Builder uses: useblacksmith/setup-docker-builder@v1 - name: Set up Python ${{ matrix.python-version }} From 3f0f0fe7ea6a67a696d0a750644b552559ae368e Mon Sep 17 00:00:00 2001 From: Tor Arvid Lund Date: Sat, 22 Nov 2025 05:39:09 +0100 Subject: [PATCH 083/169] asynpool: Don't return from inside a finally block (#10000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * asynpool: Don't return from inside a finally block * asynpool: Make the logic (mostly) backwards compatible This commit makes the logic of the AsynPool class backwards compatible, except for one change related to exception handling. Before, if an exception **other than `(OSError, EOFError)`** was raised, it would be swallowed by the return inside of the finally block. Now, such an exception would propagate out of this method. * Update celery/concurrency/asynpool.py --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/concurrency/asynpool.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index dd2f068a215..a55542e6573 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -391,6 +391,7 @@ def _flush_outqueue(self, fd, remove, process_index, on_state_change): setblocking(reader, 1) except OSError: return remove(fd) + result = None try: if reader.poll(0): task = reader.recv() @@ -398,7 +399,7 @@ def _flush_outqueue(self, fd, remove, process_index, on_state_change): task = None sleep(0.5) except (OSError, EOFError): - return remove(fd) + result = remove(fd) else: if task: on_state_change(task) @@ -406,7 +407,8 @@ def _flush_outqueue(self, fd, remove, process_index, on_state_change): try: setblocking(reader, 0) except OSError: - return remove(fd) + result = remove(fd) + return result class AsynPool(_pool.Pool): From b446910f18860531b089d6c39f974265cc24a589 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sat, 22 Nov 2025 19:51:52 +0200 Subject: [PATCH 084/169] Prepare for (pre) release: v5.6.0rc2 (#10005) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.6.0rc1 → 5.6.0rc2 * Added Changelog for v5.6.0rc2 --- .bumpversion.cfg | 2 +- Changelog.rst | 21 +++++++++++++++++++++ README.rst | 2 +- celery/__init__.py | 2 +- docs/history/changelog-5.6.rst | 21 +++++++++++++++++++++ docs/includes/introduction.txt | 2 +- 6 files changed, 46 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6a9ca0ddb28..a5911a06758 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.6.0rc1 +current_version = 5.6.0rc2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/Changelog.rst b/Changelog.rst index f73a4307e1e..168142ae8b7 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,27 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0rc2: + +5.6.0rc2 +======== + +:release-date: 2025-11-22 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Remove Python 4.0 version condition for pytest dependencies (#9993) +- Sanitize broker URL in delayed delivery logs (avoid leaking credentials) (#9997) +- Don't fail task on timeout during cold shutdown (#9678) +- Add Py39-314t to CI (#9999) +- asynpool: Don't return from inside a finally block (#10000) +- Prepare for (pre) release: v5.6.0rc2 (#10005) + .. _version-5.6.0rc1: 5.6.0rc1 diff --git a/README.rst b/README.rst index a5f9c1a6355..bcf87f9ce7e 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.6.0rc1 (recovery) +:Version: 5.6.0rc2 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index a56379df6e0..dc7a6e87edd 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'recovery' -__version__ = '5.6.0rc1' +__version__ = '5.6.0rc2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst index d16377a276d..5dce7554035 100644 --- a/docs/history/changelog-5.6.rst +++ b/docs/history/changelog-5.6.rst @@ -8,6 +8,27 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0rc2: + +5.6.0rc2 +======== + +:release-date: 2025-11-22 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Remove Python 4.0 version condition for pytest dependencies (#9993) +- Sanitize broker URL in delayed delivery logs (avoid leaking credentials) (#9997) +- Don't fail task on timeout during cold shutdown (#9678) +- Add Py39-314t to CI (#9999) +- asynpool: Don't return from inside a finally block (#10000) +- Prepare for (pre) release: v5.6.0rc2 (#10005) + .. _version-5.6.0rc1: 5.6.0rc1 diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 1fc945b4f75..c6a940e4576 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.6.0rc1 (recovery) +:Version: 5.6.0rc2 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 0932d2c06e001903b90638f7c40a14f5407c9801 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 18:54:35 +0200 Subject: [PATCH 085/169] [pre-commit.ci] pre-commit autoupdate (#10007) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/asottile/pyupgrade: v3.21.1 → v3.21.2](https://github.com/asottile/pyupgrade/compare/v3.21.1...v3.21.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65184222194..80da5253073 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/asottile/pyupgrade - rev: v3.21.1 + rev: v3.21.2 hooks: - id: pyupgrade args: ["--py38-plus"] From 1133f22181bb22223d39c0060973695c6af55643 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 30 Nov 2025 09:55:43 +0200 Subject: [PATCH 086/169] Bump mypy from 1.14.1 to 1.19.0 (#10008) Bumps [mypy](https://github.com/python/mypy) from 1.14.1 to 1.19.0. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.14.1...v1.19.0) --- updated-dependencies: - dependency-name: mypy dependency-version: 1.19.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index c4493e03440..f46d754a693 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,7 +8,7 @@ pytest-order==1.3.0 boto3>=1.26.143 moto>=4.1.11,<5.1.0 # type checking -mypy==1.14.1; platform_python_implementation=="CPython" +mypy==1.19.0; platform_python_implementation=="CPython" pre-commit>=4.0.1; python_version >= '3.9' -r extras/yaml.txt -r extras/msgpack.txt From cca11164860a1bee6ad8626c27a683b482f741eb Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 30 Nov 2025 19:20:43 +0200 Subject: [PATCH 087/169] Prepare for release: v5.6.0 (#10010) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.6.0rc2 → 5.6.0 * Added Changelog for v5.6.0 * Fix Sphinx 9.0+ compatibility (released during release checklist tests) --- .bumpversion.cfg | 2 +- Changelog.rst | 114 +++++++++++++++++++++++++++++++++ README.rst | 2 +- celery/__init__.py | 2 +- celery/contrib/sphinx.py | 33 ++++++++++ docs/history/changelog-5.6.rst | 114 +++++++++++++++++++++++++++++++++ docs/history/index.rst | 2 + docs/history/whatsnew-5.6.rst | 100 +++++++++++++++++++++++++++-- docs/includes/introduction.txt | 2 +- 9 files changed, 362 insertions(+), 9 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a5911a06758..63b58a78481 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.6.0rc2 +current_version = 5.6.0 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/Changelog.rst b/Changelog.rst index 168142ae8b7..f06d81f1674 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,120 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0: + +5.6.0 +===== + +:release-date: 2025-11-30 +:release-by: Tomer Nosrati + +Celery v5.6.0 is now available. + +Key Highlights +~~~~~~~~~~~~~~ + +See :ref:`whatsnew-5.6` for a complete overview or read the main highlights below. + +Python 3.9 Minimum Version +-------------------------- + +Celery 5.6.0 drops support for Python 3.8 (EOL). The minimum required Python +version is now 3.9. Users still on Python 3.8 must upgrade their Python version +before upgrading to Celery 5.6.0. + +Additionally, this release includes initial support for Python 3.14. + +SQS: Reverted to ``pycurl`` from ``urllib3`` +-------------------------------------------- + +The switch from ``pycurl`` to ``urllib3`` for the SQS transport (introduced in +Celery 5.5.0 via Kombu) has been reverted due to critical issues affecting SQS +users: + +- Processing throughput dropped from ~100 tasks/sec to ~3/sec in some environments +- ``UnknownOperationException`` errors causing container crash loops +- Silent message processing failures with no error logs + +Users of the SQS transport must ensure ``pycurl`` is installed. If you removed +``pycurl`` after upgrading to Celery 5.5.0, you will need to reinstall it. + +Contributed by `@auvipy `_ in +`#9620 `_. + +Security Fix: Broker Credential Leak Prevention +------------------------------------------------ + +Fixed a security issue where broker URLs containing passwords were being logged +in plaintext by the delayed delivery mechanism. Broker credentials are now +properly sanitized in all log output. + +Contributed by `@giancarloromeo `_ in +`#9997 `_. + +Memory Leak Fixes +----------------- + +Two significant memory leaks have been fixed in this release: + +**Exception Handling Memory Leak**: Fixed a critical memory leak in task exception +handling that was particularly severe on Python 3.11+ due to enhanced traceback +data. The fix properly breaks reference cycles in tracebacks to allow garbage +collection. + +Contributed by `@jaiganeshs21 `_ in +`#9799 `_. + +**Pending Result Memory Leak**: Fixed a memory leak where ``AsyncResult`` +subscriptions were not being cleaned up when results were forgotten. + +Contributed by `@tsoos99dev `_ in +`#9806 `_. + +ETA Task Memory Limit +--------------------- + +New configuration option :setting:`worker_eta_task_limit` to prevent out-of-memory +crashes when workers fetch large numbers of ETA or countdown tasks. Previously, +workers could exhaust available memory when the broker contained many scheduled tasks. + +Example usage: + +.. code-block:: python + + app.conf.worker_eta_task_limit = 1000 + +Contributed by `@sashu2310 `_ in +`#9853 `_. + +Queue Type Selection for Auto-created Queues +-------------------------------------------- + +New configuration options allow specifying the queue type and exchange type when +Celery auto-creates missing queues. This is particularly useful for RabbitMQ users +who want to use quorum queues with auto-created queues. + +Configuration options: + +- :setting:`task_create_missing_queue_type`: Sets the queue type for auto-created + queues (e.g., ``quorum``, ``classic``) +- :setting:`task_create_missing_queue_exchange_type`: Sets the exchange type for + auto-created queues + +Example usage: + +.. code-block:: python + + app.conf.task_create_missing_queue_type = 'quorum' + +Contributed by `@ghirailghiro `_ in +`#9815 `_. + +What's Changed +~~~~~~~~~~~~~~ + +- Prepare for release: v5.6.0 (#10010) + .. _version-5.6.0rc2: 5.6.0rc2 diff --git a/README.rst b/README.rst index bcf87f9ce7e..dc39a884d83 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.6.0rc2 (recovery) +:Version: 5.6.0 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index dc7a6e87edd..16c263dcecd 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'recovery' -__version__ = '5.6.0rc2' +__version__ = '5.6.0' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/celery/contrib/sphinx.py b/celery/contrib/sphinx.py index a5505ff189a..4cdeb2cb2d4 100644 --- a/celery/contrib/sphinx.py +++ b/celery/contrib/sphinx.py @@ -29,7 +29,23 @@ syntax. Use ``.. autotask::`` to alternatively manually document a task. + +Sphinx 9.0+ Compatibility +------------------------- + +Sphinx 9.0 introduced a rewritten autodoc implementation. The Celery +extension requires the legacy class-based autodoc mode to function +correctly. When using Sphinx 9.0 or later, add the following to your +:file:`conf.py`: + +.. code-block:: python + + autodoc_use_legacy_class_based = True + +The extension will automatically enable this setting if not configured, +but it is recommended to set it explicitly to avoid warnings. """ +import warnings from inspect import signature from docutils import nodes @@ -94,7 +110,24 @@ def autodoc_skip_member_handler(app, what, name, obj, skip, options): def setup(app): """Setup Sphinx extension.""" + import sphinx + app.setup_extension('sphinx.ext.autodoc') + + # Sphinx 9.0+ rewrote autodoc; TaskDocumenter requires legacy mode. + # See: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html + sphinx_version = tuple(int(x) for x in sphinx.__version__.split('.')[:2]) + if sphinx_version >= (9, 0): + if not getattr(app.config, 'autodoc_use_legacy_class_based', False): + warnings.warn( + "Sphinx 9.0+ detected. celery.contrib.sphinx requires " + "'autodoc_use_legacy_class_based = True' in conf.py. " + "Enabling it automatically.", + UserWarning, + stacklevel=2 + ) + app.config.autodoc_use_legacy_class_based = True + app.add_autodocumenter(TaskDocumenter) app.add_directive_to_domain('py', 'task', TaskDirective) app.add_config_value('celery_task_prefix', '(task)', True) diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst index 5dce7554035..d1b4294b212 100644 --- a/docs/history/changelog-5.6.rst +++ b/docs/history/changelog-5.6.rst @@ -8,6 +8,120 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.0: + +5.6.0 +===== + +:release-date: 2025-11-30 +:release-by: Tomer Nosrati + +Celery v5.6.0 is now available. + +Key Highlights +~~~~~~~~~~~~~~ + +See :ref:`whatsnew-5.6` for a complete overview or read the main highlights below. + +Python 3.9 Minimum Version +-------------------------- + +Celery 5.6.0 drops support for Python 3.8 (EOL). The minimum required Python +version is now 3.9. Users still on Python 3.8 must upgrade their Python version +before upgrading to Celery 5.6.0. + +Additionally, this release includes initial support for Python 3.14. + +SQS: Reverted to ``pycurl`` from ``urllib3`` +-------------------------------------------- + +The switch from ``pycurl`` to ``urllib3`` for the SQS transport (introduced in +Celery 5.5.0 via Kombu) has been reverted due to critical issues affecting SQS +users: + +- Processing throughput dropped from ~100 tasks/sec to ~3/sec in some environments +- ``UnknownOperationException`` errors causing container crash loops +- Silent message processing failures with no error logs + +Users of the SQS transport must ensure ``pycurl`` is installed. If you removed +``pycurl`` after upgrading to Celery 5.5.0, you will need to reinstall it. + +Contributed by `@auvipy `_ in +`#9620 `_. + +Security Fix: Broker Credential Leak Prevention +------------------------------------------------ + +Fixed a security issue where broker URLs containing passwords were being logged +in plaintext by the delayed delivery mechanism. Broker credentials are now +properly sanitized in all log output. + +Contributed by `@giancarloromeo `_ in +`#9997 `_. + +Memory Leak Fixes +----------------- + +Two significant memory leaks have been fixed in this release: + +**Exception Handling Memory Leak**: Fixed a critical memory leak in task exception +handling that was particularly severe on Python 3.11+ due to enhanced traceback +data. The fix properly breaks reference cycles in tracebacks to allow garbage +collection. + +Contributed by `@jaiganeshs21 `_ in +`#9799 `_. + +**Pending Result Memory Leak**: Fixed a memory leak where ``AsyncResult`` +subscriptions were not being cleaned up when results were forgotten. + +Contributed by `@tsoos99dev `_ in +`#9806 `_. + +ETA Task Memory Limit +--------------------- + +New configuration option :setting:`worker_eta_task_limit` to prevent out-of-memory +crashes when workers fetch large numbers of ETA or countdown tasks. Previously, +workers could exhaust available memory when the broker contained many scheduled tasks. + +Example usage: + +.. code-block:: python + + app.conf.worker_eta_task_limit = 1000 + +Contributed by `@sashu2310 `_ in +`#9853 `_. + +Queue Type Selection for Auto-created Queues +-------------------------------------------- + +New configuration options allow specifying the queue type and exchange type when +Celery auto-creates missing queues. This is particularly useful for RabbitMQ users +who want to use quorum queues with auto-created queues. + +Configuration options: + +- :setting:`task_create_missing_queue_type`: Sets the queue type for auto-created + queues (e.g., ``quorum``, ``classic``) +- :setting:`task_create_missing_queue_exchange_type`: Sets the exchange type for + auto-created queues + +Example usage: + +.. code-block:: python + + app.conf.task_create_missing_queue_type = 'quorum' + +Contributed by `@ghirailghiro `_ in +`#9815 `_. + +What's Changed +~~~~~~~~~~~~~~ + +- Prepare for release: v5.6.0 (#10010) + .. _version-5.6.0rc2: 5.6.0rc2 diff --git a/docs/history/index.rst b/docs/history/index.rst index 22cd146a1f5..fc13c0a0125 100644 --- a/docs/history/index.rst +++ b/docs/history/index.rst @@ -13,6 +13,8 @@ version please visit :ref:`changelog`. .. toctree:: :maxdepth: 2 + whatsnew-5.6 + changelog-5.6 whatsnew-5.5 changelog-5.5 whatsnew-5.4 diff --git a/docs/history/whatsnew-5.6.rst b/docs/history/whatsnew-5.6.rst index 5fe1ee31b7e..a7ea216b80a 100644 --- a/docs/history/whatsnew-5.6.rst +++ b/docs/history/whatsnew-5.6.rst @@ -37,8 +37,8 @@ While this version is **mostly** backward compatible with previous versions it's important that you read the following section as this release is a new major version. -This version is officially supported on CPython 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13. -and is also supported on PyPy3.10+. +This version is officially supported on CPython 3.9, 3.10, 3.11, 3.12 and 3.13, +and is also supported on PyPy3.11+. .. _`website`: https://celery.readthedocs.io @@ -151,7 +151,7 @@ The supported Python versions are: - CPython 3.11 - CPython 3.12 - CPython 3.13 -- PyPy3.10 (``pypy3``) +- PyPy3.11 (``pypy3``) Python 3.9 Support ------------------ @@ -180,7 +180,7 @@ SQLAlchemy 1.4.x & 2.0.x is now supported in Celery v5.6. Billiard ~~~~~~~~ -Minimum required version is now 4.2.1. +Minimum required version is now 4.2.4. Django ~~~~~~ @@ -193,4 +193,94 @@ Also added --skip-checks flag to bypass django core checks. News ==== -Will be added as we get closer to the release. +SQS: Reverted to ``pycurl`` from ``urllib3`` +-------------------------------------------- + +The switch from ``pycurl`` to ``urllib3`` for the SQS transport (introduced in +Celery 5.5.0 via Kombu) has been reverted due to critical issues affecting SQS +users. + +Security Fix: Broker Credential Leak Prevention +------------------------------------------------ + +Fixed a security issue where broker URLs containing passwords were being logged +in plaintext by the delayed delivery mechanism. Broker credentials are now +properly sanitized in all log output. + +Memory Leak Fixes +----------------- + +Two significant memory leaks have been fixed in this release: + +**Exception Handling Memory Leak**: Fixed a critical memory leak in task exception +handling that was particularly severe on Python 3.11+ due to enhanced traceback +data. The fix properly breaks reference cycles in tracebacks to allow garbage +collection. This resolves a long-standing issue that caused worker memory to grow +unbounded over time. + +**Pending Result Memory Leak**: Fixed a memory leak where ``AsyncResult`` +subscriptions were not being cleaned up when results were forgotten. This affected +users who frequently use ``AsyncResult.forget()`` in their workflows. + +ETA Task Memory Limit +--------------------- + +New configuration option to prevent out-of-memory crashes when workers fetch +large numbers of ETA or countdown tasks. Previously, workers could exhaust +available memory when the broker contained many scheduled tasks. + +Configuration option: + +- :setting:`worker_eta_task_limit`: Sets the maximum number of ETA tasks to hold + in worker memory at once (default: ``None``, unlimited) + +Example usage: + +.. code-block:: python + + app.conf.worker_eta_task_limit = 1000 + +Queue Type Selection for Auto-created Queues +-------------------------------------------- + +New configuration options allow specifying the queue type and exchange type when +Celery auto-creates missing queues. This is particularly useful for RabbitMQ users +who want to use quorum queues with auto-created queues. + +Configuration options: + +- :setting:`task_create_missing_queue_type`: Sets the queue type for auto-created + queues (e.g., ``quorum``, ``classic``) +- :setting:`task_create_missing_queue_exchange_type`: Sets the exchange type for + auto-created queues + +Example usage: + +.. code-block:: python + + app.conf.task_create_missing_queue_type = 'quorum' + +Django Connection Pool Support +------------------------------ + +Fixed an issue where Django applications using psycopg3 connection pooling would +experience ``psycopg_pool.PoolTimeout`` errors after worker forks. Celery now +properly closes Django's connection pools before forking, similar to how Django +itself handles this in its autoreload mechanism. + +Redis Backend Improvements +-------------------------- + +**Credential Provider Support**: Added the :setting:`redis_backend_credential_provider` +setting to the Redis backend. This enables integration with AWS ElastiCache using +IAM authentication and other credential provider mechanisms. + +**Client Name Support**: Added the :setting:`redis_client_name` setting to the Redis +backend, making it easier to identify Celery connections when monitoring Redis servers. + +Cold Shutdown Improvements +-------------------------- + +Fixed an issue where tasks would incorrectly fail with a timeout error during +cold shutdown. The worker now properly skips timeout failure handling during +the cold shutdown phase, allowing tasks to complete or be properly requeued. diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index c6a940e4576..5c97868a464 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.6.0rc2 (recovery) +:Version: 5.6.0 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 9f0a61c61ff7e6a7280542a03895e6fa7cb80552 Mon Sep 17 00:00:00 2001 From: anthonykuzmich7 Date: Wed, 3 Dec 2025 11:05:46 +0100 Subject: [PATCH 088/169] Fix Redis Sentinel ACL authentication support (#10013) - Extract username from sentinel URL in _params_from_url - Pass credentials (username/password) to master_for() in _get_pool - Add unit tests for ACL authentication --- Changelog.rst | 10 +++++++ celery/backends/redis.py | 9 ++++-- t/unit/backends/test_redis.py | 56 ++++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index f06d81f1674..beef6371d0e 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,16 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.1: + +5.6.1 +===== + +:release-date: TBA +:release-by: + +- Fix Redis Sentinel ACL authentication support + .. _version-5.6.0: 5.6.0 diff --git a/celery/backends/redis.py b/celery/backends/redis.py index 28e9b723102..a21a5ebfe8b 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -683,8 +683,8 @@ def _params_from_url(self, url, defaults): for param in ("host", "port", "db", "password"): connparams.pop(param) - # Adding db/password in connparams to connect to the correct instance - for param in ("db", "password"): + # Adding db/password/username in connparams to connect to the correct instance + for param in ("db", "password", "username"): if connparams['hosts'] and param in connparams['hosts'][0]: connparams[param] = connparams['hosts'][0].get(param) return connparams @@ -709,7 +709,12 @@ def _get_pool(self, **params): master_name = self._transport_options.get("master_name", None) + credentials = { + k: params[k] for k in ("username", "password") if k in params + } + return sentinel_instance.master_for( service_name=master_name, redis_class=self._get_client(), + **credentials, ).connection_pool diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index e2233b01a08..c10495e2e45 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -173,7 +173,8 @@ def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None, self.min_other_sentinels = min_other_sentinels self.connection_kwargs = connection_kwargs - def master_for(self, service_name, redis_class): + def master_for(self, service_name, redis_class, **kwargs): + self.master_for_kwargs = kwargs return random.choice(self.sentinels) @@ -1417,3 +1418,56 @@ def test_backend_ssl(self): from celery.backends.redis import SentinelManagedSSLConnection assert x.connparams['connection_class'] is SentinelManagedSSLConnection + + def test_url_with_acl_credentials(self): + x = self.Backend( + 'sentinel://myuser:mypass@github.com:123/1;' + 'sentinel://myuser:mypass@github.com:124/1', + app=self.app, + ) + assert x.connparams + assert "host" not in x.connparams + assert x.connparams['db'] == 1 + assert "port" not in x.connparams + assert x.connparams['password'] == "mypass" + assert x.connparams['username'] == "myuser" + assert len(x.connparams['hosts']) == 2 + + expected_usernames = ["myuser", "myuser"] + found_usernames = [cp['username'] for cp in x.connparams['hosts']] + assert found_usernames == expected_usernames + + def test_get_pool_with_acl_credentials(self): + x = self.Backend( + 'sentinel://myuser:mypass@github.com:123/1;' + 'sentinel://myuser:mypass@github.com:124/1', + app=self.app, + ) + with patch.object(x, '_get_sentinel_instance') as mock_get_sentinel: + mock_sentinel = Mock() + mock_sentinel.master_for.return_value = Mock(connection_pool=Mock()) + mock_get_sentinel.return_value = mock_sentinel + + x._get_pool(**x.connparams) + + mock_sentinel.master_for.assert_called_once() + call_kwargs = mock_sentinel.master_for.call_args[1] + assert call_kwargs.get('username') == 'myuser' + assert call_kwargs.get('password') == 'mypass' + + def test_get_pool_with_password_only(self): + x = self.Backend( + 'sentinel://:mypass@github.com:123/1', + app=self.app, + ) + with patch.object(x, '_get_sentinel_instance') as mock_get_sentinel: + mock_sentinel = Mock() + mock_sentinel.master_for.return_value = Mock(connection_pool=Mock()) + mock_get_sentinel.return_value = mock_sentinel + + x._get_pool(**x.connparams) + + mock_sentinel.master_for.assert_called_once() + call_kwargs = mock_sentinel.master_for.call_args[1] + assert 'username' not in call_kwargs + assert call_kwargs.get('password') == 'mypass' From fc947644289c50940c09a3d8db501d336da85404 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 Dec 2025 16:17:38 +0600 Subject: [PATCH 089/169] [pre-commit.ci] pre-commit autoupdate (#10012) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v1.18.2 → v1.19.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.18.2...v1.19.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80da5253073..006ab27a8a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.18.2 + rev: v1.19.0 hooks: - id: mypy pass_filenames: false From ab711a0b2de8fcc5aa8ec6701cdcb0530d6daebb Mon Sep 17 00:00:00 2001 From: weetster <98231586+weetster@users.noreply.github.com> Date: Fri, 5 Dec 2025 00:09:02 -0500 Subject: [PATCH 090/169] Fix: Broker heartbeats not sent during graceful shutdown (#9986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Continue firing timers while pool drains * Unit tests * flake8 fix * Code review changes * Integration test for prefork shutdown * Additional code review changes * Documentation update * Code review changes * Update documentation to specify that the behaviour described only applies to the prefork pool --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/concurrency/prefork.py | 46 ++++++++++++- celery/worker/components.py | 5 +- celery/worker/consumer/consumer.py | 2 - celery/worker/loops.py | 35 ++++------ docs/userguide/workers.rst | 6 ++ t/integration/test_prefork_shutdown.py | 89 ++++++++++++++++++++++++++ t/unit/concurrency/test_prefork.py | 79 +++++++++++++++++++++++ t/unit/worker/test_consumer.py | 3 +- 8 files changed, 238 insertions(+), 27 deletions(-) create mode 100644 t/integration/test_prefork_shutdown.py diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index b163328d0b3..aed83b8a29a 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -3,11 +3,14 @@ Pool implementation using :mod:`multiprocessing`. """ import os +import threading +import time from billiard import forking_enable from billiard.common import REMAP_SIGTERM, TERM_SIGNAME from billiard.pool import CLOSE, RUN from billiard.pool import Pool as BlockingPool +from kombu.asynchronous import get_event_loop from celery import platforms, signals from celery._state import _set_task_join_will_block, set_default_app @@ -140,7 +143,48 @@ def on_stop(self): """Gracefully stop the pool.""" if self._pool is not None and self._pool._state in (RUN, CLOSE): self._pool.close() - self._pool.join() + + # Keep firing timers (for heartbeats on async transports) while + # the pool drains. If not using an async transport, no hub exists + # and the timer thread is not created. + hub = get_event_loop() + if hub is not None: + shutdown_event = threading.Event() + + def fire_timers_loop(): + while not shutdown_event.is_set(): + try: + hub.fire_timers() + except Exception: + logger.warning( + "Exception in timer thread during prefork on_stop()", + exc_info=True, + ) + # 0.5 seconds was chosen as a balance between joining quickly + # after the pool join is complete and sleeping long enough to + # avoid excessive CPU usage. + time.sleep(0.5) + + timer_thread = threading.Thread( + target=fire_timers_loop, + daemon=True, + name="prefork-timer-shutdown", + ) + timer_thread.start() + + try: + self._pool.join() + finally: + shutdown_event.set() + timer_thread.join(timeout=1.0) + + if timer_thread.is_alive(): + logger.warning( + "Timer thread in prefork on_stop() did not terminate cleanly" + ) + else: + self._pool.join() + self._pool = None def on_terminate(self): diff --git a/celery/worker/components.py b/celery/worker/components.py index f062affb61f..d1ec1db2f3e 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -75,7 +75,10 @@ def create(self, w): return self def start(self, w): - pass + # Ensure the kombu hub's poller is initialized before the event loop starts. + # Since asynloop() no longer resets the hub on exit (to preserve timers + # during shutdown), we must initialize the poller upfront. + _ = w.hub.poller def stop(self, w): w.hub.close() diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 95e73707621..2a5d955fef3 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -456,8 +456,6 @@ def on_close(self): # to the current channel. if self.controller and self.controller.semaphore: self.controller.semaphore.clear() - if self.timer: - self.timer.clear() for bucket in self.task_buckets.values(): if bucket: bucket.clear_pending() diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 1f9e589eeef..f88cddb8d6b 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -81,28 +81,21 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, hub.propagate_errors = errors loop = hub.create_loop() - try: - while blueprint.state == RUN and obj.connection: - state.maybe_shutdown() - if heartbeat_error[0] is not None: - raise heartbeat_error[0] - - # We only update QoS when there's no more messages to read. - # This groups together qos calls, and makes sure that remote - # control commands will be prioritized over task messages. - if qos.prev != qos.value: - update_qos() - - try: - next(loop) - except StopIteration: - loop = hub.create_loop() - finally: + while blueprint.state == RUN and obj.connection: + state.maybe_shutdown() + if heartbeat_error[0] is not None: + raise heartbeat_error[0] + + # We only update QoS when there's no more messages to read. + # This groups together qos calls, and makes sure that remote + # control commands will be prioritized over task messages. + if qos.prev != qos.value: + update_qos() + try: - hub.reset() - except Exception as exc: # pylint: disable=broad-except - logger.exception( - 'Error cleaning up after event loop: %r', exc) + next(loop) + except StopIteration: + loop = hub.create_loop() def synloop(obj, connection, consumer, blueprint, hub, qos, diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 01d6491d72b..e665da489d9 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -126,6 +126,12 @@ and will call :func:`WorkController.stop() Date: Fri, 5 Dec 2025 14:41:48 +0900 Subject: [PATCH 091/169] Document confirm_publish broker transport option (#10016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: JaeHyuck Sa Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/userguide/calling.rst | 7 +++++++ docs/userguide/configuration.rst | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index d9c29dc536c..273a19bebac 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -460,6 +460,13 @@ You can handle this error too: ... except add.OperationalError as exc: ... logger.exception('Sending task raised: %r', exc) +.. note:: + + With RabbitMQ, these errors only indicate the broker is unreachable. + Messages can still be silently dropped when the broker hits resource + limits. Enable ``confirm_publish`` in :setting:`broker_transport_options` + to detect this. + .. _calling-serializers: Serializers diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index ea0e66d9705..b0bb46fa2a7 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -3123,6 +3123,14 @@ won't retry forever if the broker isn't available at the first task execution): broker_transport_options = {'max_retries': 5} +Example enabling publisher confirms (supported by the ``pyamqp`` transport). +Without this, messages can be silently dropped when the broker hits resource +limits: + +.. code-block:: python + + broker_transport_options = {'confirm_publish': True} + .. _conf-worker: Worker From a4f9beb41008085fa790b18b97aa9daf476beb8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20P=C5=99ikryl?= <2625825+petrprikryl@users.noreply.github.com> Date: Tue, 9 Dec 2025 05:44:12 +0100 Subject: [PATCH 092/169] close DB pools only in prefork mode (#10020) * close DB pools only in prefork mode * fix tests --------- Co-authored-by: petr.prikryl --- celery/fixups/django.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 80a6e3e84d7..1f5532859e3 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -199,11 +199,13 @@ def close_database(self, **kwargs: Any) -> None: self._db_recycles += 1 def _close_database(self) -> None: + is_prefork = self.app.conf.get('worker_pool', 'prefork') == "prefork" + for conn in self._db.connections.all(): try: conn.close() pool_enabled = self._settings.DATABASES.get(conn.alias, {}).get("OPTIONS", {}).get("pool") - if pool_enabled and hasattr(conn, "close_pool"): + if pool_enabled and is_prefork and hasattr(conn, "close_pool"): with contextlib.suppress(KeyError): conn.close_pool() except self.interface_errors: From cc3350ef9867a34b1f3563222bd015d123ef1ad7 Mon Sep 17 00:00:00 2001 From: Simon Kelly Date: Tue, 9 Dec 2025 12:17:06 +0200 Subject: [PATCH 093/169] Fix: Avoid unnecessary Django database connection creation during cleanup (#10015) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update database connection closing to initialized only Avoid creating and closing connections immediately by only iterating over existing connections. * update mock in django test * support old Django versions method signature * Apply suggestion from @auvipy --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/fixups/django.py | 8 +++++++- t/unit/fixups/test_django.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 1f5532859e3..6b733e6d591 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -199,9 +199,15 @@ def close_database(self, **kwargs: Any) -> None: self._db_recycles += 1 def _close_database(self) -> None: + try: + connections = self._db.connections.all(initialized_only=True) + except TypeError: + # Support Django < 4.1 + connections = self._db.connections.all() + is_prefork = self.app.conf.get('worker_pool', 'prefork') == "prefork" - for conn in self._db.connections.all(): + for conn in connections: try: conn.close() pool_enabled = self._settings.DATABASES.get(conn.alias, {}).get("OPTIONS", {}).get("pool") diff --git a/t/unit/fixups/test_django.py b/t/unit/fixups/test_django.py index 89a045dfc6a..f14348426a8 100644 --- a/t/unit/fixups/test_django.py +++ b/t/unit/fixups/test_django.py @@ -262,7 +262,7 @@ def test__close_database(self): f.interface_errors = () f._db.connections = Mock() # ConnectionHandler - f._db.connections.all.side_effect = lambda: conns + f._db.connections.all.side_effect = lambda initialized_only: conns f._close_database() conns[0].close.assert_called_with() From cb08d5042a8ce1758a7460ca6856491b36949f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20P=C5=99ikryl?= <2625825+petrprikryl@users.noreply.github.com> Date: Tue, 16 Dec 2025 12:24:23 +0100 Subject: [PATCH 094/169] reliable prefork detection (#10023) * reliable prefork detection * copilot feedback * better coverage --------- Co-authored-by: petr.prikryl --- celery/fixups/django.py | 16 ++++++++++++-- t/unit/fixups/test_django.py | 43 ++++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 6b733e6d591..65cb8eac1f2 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -12,6 +12,7 @@ from celery import _state, signals from celery.exceptions import FixupWarning, ImproperlyConfigured +from celery.worker import WorkController if TYPE_CHECKING: from types import ModuleType @@ -102,6 +103,16 @@ def on_import_modules(self, **kwargs: Any) -> None: self.worker_fixup.validate_models() def on_worker_init(self, **kwargs: Any) -> None: + worker: Optional["WorkController"] = kwargs.get("sender") + if worker: + self.worker_fixup.worker = worker + else: + warnings.warn( + "DjangoFixup.on_worker_init called without a sender (worker instance). " + "This may indicate a misconfiguration or an internal error.", + FixupWarning, + stacklevel=2, + ) self.worker_fixup.install() def now(self, utc: bool = False) -> datetime: @@ -119,8 +130,9 @@ def _now(self) -> datetime: class DjangoWorkerFixup: _db_recycles = 0 - def __init__(self, app: "Celery") -> None: + def __init__(self, app: "Celery", worker: Optional["WorkController"] = None) -> None: self.app = app + self.worker = worker or WorkController(app) self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None) self._db = cast("DjangoDBModule", import_module('django.db')) self._cache = import_module('django.core.cache') @@ -205,7 +217,7 @@ def _close_database(self) -> None: # Support Django < 4.1 connections = self._db.connections.all() - is_prefork = self.app.conf.get('worker_pool', 'prefork') == "prefork" + is_prefork = "prefork" in self.worker.pool_cls.__module__ for conn in connections: try: diff --git a/t/unit/fixups/test_django.py b/t/unit/fixups/test_django.py index f14348426a8..ade1330d534 100644 --- a/t/unit/fixups/test_django.py +++ b/t/unit/fixups/test_django.py @@ -3,6 +3,7 @@ import pytest +from celery.concurrency.thread import TaskPool as ThreadTaskPool from celery.fixups.django import DjangoFixup, DjangoWorkerFixup, FixupWarning, _maybe_close_fd, fixup from t.unit import conftest @@ -11,11 +12,11 @@ class FixupCase: Fixup = None @contextmanager - def fixup_context(self, app): + def fixup_context(self, app, **kwargs): with patch('celery.fixups.django.DjangoWorkerFixup.validate_models'): with patch('celery.fixups.django.symbol_by_name') as symbyname: with patch('celery.fixups.django.import_module') as impmod: - f = self.Fixup(app) + f = self.Fixup(app, **kwargs) yield f, impmod, symbyname @@ -150,11 +151,20 @@ def test_now(self): def test_on_worker_init(self): with self.fixup_context(self.app) as (f, _, _): with patch('celery.fixups.django.DjangoWorkerFixup') as DWF: - f.on_worker_init() + mock_worker = Mock(name="worker") + f.on_worker_init(sender=mock_worker) + assert DWF.return_value.worker == mock_worker + DWF.assert_called_with(f.app) DWF.return_value.install.assert_called_with() assert f._worker_fixup is DWF.return_value + def test_on_worker_init_warns_without_sender(self): + with self.fixup_context(self.app) as (f, _, _): + with patch("celery.fixups.django.DjangoWorkerFixup"): + with pytest.warns(FixupWarning, match="called without a sender"): + f.on_worker_init(sender=None) + class InterfaceError(Exception): pass @@ -168,9 +178,11 @@ def test_init(self): assert f def test_install(self): + worker = Mock() + worker.pool_cls = Mock(__module__='celery.concurrency.prefork') self.app.conf = {'CELERY_DB_REUSE_MAX': None} self.app.loader = Mock() - with self.fixup_context(self.app) as (f, _, _): + with self.fixup_context(self.app, worker=worker) as (f, _, _): with patch('celery.fixups.django.signals') as sigs: f.install() sigs.beat_embedded_init.connect.assert_called_with( @@ -336,6 +348,29 @@ class DJSettings: conn.close.assert_called_once_with() conn.close_pool.assert_not_called() + def test_close_database_conn_pool_thread_pool(self): + class DJSettings: + DATABASES = {} + + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + conn.alias = "default" + conn.close_pool = Mock() + f._db.connections.all = Mock(return_value=[conn]) + f._settings = DJSettings + + f._settings.DATABASES["default"] = {"OPTIONS": {"pool": True}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_called_once_with() + + conn.reset_mock() + f.worker.pool_cls = ThreadTaskPool + assert "prefork" not in ThreadTaskPool.__module__ + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_not_called() + def test_close_cache_raises_error(self): with self.fixup_context(self.app) as (f, _, _): f._cache.close_caches.side_effect = AttributeError From 170fe556eca986da69aecba10e408859e871ada0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 16 Dec 2025 17:36:48 +0600 Subject: [PATCH 095/169] [pre-commit.ci] pre-commit autoupdate (#10027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v1.19.0 → v1.19.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.19.0...v1.19.1) Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 006ab27a8a3..72da29766ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.19.0 + rev: v1.19.1 hooks: - id: mypy pass_filenames: false From 6da72bde60553a1c350eef1e17d834d004ed099d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20P=C5=99ikryl?= <2625825+petrprikryl@users.noreply.github.com> Date: Tue, 16 Dec 2025 12:42:48 +0100 Subject: [PATCH 096/169] better coverage (#10029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: petr.prikryl Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index bc8abdc6abf..f0aaa95e30d 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,9 @@ deps= bandit: bandit commands = - unit: pytest -vv --maxfail=10 --capture=no -v --cov=celery --cov-report=xml --junitxml=junit.xml -o junit_family=legacy --cov-report term {posargs} + unit: coverage run --source=celery -m pytest -vv --maxfail=10 --capture=no -v --junitxml=junit.xml -o junit_family=legacy {posargs} + unit: coverage xml + unit: coverage report integration: pytest -xsvv t/integration {posargs} smoke: pytest -xsvv t/smoke --dist=loadscope --reruns 5 --reruns-delay 10 {posargs} setenv = From f19db7008682c782d85e69b5ce736970491752ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Dec 2025 18:54:25 +0600 Subject: [PATCH 097/169] Bump mypy from 1.19.0 to 1.19.1 (#10028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [mypy](https://github.com/python/mypy) from 1.19.0 to 1.19.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.19.0...v1.19.1) --- updated-dependencies: - dependency-name: mypy dependency-version: 1.19.1 dependency-type: direct:production update-type: version-update:semver-patch ... Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/test.txt b/requirements/test.txt index f46d754a693..f2fe7e2d165 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,7 +8,7 @@ pytest-order==1.3.0 boto3>=1.26.143 moto>=4.1.11,<5.1.0 # type checking -mypy==1.19.0; platform_python_implementation=="CPython" +mypy==1.19.1; platform_python_implementation=="CPython" pre-commit>=4.0.1; python_version >= '3.9' -r extras/yaml.txt -r extras/msgpack.txt From 5f8659bdba4561796d66e938a1e2c71d4489562f Mon Sep 17 00:00:00 2001 From: SpaceShaman Date: Tue, 16 Dec 2025 15:42:20 +0100 Subject: [PATCH 098/169] Clarify 'result_extended' setting usage in tasks Added note about 'result_extended' and periodic task context. --- docs/userguide/calling.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 273a19bebac..b014357e2b6 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -812,6 +812,24 @@ setting or by using the ``ignore_result`` option: If you'd like to store additional metadata about the task in the result backend set the :setting:`result_extended` setting to ``True``. +.. note:: + + ``result_extended`` controls what *Celery* includes as extended task metadata, + but it does not automatically add scheduler-specific metadata. + For example, some integrations (e.g. :pypi:`django-celery-beat` together with + :pypi:`django-celery-results`) may record the *periodic task name* in the result + backend only when the scheduler provides it as part of the published message. + + When you call tasks manually using ``apply_async``/``delay``, that periodic task + context is usually not present unless you add it explicitly (e.g. via message + headers/properties in ``apply_async`` options). For example: + + .. code-block:: python + + result = task.apply_async( + headers={"periodic_task_name": "task_name"}, + ) + .. seealso:: For more information on tasks, please see :ref:`guide-tasks`. From 0527296acb1f1790788301d4395ba6d5ce2a9704 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Dec 2025 22:08:37 +0000 Subject: [PATCH 099/169] Bump google-cloud-firestore from 2.21.0 to 2.22.0 Bumps [google-cloud-firestore](https://github.com/googleapis/python-firestore) from 2.21.0 to 2.22.0. - [Release notes](https://github.com/googleapis/python-firestore/releases) - [Changelog](https://github.com/googleapis/python-firestore/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/python-firestore/compare/v2.21.0...v2.22.0) --- updated-dependencies: - dependency-name: google-cloud-firestore dependency-version: 2.22.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements/extras/gcs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/gcs.txt b/requirements/extras/gcs.txt index 5a3f1511c12..2cc9b4779f5 100644 --- a/requirements/extras/gcs.txt +++ b/requirements/extras/gcs.txt @@ -1,4 +1,4 @@ google-cloud-storage>=2.10.0 -google-cloud-firestore==2.21.0 +google-cloud-firestore==2.22.0 grpcio==1.75.1 From 21675299bc2d10c648448ec5221136e07c1eec1a Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 22 Dec 2025 16:24:51 +0000 Subject: [PATCH 100/169] Stop importing pytest_subtests This doesn't seem to be necessary either with pytest<9 (where pytest-subtests was a separate package) or with pytest>=9 (where subtest support is integrated into pytest). --- t/integration/test_canvas.py | 1 - t/unit/tasks/test_canvas.py | 1 - t/unit/utils/test_functional.py | 1 - 3 files changed, 3 deletions(-) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index fd036e4cb10..8d137b6a805 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -6,7 +6,6 @@ from time import monotonic, sleep import pytest -import pytest_subtests # noqa from celery import chain, chord, group, signature from celery.backends.base import BaseKeyValueStoreBackend diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 40f02e6db8a..9b97c3dd65a 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -4,7 +4,6 @@ from unittest.mock import ANY, MagicMock, Mock, call, patch, sentinel import pytest -import pytest_subtests # noqa from celery._state import _task_stack from celery.canvas import (Signature, _chain, _maybe_group, _merge_dictionaries, chain, chord, chunks, group, diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index a8c9dc1e893..c0bf626a747 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,7 +1,6 @@ import collections import pytest -import pytest_subtests # noqa from kombu.utils.functional import lazy from celery.utils.functional import (DummyContext, first, firstmethod, fun_accepts_kwargs, fun_takes_argument, From ba20bed7723c684d12ddd62d6a7c0c0d901b3351 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Mon, 22 Dec 2025 17:26:18 +0000 Subject: [PATCH 101/169] Only use exceptiongroup backport for Python < 3.11 (#10033) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/worker/consumer/delayed_delivery.py | 8 +++++--- requirements/default.txt | 2 +- t/unit/worker/test_native_delayed_delivery.py | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/celery/worker/consumer/delayed_delivery.py b/celery/worker/consumer/delayed_delivery.py index 46996ca2caf..9909be3cf27 100644 --- a/celery/worker/consumer/delayed_delivery.py +++ b/celery/worker/consumer/delayed_delivery.py @@ -3,11 +3,13 @@ This module provides the DelayedDelivery bootstep which handles setup and configuration of native delayed delivery functionality when using quorum queues. """ +import sys from typing import Iterator, List, Optional, Set, Union, ValuesView -# Backport of PEP 654 for Python versions < 3.11 -# In Python 3.11+, exceptiongroup uses the built-in ExceptionGroup -from exceptiongroup import ExceptionGroup +if sys.version_info < (3, 11): # pragma: no cover + # Backport of PEP 654 for Python versions < 3.11 + from exceptiongroup import ExceptionGroup + from kombu import Connection, Queue from kombu.transport.native_delayed_delivery import (bind_queue_to_native_delayed_delivery_exchange, declare_native_delayed_delivery_exchanges_and_queues) diff --git a/requirements/default.txt b/requirements/default.txt index 25db1a331f0..eddb0ca7229 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -6,5 +6,5 @@ click-didyoumean>=0.3.0 click-repl>=0.2.0 click-plugins>=1.1.1 python-dateutil>=2.8.2 -exceptiongroup>=1.3.0 +exceptiongroup>=1.3.0; python_version < '3.11' tzlocal diff --git a/t/unit/worker/test_native_delayed_delivery.py b/t/unit/worker/test_native_delayed_delivery.py index c5f2adcfc11..83b7d2888b7 100644 --- a/t/unit/worker/test_native_delayed_delivery.py +++ b/t/unit/worker/test_native_delayed_delivery.py @@ -1,11 +1,14 @@ import itertools +import sys from logging import LogRecord from typing import Iterator from unittest.mock import MagicMock, Mock, patch +if sys.version_info < (3, 11): # pragma: no cover + from exceptiongroup import ExceptionGroup + import pytest from amqp import NotFound -from exceptiongroup import ExceptionGroup from kombu import Exchange, Queue from kombu.utils.functional import retry_over_time From 21dbc73f81d2fae058de49e665afaa1cd92da5c0 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Mon, 29 Dec 2025 23:45:02 +0200 Subject: [PATCH 102/169] Prepare for release: v5.6.1 (#10037) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.6.0 → 5.6.1 * Added Changelog for v5.6.1 --- .bumpversion.cfg | 2 +- Changelog.rst | 19 ++++++++++++++++--- README.rst | 2 +- celery/__init__.py | 2 +- docs/history/changelog-5.6.rst | 23 +++++++++++++++++++++++ docs/includes/introduction.txt | 2 +- 6 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 63b58a78481..d187658444a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.6.0 +current_version = 5.6.1 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/Changelog.rst b/Changelog.rst index beef6371d0e..9d7a21881ba 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -13,10 +13,23 @@ an overview of what's new in Celery 5.6. 5.6.1 ===== -:release-date: TBA -:release-by: +:release-date: 2025-12-29 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ -- Fix Redis Sentinel ACL authentication support +- Fix Redis Sentinel ACL authentication support (#10013) +- Fix: Broker heartbeats not sent during graceful shutdown (#9986) +- docs #5410 -- Document confirm_publish broker transport option (#10016) +- close DB pools only in prefork mode (#10020) +- Fix: Avoid unnecessary Django database connection creation during cleanup (#10015) +- reliable prefork detection (#10023) +- better coverage (#10029) +- Docs: clarify `result_extended` vs periodic task metadata and show `headers["periodic_task_name"]` example (#10030) +- Stop importing pytest_subtests (#10032) +- Only use exceptiongroup backport for Python < 3.11 (#10033) +- Prepare for release: v5.6.1 (#10037) .. _version-5.6.0: diff --git a/README.rst b/README.rst index dc39a884d83..a34a036d433 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.6.0 (recovery) +:Version: 5.6.1 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 16c263dcecd..2e45c0e5495 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'recovery' -__version__ = '5.6.0' +__version__ = '5.6.1' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst index d1b4294b212..d861ea56517 100644 --- a/docs/history/changelog-5.6.rst +++ b/docs/history/changelog-5.6.rst @@ -8,6 +8,29 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.1: + +5.6.1 +===== + +:release-date: 2025-12-29 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix Redis Sentinel ACL authentication support (#10013) +- Fix: Broker heartbeats not sent during graceful shutdown (#9986) +- docs #5410 -- Document confirm_publish broker transport option (#10016) +- close DB pools only in prefork mode (#10020) +- Fix: Avoid unnecessary Django database connection creation during cleanup (#10015) +- reliable prefork detection (#10023) +- better coverage (#10029) +- Docs: clarify `result_extended` vs periodic task metadata and show `headers["periodic_task_name"]` example (#10030) +- Stop importing pytest_subtests (#10032) +- Only use exceptiongroup backport for Python < 3.11 (#10033) +- Prepare for release: v5.6.1 (#10037) + .. _version-5.6.0: 5.6.0 diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 5c97868a464..53187353a19 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.6.0 (recovery) +:Version: 5.6.1 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 9d6ab110d947270c4edf0b42454266d2954e653d Mon Sep 17 00:00:00 2001 From: Bruno Trindade Date: Sun, 4 Jan 2026 02:14:16 -0500 Subject: [PATCH 103/169] Fix recursive WorkController instantiation in DjangoWorkerFixup + AttributeError when pool_cls is a string (#10045) * Fix recursive WorkController instantiation in DjangoWorkerFixup - Remove worker parameter from DjangoWorkerFixup.__init__ - Set worker via on_worker_init callback instead - Add None check for worker.pool_cls to prevent AttributeError - Add regression test to prevent future recursion bugs Fixes recursive instantiation issue where DjangoWorkerFixup(app) would create WorkController(app), which in turn would create another DjangoWorkerFixup, leading to infinite recursion. The worker instance is now properly set via the on_worker_init signal callback, avoiding the circular dependency. * fix: improve prefork detection and add integration test * fix: Copilot recommendations * fix: CI unit tests + test coverage --- celery/fixups/django.py | 12 +++- requirements/test-integration.txt | 1 + t/integration/test_django_settings.py | 19 ++++++ t/integration/test_worker.py | 48 ++++++++++++++ t/unit/fixups/test_django.py | 90 ++++++++++++++++++++++++++- tox.ini | 2 +- 6 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 t/integration/test_django_settings.py diff --git a/celery/fixups/django.py b/celery/fixups/django.py index 65cb8eac1f2..5d78b381607 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -129,10 +129,10 @@ def _now(self) -> datetime: class DjangoWorkerFixup: _db_recycles = 0 + worker = None # Set via on_worker_init callback to avoid recursive WorkController instantiation - def __init__(self, app: "Celery", worker: Optional["WorkController"] = None) -> None: + def __init__(self, app: "Celery") -> None: self.app = app - self.worker = worker or WorkController(app) self.db_reuse_max = self.app.conf.get('CELERY_DB_REUSE_MAX', None) self._db = cast("DjangoDBModule", import_module('django.db')) self._cache = import_module('django.core.cache') @@ -210,6 +210,12 @@ def close_database(self, **kwargs: Any) -> None: self._close_database() self._db_recycles += 1 + def _is_prefork(self) -> bool: + if self.worker is None: + return False + pool = self.worker.pool_cls if isinstance(self.worker.pool_cls, str) else self.worker.pool_cls.__module__ + return "prefork" in pool + def _close_database(self) -> None: try: connections = self._db.connections.all(initialized_only=True) @@ -217,7 +223,7 @@ def _close_database(self) -> None: # Support Django < 4.1 connections = self._db.connections.all() - is_prefork = "prefork" in self.worker.pool_cls.__module__ + is_prefork = self._is_prefork() for conn in connections: try: diff --git a/requirements/test-integration.txt b/requirements/test-integration.txt index 50f5fdd9dcf..b915556332f 100644 --- a/requirements/test-integration.txt +++ b/requirements/test-integration.txt @@ -2,5 +2,6 @@ -r extras/azureblockblob.txt -r extras/auth.txt -r extras/memcache.txt +-r extras/django.txt pytest-rerunfailures>=11.1.2 git+https://github.com/celery/kombu.git diff --git a/t/integration/test_django_settings.py b/t/integration/test_django_settings.py new file mode 100644 index 00000000000..93de7f81139 --- /dev/null +++ b/t/integration/test_django_settings.py @@ -0,0 +1,19 @@ +import logging + +logging.info("Initializing Django settings for Celery integration tests") + +DEBUG = True +INSTALLED_APPS = [ + 'django.contrib.contenttypes', + 'django.contrib.auth', +] + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + } +} + +CELERY_BROKER_URL = 'memory://' +CELERY_RESULT_BACKEND = 'cache+memory://' diff --git a/t/integration/test_worker.py b/t/integration/test_worker.py index 9487753f4a5..23abe032ba5 100644 --- a/t/integration/test_worker.py +++ b/t/integration/test_worker.py @@ -2,6 +2,8 @@ import pytest +from celery import Celery + def test_run_worker(): with pytest.raises(subprocess.CalledProcessError) as exc_info: @@ -16,3 +18,49 @@ def test_run_worker(): "Retrying to establish a connection to the message broker after a connection " "loss has been disabled (app.conf.broker_connection_retry_on_startup=False). " "Shutting down...") != -1, output + + +def test_django_fixup_direct_worker(caplog, monkeypatch): + """Test Django fixup by directly instantiating Celery worker without subprocess.""" + import logging + + import django + + # Set logging level to capture debug messages + caplog.set_level(logging.DEBUG) + + # Configure Django settings + monkeypatch.setenv('DJANGO_SETTINGS_MODULE', 't.integration.test_django_settings') + django.setup() + + # Create Celery app with Django integration + app = Celery('test_django_direct') + app.config_from_object('django.conf:settings', namespace='CELERY') + app.autodiscover_tasks() + + # Test that we can access worker configuration without recursion errors + # This should trigger the Django fixup initialization + worker = app.Worker( + pool='solo', + concurrency=1, + loglevel='debug' + ) + + # Accessing pool_cls should not cause AttributeError + pool_cls = worker.pool_cls + assert pool_cls is not None + + # Verify pool_cls has __module__ attribute (should be a class, not a string) + assert hasattr(pool_cls, '__module__'), \ + f"pool_cls should be a class with __module__, got {type(pool_cls)}: {pool_cls}" + + # Capture and check logs + log_output = caplog.text + + # Verify no recursion-related errors in logs + assert "RecursionError" not in log_output, f"RecursionError found in logs:\n{log_output}" + assert "maximum recursion depth exceeded" not in log_output, \ + f"Recursion depth error found in logs:\n{log_output}" + + assert "AttributeError: 'str' object has no attribute '__module__'." not in log_output, \ + f"AttributeError found in logs:\n{log_output}" diff --git a/t/unit/fixups/test_django.py b/t/unit/fixups/test_django.py index ade1330d534..a75dbd90c1e 100644 --- a/t/unit/fixups/test_django.py +++ b/t/unit/fixups/test_django.py @@ -16,7 +16,10 @@ def fixup_context(self, app, **kwargs): with patch('celery.fixups.django.DjangoWorkerFixup.validate_models'): with patch('celery.fixups.django.symbol_by_name') as symbyname: with patch('celery.fixups.django.import_module') as impmod: + worker = Mock() + worker.pool_cls = Mock(__module__='celery.concurrency.prefork') f = self.Fixup(app, **kwargs) + f.worker = worker yield f, impmod, symbyname @@ -178,11 +181,9 @@ def test_init(self): assert f def test_install(self): - worker = Mock() - worker.pool_cls = Mock(__module__='celery.concurrency.prefork') self.app.conf = {'CELERY_DB_REUSE_MAX': None} self.app.loader = Mock() - with self.fixup_context(self.app, worker=worker) as (f, _, _): + with self.fixup_context(self.app) as (f, _, _): with patch('celery.fixups.django.signals') as sigs: f.install() sigs.beat_embedded_init.connect.assert_called_with( @@ -286,6 +287,37 @@ def test__close_database(self): with pytest.raises(KeyError): f._close_database() + def test__close_database_django_pre_41(self): + """Test that Django < 4.1 (without initialized_only parameter) is handled.""" + with self.fixup_context(self.app) as (f, _, _): + conns = [Mock(), Mock()] + f.DatabaseError = KeyError + f.interface_errors = () + + # Mock Django < 4.1 behavior: connections.all() doesn't accept initialized_only + f._db.connections = Mock() + + def all_without_initialized_only(**kwargs): + if 'initialized_only' in kwargs: + raise TypeError("all() got an unexpected keyword argument 'initialized_only'") + return conns + + f._db.connections.all = Mock(side_effect=all_without_initialized_only) + + # Should fall back to calling all() without initialized_only + f._close_database() + + # Verify it was called twice: first with initialized_only (raises), then without + assert f._db.connections.all.call_count == 2 + # First call with initialized_only=True + f._db.connections.all.assert_any_call(initialized_only=True) + # Second call without arguments (fallback) + f._db.connections.all.assert_any_call() + + # Verify connections were closed + conns[0].close.assert_called_with() + conns[1].close.assert_called_with() + def test_close_database_always_closes_connections(self): with self.fixup_context(self.app) as (f, _, _): conn = Mock() @@ -410,3 +442,55 @@ def test_django_setup(self, patching): f = self.Fixup(self.app) f.django_setup() django.setup.assert_called_with() + + def test__is_prefork(self): + with self.fixup_context(self.app) as (f, _, _): + f.worker.pool_cls = Mock(__module__='celery.concurrency.prefork') + assert f._is_prefork() + + f.worker.pool_cls = "prefork" + assert f._is_prefork() + + f.worker.pool_cls = Mock(__module__='celery.concurrency.thread') + assert not f._is_prefork() + + f.worker = None + assert not f._is_prefork() + + def test_no_recursive_worker_instantiation(self, patching): + """Regression test: DjangoWorkerFixup must not create a WorkController in __init__. + + Historically, DjangoWorkerFixup.__init__ instantiated a WorkController when + called with worker=None, which could cause recursive instantiation when + invoked from worker lifecycle signals. + + This test verifies the fixed behavior: + - DjangoWorkerFixup(app, worker=None) must not create a WorkController + - It should instead leave self.worker unset and rely on on_worker_init + to attach the actual worker instance later + """ + from celery.worker import WorkController + + patching('celery.fixups.django.symbol_by_name') + patching('celery.fixups.django.import_module') + patching.modules('django', 'django.db', 'django.core.checks') + + # Track WorkController instantiations + instantiation_count = {'count': 0} + original_init = WorkController.__init__ + + def tracking_init(self_worker, *args, **kwargs): + instantiation_count['count'] += 1 + return original_init(self_worker, *args, **kwargs) + + with patch.object(WorkController, '__init__', tracking_init): + # Creating DjangoWorkerFixup without a worker argument + # should NOT create a WorkController instance + DjangoWorkerFixup(self.app) + + # EXPECTED: 0 WorkController instances created + assert instantiation_count['count'] == 0, ( + f"DjangoWorkerFixup(app) should NOT create a WorkController, " + f"but {instantiation_count['count']} instance(s) were created. " + f"This is the root cause of the recursion bug." + ) diff --git a/tox.ini b/tox.ini index f0aaa95e30d..ef31b460ac6 100644 --- a/tox.ini +++ b/tox.ini @@ -84,7 +84,7 @@ setenv = dynamodb: AWS_SECRET_ACCESS_KEY=test_aws_secret_key azureblockblob: TEST_BROKER=redis:// - azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; + azureblockblob: TEST_BACKEND="azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" basepython = 3.9: python3.9 From 333a82f746734151c5cbe848916085455b0a3748 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 4 Jan 2026 14:11:56 +0200 Subject: [PATCH 104/169] Bugfix: Revoked tasks now immediately update backend status to REVOKED (#9869) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Potential fix for Issue #9844 * Update t/unit/worker/test_control.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Cleanup and docs --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- celery/worker/control.py | 7 +++++ docs/userguide/workers.rst | 10 ++++++- t/unit/worker/test_control.py | 55 ++++++++++++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/celery/worker/control.py b/celery/worker/control.py index 8f9fc4f92ba..547ce7989b6 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -216,6 +216,13 @@ def _revoke(state, task_ids, terminate=False, signal=None, **kwargs): terminated = set() worker_state.revoked.update(task_ids) + + for task_id in task_ids: + try: + state.app.backend.mark_as_revoked(task_id, reason='revoked', store_result=True) + except Exception as exc: + logger.warning('Failed to mark task %s as revoked in backend: %s', task_id, exc) + if terminate: signum = _signals.signum(signal or TERM_SIGNAME) for request in _find_requests_by_id(task_ids): diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index e665da489d9..024b3aa7078 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -108,7 +108,7 @@ Worker Shutdown We will use the terms *Warm, Soft, Cold, Hard* to describe the different stages of worker shutdown. The worker will initiate the shutdown process when it receives the :sig:`TERM` or :sig:`QUIT` signal. -The :sig:`INT` (Ctrl-C) signal is also handled during the shutdown process and always triggers the +The :sig:`INT` (Ctrl-C) signal is also handled during the shutdown process and always triggers the next stage of the shutdown process. .. _worker-warm-shutdown: @@ -544,6 +544,14 @@ Library. Terminating a task also revokes it. +.. versionchanged:: 5.6 + + When a task is revoked, the result backend is now immediately updated + to reflect the ``REVOKED`` status. Previously, the backend was only + updated when a worker attempted to process the revoked task, which + could leave tasks with ETA/countdown in ``PENDING`` status indefinitely + if the worker was shut down before the scheduled time. + **Example** .. code-block:: pycon diff --git a/t/unit/worker/test_control.py b/t/unit/worker/test_control.py index 6d7e923d2db..5f4c690f4b1 100644 --- a/t/unit/worker/test_control.py +++ b/t/unit/worker/test_control.py @@ -4,7 +4,7 @@ from collections import defaultdict from datetime import datetime, timedelta from queue import Queue as FastQueue -from unittest.mock import Mock, call, patch +from unittest.mock import Mock, PropertyMock, call, patch import pytest from kombu import pidbox @@ -815,3 +815,56 @@ def test_query_task(self): assert ret[req1.id][0] == 'reserved' finally: worker_state.reserved_requests.clear() + + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) + def test_revoke_backend_status_update(self): + state = self.create_state() + task_ids = ['task-1', 'task-2'] + + control._revoke(state, task_ids) + + assert self.app.backend.mark_as_revoked.call_count == 2 + calls = self.app.backend.mark_as_revoked.call_args_list + assert calls[0] == (('task-1',), {'reason': 'revoked', 'store_result': True}) + assert calls[1] == (('task-2',), {'reason': 'revoked', 'store_result': True}) + + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) + def test_revoke_backend_failure_defensive(self): + self.app.backend.mark_as_revoked.side_effect = Exception("Backend error") + state = self.create_state() + + control._revoke(state, ['task-1']) + + assert 'task-1' in worker_state.revoked + + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) + def test_revoke_terminate_backend_update(self): + state = self.create_state() + + with patch('celery.worker.control._find_requests_by_id', return_value=[]): + control._revoke(state, ['task-1'], terminate=True) + + self.app.backend.mark_as_revoked.assert_called_once_with( + 'task-1', reason='revoked', store_result=True + ) + + def test_revoke_by_stamped_headers_terminates_matching_request(self): + state = self.create_state() + state.consumer = Mock() + + request = Mock() + request.id = 'task-with-stamp' + request.stamps = {'monitoring_id': 'test-123'} + request.terminate = Mock() + + worker_state.active_requests.add(request) + + headers = {'monitoring_id': 'test-123'} + + with patch('celery.worker.control._signals.signum', return_value=15): + control.revoke_by_stamped_headers(state, headers, terminate=True) + + request.terminate.assert_called_once() + assert 'test-123' in worker_state.revoked_stamps.get('monitoring_id', []) + + worker_state.active_requests.clear() From 6a43c846f183ef0cbade24f4b9a8f7a6ea113b44 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 4 Jan 2026 14:35:33 +0200 Subject: [PATCH 105/169] Prepare for release: v5.6.2 (#10049) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.6.1 → 5.6.2 * Added Changelog for v5.6.2 --- .bumpversion.cfg | 2 +- Changelog.rst | 15 +++++++++++++++ README.rst | 2 +- celery/__init__.py | 2 +- docs/history/changelog-5.6.rst | 15 +++++++++++++++ docs/includes/introduction.txt | 2 +- 6 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d187658444a..429ef97e7aa 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.6.1 +current_version = 5.6.2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/Changelog.rst b/Changelog.rst index 9d7a21881ba..73bc9ba3671 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,21 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.2: + +5.6.2 +===== + +:release-date: 2026-01-04 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix recursive WorkController instantiation in DjangoWorkerFixup + AttributeError when pool_cls is a string (#10045) +- Bugfix: Revoked tasks now immediately update backend status to REVOKED (#9869) +- Prepare for release: v5.6.2 (#10049) + .. _version-5.6.1: 5.6.1 diff --git a/README.rst b/README.rst index a34a036d433..1fb99f91793 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.6.1 (recovery) +:Version: 5.6.2 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 2e45c0e5495..b89616415b5 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -17,7 +17,7 @@ SERIES = 'recovery' -__version__ = '5.6.1' +__version__ = '5.6.2' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst index d861ea56517..a56a1eb531d 100644 --- a/docs/history/changelog-5.6.rst +++ b/docs/history/changelog-5.6.rst @@ -8,6 +8,21 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.2: + +5.6.2 +===== + +:release-date: 2026-01-04 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix recursive WorkController instantiation in DjangoWorkerFixup + AttributeError when pool_cls is a string (#10045) +- Bugfix: Revoked tasks now immediately update backend status to REVOKED (#9869) +- Prepare for release: v5.6.2 (#10049) + .. _version-5.6.1: 5.6.1 diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 53187353a19..01663e64a6e 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.6.1 (recovery) +:Version: 5.6.2 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 87f117612ee578300c92a4f6eb7d38902ad14a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20Anh=20B=C3=ACnh?= Date: Tue, 6 Jan 2026 01:42:00 +0700 Subject: [PATCH 106/169] Fix Django worker recursion bug + defensive checks for pool_cls.__module__ (#10048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix recursive WorkController instantiation in DjangoWorkerFixup - Remove worker parameter from DjangoWorkerFixup.__init__ - Set worker via on_worker_init callback instead - Add None check for worker.pool_cls to prevent AttributeError - Add regression test to prevent future recursion bugs Fixes recursive instantiation issue where DjangoWorkerFixup(app) would create WorkController(app), which in turn would create another DjangoWorkerFixup, leading to infinite recursion. The worker instance is now properly set via the on_worker_init signal callback, avoiding the circular dependency. * fix: improve prefork detection and add integration test * fix: Copilot recommendations * Add defensive checks for pool_cls.__module__ in additional locations Extends the fix from #10045 by adding defensive checks for when pool_cls is a string instead of a class in two additional locations: - celery/contrib/testing/worker.py: TestWorkController.__init__ - celery/worker/components.py: Beat.create This prevents AttributeError: 'str' object has no attribute '__module__' when pool_cls is passed as a string (e.g., 'prefork', 'gevent', 'eventlet'). Also adds unit tests to verify the defensive checks work correctly. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add unit tests for TestWorkController with string pool_cls Add tests to verify defensive handling of string pool_cls in TestWorkController.__init__, addressing reviewer feedback. --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/contrib/testing/worker.py | 4 +++- celery/worker/components.py | 5 ++++- t/unit/contrib/test_worker.py | 37 ++++++++++++++++++++++++++++++++ t/unit/worker/test_components.py | 14 ++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/celery/contrib/testing/worker.py b/celery/contrib/testing/worker.py index 46eac75fd64..905a6a9ccdc 100644 --- a/celery/contrib/testing/worker.py +++ b/celery/contrib/testing/worker.py @@ -42,7 +42,9 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.pool_cls.__module__.split('.')[-1] == 'prefork': + # Defensive check: pool_cls may be a string (e.g., 'prefork') or a class + pool_module = self.pool_cls if isinstance(self.pool_cls, str) else self.pool_cls.__module__ + if pool_module.split('.')[-1] == 'prefork': from billiard import Queue self.logger_queue = Queue() self.pid = os.getpid() diff --git a/celery/worker/components.py b/celery/worker/components.py index d1ec1db2f3e..f60abe98a9c 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -194,7 +194,10 @@ def __init__(self, w, beat=False, **kwargs): def create(self, w): from celery.beat import EmbeddedService - if w.pool_cls.__module__.endswith(('gevent', 'eventlet')): + + # Defensive check: pool_cls may be a string (e.g., 'gevent') or a class + pool_module = w.pool_cls if isinstance(w.pool_cls, str) else w.pool_cls.__module__ + if pool_module.endswith(('gevent', 'eventlet')): raise ImproperlyConfigured(ERR_B_GREEN) b = w.beat = EmbeddedService(w.app, schedule_filename=w.schedule_filename, diff --git a/t/unit/contrib/test_worker.py b/t/unit/contrib/test_worker.py index 4534317ae83..fe4ac2ef8e5 100644 --- a/t/unit/contrib/test_worker.py +++ b/t/unit/contrib/test_worker.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock, patch + import pytest # this import adds a @shared_task, which uses connect_on_app_finalize @@ -57,3 +59,38 @@ def test_start_worker_with_hostname_config(self): result = self.add.s(1, 2).apply_async() val = result.get(timeout=5) assert val == 3 + + +class test_TestWorkController: + + @patch('celery.contrib.testing.worker.worker.WorkController.__init__') + def test_init_with_string_pool_cls_prefork(self, mock_super_init): + mock_super_init.return_value = None + controller = object.__new__(TestWorkController) + controller._on_started = None + controller.pool_cls = 'prefork' + with patch('celery.contrib.testing.worker.logging.handlers.QueueListener') as mock_listener: + with patch('billiard.Queue') as mock_queue: + with patch.dict('sys.modules', {'tblib': None, 'tblib.pickling_support': None}): + mock_queue.return_value = Mock() + controller.__init__(app=Mock()) + mock_listener.assert_called_once() + assert controller.logger_queue is not None + + @patch('celery.contrib.testing.worker.worker.WorkController.__init__') + def test_init_with_string_pool_cls_solo(self, mock_super_init): + mock_super_init.return_value = None + controller = object.__new__(TestWorkController) + controller._on_started = None + controller.pool_cls = 'solo' + controller.__init__(app=Mock()) + assert controller.logger_queue is None + + @patch('celery.contrib.testing.worker.worker.WorkController.__init__') + def test_init_with_string_pool_cls_gevent(self, mock_super_init): + mock_super_init.return_value = None + controller = object.__new__(TestWorkController) + controller._on_started = None + controller.pool_cls = 'gevent' + controller.__init__(app=Mock()) + assert controller.logger_queue is None diff --git a/t/unit/worker/test_components.py b/t/unit/worker/test_components.py index 739808e4311..ea43d3d5f75 100644 --- a/t/unit/worker/test_components.py +++ b/t/unit/worker/test_components.py @@ -89,3 +89,17 @@ def test_create__green(self): w.pool_cls.__module__ = 'foo_gevent' with pytest.raises(ImproperlyConfigured): Beat(w).create(w) + + def test_create__green_string_pool_cls(self): + """Test Beat.create raises ImproperlyConfigured when pool_cls is a string like 'gevent'.""" + w = Mock(name='w') + w.pool_cls = 'gevent' # pool_cls can be a string instead of a class + with pytest.raises(ImproperlyConfigured): + Beat(w).create(w) + + def test_create__green_string_pool_cls_eventlet(self): + """Test Beat.create raises ImproperlyConfigured when pool_cls is 'eventlet'.""" + w = Mock(name='w') + w.pool_cls = 'eventlet' + with pytest.raises(ImproperlyConfigured): + Beat(w).create(w) From 033983029d40c6d4f6da617cee33c4626e2d5649 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Jan 2026 00:45:32 +0200 Subject: [PATCH 107/169] Update elasticsearch requirement from <=9.1.2 to <=9.2.1 (#10053) Updates the requirements on [elasticsearch](https://github.com/elastic/elasticsearch-py) to permit the latest version. - [Release notes](https://github.com/elastic/elasticsearch-py/releases) - [Commits](https://github.com/elastic/elasticsearch-py/compare/0.4.1...v9.2.1) --- updated-dependencies: - dependency-name: elasticsearch dependency-version: 9.2.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/extras/elasticsearch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index 605852ada53..80e152095b1 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ -elasticsearch<=9.1.2 +elasticsearch<=9.2.1 elastic-transport<=9.1.0 From 1780e9fbfd32579a918d39c48d9bbf3b0e7222bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Asif=20Saif=20Uddin=20=7B=22Auvi=22=3A=22=E0=A6=85?= =?UTF-8?q?=E0=A6=AD=E0=A6=BF=22=7D?= Date: Tue, 6 Jan 2026 12:42:40 +0000 Subject: [PATCH 108/169] Revert "Update elasticsearch requirement from <=9.1.2 to <=9.2.1 (#10053)" (#10054) This reverts commit 7c6c3bd3254e6326d718c1f12289c26c6ee9c1a5. --- requirements/extras/elasticsearch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index 80e152095b1..605852ada53 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ -elasticsearch<=9.2.1 +elasticsearch<=9.1.2 elastic-transport<=9.1.0 From 3442e3bf52c8526a84b4be5a3366453ffaaf5ae0 Mon Sep 17 00:00:00 2001 From: Vladislav Trubchik <128217623+jorsyk@users.noreply.github.com> Date: Sat, 10 Jan 2026 20:44:30 +0300 Subject: [PATCH 109/169] Docs: Update user_preload_options example to use click. (#10056) * git commit -m "Docs: Fix user_preload_options example (replace deprecated Option with click.option)" * Corrections based on comments --- docs/userguide/signals.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 7aeea8adbf8..35b57680cde 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -820,22 +820,25 @@ It can be used to add additional command-line arguments to the .. code-block:: python - from celery import Celery - from celery import signals - from celery.bin.base import Option + from celery import Celery, signals + from click import Option app = Celery() + + # Celery 5.0+ uses click for its command-line interface. + # Use click.option to add new command-line arguments. app.user_options['preload'].add(Option( - '--monitoring', action='store_true', + ('--monitoring',), is_flag=True, help='Enable our external monitoring utility, blahblah', )) @signals.user_preload_options.connect def handle_preload_options(options, **kwargs): - if options['monitoring']: + if options.get('monitoring'): enable_monitoring() + Sender is the :class:`~celery.bin.base.Command` instance, and the value depends on the program that was called (e.g., for the umbrella command it'll be a :class:`~celery.bin.celery.CeleryCommand`) object). From 16d7ae10864c21f35ae8b394ada4724500b9a3c6 Mon Sep 17 00:00:00 2001 From: Vladislav Trubchik <128217623+jorsyk@users.noreply.github.com> Date: Sat, 10 Jan 2026 21:43:35 +0300 Subject: [PATCH 110/169] Fix invalid configuration key "bootstrap_servers" in Kafka demo (#10060) * git commit -m "Docs: Fix user_preload_options example (replace deprecated Option with click.option)" * Corrections based on comments * Fix: change bootstrap_servers to bootstrap.servers in Kafka config --- docs/getting-started/backends-and-brokers/kafka.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/backends-and-brokers/kafka.rst b/docs/getting-started/backends-and-brokers/kafka.rst index e5b0ea0b68e..eb6d540532c 100644 --- a/docs/getting-started/backends-and-brokers/kafka.rst +++ b/docs/getting-started/backends-and-brokers/kafka.rst @@ -41,7 +41,7 @@ For celeryconfig.py: "sasl.password": sasl_password, "security.protocol": "SASL_SSL", "sasl.mechanism": "SCRAM-SHA-512", - "bootstrap_servers": "broker:9094", + "bootstrap.servers": "broker:9094", } }) From 83f85e639107f9351cb0543ded888915062b5c31 Mon Sep 17 00:00:00 2001 From: Ilyas Timour Date: Sat, 17 Jan 2026 06:57:45 +0100 Subject: [PATCH 111/169] Fix broken images on PyPI page (#10066) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 1fb99f91793..a71e7c74f7a 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Sponsors Blacksmith ---------- -.. image:: ./docs/images/blacksmith-logo-white-on-black.svg +.. image:: https://github.com/celery/celery/blob/main/docs/images/blacksmith-logo-white-on-black.svg :alt: Blacksmith logo :width: 240px :target: https://blacksmith.sh/ @@ -47,7 +47,7 @@ Blacksmith CloudAMQP --------- -.. image:: ./docs/images/cloudamqp-logo-lightbg.svg +.. image:: https://github.com/celery/celery/blob/main/docs/images/cloudamqp-logo-lightbg.svg :alt: CloudAMQP logo :width: 240px :target: https://www.cloudamqp.com/ From bb7eb389b237aaa9851d502e457ebce1432dec2d Mon Sep 17 00:00:00 2001 From: sue Date: Thu, 22 Jan 2026 06:03:54 -0500 Subject: [PATCH 112/169] Remove broken reference. --- README.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.rst b/README.rst index a71e7c74f7a..2d8c0d6e8a8 100644 --- a/README.rst +++ b/README.rst @@ -536,11 +536,6 @@ documentation. .. _`Contributing to Celery`: https://docs.celeryq.dev/en/stable/contributing.html -|oc-contributors| - -.. |oc-contributors| image:: https://opencollective.com/celery/contributors.svg?width=890&button=false - :target: https://github.com/celery/celery/graphs/contributors - Backers ------- From e2f9d3b814b8493527d2d506f9ee8648297a77fb Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Sun, 25 Jan 2026 19:25:47 +0200 Subject: [PATCH 113/169] Removed --dist=loadscope from smoke tests (#10073) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ef31b460ac6..19697da9e1b 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ commands = unit: coverage xml unit: coverage report integration: pytest -xsvv t/integration {posargs} - smoke: pytest -xsvv t/smoke --dist=loadscope --reruns 5 --reruns-delay 10 {posargs} + smoke: pytest -xsvv t/smoke --reruns 5 --reruns-delay 10 {posargs} setenv = PIP_EXTRA_INDEX_URL=https://celery.github.io/celery-wheelhouse/repo/simple/ BOTO_CONFIG = /dev/null From f78be8032923e56e9f669e33edd1f81e207caa16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=84=8B=E1=85=B5=E1=84=80=E1=85=A1=E1=86=BC=E1=84=8B?= =?UTF-8?q?=E1=85=B3=E1=86=AB?= Date: Wed, 28 Jan 2026 00:54:03 +0900 Subject: [PATCH 114/169] Docs: Clarify task_retry signal args may be None (#9851) Add a note to the task_retry signal documentation clarifying that only the request argument is guaranteed to be provided. The reason and einfo arguments may be None or not provided in certain scenarios, such as when a task is cancelled and retried. Fixes #9851 --- docs/userguide/signals.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 35b57680cde..28bc988c481 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -239,6 +239,12 @@ Provides arguments: Detailed exception information, including traceback (a :class:`billiard.einfo.ExceptionInfo` object). +.. note:: + + Only the ``request`` argument is guaranteed to be provided in all cases. + The ``reason`` and ``einfo`` arguments may be ``None`` or not provided + in certain scenarios, such as when a task is cancelled and retried. + Signal handlers should not assume these arguments are always present. .. signal:: task_success From 6f17d8badbdbde95679a07b2d6e1f7cf247157e7 Mon Sep 17 00:00:00 2001 From: sbc-khacnha Date: Tue, 3 Feb 2026 17:13:58 +0700 Subject: [PATCH 115/169] Update example for Django (#10081) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * example - Update tasks.py demo * Update examples/django/demoapp/tasks.py * Update examples/django/demoapp/tasks.py * Update examples/django/demoapp/tasks.py --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- examples/django/demoapp/tasks.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/examples/django/demoapp/tasks.py b/examples/django/demoapp/tasks.py index c16b76b4c4f..16281a1b55e 100644 --- a/examples/django/demoapp/tasks.py +++ b/examples/django/demoapp/tasks.py @@ -30,3 +30,23 @@ def rename_widget(widget_id, name): w = Widget.objects.get(id=widget_id) w.name = name w.save() + + +@shared_task( + bind=True, + autoretry_for=(Exception,), + retry_kwargs={"max_retries": 2, "countdown": 10 * 60}, # retry up to 2 times with 10 minutes between retries +) +def error_task(self): + raise Exception("Test error") + + +@shared_task( + bind=True, + autoretry_for=(Exception,), + retry_backoff=5, # Factor in seconds (first retry: 5s, second: 10s, third: 20s, etc.) + retry_jitter=False, # Set False to disable randomization (use exact values: 5s, 10s, 20s) + retry_kwargs={"max_retries": 3}, +) +def error_backoff_test(self): + raise Exception("Test error") From e35c2e679c2449156d9dee627511ef2a1bd58ed4 Mon Sep 17 00:00:00 2001 From: Colin Watson Date: Wed, 4 Feb 2026 13:22:17 +0000 Subject: [PATCH 116/169] Make tests compatible with pymongo >= 4.16 (#10074) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `InvalidDocument` now requires a `message` argument; it was previously optional. Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- t/unit/backends/test_mongodb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index 0c29111654b..075ce3d4862 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -390,7 +390,7 @@ def test_store_result(self, mock_get_database): upsert=True) assert sentinel.result == ret_val - mock_collection.replace_one.side_effect = InvalidDocument() + mock_collection.replace_one.side_effect = InvalidDocument("bad") with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) @@ -417,7 +417,7 @@ def test_store_result_with_request(self, mock_get_database): assert parameters['parent_id'] == sentinel.parent_id assert sentinel.result == ret_val - mock_collection.replace_one.side_effect = InvalidDocument() + mock_collection.replace_one.side_effect = InvalidDocument("bad") with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) From 54b7a41d38e47c191a38e557e2047eb5967eeb61 Mon Sep 17 00:00:00 2001 From: Isabelle COWAN-BERGMAN Date: Wed, 11 Feb 2026 10:56:05 +0100 Subject: [PATCH 117/169] fix: source install of cassandra-driver (#10105) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: source install of cassandra-driver * fix: rename build-constraints.txt to constraints.txt * fix: apply constraints to docker build * fix: switch to using PIP_CONSTRAINTS environment variable * fix: change install command to pass build-constraint * fix: pass --build-constraint in docker build * Update requirements/constraints.txt Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Revert "Update requirements/constraints.txt" This reverts commit 16b08b86ded0a3bc20fbec9489ca160d7c93ff4c. * Revert "Apply suggestions from code review" This reverts commit 6e60cc7c9374286ef9342e975370fc5a30f9739d. --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docker/Dockerfile | 18 ++++++++++++------ requirements/constraints.txt | 3 +++ tox.ini | 2 ++ 3 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 requirements/constraints.txt diff --git a/docker/Dockerfile b/docker/Dockerfile index 5c8bcf50902..cebba9314ac 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -104,7 +104,8 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test.txt + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.12 -m pip install -r requirements/default.txt \ @@ -114,7 +115,8 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test.txt + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.11 -m pip install -r requirements/default.txt \ @@ -124,7 +126,8 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test.txt + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.10 -m pip install -r requirements/default.txt \ @@ -134,7 +137,8 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test.txt + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec python3.9 -m pip install -r requirements/default.txt \ @@ -144,7 +148,8 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test.txt + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ pyenv exec pypy3.11 -m pip install -r requirements/default.txt \ @@ -154,7 +159,8 @@ RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test.txt + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt COPY --chown=1000:1000 . $HOME/celery diff --git a/requirements/constraints.txt b/requirements/constraints.txt new file mode 100644 index 00000000000..0972b9f80cd --- /dev/null +++ b/requirements/constraints.txt @@ -0,0 +1,3 @@ +# Source install of cassandra-driver is broken in setuptools >=82.0.0, see +# https://github.com/apache/cassandra-python-driver/pull/1268 +setuptools<82.0.0 diff --git a/tox.ini b/tox.ini index 19697da9e1b..4cf856837d6 100644 --- a/tox.ini +++ b/tox.ini @@ -43,6 +43,8 @@ deps= lint: pre-commit bandit: bandit +install_command = python -I -m pip install {opts} {packages} --build-constraint {toxinidir}/requirements/constraints.txt + commands = unit: coverage run --source=celery -m pytest -vv --maxfail=10 --capture=no -v --junitxml=junit.xml -o junit_family=legacy {posargs} unit: coverage xml From 5d0567765a8e8533b34f2ed5ba4469a52467ad21 Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Wed, 11 Feb 2026 03:56:35 -0800 Subject: [PATCH 118/169] fix: register task cross-reference role in Sphinx extension (#10100) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: register task cross-reference role in Sphinx extension The Sphinx extension registered the task directive but not the corresponding role, causing 'Unknown interpreted text role "task"' errors when using :task:`my.task.function` syntax in documentation. Register PyXRefRole with fix_parens=True so task references render with parentheses like function references. Fixes #9926 * Add regression test for :task: cross-reference role Add :task:`foo.bar` usage to the test fixture RST and assert the cross-reference renders in the generated HTML output. --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/contrib/sphinx.py | 3 ++- t/unit/contrib/proj/contents.rst | 2 ++ t/unit/contrib/test_sphinx.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/celery/contrib/sphinx.py b/celery/contrib/sphinx.py index 4cdeb2cb2d4..0b1e6389af9 100644 --- a/celery/contrib/sphinx.py +++ b/celery/contrib/sphinx.py @@ -49,7 +49,7 @@ from inspect import signature from docutils import nodes -from sphinx.domains.python import PyFunction +from sphinx.domains.python import PyFunction, PyXRefRole from sphinx.ext.autodoc import FunctionDocumenter from celery.app.task import BaseTask @@ -130,6 +130,7 @@ def setup(app): app.add_autodocumenter(TaskDocumenter) app.add_directive_to_domain('py', 'task', TaskDirective) + app.add_role_to_domain('py', 'task', PyXRefRole(fix_parens=True)) app.add_config_value('celery_task_prefix', '(task)', True) app.connect('autodoc-skip-member', autodoc_skip_member_handler) diff --git a/t/unit/contrib/proj/contents.rst b/t/unit/contrib/proj/contents.rst index 5ba93e82eba..43701b1a21b 100644 --- a/t/unit/contrib/proj/contents.rst +++ b/t/unit/contrib/proj/contents.rst @@ -5,3 +5,5 @@ Documentation .. automodule:: foo :members: + +Cross-reference test: :task:`foo.bar` diff --git a/t/unit/contrib/test_sphinx.py b/t/unit/contrib/test_sphinx.py index 0a5abceab91..dcdceb16562 100644 --- a/t/unit/contrib/test_sphinx.py +++ b/t/unit/contrib/test_sphinx.py @@ -28,3 +28,6 @@ def test_sphinx(): 'This task is in a different module!' not in contents ) + # Verify :task: cross-reference role resolves to a link + assert 'foo.bar' in contents + assert 'Cross-reference test' in contents From f2e37f016d90b5f352452460ebde68b2e022b9e5 Mon Sep 17 00:00:00 2001 From: Isabelle COWAN-BERGMAN Date: Wed, 11 Feb 2026 13:30:18 +0100 Subject: [PATCH 119/169] fix: avoid cycle detection in native delayed delivery (#10095) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: avoid cycle detection in native delayed delivery * Avoid RabbitMQ <4.0.1 dead-lettering cycle detection when retrying with native delayed-delivery. * chore: fix typo, rename, unused variable --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/task.py | 21 ++- ...t_rabbitmq_quorum_queue_cycle_detection.py | 148 ++++++++++++++++++ t/unit/tasks/test_tasks.py | 31 ++++ 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 t/integration/test_rabbitmq_quorum_queue_cycle_detection.py diff --git a/celery/app/task.py b/celery/app/task.py index 3ab54ad623e..79cf9cb7f4a 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -36,6 +36,17 @@ R_UNBOUND_TASK = '' R_INSTANCE = '<@task: {0.name} of {app}{flags}>' +# Filtered headers relating to dead-lettering in RabbitMQ. +X_DEATH_HEADERS = { + 'x-death', + 'x-first-death-exchange', + 'x-first-death-queue', + 'x-first-death-reason', + 'x-last-death-exchange', + 'x-last-death-queue', + 'x-last-death-reason', +} + #: Here for backwards compatibility as tasks no longer use a custom meta-class. TaskType = type @@ -123,6 +134,14 @@ def get(self, key, default=None): def __repr__(self): return f'' + def _filter_x_death_headers(self, headers): + """Filter out X-Death headers to prevent RabbitMQ cycle detection.""" + headers = headers.copy() if headers else {} + for x_death_header in X_DEATH_HEADERS: + headers.pop(x_death_header, None) + + return headers + def as_execution_options(self): limit_hard, limit_soft = self.timelimit or (None, None) execution_options = { @@ -139,7 +158,7 @@ def as_execution_options(self): 'expires': self.expires, 'soft_time_limit': limit_soft, 'time_limit': limit_hard, - 'headers': self.headers, + 'headers': self._filter_x_death_headers(self.headers), 'retries': self.retries, 'reply_to': self.reply_to, 'replaced_task_nesting': self.replaced_task_nesting, diff --git a/t/integration/test_rabbitmq_quorum_queue_cycle_detection.py b/t/integration/test_rabbitmq_quorum_queue_cycle_detection.py new file mode 100644 index 00000000000..f0bed95afc5 --- /dev/null +++ b/t/integration/test_rabbitmq_quorum_queue_cycle_detection.py @@ -0,0 +1,148 @@ +""" +Integration tests for RabbitMQ cycle detection bug with quorum queues. + +This reproduces the conditions for +https://github.com/celery/celery/issues/9867 when running against RabbitMQ +<4.0.1. +""" + +import os + +import pytest +from kombu import Exchange, Queue + +from celery import Celery, exceptions +from celery.contrib.testing.worker import start_worker + + +class TaskFailedException(Exception): + """Test exception for tasks that should fail and retry.""" + pass + + +def get_rabbitmq_url(): + """Get RabbitMQ broker URL from environment.""" + user = os.environ.get("RABBITMQ_DEFAULT_USER", "guest") + password = os.environ.get("RABBITMQ_DEFAULT_PASSWORD", "guest") + return os.environ.get( + "TEST_BROKER", f"pyamqp://{user}:{password}@localhost:5672//") + + +def get_backend_url(): + """Get backend URL from environment.""" + return os.environ.get("TEST_BACKEND", "rpc") + + +def create_test_app_with_quorum_queues(): + """ + Create Celery app configured for quorum queue testing. + + Returns: + Tuple of (app, queue_name) + """ + broker_url = get_rabbitmq_url() + backend_url = get_backend_url() + + app = Celery( + "test_cycle_detection", + broker=broker_url, + backend=backend_url, + ) + + # Configure queue with quorum queue type + queue_name = 'test_cycle_queue' + default_exchange = Exchange('celery.topic', type='topic') + + app.conf.task_queues = [ + Queue( + queue_name, + exchange=default_exchange, + routing_key=queue_name, + queue_arguments={'x-queue-type': 'quorum'} + ), + ] + + app.conf.task_default_queue = queue_name + + # Recommended settings for quorum queues + app.conf.broker_transport_options = {"confirm_publish": True} + + # Enable result extended to track retries + app.conf.result_extended = True + + return app + + +@pytest.mark.amqp +@pytest.mark.timeout(35) +def test_countdown_task_with_retry(): + """Test where a countdown task gets retried with the same delay.""" + app = create_test_app_with_quorum_queues() + + @app.task(bind=True, default_retry_delay=3, max_retries=1) + def countdown_task_with_retry(self): + """Task that fails once then succeeds.""" + # First attempt: fail to trigger retry + if self.request.retries < 1: + raise self.retry(exc=TaskFailedException("Simulated failure")) + + return self.request.retries + + with start_worker( + app, + loglevel="INFO", + shutdown_timeout=15, + ): + # Execute task with countdown=3 (same as default_retry_delay) + # We need to hit the same delayed buckets to trigger the cycle detection + # in RabbitMQ. + result = countdown_task_with_retry.apply_async(countdown=3) + + try: + # Wait for the task to complete, we must wait at least long enough + # for the countdown to be reached and the retry to occur (6s). + value = result.get(timeout=15) + except exceptions.TimeoutError as e: + pytest.fail( + f"Task was silently dropped by RabbitMQ due to cycle detection. " + f"Exception: {e!r}." + ) + + assert value == 1, \ + f"Expected task to succeed after 1 retry, got: {value}" + + +@pytest.mark.amqp +@pytest.mark.timeout(35) +def test_non_eta_task_with_multiple_retries(): + """Test where a task gets retried twice with the same delay.""" + app = create_test_app_with_quorum_queues() + + @app.task(bind=True, max_retries=2) + def non_eta_task_with_retries(self): + """Task that fails twice then succeeds.""" + # First and second attempts: fail to trigger retry + if self.request.retries < 2: + raise self.retry(exc=TaskFailedException("Simulated failure"), countdown=3) + + return self.request.retries + + with start_worker( + app, + loglevel="INFO", + shutdown_timeout=15, + ): + result = non_eta_task_with_retries.apply_async() + + try: + # Wait for the task to complete, we must wait at least long enough + # for the two retries to occur (6s). + value = result.get(timeout=15) + except exceptions.TimeoutError as e: + pytest.fail( + f"Task was silently dropped by RabbitMQ due to cycle detection. " + f"Exception: {e!r}" + ) + + assert value == 2, \ + f"Expected task to succeed after 2 retries, got: {value}" diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 720394641c8..13b39739e5f 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -424,6 +424,37 @@ def test_signature_from_request__passes_headers(self): sig = self.retry_task.signature_from_request() assert sig.options['headers']['custom'] == 10.1 + def test_signature_from_request__filters_x_death_headers(self): + """ + Test that X-Death headers are filtered out during retries to prevent + RabbitMQ cycle detection. + """ + + self.retry_task.push_request() + self.retry_task.request.headers = { + 'custom': 10.1, + 'x-death': [{'count': 1, 'queue': 'celery_delayed_0'}], + 'x-first-death-exchange': 'celery_delayed_0', + 'x-first-death-queue': 'celery_delayed_0', + 'x-first-death-reason': 'expired', + 'x-last-death-exchange': 'celery_delayed_0', + 'x-last-death-queue': 'celery_delayed_0', + 'x-last-death-reason': 'expired', + } + sig = self.retry_task.signature_from_request() + + # Custom headers should be preserved + assert sig.options['headers']['custom'] == 10.1 + + # X-Death related headers should be filtered out + assert 'x-death' not in sig.options['headers'] + assert 'x-first-death-exchange' not in sig.options['headers'] + assert 'x-first-death-queue' not in sig.options['headers'] + assert 'x-first-death-reason' not in sig.options['headers'] + assert 'x-last-death-exchange' not in sig.options['headers'] + assert 'x-last-death-queue' not in sig.options['headers'] + assert 'x-last-death-reason' not in sig.options['headers'] + def test_signature_from_request__delivery_info(self): self.retry_task.push_request() self.retry_task.request.delivery_info = { From e78bc780111763cf8a9eced2b3cb0f642688427e Mon Sep 17 00:00:00 2001 From: Matthew Riddle Date: Sat, 14 Feb 2026 08:42:42 +0100 Subject: [PATCH 120/169] fix(asynpool): avoid AttributeError when proc lacks _sentinel_poll (#10086) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use getattr for safe access to _sentinel_poll in _untrack_child_process. A race condition during cold shutdown can cause this method to be called with a process that never had _sentinel_poll set or had it cleared. Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/concurrency/asynpool.py | 9 +++++---- t/unit/concurrency/test_prefork.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index a55542e6573..30a761f43c4 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -513,10 +513,11 @@ def _track_child_process(self, proc, hub): self._event_process_exit, hub, proc) def _untrack_child_process(self, proc, hub): - if proc._sentinel_poll is not None: - fd, proc._sentinel_poll = proc._sentinel_poll, None - hub.remove(fd) - os.close(fd) + sentinel_poll = getattr(proc, '_sentinel_poll', None) + if sentinel_poll is not None: + proc._sentinel_poll = None + hub.remove(sentinel_poll) + os.close(sentinel_poll) def register_with_event_loop(self, hub): """Register the async pool with the current event loop.""" diff --git a/t/unit/concurrency/test_prefork.py b/t/unit/concurrency/test_prefork.py index f72655c0e48..51b5214216a 100644 --- a/t/unit/concurrency/test_prefork.py +++ b/t/unit/concurrency/test_prefork.py @@ -488,6 +488,31 @@ def test_before_create_process_signal(self, create_process): sender=pool, ) + def test_untrack_child_process_without_sentinel_poll(self): + """_untrack_child_process must not raise when proc lacks _sentinel_poll. + + Race condition during cold shutdown can cause _untrack_child_process to + be called with a process that never had _sentinel_poll set or had it + cleared. Use getattr for safe access. + """ + pytest.importorskip('multiprocessing') + pool = asynpool.AsynPool(processes=1, threads=False) + hub = Mock(name='hub') + proc = object() # No _sentinel_poll attribute + pool._untrack_child_process(proc, hub) # Should not raise AttributeError + hub.remove.assert_not_called() + + def test_untrack_child_process_with_sentinel_poll(self): + """_untrack_child_process cleans up when proc has _sentinel_poll set.""" + pytest.importorskip('multiprocessing') + pool = asynpool.AsynPool(processes=1, threads=False) + hub = Mock(name='hub') + fd = os.open(os.devnull, os.O_RDONLY) + proc = Mock(_sentinel_poll=fd) + pool._untrack_child_process(proc, hub) + hub.remove.assert_called_once_with(fd) + assert proc._sentinel_poll is None + @t.skip.if_win32 class test_ResultHandler: From f176fc6f2c6b9c831494b01c0d98c8ffcacb2b6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Feb 2026 13:45:54 +0600 Subject: [PATCH 121/169] Update elastic-transport requirement from <=9.1.0 to <=9.2.1 (#10052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates the requirements on [elastic-transport](https://github.com/elastic/elastic-transport-python) to permit the latest version. - [Release notes](https://github.com/elastic/elastic-transport-python/releases) - [Changelog](https://github.com/elastic/elastic-transport-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/elastic/elastic-transport-python/compare/0.1.0b0...v9.2.1) --- updated-dependencies: - dependency-name: elastic-transport dependency-version: 9.2.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/elasticsearch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index 605852ada53..ca70a7df23d 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ elasticsearch<=9.1.2 -elastic-transport<=9.1.0 +elastic-transport<=9.2.1 From fde34c8d40f44a81e413be1694bb689c42910350 Mon Sep 17 00:00:00 2001 From: Harikrishna KP Date: Sat, 14 Feb 2026 13:21:27 +0530 Subject: [PATCH 122/169] fix dusk_astronomical horizon set to +18 instead of -18 (#10121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/schedules.py | 2 +- t/unit/app/test_schedules.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/celery/schedules.py b/celery/schedules.py index 010b3396fa8..dc6cb695394 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -764,7 +764,7 @@ class solar(BaseSchedule): 'sunset': '-0:34', 'dusk_civil': '-6', 'dusk_nautical': '-12', - 'dusk_astronomical': '18', + 'dusk_astronomical': '-18', } _methods = { 'dawn_astronomical': 'next_rising', diff --git a/t/unit/app/test_schedules.py b/t/unit/app/test_schedules.py index 63689831bdf..924a72d3360 100644 --- a/t/unit/app/test_schedules.py +++ b/t/unit/app/test_schedules.py @@ -76,6 +76,16 @@ def test_invalid_event(self): with pytest.raises(ValueError): solar('asdqwewqew', 60, 60, app=self.app) + def test_dusk_horizons_are_negative(self): + """All dusk events should have negative horizons (sun below horizon).""" + for event in ('dusk_civil', 'dusk_nautical', 'dusk_astronomical'): + s = solar(event, 50, 10, app=self.app) + horizon = float(s.cal.horizon) + assert horizon < 0, ( + f"{event} horizon should be negative (below horizon), " + f"got {s.cal.horizon}" + ) + def test_event_uses_center(self): s = solar('solar_noon', 60, 60, app=self.app) for ev, is_center in s._use_center_l.items(): From 8aa1a8080511cd21b629dbffb2df98b75e47b6d5 Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Sun, 15 Feb 2026 21:56:12 +0800 Subject: [PATCH 123/169] Fix/10106 onupdate col use lambda func (#10108) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: use lambda function in date_done col * test: date_done col is a callable function * refactor: extract the function for better control * fix: check default is not None for better interpret --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/database/models.py | 17 ++++++++++++--- t/unit/backends/test_database.py | 34 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/celery/backends/database/models.py b/celery/backends/database/models.py index ddc18747bac..ff193f7aeb0 100644 --- a/celery/backends/database/models.py +++ b/celery/backends/database/models.py @@ -10,9 +10,20 @@ __all__ = ('Task', 'TaskExtended', 'TaskSet') + DialectSpecificInteger = sa.Integer().with_variant(sa.BigInteger, 'mssql') +def _get_utc_now(): + """Return current UTC datetime. + + This helper is used as a callable for SQLAlchemy column defaults + to ensure the timestamp is evaluated at INSERT/UPDATE time, + not at module import time. + """ + return datetime.now(timezone.utc) + + class Task(ResultModelBase): """Task result/status.""" @@ -24,8 +35,8 @@ class Task(ResultModelBase): task_id = sa.Column(sa.String(155), unique=True) status = sa.Column(sa.String(50), default=states.PENDING) result = sa.Column(PickleType, nullable=True) - date_done = sa.Column(sa.DateTime, default=datetime.now(timezone.utc), - onupdate=datetime.now(timezone.utc), nullable=True) + date_done = sa.Column(sa.DateTime, default=_get_utc_now, + onupdate=_get_utc_now, nullable=True) traceback = sa.Column(sa.Text, nullable=True) def __init__(self, task_id): @@ -86,7 +97,7 @@ class TaskSet(ResultModelBase): autoincrement=True, primary_key=True) taskset_id = sa.Column(sa.String(155), unique=True) result = sa.Column(PickleType, nullable=True) - date_done = sa.Column(sa.DateTime, default=datetime.now(timezone.utc), + date_done = sa.Column(sa.DateTime, default=_get_utc_now, nullable=True) def __init__(self, taskset_id, result): diff --git a/t/unit/backends/test_database.py b/t/unit/backends/test_database.py index 2a738731c07..d59df1cbc8f 100644 --- a/t/unit/backends/test_database.py +++ b/t/unit/backends/test_database.py @@ -65,6 +65,40 @@ def test_for_mssql_dialect(self): assert isinstance(compiled_type, Integer) +@skip.if_pypy +class test_DateDoneColumnDefaults: + """Test that date_done column defaults are callables, not fixed values. + + The default and onupdate values must be callables (lambdas) so that + datetime.now() is evaluated per-row at INSERT/UPDATE time, not once + at module import time. + """ + + def test_task_date_done_default_is_callable(self): + """Task.date_done default should be a callable.""" + col = Task.__table__.columns['date_done'] + assert col.default is not None, \ + "Task.date_done should have a default" + assert callable(col.default.arg), \ + "Task.date_done default should be a callable, not a fixed datetime" + + def test_task_date_done_onupdate_is_callable(self): + """Task.date_done onupdate should be a callable (lambda).""" + col = Task.__table__.columns['date_done'] + assert col.onupdate is not None, \ + "Task.date_done should have an onupdate" + assert callable(col.onupdate.arg), \ + "Task.date_done onupdate should be a callable, not a fixed datetime" + + def test_taskset_date_done_default_is_callable(self): + """TaskSet.date_done default should be a callable.""" + col = TaskSet.__table__.columns['date_done'] + assert col.default is not None, \ + "TaskSet.date_done should have a default" + assert callable(col.default.arg), \ + "TaskSet.date_done default should be a callable, not a fixed datetime" + + @skip.if_pypy class test_DatabaseBackend: From 2ac70700b9a0404df6de689ee631cf84e904ee1f Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Mon, 16 Feb 2026 00:27:04 +0800 Subject: [PATCH 124/169] Fix warm shutdown RuntimeError with eventlet>=0.37.0 (#10083) (#10123) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: save the original os write before import eventlet * test: safe say use original os write --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/__init__.py | 8 ++++++++ celery/apps/worker.py | 10 ++++++++-- t/unit/worker/test_worker.py | 23 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/celery/__init__.py b/celery/__init__.py index b89616415b5..36bdb65bc67 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -15,6 +15,13 @@ # Lazy loading from . import local +# Save original os.write before eventlet/gevent can monkey-patch it. +# This is needed for signal handlers (e.g., SIGINT) which may run inside +# the eventlet hub's event loop. Using the patched os.write from within +# the hub causes: RuntimeError('do not call blocking functions from the mainloop') +# See: https://github.com/celery/celery/issues/10083 +_original_os_write = os.write + SERIES = 'recovery' __version__ = '5.6.2' @@ -169,4 +176,5 @@ def maybe_patch_concurrency(argv=None, short_opts=None, version_info=version_info, maybe_patch_concurrency=maybe_patch_concurrency, _find_option_with_arg=_find_option_with_arg, + _original_os_write=_original_os_write, ) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 7286d4b8543..0a49a878909 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -17,7 +17,7 @@ from billiard.process import current_process from kombu.utils.encoding import safe_str -from celery import VERSION_BANNER, platforms, signals +from celery import VERSION_BANNER, _original_os_write, platforms, signals from celery.app import trace from celery.loaders.app import AppLoader from celery.platforms import EX_FAILURE, EX_OK, check_privileges, isatty @@ -78,8 +78,14 @@ def active_thread_count(): def safe_say(msg, f=sys.__stderr__): + """ + Uses the original (unpatched) os.write to avoid issues with eventlet/gevent + monkey-patching. When using eventlet>=0.37.0, the patched os.write calls + hubs.trampoline() which raises RuntimeError if called from within the + hub's event loop (e.g., during signal handling). + """ if hasattr(f, 'fileno') and f.fileno() is not None: - os.write(f.fileno(), f'\n{msg}\n'.encode()) + _original_os_write(f.fileno(), f'\n{msg}\n'.encode()) class Worker(WorkController): diff --git a/t/unit/worker/test_worker.py b/t/unit/worker/test_worker.py index c14c3c89f55..f0bc1ca8c5f 100644 --- a/t/unit/worker/test_worker.py +++ b/t/unit/worker/test_worker.py @@ -1242,3 +1242,26 @@ def test_safe_say_writes_to_std_out(self, capfd): captured = capfd.readouterr() assert "\nout\n" == captured.out assert "" == captured.err + + def test_safe_say_uses_original_os_write(self): + from celery import _original_os_write + from celery.apps.worker import _original_os_write as worker_os_write + + assert _original_os_write is not None + assert callable(_original_os_write) + assert worker_os_write is _original_os_write + assert _original_os_write.__name__ == 'write' + + def test_safe_say_works_with_patched_os_write(self, capfd): + original_write = os.write + + def patched_write(fd, data): + raise RuntimeError("do not call blocking functions from the mainloop") + + try: + os.write = patched_write + safe_say("test message") + captured = capfd.readouterr() + assert "\ntest message\n" == captured.err + finally: + os.write = original_write From 9df38ec3e4257c52e60a29f384c80fba7ae6f1a2 Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Mon, 16 Feb 2026 23:12:31 +0800 Subject: [PATCH 125/169] Fix 10109 db backend connection health (#10124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Harden database backend retries against stale SQLAlchemy connections. Enable pool health defaults for the SQLAlchemy result backend and invalidate cached engines/sessions whenever retryable database errors occur so retries reconnect cleanly instead of reusing dead connections. * Update celery/backends/database/__init__.py * Update celery/backends/base.py * fix: remove the default value and extend the engine options with passing value * doc: update the database engine options * feat: call session invalidate if get task meta fail with retryable error * test: retryable error and retry hook * Update docs/userguide/configuration.rst * fix: wrap the on retryable error --------- Co-authored-by: Harshang Akabari Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/defaults.py | 4 + celery/backends/base.py | 16 ++++ celery/backends/database/__init__.py | 35 ++++++- celery/backends/database/session.py | 7 ++ docs/userguide/configuration.rst | 12 ++- t/unit/backends/test_base.py | 54 +++++++++++ t/unit/backends/test_database.py | 132 +++++++++++++++++++++++++++ 7 files changed, 255 insertions(+), 5 deletions(-) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 77fcfd02196..480667eeced 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -247,6 +247,10 @@ def __repr__(self): database=Namespace( url=Option(old={'celery_result_dburi'}), engine_options=Option( + { + 'pool_pre_ping': True, + 'pool_recycle': 3600, + }, type='dict', old={'celery_result_engine_options'}, ), short_lived_sessions=Option( diff --git a/celery/backends/base.py b/celery/backends/base.py index c80591de19c..69bfa550f6c 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -629,6 +629,12 @@ def store_result(self, task_id, result, state, if self.always_retry and self.exception_safe_to_retry(exc): if retries < self.max_retries: retries += 1 + try: + self.on_backend_retryable_error(exc) + except Exception: + logger.exception( + "on_backend_retryable_error hook failed; continuing retry loop", + ) # get_exponential_backoff_interval computes integers # and time.sleep accept floats for sub second sleep @@ -689,6 +695,10 @@ def exception_safe_to_retry(self, exc): """ return False + def on_backend_retryable_error(self, exc): + """Hook called before retrying a recoverable backend exception.""" + return None + def get_task_meta(self, task_id, cache=True): """Get task meta from backend. @@ -710,6 +720,12 @@ def get_task_meta(self, task_id, cache=True): if self.always_retry and self.exception_safe_to_retry(exc): if retries < self.max_retries: retries += 1 + try: + self.on_backend_retryable_error(exc) + except Exception: + logger.exception( + "on_backend_retryable_error hook failed; continuing retry loop", + ) # get_exponential_backoff_interval computes integers # and time.sleep accept floats for sub second sleep diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index df03db56d38..97a96e49813 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -13,7 +13,7 @@ from .session import SessionManager try: - from sqlalchemy.exc import DatabaseError, InvalidRequestError + from sqlalchemy.exc import DatabaseError, InterfaceError, InvalidRequestError from sqlalchemy.orm.exc import StaleDataError except ImportError: raise ImproperlyConfigured( @@ -24,6 +24,13 @@ __all__ = ('DatabaseBackend',) +RETRYABLE_DB_ERRORS = ( + DatabaseError, + InterfaceError, + InvalidRequestError, + StaleDataError, +) + @contextmanager def session_cleanup(session): @@ -45,7 +52,16 @@ def _inner(*args, **kwargs): for retries in range(max_retries): try: return fun(*args, **kwargs) - except (DatabaseError, InvalidRequestError, StaleDataError): + except RETRYABLE_DB_ERRORS as exc: + backend = args[0] if args else None + on_retryable_error = getattr(backend, 'on_backend_retryable_error', None) + if callable(on_retryable_error): + try: + on_retryable_error(exc) + except Exception: + logger.exception( + "on_backend_retryable_error hook failed; continuing retry loop", + ) logger.warning( 'Failed operation %s. Retrying %s more times.', fun.__name__, max_retries - retries - 1, @@ -77,9 +93,14 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): self.task_cls = TaskExtended self.url = url or dburi or conf.database_url + + # Merge engine options: defaults from config <- constructor overrides + # The defaults (pool_pre_ping=True, pool_recycle=3600) are defined in + # celery/app/defaults.py under database_engine_options self.engine_options = dict( - engine_options or {}, - **conf.database_engine_options or {}) + conf.database_engine_options or {}, + **(engine_options or {}) + ) self.short_lived_sessions = kwargs.get( 'short_lived_sessions', conf.database_short_lived_sessions) @@ -108,6 +129,12 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): def extended_result(self): return self.app.conf.find_value_for_key('extended', 'result') + def exception_safe_to_retry(self, exc): + return isinstance(exc, RETRYABLE_DB_ERRORS) + + def on_backend_retryable_error(self, exc): + self.session_manager.invalidate(self.url) + def _create_tables(self): """Create the task and taskset tables.""" self.ResultSession() diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index 415d4623e00..f4f3ca6a2f5 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -60,6 +60,13 @@ def create_session(self, dburi, short_lived_sessions=False, **kwargs): return engine, self._sessions[dburi] return engine, sessionmaker(bind=engine) + def invalidate(self, dburi): + """Dispose cached engine/session state for a database URI.""" + self._sessions.pop(dburi, None) + engine = self._engines.pop(dburi, None) + if engine is not None: + engine.dispose() + def prepare_models(self, engine): if not self.prepared: # SQLAlchemy will check if the items exist before trying to diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index b0bb46fa2a7..270dd187843 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -1017,7 +1017,14 @@ Default: True by default. ``database_engine_options`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Default: ``{}`` (empty mapping). +Default: ``{'pool_pre_ping': True, 'pool_recycle': 3600}`` + +.. versionchanged:: 5.7 + + The default was changed from ``{}`` to include ``pool_pre_ping=True`` + and ``pool_recycle=3600`` for improved connection health handling. + This helps prevent stale connection errors such as + ``(OperationalError) (2006, 'MySQL server has gone away')``. To specify additional SQLAlchemy database engine options you can use the :setting:`database_engine_options` setting:: @@ -1025,6 +1032,9 @@ the :setting:`database_engine_options` setting:: # echo enables verbose logging from SQLAlchemy. app.conf.database_engine_options = {'echo': True} + # To disable the default pool health options: + app.conf.database_engine_options = {'pool_pre_ping': False, 'pool_recycle': None} + .. setting:: database_short_lived_sessions ``database_short_lived_sessions`` diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index ce25ff72ad8..44f3f0cbdc7 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -1492,6 +1492,7 @@ def test_get_with_retries(self): b = BaseBackend(app=self.app) b.exception_safe_to_retry = lambda exc: True b._sleep = Mock() + b.on_backend_retryable_error = Mock() b._get_task_meta_for = Mock() b._get_task_meta_for.side_effect = [ Exception("failed"), @@ -1500,6 +1501,7 @@ def test_get_with_retries(self): res = b.get_task_meta(sentinel.task_id) assert res == {'status': states.SUCCESS, 'result': 42} assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() finally: self.app.conf.result_backend_always_retry = prev @@ -1554,6 +1556,7 @@ def test_store_result_never_retries(self): b = BaseBackend(app=self.app) b.exception_safe_to_retry = lambda exc: True b._sleep = Mock() + b.on_backend_retryable_error = Mock() b._get_task_meta_for = Mock() b._get_task_meta_for.return_value = { 'status': states.RETRY, @@ -1583,6 +1586,7 @@ def test_store_result_with_retries(self): b = BaseBackend(app=self.app) b.exception_safe_to_retry = lambda exc: True b._sleep = Mock() + b.on_backend_retryable_error = Mock() b._get_task_meta_for = Mock() b._get_task_meta_for.return_value = { 'status': states.RETRY, @@ -1600,6 +1604,56 @@ def test_store_result_with_retries(self): res = b.store_result(sentinel.task_id, 42, states.SUCCESS) assert res == 42 assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() + finally: + self.app.conf.result_backend_always_retry = prev + + def test_get_with_retries_hook_failure_continues(self): + self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry + + try: + b = BaseBackend(app=self.app) + b.exception_safe_to_retry = lambda exc: True + b._sleep = Mock() + b.on_backend_retryable_error = Mock(side_effect=RuntimeError("hook failed")) + b._get_task_meta_for = Mock() + b._get_task_meta_for.side_effect = [ + Exception("failed"), + {'status': states.SUCCESS, 'result': 42} + ] + res = b.get_task_meta(sentinel.task_id) + assert res == {'status': states.SUCCESS, 'result': 42} + assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() + finally: + self.app.conf.result_backend_always_retry = prev + + def test_store_result_with_retries_hook_failure_continues(self): + self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry + + try: + b = BaseBackend(app=self.app) + b.exception_safe_to_retry = lambda exc: True + b._sleep = Mock() + b.on_backend_retryable_error = Mock(side_effect=RuntimeError("hook failed")) + b._get_task_meta_for = Mock() + b._get_task_meta_for.return_value = { + 'status': states.RETRY, + 'result': { + "exc_type": "Exception", + "exc_message": ["failed"], + "exc_module": "builtins", + }, + } + b._store_result = Mock() + b._store_result.side_effect = [ + Exception("failed"), + 42 + ] + res = b.store_result(sentinel.task_id, 42, states.SUCCESS) + assert res == 42 + assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() finally: self.app.conf.result_backend_always_retry = prev diff --git a/t/unit/backends/test_database.py b/t/unit/backends/test_database.py index d59df1cbc8f..b922962af74 100644 --- a/t/unit/backends/test_database.py +++ b/t/unit/backends/test_database.py @@ -126,11 +126,115 @@ def raises(): raises(max_retries=5) assert calls[0] == 5 + def test_retry_helper_calls_on_backend_retryable_error(self): + from celery.backends.database import DatabaseError + + calls = [0] + hook_calls = [] + + mock_backend = Mock() + mock_backend.on_backend_retryable_error = Mock(side_effect=lambda exc: hook_calls.append(exc)) + + @retry + def raises_with_backend(backend): + calls[0] += 1 + raise DatabaseError(1, 2, 3) + + with pytest.raises(DatabaseError): + raises_with_backend(mock_backend, max_retries=3) + + assert calls[0] == 3 + assert mock_backend.on_backend_retryable_error.call_count == 3 + for exc in hook_calls: + assert isinstance(exc, DatabaseError) + + def test_retry_helper_without_hook(self): + from celery.backends.database import DatabaseError + + calls = [0] + + mock_backend = Mock(spec=[]) + + @retry + def raises_with_backend(backend): + calls[0] += 1 + raise DatabaseError(1, 2, 3) + + with pytest.raises(DatabaseError): + raises_with_backend(mock_backend, max_retries=3) + + assert calls[0] == 3 + + def test_retry_helper_hook_failure_continues(self): + from celery.backends.database import DatabaseError + + calls = [0] + + mock_backend = Mock() + mock_backend.on_backend_retryable_error = Mock(side_effect=RuntimeError("hook failed")) + + @retry + def raises_with_backend(backend): + calls[0] += 1 + raise DatabaseError(1, 2, 3) + + with pytest.raises(DatabaseError): + raises_with_backend(mock_backend, max_retries=3) + + assert calls[0] == 3 + assert mock_backend.on_backend_retryable_error.call_count == 3 + def test_missing_dburi_raises_ImproperlyConfigured(self): self.app.conf.database_url = None with pytest.raises(ImproperlyConfigured): DatabaseBackend(app=self.app) + def test_engine_options_include_pool_health_defaults(self): + tb = DatabaseBackend(self.uri, app=self.app) + assert tb.engine_options["pool_pre_ping"] is True + assert tb.engine_options["pool_recycle"] == 3600 + + def test_engine_options_explicit_values_override_defaults(self): + self.app.conf.database_engine_options = {"pool_pre_ping": False} + tb = DatabaseBackend( + self.uri, + app=self.app, + engine_options={"pool_recycle": 15}, + ) + assert tb.engine_options["pool_pre_ping"] is False + assert tb.engine_options["pool_recycle"] == 15 + + def test_exception_safe_to_retry(self): + from celery.backends.database import DatabaseError, InvalidRequestError, StaleDataError + + tb = DatabaseBackend(self.uri, app=self.app) + assert tb.exception_safe_to_retry(DatabaseError("", "", Exception("db error"))) + assert tb.exception_safe_to_retry(InvalidRequestError()) + assert tb.exception_safe_to_retry(StaleDataError()) + assert not tb.exception_safe_to_retry(RuntimeError("not retryable")) + + def test_exception_safe_to_retry_with_interface_error(self): + from celery.backends.database import InterfaceError + + tb = DatabaseBackend(self.uri, app=self.app) + assert tb.exception_safe_to_retry(InterfaceError("", None, Exception("connection lost"))) + + def test_on_backend_retryable_error_invalidates_session(self): + tb = DatabaseBackend(self.uri, app=self.app) + tb.session_manager.invalidate = Mock() + + tb.on_backend_retryable_error(RuntimeError("retryable")) + tb.session_manager.invalidate.assert_called_once_with(tb.url) + + def test_on_backend_retryable_error_called_with_exception(self): + tb = DatabaseBackend(self.uri, app=self.app) + tb.session_manager.invalidate = Mock() + mock_exc = RuntimeError("connection lost") + + tb.on_backend_retryable_error(mock_exc) + + tb.session_manager.invalidate.assert_called_once_with(tb.url) + def test_table_schema_config(self): self.app.conf.database_table_schemas = { 'task': 'foo', @@ -445,6 +549,34 @@ def test_get_engine_kwargs(self, create_engine): engine2 = s.get_engine('dburi', foo=1) assert engine2 is engine + def test_invalidate_disposes_cached_engine(self): + s = SessionManager() + engine = Mock() + s._engines['dburi'] = engine + s._sessions['dburi'] = Mock() + + s.invalidate('dburi') + + assert 'dburi' not in s._engines + assert 'dburi' not in s._sessions + engine.dispose.assert_called_once_with() + + def test_invalidate_nonexistent_dburi_is_noop(self): + s = SessionManager() + s.invalidate('nonexistent-dburi') + assert 'nonexistent-dburi' not in s._engines + assert 'nonexistent-dburi' not in s._sessions + + def test_invalidate_only_engine_cached(self): + s = SessionManager() + engine = Mock() + s._engines['dburi'] = engine + + s.invalidate('dburi') + + assert 'dburi' not in s._engines + engine.dispose.assert_called_once_with() + @patch('celery.backends.database.session.sessionmaker') def test_create_session_forked(self, sessionmaker): s = SessionManager() From 30b51243f603ce828f1fae3e5221f925341b1505 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Feb 2026 11:55:56 +0600 Subject: [PATCH 126/169] Bump grpcio from 1.75.1 to 1.76.0 (#9962) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [grpcio](https://github.com/grpc/grpc) from 1.75.1 to 1.76.0. - [Release notes](https://github.com/grpc/grpc/releases) - [Changelog](https://github.com/grpc/grpc/blob/master/doc/grpc_release_schedule.md) - [Commits](https://github.com/grpc/grpc/compare/v1.75.1...v1.76.0) --- updated-dependencies: - dependency-name: grpcio dependency-version: 1.76.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/gcs.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/extras/gcs.txt b/requirements/extras/gcs.txt index 2cc9b4779f5..804aa883ab7 100644 --- a/requirements/extras/gcs.txt +++ b/requirements/extras/gcs.txt @@ -1,4 +1,5 @@ google-cloud-storage>=2.10.0 +grpcio==1.76.0 google-cloud-firestore==2.22.0 -grpcio==1.75.1 + From e0a9716b27d480293f83e4abbfd11cb71f3e950e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Feb 2026 22:13:48 +0000 Subject: [PATCH 127/169] Bump cryptography from 46.0.3 to 46.0.5 Bumps [cryptography](https://github.com/pyca/cryptography) from 46.0.3 to 46.0.5. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/46.0.3...46.0.5) --- updated-dependencies: - dependency-name: cryptography dependency-version: 46.0.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements/extras/auth.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/auth.txt b/requirements/extras/auth.txt index 7637ae07bf4..ed207ea0cdb 100644 --- a/requirements/extras/auth.txt +++ b/requirements/extras/auth.txt @@ -1 +1 @@ -cryptography==46.0.3 +cryptography==46.0.5 From 3026b482c19e39760bb9ed2b5e46d2c93b82d10c Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Sat, 21 Feb 2026 17:01:43 +0800 Subject: [PATCH 128/169] Database Backend filter unsupport sql engine arguments with nullpool #7355 (#10134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: add the max overflow to unsupport kwargs * test: create engine with nullpool * fix: typo and filter echo pool either --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/database/session.py | 7 +++++-- t/unit/backends/test_database.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index f4f3ca6a2f5..d4b6496a7ed 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -48,8 +48,11 @@ def get_engine(self, dburi, **kwargs): engine = self._engines[dburi] = create_engine(dburi, **kwargs) return engine else: - kwargs = {k: v for k, v in kwargs.items() if - not k.startswith('pool')} + unsupported_nullpool_kwargs = {'max_overflow', 'echo_pool'} + kwargs = { + k: v for k, v in kwargs.items() + if not k.startswith('pool') and k not in unsupported_nullpool_kwargs + } return create_engine(dburi, poolclass=NullPool, **kwargs) def create_session(self, dburi, short_lived_sessions=False, **kwargs): diff --git a/t/unit/backends/test_database.py b/t/unit/backends/test_database.py index b922962af74..346610b9886 100644 --- a/t/unit/backends/test_database.py +++ b/t/unit/backends/test_database.py @@ -633,3 +633,21 @@ def raise_err(bind): manager.prepare_models(engine) assert mock_create_all.call_count == PREPARE_MODELS_MAX_RETRIES + 1 + + @patch('celery.backends.database.session.create_engine') + def test_get_engine_filters_nullpool_unsupported_kwargs(self, mock_create_engine): + """ + Test that QueuePool-specific kwargs (like pool_size and max_overflow) + are filtered out when creating an engine with NullPool. + """ + from celery.backends.database.session import NullPool + + s = SessionManager() + s.forked = False # Ensure we're in the non-forked code path + + s.get_engine('dburi', echo_pool=True, pool_size=10, max_overflow=5) + + mock_create_engine.assert_called_once_with( + 'dburi', + poolclass=NullPool, + ) From 4b2794f75c2fa97ef85a15db19feff0bb6056356 Mon Sep 17 00:00:00 2001 From: Kadir Can Ozden <101993364+bysiber@users.noreply.github.com> Date: Sat, 21 Feb 2026 19:01:20 +0300 Subject: [PATCH 129/169] fix(beat): correct argument order in Service.__reduce__ (#10137) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Service.__reduce__ passed arguments in wrong order (max_interval, schedule_filename, scheduler_cls, app) but __init__ expects (app, max_interval, schedule_filename, scheduler_cls). This caused app to receive the value of max_interval when unpickling, leading to errors when deserializing beat Service instances. Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/beat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/celery/beat.py b/celery/beat.py index 93203bf0f89..fbda26e9f42 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -627,8 +627,8 @@ def __init__(self, app, max_interval=None, schedule_filename=None, self._is_stopped = Event() def __reduce__(self): - return self.__class__, (self.max_interval, self.schedule_filename, - self.scheduler_cls, self.app) + return self.__class__, (self.app, self.max_interval, + self.schedule_filename, self.scheduler_cls) def start(self, embedded_process=False): info('beat: Starting...') From 023827e1be611b14439be303fd653786321ab027 Mon Sep 17 00:00:00 2001 From: rohan436 Date: Sun, 22 Feb 2026 11:49:24 +0800 Subject: [PATCH 130/169] ci: declare explicit read-only token permissions in workflow jobs --- .github/workflows/linter.yml | 3 +++ .github/workflows/semgrep.yml | 4 ++++ .github/workflows/smoke-tests.yml | 3 +++ 3 files changed, 10 insertions(+) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 498b950d377..249592d4fb3 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -2,6 +2,9 @@ name: Linter on: [pull_request, workflow_dispatch] +permissions: + contents: read + jobs: linter: runs-on: blacksmith-4vcpu-ubuntu-2204 diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index 42fd5fcb02e..2f1a2309eeb 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -12,6 +12,10 @@ on: workflow_dispatch: name: Semgrep + +permissions: + contents: read + jobs: semgrep: name: Scan diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index b23cc833e54..1116b2af2ea 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -13,6 +13,9 @@ on: type: string default: '["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"]' +permissions: + contents: read + jobs: testing-with: runs-on: blacksmith-4vcpu-ubuntu-2404 From d8cbbec4b133efac0041d60e7b4bc75caae5ec3a Mon Sep 17 00:00:00 2001 From: cui Date: Sun, 22 Feb 2026 15:15:23 +0800 Subject: [PATCH 131/169] chore: 'boto3to' to 'boto3 to' (#10133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/backends/s3.py b/celery/backends/s3.py index ea04ae373d1..e53bd84a7f2 100644 --- a/celery/backends/s3.py +++ b/celery/backends/s3.py @@ -32,7 +32,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) if not boto3 or not botocore: - raise ImproperlyConfigured('You must install boto3' + raise ImproperlyConfigured('You must install boto3 ' 'to use s3 backend') conf = self.app.conf From 7f04c80aac48ca70d8acd7d77fc955068a82b08a Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Sun, 22 Feb 2026 19:31:39 +0800 Subject: [PATCH 132/169] Database Backend: Add missing index on date_done (Fixes #10097) (#10098) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Database Backend: Add missing index on date_done (Fixes #10097) * test: Add unit tests for date_done index on Task and TaskSet models * docs: document date_done index schema change * doc: add the migration schema --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/database/models.py | 4 ++-- docs/userguide/configuration.rst | 32 ++++++++++++++++++++++++++++++ t/unit/backends/test_database.py | 12 +++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/celery/backends/database/models.py b/celery/backends/database/models.py index ff193f7aeb0..f8ee6239349 100644 --- a/celery/backends/database/models.py +++ b/celery/backends/database/models.py @@ -36,7 +36,7 @@ class Task(ResultModelBase): status = sa.Column(sa.String(50), default=states.PENDING) result = sa.Column(PickleType, nullable=True) date_done = sa.Column(sa.DateTime, default=_get_utc_now, - onupdate=_get_utc_now, nullable=True) + onupdate=_get_utc_now, nullable=True, index=True) traceback = sa.Column(sa.Text, nullable=True) def __init__(self, task_id): @@ -98,7 +98,7 @@ class TaskSet(ResultModelBase): taskset_id = sa.Column(sa.String(155), unique=True) result = sa.Column(PickleType, nullable=True) date_done = sa.Column(sa.DateTime, default=_get_utc_now, - nullable=True) + nullable=True, index=True) def __init__(self, taskset_id, result): self.taskset_id = taskset_id diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 270dd187843..5642480eafc 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -995,6 +995,38 @@ strings (this is the part of the URI that comes after the ``db+`` prefix). .. _`Connection String`: http://www.sqlalchemy.org/docs/core/engines.html#database-urls +.. note:: + + If you are upgrading from Celery 5.6 or earlier, the ``date_done`` column + in ``celery_taskmeta`` and ``celery_tasksetmeta`` tables does not have a + database index. The built-in periodic task ``celery.backend_cleanup`` + queries on ``date_done`` to delete expired task results, so adding an + index significantly improves cleanup performance on large tables. + + Since SQLAlchemy's ``create_all()`` will not alter existing tables, you + will need to update your database schema. If you are using Alembic for + schema migrations, you can generate an empty revision and apply the + following operations: + + .. code-block:: python + + from alembic import op + + def upgrade(): + op.create_index('ix_celery_taskmeta_date_done', 'celery_taskmeta', ['date_done']) + op.create_index('ix_celery_tasksetmeta_date_done', 'celery_tasksetmeta', ['date_done']) + + def downgrade(): + op.drop_index('ix_celery_tasksetmeta_date_done', table_name='celery_tasksetmeta') + op.drop_index('ix_celery_taskmeta_date_done', table_name='celery_taskmeta') + + Otherwise, you can add the indexes manually using SQL: + + .. code-block:: sql + + CREATE INDEX ix_celery_taskmeta_date_done ON celery_taskmeta (date_done); + CREATE INDEX ix_celery_tasksetmeta_date_done ON celery_tasksetmeta (date_done); + .. setting:: database_create_tables_at_setup ``database_create_tables_at_setup`` diff --git a/t/unit/backends/test_database.py b/t/unit/backends/test_database.py index 346610b9886..ffff028a755 100644 --- a/t/unit/backends/test_database.py +++ b/t/unit/backends/test_database.py @@ -66,6 +66,18 @@ def test_for_mssql_dialect(self): @skip.if_pypy +class test_DateDoneIndex: + """Test that date_done columns have index=True on Task and TaskSet models.""" + + def test_task_date_done_has_index(self): + col = Task.__table__.columns['date_done'] + assert col.index is True, "Task.date_done should have index=True" + + def test_taskset_date_done_has_index(self): + col = TaskSet.__table__.columns['date_done'] + assert col.index is True, "TaskSet.date_done should have index=True" + + class test_DateDoneColumnDefaults: """Test that date_done column defaults are callables, not fixed values. From a0efe3e20d0de73a76e3559798e7c0ec6d371251 Mon Sep 17 00:00:00 2001 From: Rohan Santhosh Date: Mon, 23 Feb 2026 12:20:39 +0800 Subject: [PATCH 133/169] docs: fix typo in contributing guide (#10141) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: rohan436 Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- CONTRIBUTING.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1f7e665a6ef..34b74832081 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -74,7 +74,7 @@ We should always be open to collaboration. Your work should be done transparently and patches from Celery should be given back to the community when they're made, not just when the distribution releases. If you wish to work on new code for existing upstream projects, at least keep those -projects informed of your ideas and progress. It many not be possible to +projects informed of your ideas and progress. It may not be possible to get consensus from upstream, or even from your colleagues about the correct implementation for an idea, so don't feel obliged to have that agreement before you begin, but at least keep the outside world informed of your work, @@ -1509,4 +1509,3 @@ following: .. _`bundles`: https://docs.celeryq.dev/en/latest/getting-started/introduction.html#bundles .. _`report an issue`: https://docs.celeryq.dev/en/latest/contributing.html#reporting-bugs - From bb909f003cb384b211e58cedaff165092bb188a0 Mon Sep 17 00:00:00 2001 From: William David Edwards Date: Mon, 23 Feb 2026 07:24:50 +0100 Subject: [PATCH 134/169] Refer to Flower / Prometheus for monitoring (#10140) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refer to Flower / Prometheus for monitoring Closes https://github.com/celery/celery/discussions/10125 * Update docs/userguide/monitoring.rst --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/userguide/monitoring.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index 66cb6f00871..361cfe06172 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -460,6 +460,18 @@ The default queue is named `celery`. To get all available queues, invoke: hosts), but this won't affect the monitoring events used by for example Flower as Redis pub/sub commands are global rather than database based. +.. _monitoring-prometheus: + +Prometheus +========= + +While Prometheus monitoring is not a native part of Celery, +you can easily monitor your Celery workers using Prometheus via Flower. +Flower also provides pre-made Grafana dashboards to easily graph the amount +of tasks, workers and other instrumental statistics. + +To set up Prometheus, refer to the Flower documentation: https://flower.readthedocs.io/en/latest/prometheus-integration.html + .. _monitoring-munin: Munin From 55088e1c523fc2b33a44cdc86e1b9a8662143235 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 16:51:43 +0000 Subject: [PATCH 135/169] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 7.0.0 → 8.0.0](https://github.com/pycqa/isort/compare/7.0.0...8.0.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72da29766ed..d05bdca4c98 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: - id: mixed-line-ending - repo: https://github.com/pycqa/isort - rev: 7.0.0 + rev: 8.0.0 hooks: - id: isort From ce5269255af75faea0fdf6a65ce3721d60edc4a2 Mon Sep 17 00:00:00 2001 From: Rohan Santhosh <181558744+Rohan5commit@users.noreply.github.com> Date: Wed, 25 Feb 2026 11:43:54 +0800 Subject: [PATCH 136/169] docs: remove duplicated words in broker and routing docs --- docs/getting-started/backends-and-brokers/rabbitmq.rst | 2 +- docs/getting-started/backends-and-brokers/redis.rst | 2 +- docs/userguide/routing.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/backends-and-brokers/rabbitmq.rst b/docs/getting-started/backends-and-brokers/rabbitmq.rst index 2afc3fa3291..b3f3d722a74 100644 --- a/docs/getting-started/backends-and-brokers/rabbitmq.rst +++ b/docs/getting-started/backends-and-brokers/rabbitmq.rst @@ -223,7 +223,7 @@ To migrate from classic mirrored queues to quorum queues, please refer to Rabbit Limitations ----------- -Disabling global QoS means that the the per-channel QoS is now static. +Disabling global QoS means that the per-channel QoS is now static. This means that some Celery features won't work when using Quorum Queues. Autoscaling relies on increasing and decreasing the prefetch count whenever a new process is instantiated diff --git a/docs/getting-started/backends-and-brokers/redis.rst b/docs/getting-started/backends-and-brokers/redis.rst index aec1232f3f0..a618488fd5c 100644 --- a/docs/getting-started/backends-and-brokers/redis.rst +++ b/docs/getting-started/backends-and-brokers/redis.rst @@ -268,7 +268,7 @@ Group result ordering Versions of Celery up to and including 4.4.6 used an unsorted list to store result objects for groups in the Redis backend. This can cause those results to -be be returned in a different order to their associated tasks in the original +be returned in a different order to their associated tasks in the original group instantiation. Celery 4.4.7 introduced an opt-in behaviour which fixes this issue and ensures that group results are returned in the same order the tasks were defined, matching the behaviour of other backends. In Celery 5.0 diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index a5d58755427..f88d64b0f29 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -276,7 +276,7 @@ This means that even though there are 10 (0-9) priority levels, these are consolidated into 4 levels by default to save resources. This means that a queue named celery will really be split into 4 queues. -The highest priority queue will be named celery, and the the other queues will +The highest priority queue will be named celery, and the other queues will have a separator (by default `\x06\x16`) and their priority number appended to the queue name. From bfd5337a99fc1783cbc106dcc70c201b2c159b47 Mon Sep 17 00:00:00 2001 From: Kelson Brito Date: Wed, 25 Feb 2026 01:49:05 -0600 Subject: [PATCH 137/169] docs: fix stale version reference and grammar in README (#10145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2d8c0d6e8a8..6867e8d84ef 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ CloudAMQP :width: 240px :target: https://www.cloudamqp.com/ -`CloudAMQP `_ is a industry leading RabbitMQ as a service provider. +`CloudAMQP `_ is an industry leading RabbitMQ as a service provider. If you need highly available message queues, a perfect choice would be to use CloudAMQP. With 24,000+ running instances, CloudAMQP is the leading hosting provider of RabbitMQ, with customers all over the world. @@ -154,7 +154,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery v5.5.x coming from previous versions then you should read our +new to Celery v5.6.x coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ From 3b451ef27cb1c7411bd4f9de93f9775ee0e8e33b Mon Sep 17 00:00:00 2001 From: Rohan Santhosh Date: Sat, 28 Feb 2026 13:01:33 +0800 Subject: [PATCH 138/169] docs: fix wording in Celery 5.3 worker pool notes (#10149) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: fix wording in 5.3 worker pool notes Signed-off-by: rohan436 * Update docs/history/whatsnew-5.3.rst --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/history/whatsnew-5.3.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/history/whatsnew-5.3.rst b/docs/history/whatsnew-5.3.rst index 4ccccb69224..174d31c4082 100644 --- a/docs/history/whatsnew-5.3.rst +++ b/docs/history/whatsnew-5.3.rst @@ -249,9 +249,9 @@ A switch have been made to zoneinfo for handling timezone data instead of pytz. Support for out-of-tree worker pool implementations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Prior to version 5.3, Celery had a fixed notion of the worker pool types it supports. -Celery v5.3.0 introduces the the possibility of an out-of-tree worker pool implementation. -This feature ensure that the current worker pool implementations consistently call into -BasePool._get_info(), and enhance it to report the work pool class in use via the +Celery v5.3.0 introduces the possibility of an out-of-tree worker pool implementation. +This feature ensures that the current worker pool implementations consistently call into +BasePool._get_info(), and enhances it to report the worker pool class in use via the "celery inspect stats" command. For example: $ celery -A ... inspect stats @@ -348,4 +348,3 @@ environment and is not safe for production use at the moment. - From 97124796a3695fa340fd2e505415f4b871fd3e6c Mon Sep 17 00:00:00 2001 From: Rohan Santhosh Date: Sat, 28 Feb 2026 13:30:42 +0800 Subject: [PATCH 139/169] docs: fix duplicated word in 3.1 changelog entry Signed-off-by: Rohan5commit <181558744+Rohan5commit@users.noreply.github.com> --- docs/history/changelog-3.1.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index 4bb58c4f5a4..351faff9e15 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -1031,7 +1031,7 @@ News - **Beat**: No longer attempts to upgrade a newly created database file (Issue #1923). -- **Beat**: New setting :setting:``CELERYBEAT_SYNC_EVERY`` can be be used +- **Beat**: New setting :setting:``CELERYBEAT_SYNC_EVERY`` can be used to control file sync by specifying the number of tasks to send between each sync. @@ -1733,4 +1733,4 @@ Fixes :release-date: 2013-11-09 11:00 p.m. UTC :release-by: Ask Solem -See :ref:`whatsnew-3.1`. +See :ref:`whatsnew-3.1`. \ No newline at end of file From f288893a1b280625d5184302dc3b5cb5cd4ccb61 Mon Sep 17 00:00:00 2001 From: Rohan Santhosh Date: Sat, 28 Feb 2026 21:57:18 +0800 Subject: [PATCH 140/169] docs: fix changelog typo (context manager) (#10144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- Changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.rst b/Changelog.rst index 73bc9ba3671..2ff3ca1968d 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -276,7 +276,7 @@ What's Changed - Add xfail test for RabbitMQ quorum queue global QoS race condition (#9770) - fix: (#8786) time out when chord header fails with group body (#9788) - Fix #9738 : Add root_id and parent_id to .apply() (#9784) -- Replace DelayedDelivery connection creation to use context manger (#9793) +- Replace DelayedDelivery connection creation to use context manager (#9793) - Fix #9794: Pydantic integration fails with __future__.annotations. (#9795) - add go and rust implementation in docs (#9800) - Fix memory leak in exception handling (Issue #8882) (#9799) From 5597771e74ee59e2e82a12513223159f5b1ce142 Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Sun, 1 Mar 2026 12:29:48 +0800 Subject: [PATCH 141/169] Fix/10096 worker fails to reconnect after redis failover (#10151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: trigger the hub reset on error * fix: jobs stuck in cache when synack is disabled * fix: prevent infinite loop when worker process is dead * fix: pass the synack to pool * fix: advance _write_ack generators in flush() instead of dropping them * test: add missing coverage for flush() _write_ack and gen_not_started paths * fix: sitch the synack to True --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/concurrency/asynpool.py | 76 ++++++---- celery/worker/loops.py | 42 ++++-- t/unit/concurrency/test_prefork.py | 225 ++++++++++++++++++++++++++++- t/unit/worker/test_loops.py | 56 +++++++ 4 files changed, 349 insertions(+), 50 deletions(-) diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index 30a761f43c4..10783847a96 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -472,7 +472,7 @@ def __init__(self, processes=None, synack=False, self.write_stats = Counter() - super().__init__(processes, *args, **kwargs) + super().__init__(processes, *args, synack=synack, **kwargs) for proc in self._pool: # create initial mappings, these will be updated @@ -1000,11 +1000,14 @@ def flush(self): if self._state == TERMINATE: return # cancel all tasks that haven't been accepted so that NACK is sent - # if synack is enabled. - if self.synack: - for job in self._cache.values(): - if not job._accepted: + # if synack is enabled, otherwise discard them from the cache + # since they will be redelivered by the broker. + for job in tuple(self._cache.values()): + if not job._accepted: + if self.synack: job._cancel() + else: + job.discard() # clear the outgoing buffer as the tasks will be redelivered by # the broker anyway. @@ -1029,36 +1032,47 @@ def flush(self): if writer is not None: owned_by[writer] = job - if not self._active_writers: - self._cache.clear() - else: - while self._active_writers: - writers = list(self._active_writers) - for gen in writers: - if (gen.__name__ == '_write_job' and - gen_not_started(gen)): - # hasn't started writing the job so can - # discard the task, but we must also remove - # it from the Pool._cache. - try: - job = owned_by[gen] - except KeyError: - pass - else: - # removes from Pool._cache - job.discard() - self._active_writers.discard(gen) + while self._active_writers: + writers = list(self._active_writers) + for gen in writers: + if (gen.__name__ == '_write_job' and + gen_not_started(gen)): + # hasn't started writing the job so can + # discard the task, but we must also remove + # it from the Pool._cache. + try: + job = owned_by[gen] + except KeyError: + pass else: + # removes from Pool._cache + job.discard() + self._active_writers.discard(gen) + else: + try: + job = owned_by[gen] + except KeyError: + # Generator not in owned_by — not a _write_job + # (e.g. a _write_ack coroutine added by send_ack()). + # These *MUST* complete or the worker process will + # hang waiting for the ack. Advance it one step; + # the generator raises StopIteration/OSError when + # done or when the peer process has already died. try: - job = owned_by[gen] - except KeyError: - pass + next(gen) + except (StopIteration, OSError, EOFError): + self._active_writers.discard(gen) + else: + job_proc = job._write_to + if job_proc._is_alive(): + # _flush_writer calls + # _active_writers.discard(gen) in its finally. + self._flush_writer(job_proc, gen) else: - job_proc = job._write_to - if job_proc._is_alive(): - self._flush_writer(job_proc, gen) - + # Process is dead, job will never + # complete - discard from cache. job.discard() + self._active_writers.discard(gen) # workers may have exited in the meantime. self.maintain_pool() sleep(next(intervals)) # don't busyloop diff --git a/celery/worker/loops.py b/celery/worker/loops.py index f88cddb8d6b..ed33a8c64af 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -81,21 +81,35 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, hub.propagate_errors = errors loop = hub.create_loop() - while blueprint.state == RUN and obj.connection: - state.maybe_shutdown() - if heartbeat_error[0] is not None: - raise heartbeat_error[0] - - # We only update QoS when there's no more messages to read. - # This groups together qos calls, and makes sure that remote - # control commands will be prioritized over task messages. - if qos.prev != qos.value: - update_qos() - + try: + while blueprint.state == RUN and obj.connection: + state.maybe_shutdown() + if heartbeat_error[0] is not None: + raise heartbeat_error[0] + + # We only update QoS when there's no more messages to read. + # This groups together qos calls, and makes sure that remote + # control commands will be prioritized over task messages. + if qos.prev != qos.value: + update_qos() + + try: + next(loop) + except StopIteration: + loop = hub.create_loop() + except Exception: + # Reset the hub on error (e.g. connection loss) to clean up + # stale file descriptors and callbacks from the old connection. + # We intentionally do NOT reset on normal exit (graceful shutdown) + # so that timers (e.g. heartbeat) keep firing while the pool drains. + # WorkerShutdown/WorkerTerminate extend SystemExit (not Exception) + # so they won't be caught here. try: - next(loop) - except StopIteration: - loop = hub.create_loop() + hub.reset() + except Exception as exc: # pylint: disable=broad-except + logger.exception( + 'Error cleaning up after event loop: %r', exc) + raise def synloop(obj, connection, consumer, blueprint, hub, qos, diff --git a/t/unit/concurrency/test_prefork.py b/t/unit/concurrency/test_prefork.py index 51b5214216a..8f266f78302 100644 --- a/t/unit/concurrency/test_prefork.py +++ b/t/unit/concurrency/test_prefork.py @@ -3,7 +3,7 @@ import socket import tempfile from itertools import cycle -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import pytest from billiard.pool import ApplyResult @@ -513,12 +513,227 @@ def test_untrack_child_process_with_sentinel_poll(self): hub.remove.assert_called_once_with(fd) assert proc._sentinel_poll is None + @t.skip.if_pypy + def test_flush_no_synack_discards_unaccepted_jobs(self): + """flush() should discard unaccepted jobs when synack is disabled. -@t.skip.if_win32 -class test_ResultHandler: + Previously, flush() only handled the synack case. Without synack, + unaccepted jobs were never cleaned from the cache, leading to stale + entries. + """ + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + job1 = Mock(name='job1') + job1._accepted = False + job1._writer.return_value = None + job2 = Mock(name='job2') + job2._accepted = True + job2._writer.return_value = None + + pool._cache = {1: job1, 2: job2} + pool.outbound_buffer.clear() + pool._active_writers.clear() - def setup_method(self): - pytest.importorskip('multiprocessing') + pool.flush() + + job1.discard.assert_called_once() + job2.discard.assert_not_called() + + @t.skip.if_pypy + def test_flush_synack_cancels_unaccepted_jobs(self): + """flush() should call _cancel() on unaccepted jobs when synack is enabled.""" + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + job1 = Mock(name='job1') + job1._accepted = False + job1._writer.return_value = None + job2 = Mock(name='job2') + job2._accepted = True + job2._writer.return_value = None + + pool._cache = {1: job1, 2: job2} + pool.outbound_buffer.clear() + pool._active_writers.clear() + + pool.flush() + + job1._cancel.assert_called_once() + job1.discard.assert_not_called() + job2._cancel.assert_not_called() + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_dead_process_discards_active_writer(self, _create_worker_process): + """flush() must discard generator from _active_writers when process is dead. + + Previously, when a process was dead, the generator was never removed + from _active_writers, causing an infinite loop in the while loop. + """ + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + # Create a mock generator (already started, so not gen_not_started) + gen = Mock(name='gen') + gen.__name__ = '_write_job' + # Simulate a started generator + with patch.object(asynpool, 'gen_not_started', return_value=False): + proc = Mock(name='proc') + proc._is_alive.return_value = False # Process is dead + + job = Mock(name='job') + job._accepted = True + job._write_to = proc + job._writer.return_value = gen + + pool._cache = {1: job} + pool._active_writers = {gen} + pool.outbound_buffer.clear() + + pool.flush() + + # Generator should have been removed from active_writers + assert gen not in pool._active_writers + # Job should have been discarded since process is dead + job.discard.assert_called() + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_alive_process_flushes_writer(self, _create_worker_process): + """flush() should call _flush_writer when process is still alive.""" + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + gen = Mock(name='gen') + gen.__name__ = '_write_job' + + with patch.object(asynpool, 'gen_not_started', return_value=False): + proc = Mock(name='proc') + proc._is_alive.return_value = True + + job = Mock(name='job') + job._accepted = True + job._write_to = proc + job._writer.return_value = gen + + pool._cache = {1: job} + pool._active_writers = {gen} + pool.outbound_buffer.clear() + + with patch.object(pool, '_flush_writer') as mock_flush: + # _flush_writer removes from _active_writers in its finally + def side_effect(p, g): + pool._active_writers.discard(g) + mock_flush.side_effect = side_effect + + pool.flush() + + mock_flush.assert_called_once_with(proc, gen) + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_write_ack_coroutine_is_advanced_not_dropped(self, _create_worker_process): + """flush() must not silently drop _write_ack generators (synack mode). + + _write_ack coroutines are added to _active_writers by send_ack() but + are NOT mapped in owned_by (which is built from _cache job writers only). + Dropping them mid-write leaves a partially-written ack on the synq pipe + and hangs the worker process waiting for the ack that never arrives. + flush() must advance them to completion instead. + """ + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + # Simulate a _write_ack generator (name != '_write_job', not in owned_by) + ack_gen = Mock(name='ack_gen') + ack_gen.__name__ = '_write_ack' + # First call to next() returns normally (still writing), + # second raises StopIteration (write complete). + ack_gen.__next__ = Mock(side_effect=[None, StopIteration()]) + + pool._cache = {} + pool._active_writers = {ack_gen} + pool.outbound_buffer.clear() + + pool.flush() + + # Generator should have been advanced (not just silently discarded) + assert ack_gen.__next__.called + # And removed once it signalled completion + assert ack_gen not in pool._active_writers + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_write_ack_coroutine_handles_oserror(self, _create_worker_process): + """flush() should discard the coroutine if OSError is raised during next().""" + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + ack_gen = MagicMock(name='ack_gen') + ack_gen.__name__ = '_write_ack' + ack_gen.__next__.side_effect = OSError() + + pool._cache = {} + pool._active_writers = {ack_gen} + pool.outbound_buffer.clear() + pool.flush() + + assert ack_gen not in pool._active_writers + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_write_ack_coroutine_handles_eoferror(self, _create_worker_process): + """flush() should discard the coroutine if EOFError is raised during next().""" + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + ack_gen = MagicMock(name='ack_gen') + ack_gen.__name__ = '_write_ack' + ack_gen.__next__.side_effect = EOFError() + + pool._cache = {} + pool._active_writers = {ack_gen} + pool.outbound_buffer.clear() + pool.flush() + + assert ack_gen not in pool._active_writers + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_not_started_write_job_is_discarded(self, _create_worker_process): + """flush() should discard a _write_job generator that has not started yet. + + When gen_not_started() returns True the job has not been written to the + pipe at all, so it is safe to discard it and let the broker redeliver. + """ + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + gen = Mock(name='gen') + gen.__name__ = '_write_job' + + with patch.object(asynpool, 'gen_not_started', return_value=True): + job = Mock(name='job') + job._accepted = True + job._writer.return_value = gen + + pool._cache = {1: job} + pool._active_writers = {gen} + pool.outbound_buffer.clear() + + pool.flush() + + job.discard.assert_called_once() + assert gen not in pool._active_writers def test_process_result(self): x = asynpool.ResultHandler( diff --git a/t/unit/worker/test_loops.py b/t/unit/worker/test_loops.py index 754a3a119c7..42369b01960 100644 --- a/t/unit/worker/test_loops.py +++ b/t/unit/worker/test_loops.py @@ -453,6 +453,62 @@ def test_no_heartbeat_support(self): x.hub.timer.call_repeatedly.assert_not_called() + def test_hub_reset_on_connection_error(self): + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()') + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + poller = x.hub.poller + poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.reset.assert_called_once() + + def test_hub_not_reset_on_graceful_shutdown(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + x.hub.on_tick.add(x.closer(mod=2)) + asynloop(*x.args) + x.hub.reset.assert_not_called() + + def test_hub_not_reset_on_worker_shutdown(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_stop = 303 + try: + with pytest.raises(WorkerShutdown): + asynloop(*x.args) + finally: + state.should_stop = None + x.hub.reset.assert_not_called() + + def test_hub_not_reset_on_worker_terminate(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_terminate = True + try: + with pytest.raises(WorkerTerminate): + asynloop(*x.args) + finally: + state.should_terminate = None + x.hub.reset.assert_not_called() + + def test_hub_reset_error_still_reraises_original(self): + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()', side_effect=RuntimeError('reset failed')) + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + poller = x.hub.poller + poller.poll.return_value = [] + # The original socket.error should still be raised, not the RuntimeError from reset() + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.reset.assert_called_once() + class test_synloop: From 6b47d98cf4839be07e99d55f88baddd9a36850eb Mon Sep 17 00:00:00 2001 From: Br1an <932039080@qq.com> Date: Sun, 1 Mar 2026 17:37:52 +0800 Subject: [PATCH 142/169] Improve on_after_finalize signal documentation (#10155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Improve on_after_finalize signal documentation The existing documentation just repeated the signal name without explaining what 'finalized' means. Updated both the class attribute docstring and the reference docs to explain that finalization evaluates pending task decorators, loads built-in tasks, and binds all tasks to the app — making it the earliest point where the full task registry is available. Fixes #7280 * Update docs/reference/celery.rst * Update celery/app/base.py --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/base.py | 6 +++++- docs/reference/celery.rst | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 71ce9329d81..13a2f3862a1 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -308,7 +308,11 @@ class name. #: Signal sent after app has prepared the configuration. on_after_configure = None - #: Signal sent after app has been finalized. + #: Signal sent after the app has been finalized (i.e., all pending + #: task decorators have been evaluated, built-in tasks loaded, and + #: every currently registered task has been bound to the app). This is + #: the earliest point at which the task registry is initialized/stable + #: and safe to inspect for tasks currently registered with this app. on_after_finalize = None #: Signal sent by every new process after fork. diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index 65c778cecd6..962fe0a1f35 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -126,7 +126,14 @@ and creating Celery applications. .. data:: on_after_finalize - Signal sent after app has been finalized. + Signal sent after the app has been finalized — that is, after all + pending task decorators have been evaluated, built-in tasks loaded, + and every task registered at that point has been bound to the app. + At this stage the task registry is initialized and stable enough to + import and inspect task objects reliably. + + See :meth:`~celery.Celery.finalize` for more details on what + finalization does. .. data:: on_after_fork From 1ab31a274adeb73216ce48a6f470576c353f2590 Mon Sep 17 00:00:00 2001 From: Br1an <932039080@qq.com> Date: Sun, 1 Mar 2026 17:47:20 +0800 Subject: [PATCH 143/169] Add non-commutative example to clarify partial arg ordering in canvas docs (#10157) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add non-commutative example to clarify partial arg ordering in canvas docs The existing partial examples use add(x, y), which is commutative. This makes it unclear whether delay() prepends or appends its arguments, since add(4, 2) == add(2, 4). Add a note with a subtract example to make the prepend behavior obvious. Fixes #6484 * docs: add inline subtract task definition per review feedback Add an explicit @app.task definition for subtract(x, y) so the example is self-contained and readers don't wonder where the task comes from. * docs: make subtract example explicitly hypothetical per review Remove the separate task definition code block and instead describe subtract inline as 'subtract(x, y) -> x - y', making it clear this is a hypothetical example without needing a full definition. * Update docs/userguide/canvas.rst --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/userguide/canvas.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 82b0e1521b6..1383bdb9a25 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -137,6 +137,24 @@ creates partials: >>> partial.delay(4) # 4 + 2 >>> partial.apply_async((4,)) # same + .. note:: + + Additional args passed to ``delay``/``apply_async`` are **prepended** + to the signature args. Since ``add`` is commutative, the ordering may + not be obvious. A non-commutative task like + ``subtract(x, y) -> x - y`` makes this clear: + + .. code-block:: python + + @app.task + def subtract(x, y): + return x - y + + partial = subtract.s(10) # incomplete: second arg only + partial.delay(30) # -> subtract(30, 10) = 20 + Here ``delay(30)`` prepends ``30`` as the first argument, resulting + in ``subtract(30, 10)`` — not ``subtract(10, 30)``. + - Any keyword arguments added will be merged with the kwargs in the signature, with the new keyword arguments taking precedence: From 59f6c2d6197f15c19ab6e4448ba16a1ca447eeef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:48:46 +0600 Subject: [PATCH 144/169] Bump google-cloud-firestore from 2.22.0 to 2.23.0 (#10126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [google-cloud-firestore](https://github.com/googleapis/python-firestore) from 2.22.0 to 2.23.0. - [Release notes](https://github.com/googleapis/python-firestore/releases) - [Changelog](https://github.com/googleapis/python-firestore/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/python-firestore/compare/v2.22.0...v2.23.0) --- updated-dependencies: - dependency-name: google-cloud-firestore dependency-version: 2.23.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/extras/gcs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/gcs.txt b/requirements/extras/gcs.txt index 804aa883ab7..64d2ff93e10 100644 --- a/requirements/extras/gcs.txt +++ b/requirements/extras/gcs.txt @@ -1,5 +1,5 @@ google-cloud-storage>=2.10.0 grpcio==1.76.0 -google-cloud-firestore==2.22.0 +google-cloud-firestore==2.23.0 From 00412047e4ab7adbbcec12c65a8f80956b031961 Mon Sep 17 00:00:00 2001 From: Tatul Danielyan Date: Mon, 2 Mar 2026 14:53:50 +0400 Subject: [PATCH 145/169] Remove redundant test_isa_mapping test (#10103) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test_isa_mapping test checks that ConfigurationView is a subclass of Mapping, but this is already implied by test_isa_mutable_mapping which checks for MutableMapping (a subclass of Mapping). Since ConfigurationView inherits from ChainMap which is a MutableMapping, the Mapping check is always True and fully redundant. Fixes #10077 Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- t/unit/utils/test_collections.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/t/unit/utils/test_collections.py b/t/unit/utils/test_collections.py index 2f183899017..9f5b0b6a492 100644 --- a/t/unit/utils/test_collections.py +++ b/t/unit/utils/test_collections.py @@ -128,10 +128,6 @@ def test_len(self): self.view.clear() assert len(self.view) == 2 - def test_isa_mapping(self): - from collections.abc import Mapping - assert issubclass(ConfigurationView, Mapping) - def test_isa_mutable_mapping(self): from collections.abc import MutableMapping assert issubclass(ConfigurationView, MutableMapping) From 59e45db52084b55f42df1943c730fa0422a4ed48 Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Mon, 2 Mar 2026 23:53:55 +0200 Subject: [PATCH 146/169] Upgrade pytest-celery to >=1.3.0 and adopt PYTEST_CELERY_PKG build arg (#10162) - Update pytest-celery version pin to >=1.3.0 (no upper bound) in requirements/test.txt and requirements/extras/pytest.txt - Adopt PYTEST_CELERY_PKG build arg pattern in smoke test Dockerfiles (dev and pypi) for configurable install source --- requirements/extras/pytest.txt | 2 +- requirements/test.txt | 2 +- t/smoke/workers/docker/dev | 3 ++- t/smoke/workers/docker/pypi | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/requirements/extras/pytest.txt b/requirements/extras/pytest.txt index 01fe3ab8c5e..0dff7152678 100644 --- a/requirements/extras/pytest.txt +++ b/requirements/extras/pytest.txt @@ -1 +1 @@ -pytest-celery[all]>=1.2.0,<1.3.0 +pytest-celery[all]>=1.3.0 diff --git a/requirements/test.txt b/requirements/test.txt index f2fe7e2d165..cdba2da75f9 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ pytest==8.4.2 -pytest-celery[all]>=1.2.0,<1.3.0 +pytest-celery[all]>=1.3.0 pytest-rerunfailures>=15.0; python_version >= "3.9" pytest-subtests>=0.14.1; python_version >= "3.9" pytest-timeout==2.4.0 diff --git a/t/smoke/workers/docker/dev b/t/smoke/workers/docker/dev index 015be6deebb..51041dbb0c6 100644 --- a/t/smoke/workers/docker/dev +++ b/t/smoke/workers/docker/dev @@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y build-essential \ ARG CELERY_LOG_LEVEL=INFO ARG CELERY_WORKER_NAME=celery_dev_worker ARG CELERY_WORKER_QUEUE=celery +ARG PYTEST_CELERY_PKG="pytest-celery" ENV LOG_LEVEL=$CELERY_LOG_LEVEL ENV WORKER_NAME=$CELERY_WORKER_NAME ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE @@ -39,7 +40,7 @@ COPY --chown=test_user:test_user . /celery RUN pip install --no-cache-dir --upgrade \ pip \ -e /celery[redis,pymemcache,pydantic,sqs] \ - pytest-celery>=1.1.3 + "${PYTEST_CELERY_PKG}" # The workdir must be /app WORKDIR /app diff --git a/t/smoke/workers/docker/pypi b/t/smoke/workers/docker/pypi index d0b2c21aa48..9e1f94d64d5 100644 --- a/t/smoke/workers/docker/pypi +++ b/t/smoke/workers/docker/pypi @@ -24,6 +24,7 @@ ARG CELERY_VERSION="" ARG CELERY_LOG_LEVEL=INFO ARG CELERY_WORKER_NAME=celery_tests_worker ARG CELERY_WORKER_QUEUE=celery +ARG PYTEST_CELERY_PKG="pytest-celery" ENV PIP_VERSION=$CELERY_VERSION ENV LOG_LEVEL=$CELERY_LOG_LEVEL ENV WORKER_NAME=$CELERY_WORKER_NAME @@ -38,7 +39,7 @@ EXPOSE 5678 RUN pip install --no-cache-dir --upgrade \ pip \ celery[redis,pymemcache]${CELERY_VERSION:+==$CELERY_VERSION} \ - pytest-celery[sqs]>=1.1.3 \ + "${PYTEST_CELERY_PKG}" \ pydantic>=2.4 # The workdir must be /app From f0c7de04a4a02ead8dc504d8f268189e5ac47f7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 23:41:15 +0000 Subject: [PATCH 147/169] Chore(deps): Update elasticsearch requirement from <=9.1.2 to <=9.3.0 Updates the requirements on [elasticsearch](https://github.com/elastic/elasticsearch-py) to permit the latest version. - [Release notes](https://github.com/elastic/elasticsearch-py/releases) - [Commits](https://github.com/elastic/elasticsearch-py/compare/0.4.1...v9.3.0) --- updated-dependencies: - dependency-name: elasticsearch dependency-version: 9.3.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements/extras/elasticsearch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index ca70a7df23d..5362f230427 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ -elasticsearch<=9.1.2 +elasticsearch<=9.3.0 elastic-transport<=9.2.1 From 61d8d388a977ead4f41d4c1e21162aaddb229424 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Mar 2026 13:10:10 +0600 Subject: [PATCH 148/169] [pre-commit.ci] pre-commit autoupdate (#10161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/isort: 8.0.0 → 8.0.1](https://github.com/pycqa/isort/compare/8.0.0...8.0.1) Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d05bdca4c98..3e06a8431aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: - id: mixed-line-ending - repo: https://github.com/pycqa/isort - rev: 8.0.0 + rev: 8.0.1 hooks: - id: isort From b69718dee1b80edd9dbb91dcf26e0039fb4fab14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Mar 2026 15:19:13 +0600 Subject: [PATCH 149/169] Bump isort from 6.1.0 to 7.0.0 (#10051) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [isort](https://github.com/PyCQA/isort) from 6.1.0 to 7.0.0. - [Release notes](https://github.com/PyCQA/isort/releases) - [Changelog](https://github.com/PyCQA/isort/blob/main/CHANGELOG.md) - [Commits](https://github.com/PyCQA/isort/compare/6.1.0...7.0.0) --- updated-dependencies: - dependency-name: isort dependency-version: 7.0.0 dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 5855800eadf..44d852a369c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,4 +2,4 @@ git+https://github.com/celery/py-amqp.git git+https://github.com/celery/kombu.git git+https://github.com/celery/billiard.git vine>=5.0.0 -isort==6.1.0 +isort==7.0.0 From 4ec4dcaa84d7469fff8a3473a1eb8682b0f9a7af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=82=AC=EC=9E=AC=ED=98=81?= Date: Tue, 3 Mar 2026 18:29:55 +0900 Subject: [PATCH 150/169] Remove deprecated args from redis get_connection call (#10036) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: JaeHyuck Sa Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/redis.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/celery/backends/redis.py b/celery/backends/redis.py index a21a5ebfe8b..d31eb66b143 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -117,9 +117,7 @@ def _reconnect_pubsub(self): if self.subscribed_to: self._pubsub.subscribe(*self.subscribed_to) else: - self._pubsub.connection = self._pubsub.connection_pool.get_connection( - 'pubsub', self._pubsub.shard_hint - ) + self._pubsub.connection = self._pubsub.connection_pool.get_connection() # even if there is nothing to subscribe, we should not lose the callback after connecting. # The on_connect callback will re-subscribe to any channels we previously subscribed to. self._pubsub.connection.register_connect_callback(self._pubsub.on_connect) From 5c70c1d3c02a3819096fd39deca9599ffe6ee695 Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Sun, 8 Mar 2026 11:25:40 +0800 Subject: [PATCH 151/169] Fix #6912 rpc backend reconnection error (#10179) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add handle connection errors retry function * fix: wrap the drain event with handle connection errors * fix: catch the oserror for retry the connection * test: drain events connection and channel errors * test: drainer without greenlet * fix: sleep after catch the os error * fix: move the importorskip to module-level calls * Update celery/backends/rpc.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: start the comsumer with self._no_ack --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/asynchronous.py | 23 ++ celery/backends/rpc.py | 63 ++++- t/unit/backends/test_asynchronous.py | 372 ++++++++++++++++++++++++++- t/unit/backends/test_rpc.py | 114 ++++++++ 4 files changed, 567 insertions(+), 5 deletions(-) diff --git a/celery/backends/asynchronous.py b/celery/backends/asynchronous.py index a5e0e5d4036..0413afecc8a 100644 --- a/celery/backends/asynchronous.py +++ b/celery/backends/asynchronous.py @@ -79,6 +79,18 @@ def drain_events_until(self, p, timeout=None, interval=1, on_interval=None, wait yield self.wait_for(p, wait, timeout=interval) except socket.timeout: pass + except OSError: + # Recoverable connection error (e.g. broker restart). + # drain_events handles reconnection internally; if an + # OSError still leaks through, we log, sleep for one + # interval, and continue rather than spinning hot. + logging.warning( + 'Drainer: connection error during drain_events, ' + 'will retry on next loop iteration.', + exc_info=True, + ) + time.sleep(interval) + if on_interval: on_interval() if p.ready: # got event on the wanted channel. @@ -119,6 +131,17 @@ def run(self): self._send_drain_complete_event() except socket.timeout: pass + except OSError: + # Recoverable connection errors (e.g. broker restart) + # are handled inside drain_events via reconnection. + # If something still leaks through, we log, back off + # briefly, and retry instead of spinning hot. + logging.warning( + 'Drainer: connection error during drain_events, ' + 'will retry on next loop iteration.', + exc_info=True, + ) + time.sleep(1) except Exception as e: self._exc = e raise diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 927c7f517fa..42fef2072c5 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -2,7 +2,9 @@ RPC-style result backend, using reply-to and one queue per client. """ +import logging import time +from contextlib import contextmanager import kombu from kombu.common import maybe_declare @@ -17,6 +19,8 @@ __all__ = ('BacklogLimitExceeded', 'RPCBackend') +logger = logging.getLogger(__name__) + E_NO_CHORD_SUPPORT = """ The "rpc" result backend does not support chords! @@ -40,12 +44,14 @@ class ResultConsumer(BaseResultConsumer): _connection = None _consumer = None + _no_ack = True def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._create_binding = self.backend._create_binding def start(self, initial_task_id, no_ack=True, **kwargs): + self._no_ack = no_ack self._connection = self.app.connection() initial_queue = self._create_binding(initial_task_id) self._consumer = self.Consumer( @@ -54,12 +60,67 @@ def start(self, initial_task_id, no_ack=True, **kwargs): accept=self.accept) self._consumer.consume() + @contextmanager + def _handle_connection_errors(self): + """Context manager that catches broker connection/channel errors and reconnects.""" + try: + yield + except (self._connection.connection_errors + + self._connection.channel_errors) as exc: + logger.warning( + 'RPC result consumer: connection lost (%s), ' + 'attempting to reconnect...', exc, + ) + self._reconnect() + def drain_events(self, timeout=None): if self._connection: - return self._connection.drain_events(timeout=timeout) + with self._handle_connection_errors(): + return self._connection.drain_events(timeout=timeout) elif timeout: time.sleep(timeout) + def _reconnect(self): + """Close the stale connection and rebuild the consumer. + + Re-subscribes to every queue that the old consumer was listening on + so that pending results can still be drained. + """ + old_queues = [] + if self._consumer is not None: + old_queues = list(self._consumer.queues) + try: + self._consumer.cancel() + except Exception: + logger.debug( + 'RPC result consumer: error while cancelling stale ' + 'consumer during reconnect', + exc_info=True, + ) + + if self._connection is not None: + try: + self._connection.close() + except Exception: + logger.debug( + 'RPC result consumer: error while closing stale ' + 'connection during reconnect', + exc_info=True, + ) + self._connection = None + + # Establish a fresh connection and consumer. + self._connection = self.app.connection() + self._consumer = self.Consumer( + self._connection.default_channel, + old_queues, + callbacks=[self.on_state_change], + no_ack=self._no_ack, + accept=self.accept, + ) + self._consumer.consume() + logger.info('RPC result consumer: reconnected successfully.') + def stop(self): try: self._consumer.cancel() diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index e5dc27eec62..26acf96ccf1 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -1,3 +1,4 @@ +import logging import os import socket import sys @@ -8,17 +9,325 @@ import pytest from vine import promise -from celery.backends.asynchronous import E_CELERY_RESTART_REQUIRED, BaseResultConsumer +from celery.backends.asynchronous import E_CELERY_RESTART_REQUIRED, BaseResultConsumer, greenletDrainer from celery.backends.base import Backend from celery.utils import cached_property -pytest.importorskip('gevent') -pytest.importorskip('eventlet') +# ---- helpers --------------------------------------------------------------- + + +def _make_consumer(app, environment='default'): + """Create a BaseResultConsumer with a mocked drainer environment.""" + with patch('celery.backends.asynchronous.detect_environment') as det: + det.return_value = environment + backend = Backend(app) + consumer = BaseResultConsumer( + backend, app, backend.accept, + pending_results={}, pending_messages={}, + ) + return consumer + + +# --------------------------------------------------------------------------- +# 1. Drainer (default / synchronous) -- no gevent / eventlet needed +# --------------------------------------------------------------------------- + +class test_Drainer_without_greenlets: + + # -- drain_events_until: normal flow ------------------------------------ + + def test_drain_fulfils_promise(self, app): + """Loop exits once the promise is fulfilled.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] >= 3: + p('done') + + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5): + pass + + assert p.ready + assert calls[0] >= 3 + + def test_drain_calls_on_interval(self, app): + """on_interval callback is invoked every iteration.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + on_interval = Mock() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] >= 3: + p('done') + + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5, + on_interval=on_interval): + pass + + assert on_interval.call_count >= 2 + + def test_drain_raises_timeout(self, app): + """socket.timeout raised when total elapsed time exceeds *timeout*.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + + def wait(timeout=None): + time.sleep(0.02) + + with pytest.raises(socket.timeout): + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=0.05): + pass + + assert not p.ready + + def test_drain_uses_result_consumer_drain_events_by_default(self, app): + """When *wait* is None, result_consumer.drain_events is used.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def mock_drain(timeout=None): + calls[0] += 1 + if calls[0] >= 2: + p('done') + + consumer.drain_events = mock_drain + + for _ in drainer.drain_events_until(p, interval=0.01, timeout=5): + pass + + assert p.ready + assert calls[0] >= 2 + + # -- drain_events_until: socket.timeout from wait ----------------------- + + def test_drain_swallows_socket_timeout_from_wait(self, app): + """socket.timeout raised inside wait() must be silently caught.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] <= 2: + raise socket.timeout('idle') + p('done') + + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5): + pass + + assert p.ready + + # -- drain_events_until: OSError from wait ------------------------------ + + def test_drain_catches_oserror_and_logs(self, app): + """OSError from wait() must be caught, logged, loop continues.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] <= 2: + raise OSError('broker away') + p('done') + + with patch.object(logging, 'warning') as mock_warn: + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5): + pass + + assert p.ready + assert mock_warn.call_count >= 2 + + # -- wait_for ----------------------------------------------------------- + + def test_wait_for_calls_wait_with_timeout(self, app): + """Drainer.wait_for delegates to the wait callback.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + wait = Mock() + drainer.wait_for(p, wait, timeout=0.5) + wait.assert_called_once_with(timeout=0.5) + + +# --------------------------------------------------------------------------- +# 2. greenletDrainer -- tested synchronously (no real greenlet spawning) +# --------------------------------------------------------------------------- + +class test_greenletDrainer: + + def _make_greenlet_drainer(self, app): + consumer = _make_consumer(app) + drainer = greenletDrainer(consumer) + return drainer + + # -- run: normal stop --------------------------------------------------- + + def test_run_exits_when_stopped(self, app): + """run() exits cleanly when _stopped is set.""" + drainer = self._make_greenlet_drainer(app) + calls = [0] + + def drain(timeout=None): + calls[0] += 1 + if calls[0] >= 3: + drainer._stopped.set() + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + drainer.run() + + assert drainer._shutdown.is_set() + assert drainer._exc is None + + # -- run: socket.timeout is swallowed ----------------------------------- + + def test_run_swallows_socket_timeout(self, app): + """socket.timeout inside run() must be silently caught.""" + drainer = self._make_greenlet_drainer(app) + calls = [0] + + def drain(timeout=None): + calls[0] += 1 + if calls[0] <= 3: + raise socket.timeout('idle') + drainer._stopped.set() + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + drainer.run() + + assert calls[0] >= 4 + assert drainer._exc is None + + # -- run: OSError is caught and logged ---------------------------------- + + def test_run_catches_oserror_and_logs(self, app): + """OSError in run() must be caught/logged, loop continues.""" + drainer = self._make_greenlet_drainer(app) + calls = [0] + + def drain(timeout=None): + calls[0] += 1 + if calls[0] <= 3: + raise OSError('connection reset') + drainer._stopped.set() + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + + with patch.object(logging, 'warning') as mock_warn, \ + patch('celery.backends.asynchronous.time.sleep') as mock_sleep: + drainer.run() + + assert calls[0] >= 4 + assert mock_warn.call_count >= 3 + # backoff sleep should have been called once per OSError + assert mock_sleep.call_count >= 3 + assert drainer._exc is None + + # -- run: unexpected Exception is stored and re-raised ------------------ + + def test_run_stores_and_reraises_unexpected_exception(self, app): + """Non-OSError / non-timeout exceptions must propagate and be stored.""" + drainer = self._make_greenlet_drainer(app) + + def drain(timeout=None): + raise RuntimeError('unexpected') + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + + with pytest.raises(RuntimeError, match='unexpected'): + drainer.run() + + assert drainer._exc is not None + assert drainer._shutdown.is_set() + + # -- _ensure_not_shut_down ---------------------------------------------- + + def test_ensure_not_shut_down_raises_stored_exc(self, app): + """If run() failed, _ensure_not_shut_down re-raises the exception.""" + drainer = self._make_greenlet_drainer(app) + drainer._shutdown.set() + drainer._exc = ValueError('boom') + + with pytest.raises(ValueError, match='boom'): + drainer._ensure_not_shut_down() + + def test_ensure_not_shut_down_raises_restart_msg(self, app): + """If stopped cleanly, _ensure_not_shut_down raises restart msg.""" + drainer = self._make_greenlet_drainer(app) + drainer._shutdown.set() + drainer._exc = None + + with pytest.raises(Exception, match=E_CELERY_RESTART_REQUIRED): + drainer._ensure_not_shut_down() + + def test_ensure_not_shut_down_noop_when_running(self, app): + """No-op when _shutdown is not set.""" + drainer = self._make_greenlet_drainer(app) + # Should not raise + drainer._ensure_not_shut_down() + + # -- start / stop ------------------------------------------------------- + + def test_start_spawns_and_waits(self, app): + """start() calls spawn(run) and waits for _started.""" + drainer = self._make_greenlet_drainer(app) + + def fake_spawn(func): + # Run synchronously with immediate stop. + drainer._stopped.set() + func() + + drainer.spawn = fake_spawn + drainer.result_consumer.drain_events = Mock( + side_effect=lambda timeout=None: drainer._stopped.set() + ) + drainer.start() + + assert drainer._started.is_set() + assert drainer._shutdown.is_set() + + def test_start_raises_if_already_shut_down(self, app): + """start() raises if drainer already completed.""" + drainer = self._make_greenlet_drainer(app) + drainer._shutdown.set() + + with pytest.raises(Exception, match=E_CELERY_RESTART_REQUIRED): + drainer.start() + + def test_stop_signals_and_waits(self, app): + """stop() sets _stopped and waits for _shutdown.""" + drainer = self._make_greenlet_drainer(app) + # Pre-set _shutdown so wait returns immediately. + drainer._shutdown.set() + drainer.stop() + + assert drainer._stopped.is_set() + + +# --------------------------------------------------------------------------- +# 3. Integration tests with real greenlet runtimes (gevent + eventlet) +# --------------------------------------------------------------------------- @pytest.fixture(autouse=True) def setup_eventlet(): - # By default eventlet will patch the DNS resolver when imported. os.environ.update(EVENTLET_NO_GREENDNS='yes') @@ -141,6 +450,40 @@ def test_drain_timeout(self): assert not p.ready, 'Promise should remain un-fulfilled' assert on_interval.call_count < 20, 'Should have limited number of calls to on_interval' + def test_drain_catches_and_logs_oserror(self): + p = promise() + + def fulfill(): + self.sleep(self.interval * 2) + p('done') + + t = self.schedule_thread(fulfill) + + state = {'n': 0} + + def flaky(*args, **kwargs): + state['n'] += 1 + if state['n'] == 1: + raise OSError('simulated broker restart') + # Yield to hub so the promise thread can run. + self.result_consumer_drain_events( + timeout=kwargs.get('timeout', None), + ) + + with patch.object( + self.drainer.result_consumer, 'drain_events', + side_effect=flaky, + ): + with patch('logging.warning') as mock_warn: + for _ in self.drainer.drain_events_until( + p, interval=self.interval, + timeout=self.MAX_TIMEOUT): + pass + + self.teardown_thread(t) + assert p.ready + assert mock_warn.called + class GreenletDrainerTests(DrainerTests): def test_drain_raises_when_greenlet_already_exited(self): @@ -182,6 +525,25 @@ def test_start_raises_if_drainer_already_stopped(self): self.teardown_thread(thread) + def test_run_catches_and_logs_oserror(self): + def flaky(*args, **kwargs): + if not hasattr(flaky, '_raised'): + flaky._raised = True + raise OSError('simulated broker restart in greenlet') + self.drainer._stopped.set() + + with patch.object( + self.drainer.result_consumer, 'drain_events', + side_effect=flaky, + ): + with patch('logging.warning') as mock_warn: + t = self.schedule_thread(self.drainer.run) + self.teardown_thread(t) + + assert mock_warn.called + assert 'connection error during drain_events' in mock_warn.call_args[0][0] + assert self.drainer._exc is None + @pytest.mark.skipif( sys.platform == "win32", @@ -190,6 +552,7 @@ def test_start_raises_if_drainer_already_stopped(self): class test_EventletDrainer(GreenletDrainerTests): @pytest.fixture(autouse=True) def setup_drainer(self): + pytest.importorskip('eventlet') self.drainer = self.get_drainer('eventlet') @cached_property @@ -245,6 +608,7 @@ def teardown_thread(self, thread): class test_GeventDrainer(GreenletDrainerTests): @pytest.fixture(autouse=True) def setup_drainer(self): + pytest.importorskip('gevent') self.drainer = self.get_drainer('gevent') @cached_property diff --git a/t/unit/backends/test_rpc.py b/t/unit/backends/test_rpc.py index 5d37689a31d..232ea05f6d4 100644 --- a/t/unit/backends/test_rpc.py +++ b/t/unit/backends/test_rpc.py @@ -20,6 +20,120 @@ def test_drain_events_before_start(self): # drain_events shouldn't crash when called before start consumer.drain_events(0.001) + def test_drain_events_reconnects_on_connection_error(self): + consumer = self.get_consumer() + # Simulate a started consumer with a live connection. + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (OSError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = OSError( + 'Server unexpectedly closed connection' + ) + consumer._connection = mock_conn + + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [Mock(name='queue1')] + consumer._consumer = mock_consumer + + # Patch app.connection() to return a fresh mock connection + # and Consumer to return a mock consumer. + new_conn = Mock(name='new_connection') + new_kombu_consumer = Mock(name='new_kombu_consumer') + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=new_kombu_consumer) + + # drain_events should NOT raise; it should reconnect instead. + consumer.drain_events(timeout=1) + + # Old connection should be closed. + mock_conn.close.assert_called_once() + # New connection should be established. + consumer.app.connection.assert_called_once() + assert consumer._connection is new_conn + # New consumer should be consuming. + assert consumer._consumer is new_kombu_consumer + new_kombu_consumer.consume.assert_called_once() + + def test_drain_events_reconnect_preserves_queues(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (ConnectionError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = ConnectionError('reset') + consumer._connection = mock_conn + + queue1, queue2 = Mock(name='q1'), Mock(name='q2') + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [queue1, queue2] + consumer._consumer = mock_consumer + + new_conn = Mock(name='new_connection') + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=Mock(name='new_kombu_consumer')) + + consumer.drain_events(timeout=1) + + # The new Consumer should have been created with both old queues. + new_consumer_call = consumer.Consumer.call_args + assert list(new_consumer_call[0][1]) == [queue1, queue2] + + def test_drain_events_no_reconnect_on_other_errors(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (OSError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = RuntimeError('unexpected') + consumer._connection = mock_conn + + with pytest.raises(RuntimeError, match='unexpected'): + consumer.drain_events(timeout=1) + + def test_reconnect_handles_close_failures_gracefully(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.close.side_effect = OSError('already closed') + consumer._connection = mock_conn + + mock_consumer = Mock(name='consumer') + mock_consumer.cancel.side_effect = OSError('channel gone') + mock_consumer.queues = [Mock(name='queue1')] + consumer._consumer = mock_consumer + + new_conn = Mock(name='new_connection') + new_kombu_consumer = Mock(name='new_kombu_consumer') + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=new_kombu_consumer) + + # _reconnect should NOT raise even if cancel/close fail + consumer._reconnect() + + assert consumer._connection is new_conn + new_kombu_consumer.consume.assert_called_once() + + def test_drain_events_channel_error_triggers_reconnect(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.connection_errors = () + mock_conn.channel_errors = (KeyError,) + mock_conn.drain_events.side_effect = KeyError('channel closed') + consumer._connection = mock_conn + + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [] + consumer._consumer = mock_consumer + + new_conn = Mock(name='new_connection') + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=Mock(name='new_kombu_consumer')) + + consumer.drain_events(timeout=1) + + assert consumer._connection is new_conn + class test_RPCBackend: From b3a8f9dc2056c024f560eb030f7a4afb7eae9e8a Mon Sep 17 00:00:00 2001 From: Doug Richardson Date: Sun, 8 Mar 2026 04:03:14 -0700 Subject: [PATCH 152/169] Fix NameError with TYPE_CHECKING annotations on Python 3.14+ (PEP 649) (#10165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix NameError with TYPE_CHECKING annotations on Python 3.14+ (PEP 649) In Python 3.14, annotations are deferred by default (PEP 649). Two sites in Celery eagerly triggered annotation evaluation, raising NameError for types only imported under TYPE_CHECKING: 1. `fun.__annotations__` in `_task_from_fun` (celery/app/base.py): Use `inspect.get_annotations(fun, format=Format.STRING)` on 3.14+ to return annotations as strings without evaluation. 2. `inspect.getfullargspec(fun)` in `head_from_fun` (utils/functional.py): On 3.14+, getfullargspec internally evaluates annotations and raises TypeError for unresolvable names. Fall back to reading the function's __code__ object directly (which is annotation-free) when that happens. Fixes https://github.com/celery/celery/discussions/10099 Assisted by AI * Remove unused annotationlib import in functional.py * Simplify _getfullargspec: use inspect.signature with Format.STRING Replace the __code__ introspection fallback with a cleaner approach: use inspect.signature(target, annotation_format=Format.STRING) which avoids annotation evaluation and handles all argument kinds directly. For bound methods, pass __func__ so that 'self' is included in args, preserving the behaviour of getfullargspec on older Python versions. * Skip PEP 649 regression tests on Python < 3.14 These tests exercise deferred annotation evaluation introduced by PEP 649, which is only the default on Python 3.14+. Skipping on earlier versions avoids NameError at exec time when annotations are evaluated eagerly. Assisted by AI * Update celery/app/base.py --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/base.py | 20 +++++++++++++++- celery/utils/functional.py | 42 ++++++++++++++++++++++++++++++++- t/unit/app/test_app.py | 20 ++++++++++++++++ t/unit/utils/test_functional.py | 16 +++++++++++++ 4 files changed, 96 insertions(+), 2 deletions(-) diff --git a/celery/app/base.py b/celery/app/base.py index 13a2f3862a1..fe18784809d 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -57,6 +57,24 @@ logger = get_logger(__name__) +if sys.version_info >= (3, 14): + import annotationlib + + def _get_annotations(fun): + # In Python 3.14+, annotations are deferred by default (PEP 649). + # Accessing fun.__annotations__ (or inspect.get_annotations without a + # format) evaluates them and may raise NameError for types only + # available under TYPE_CHECKING. To preserve previous behavior, first + # try to return evaluated annotations; if that fails with NameError, + # fall back to returning stringified annotations instead. + try: + return inspect.get_annotations(fun) + except NameError: + return inspect.get_annotations(fun, format=annotationlib.Format.STRING) +else: + def _get_annotations(fun): + return fun.__annotations__ + BUILTIN_FIXUPS = { 'celery.fixups.django:fixup', } @@ -594,7 +612,7 @@ def _task_from_fun( '_decorated': True, '__doc__': fun.__doc__, '__module__': fun.__module__, - '__annotations__': fun.__annotations__, + '__annotations__': _get_annotations(fun), '__header__': self.type_checker(fun, bound=bind), '__wrapped__': run}, **options))() # for some reason __qualname__ cannot be set in type() diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 5fb0d6339e5..762685b60cd 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -1,5 +1,6 @@ """Functional-style utilities.""" import inspect +import sys from collections import UserList from functools import partial from itertools import islice, tee, zip_longest @@ -311,6 +312,45 @@ def _argsfromspec(spec, replace_defaults=True): ])) +if sys.version_info >= (3, 14): + import annotationlib as _annotationlib + + def _getfullargspec(fun): + # In Python 3.14+, inspect.getfullargspec evaluates annotations by default + # (PEP 649), raising NameError for TYPE_CHECKING-only types. We don't need + # annotations here, so use Format.STRING to avoid evaluation. + # For bound methods, use __func__ so that 'self' is included in args, + # matching the behaviour of getfullargspec on older Python versions. + target = getattr(fun, '__func__', fun) + sig = inspect.signature(target, annotation_format=_annotationlib.Format.STRING) + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults = [], None, None, [], [], {} + for name, param in sig.parameters.items(): + kind = param.kind + if kind in (param.POSITIONAL_ONLY, param.POSITIONAL_OR_KEYWORD): + args.append(name) + if param.default is not param.empty: + defaults.append(param.default) + elif kind == param.VAR_POSITIONAL: + varargs = name + elif kind == param.KEYWORD_ONLY: + kwonlyargs.append(name) + if param.default is not param.empty: + kwonlydefaults[name] = param.default + elif kind == param.VAR_KEYWORD: + varkw = name + return inspect.FullArgSpec( + args=args, + varargs=varargs, + varkw=varkw, + defaults=tuple(defaults) or None, + kwonlyargs=kwonlyargs, + kwonlydefaults=kwonlydefaults or None, + annotations={}, + ) +else: + _getfullargspec = inspect.getfullargspec + + def head_from_fun(fun: Callable[..., Any], bound: bool = False) -> str: """Generate signature function from actual function.""" # we could use inspect.Signature here, but that implementation @@ -329,7 +369,7 @@ def head_from_fun(fun: Callable[..., Any], bound: bool = False) -> str: name = fun.__name__ definition = FUNHEAD_TEMPLATE.format( fun_name=name, - fun_args=_argsfromspec(inspect.getfullargspec(fun)), + fun_args=_argsfromspec(_getfullargspec(fun)), fun_value=1, ) logger.debug(definition) diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index 32cc338c336..dc5a4cc417e 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -831,6 +831,26 @@ def foo(parameter: int) -> None: assert typing.get_type_hints(foo) == { 'parameter': int, 'return': type(None)} + @pytest.mark.skipif(sys.version_info < (3, 14), reason="PEP 649 deferred annotations require Python 3.14+") + def test_task_with_type_checking_annotation(self): + # Regression test for https://github.com/celery/celery/discussions/10099 + # On Python 3.14+, annotations are deferred (PEP 649). Registering a task + # whose annotations reference TYPE_CHECKING-only types must not raise NameError. + local = {} + exec( + 'def foo(args: Sequence[str], x: int = 0): return args', + {'app': None}, + local, + ) + raw_fun = local['foo'] + + with self.Celery() as app: + task = app.task(raw_fun) + result = task.apply(args=(['hello'],)) + assert result.result == ['hello'] + # Annotations should be stored as strings, not evaluated + assert task.__annotations__['args'] == 'Sequence[str]' + def test_annotate_decorator(self): from celery.app.task import Task diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index c0bf626a747..fc2baa861e4 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,4 +1,5 @@ import collections +import sys import pytest from kombu.utils.functional import lazy @@ -368,6 +369,21 @@ def test_kwonly_required_args(self): g(b=3) + @pytest.mark.skipif(sys.version_info < (3, 14), reason="PEP 649 deferred annotations require Python 3.14+") + def test_type_checking_annotation(self): + # Regression test for https://github.com/celery/celery/discussions/10099 + # On Python 3.14+, annotations are deferred (PEP 649). Functions with + # annotations referencing TYPE_CHECKING-only types must not raise NameError. + local = {} + exec('def f(args: Sequence[str], x: int = 0): return args', {}, local) + f = local['f'] + + g = head_from_fun(f) + with pytest.raises(TypeError): + g() + g(1) + g(1, 2) + class test_fun_takes_argument: From d232c4132534a4e6fe475770b61f82036bc86ba1 Mon Sep 17 00:00:00 2001 From: William <3538066+tsangwailam@users.noreply.github.com> Date: Sun, 8 Mar 2026 11:05:27 +0000 Subject: [PATCH 153/169] docs: Add elaboration on prefetch multiplier settings (worker_prefetch_multiplier) and worker_eta_task_limit (#10181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: add description for prefetch multiplier related to worker_eta_task_limit * docs: add description for prefetch multiplier related to worker_eta_task_limit * Update docs/userguide/configuration.rst * Update docs/glossary.rst * Update docs/userguide/configuration.rst * Update docs/glossary.rst --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/glossary.rst | 7 +++++++ docs/userguide/configuration.rst | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/glossary.rst b/docs/glossary.rst index 0fe3988efad..a145d8ba98b 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -106,6 +106,13 @@ Glossary :setting:`worker_prefetch_multiplier` setting, which is multiplied by the number of pool slots (threads/processes/greenthreads). + .. note:: + If you are using eta or countdown tasks, the :setting:`worker_prefetch_multiplier` + still determines the base prefetch count. The :setting:`worker_eta_task_limit` + setting, when enabled, instead caps the total number of unacknowledged + messages the worker will hold (including eta/countdown tasks). See + :setting:`worker_eta_task_limit`. + `prefetch count` Maximum number of unacknowledged messages a consumer can hold and if exceeded the transport shouldn't deliver any more messages to that diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index 5642480eafc..0c93abd6cab 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -3302,8 +3302,18 @@ memory, potentially causing out-of-memory issues. .. note:: - Tasks with ETA/countdown aren't affected by prefetch limits. - + Tasks with ETA/countdown are fetched into memory and scheduled on an internal + timer, so they are not constrained by the per-process prefetch window derived + from :setting:`worker_prefetch_multiplier` in the same way as immediately + executed tasks. This is why ``--prefetch-multiplier=1`` can appear to have no + effect when many ETA/countdown tasks are present. + + :setting:`worker_eta_task_limit` configures the maximum number of ETA/countdown + tasks a worker will hold in memory and also sets an overall cap on + unacknowledged messages via kombu's QoS ``max_prefetch``. If the prefetch count + implied by :setting:`worker_prefetch_multiplier` would exceed this cap, the + worker will stop consuming new messages until previously received tasks have + been acknowledged. .. setting:: worker_disable_prefetch ``worker_disable_prefetch`` From b34c1e1027fcbf11fa3efab106dbe8d77257704e Mon Sep 17 00:00:00 2001 From: Feliks Borzik Date: Sun, 8 Mar 2026 11:10:32 +0000 Subject: [PATCH 154/169] =?UTF-8?q?Fix=20O(K=C2=B2)=20message=20bloat=20in?= =?UTF-8?q?=20a=20chain=20of=20chords=20(#10171)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix multiple chord chaining * Apply suggestions from code review * fix test name --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/canvas.py | 6 +++--- t/integration/test_canvas.py | 42 ++++++++++++++++++++++++++++++++++++ t/unit/tasks/test_canvas.py | 34 +++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/celery/canvas.py b/celery/canvas.py index 396eb7d307b..5d701839c1f 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -989,13 +989,13 @@ def __or__(self, other): sig.tasks[-2].body = sig.tasks[-2].body | sig.tasks[-1] sig.tasks = sig.tasks[:-1] return sig - elif self.tasks and isinstance(self.tasks[-1], chord): - # CHAIN [last item is chord] -> chain with chord body. + elif self.tasks and isinstance(self.tasks[-1], chord) and not isinstance(other, chord): + # CHAIN [last item is chord] | TASK -> chain with chord body. sig = self.clone() sig.tasks[-1].body = sig.tasks[-1].body | other return sig else: - # chain | task -> chain + # chain | task/chord -> chain # use type(self) for _chain subclasses return type(self)(seq_concat_item( self.unchain_tasks(), other), app=self._app) diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 8d137b6a805..729c94e5d61 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -440,6 +440,48 @@ def test_chain_of_chords_with_two_tasks(self, manager): res = c() assert res.get(timeout=TIMEOUT) == 12 + @flaky + def test_chain_of_explicit_chords(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + c1 = chain( + chord(group(add.si(1, 0), add.si(1, 0)), tsum.s()), + chord(group(add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(0), add.s(0)), tsum.s()), + ) + c2 = chain( + chord(group(add.s(10), add.s(10)), tsum.s()), + chord(group(add.s(0), add.s(0)), tsum.s()), + chord(group(add.s(1), add.s(1)), tsum.s()), + ) + c = c1 | c2 + res = c() + assert res.get(timeout=TIMEOUT) == 178 + + @flaky + def test_chain_of_nine_chords(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + c = chain( + chord(group(add.si(1, 0), add.si(1, 0), add.si(1, 0)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(0), add.s(0), add.s(0)), tsum.s()), + ) + res = c() + assert res.get(timeout=TIMEOUT) == 29520 + @flaky def test_chain_of_a_chord_and_a_group_with_two_tasks(self, manager): try: diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 9b97c3dd65a..144ee625be7 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -592,6 +592,40 @@ def test_chain_of_chord_upgrade_on_chaining__protocol_3(self): ), "Chord followed by a group should be upgraded to a single chord with chained body." assert len(c.tasks) == 6 + def test_chain_of_chords_stays_flat(self): + c = chain( + chord([signature('h1'), signature('h2')], signature('b1'), app=self.app), + chord([signature('h3'), signature('h4')], signature('b2'), app=self.app), + chord([signature('h5'), signature('h6')], signature('b3'), app=self.app), + ) + assert isinstance(c, _chain) + assert len(c.tasks) == 3 + for task in c.tasks: + assert isinstance(task, chord) + assert not isinstance(c.tasks[0].body, _chain) + assert not isinstance(c.tasks[1].body, _chain) + assert not isinstance(c.tasks[2].body, _chain) + + def test_chain_of_chords_serialized_size_constant(self): + chords = [ + chord([signature(f'h{i}_{j}') for j in range(3)], + signature(f'b{i}'), app=self.app) + for i in range(6) + ] + c = chain(*chords) + assert isinstance(c, _chain) + sizes = [len(json.dumps(task.__json__())) for task in c.tasks] + assert max(sizes) == min(sizes), ( + f"Chord sizes not constant across chain: {sizes}" + ) + + def test_chord_or_task_still_nests(self): + c = chord([signature('h1')], signature('b1'), app=self.app) + t = signature('t1') + result = chain(c) | t + assert isinstance(result, _chain) + assert isinstance(result.tasks[0].body, _chain) + def test_apply_options(self): class static(Signature): From 37f16e01293e3d34fe06d108384ef329638ee39e Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Sun, 8 Mar 2026 22:01:48 +0800 Subject: [PATCH 155/169] test: mock the channel and connection errors in a more elegant way (#10178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- t/unit/worker/test_bootsteps.py | 4 ++++ t/unit/worker/test_consumer.py | 10 ++++++++++ t/unit/worker/test_worker.py | 4 ++++ 3 files changed, 18 insertions(+) diff --git a/t/unit/worker/test_bootsteps.py b/t/unit/worker/test_bootsteps.py index 4a33f44da35..6443d660626 100644 --- a/t/unit/worker/test_bootsteps.py +++ b/t/unit/worker/test_bootsteps.py @@ -122,6 +122,8 @@ def test_interface(self): def test_start_stop_shutdown(self): consumer = Mock() self.connection = Mock() + self.connection.connection_errors = () + self.connection.channel_errors = () class Step(bootsteps.ConsumerStep): @@ -141,6 +143,8 @@ def get_consumers(self, c): def test_start_no_consumers(self): self.connection = Mock() + self.connection.connection_errors = () + self.connection.channel_errors = () class Step(bootsteps.ConsumerStep): diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index df3f478ce27..68e12cc3a0d 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -534,6 +534,8 @@ def test_disable_prefetch_not_enabled(self): consumer.controller.max_concurrency = None consumer.initial_prefetch_count = 16 consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () consumer.connection.default_channel = Mock() consumer.connection.transport = Mock() consumer.connection.transport.driver_type = 'redis' @@ -571,6 +573,8 @@ def test_disable_prefetch_enabled_basic(self): consumer.controller.max_concurrency = None consumer.initial_prefetch_count = 16 consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () consumer.connection.default_channel = Mock() consumer.connection.transport = Mock() consumer.connection.transport.driver_type = 'redis' @@ -611,6 +615,8 @@ def test_disable_prefetch_respects_reserved_requests_limit(self): consumer.controller.max_concurrency = None consumer.initial_prefetch_count = 16 consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () consumer.connection.default_channel = Mock() consumer.connection.transport = Mock() consumer.connection.transport.driver_type = 'redis' @@ -651,6 +657,8 @@ def test_disable_prefetch_respects_autoscale_max_concurrency(self): consumer.controller.max_concurrency = 2 # Lower than pool processes consumer.initial_prefetch_count = 16 consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () consumer.connection.default_channel = Mock() consumer.connection.transport = Mock() consumer.connection.transport.driver_type = 'redis' @@ -691,6 +699,8 @@ def test_disable_prefetch_ignored_for_non_redis_brokers(self): consumer.controller.max_concurrency = None consumer.initial_prefetch_count = 16 consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () consumer.connection.default_channel = Mock() consumer.connection.transport = Mock() consumer.connection.transport.driver_type = 'amqp' # RabbitMQ diff --git a/t/unit/worker/test_worker.py b/t/unit/worker/test_worker.py index f0bc1ca8c5f..d0459adf9ba 100644 --- a/t/unit/worker/test_worker.py +++ b/t/unit/worker/test_worker.py @@ -111,6 +111,8 @@ def LoopConsumer(self, buffer=None, controller=None, timer=None, app=None, c.task_consumer = Mock(name='.task_consumer') c.qos = QoS(c.task_consumer.qos, 10) c.connection = Mock(name='.connection') + c.connection.connection_errors = () + c.connection.channel_errors = () c.controller = c.app.WorkController() c.heart = Mock(name='.heart') c.controller.consumer = c @@ -249,6 +251,8 @@ def _get_on_message(self, c): c.task_consumer = Mock() c.event_dispatcher = mock_event_dispatcher() c.connection = Mock(name='.connection') + c.connection.connection_errors = () + c.connection.channel_errors = () c.connection.get_heartbeat_interval.return_value = 0 c.connection.drain_events.side_effect = WorkerShutdown() From 67ab3d27d4d7bab48b1141c677ae30a712781a20 Mon Sep 17 00:00:00 2001 From: Rana Aurangzaib Date: Mon, 9 Mar 2026 14:06:27 +0300 Subject: [PATCH 156/169] fix(trace): dispatch chain/callbacks on dedup fast-path for redelivered tasks (#10159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(trace): dispatch chain/callbacks on dedup fast-path for redelivered tasks (#9835) When a redelivered task is deduplicated via backend state check, the dedup fast-path returned immediately without dispatching the chain or callbacks from the message. This caused chains to be permanently lost when a worker crashed after mark_as_done but before the broker ack. Also replace chain.pop() with non-mutating chain[-1] + chain[:-1] to prevent in-place corruption of task_request.chain on retry/redelivery. Closes #9835 * fix(trace): harden dedup dispatch and add successful_requests guard (#9835) - Move successful_requests.add() inside try block so failed dispatches allow retry on next redelivery - Handle group callbacks in dedup path matching normal success path (Issue #1936) - Pass priority to callback dispatch in dedup path - Re-raise MemoryError from dedup dispatch exception handler - Escalate dispatch failure log from warning to error - Remove leftover fault injection debug code from chain dispatch - Add tests for multi-element chain, successful_requests guard, combined chain+callbacks, and dispatch failure logging * fix(test): clean up successful_requests and align task_id in dedup tests - Add successful_requests.discard(task_id) to tests that trigger the backend-dedup path to avoid leaking global state - Pass task_id to trace() in test_chain_dispatch_does_not_mutate to match request['id'] * fix(trace): guard dedup dispatch with stored children to avoid duplicates Skip chain/callback dispatch when the stored result meta already contains children, indicating the previous worker successfully dispatched them before storing the result. * fix(test): patch maybe_signature in chain mutation test to avoid side effects * fix(trace): match normal path dispatch order in dedup branch (callbacks before chain) * fix(test): add test for children guard skipping dedup dispatch * Update celery/app/trace.py * fix(trace): raise Reject(requeue=True) on dedup dispatch failure instead of acking * fix(trace): track push_task/push_request separately to unwind partial pushes * refactor(trace): extract shared dispatch helper, remove dead children persistence Extract callback/chain dispatch logic into _dispatch_callbacks_and_chain() helper inside build_tracer, called from both the normal success path and the dedup fast-path. This eliminates duplicated dispatch code that could drift between the two paths. Remove the push_task/push_request context wrapper and children persistence block from the dedup path — store_result is silently blocked by the KV-backend SUCCESS guard so children never persist, and the _meta dict mutation was a local no-op. * test(trace): cover MemoryError propagation and Reject passthrough in dedup path Add tests for two uncovered lines flagged by Codecov: - MemoryError during dedup dispatch propagates directly (not wrapped in Reject) - Reject from dedup dispatch propagates through the module-level trace_task wrapper * test(integration): add integration tests for dedup chain/callback dispatch Add 4 integration tests exercising the full worker pipeline with Redis broker/backend to verify chain and callback dispatch on the dedup fast-path (issue #9835). New integration tasks: store_success_then_reject (simulates redelivery by storing SUCCESS then raising Reject) and reject_then_succeed (tests normal reject passthrough). Also address review findings: consolidate redundant _get_task_meta() calls, document the _children guard and partial-dispatch limitations, strengthen root_id assertion, and add unit tests for root_id fallback and empty-chain edge case. * refactor(test): move dedup integration tests to standalone file, fix CI - Move test_dedup_chain_dispatch to its own file following test_prefork_shutdown.py pattern (celery_session_app + start_worker) - Add test_dedup_chain_dispatch.py to CI integration test matrix - Extract flaky/timeout boilerplate from test_canvas.py to conftest.py - Fix skip condition to use backend.persistent instead of startswith('redis') - Wrap dedup_worker fixture teardown in try/finally - Fix callback test to assert actual callback result via AsyncResult - Move inline Reject import to module level in tasks.py - Add unit test for backend-read failure during dedup check * style: remove trailing blank line in test_canvas.py * Update t/integration/test_dedup_chain_dispatch.py --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- .github/workflows/python-package.yml | 1 + celery/app/trace.py | 131 ++++-- t/integration/conftest.py | 17 + t/integration/tasks.py | 23 +- t/integration/test_canvas.py | 16 +- t/integration/test_dedup_chain_dispatch.py | 87 ++++ t/unit/tasks/test_trace.py | 517 +++++++++++++++++++++ 7 files changed, 739 insertions(+), 53 deletions(-) create mode 100644 t/integration/test_dedup_chain_dispatch.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6235f5cdd82..66f034e7058 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -95,6 +95,7 @@ jobs: module: [ 'test_backend.py', 'test_canvas.py', + 'test_dedup_chain_dispatch.py', 'test_inspect.py', 'test_loader.py', 'test_mem_leak_in_exception_handling.py', diff --git a/celery/app/trace.py b/celery/app/trace.py index b6289709365..ecd26994288 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -409,6 +409,55 @@ def on_error(request, exc, state=FAILURE, call_errbacks=True): ) return I, R, I.state, I.retval + def _dispatch_callbacks_and_chain( + retval, callbacks, chain, parent_id, root_id, priority, + ): + """Dispatch callbacks and chain for a completed task. + + Dispatches link callbacks and then the next chain step. + Does NOT fire task lifecycle signals (on_success, task_postrun) + or call mark_as_done — callers handle those separately. + + Note: dispatch is not atomic. If callbacks succeed but the + chain step fails (or vice-versa), a Reject + redeliver may + re-dispatch the already-sent callbacks. This is acceptable + under Celery's at-least-once delivery model. + """ + if callbacks: + if len(callbacks) > 1: + sigs, groups = [], [] + for sig in callbacks: + sig = signature(sig, app=app) + if isinstance(sig, group): + groups.append(sig) + else: + sigs.append(sig) + for group_ in groups: + group_.apply_async( + (retval,), + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + if sigs: + group(sigs, app=app).apply_async( + (retval,), + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + else: + signature(callbacks[0], app=app).apply_async( + (retval,), + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + if chain: + _chsig = signature(chain[-1], app=app) + _chsig.apply_async( + (retval,), chain=chain[:-1], + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + def trace_task(uuid, args, kwargs, request=None): # R - is the possibly prepared return value. # I - is the Info object. @@ -452,6 +501,41 @@ def trace_task(uuid, args, kwargs, request=None): 'name': get_task_name(task_request, name), 'description': 'Task already completed successfully.' }) + _root_id = task_request.root_id or uuid + _priority = task_request.delivery_info.get('priority') if \ + inherit_parent_priority else None + try: + _meta = r._get_task_meta() + stored_retval = _meta.get('result') + # Children are populated by mark_as_done on the + # original execution. If present, callbacks were + # already dispatched — skip to avoid duplicates. + # Requires the backend to persist extended result + # metadata (result_extended=True). + _children = _meta.get('children') + _callbacks = task_request.callbacks + _chain = task_request.chain + if (_callbacks or _chain) and not _children: + _dispatch_callbacks_and_chain( + stored_retval, _callbacks, _chain, + parent_id=uuid, root_id=_root_id, + priority=_priority, + ) + successful_requests.add(task_request.id) + except MemoryError: + raise + except Exception as exc: + # Permanent failures (malformed signature, etc.) + # will requeue indefinitely. Broker-level + # dead-letter / max-delivery-count policies are + # the intended circuit-breaker. + logger.error( + 'Failed to dispatch chain/callbacks for ' + 'deduplicated task %s', + task_request.id, + exc_info=True, + ) + raise Reject(exc, requeue=True) return trace_ok_t(R, I, T, Rstr) push_task(task) @@ -510,43 +594,12 @@ def trace_task(uuid, args, kwargs, request=None): # separately, so need to call them separately # so that the trail's not added multiple times :( # (Issue #1936) - callbacks = task.request.callbacks - if callbacks: - if len(task.request.callbacks) > 1: - sigs, groups = [], [] - for sig in callbacks: - sig = signature(sig, app=app) - if isinstance(sig, group): - groups.append(sig) - else: - sigs.append(sig) - for group_ in groups: - group_.apply_async( - (retval,), - parent_id=uuid, root_id=root_id, - priority=task_priority - ) - if sigs: - group(sigs, app=app).apply_async( - (retval,), - parent_id=uuid, root_id=root_id, - priority=task_priority - ) - else: - signature(callbacks[0], app=app).apply_async( - (retval,), parent_id=uuid, root_id=root_id, - priority=task_priority - ) - - # execute first task in chain - chain = task_request.chain - if chain: - _chsig = signature(chain.pop(), app=app) - _chsig.apply_async( - (retval,), chain=chain, - parent_id=uuid, root_id=root_id, - priority=task_priority - ) + _dispatch_callbacks_and_chain( + retval, task.request.callbacks, + task_request.chain, + parent_id=uuid, root_id=root_id, + priority=task_priority, + ) task.backend.mark_as_done( uuid, retval, task_request, publish_result, ) @@ -597,6 +650,8 @@ def trace_task(uuid, args, kwargs, request=None): exc_info=True) except MemoryError: raise + except Reject: + raise except Exception as exc: _signal_internal_error(task, uuid, args, kwargs, request, exc) if eager: @@ -616,6 +671,8 @@ def trace_task(task, uuid, args, kwargs, request=None, **opts): if task.__trace__ is None: task.__trace__ = build_tracer(task.name, task, **opts) return task.__trace__(uuid, args, kwargs, request) + except Reject: + raise except Exception as exc: _signal_internal_error(task, uuid, args, kwargs, request, exc) return trace_ok_t(report_internal_error(task, exc), TraceInfo(FAILURE, exc), 0.0, None) diff --git a/t/integration/conftest.py b/t/integration/conftest.py index 2383cb2d9b6..1e05302b75e 100644 --- a/t/integration/conftest.py +++ b/t/integration/conftest.py @@ -8,6 +8,7 @@ from celery.contrib.pytest import celery_app, celery_session_worker from celery.contrib.testing.manager import Manager +from celery.exceptions import TimeoutError from t.integration.tasks import get_redis_connection # we have to import the pytest plugin fixtures here, @@ -20,9 +21,25 @@ TEST_BROKER = os.environ.get('TEST_BROKER', 'pyamqp://') TEST_BACKEND = os.environ.get('TEST_BACKEND', 'redis://') +RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) + + +def is_retryable_exception(exc): + return isinstance(exc, RETRYABLE_EXCEPTIONS) + + +_flaky = pytest.mark.flaky(reruns=5, reruns_delay=1, cause=is_retryable_exception) +_timeout = pytest.mark.timeout(timeout=300) + + +def flaky(fn): + return _timeout(_flaky(fn)) + + __all__ = ( 'celery_app', 'celery_session_worker', + 'flaky', 'get_active_redis_channels', ) diff --git a/t/integration/tasks.py b/t/integration/tasks.py index ff823b96cbc..4bd874a8882 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -6,7 +6,7 @@ from celery import Signature, Task, chain, chord, group, shared_task from celery.canvas import signature -from celery.exceptions import SoftTimeLimitExceeded +from celery.exceptions import Reject, SoftTimeLimitExceeded from celery.utils.log import get_task_logger LEGACY_TASKS_DISABLED = True @@ -522,6 +522,27 @@ def replace_with_stamped_task(self: StampedTaskOnReplace, replace_with=None): self.replace(signature(replace_with)) +@shared_task(bind=True, acks_late=True) +def store_success_then_reject(self): + """First delivery: store SUCCESS manually, then Reject to trigger redelivery. + Second delivery: dedup finds SUCCESS, dispatches chain.""" + from celery.backends.base import states + if not self.request.delivery_info.get('redelivered'): + self.backend.store_result(self.request.id, 'first-pass', states.SUCCESS) + raise Reject(requeue=True) + # When dedup is enabled the fast-path intercepts before reaching here, + # so 'dedup-pass' is only returned when dedup is disabled. + return 'dedup-pass' + + +@shared_task(bind=True, acks_late=True) +def reject_then_succeed(self): + """First delivery: Reject(requeue=True). Second delivery: succeed normally.""" + if not self.request.delivery_info.get('redelivered'): + raise Reject(requeue=True) + return 'second-pass' + + @shared_task(soft_time_limit=2, time_limit=1) def soft_time_limit_must_exceed_time_limit(): pass diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index 729c94e5d61..b1daaae8619 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -15,7 +15,7 @@ from celery.signals import before_task_publish, task_received from . import tasks -from .conftest import TEST_BACKEND, check_for_logs, get_active_redis_channels, get_redis_connection +from .conftest import TEST_BACKEND, check_for_logs, flaky, get_active_redis_channels, get_redis_connection from .tasks import (ExpectedException, StampOnReplace, add, add_chord_to_chord, add_replaced, add_to_all, add_to_all_to_chord, build_chain_inside_task, collect_ids, delayed_sum, delayed_sum_with_soft_guard, errback_new_style, errback_old_style, fail, fail_replaced, identity, @@ -24,22 +24,8 @@ replace_with_stamped_task, retry_once, return_exception, return_priority, second_order_replace1, tsum, write_to_file_and_return_int, xsum) -RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) - - -def is_retryable_exception(exc): - return isinstance(exc, RETRYABLE_EXCEPTIONS) - - TIMEOUT = 60 -_flaky = pytest.mark.flaky(reruns=5, reruns_delay=1, cause=is_retryable_exception) -_timeout = pytest.mark.timeout(timeout=300) - - -def flaky(fn): - return _timeout(_flaky(fn)) - def await_redis_echo(expected_msgs, redis_key="redis-echo", timeout=TIMEOUT): """ diff --git a/t/integration/test_dedup_chain_dispatch.py b/t/integration/test_dedup_chain_dispatch.py new file mode 100644 index 00000000000..013288cc589 --- /dev/null +++ b/t/integration/test_dedup_chain_dispatch.py @@ -0,0 +1,87 @@ +"""Integration tests for chain/callback dispatch on the dedup fast-path. + +When ``worker_deduplicate_successful_tasks=True`` and +``task_acks_late=True``, a redelivered task that hits the dedup +fast-path in ``trace.py`` must still dispatch its chain and link +callbacks. + +See https://github.com/celery/celery/issues/9835 +""" + +import pytest + +from celery import chain +from celery.contrib.testing.worker import start_worker +from celery.result import AsyncResult + +from .conftest import flaky +from .tasks import add, identity, reject_then_succeed, store_success_then_reject + +TIMEOUT = 60 + + +@pytest.fixture() +def dedup_worker(celery_session_app): + """Solo worker with dedup enabled. + + Temporarily enables ``worker_deduplicate_successful_tasks`` on the + session app, starts a solo worker, and restores the original + setting on teardown. + """ + if not celery_session_app.backend.persistent: + raise pytest.skip('Requires a persistent result backend.') + + orig_dedup = celery_session_app.conf.worker_deduplicate_successful_tasks + orig_acks_late = celery_session_app.conf.task_acks_late + celery_session_app.conf.worker_deduplicate_successful_tasks = True + celery_session_app.conf.task_acks_late = True + + try: + with start_worker( + celery_session_app, + pool='solo', + concurrency=1, + perform_ping_check=False, + shutdown_timeout=TIMEOUT, + ) as worker: + yield worker + finally: + celery_session_app.conf.worker_deduplicate_successful_tasks = orig_dedup + celery_session_app.conf.task_acks_late = orig_acks_late + + +class test_dedup_chain_dispatch: + """Test chain/callback dispatch on the dedup fast-path.""" + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_chain_completes_with_dedup_enabled(self): + """Smoke test: a normal chain works when dedup is on.""" + c = chain(add.s(2, 3), add.s(5)) + assert c().get(timeout=TIMEOUT) == 10 + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_reject_requeue_completes_chain(self): + """Reject passthrough: chain completes after rejection + redelivery.""" + c = chain(reject_then_succeed.s(), identity.s()) + assert c().get(timeout=TIMEOUT) == 'second-pass' + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_dedup_dispatches_chain_on_redelivery(self): + """Core test: dedup fast-path dispatches the chain.""" + c = chain(store_success_then_reject.s(), identity.s()) + assert c().get(timeout=TIMEOUT) == 'first-pass' + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_dedup_dispatches_callback_on_redelivery(self, celery_session_app): + """Dedup fast-path dispatches link callbacks.""" + import uuid as _uuid + cb_id = _uuid.uuid4().hex + sig = store_success_then_reject.s() + sig.link(identity.s().set(task_id=cb_id)) + sig.apply_async() + cb_result = AsyncResult(cb_id, app=celery_session_app) + assert cb_result.get(timeout=TIMEOUT) == 'first-pass' diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index cd0c8c6901e..641fe25d7e3 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -14,6 +14,7 @@ from celery.backends.base import BaseDictBackend from celery.backends.cache import CacheBackend from celery.exceptions import BackendGetMetaError, Ignore, Reject, Retry +from celery.result import AsyncResult from celery.states import PENDING from celery.worker.state import successful_requests @@ -564,6 +565,522 @@ def add(x, y): successful_requests.clear() self.app.conf.worker_deduplicate_successful_tasks = False + def test_deduplicate_successful_tasks__backend_dedup_dispatches_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_apply.assert_called_once() + call_args = mock_apply.call_args + assert call_args[0] == ((2,),) + assert call_args[1]['parent_id'] == task_id + assert call_args[1]['root_id'] == task_id + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_multi_element_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + step2 = self.add.s(20) + step3 = self.add.s(30) + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [step3, step2], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_apply.assert_called_once() + call_args = mock_apply.call_args + assert call_args[1]['chain'] == [step3] + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_adds_to_successful_requests(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + successful_requests.discard(task_id) + + request_dedup = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + } + with patch('celery.canvas.maybe_signature'): + trace(self.app, add, (1, 1), task_id=task_id, request=request_dedup) + + assert task_id in successful_requests + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_dispatch_failure_skips_successful_requests(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + successful_requests.discard(task_id) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = RuntimeError('broker down') + with patch('celery.app.trace.logger'): + with pytest.raises(Reject): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + + assert task_id not in successful_requests + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__inmemory_dedup_skips_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + + task_id = str(uuid4()) + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + successful_requests.add(task_id) + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_apply.assert_not_called() + + successful_requests.clear() + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_chain_dispatch_does_not_mutate_request_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + + chain_list = [self.add.s(10), self.add.s(20)] + original_length = len(chain_list) + task_id = str(uuid4()) + request = { + 'id': task_id, + 'delivery_info': {'redelivered': False}, + 'chain': chain_list, + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async = Mock() + trace(self.app, add, (1, 1), task_id=task_id, request=request) + call_args = mock_signature.return_value.apply_async.call_args + assert call_args[1]['chain'] == chain_list[:-1] + assert len(chain_list) == original_length + + def test_deduplicate_successful_tasks__backend_dedup_dispatches_callbacks(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_callbacks = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'callbacks': [self.add.s(99)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_callbacks) + mock_apply.assert_called_once() + call_args = mock_apply.call_args + assert call_args[0] == ((2,),) + assert call_args[1]['parent_id'] == task_id + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_chain_and_callbacks(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_both = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + 'callbacks': [self.add.s(99)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_both) + assert mock_apply.call_count == 2 + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_skips_when_children_present(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + 'callbacks': [self.add.s(99)], + } + + meta_with_children = { + 'status': 'SUCCESS', 'result': 2, + 'children': [('some-child-id', None)], + } + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + with patch('celery.result.AsyncResult._get_task_meta', + return_value=meta_with_children): + trace(self.app, add, (1, 1), task_id=task_id, + request=request_with_chain) + mock_apply.assert_not_called() + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_dispatch_failure_logged(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = RuntimeError('broker down') + with patch('celery.app.trace.logger') as mock_logger: + with pytest.raises(Reject): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_logger.error.assert_called_once() + assert 'deduplicated task' in mock_logger.error.call_args[0][0] + + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_memory_error_propagates(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = MemoryError() + with pytest.raises(MemoryError): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__reject_propagates_through_trace_task(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + add.__trace__ = None + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = RuntimeError('broker down') + with patch('celery.app.trace.logger'): + with pytest.raises(Reject): + trace_task(add, task_id, (1, 1), {}, request=request_with_chain, app=self.app) + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__root_id_fallback(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_no_root_id = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_no_root_id) + call_args = mock_apply.call_args + assert call_args[1]['root_id'] == task_id + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__empty_chain_skips_dispatch(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_empty_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [], + 'callbacks': [], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_empty_chain) + mock_apply.assert_not_called() + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_read_failure_rejects(self): + """When _get_task_meta() fails after state==SUCCESS, the exception + is caught and re-raised as Reject(requeue=True).""" + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + successful_requests.discard(task_id) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + # First call to _get_task_meta (from r.state) returns normally; + # second call (line 508 in trace.py) raises to simulate a + # transient backend failure during dispatch. + original = AsyncResult._get_task_meta + call_count = 0 + + def fail_on_second_call(self_): + nonlocal call_count + call_count += 1 + if call_count >= 2: + raise ConnectionError('redis gone') + return original(self_) + + with patch.object(AsyncResult, '_get_task_meta', fail_on_second_call): + with patch('celery.app.trace.logger'): + with pytest.raises(Reject): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + + assert task_id not in successful_requests + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + class test_TraceInfo(TraceCase): class TI(TraceInfo): From 2e51ec6782ae3a9d030a94298470224ce7cf2a2a Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Tue, 10 Mar 2026 15:47:22 +0800 Subject: [PATCH 157/169] Extract `reconnect_on_error` to `BaseResultConsumer` (#10189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: extract the share reconnect function * test: reconnect on error in asynchronous * fix: reset the connection error and add unittest * fix: lint --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/backends/asynchronous.py | 42 +++++++++++++++ celery/backends/redis.py | 19 ++----- celery/backends/rpc.py | 28 +++++----- t/unit/backends/test_asynchronous.py | 78 ++++++++++++++++++++++++++++ t/unit/backends/test_rpc.py | 35 +++++++++++++ 5 files changed, 171 insertions(+), 31 deletions(-) diff --git a/celery/backends/asynchronous.py b/celery/backends/asynchronous.py index 0413afecc8a..c5f292ceb6e 100644 --- a/celery/backends/asynchronous.py +++ b/celery/backends/asynchronous.py @@ -5,6 +5,7 @@ import threading import time from collections import deque +from contextlib import contextmanager from queue import Empty from time import sleep from weakref import WeakKeyDictionary @@ -13,10 +14,18 @@ from celery import states from celery.exceptions import TimeoutError +from celery.utils.log import get_logger from celery.utils.threads import THREAD_TIMEOUT_MAX E_CELERY_RESTART_REQUIRED = "Celery must be restarted because a shutdown signal was detected." +E_RETRY_LIMIT_EXCEEDED = """ +Retry limit exceeded while trying to reconnect to the Celery result store +backend. The Celery application must be restarted. +""" + +logger = get_logger(__name__) + __all__ = ( 'AsyncBackendMixin', 'BaseResultConsumer', 'Drainer', 'register_drainer', @@ -307,6 +316,11 @@ def is_async(self): class BaseResultConsumer: """Manager responsible for consuming result messages.""" + #: Tuple of transport-layer exceptions that signal a lost connection. + #: Subclasses should override this with the appropriate exception types + #: so that :meth:`reconnect_on_error` can catch and recover from them. + _connection_errors = () + def __init__(self, backend, app, accept, pending_results, pending_messages): self.backend = backend @@ -321,6 +335,34 @@ def __init__(self, backend, app, accept, def start(self, initial_task_id, **kwargs): raise NotImplementedError() + @contextmanager + def reconnect_on_error(self): + """Context manager that catches connection errors and reconnects. + + Wraps a block of code so that any :attr:`_connection_errors` raised + inside it trigger a call to :meth:`_reconnect`. If reconnection + itself raises a connection error the consumer is considered + unrecoverable and a :exc:`RuntimeError` is raised to signal that + the Celery application must be restarted. + """ + try: + yield + except self._connection_errors: + try: + self._reconnect() + except self._connection_errors as exc: + logger.critical(E_RETRY_LIMIT_EXCEEDED) + raise RuntimeError(E_RETRY_LIMIT_EXCEEDED) from exc + + def _reconnect(self): + """Re-establish the backend connection. + + Subclasses must override this method to perform the transport-specific + reconnection logic that should be executed when a connection error is + caught by :meth:`reconnect_on_error`. + """ + pass + def stop(self): pass diff --git a/celery/backends/redis.py b/celery/backends/redis.py index d31eb66b143..154285f2a7e 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -1,6 +1,5 @@ """Redis result store backend.""" import time -from contextlib import contextmanager from functools import partial from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED from urllib.parse import unquote @@ -72,11 +71,6 @@ E_LOST = 'Connection to Redis lost: Retry (%s/%s) %s.' -E_RETRY_LIMIT_EXCEEDED = """ -Retry limit exceeded while trying to reconnect to the Celery redis result \ -store backend. The Celery application must be restarted. -""" - logger = get_logger(__name__) @@ -122,16 +116,9 @@ def _reconnect_pubsub(self): # The on_connect callback will re-subscribe to any channels we previously subscribed to. self._pubsub.connection.register_connect_callback(self._pubsub.on_connect) - @contextmanager - def reconnect_on_error(self): - try: - yield - except self._connection_errors: - try: - self._ensure(self._reconnect_pubsub, ()) - except self._connection_errors as e: - logger.critical(E_RETRY_LIMIT_EXCEEDED) - raise RuntimeError(E_RETRY_LIMIT_EXCEEDED) from e + def _reconnect(self): + """Re-establish the Redis pub/sub connection with retry.""" + self._ensure(self._reconnect_pubsub, ()) def _maybe_cancel_ready_task(self, meta): if meta['status'] in states.READY_STATES: diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 42fef2072c5..577eb7d404f 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -4,7 +4,6 @@ """ import logging import time -from contextlib import contextmanager import kombu from kombu.common import maybe_declare @@ -53,6 +52,10 @@ def __init__(self, *args, **kwargs): def start(self, initial_task_id, no_ack=True, **kwargs): self._no_ack = no_ack self._connection = self.app.connection() + self._connection_errors = ( + self._connection.connection_errors + + self._connection.channel_errors + ) initial_queue = self._create_binding(initial_task_id) self._consumer = self.Consumer( self._connection.default_channel, [initial_queue], @@ -60,22 +63,9 @@ def start(self, initial_task_id, no_ack=True, **kwargs): accept=self.accept) self._consumer.consume() - @contextmanager - def _handle_connection_errors(self): - """Context manager that catches broker connection/channel errors and reconnects.""" - try: - yield - except (self._connection.connection_errors - + self._connection.channel_errors) as exc: - logger.warning( - 'RPC result consumer: connection lost (%s), ' - 'attempting to reconnect...', exc, - ) - self._reconnect() - def drain_events(self, timeout=None): if self._connection: - with self._handle_connection_errors(): + with self.reconnect_on_error(): return self._connection.drain_events(timeout=timeout) elif timeout: time.sleep(timeout) @@ -86,6 +76,10 @@ def _reconnect(self): Re-subscribes to every queue that the old consumer was listening on so that pending results can still be drained. """ + logger.warning( + 'RPC result consumer: connection lost, attempting to reconnect...', + exc_info=True, + ) old_queues = [] if self._consumer is not None: old_queues = list(self._consumer.queues) @@ -111,6 +105,10 @@ def _reconnect(self): # Establish a fresh connection and consumer. self._connection = self.app.connection() + self._connection_errors = ( + self._connection.connection_errors + + self._connection.channel_errors + ) self._consumer = self.Consumer( self._connection.default_channel, old_queues, diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index 26acf96ccf1..05a0557379f 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -633,3 +633,81 @@ def schedule_thread(self, thread): def teardown_thread(self, thread): import gevent gevent.wait([thread]) + + +class test_BaseResultConsumer_reconnect: + + def _make_consumer(self, app): + return _make_consumer(app) + + def test_reconnect_on_error_no_exception_passes_through(self, app): + consumer = self._make_consumer(app) + result = [] + with consumer.reconnect_on_error(): + result.append('ok') + assert result == ['ok'] + + def test_reconnect_on_error_ignores_non_connection_error(self, app): + consumer = self._make_consumer(app) + with pytest.raises(ValueError): + with consumer.reconnect_on_error(): + raise ValueError('unrelated') + + def test_reconnect_on_error_default_connection_errors_empty(self, app): + consumer = self._make_consumer(app) + assert consumer._connection_errors == () + + class FakeConnError(Exception): + pass + + with pytest.raises(FakeConnError): + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + def test_reconnect_on_error_calls_reconnect_on_connection_error(self, app): + consumer = self._make_consumer(app) + + class FakeConnError(Exception): + pass + + consumer._connection_errors = (FakeConnError,) + consumer._reconnect = Mock() + + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + consumer._reconnect.assert_called_once_with() + + def test_reconnect_on_error_raises_runtime_when_reconnect_also_fails(self, app): + consumer = self._make_consumer(app) + + class FakeConnError(Exception): + pass + + consumer._connection_errors = (FakeConnError,) + consumer._reconnect = Mock(side_effect=FakeConnError('still down')) + + with pytest.raises(RuntimeError, match='Retry limit exceeded'): + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + def test_reconnect_on_error_runtime_chained_from_connection_error(self, app): + consumer = self._make_consumer(app) + + class FakeConnError(Exception): + pass + + consumer._connection_errors = (FakeConnError,) + original = FakeConnError('still down') + consumer._reconnect = Mock(side_effect=original) + + with pytest.raises(RuntimeError) as exc_info: + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + assert exc_info.value.__cause__ is original + + def test_reconnect_base_implementation_is_noop(self, app): + consumer = self._make_consumer(app) + + assert consumer._reconnect() is None diff --git a/t/unit/backends/test_rpc.py b/t/unit/backends/test_rpc.py index 232ea05f6d4..1c09f347b25 100644 --- a/t/unit/backends/test_rpc.py +++ b/t/unit/backends/test_rpc.py @@ -30,6 +30,7 @@ def test_drain_events_reconnects_on_connection_error(self): 'Server unexpectedly closed connection' ) consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors mock_consumer = Mock(name='consumer') mock_consumer.queues = [Mock(name='queue1')] @@ -38,6 +39,8 @@ def test_drain_events_reconnects_on_connection_error(self): # Patch app.connection() to return a fresh mock connection # and Consumer to return a mock consumer. new_conn = Mock(name='new_connection') + new_conn.connection_errors = (OSError,) + new_conn.channel_errors = () new_kombu_consumer = Mock(name='new_kombu_consumer') consumer.app = Mock() consumer.app.connection.return_value = new_conn @@ -62,6 +65,7 @@ def test_drain_events_reconnect_preserves_queues(self): mock_conn.channel_errors = () mock_conn.drain_events.side_effect = ConnectionError('reset') consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors queue1, queue2 = Mock(name='q1'), Mock(name='q2') mock_consumer = Mock(name='consumer') @@ -69,6 +73,8 @@ def test_drain_events_reconnect_preserves_queues(self): consumer._consumer = mock_consumer new_conn = Mock(name='new_connection') + new_conn.connection_errors = (ConnectionError,) + new_conn.channel_errors = () consumer.app = Mock() consumer.app.connection.return_value = new_conn consumer.Consumer = Mock(return_value=Mock(name='new_kombu_consumer')) @@ -86,6 +92,7 @@ def test_drain_events_no_reconnect_on_other_errors(self): mock_conn.channel_errors = () mock_conn.drain_events.side_effect = RuntimeError('unexpected') consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors with pytest.raises(RuntimeError, match='unexpected'): consumer.drain_events(timeout=1) @@ -102,6 +109,8 @@ def test_reconnect_handles_close_failures_gracefully(self): consumer._consumer = mock_consumer new_conn = Mock(name='new_connection') + new_conn.connection_errors = (OSError,) + new_conn.channel_errors = () new_kombu_consumer = Mock(name='new_kombu_consumer') consumer.app = Mock() consumer.app.connection.return_value = new_conn @@ -120,12 +129,15 @@ def test_drain_events_channel_error_triggers_reconnect(self): mock_conn.channel_errors = (KeyError,) mock_conn.drain_events.side_effect = KeyError('channel closed') consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors mock_consumer = Mock(name='consumer') mock_consumer.queues = [] consumer._consumer = mock_consumer new_conn = Mock(name='new_connection') + new_conn.connection_errors = () + new_conn.channel_errors = (KeyError,) consumer.app = Mock() consumer.app.connection.return_value = new_conn consumer.Consumer = Mock(return_value=Mock(name='new_kombu_consumer')) @@ -134,6 +146,29 @@ def test_drain_events_channel_error_triggers_reconnect(self): assert consumer._connection is new_conn + def test_drain_events_raises_runtime_when_reconnect_also_fails(self): + consumer = self.get_consumer() + + class FakeConnError(Exception): + pass + + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (FakeConnError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = FakeConnError('dropped') + consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors + + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [] + consumer._consumer = mock_consumer + + consumer.app = Mock() + consumer.app.connection.side_effect = FakeConnError('still down') + + with pytest.raises(RuntimeError, match='Retry limit exceeded'): + consumer.drain_events(timeout=1) + class test_RPCBackend: From f0dbcb059326dc7f7a21a2bfb2800ed7559fb9cf Mon Sep 17 00:00:00 2001 From: Eric Buehl <715650+ericbuehl@users.noreply.github.com> Date: Tue, 10 Mar 2026 08:03:43 -0700 Subject: [PATCH 158/169] pep 649 (#10187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/utils/functional.py | 6 ++++++ t/unit/utils/test_functional.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 762685b60cd..f9a4d4600d5 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -399,6 +399,12 @@ def fun_takes_argument(name, fun, position=None): def fun_accepts_kwargs(fun): """Return true if function accepts arbitrary keyword arguments.""" + # inspect.signature evaluates annotations in Python 3.14+ (PEP 649), + # which raises NameError for types only imported under TYPE_CHECKING. + # Check co_flags directly to avoid touching annotations entirely. + code = getattr(fun, '__code__', None) + if code is not None: + return bool(code.co_flags & inspect.CO_VARKEYWORDS) return any( p for p in inspect.signature(fun).parameters.values() if p.kind == p.VAR_KEYWORD diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index fc2baa861e4..3b97a12b2b9 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -487,6 +487,25 @@ def test_accepts(self, fun): def test_rejects(self, fun): assert not fun_accepts_kwargs(fun) + @pytest.mark.skipif(sys.version_info < (3, 14), reason="PEP 649 deferred annotations require Python 3.14+") + def test_type_checking_annotation(self): + # Regression test for https://github.com/celery/celery/discussions/10099 + # On Python 3.14+, annotations are deferred (PEP 649). Calling + # fun_accepts_kwargs on a function whose annotations reference + # TYPE_CHECKING-only types must not raise NameError. + # + # This reproduces the failure seen with on_after_finalize.connect: + # def setup_periodic_tasks(sender: Celery, **kwargs: object) -> None: ... + # where 'Celery' is only imported under TYPE_CHECKING. + local = {} + exec('def f(sender: Celery, **kwargs: object) -> None: pass', {}, local) + f = local['f'] + assert fun_accepts_kwargs(f) is True + + exec('def g(sender: Celery) -> None: pass', {}, local) + g = local['g'] + assert fun_accepts_kwargs(g) is False + @pytest.mark.parametrize('value,expected', [ (5, True), From a9a2d4cecaf0e58c401ad6f68f022afa19770ac2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2026 21:04:26 +0600 Subject: [PATCH 159/169] [pre-commit.ci] pre-commit autoupdate (#10186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/codespell-project/codespell: v2.4.1 → v2.4.2](https://github.com/codespell-project/codespell/compare/v2.4.1...v2.4.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e06a8431aa..2b1a24a66fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: exclude: ^celery/app/task\.py$|^celery/backends/cache\.py$ - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell # See pyproject.toml for args args: [--toml, pyproject.toml, --write-changes] From 6ee6230cd80ef6c3e7482e1f4cd970fbb0629b23 Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Wed, 11 Mar 2026 23:15:14 +0800 Subject: [PATCH 160/169] Fix#9722 friendly status errors for CLI (#10190) * fix: show user-friendly broker errors for remote CLI commands Convert broker connection failures in `status` into concise actionable CLI errors instead of tracebacks, and apply the same error handling path to related remote CLI flows. Add regression coverage for broker-unreachable and unexpected-error handling in status/graph/events command paths. * test: handle remote command error * chore: use cli_runner in click package --------- Co-authored-by: Harshang Akabari --- celery/bin/base.py | 23 ++++++ celery/bin/control.py | 30 ++++--- celery/bin/events.py | 22 +++--- celery/bin/graph.py | 14 +++- t/unit/bin/test_control.py | 155 +++++++++++++++++++++++++++++++++++++ 5 files changed, 221 insertions(+), 23 deletions(-) diff --git a/celery/bin/base.py b/celery/bin/base.py index 61cc37a0291..be3999236bc 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -8,9 +8,12 @@ import click from click import Context, ParamType +from kombu.exceptions import OperationalError from kombu.utils.objects import cached_property from celery._state import get_current_app +from celery.exceptions import CeleryCommandException +from celery.platforms import EX_UNAVAILABLE from celery.signals import user_preload_options from celery.utils import text from celery.utils.log import mlevel @@ -117,6 +120,26 @@ def say_chat(self, direction, title, body='', show_body=False): self.echo(body) +def handle_remote_command_error(command: str, exc: Exception) -> None: + if isinstance(exc, click.ClickException): + raise + + if isinstance(exc, OperationalError): + raise CeleryCommandException( + message=( + 'Could not connect to the message broker. ' + 'Please make sure your broker (e.g., RabbitMQ or Redis) is running and ' + f'the connection settings are correct. Reason: {exc}' + ), + exit_code=EX_UNAVAILABLE, + ) from exc + + raise CeleryCommandException( + message=f'Unable to run the `{command}` command. Reason: {exc}', + exit_code=EX_UNAVAILABLE, + ) from exc + + def handle_preload_options(f): """Extract preload options and return a wrapped callable.""" def caller(ctx, *args, **kwargs): diff --git a/celery/bin/control.py b/celery/bin/control.py index 38a917ea0f2..48681a9f28e 100644 --- a/celery/bin/control.py +++ b/celery/bin/control.py @@ -5,7 +5,8 @@ import click from kombu.utils.json import dumps -from celery.bin.base import COMMA_SEPARATED_LIST, CeleryCommand, CeleryOption, handle_preload_options +from celery.bin.base import (COMMA_SEPARATED_LIST, CeleryCommand, CeleryOption, handle_preload_options, + handle_remote_command_error) from celery.exceptions import CeleryCommandException from celery.platforms import EX_UNAVAILABLE from celery.utils import text @@ -128,9 +129,12 @@ def _get_commands_of_type(type_: _RemoteControlType) -> dict: def status(ctx, timeout, destination, json, **kwargs): """Show list of workers that are online.""" callback = None if json else partial(_say_remote_command_reply, ctx) - replies = ctx.obj.app.control.inspect(timeout=timeout, - destination=destination, - callback=callback).ping() + try: + replies = ctx.obj.app.control.inspect(timeout=timeout, + destination=destination, + callback=callback).ping() + except Exception as exc: + handle_remote_command_error('status', exc) if not replies: raise CeleryCommandException( @@ -183,7 +187,10 @@ def inspect(ctx, command, timeout, destination, json, **kwargs): inspect = ctx.obj.app.control.inspect(timeout=timeout, destination=destination, callback=callback) - replies = inspect._request(command, **arguments) + try: + replies = inspect._request(command, **arguments) + except Exception as exc: + handle_remote_command_error(f'inspect {command}', exc) if not replies: raise CeleryCommandException( @@ -236,11 +243,14 @@ def control(ctx, command, timeout, destination, json): show_reply=True) args = ctx.args arguments = _compile_arguments(command, args) - replies = ctx.obj.app.control.broadcast(command, timeout=timeout, - destination=destination, - callback=callback, - reply=True, - arguments=arguments) + try: + replies = ctx.obj.app.control.broadcast(command, timeout=timeout, + destination=destination, + callback=callback, + reply=True, + arguments=arguments) + except Exception as exc: + handle_remote_command_error(f'control {command}', exc) if not replies: raise CeleryCommandException( diff --git a/celery/bin/events.py b/celery/bin/events.py index 89470838bcc..c69f4ae7552 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -4,7 +4,8 @@ import click -from celery.bin.base import LOG_LEVEL, CeleryDaemonCommand, CeleryOption, handle_preload_options +from celery.bin.base import (LOG_LEVEL, CeleryDaemonCommand, CeleryOption, handle_preload_options, + handle_remote_command_error) from celery.platforms import detached, set_process_title, strargv @@ -82,13 +83,16 @@ def _run_evtop(app): def events(ctx, dump, camera, detach, frequency, maxrate, loglevel, **kwargs): """Event-stream utilities.""" app = ctx.obj.app - if dump: - return _run_evdump(app) + try: + if dump: + return _run_evdump(app) - if camera: - return _run_evcam(camera, app=app, freq=frequency, maxrate=maxrate, - loglevel=loglevel, - detach=detach, - **kwargs) + if camera: + return _run_evcam(camera, app=app, freq=frequency, maxrate=maxrate, + loglevel=loglevel, + detach=detach, + **kwargs) - return _run_evtop(app) + return _run_evtop(app) + except Exception as exc: + handle_remote_command_error('events', exc) diff --git a/celery/bin/graph.py b/celery/bin/graph.py index d4d6f16205f..eb1baaa4aa0 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -4,7 +4,7 @@ import click -from celery.bin.base import CeleryCommand, handle_preload_options +from celery.bin.base import CeleryCommand, handle_preload_options, handle_remote_command_error from celery.utils.graph import DependencyGraph, GraphFormatter @@ -154,7 +154,10 @@ def maybe_abbr(l, name, max=Wmax): workers = args['nodes'] threads = args.get('threads') or [] except KeyError: - replies = app.control.inspect().stats() or {} + try: + replies = app.control.inspect().stats() or {} + except Exception as exc: + handle_remote_command_error('graph workers', exc) workers, threads = [], [] for worker, reply in replies.items(): workers.append(worker) @@ -171,8 +174,11 @@ def maybe_abbr(l, name, max=Wmax): list(range(int(threads))), 'P', Tmax, ) - broker = Broker(args.get( - 'broker', app.connection_for_read().as_uri())) + try: + broker_uri = args.get('broker', app.connection_for_read().as_uri()) + except Exception as exc: + handle_remote_command_error('graph workers', exc) + broker = Broker(broker_uri) backend = Backend(backend) if backend else None deps = DependencyGraph(formatter=Formatter()) deps.add_arc(broker) diff --git a/t/unit/bin/test_control.py b/t/unit/bin/test_control.py index 74f6e4fb1ca..217b37a8c0d 100644 --- a/t/unit/bin/test_control.py +++ b/t/unit/bin/test_control.py @@ -4,6 +4,7 @@ import pytest from click.testing import CliRunner +from kombu.exceptions import OperationalError from celery.bin.celery import celery from celery.platforms import EX_UNAVAILABLE @@ -80,3 +81,157 @@ def test_listing_remote_commands(celery_cmd, expected_regex, isolated_cli_runner ) assert res.exit_code == 0, (res, res.stdout) assert expected_regex.search(res.stdout) + + +def test_status_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.app.control.Inspect.ping', + side_effect=OperationalError('[Errno 61] Connection refused')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'status'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: [Errno 61] Connection refused' in res.output + assert 'Traceback' not in res.output + + +def test_status_unexpected_error_is_summarized(cli_runner: CliRunner): + with patch('celery.app.control.Inspect.ping', + side_effect=RuntimeError('boom')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'status'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Unable to run the `status` command. Reason: boom' in res.output + assert 'Traceback' not in res.output + + +def test_graph_workers_shows_friendly_error_when_broker_unreachable( + cli_runner: CliRunner, +): + with patch('celery.app.control.Inspect.stats', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'graph', 'workers'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + + +def test_events_dump_shows_friendly_error_when_broker_unreachable( + cli_runner: CliRunner, +): + with patch('celery.bin.events._run_evdump', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'events', '--dump'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + + +def test_handle_remote_command_error_reraises_click_exception(): + """base.py: bare ``raise`` inside the ClickException branch must be covered.""" + import click + + from celery.bin.base import handle_remote_command_error + + original = click.ClickException('original click error') + with pytest.raises(click.ClickException) as exc_info: + try: + raise original + except Exception as exc: + handle_remote_command_error('any', exc) + assert exc_info.value is original + + +def test_inspect_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.app.control.Inspect._request', + side_effect=OperationalError('connection refused')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'inspect', *_INSPECT_OPTIONS, 'custom_inspect_cmd', '1'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection refused' in res.output + assert 'Traceback' not in res.output + + +def test_inspect_unexpected_error_is_summarized(cli_runner: CliRunner): + with patch('celery.app.control.Inspect._request', + side_effect=RuntimeError('inspect boom')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'inspect', *_INSPECT_OPTIONS, 'custom_inspect_cmd', '1'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Unable to run the `inspect custom_inspect_cmd` command. Reason: inspect boom' in res.output + assert 'Traceback' not in res.output + + +def test_control_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.app.control.Control.broadcast', + side_effect=OperationalError('connection refused')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'control', *_INSPECT_OPTIONS, 'custom_control_cmd', '1', '2'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection refused' in res.output + assert 'Traceback' not in res.output + + +def test_control_unexpected_error_is_summarized(cli_runner: CliRunner): + with patch('celery.app.control.Control.broadcast', + side_effect=RuntimeError('control boom')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'control', *_INSPECT_OPTIONS, 'custom_control_cmd', '1', '2'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Unable to run the `control custom_control_cmd` command. Reason: control boom' in res.output + assert 'Traceback' not in res.output + + +def test_events_camera_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.bin.events._run_evcam', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'events', '--camera', 'myapp.MyCameraClass'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + assert 'Traceback' not in res.output + + +def test_events_evtop_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.bin.events._run_evtop', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'events'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + assert 'Traceback' not in res.output From 9a270925546ed9d0ca0303fb5006edc86b705fd9 Mon Sep 17 00:00:00 2001 From: Kian Anbarestani <145364424+KianAnbarestani@users.noreply.github.com> Date: Thu, 12 Mar 2026 18:21:00 +0330 Subject: [PATCH 161/169] docs: clarify after_return behavior for retried tasks (#10192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: clarify after_return behavior for retried tasks * Update docs/userguide/tasks.rst * docs: clarify after_return ordering for terminal states --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/userguide/tasks.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 3dfdbd58093..8144f5db0fd 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1596,7 +1596,8 @@ The following diagram shows the exact order of execution: │ 4. on_success() OR ← Outcome-specific handler │ │ on_retry() OR │ │ │ on_failure() │ │ - │ 5. after_return() ← Always runs last │ + │ 5. after_return() ← Runs last on terminal states │ + │ (skipped for RETRY/REJECTED/IGNORED) │ └───────────────────────────────────────────────────────────────┘ .. important:: @@ -1606,7 +1607,9 @@ The following diagram shows the exact order of execution: - All handlers run in the **same worker process** as your task - ``before_start`` **blocks** the task - ``run()`` won't start until it completes - Result backend is updated **before** ``on_success``/``on_failure`` - other clients can see the task as finished while handlers are still running - - ``after_return`` **always** executes, regardless of task outcome + - ``after_return`` executes when the task reaches a terminal state. + It does not run for ``RETRY``, ``REJECTED``, or ``IGNORED``. If you need + a hook that fires on every attempt, use the :signal:`task_postrun` signal. Available handlers ~~~~~~~~~~~~~~~~~~ @@ -1687,8 +1690,13 @@ Available handlers Handler called after the task returns. .. note:: - Executes **after** ``on_success``/``on_retry``/``on_failure``. This is the - final hook in the task lifecycle and **always** runs, regardless of outcome. + Executes after the outcome-specific handler when the task reaches a + terminal state. + + In practice, this means it runs after ``on_success`` or ``on_failure``. + It is not executed for ``RETRY``, ``REJECTED``, or ``IGNORED`` states. + If a hook is needed for every attempt, consider using the + :signal:`task_postrun` signal. :param status: Current task state. :param retval: Task return value/exception. From f45f62beb3b16ae960944f8c97de13ccf15f2d0a Mon Sep 17 00:00:00 2001 From: Br1an <932039080@qq.com> Date: Sun, 15 Mar 2026 18:31:43 +0800 Subject: [PATCH 162/169] Add compression header to message protocol docs (#10156) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add compression header to message protocol docs Document the optional 'compression' header field in the version 2 message protocol definition. * docs: clarify compression header is optional with examples Add inline comment noting the compression header is omitted when no compression is used, with example compressor names. --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/internals/protocol.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 72f461dc936..00d7b4b34e7 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -49,7 +49,8 @@ Definition 'argsrepr': str repr(args), 'kwargsrepr': str repr(kwargs), 'origin': str nodename, - 'replaced_task_nesting': int + 'replaced_task_nesting': int, + 'compression': string compression_method (optional; omitted when no compression is used, matches kombu compressor names such as 'zlib', 'bzip2', 'gzip'), } body = ( From ada2da7475a5fa9f9ad079149a5d6864634abc28 Mon Sep 17 00:00:00 2001 From: Rohan Santhosh Kumar <181558744+Rohan5commit@users.noreply.github.com> Date: Sun, 15 Mar 2026 18:33:02 +0800 Subject: [PATCH 163/169] docs: fix duplicated word in bootsteps comment\n\nSigned-off-by: Rohan Santhosh (#10153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/bootsteps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/celery/bootsteps.py b/celery/bootsteps.py index 878560624d1..dcd2dc0875a 100644 --- a/celery/bootsteps.py +++ b/celery/bootsteps.py @@ -303,7 +303,7 @@ class Step(metaclass=StepType): #: Set this to true if the step is enabled based on some condition. conditional = False - #: List of other steps that that must be started before this step. + #: List of other steps that must be started before this step. #: Note that all dependencies must be in the same blueprint. requires = () From d23be53f6f3600d48df35a797c63eb1c7d4d4b97 Mon Sep 17 00:00:00 2001 From: Br1an <932039080@qq.com> Date: Sun, 15 Mar 2026 18:35:01 +0800 Subject: [PATCH 164/169] Remove outdated autoreloader section from extending docs (#10154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The autoreloader component (celery.worker.autoreloader) was removed from the codebase, but the documentation section still referenced it. This removes the misleading autoreloader attribute section from the worker extensions guide. Fixes #7320 Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- docs/userguide/extending.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index ea8c0462598..32c28620376 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -220,21 +220,6 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.autoscaler:Autoscaler',) -.. _extending-worker-autoreloader: - -.. attribute:: autoreloader - - :class:`~celery.worker.autoreloder.Autoreloader` used to automatically - reload use code when the file-system changes. - - This is only defined if the ``autoreload`` argument is enabled. - Your worker bootstep must require the `Autoreloader` bootstep to use this; - - .. code-block:: python - - class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.autoreloader:Autoreloader',) - Example worker bootstep ----------------------- From c3c19c31dc3e21f16d4d85a8ba8401a9223ace09 Mon Sep 17 00:00:00 2001 From: Patrick Lin Date: Sun, 15 Mar 2026 18:51:04 +0800 Subject: [PATCH 165/169] Fix: prioritize request ignore_result over task definition (#10184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: prioritize request ignore_result over task definition * fix: add comment to explain logic of get_actual_ignore_result * test: add missing tests for Request.ignore_result fallback This commit adds the missing unit tests to verify the priority logic for 'ignore_result' within the Request class, ensuring it correctly falls back to the task property when not specified in the message. * fix: pre-commit error * fix: add missing tests for handle_error_state while req is none * fix(worker): add test to ensure request.store_errors respects task.ignore_result * fix(docs): clarify fallback behavior for ignore_result and store_errors * fix(worker): handle ignore_result=None in request headers --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/app/trace.py | 44 +++++++++++++++----- celery/worker/request.py | 7 +++- docs/userguide/tasks.rst | 7 ++++ t/unit/tasks/test_trace.py | 76 +++++++++++++++++++++++++++++++++++ t/unit/worker/test_request.py | 68 ++++++++++++++++++++++++++++--- 5 files changed, 184 insertions(+), 18 deletions(-) diff --git a/celery/app/trace.py b/celery/app/trace.py index ecd26994288..0641415c236 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -154,6 +154,28 @@ def get_task_name(request, default): return getattr(request, 'shadow', None) or default +def get_actual_ignore_result(task, req): + """Return the effective ignore_result, with request overriding task. + + If req provides an explicit ignore_result, that value is used; + otherwise task.ignore_result is returned. + """ + if req is None: + return task.ignore_result + + actual = getattr(req, 'ignore_result', None) + + # Context defines `ignore_result = False` at class level (see Context + # in celery/app/task.py). getattr() above would return the class default + # (False) even when the request never set it explicitly, making it + # impossible to distinguish "override=False" from "not set". We check + # __dict__ to detect only instance-level (i.e., explicitly set) values. + if isinstance(req, Context) and 'ignore_result' not in req.__dict__: + actual = None + + return actual if actual is not None else task.ignore_result + + class TraceInfo: """Information about task execution.""" @@ -165,7 +187,9 @@ def __init__(self, state, retval=None): def handle_error_state(self, task, req, eager=False, call_errbacks=True): - if task.ignore_result: + ignore_result = get_actual_ignore_result(task, req) + + if ignore_result: store_errors = task.store_errors_even_if_ignored elif eager and task.store_eager_result: store_errors = True @@ -353,16 +377,6 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, fun = task if task_has_custom(task, '__call__') else task.run loader = loader or app.loader - ignore_result = task.ignore_result - track_started = task.track_started - track_started = not eager and (task.track_started and not ignore_result) - - # #6476 - if eager and not ignore_result and task.store_eager_result: - publish_result = True - else: - publish_result = not eager and not ignore_result - deduplicate_successful_tasks = ((app.conf.task_acks_late or task.acks_late) and app.conf.worker_deduplicate_successful_tasks and app.backend.persistent) @@ -483,6 +497,14 @@ def trace_task(uuid, args, kwargs, request=None): task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) + ignore_result = get_actual_ignore_result(task, task_request) + track_started = not eager and (task.track_started and not ignore_result) + # #6476 + if eager and not ignore_result and task.store_eager_result: + publish_result = True + else: + publish_result = not eager and not ignore_result + redelivered = (task_request.delivery_info and task_request.delivery_info.get('redelivered', False)) if deduplicate_successful_tasks and redelivered: diff --git a/celery/worker/request.py b/celery/worker/request.py index 2b975266f68..4eaab75675e 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -125,7 +125,10 @@ def __init__(self, message, on_ack=noop, self._eventer = eventer self._connection_errors = connection_errors or () self._task = task or self._app.tasks[self._type] - self._ignore_result = self._request_dict.get('ignore_result', False) + ignore_result = self._request_dict.get('ignore_result', None) + if ignore_result is None: + ignore_result = self._task.ignore_result + self._ignore_result = ignore_result # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. @@ -287,7 +290,7 @@ def tzlocal(self): @property def store_errors(self): - return (not self.task.ignore_result or + return (not self._ignore_result or self.task.store_errors_even_if_ignored) @property diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 8144f5db0fd..0a29e0982d9 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1042,6 +1042,13 @@ General If :const:`True`, errors will be stored even if the task is configured to ignore results. + .. versionchanged:: 5.7 + Previously, if the ``ignore_result`` key was missing from the request + message, ``store_errors`` would default to ``True``, ignoring the + task's own ``ignore_result`` setting. The worker now correctly + falls back to ``Task.ignore_result`` when no per-request override + is present. + .. attribute:: Task.serializer A string identifying the default serialization diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index 641fe25d7e3..5f7172383dd 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -1081,6 +1081,53 @@ def fail_on_second_call(self_): successful_requests.discard(task_id) self.app.conf.worker_deduplicate_successful_tasks = False + def test_ignore_result_priority__request_overrides_task_true(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + add.ignore_result = True + request = {'ignore_result': False} + + self.trace(add, (2, 2), {}, request=request, eager=False) + + add.backend.mark_as_done.assert_called_with(ANY, 4, ANY, True) + + def test_ignore_result_priority__request_overrides_task_false(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + add.ignore_result = False + request = {'ignore_result': True} + + self.trace(add, (2, 2), {}, request=request, eager=False) + + add.backend.mark_as_done.assert_called_with(ANY, 4, ANY, False) + + def test_ignore_result_priority__request_overrides_app_config(self): + prev_ignore = self.app.conf.task_ignore_result + + try: + self.app.conf.task_ignore_result = True + + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + + assert add.ignore_result is True + + request = {'ignore_result': False} + self.trace(add, (2, 2), {}, request=request, eager=False) + + add.backend.mark_as_done.assert_called_with(ANY, 4, ANY, True) + finally: + self.app.conf.task_ignore_result = prev_ignore + class test_TraceInfo(TraceCase): class TI(TraceInfo): @@ -1123,6 +1170,35 @@ def test_handle_error_for_eager_saved_to_backend(self): call_errbacks=True, ) + def test_handle_error_state_missing_request_store_errors_false_while_task_ignore_result_true(self): + x = self.TI(states.FAILURE) + x.handle_failure = Mock() + + self.add.ignore_result = True + self.add.store_errors_even_if_ignored = False + + x.handle_error_state(self.add, None) + x.handle_failure.assert_called_once_with( + self.add, + None, + store_errors=False, + call_errbacks=True, + ) + + def test_handle_error_state_missing_request_store_errors_true_while_task_ignore_result_false(self): + x = self.TI(states.FAILURE) + x.handle_failure = Mock() + + self.add.ignore_result = False + + x.handle_error_state(self.add, None) + x.handle_failure.assert_called_once_with( + self.add, + None, + store_errors=True, + call_errbacks=True, + ) + @patch('celery.app.trace.ExceptionInfo') def test_handle_reject(self, ExceptionInfo): x = self.TI(states.FAILURE) diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index fb4354942c3..1a57f2d4212 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -188,13 +188,14 @@ def get_request(self, sig, Request=Request, exclude_headers=None, + headers=None, **kwargs): msg = self.task_message_from_sig(self.app, sig) - headers = None - if exclude_headers: - headers = msg.headers - for header in exclude_headers: - headers.pop(header) + if headers is None: + headers = msg.headers.copy() + if exclude_headers: + for header in exclude_headers: + headers.pop(header, None) return Request( msg, on_ack=Mock(name='on_ack'), @@ -443,6 +444,34 @@ def test_tzlocal_is_cached(self): req._tzlocal = 'foo' assert req.tzlocal == 'foo' + def test_ignore_result_from_request_true(self): + req = self.get_request(self.add.s(2, 2).set(ignore_result=True)) + assert req.ignore_result is True + + def test_ignore_result_from_request_false(self): + req = self.get_request(self.add.s(2, 2).set(ignore_result=False)) + assert req.ignore_result is False + + def test_ignore_result_default_from_task_false(self): + req = self.get_request(self.add.s(2, 2)) + assert req.ignore_result is False + + def test_ignore_result_default_from_task_true(self): + self.add.ignore_result = True + try: + req = self.get_request(self.add.s(2, 2)) + assert req.ignore_result is True + finally: + self.add.ignore_result = False + + def test_ignore_result_request_overrides_task_true(self): + self.add.ignore_result = True + try: + req = self.get_request(self.add.s(2, 2).set(ignore_result=False)) + assert req.ignore_result is False + finally: + self.add.ignore_result = False + def test_task_wrapper_repr(self): assert repr(self.xRequest()) @@ -455,6 +484,35 @@ def test_sets_store_errors(self): job = self.xRequest() assert job.store_errors + def test_store_errors_default(self): + self.mytask.ignore_result = False + job = self.xRequest() + assert job.store_errors + + def test_store_errors_task_ignore_result(self): + self.mytask.ignore_result = True + job = self.xRequest() + assert not job.store_errors + + def test_store_errors_request_overrides_task_ignore_result(self): + self.mytask.ignore_result = True + msg = self.task_message_from_sig(self.app, self.mytask.s()) + headers = msg.headers.copy() + headers['ignore_result'] = False + job = self.get_request( + self.mytask.s(), + headers=headers, + ) + assert job.store_errors + + def test_ignore_result_from_request_none(self): + self.mytask.ignore_result = True + msg = self.task_message_from_sig(self.app, self.mytask.s()) + headers = msg.headers.copy() + headers['ignore_result'] = None + job = self.get_request(self.mytask.s(), headers=headers) + assert job._ignore_result is True + def test_send_event(self): job = self.xRequest() job.eventer = Mock(name='.eventer') From d06de5f047620b0ea2bdbdb3c0c56137b79ae9a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:23:36 +0600 Subject: [PATCH 166/169] Chore(deps): Bump nick-fields/retry from 3 to 4 (#10213) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [nick-fields/retry](https://github.com/nick-fields/retry) from 3 to 4. - [Release notes](https://github.com/nick-fields/retry/releases) - [Commits](https://github.com/nick-fields/retry/compare/v3...v4) --- updated-dependencies: - dependency-name: nick-fields/retry dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- .github/workflows/integration-tests.yml | 2 +- .github/workflows/smoke-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c3cf3258e0f..e3f411bc7ea 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -63,7 +63,7 @@ jobs: - name: > Run tox for "${{ matrix.python-version }}-integration-${{ matrix.toxenv }}-${{ inputs.module_name }}" - uses: nick-fields/retry@v3 + uses: nick-fields/retry@v4 with: timeout_minutes: 15 max_attempts: 5 diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index 1116b2af2ea..d40307c578d 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -51,7 +51,7 @@ jobs: run: python -m pip install --upgrade pip tox tox-gh-actions - name: Run tox for "${{ matrix.python-version }}-smoke-${{ inputs.module_name }}" - uses: nick-fields/retry@v3 + uses: nick-fields/retry@v4 with: timeout_minutes: 20 max_attempts: 5 From a989e8cf8876274b1f0612abffeeb2e9995ed321 Mon Sep 17 00:00:00 2001 From: ChickenBenny Date: Thu, 26 Mar 2026 13:24:37 +0800 Subject: [PATCH 167/169] fix: clear the timer while catch the exception (#10218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: clear the timer while catch the exception * fix: move the timer clear to another try catch block * Update t/unit/worker/test_loops.py --------- Co-authored-by: Asif Saif Uddin {"Auvi":"অভি"} --- celery/worker/loops.py | 18 +++++++++ t/unit/worker/test_loops.py | 81 +++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/celery/worker/loops.py b/celery/worker/loops.py index ed33a8c64af..5c36f15da6d 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -100,6 +100,12 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, except Exception: # Reset the hub on error (e.g. connection loss) to clean up # stale file descriptors and callbacks from the old connection. + # Also clear the timer queue so that stale periodic entries added by + # register_with_event_loop (e.g. maybe_restore_messages) do not fire + # against the broken connection after reconnect and trigger another + # crash before the new connection is fully established. + # All hub timers are re-registered during blueprint.start() once this + # exception propagates and the consumer reconnects. # We intentionally do NOT reset on normal exit (graceful shutdown) # so that timers (e.g. heartbeat) keep firing while the pool drains. # WorkerShutdown/WorkerTerminate extend SystemExit (not Exception) @@ -109,6 +115,18 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, except Exception as exc: # pylint: disable=broad-except logger.exception( 'Error cleaning up after event loop: %r', exc) + # Clear stale timer entries accumulated across reconnects (e.g. + # maybe_restore_messages registered via call_repeatedly). Without + # this, each reconnect appends a new entry; all of them fire during + # the reconnect window, raise again, and trigger another restart. + # Use a separate try/except so this always runs even if hub.reset() + # raised above. Timers are re-registered by register_with_event_loop + # when blueprint.start() is called after reconnect. + try: + hub.timer.clear() + except Exception as exc: # pylint: disable=broad-except + logger.exception( + 'Error clearing hub timer after event loop: %r', exc) raise diff --git a/t/unit/worker/test_loops.py b/t/unit/worker/test_loops.py index 42369b01960..26e682a6485 100644 --- a/t/unit/worker/test_loops.py +++ b/t/unit/worker/test_loops.py @@ -466,6 +466,87 @@ def test_hub_reset_on_connection_error(self): asynloop(*x.args) x.hub.reset.assert_called_once() + def test_hub_timer_cleared_on_connection_error(self): + # Stale timer entries (e.g. maybe_restore_messages) must be cleared + # when the event loop exits due to a connection error. Without this, + # entries accumulated across reconnects can fire against the broken + # connection and crash the loop again before the new connection is + # fully established, causing a rapid restart loop. + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()') + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + x.hub.poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.timer.clear.assert_called_once() + + def test_hub_timer_not_cleared_on_graceful_shutdown(self): + # On graceful shutdown the timer queue must be left intact so that + # periodic timers (e.g. heartbeat) keep firing while the pool drains. + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + x.hub.on_tick.add(x.closer(mod=2)) + asynloop(*x.args) + x.hub.timer.clear.assert_not_called() + + def test_hub_timer_not_cleared_on_worker_shutdown(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_stop = 303 + try: + with pytest.raises(WorkerShutdown): + asynloop(*x.args) + finally: + state.should_stop = None + x.hub.timer.clear.assert_not_called() + + def test_hub_timer_not_cleared_on_worker_terminate(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_terminate = True + try: + with pytest.raises(WorkerTerminate): + asynloop(*x.args) + finally: + state.should_terminate = None + x.hub.timer.clear.assert_not_called() + + def test_hub_timer_clear_error_still_reraises_original(self): + # If hub.timer.clear() itself raises, the original connection error + # must still be propagated, not the cleanup error. + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()') + x.hub.timer.clear = Mock( + name='hub.timer.clear()', side_effect=RuntimeError('clear failed') + ) + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + x.hub.poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.timer.clear.assert_called_once() + + def test_hub_timer_cleared_even_when_reset_raises(self): + # hub.timer.clear() must still be called even if hub.reset() raises. + # The two cleanup calls are in separate try/except blocks so that a + # failure in hub.reset() does not prevent stale timer entries from + # being discarded, avoiding stale timers persisting after a reset error. + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()', side_effect=RuntimeError('reset failed')) + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + x.hub.poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.timer.clear.assert_called_once() + def test_hub_not_reset_on_graceful_shutdown(self): x = X(self.app) x.hub.reset = Mock(name='hub.reset()') From 3f4d8d795ad128bd7430cc5dc174a802cded425c Mon Sep 17 00:00:00 2001 From: Tomer Nosrati Date: Thu, 26 Mar 2026 12:09:52 +0000 Subject: [PATCH 168/169] Prepare for release: v5.6.3 (#10221) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 5.6.2 → 5.6.3 * ci: enable CI on v5.6.x branch * Added Changelog for v5.6.3 --- .bumpversion.cfg | 2 +- .github/workflows/python-package.yml | 4 +- Changelog.rst | 63 ++++++++++++++++++++++++++++ README.rst | 2 +- celery/__init__.py | 2 +- docs/includes/introduction.txt | 2 +- 6 files changed, 69 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 429ef97e7aa..f8d0714eff7 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.6.2 +current_version = 5.6.3 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 66f034e7058..997585a0b9a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,7 +5,7 @@ name: Celery on: push: - branches: [ 'main'] + branches: [ 'main', 'v5.6.x' ] paths: - '**.py' - '**.txt' @@ -13,7 +13,7 @@ on: - '**.toml' - "tox.ini" pull_request: - branches: [ 'main' ] + branches: [ 'main', 'v5.6.x' ] paths: - '**.py' - '**.txt' diff --git a/Changelog.rst b/Changelog.rst index 2ff3ca1968d..39d6192d845 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -8,6 +8,69 @@ This document contains change notes for bugfix & new features in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for an overview of what's new in Celery 5.6. +.. _version-5.6.3: + +5.6.3 +===== + +:release-date: 2026-03-26 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix Django worker recursion bug + defensive checks for pool_cls.__module__ (#10048) +- Docs: Update user_preload_options example to use click. (#10056) +- Fix invalid configuration key "bootstrap_servers" in Kafka demo (#10060) +- Fix broken images on PyPI page (#10066) +- Remove broken reference. (#10071) +- Removed --dist=loadscope from smoke tests (#10073) +- Docs: Clarify task_retry signal args may be None (#10076) +- Update example for Django (#10081) +- Make tests compatible with pymongo >= 4.16 (#10074) +- fix: source install of cassandra-driver (#10105) +- fix: register task cross-reference role in Sphinx extension (#10100) +- fix: avoid cycle detection in native delayed delivery (#10095) +- fix(asynpool): avoid AttributeError when proc lacks _sentinel_poll (#10086) +- fix dusk_astronomical horizon sign (+18 -> -18) (#10121) +- Fix/10106 onupdate col use lambda func (#10108) +- Fix warm shutdown RuntimeError with eventlet>=0.37.0 (#10083) (#10123) +- Fix 10109 db backend connection health (#10124) +- Database Backend filter unsupport sql engine arguments with nullpool #7355 (#10134) +- fix(beat): correct argument order in Service.__reduce__ (#10137) +- ci: declare explicit read-only token permissions in workflow jobs (#10139) +- chore: 'boto3to' to 'boto3 to' (#10133) +- Database Backend: Add missing index on date_done (Fixes #10097) (#10098) +- docs: fix typo in CONTRIBUTING.rst (#10141) +- Refer to Flower / Prometheus for monitoring (#10140) +- docs: remove duplicated words in broker and routing docs (#10146) +- docs: fix stale version reference and grammar in README (#10145) +- docs: fix wording in Celery 5.3 worker pool notes (#10149) +- docs: fix duplicated wording in 3.1 changelog entry (#10152) +- docs: fix changelog typo in context manager wording (#10144) +- Fix/10096 worker fails to reconnect after redis failover (#10151) +- Improve on_after_finalize signal documentation (#10155) +- Add non-commutative example to clarify partial arg ordering in canvas docs (#10157) +- Remove redundant test_isa_mapping test (fixes #10077) (#10103) +- Upgrade pytest-celery to >=1.3.0 and adopt PYTEST_CELERY_PKG build arg (#10162) +- Remove deprecated args from redis get_connection call (#10036) +- Fix #6912 rpc backend reconnection error (#10179) +- Fix NameError with TYPE_CHECKING annotations on Python 3.14+ (PEP 649) (#10165) +- docs: Add elaboration on prefetch multiplier settings (worker_prefetch_multiplier) and worker_eta_task_limit (#10181) +- Fix O(K²) message bloat in a chain of chords (#10171) +- Fix mock connection interfaces to prevent `TypeError` during exception handling (#10178) +- fix(trace): dispatch chain/callbacks on dedup fast-path for redelivered tasks (#10159) +- Extract `reconnect_on_error` to `BaseResultConsumer` (#10189) +- pep 649 (#10187) +- Fix#9722 friendly status errors for CLI (#10190) +- docs: clarify after_return behavior for retried tasks (#10192) +- Add compression header to message protocol docs (#10156) +- docs: fix duplicated word in bootsteps comment (#10153) +- Remove outdated autoreloader section from extending docs (#10154) +- Fix: prioritize request ignore_result over task definition (#10184) +- fix: clear the timer while catch the exception (#10218) +- Prepare for release: v5.6.3 (#10221) + .. _version-5.6.2: 5.6.2 diff --git a/README.rst b/README.rst index 6867e8d84ef..a3586020062 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.6.2 (recovery) +:Version: 5.6.3 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/celery/__init__.py b/celery/__init__.py index 36bdb65bc67..a8a798c288e 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -24,7 +24,7 @@ SERIES = 'recovery' -__version__ = '5.6.2' +__version__ = '5.6.3' __author__ = 'Ask Solem' __contact__ = 'auvipy@gmail.com' __homepage__ = 'https://docs.celeryq.dev/' diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 01663e64a6e..17350312e5a 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.6.2 (recovery) +:Version: 5.6.3 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ From 54a2976ca69bdc4e851db907f0c7a80df946732c Mon Sep 17 00:00:00 2001 From: Kaustav Das Sharma Date: Mon, 6 Jul 2026 20:39:40 -0700 Subject: [PATCH 169/169] Rebuild gumloop-celery on celery 5.6.3 Fresh branch from v5.6.3 (kombu>=5.6.0 -> unlocks redis 5.3.1, redis-py#3557). Re-applies the gumloop delta: spawn pool + alias, GMLP-9012 ready_worker_limit recycling gate, and packaging identity. disable-prefetch (old PR #1) is dropped because celery upstreamed it in 5.6.x (#9863/#9919). --- README.MD | 31 ++++++++ celery/__init__.py | 8 +- celery/concurrency/__init__.py | 1 + celery/concurrency/spawn.py | 19 +++++ celery/worker/consumer/tasks.py | 26 ++++++- .../reference/celery.concurrency.spawn.rst | 11 +++ docs/internals/reference/index.rst | 1 + docs/userguide/concurrency/index.rst | 3 + docs/userguide/concurrency/spawn.rst | 18 +++++ setup.py | 7 +- t/integration/test_spawn_pool.py | 20 +++++ t/unit/concurrency/test_concurrency.py | 2 + t/unit/concurrency/test_spawn.py | 77 +++++++++++++++++++ t/unit/worker/test_consumer.py | 31 ++++++++ 14 files changed, 245 insertions(+), 10 deletions(-) create mode 100644 README.MD create mode 100644 celery/concurrency/spawn.py create mode 100644 docs/internals/reference/celery.concurrency.spawn.rst create mode 100644 docs/userguide/concurrency/spawn.rst create mode 100644 t/integration/test_spawn_pool.py create mode 100644 t/unit/concurrency/test_spawn.py diff --git a/README.MD b/README.MD new file mode 100644 index 00000000000..08b85e8bfcb --- /dev/null +++ b/README.MD @@ -0,0 +1,31 @@ +# Gumloop Fork of the Official Celery SDK + +This fork was created for use with gumloop since the official package does not have spawn support for concurrency. + +The core should be rebased from `main` of the official repository from time to time. + +### Official Repository + +[Official Repository URL](https://github.com/celery/celery) + +## Building and Deploying + +### Building the Package + +To build the package: + +```bash +python -m build +``` + +This will create distribution packages in the `dist/` directory. + +### Deploying to Artifact Registry + +To deploy to Google Artifact Registry: + +```bash +python -m twine upload --repository-url https://us-west1-python.pkg.dev/agenthub-dev/gumloop/ dist/* --skip-existing +``` + +You'll need appropriate authentication credentials configured for the Artifact Registry repository. diff --git a/celery/__init__.py b/celery/__init__.py index a8a798c288e..3467851685f 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -24,10 +24,10 @@ SERIES = 'recovery' -__version__ = '5.6.3' -__author__ = 'Ask Solem' -__contact__ = 'auvipy@gmail.com' -__homepage__ = 'https://docs.celeryq.dev/' +__version__ = '5.6.3+gumloop_0.2.0' +__author__ = 'Rahul Behal' +__contact__ = 'rahul@gumloop.com' +__homepage__ = 'https://github.com/gumloop/gumloop-celery' __docformat__ = 'restructuredtext' __keywords__ = 'task job queue distributed messaging actor' diff --git a/celery/concurrency/__init__.py b/celery/concurrency/__init__.py index 4953f463f01..36ba31c66ee 100644 --- a/celery/concurrency/__init__.py +++ b/celery/concurrency/__init__.py @@ -10,6 +10,7 @@ ALIASES = { 'prefork': 'celery.concurrency.prefork:TaskPool', + 'spawn': 'celery.concurrency.spawn:TaskPool', 'eventlet': 'celery.concurrency.eventlet:TaskPool', 'gevent': 'celery.concurrency.gevent:TaskPool', 'solo': 'celery.concurrency.solo:TaskPool', diff --git a/celery/concurrency/spawn.py b/celery/concurrency/spawn.py new file mode 100644 index 00000000000..a0a87c8b46f --- /dev/null +++ b/celery/concurrency/spawn.py @@ -0,0 +1,19 @@ +"""Spawn execution pool.""" +import os + +import billiard + +from .prefork import TaskPool as PreforkTaskPool + +__all__ = ("TaskPool",) + + +class TaskPool(PreforkTaskPool): + """Multiprocessing Pool using the ``spawn`` start method.""" + + start_method = "spawn" + + def on_start(self): + billiard.set_start_method(self.start_method, force=True) + os.environ.setdefault("FORKED_BY_MULTIPROCESSING", "1") + super().on_start() diff --git a/celery/worker/consumer/tasks.py b/celery/worker/consumer/tasks.py index b017cf52838..6aabbf620c7 100644 --- a/celery/worker/consumer/tasks.py +++ b/celery/worker/consumer/tasks.py @@ -26,6 +26,26 @@ def __init__(self, c, **kwargs): c.task_consumer = c.qos = None super().__init__(c, **kwargs) + @staticmethod + def ready_worker_limit(consumer): + """Configured concurrency, capped by the number of workers ready to run. + + Excludes children that are recycling/cold-starting (not yet in the + pool's ``_fileno_to_inq``) so we don't fetch a task into a slot that + can't run it. Guarded: any error, or a pool without that attribute, + falls back to the configured concurrency, keeping this purely additive. + """ + limit = getattr(consumer.controller, "max_concurrency", None) + if limit is None: + limit = consumer.pool.num_processes + try: + ready = getattr(getattr(consumer.pool, "_pool", None), "_fileno_to_inq", None) + if ready is not None: + limit = min(limit, len(ready)) + except Exception: + pass + return limit + def start(self, c): """Start task consumer.""" c.update_strategies() @@ -69,9 +89,9 @@ def set_prefetch_count(prefetch_count): original_can_consume = channel_qos.can_consume def can_consume(self): - # Prefer autoscaler's max_concurrency if set; otherwise fall back to pool size - limit = getattr(c.controller, "max_concurrency", None) or c.pool.num_processes - if len(state.reserved_requests) >= limit: + # Gate on workers that have completed the WORKER_UP handshake so a + # recycling/cold-starting slot doesn't pull a task it can't run. + if len(state.reserved_requests) >= Tasks.ready_worker_limit(c): return False return original_can_consume() diff --git a/docs/internals/reference/celery.concurrency.spawn.rst b/docs/internals/reference/celery.concurrency.spawn.rst new file mode 100644 index 00000000000..a49621571a5 --- /dev/null +++ b/docs/internals/reference/celery.concurrency.spawn.rst @@ -0,0 +1,11 @@ +============================================================== + ``celery.concurrency.spawn`` +============================================================== + +.. contents:: + :local: +.. currentmodule:: celery.concurrency.spawn + +.. automodule:: celery.concurrency.spawn + :members: + :undoc-members: diff --git a/docs/internals/reference/index.rst b/docs/internals/reference/index.rst index 483ea193444..f740348bed9 100644 --- a/docs/internals/reference/index.rst +++ b/docs/internals/reference/index.rst @@ -17,6 +17,7 @@ celery.concurrency celery.concurrency.solo celery.concurrency.prefork + celery.concurrency.spawn celery.concurrency.eventlet celery.concurrency.gevent celery.concurrency.thread diff --git a/docs/userguide/concurrency/index.rst b/docs/userguide/concurrency/index.rst index d0355fdfb80..bdb04f6c67c 100644 --- a/docs/userguide/concurrency/index.rst +++ b/docs/userguide/concurrency/index.rst @@ -20,6 +20,8 @@ Overview of Concurrency Options - `prefork`: The default option, ideal for CPU-bound tasks and most use cases. It is robust and recommended unless there's a specific need for another model. +- `spawn`: Uses Python's ``spawn`` start method. Helpful when libraries are not + fork-safe, for example when using CUDA. - `eventlet` and `gevent`: Designed for IO-bound tasks, these models use greenlets for high concurrency. Note that certain features, like `soft_timeout`, are not available in these modes. These have detailed documentation pages @@ -35,6 +37,7 @@ Overview of Concurrency Options eventlet gevent + spawn .. note:: While alternative models like `eventlet` and `gevent` are available, they diff --git a/docs/userguide/concurrency/spawn.rst b/docs/userguide/concurrency/spawn.rst new file mode 100644 index 00000000000..27a28cb5dbb --- /dev/null +++ b/docs/userguide/concurrency/spawn.rst @@ -0,0 +1,18 @@ +.. _concurrency-spawn: + +======================= + Spawn Start Method +======================= + +Celery can use Python's ``spawn`` start method to create worker processes. +This is useful when libraries used by your tasks are not fork-safe, for +example when working with CUDA. + +Enable this pool using the :option:`celery worker -P` option: + +.. code-block:: console + + $ celery -A proj worker -P spawn -c 4 + +When using ``spawn`` each worker starts in a fresh Python interpreter +so any global state must be initialized in the child process. diff --git a/setup.py b/setup.py index bbd55cbf0d7..f9074260871 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ import setuptools -NAME = 'celery' +NAME = 'gumloop-celery' # -*- Extras -*- @@ -64,7 +64,7 @@ def parse_dist_meta(): """Extract metadata information from ``$dist/__init__.py``.""" pats = {re_meta: _add_default, re_doc: _add_doc} here = os.path.abspath(os.path.dirname(__file__)) - with open(os.path.join(here, NAME, '__init__.py')) as meta_fh: + with open(os.path.join(here, 'celery', '__init__.py')) as meta_fh: distmeta = {} for line in meta_fh: if line.strip() == '# -eof meta-': @@ -178,5 +178,6 @@ def long_description(): "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Operating System :: OS Independent" - ] + ], + provides=['celery'] ) diff --git a/t/integration/test_spawn_pool.py b/t/integration/test_spawn_pool.py new file mode 100644 index 00000000000..912f2fdd259 --- /dev/null +++ b/t/integration/test_spawn_pool.py @@ -0,0 +1,20 @@ +import pytest + +from .tasks import add + + +@pytest.fixture(scope="session") +def celery_worker_pool(): + return "spawn" + + +@pytest.mark.flaky(reruns=5, reruns_delay=2) +def test_basic_task_spawn(manager): + results = [] + for i in range(5): + results.append([i + i, add.delay(i, i)]) + for expected, result in results: + assert result.get(timeout=10) == expected + assert result.status == "SUCCESS" + assert result.ready() is True + assert result.successful() is True diff --git a/t/unit/concurrency/test_concurrency.py b/t/unit/concurrency/test_concurrency.py index ba80aa98ec5..71a755ccc8b 100644 --- a/t/unit/concurrency/test_concurrency.py +++ b/t/unit/concurrency/test_concurrency.py @@ -163,6 +163,7 @@ class test_get_available_pool_names: def test_no_concurrent_futures__returns_no_threads_pool_name(self): expected_pool_names = ( 'prefork', + 'spawn', 'eventlet', 'gevent', 'solo', @@ -176,6 +177,7 @@ def test_no_concurrent_futures__returns_no_threads_pool_name(self): def test_concurrent_futures__returns_threads_pool_name(self): expected_pool_names = ( 'prefork', + 'spawn', 'eventlet', 'gevent', 'solo', diff --git a/t/unit/concurrency/test_spawn.py b/t/unit/concurrency/test_spawn.py new file mode 100644 index 00000000000..4db01596e34 --- /dev/null +++ b/t/unit/concurrency/test_spawn.py @@ -0,0 +1,77 @@ +import os +from unittest.mock import Mock, patch + +from celery.concurrency import spawn + + +class MockPool: + """Mock pool that prevents actual process creation.""" + started = False + closed = False + joined = False + terminated = False + _state = None + + def __init__(self, *args, **kwargs): + self.started = True + self._timeout_handler = Mock() + self._result_handler = Mock() + self.maintain_pool = Mock() + self._state = 1 # RUN state + self._processes = kwargs.get('processes', 1) + self._proc_alive_timeout = kwargs.get('proc_alive_timeout') + self._pool = [Mock(pid=i) for i in range(self._processes)] + + def close(self): + self.closed = True + self._state = 'CLOSE' + + def join(self): + self.joined = True + + def terminate(self): + self.terminated = True + + def did_start_ok(self): + return True + + def apply_async(self, *args, **kwargs): + pass + + def terminate_job(self, *args, **kwargs): + pass + + def restart(self, *args, **kwargs): + pass + + def register_with_event_loop(self, loop): + pass + + def flush(self): + pass + + def grow(self, n=1): + self._processes += n + + def shrink(self, n=1): + self._processes -= n + + +class TestTaskPool(spawn.TaskPool): + """TaskPool that uses MockPool instead of real billiard pools.""" + Pool = MockPool + BlockingPool = MockPool + + +class test_spawn_TaskPool: + @patch('billiard.set_start_method') + @patch('billiard.forking_enable') + def test_on_start_sets_spawn(self, mock_forking_enable, set_method): + pool = TestTaskPool(1) + with patch.dict(os.environ, {}, clear=True): + pool.on_start() + set_method.assert_called_with('spawn', force=True) + assert os.environ['FORKED_BY_MULTIPROCESSING'] == '1' + # Verify the pool was created + assert pool._pool is not None + assert pool._pool.started diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index 68e12cc3a0d..8cdb908795f 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -1004,6 +1004,37 @@ def test_qos_with_zero_worker_eta_task_limit(self): assert len(args) == 2 # callback and initial_value assert kwargs.get('max_prefetch') == 0 + def test_ready_worker_limit_caps_at_ready_workers(self): + # Concurrency 3 but only 2 workers have completed the WORKER_UP + # handshake (one recycling), so the limit is capped at 2. + c = self.c + c.controller.max_concurrency = 3 + c.pool._pool._fileno_to_inq = {10: 'w1', 11: 'w2'} + assert Tasks.ready_worker_limit(c) == 2 + + def test_ready_worker_limit_full_when_all_ready(self): + c = self.c + c.controller.max_concurrency = 3 + c.pool._pool._fileno_to_inq = {10: 'w1', 11: 'w2', 12: 'w3'} + assert Tasks.ready_worker_limit(c) == 3 + + def test_ready_worker_limit_without_prefork_pool(self): + # Pools without _fileno_to_inq (solo/eventlet/gevent) keep the prior + # behaviour: the configured concurrency. + c = self.c + c.controller.max_concurrency = None + c.pool.num_processes = 3 + c.pool._pool = object() + assert Tasks.ready_worker_limit(c) == 3 + + def test_ready_worker_limit_falls_back_on_introspection_error(self): + # If reading the ready set raises, fall back to configured concurrency + # rather than breaking can_consume (purely additive). + c = self.c + c.controller.max_concurrency = 3 + c.pool._pool._fileno_to_inq = object() # len() raises + assert Tasks.ready_worker_limit(c) == 3 + class test_Agent: