diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 00000000000..f8d0714eff7 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,14 @@ +[bumpversion] +current_version = 5.6.3 +commit = True +tag = True +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P[a-z\d]+)? +serialize = + {major}.{minor}.{patch}{releaselevel} + {major}.{minor}.{patch} + +[bumpversion:file:celery/__init__.py] + +[bumpversion:file:docs/includes/introduction.txt] + +[bumpversion:file:README.rst] diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..6f04c910819 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,33 @@ +.DS_Store +*.pyc +*$py.class +*~ +.*.sw[pon] +dist/ +*.egg-info +*.egg +*.egg/ +*.eggs/ +build/ +.build/ +_build/ +pip-log.txt +.directory +erl_crash.dump +*.db +Documentation/ +.tox/ +.ropeproject/ +.project +.pydevproject +.idea/ +.coverage +celery/tests/cover/ +.ve* +cover/ +.vagrant/ +.cache/ +htmlcov/ +coverage.xml +test.db +.git/ diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000000..140566f1819 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +# http://editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 +end_of_line = lf +max_line_length = 117 + +[Makefile] +indent_style = tab diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000000..55c5ce97aa7 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,8 @@ +# These are supported funding model platforms + +github: celery +patreon: +open_collective: celery +ko_fi: # Replace with a single Ko-fi username +tidelift: "pypi/celery" +custom: # Replace with a single custom sponsorship URL diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000000..f9317a3f35a --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,4 @@ + diff --git a/.github/ISSUE_TEMPLATE/Bug-Report.md b/.github/ISSUE_TEMPLATE/Bug-Report.md new file mode 100644 index 00000000000..6ec1556e0b7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Bug-Report.md @@ -0,0 +1,166 @@ +--- +name: Bug Report +about: Is something wrong with Celery? +title: '' +labels: 'Issue Type: Bug Report' +assignees: '' + +--- + + +# Checklist + +- [ ] I have verified that the issue exists against the `main` branch of Celery. +- [ ] This has already been asked to the [discussions forum](https://github.com/celery/celery/discussions) first. +- [ ] I have read the relevant section in the + [contribution guide](https://docs.celeryq.dev/en/main/contributing.html#other-bugs) + on reporting bugs. +- [ ] I have checked the [issues list](https://github.com/celery/celery/issues?q=is%3Aissue+label%3A%22Issue+Type%3A+Bug+Report%22+-label%3A%22Category%3A+Documentation%22) + for similar or identical bug reports. +- [ ] I have checked the [pull requests list](https://github.com/celery/celery/pulls?q=is%3Apr+label%3A%22PR+Type%3A+Bugfix%22+-label%3A%22Category%3A+Documentation%22) + for existing proposed fixes. +- [ ] I have checked the [commit log](https://github.com/celery/celery/commits/main) + to find out if the bug was already fixed in the main branch. +- [ ] I have included all related issues and possible duplicate issues + in this issue (If there are none, check this box anyway). +- [ ] I have tried to reproduce the issue with [pytest-celery](https://docs.celeryq.dev/projects/pytest-celery/en/latest/userguide/celery-bug-report.html) and added the reproduction script below. + +## Mandatory Debugging Information + +- [ ] I have included the output of ``celery -A proj report`` in the issue. + (if you are not able to do this, then at least specify the Celery + version affected). +- [ ] I have verified that the issue exists against the `main` branch of Celery. +- [ ] I have included the contents of ``pip freeze`` in the issue. +- [ ] I have included all the versions of all the external dependencies required + to reproduce this bug. + +## Optional Debugging Information + +- [ ] I have tried reproducing the issue on more than one Python version + and/or implementation. +- [ ] I have tried reproducing the issue on more than one message broker and/or + result backend. +- [ ] I have tried reproducing the issue on more than one version of the message + broker and/or result backend. +- [ ] I have tried reproducing the issue on more than one operating system. +- [ ] I have tried reproducing the issue on more than one workers pool. +- [ ] I have tried reproducing the issue with autoscaling, retries, + ETA/Countdown & rate limits disabled. +- [ ] I have tried reproducing the issue after downgrading + and/or upgrading Celery and its dependencies. + +## Related Issues and Possible Duplicates + + +#### Related Issues + +- None + +#### Possible Duplicates + +- None + +## Environment & Settings + +**Celery version**: + +
+celery report Output: +

+ +``` +``` + +

+
+ +# Steps to Reproduce + +## Required Dependencies + +- **Minimal Python Version**: N/A or Unknown +- **Minimal Celery Version**: N/A or Unknown +- **Minimal Kombu Version**: N/A or Unknown +- **Minimal Broker Version**: N/A or Unknown +- **Minimal Result Backend Version**: N/A or Unknown +- **Minimal OS and/or Kernel Version**: N/A or Unknown +- **Minimal Broker Client Version**: N/A or Unknown +- **Minimal Result Backend Client Version**: N/A or Unknown + +### Python Packages + +
+pip freeze Output: +

+ +``` +``` + +

+
+ +### Other Dependencies + +
+

+N/A +

+
+ +## Minimally Reproducible Test Case + + +
+

+ +```python +``` + +

+
+ +# Expected Behavior + + +# Actual Behavior + diff --git a/.github/ISSUE_TEMPLATE/Documentation-Bug-Report.md b/.github/ISSUE_TEMPLATE/Documentation-Bug-Report.md new file mode 100644 index 00000000000..97f341dbc40 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Documentation-Bug-Report.md @@ -0,0 +1,56 @@ +--- +name: Documentation Bug Report +about: Is something wrong with our documentation? +title: '' +labels: 'Category: Documentation, Issue Type: Bug Report' +assignees: '' + +--- + + +# Checklist + + +- [ ] I have checked the [issues list](https://github.com/celery/celery/issues?utf8=%E2%9C%93&q=is%3Aissue+label%3A%22Category%3A+Documentation%22+) + for similar or identical bug reports. +- [ ] I have checked the [pull requests list](https://github.com/celery/celery/pulls?q=is%3Apr+label%3A%22Category%3A+Documentation%22) + for existing proposed fixes. +- [ ] I have checked the [commit log](https://github.com/celery/celery/commits/main) + to find out if the bug was already fixed in the main branch. +- [ ] I have included all related issues and possible duplicate issues in this issue + (If there are none, check this box anyway). + +## Related Issues and Possible Duplicates + + +#### Related Issues + +- None + +#### Possible Duplicates + +- None + +# Description + + +# Suggestions + diff --git a/.github/ISSUE_TEMPLATE/Enhancement.md b/.github/ISSUE_TEMPLATE/Enhancement.md new file mode 100644 index 00000000000..363f4630628 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Enhancement.md @@ -0,0 +1,94 @@ +--- +name: Enhancement +about: Do you want to improve an existing feature? +title: '' +labels: 'Issue Type: Enhancement' +assignees: '' + +--- + + +# Checklist + + +- [ ] I have checked the [issues list](https://github.com/celery/celery/issues?q=is%3Aissue+label%3A%22Issue+Type%3A+Enhancement%22+-label%3A%22Category%3A+Documentation%22) + for similar or identical enhancement to an existing feature. +- [ ] I have checked the [pull requests list](https://github.com/celery/celery/pulls?q=is%3Apr+label%3A%22Issue+Type%3A+Enhancement%22+-label%3A%22Category%3A+Documentation%22) + for existing proposed enhancements. +- [ ] I have checked the [commit log](https://github.com/celery/celery/commits/main) + to find out if the same enhancement was already implemented in the + main branch. +- [ ] I have included all related issues and possible duplicate issues in this issue + (If there are none, check this box anyway). + +## Related Issues and Possible Duplicates + + +#### Related Issues + +- None + +#### Possible Duplicates + +- None + +# Brief Summary + + +# Design + +## Architectural Considerations + +None + +## Proposed Behavior + + +## Proposed UI/UX + + +## Diagrams + +N/A + +## Alternatives + +None diff --git a/.github/ISSUE_TEMPLATE/Feature-Request.md b/.github/ISSUE_TEMPLATE/Feature-Request.md new file mode 100644 index 00000000000..5de9452a55c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Feature-Request.md @@ -0,0 +1,93 @@ +--- +name: Feature Request +about: Do you need a new feature? +title: '' +labels: 'Issue Type: Feature Request' +assignees: '' + +--- + + +# Checklist + + +- [ ] I have checked the [issues list](https://github.com/celery/celery/issues?utf8=%E2%9C%93&q=is%3Aissue+label%3A%22Issue+Type%3A+Feature+Request%22+) + for similar or identical feature requests. +- [ ] I have checked the [pull requests list](https://github.com/celery/celery/pulls?utf8=%E2%9C%93&q=is%3Apr+label%3A%22PR+Type%3A+Feature%22+) + for existing proposed implementations of this feature. +- [ ] I have checked the [commit log](https://github.com/celery/celery/commits/main) + to find out if the same feature was already implemented in the + main branch. +- [ ] I have included all related issues and possible duplicate issues + in this issue (If there are none, check this box anyway). + +## Related Issues and Possible Duplicates + + +#### Related Issues + +- None + +#### Possible Duplicates + +- None + +# Brief Summary + + +# Design + +## Architectural Considerations + +None + +## Proposed Behavior + + +## Proposed UI/UX + + +## Diagrams + +N/A + +## Alternatives + +None diff --git a/.github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md b/.github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md new file mode 100644 index 00000000000..fcc81ec0aa9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Major-Version-Release-Checklist.md @@ -0,0 +1,48 @@ +--- +name: Major Version Release Checklist +about: About to release a new major version? (Maintainers Only!) +title: '' +labels: '' +assignees: '' + +--- + +Version: +Release PR: + +# Description + + + +# Checklist + +- [ ] Release PR drafted +- [ ] Milestone is 100% done +- [ ] Merge Freeze +- [ ] Release PR reviewed +- [ ] The main branch build passes + + [![Build Status](https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg)](https://github.com/celery/celery/actions/workflows/python-package.yml) +- [ ] Release Notes +- [ ] What's New + +# Process + +# Alphas + + +- [ ] Alpha 1 + +## Betas + + +- [ ] Beta 1 + +## Release Candidates + + +- [ ] RC 1 + +# Release Blockers + +# Potential Release Blockers diff --git a/.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md b/.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md new file mode 100644 index 00000000000..63e91a5d87c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Minor-Version-Release-Checklist.md @@ -0,0 +1,136 @@ +--- +name: Minor Version Release Checklist +about: About to release a new minor version? (Maintainers Only!) +title: '' +labels: '' +assignees: '' + +--- + +# Minor Release Overview: v + +This issue will summarize the status and discussion in preparation for the new release. It will be used to track the progress of the release and to ensure that all the necessary steps are taken. It will serve as a checklist for the release and will be used to communicate the status of the release to the community. + +> ⚠️ **Warning:** The release checklist is a living document. It will be updated as the release progresses. Please check back often to ensure that you are up to date with the latest information. + +## Checklist +- [ ] Codebase Stability +- [ ] Breaking Changes Validation +- [ ] Compile Changelog +- [ ] Release +- [ ] Release Announcement + +# Release Details +The release manager is responsible for completing the release end-to-end ensuring that all the necessary steps are taken and that the release is completed in a timely manner. This is usually the owner of the release issue but may be assigned to a different maintainer if necessary. + +- Release Manager: +- Release Date: +- Release Branch: `main` + +# Release Steps +The release manager is expected to execute the checklist below. The release manager is also responsible for ensuring that the checklist is updated as the release progresses. Any changes or issues should be communicated under this issue for centralized tracking. + +# Potential Release Blockers + +## 1. Codebase Stability +- [ ] The `main` branch build passes + + [![Build Status](https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg)](https://github.com/celery/celery/actions/workflows/python-package.yml) + +## 2. Breaking Changes Validation +A patch release should not contain any breaking changes. The release manager is responsible for reviewing all of the merged PRs since the last release to ensure that there are no breaking changes. If there are any breaking changes, the release manager should discuss with the maintainers to determine the best course of action if an obvious solution is not apparent. + +## 3. Compile Changelog +The release changelog is set in two different places: +1. The [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) that uses the RST format. +2. The GitHub Release auto-generated changelog that uses the Markdown format. This is auto-generated by the GitHub Draft Release UI. + +> ⚠️ **Warning:** The pre-commit changes should not be included in the changelog. + +To generate the changelog automatically, [draft a new release](https://github.com/celery/celery/releases/new) on GitHub using a fake new version tag for the automatic changelog generation. Notice the actual tag creation is done **on publish** so we can use that to generate the changelog and then delete the draft release without publishing it thus avoiding creating a new tag. + +- Create a new tag +CleanShot 2023-09-05 at 22 06 24@2x + +- Generate Markdown release notes +CleanShot 2023-09-05 at 22 13 39@2x + +- Copy the generated release notes. + +- Delete the draft release without publishing it. + +### 3.1 Changelog.rst +Once you have the actual changes, you need to convert it to rst format and add it to the [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) file. The new version block needs to follow the following format: +```rst +.. _version-x.y.z: + +x.y.z +===== + +:release-date: YYYY-MM-DD HH:MM P.M/A.M TimeZone +:release-by: Release Manager Name + +Changes list in RST format. +``` + +These changes will reflect in the [Change history](https://docs.celeryq.dev/en/stable/changelog.html) section of the documentation. + +### 3.2 Changelog PR +The changes to the [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) file should be submitted as a PR. This will PR should be the last merged PR before the release. + +## 4. Release +### 4.1 Prepare releasing environment +Before moving forward with the release, the release manager should ensure that bumpversion and twine are installed. These are required to publish the release. + +### 4.2 Bump version +The release manager should bump the version using the following command: +```bash +bumpversion patch +``` +The changes should be pushed directly to main by the release manager. + +At this point, the git log should appear somewhat similar to this: +``` +commit XXX (HEAD -> main, tag: vX.Y.Z, upstream/main, origin/main) +Author: Release Manager +Date: YYY + + Bump version: a.b.c → x.y.z + +commit XXX +Author: Release Manager +Date: YYY + + Added changelog for vX.Y.Z (#1234) +``` +If everything looks good, the bump version commit can be directly pushed to `main`: +```bash +git push origin main --tags +``` + +### 4.3 Publish release to PyPI +The release manager should publish the release to PyPI using the following commands running under the root directory of the repository: +```bash +python setup.py clean build sdist bdist_wheel +``` +If the build is successful, the release manager should publish the release to PyPI using the following command: +```bash +twine upload dist/celery-X.Y.Z* +``` + +> ⚠️ **Warning:** The release manager should double check that the release details are correct (project/version) before publishing the release to PyPI. + +> ⚠️ **Critical Reminder:** Should the released package prove to be faulty or need retraction for any reason, do not delete it from PyPI. The appropriate course of action is to "yank" the release. + +## Release Announcement +After the release is published, the release manager should create a new GitHub Release and set it as the latest release. + +CleanShot 2023-09-05 at 22 51 24@2x + +### Add Release Notes +On a per-case basis, the release manager may also attach an additional release note to the auto-generated release notes. This is usually done when there are important changes that are not reflected in the auto-generated release notes. + +### OpenCollective Update +After successfully publishing the new release, the release manager is responsible for announcing it on the project's OpenCollective [page](https://opencollective.com/celery/updates). This is to engage with the community and keep backers and sponsors in the loop. + + diff --git a/.github/ISSUE_TEMPLATE/Patch-Version-Release-Checklist.md b/.github/ISSUE_TEMPLATE/Patch-Version-Release-Checklist.md new file mode 100644 index 00000000000..0140d93e1c3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/Patch-Version-Release-Checklist.md @@ -0,0 +1,136 @@ +--- +name: Patch Version Release Checklist +about: About to release a new patch version? (Maintainers Only!) +title: '' +labels: '' +assignees: '' + +--- + +# Patch Release Overview: v + +This issue will summarize the status and discussion in preparation for the new release. It will be used to track the progress of the release and to ensure that all the necessary steps are taken. It will serve as a checklist for the release and will be used to communicate the status of the release to the community. + +> ⚠️ **Warning:** The release checklist is a living document. It will be updated as the release progresses. Please check back often to ensure that you are up to date with the latest information. + +## Checklist +- [ ] Codebase Stability +- [ ] Breaking Changes Validation +- [ ] Compile Changelog +- [ ] Release +- [ ] Release Announcement + +# Release Details +The release manager is responsible for completing the release end-to-end ensuring that all the necessary steps are taken and that the release is completed in a timely manner. This is usually the owner of the release issue but may be assigned to a different maintainer if necessary. + +- Release Manager: +- Release Date: +- Release Branch: `main` + +# Release Steps +The release manager is expected to execute the checklist below. The release manager is also responsible for ensuring that the checklist is updated as the release progresses. Any changes or issues should be communicated under this issue for centralized tracking. + +## 1. Codebase Stability +- [ ] The `main` branch build passes + + [![Build Status](https://github.com/celery/celery/actions/workflows/python-package.yml/badge.svg)](https://github.com/celery/celery/actions/workflows/python-package.yml) + +## 2. Breaking Changes Validation +A patch release should not contain any breaking changes. The release manager is responsible for reviewing all of the merged PRs since the last release to ensure that there are no breaking changes. If there are any breaking changes, the release manager should discuss with the maintainers to determine the best course of action if an obvious solution is not apparent. + +## 3. Compile Changelog +The release changelog is set in two different places: +1. The [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) that uses the RST format. +2. The GitHub Release auto-generated changelog that uses the Markdown format. This is auto-generated by the GitHub Draft Release UI. + +> ⚠️ **Warning:** The pre-commit changes should not be included in the changelog. + +To generate the changelog automatically, [draft a new release](https://github.com/celery/celery/releases/new) on GitHub using a fake new version tag for the automatic changelog generation. Notice the actual tag creation is done **on publish** so we can use that to generate the changelog and then delete the draft release without publishing it thus avoiding creating a new tag. + +- Create a new tag +CleanShot 2023-09-05 at 22 06 24@2x + +- Generate Markdown release notes +CleanShot 2023-09-05 at 22 13 39@2x + +- Copy the generated release notes. + +- Delete the draft release without publishing it. + +### 3.1 Changelog.rst +Once you have the actual changes, you need to convert it to rst format and add it to the [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) file. The new version block needs to follow the following format: +```rst +.. _version-x.y.z: + +x.y.z +===== + +:release-date: YYYY-MM-DD HH:MM P.M/A.M TimeZone +:release-by: Release Manager Name + +Changes list in RST format. +``` + +These changes will reflect in the [Change history](https://docs.celeryq.dev/en/stable/changelog.html) section of the documentation. + +### 3.2 Changelog PR +The changes to the [Changelog.rst](https://github.com/celery/celery/blob/main/Changelog.rst) file should be submitted as a PR. This will PR should be the last merged PR before the release. + +## 4. Release +### 4.1 Prepare releasing environment +Before moving forward with the release, the release manager should ensure that bumpversion and twine are installed. These are required to publish the release. + +### 4.2 Bump version +The release manager should bump the version using the following command: +```bash +bumpversion patch +``` +The changes should be pushed directly to main by the release manager. + +At this point, the git log should appear somewhat similar to this: +``` +commit XXX (HEAD -> main, tag: vX.Y.Z, upstream/main, origin/main) +Author: Release Manager +Date: YYY + + Bump version: a.b.c → x.y.z + +commit XXX +Author: Release Manager +Date: YYY + + Added changelog for vX.Y.Z (#1234) +``` +If everything looks good, the bump version commit can be directly pushed to `main`: +```bash +git push origin main --tags +``` + +### 4.3 Publish release to PyPI +The release manager should publish the release to PyPI using the following commands running under the root directory of the repository: +```bash +python setup.py clean build sdist bdist_wheel +``` +If the build is successful, the release manager should publish the release to PyPI using the following command: +```bash +twine upload dist/celery-X.Y.Z* +``` + +> ⚠️ **Warning:** The release manager should double check that the release details are correct (project/version) before publishing the release to PyPI. + +> ⚠️ **Critical Reminder:** Should the released package prove to be faulty or need retraction for any reason, do not delete it from PyPI. The appropriate course of action is to "yank" the release. + +## Release Announcement +After the release is published, the release manager should create a new GitHub Release and set it as the latest release. + +CleanShot 2023-09-05 at 22 51 24@2x + +### Add Release Notes +On a per-case basis, the release manager may also attach an additional release note to the auto-generated release notes. This is usually done when there are important changes that are not reflected in the auto-generated release notes. + +### OpenCollective Update +After successfully publishing the new release, the release manager is responsible for announcing it on the project's OpenCollective [page](https://opencollective.com/celery/updates). This is to engage with the community and keep backers and sponsors in the loop. + + +# Release Blockers + \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..44099454b10 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,13 @@ +blank_issues_enabled: false +contact_links: + - name: Kombu Issue Tracker + url: https://github.com/celery/kombu/issues/ + about: If this issue only involves Kombu, please open a new issue there. + - name: Billiard Issue Tracker + url: https://github.com/celery/billiard/issues/ + about: If this issue only involves Billiard, please open a new issue there. + - name: py-amqp Issue Tracker + url: https://github.com/celery/py-amqp/issues/ + about: If this issue only involves py-amqp, please open a new issue there. + - name: pytest-celery Issue Tracker + url: https://github.com/celery/pytest-celery/issues/ diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000000..f9e0765d935 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,14 @@ +*Note*: Before submitting this pull request, please review our [contributing +guidelines](https://docs.celeryq.dev/en/main/contributing.html). + +## Description + + diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000000..bab8f8dcd2e --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,567 @@ +# GitHub Copilot PR Review Guide + +Conservative, question-first review guidance to keep feedback useful, low-noise, and maintainable for a large, long-lived project. + +## Purpose and scope + +- Role: Assist maintainers during PR reviews with concise, question-first feedback that nudges good decisions and documents rationale for posterity. +- Objectives: Surface user-facing behavior changes needing docs; highlight backward-compatibility risks; keep scope focused; encourage consistency and cleanup; optionally suggest tests and local tox usage. +- Principles: Very-high confidence or question-first; bottom-line first; avoid style/lint remarks; avoid prescriptive internal rules unless unambiguous; minimize noise. +- When to ask vs. assert: Ask by default; assert only for obvious issues (e.g., debug leftovers) or when a strict rule clearly applies. +- When to stay silent: Formatting-only changes, comments-only diffs, tests-only edits, or strictly internal refactors with no user-facing impact. + +### What "question-first" means + +- Default to asking when not 90%+ confident; assert only for obvious issues or clear, documented policy. +- Lead with a concise question that contains the bottom-line ask and one-sentence rationale. +- Make it easy to answer: yes/no + suggested next step (e.g., "Should we add versionchanged::?"). +- Avoid prescribing exact code; clarify intent and offer options when needed. +- If confirmed user-facing, follow docs/versioning guidance; if internal-only, prefer consistency and brief rationale. +- One comment per theme; do not repeat after it is addressed. + +## Collaboration contract (Copilot alongside maintainers) + +- Assist maintainers; do not decide. Questions by default; assertions only on clear policy violations or obvious mistakes. +- Never block the review; comments are non-binding prompts for the human reviewer. +- Keep comments atomic and actionable; include the bottom-line ask and, when helpful, a suggested next step. +- Avoid prescriptive code changes unless asked; prefer intent-focused guidance and options. +- Respect repository conventions and CI; skip style/lint feedback that automation enforces. +- Ask once per theme and stop after it's addressed; avoid repetition and noise. + +## Reviewer persona and behavior + +- Prefer question-first comments; assert only with very-high confidence. +- Bottom line first, then brief rationale, then the ask. +- Avoid style/lint remarks (CI handles these). +- Avoid prescriptive internal rules unless policy is unambiguous. +- Keep comments short, technical, specific. + +## Response formatting for Copilot + +- Use standard GitHub Markdown in comments; keep them concise and technical. +- Use fenced code blocks with explicit language where possible: ```diff, ```python, ```sh, ```yaml, ```toml, ```ini, ```rst, or ```text. +- Prefer small unified diffs (```diff) when referencing exact changes; include only the minimal hunk needed. +- Avoid emojis and decorative formatting; focus on clarity and actionability. +- One comment per theme; avoid repetition once addressed. +- When referencing files/lines, include a GitHub permalink to exact lines or ranges (Copy permalink) using commit-SHA anchored URLs, e.g., https://github.com/celery/celery/blob//celery/app/base.py#L820-L860. + +## High-signal focus areas (question-first by default) + +### 1) Backward compatibility risk + +Triggers include: +- Signature/default changes in user-facing APIs (added/removed/renamed params; changed defaults; narrowed/broadened accepted types). +- Return type/shape/order changes (e.g., list -> iterator/generator; tuple -> dict; stable order -> undefined order). +- Exceptions/validation changes (exception type changed; now raises where it previously passed). +- Config/CLI/ENV defaults that alter behavior (e.g., task_acks_late, timeouts, default_queue/default_exchange/default_routing_key, CLI flag defaults). +- Wire/persistence schema changes (task headers/stamping, message/result schema, serialization/content type, visibility-timeout semantics). +- Removing/deprecating public APIs without a documented deprecation window, alias, or compatibility layer. + +What to look for (detectors): +- Param removed/renamed or default flipped in a public signature (or apply_async/send_task options). +- Return type/shape/order changed in code, docstrings, or tests (yield vs list; mapping vs tuple). +- Exception types changed in raise paths or surfaced in tests/docs. +- Defaults changed in celery/app/defaults.py or via config/CLI/ENV resolution. +- Changes to headers/stamps/message body/result schema or serialization in amqp/backend paths. +- Public symbol/behavior removal with no deprecation entry. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This appears to be a user-facing behavior change (X -> Y), which could break existing users because . Is this intended?" + - If yes: Could we add migration guidance in the PR description and docs (versionchanged::), and consider a compat/deprecation path (e.g., alias/flag) through vX.Y? + - If no: Would reverting to the previous behavior and adding a regression test make sense, or alternatively guarding this behind a feature flag until we can provide a proper deprecation path?" + +Examples: +- Case A: Config default change (task_acks_late) + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- acks_late=Option(False, type='bool'), ++ acks_late=Option(True, type='bool'), + ``` + + - Why it matches: Flipping this default changes when tasks are acknowledged; can impact delivery semantics, retries, and failure handling for users not explicitly setting it. + - Example comment: "I see task_acks_late default changed False -> True; this could change delivery/retry semantics for users relying on the current default. Is this intended? If yes, could we add migration guidance and a versionchanged:: entry, and consider a transition plan (e.g., keep False unless explicitly opted in) through vX.Y? If not, should we revert and add a regression test?" + +- Case B: Return type change (list -> iterator) + - Diff (illustrative): + + ```diff + --- a/celery/app/builtins.py + +++ b/celery/app/builtins.py + @@ +- return [task(item) for item in it] ++ return (task(item) for item in it) + ``` + + - Why it matches: Changing to a generator would break callers that rely on len(), indexing, multiple passes, or list operations. + - Example comment: "I see the return type changed from list to iterator; this can break callers relying on len() or multiple passes. Is this intended? If yes, could we document (versionchanged::), add migration notes, and consider returning a list for one release or gating behind an opt-in flag? If not, let's keep returning a list and add a test to prevent regressions." + +- Case C: Exception type change (TypeError -> ValueError) on argument checking + - Diff (illustrative): + + ```diff + --- a/celery/some_module.py + +++ b/celery/some_module.py + @@ +- raise TypeError("bad arguments") ++ raise ValueError("bad arguments") + ``` + + - Why it matches: Changing the raised exception type breaks existing handlers and test expectations that catch TypeError. + - Example comment: "I see the raised exception changed TypeError -> ValueError; this can break existing error handlers/tests. Is this intended? If yes, could we document with versionchanged:: and suggest catching both for a transition period? If not, keep TypeError and add a test ensuring the type stays consistent." + +- Case D: Routing defaults change that silently reroutes tasks + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- default_queue=Option('celery'), ++ default_queue=Option('celery_v2'), + ``` + + - Why it matches: Changing default_queue (or introducing a non-None default in a call path) can reroute tasks for users who did not specify queue explicitly. + - Example comment: "I see default_queue changed 'celery' -> 'celery_v2'; this may silently reroute tasks for users not specifying queue. Is this intended? If yes, please add migration guidance and a versionchanged:: entry, and consider keeping a compat alias or opt-in flag through vX.Y. If not, revert and add a regression test verifying routing is unchanged when queue is omitted." + +### 2) Documentation versioning (strict but question-first) + +Triggers include: +- New/removed/renamed configuration setting or environment variable. +- Changed default of a documented setting. +- Behavior change in a documented feature (signals, CLI flags, return values, error behavior). +- Added/removed/renamed parameter in a documented API that users call directly. + +What to look for (detectors): +- Defaults changed in celery/app/defaults.py or docs without corresponding docs/whatsnew updates. +- Missing Sphinx directives (versionchanged::/versionadded::) in relevant docs when behavior/settings change. +- Public signatures changed (method/function params) without doc updates or deprecation notes. +- CLI help/defaults changed without docs alignment. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This appears to be a user-facing change (X -> Y). Is this intended? + - If yes: Should we add docs updates (versionchanged::/versionadded::) and a short migration note? + - If no: Should we revert or adjust the docs/code so they remain consistent until we can introduce a documented change?" + +Examples: +- Case A: Changed default of a documented setting (task_time_limit) + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- task_time_limit=Option(300, type='int'), ++ task_time_limit=Option(600, type='int'), + ``` + + - Why it matches: The default is documented and affects runtime behavior; changing it impacts users who relied on the previous default. + - Example comment: "I see task_time_limit default changed 300 -> 600; is this intended? If yes, should we add versionchanged:: in the docs and a brief migration note? If not, should we revert or defer behind a release note with guidance?" + +- Case B: New setting introduced (CELERY_FOO) + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ ++ foo=Option(False, type='bool'), # new + ``` + + - Why it matches: New documented configuration requires docs (usage, default, examples) and possibly a whatsnew entry. + - Example comment: "A new setting (celery.foo) is introduced. Should we add docs (reference + usage) and a versionadded:: note?" + +- Case C: Public API parameter renamed + - Diff (illustrative): + + ```diff + --- a/celery/app/task.py + +++ b/celery/app/task.py + @@ +- def apply_async(self, args=None, kwargs=None, routing_key=None, **options): ++ def apply_async(self, args=None, kwargs=None, route_key=None, **options): + ``` + + - Why it matches: Renamed parameter breaks user code and docs; requires docs changes and possibly a deprecation alias. + - Example comment: "apply_async param routing_key -> route_key is user-facing. Is this intended? If yes, can we add docs updates (versionchanged::) and consider an alias/deprecation path? If not, should we keep routing_key and add a regression test?" + +### 3) Scope and coherence + +Triggers include: +- Mixed concerns in a single PR (refactor/move/rename + behavior change). +- Large formatting sweep bundled with functional changes. +- Multiple unrelated features or modules changed together. + +What to look for (detectors): +- File renames/moves and non-trivial logic changes in the same PR. +- Many formatting-only hunks (whitespace/quotes/import order) mixed with logic edits. +- Multiple features or modules modified without a unifying rationale. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This PR appears to mix refactor/moves with functional changes. Would splitting the concerns improve focus and reviewability? + - If yes: Could we split into (A) refactor-only and (B) behavior change, or at least separate commits? + - If no: Could we provide a brief rationale and ensure commit messages clearly separate concerns?" + +Examples: +- Case A: Move + behavior change in the same change + - Diff (illustrative): + + ```diff + --- a/celery/old_module.py + +++ b/celery/new_module.py + @@ +- def handle(msg): +- return process(msg) ++ def handle(msg): ++ if msg.priority > 5: ++ return fast_path(msg) ++ return process(msg) + ``` + + - Why it matches: Relocation plus logic change complicates review and rollback. + - Example comment: "This includes both move and behavior change. Could we split the move (no-op) and the logic change into separate commits/PRs?" + +- Case B: Formatting sweep + logic change + - Diff (illustrative): + + ```diff + --- a/celery/module.py + +++ b/celery/module.py + @@ +- def f(x,y): return x+y ++ def f(x, y): ++ return x + y ++ ++ def g(x): ++ return x * 2 # new behavior + ``` + + - Why it matches: Formatting noise hides behavior changes. + - Example comment: "There is a formatting sweep plus a new function. Could we isolate logic changes so the diff is high-signal?" + +- Case C: Unrelated rename grouped with feature + - Diff (illustrative): + + ```diff + --- a/celery/feature.py + +++ b/celery/feature.py + @@ +- def add_user(u): ++ def create_user(u): # rename + ... + --- a/celery/other.py + +++ b/celery/other.py + @@ ++ def implement_new_queue(): ++ ... + ``` + + - Why it matches: Unrelated rename grouped with new feature reduces clarity. + - Example comment: "Can we separate the rename from the new feature so history and review stay focused?" + +### 4) Debug/development leftovers + +Triggers include: +- `print`, `pdb`/`breakpoint()`, commented-out blocks, temporary tracing/logging. +- Accidental debug helpers left in code (timers, counters). + +What to look for (detectors): +- `import pdb`, `pdb.set_trace()`, `breakpoint()`; new `print()` statements. +- `logger.debug(...)` with TODO/temporary text; excessive logging added. +- Large commented-out blocks or dead code left behind. +- Unused variables added for debugging only. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This looks like debug/temporary code. Can we remove it before merge? + - If yes: Please drop these lines (or guard behind a verbose flag). + - If no: Could you share why it’s needed and add a comment/guard to ensure it won’t leak in production?" + +Examples: +- Case A: Interactive debugger left in + - Diff (illustrative): + + ```diff + --- a/celery/worker.py + +++ b/celery/worker.py + @@ ++ import pdb ++ pdb.set_trace() + ``` + + - Why it matches: Debugger halts execution in production. + - Example comment: "Debugger calls found; can we remove them before merge?" + +- Case B: Temporary print/log statements + - Diff (illustrative): + + ```diff + --- a/celery/module.py + +++ b/celery/module.py + @@ +- result = compute(x) ++ result = compute(x) ++ print("DEBUG:", result) + ``` + + - Why it matches: Adds noisy output; not suitable for production. + - Example comment: "Temporary prints detected; could we remove or convert to a guarded debug log?" + +- Case C: Commented-out block + - Diff (illustrative): + + ```diff + --- a/celery/module.py + +++ b/celery/module.py + @@ ++ # old approach ++ # data = fetch_old() ++ # process_old(data) + ``` + + - Why it matches: Dead code should be removed for clarity and git history provides recovery. + - Example comment: "Large commented block detected; can we remove it and rely on git history if needed?" + +### 5) "Cover the other ends" for fixes + +Triggers include: +- Fix applied in one place while similar call sites/patterns remain elsewhere. +- Fix made in a wrapper/entry-point but not in the underlying helper used elsewhere. + +What to look for (detectors): +- Duplicate/similar functions that share the same bug but were not updated. +- Shared helpers where only one call path was fixed. +- Tests cover only the changed path but not sibling paths. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This fix updates one call site, but similar sites seem to exist (A/B). Were those reviewed? + - If yes: Could we update them in this PR or in a follow-up with references? + - If no: Would you like pointers on where similar patterns live (grep/symbol refs)?" + +Examples: +- Case A: Fix applied to one module; another equivalent module remains unchanged + - Diff (illustrative): + + ```diff + --- a/celery/foo.py + +++ b/celery/foo.py + @@ +- result = do_work(x) ++ result = do_work(x, safe=True) + ``` + + - Why it matches: bar.py uses the same pattern and likely needs the same safety flag. + - Example comment: "foo.py updated to pass safe=True; bar.py appears to call do_work similarly without the flag. Should we update bar.py too or open a follow-up?" + +- Case B: Wrapper fixed, helper not fixed + - Diff (illustrative): + + ```diff + --- a/celery/api.py + +++ b/celery/api.py + @@ +- def submit(task): +- return _publish(task) ++ def submit(task): ++ return _publish(task, retry=True) + ``` + + - Why it matches: Other entry points call _publish directly and still miss retry=True. + - Example comment: "submit() now passes retry=True, but direct _publish callers won't. Should we fix those call sites or update _publish's default?" + +### 6) Consistency and organization (not lint/style) + +Triggers include: +- New code diverges from nearby structural patterns (module layout, naming, docstrings, imports organization). +- Logger usage/structure differs from the rest of the module. +- Module/API structure inconsistent with sibling modules. + +What to look for (detectors): +- Different naming conventions (CamelCase vs snake_case) near similar code. +- Docstring style/sections differ from adjacent functions/classes. +- Logger names/patterns inconsistent with module-level practice. +- Module splitting/placement differs from sibling feature modules without rationale. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This code diverges from nearby patterns (X). Was that intentional? + - If yes: Could we add a brief rationale in the PR description and consider documenting the new pattern? + - If no: Should we align with the surrounding approach for consistency?" + +Examples: +- Case A: Naming deviates from local convention + - Diff (illustrative): + + ```diff + --- a/celery/jobs.py + +++ b/celery/jobs.py + @@ +- def CreateTask(payload): ++ def create_task(payload): + ... + ``` + + - Why it matches: Local code uses snake_case; CamelCase function name is inconsistent. + - Example comment: "Local convention is snake_case; should we rename to create_task for consistency?" + +- Case B: Logger name/prefix inconsistent + - Diff (illustrative): + + ```diff + --- a/celery/worker.py + +++ b/celery/worker.py + @@ +- log = logging.getLogger("celery.worker") ++ log = logging.getLogger("celery.custom") + ``` + + - Why it matches: Module logger naming differs from the standard. + - Example comment: "Module loggers typically use 'celery.worker'; should we align the logger name here?" + +- Case C: Module layout divergence + - Diff (illustrative): + + ```diff + --- a/celery/feature/__init__.py + +++ b/celery/feature/__init__.py + @@ ++ from .impl import Feature # new public import + ``` + + - Why it matches: New public import/path differs from sibling modules. + - Example comment: "Exposing Feature at package root differs from siblings; was that intentional, or should we keep imports local?" + +### 7) Tests and local workflow (optional nudges) + +Triggers include: +- Behavior change, bug fix, or CI failures without corresponding tests/updates. + +What to look for (detectors): +- Code changes that alter behavior with no new/updated tests. +- API/signature changes with tests still asserting old behavior. +- Failing CI areas that need local reproduction guidance. + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "Since behavior changes here, could we add/update a focused unit test that fails before and passes after? + - If yes: A small unit test should suffice; consider narrowing with -k. + - If no: Could you share rationale (e.g., covered by integration/smoke), and note how to reproduce locally?" + +Suggested commands: +- `tox -e lint` +- `tox -e 3.13-unit` +- `tox -e 3.13-integration-rabbitmq_redis` (ensure local RabbitMQ and Redis containers are running) +- `tox -e 3.13-smoke -- -n auto` +- Narrow scope: `tox -e 3.13-unit -- -k ` + +Examples: +- Case A: Bug fix without a regression test + - Diff (illustrative): + + ```diff + --- a/celery/utils.py + +++ b/celery/utils.py + @@ +- return retry(task) ++ return retry(task, backoff=True) + ``` + + - Why it matches: Behavior changed; add a unit test asserting backoff path. + - Example comment: "New backoff behavior added; can we add a unit test that fails before and passes after this change?" + +- Case B: API/signature changed; tests not updated + - Diff (illustrative): + + ```diff + --- a/celery/app/task.py + +++ b/celery/app/task.py + @@ +- def apply_async(self, args=None, kwargs=None, routing_key=None, **options): ++ def apply_async(self, args=None, kwargs=None, route_key=None, **options): + ``` + + - Why it matches: Tests/callers may still pass routing_key. + - Example comment: "apply_async param rename detected; can we update tests and add a note in the PR description on migration?" + +- Case C: Provide local reproduction guidance for CI failures + - Example comment: "CI failures indicate tests in module X. To iterate locally: + - `tox -e 3.13-unit -- -k ` + - If integration-related: `tox -e 3.13-integration-rabbitmq_redis` (ensure services run) + - For smoke: `tox -e 3.13-smoke -- -n auto`" + +### 8) Ecosystem awareness (non-prescriptive) + +Triggers include: +- Changes to internal components or cross-project boundaries (kombu/amqp, backends, transports). +- Acknowledge/visibility-timeout semantics modified; stamped headers or message schema altered. +- Serialization/content-type defaults changed; transport-specific behavior altered. + +What to look for (detectors): +- Edits to amqp producer/consumer internals; ack/requeue/visibility logic. +- Changes to stamped_headers handling or task message headers/body schema. +- Defaults that affect interop (content_type/serializer, queue types, exchange kinds). + +Comment pattern (question-first; handle both "if yes" and "if no"): +- "This touches internal messaging/interop semantics and may affect the ecosystem. Could you share the rationale and cross-component considerations? + - If yes: Could we add focused tests (publish/consume round-trip) and a brief docs/whatsnew note? + - If no: Should we revert or gate behind a feature flag until we coordinate across components?" + +Examples: +- Case A: Stamped headers behavior changed + - Diff (illustrative): + + ```diff + --- a/celery/app/base.py + +++ b/celery/app/base.py + @@ +- stamped_headers = options.pop('stamped_headers', []) ++ stamped_headers = options.pop('stamped_headers', ['trace_id']) + ``` + + - Why it matches: Default stamped headers alter on-the-wire metadata; other tools may not expect it. + - Example comment: "Default stamped_headers now include 'trace_id'; is this intended? If yes, can we add tests/docs and note interop impact? If not, should we keep [] and document opt-in?" + +- Case B: Ack/visibility semantics tweaked + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- acks_on_failure_or_timeout=Option(True, type='bool'), ++ acks_on_failure_or_timeout=Option(False, type='bool'), + ``` + + - Why it matches: Changes worker/broker interaction; can affect redelivery and failure semantics. + - Example comment: "acks_on_failure_or_timeout True -> False affects redelivery; is this intended? If yes, could we add tests and a docs note? If not, revert and add a regression test?" + +- Case C: Serialization/content-type default changed + - Diff (illustrative): + + ```diff + --- a/celery/app/defaults.py + +++ b/celery/app/defaults.py + @@ +- serializer=Option('json'), ++ serializer=Option('yaml'), + ``` + + - Why it matches: Affects compatibility with consumers/producers; security considerations for yaml. + - Example comment: "Serializer default json -> yaml changes interop/security profile. Is this intended? If yes, please document risks and add tests; if not, keep json." + +## What to avoid commenting on + +- Style/formatting/line length (lint/CI already enforce repo standards). +- Dependency management specifics. +- Over-specific internal patterns unless explicitly documented policy. +- Repeating the same point after it has been addressed. + +## Noise control (without hard caps) + +- Group related questions into one concise comment per theme when possible. +- Ask once per issue; don't repeat after the contributor responds/updates. +- Skip commentary on pure formatting, comment-only diffs, tests-only edits, or private helper refactors with no user-facing impact. + +## PR title and description (nice-to-have) + +- If title/description don't reflect the change, suggest a concise rewrite that helps future "What's New" compilation - helpful, never blocking. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000..47a31bc9d65 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/opencollective.yml b/.github/opencollective.yml new file mode 100644 index 00000000000..be703c8b871 --- /dev/null +++ b/.github/opencollective.yml @@ -0,0 +1,18 @@ +collective: celery +tiers: + - tiers: '*' + labels: ['Backer ❤️'] + message: 'Hey . Thank you for supporting the project!:heart:' + - tiers: ['Basic Sponsor', 'Sponsor', 'Silver Sponsor', 'Gold Sponsor'] + labels: ['Sponsor ❤️'] + message: | + Thank you for sponsoring the project!:heart::heart::heart: + Resolving this issue is one of our top priorities. + One of @celery/core-developers will triage it shortly. +invitation: | + Hey :wave:, + Thank you for opening an issue. We will get back to you as soon as we can. + Also, check out our [Open Collective]() and consider backing us - every little helps! + + We also offer priority support for our sponsors. + If you require immediate assistance please consider sponsoring us. diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000000..3f014e19107 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,71 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + workflow_dispatch: + + + +jobs: + analyze: + name: Analyze + runs-on: blacksmith-4vcpu-ubuntu-2204 + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v4 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000000..43fde7bb735 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,76 @@ +name: Docker + +on: + pull_request: + branches: [ 'main'] + paths: + - '**.py' + - '**.txt' + - '**.toml' + - '/docker/**' + - '.github/workflows/docker.yml' + - 'docker/Dockerfile' + - 't/smoke/workers/docker/**' + push: + branches: [ 'main'] + paths: + - '**.py' + - '**.txt' + - '**.toml' + - '/docker/**' + - '.github/workflows/docker.yml' + - 'docker/Dockerfile' + - 't/smoke/workers/docker/**' + workflow_dispatch: + + +jobs: + docker-build: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v6 + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 + - name: Build Docker container + run: make docker-build + + smoke-tests_dev: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 + - name: "Build smoke tests container: dev" + run: docker build -f t/smoke/workers/docker/dev . + + smoke-tests_latest: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 + - name: "Build smoke tests container: latest" + run: docker build -f t/smoke/workers/docker/pypi . + + smoke-tests_pypi: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 + - name: "Build smoke tests container: pypi" + run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="5" . + + smoke-tests_legacy: + runs-on: blacksmith-4vcpu-ubuntu-2204 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 + - name: "Build smoke tests container: legacy" + run: docker build -f t/smoke/workers/docker/pypi --build-arg CELERY_VERSION="4" . diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 00000000000..e3f411bc7ea --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,72 @@ +name: Integration Tests + +on: + workflow_call: + inputs: + module_name: + description: 'Name of the test module to run (e.g., test_backend.py)' + required: true + type: string + python_versions: + description: 'JSON array of Python versions to test' + required: false + type: string + default: '["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"]' + tox_environments: + description: 'JSON array of tox environments to test' + required: false + type: string + default: '["redis", "rabbitmq", "rabbitmq_redis"]' + +jobs: + testing-with: + timeout-minutes: 240 + runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(inputs.python_versions) }} + toxenv: ${{ fromJson(inputs.tox_environments) }} + + services: + redis: + image: redis + ports: + - 6379:6379 + env: + REDIS_HOST: localhost + REDIS_PORT: 6379 + rabbitmq: + image: rabbitmq:management + ports: + - 5672:5672 + - 15672:15672 + env: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + + steps: + - name: Install apt packages + run: | + sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev + + - uses: actions/checkout@v6 + - name: Set up Python ${{ matrix.python-version }} + uses: useblacksmith/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: 'pip' + cache-dependency-path: '**/setup.py' + - name: Install tox + run: python -m pip install --upgrade pip 'tox' tox-gh-actions + - name: > + Run tox for + "${{ matrix.python-version }}-integration-${{ matrix.toxenv }}-${{ inputs.module_name }}" + uses: nick-fields/retry@v4 + with: + timeout_minutes: 15 + max_attempts: 5 + retry_wait_seconds: 0 + command: | + tox --verbose --verbose -e "${{ matrix.python-version }}-integration-${{ matrix.toxenv }}" -- -k ${{ inputs.module_name }} -vv diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 00000000000..249592d4fb3 --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,17 @@ +name: Linter + +on: [pull_request, workflow_dispatch] + +permissions: + contents: read + +jobs: + linter: + runs-on: blacksmith-4vcpu-ubuntu-2204 + steps: + + - name: Checkout branch + uses: actions/checkout@v6 + + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 00000000000..997585a0b9a --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,137 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Celery + +on: + push: + branches: [ 'main', 'v5.6.x' ] + paths: + - '**.py' + - '**.txt' + - '.github/workflows/python-package.yml' + - '**.toml' + - "tox.ini" + pull_request: + branches: [ 'main', 'v5.6.x' ] + paths: + - '**.py' + - '**.txt' + - '**.toml' + - '.github/workflows/python-package.yml' + - "tox.ini" + workflow_dispatch: + + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + Unit: + + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.11'] + os: ["blacksmith-4vcpu-ubuntu-2404", "windows-latest"] + exclude: + - python-version: '3.9' + os: "windows-latest" + - python-version: '3.10' + os: "windows-latest" + - python-version: '3.11' + os: "windows-latest" + - python-version: '3.12' + os: "windows-latest" + - python-version: '3.13' + os: "windows-latest" + - python-version: '3.14t' + os: "windows-latest" + - python-version: 'pypy3.11' + os: "windows-latest" + + steps: + - name: Install apt packages + if: startsWith(matrix.os, 'blacksmith-4vcpu-ubuntu') + run: | + sudo apt-get update && sudo apt-get install -f libcurl4-openssl-dev libssl-dev libgnutls28-dev httping expect libmemcached-dev + - uses: actions/checkout@v6 + - name: Set up Python ${{ matrix.python-version }} + uses: useblacksmith/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: 'pip' + cache-dependency-path: '**/setup.py' + + - name: Install tox + run: python -m pip install --upgrade pip 'tox' tox-gh-actions + - name: > + Run tox for + "${{ matrix.python-version }}-unit" + timeout-minutes: 30 + run: | + tox --verbose --verbose + + - uses: codecov/codecov-action@v5 + with: + flags: unittests # optional + fail_ci_if_error: true # optional (default = false) + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true # optional (default = false) + + - name: Upload test results to Codecov + if: ${{ !cancelled() }} + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + Integration-tests: + needs: [Unit] + if: needs.Unit.result == 'success' + strategy: + matrix: + module: [ + 'test_backend.py', + 'test_canvas.py', + 'test_dedup_chain_dispatch.py', + 'test_inspect.py', + 'test_loader.py', + 'test_mem_leak_in_exception_handling.py', + 'test_quorum_queue_qos_cluster_simulation.py', + 'test_rabbitmq_chord_unlock_routing.py', + 'test_rabbitmq_default_queue_type_fallback.py', + 'test_security.py', + 'test_serialization.py', + 'test_tasks.py', + 'test_worker.py' + ] + uses: ./.github/workflows/integration-tests.yml + with: + module_name: ${{ matrix.module }} + + Smoke-tests: + needs: [Unit] + if: needs.Unit.result == 'success' + strategy: + matrix: + module: [ + 'test_broker_failover.py', + 'test_worker_failover.py', + 'test_native_delayed_delivery.py', + 'test_quorum_queues.py', + 'test_hybrid_cluster.py', + 'test_revoke.py', + 'test_visitor.py', + 'test_canvas.py', + 'test_consumer.py', + 'test_control.py', + 'test_signals.py', + 'test_tasks.py', + 'test_thread_safe.py', + 'test_worker.py' + ] + uses: ./.github/workflows/smoke-tests.yml + with: + module_name: ${{ matrix.module }} diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml new file mode 100644 index 00000000000..2f1a2309eeb --- /dev/null +++ b/.github/workflows/semgrep.yml @@ -0,0 +1,29 @@ +on: + pull_request: {} + push: + branches: + - main + - master + paths: + - .github/workflows/semgrep.yml + schedule: + # random HH:MM to avoid a load spike on GitHub Actions at 00:00 + - cron: 44 6 * * * + workflow_dispatch: + +name: Semgrep + +permissions: + contents: read + +jobs: + semgrep: + name: Scan + runs-on: blacksmith-4vcpu-ubuntu-2204 + env: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + container: + image: returntocorp/semgrep + steps: + - uses: actions/checkout@v6 + - run: semgrep ci diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml new file mode 100644 index 00000000000..d40307c578d --- /dev/null +++ b/.github/workflows/smoke-tests.yml @@ -0,0 +1,60 @@ +name: Smoke Tests + +on: + workflow_call: + inputs: + module_name: + description: 'Name of the test module to run (e.g., test_broker_failover.py)' + required: true + type: string + python_versions: + description: 'JSON array of Python versions to test' + required: false + type: string + default: '["3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"]' + +permissions: + contents: read + +jobs: + testing-with: + runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJson(inputs.python_versions) }} + + steps: + - name: Fetch Docker Images + run: | + docker pull redis:latest + docker pull rabbitmq:latest + + - name: Install apt packages + run: | + sudo apt update + sudo apt-get install -y procps # Install procps to enable sysctl + sudo sysctl -w vm.overcommit_memory=1 + + - uses: actions/checkout@v6 + - name: Setup Docker Builder + uses: useblacksmith/setup-docker-builder@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: useblacksmith/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + cache: 'pip' + cache-dependency-path: '**/setup.py' + + - name: Install tox + run: python -m pip install --upgrade pip tox tox-gh-actions + + - name: Run tox for "${{ matrix.python-version }}-smoke-${{ inputs.module_name }}" + uses: nick-fields/retry@v4 + with: + timeout_minutes: 20 + max_attempts: 5 + retry_wait_seconds: 60 + command: | + tox --verbose --verbose -e "${{ matrix.python-version }}-smoke" -- -k ${{ inputs.module_name }} -n auto diff --git a/.gitignore b/.gitignore index 677430265ab..f70de56dce0 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ coverage.xml test.db pip-wheel-metadata/ .python-version +.tool-versions .vscode/ integration-tests-config.json [0-9]* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..2b1a24a66fd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +repos: + - repo: https://github.com/asottile/pyupgrade + rev: v3.21.2 + hooks: + - id: pyupgrade + args: ["--py38-plus"] + + - repo: https://github.com/PyCQA/flake8 + rev: 7.3.0 + hooks: + - id: flake8 + + - repo: https://github.com/asottile/yesqa + rev: v1.5.0 + hooks: + - id: yesqa + exclude: ^celery/app/task\.py$|^celery/backends/cache\.py$ + + - repo: https://github.com/codespell-project/codespell + rev: v2.4.2 + hooks: + - id: codespell # See pyproject.toml for args + args: [--toml, pyproject.toml, --write-changes] + additional_dependencies: + - tomli + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + exclude: helm-chart/templates/ + - id: mixed-line-ending + + - repo: https://github.com/pycqa/isort + rev: 8.0.1 + hooks: + - id: isort + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.19.1 + hooks: + - id: mypy + pass_filenames: false diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000000..b296878a8d8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,26 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-20.04 + tools: + python: "3.9" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +# formats: +# - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - method: pip + path: . + - requirements: requirements/docs.txt diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1f7e665a6ef..34b74832081 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -74,7 +74,7 @@ We should always be open to collaboration. Your work should be done transparently and patches from Celery should be given back to the community when they're made, not just when the distribution releases. If you wish to work on new code for existing upstream projects, at least keep those -projects informed of your ideas and progress. It many not be possible to +projects informed of your ideas and progress. It may not be possible to get consensus from upstream, or even from your colleagues about the correct implementation for an idea, so don't feel obliged to have that agreement before you begin, but at least keep the outside world informed of your work, @@ -1509,4 +1509,3 @@ following: .. _`bundles`: https://docs.celeryq.dev/en/latest/getting-started/introduction.html#bundles .. _`report an issue`: https://docs.celeryq.dev/en/latest/contributing.html#reporting-bugs - diff --git a/Changelog.rst b/Changelog.rst index 1eba0c056b2..39d6192d845 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -5,8 +5,370 @@ ================ This document contains change notes for bugfix & new features -in the main branch & 5.5.x series, please see :ref:`whatsnew-5.5` for -an overview of what's new in Celery 5.5. +in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for +an overview of what's new in Celery 5.6. + +.. _version-5.6.3: + +5.6.3 +===== + +:release-date: 2026-03-26 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix Django worker recursion bug + defensive checks for pool_cls.__module__ (#10048) +- Docs: Update user_preload_options example to use click. (#10056) +- Fix invalid configuration key "bootstrap_servers" in Kafka demo (#10060) +- Fix broken images on PyPI page (#10066) +- Remove broken reference. (#10071) +- Removed --dist=loadscope from smoke tests (#10073) +- Docs: Clarify task_retry signal args may be None (#10076) +- Update example for Django (#10081) +- Make tests compatible with pymongo >= 4.16 (#10074) +- fix: source install of cassandra-driver (#10105) +- fix: register task cross-reference role in Sphinx extension (#10100) +- fix: avoid cycle detection in native delayed delivery (#10095) +- fix(asynpool): avoid AttributeError when proc lacks _sentinel_poll (#10086) +- fix dusk_astronomical horizon sign (+18 -> -18) (#10121) +- Fix/10106 onupdate col use lambda func (#10108) +- Fix warm shutdown RuntimeError with eventlet>=0.37.0 (#10083) (#10123) +- Fix 10109 db backend connection health (#10124) +- Database Backend filter unsupport sql engine arguments with nullpool #7355 (#10134) +- fix(beat): correct argument order in Service.__reduce__ (#10137) +- ci: declare explicit read-only token permissions in workflow jobs (#10139) +- chore: 'boto3to' to 'boto3 to' (#10133) +- Database Backend: Add missing index on date_done (Fixes #10097) (#10098) +- docs: fix typo in CONTRIBUTING.rst (#10141) +- Refer to Flower / Prometheus for monitoring (#10140) +- docs: remove duplicated words in broker and routing docs (#10146) +- docs: fix stale version reference and grammar in README (#10145) +- docs: fix wording in Celery 5.3 worker pool notes (#10149) +- docs: fix duplicated wording in 3.1 changelog entry (#10152) +- docs: fix changelog typo in context manager wording (#10144) +- Fix/10096 worker fails to reconnect after redis failover (#10151) +- Improve on_after_finalize signal documentation (#10155) +- Add non-commutative example to clarify partial arg ordering in canvas docs (#10157) +- Remove redundant test_isa_mapping test (fixes #10077) (#10103) +- Upgrade pytest-celery to >=1.3.0 and adopt PYTEST_CELERY_PKG build arg (#10162) +- Remove deprecated args from redis get_connection call (#10036) +- Fix #6912 rpc backend reconnection error (#10179) +- Fix NameError with TYPE_CHECKING annotations on Python 3.14+ (PEP 649) (#10165) +- docs: Add elaboration on prefetch multiplier settings (worker_prefetch_multiplier) and worker_eta_task_limit (#10181) +- Fix O(K²) message bloat in a chain of chords (#10171) +- Fix mock connection interfaces to prevent `TypeError` during exception handling (#10178) +- fix(trace): dispatch chain/callbacks on dedup fast-path for redelivered tasks (#10159) +- Extract `reconnect_on_error` to `BaseResultConsumer` (#10189) +- pep 649 (#10187) +- Fix#9722 friendly status errors for CLI (#10190) +- docs: clarify after_return behavior for retried tasks (#10192) +- Add compression header to message protocol docs (#10156) +- docs: fix duplicated word in bootsteps comment (#10153) +- Remove outdated autoreloader section from extending docs (#10154) +- Fix: prioritize request ignore_result over task definition (#10184) +- fix: clear the timer while catch the exception (#10218) +- Prepare for release: v5.6.3 (#10221) + +.. _version-5.6.2: + +5.6.2 +===== + +:release-date: 2026-01-04 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix recursive WorkController instantiation in DjangoWorkerFixup + AttributeError when pool_cls is a string (#10045) +- Bugfix: Revoked tasks now immediately update backend status to REVOKED (#9869) +- Prepare for release: v5.6.2 (#10049) + +.. _version-5.6.1: + +5.6.1 +===== + +:release-date: 2025-12-29 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix Redis Sentinel ACL authentication support (#10013) +- Fix: Broker heartbeats not sent during graceful shutdown (#9986) +- docs #5410 -- Document confirm_publish broker transport option (#10016) +- close DB pools only in prefork mode (#10020) +- Fix: Avoid unnecessary Django database connection creation during cleanup (#10015) +- reliable prefork detection (#10023) +- better coverage (#10029) +- Docs: clarify `result_extended` vs periodic task metadata and show `headers["periodic_task_name"]` example (#10030) +- Stop importing pytest_subtests (#10032) +- Only use exceptiongroup backport for Python < 3.11 (#10033) +- Prepare for release: v5.6.1 (#10037) + +.. _version-5.6.0: + +5.6.0 +===== + +:release-date: 2025-11-30 +:release-by: Tomer Nosrati + +Celery v5.6.0 is now available. + +Key Highlights +~~~~~~~~~~~~~~ + +See :ref:`whatsnew-5.6` for a complete overview or read the main highlights below. + +Python 3.9 Minimum Version +-------------------------- + +Celery 5.6.0 drops support for Python 3.8 (EOL). The minimum required Python +version is now 3.9. Users still on Python 3.8 must upgrade their Python version +before upgrading to Celery 5.6.0. + +Additionally, this release includes initial support for Python 3.14. + +SQS: Reverted to ``pycurl`` from ``urllib3`` +-------------------------------------------- + +The switch from ``pycurl`` to ``urllib3`` for the SQS transport (introduced in +Celery 5.5.0 via Kombu) has been reverted due to critical issues affecting SQS +users: + +- Processing throughput dropped from ~100 tasks/sec to ~3/sec in some environments +- ``UnknownOperationException`` errors causing container crash loops +- Silent message processing failures with no error logs + +Users of the SQS transport must ensure ``pycurl`` is installed. If you removed +``pycurl`` after upgrading to Celery 5.5.0, you will need to reinstall it. + +Contributed by `@auvipy `_ in +`#9620 `_. + +Security Fix: Broker Credential Leak Prevention +------------------------------------------------ + +Fixed a security issue where broker URLs containing passwords were being logged +in plaintext by the delayed delivery mechanism. Broker credentials are now +properly sanitized in all log output. + +Contributed by `@giancarloromeo `_ in +`#9997 `_. + +Memory Leak Fixes +----------------- + +Two significant memory leaks have been fixed in this release: + +**Exception Handling Memory Leak**: Fixed a critical memory leak in task exception +handling that was particularly severe on Python 3.11+ due to enhanced traceback +data. The fix properly breaks reference cycles in tracebacks to allow garbage +collection. + +Contributed by `@jaiganeshs21 `_ in +`#9799 `_. + +**Pending Result Memory Leak**: Fixed a memory leak where ``AsyncResult`` +subscriptions were not being cleaned up when results were forgotten. + +Contributed by `@tsoos99dev `_ in +`#9806 `_. + +ETA Task Memory Limit +--------------------- + +New configuration option :setting:`worker_eta_task_limit` to prevent out-of-memory +crashes when workers fetch large numbers of ETA or countdown tasks. Previously, +workers could exhaust available memory when the broker contained many scheduled tasks. + +Example usage: + +.. code-block:: python + + app.conf.worker_eta_task_limit = 1000 + +Contributed by `@sashu2310 `_ in +`#9853 `_. + +Queue Type Selection for Auto-created Queues +-------------------------------------------- + +New configuration options allow specifying the queue type and exchange type when +Celery auto-creates missing queues. This is particularly useful for RabbitMQ users +who want to use quorum queues with auto-created queues. + +Configuration options: + +- :setting:`task_create_missing_queue_type`: Sets the queue type for auto-created + queues (e.g., ``quorum``, ``classic``) +- :setting:`task_create_missing_queue_exchange_type`: Sets the exchange type for + auto-created queues + +Example usage: + +.. code-block:: python + + app.conf.task_create_missing_queue_type = 'quorum' + +Contributed by `@ghirailghiro `_ in +`#9815 `_. + +What's Changed +~~~~~~~~~~~~~~ + +- Prepare for release: v5.6.0 (#10010) + +.. _version-5.6.0rc2: + +5.6.0rc2 +======== + +:release-date: 2025-11-22 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Remove Python 4.0 version condition for pytest dependencies (#9993) +- Sanitize broker URL in delayed delivery logs (avoid leaking credentials) (#9997) +- Don't fail task on timeout during cold shutdown (#9678) +- Add Py39-314t to CI (#9999) +- asynpool: Don't return from inside a finally block (#10000) +- Prepare for (pre) release: v5.6.0rc2 (#10005) + +.. _version-5.6.0rc1: + +5.6.0rc1 +======== + +:release-date: 2025-11-02 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Add support for Django Connection pool (#9953) +- Pin tblib to ==3.1.0 (#9967) +- fix(worker): continue to attempt to bind other queues after a native delayed delivery binding failure has occurred (#9959) +- Handle UnpicklingError in persistent scheduler initialization (#9952) +- Bug Fix: Nested Chords Fail When Using django-celery-results with a Redis Backend (#9950) +- Add support pymongo 4.12 (#9665) +- Make tests compatible with pymongo >= 4.14 (#9968) +- tblib updated from 3.1.0 to 3.2.0 (#9970) +- Fix remaining function typing and docstring (#9971) +- Fix regex pattern in version parsing and remove duplicate entry in __all__ (#9978) +- Bump Kombu to v5.6.0 and removed <5.7 limit on kombu (#9981) +- Prepare for (pre) release: v5.6.0rc1 (#9982) + +.. _version-5.6.0b2: + +5.6.0b2 +======= + +:release-date: 2025-10-20 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- GitHub Actions: Test on Python 3.14 release candidate 2 (#9891) +- Update pypy to python 3.11 (#9896) +- Feature: Add support credential_provider to Redis Backend (#9879) +- Celery.timezone: try tzlocal.get_localzone() before using LocalTimezone (#9862) +- Run integration tests on Python 3.14 (#9903) +- Fix arithmetic overflow for MSSQL result backend (#9904) +- Add documentation for task_id param for apply_async function (#9906) +- Support redis client name (#9900) +- Bump Kombu to v5.6.0rc1 (#9918) +- Fix broker connection retry attempt counter in the error log (#9911) +- fix: restrict disable-prefetch feature to Redis brokers only (#9919) +- fix(): preserve group order in replaced signature (#9910) +- Remove Python 3.8 from CI workflow (#9930) +- Update default Python versions in integration tests (#9931) +- Update tox.ini to remove Python 3.8 (#9932) +- Remove Python 3.8 from Dockerfile (#9933) +- Update Python version requirement to 3.9 (#9935) +- Update pypy version from 3.10 to 3.11 in Dockerfile (#9934) +- Flake8 fixes (#9955) +- Remove test-pypy3.txt from Dockerfile dependencies (#9939) +- Remove backports.zoneinfo for Python 3.9 compatibility (#9956) +- Update pytest-cov version for Python compatibility (#9957) +- Update pytest-rerunfailures and pre-commit versions (#9958) +- Prepare for (pre) release: v5.6.0b2 (#9938) + +.. _version-5.6.0b1: + +5.6.0b1 +======= + +:release-date: 2025-09-15 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- docs: mention of json serializer recursive reference message size blowup (#5000) (#9743) +- docs: typo in canvas.rst (#9744) +- Makes _on_retry return a float as required to be used as errback on retry_over_time (#9741) +- Update canvas.rst doc calculation order for callback (#9758) +- Updated Blacksmith logo (#9763) +- Made the Sponsors logos link to their website (#9764) +- add missing cloudamqp logo (#9767) +- Improve sponsor visibility (#9768) +- fix: (#9773) task_id must not be empty with chain as body of a chord (#9774) +- Update setup.py to fix deprecation warning (#9771) +- Adds integration test for chord_unlock bug when routed to quorum/topic queue (#9766) +- Add xfail test for default queue/exchange fallback ignoring task_default_* settings (#9765) +- Add xfail test for RabbitMQ quorum queue global QoS race condition (#9770) +- fix: (#8786) time out when chord header fails with group body (#9788) +- Fix #9738 : Add root_id and parent_id to .apply() (#9784) +- Replace DelayedDelivery connection creation to use context manager (#9793) +- Fix #9794: Pydantic integration fails with __future__.annotations. (#9795) +- add go and rust implementation in docs (#9800) +- Fix memory leak in exception handling (Issue #8882) (#9799) +- Fix handlers docs (Issue #9787) (#9804) +- Remove importlib_metadata leftovers (#9791) +- Update timeout minutes for smoke tests CI (#9807) +- Revert "Remove dependency on `pycurl`" (#9620) +- Add Blacksmith Docker layer caching to all Docker builds (#9840) +- Bump Kombu to v5.6.0b1 (#9839) +- Disable pytest-xdist for smoke tests and increase retries (CI ONLY) (#9842) +- Fix Python 3.13 compatibility in events dumper (#9826) +- Dockerfile Build Optimizations (#9733) +- Migrated from useblacksmith/build-push-action@v1 to useblacksmith/setup-docker-builder@v1 in the CI (#9846) +- Remove incorrect example (#9854) +- Revert "Use Django DB max age connection setting" (#9824) +- Fix pending_result memory leak (#9806) +- Update python-package.yml (#9856) +- Bump Kombu to v5.6.0b2 (#9858) +- Refactor integration and smoke tests CI (#9855) +- Fix `AsyncResult.forget()` with couchdb backend method raises `TypeError: a bytes-like object is required, not 'str'` (#9865) +- Improve Docs for SQS Authentication (#9868) +- Added `.github/copilot-instructions.md` for GitHub Copilot (#9874) +- misc: credit removal (#9877) +- Choose queue type and exchange type when creating missing queues (fix #9671) (#9815) +- fix: prevent celery from hanging due to spawned greenlet errors in greenlet drainers (#9371) +- Feature/disable prefetch fixes (#9863) +- Add worker_eta_task_limit configuration to manage ETA task memory usage (#9853) +- Update runner version in Docker workflow (#9884) +- Prepare for (pre) release: v5.6.0b1 (#9890) .. _version-5.5.3: diff --git a/Makefile b/Makefile index d28ac57dcf7..6e2eb420942 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ authorcheck: .PHONY: docker-build docker-build: - @docker compose -f docker/docker-compose.yml build + @DOCKER_BUILDKIT=1 docker compose -f docker/docker-compose.yml build .PHONY: docker-lint docker-lint: diff --git a/README.rst b/README.rst index 8415508638d..a3586020062 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |build-status| |coverage| |license| |wheel| |semgrep| |pyversion| |pyimp| |ocbackerbadge| |ocsponsorbadge| -:Version: 5.5.3 (immunity) +:Version: 5.6.3 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ @@ -37,7 +37,7 @@ Sponsors Blacksmith ---------- -.. image:: ./docs/images/blacksmith-logo-white-on-black.svg +.. image:: https://github.com/celery/celery/blob/main/docs/images/blacksmith-logo-white-on-black.svg :alt: Blacksmith logo :width: 240px :target: https://blacksmith.sh/ @@ -47,12 +47,12 @@ Blacksmith CloudAMQP --------- -.. image:: ./docs/images/cloudamqp-logo-lightbg.svg +.. image:: https://github.com/celery/celery/blob/main/docs/images/cloudamqp-logo-lightbg.svg :alt: CloudAMQP logo :width: 240px :target: https://www.cloudamqp.com/ -`CloudAMQP `_ is a industry leading RabbitMQ as a service provider. +`CloudAMQP `_ is an industry leading RabbitMQ as a service provider. If you need highly available message queues, a perfect choice would be to use CloudAMQP. With 24,000+ running instances, CloudAMQP is the leading hosting provider of RabbitMQ, with customers all over the world. @@ -154,7 +154,7 @@ Get Started =========== If this is the first time you're trying to use Celery, or you're -new to Celery v5.5.x coming from previous versions then you should read our +new to Celery v5.6.x coming from previous versions then you should read our getting started tutorials: - `First steps with Celery`_ @@ -536,11 +536,6 @@ documentation. .. _`Contributing to Celery`: https://docs.celeryq.dev/en/stable/contributing.html -|oc-contributors| - -.. |oc-contributors| image:: https://opencollective.com/celery/contributors.svg?width=890&button=false - :target: https://github.com/celery/celery/graphs/contributors - Backers ------- diff --git a/celery/__init__.py b/celery/__init__.py index 4739b81fb8c..3467851685f 100644 --- a/celery/__init__.py +++ b/celery/__init__.py @@ -15,9 +15,16 @@ # Lazy loading from . import local -SERIES = 'immunity' +# Save original os.write before eventlet/gevent can monkey-patch it. +# This is needed for signal handlers (e.g., SIGINT) which may run inside +# the eventlet hub's event loop. Using the patched os.write from within +# the hub causes: RuntimeError('do not call blocking functions from the mainloop') +# See: https://github.com/celery/celery/issues/10083 +_original_os_write = os.write -__version__ = '5.5.3+gumloop_0.1.5' +SERIES = 'recovery' + +__version__ = '5.6.3+gumloop_0.2.0' __author__ = 'Rahul Behal' __contact__ = 'rahul@gumloop.com' __homepage__ = 'https://github.com/gumloop/gumloop-celery' @@ -42,7 +49,7 @@ # bumpversion can only search for {current_version} # so we have to parse the version here. _temp = re.match( - r'(\d+)\.(\d+).(\d+)(.+)?', __version__).groups() + r'(\d+)\.(\d+)\.(\d+)(.+)?', __version__).groups() VERSION = version_info = version_info_t( int(_temp[0]), int(_temp[1]), int(_temp[2]), _temp[3] or '', '') del _temp @@ -169,4 +176,5 @@ def maybe_patch_concurrency(argv=None, short_opts=None, version_info=version_info, maybe_patch_concurrency=maybe_patch_concurrency, _find_option_with_arg=_find_option_with_arg, + _original_os_write=_original_os_write, ) diff --git a/celery/app/amqp.py b/celery/app/amqp.py index 8dcec363053..6caedc5c5c6 100644 --- a/celery/app/amqp.py +++ b/celery/app/amqp.py @@ -46,6 +46,13 @@ class Queues(dict): create_missing (bool): By default any unknown queues will be added automatically, but if this flag is disabled the occurrence of unknown queues in `wanted` will raise :exc:`KeyError`. + create_missing_queue_type (str): Type of queue to create for missing queues. + Must be either 'classic' (default) or 'quorum'. If set to 'quorum', + the broker will declare new queues using the quorum type. + create_missing_queue_exchange_type (str): Type of exchange to use + when creating missing queues. If not set, the default exchange type + will be used. If set, the exchange type will be set to this value + when creating missing queues. max_priority (int): Default x-max-priority for queues with none set. """ @@ -53,14 +60,19 @@ class Queues(dict): #: The rest of the queues are then used for routing only. _consume_from = None - def __init__(self, queues=None, default_exchange=None, - create_missing=True, autoexchange=None, - max_priority=None, default_routing_key=None): + def __init__( + self, queues=None, default_exchange=None, + create_missing=True, create_missing_queue_type=None, + create_missing_queue_exchange_type=None, autoexchange=None, + max_priority=None, default_routing_key=None, + ): super().__init__() self.aliases = WeakValueDictionary() self.default_exchange = default_exchange self.default_routing_key = default_routing_key self.create_missing = create_missing + self.create_missing_queue_type = create_missing_queue_type + self.create_missing_queue_exchange_type = create_missing_queue_exchange_type self.autoexchange = Exchange if autoexchange is None else autoexchange self.max_priority = max_priority if queues is not None and not isinstance(queues, Mapping): @@ -181,7 +193,21 @@ def deselect(self, exclude): self._consume_from.pop(queue, None) def new_missing(self, name): - return Queue(name, self.autoexchange(name), name) + queue_arguments = None + if self.create_missing_queue_type and self.create_missing_queue_type != "classic": + if self.create_missing_queue_type not in ("classic", "quorum"): + raise ValueError( + f"Invalid queue type '{self.create_missing_queue_type}'. " + "Valid types are 'classic' and 'quorum'." + ) + queue_arguments = {"x-queue-type": self.create_missing_queue_type} + + if self.create_missing_queue_exchange_type: + exchange = Exchange(name, self.create_missing_queue_exchange_type) + else: + exchange = self.autoexchange(name) + + return Queue(name, exchange, name, queue_arguments=queue_arguments) @property def consume_from(self): @@ -238,14 +264,18 @@ def create_task_message(self): def send_task_message(self): return self._create_task_sender() - def Queues(self, queues, create_missing=None, - autoexchange=None, max_priority=None): + def Queues(self, queues, create_missing=None, create_missing_queue_type=None, + create_missing_queue_exchange_type=None, autoexchange=None, max_priority=None): # Create new :class:`Queues` instance, using queue defaults # from the current configuration. conf = self.app.conf default_routing_key = conf.task_default_routing_key if create_missing is None: create_missing = conf.task_create_missing_queues + if create_missing_queue_type is None: + create_missing_queue_type = conf.task_create_missing_queue_type + if create_missing_queue_exchange_type is None: + create_missing_queue_exchange_type = conf.task_create_missing_queue_exchange_type if max_priority is None: max_priority = conf.task_queue_max_priority if not queues and conf.task_default_queue: @@ -259,8 +289,14 @@ def Queues(self, queues, create_missing=None, autoexchange = (self.autoexchange if autoexchange is None else autoexchange) return self.queues_cls( - queues, self.default_exchange, create_missing, - autoexchange, max_priority, default_routing_key, + queues, + default_exchange=self.default_exchange, + create_missing=create_missing, + create_missing_queue_type=create_missing_queue_type, + create_missing_queue_exchange_type=create_missing_queue_exchange_type, + autoexchange=autoexchange, + max_priority=max_priority, + default_routing_key=default_routing_key, ) def Router(self, queues=None, create_missing=None): diff --git a/celery/app/base.py b/celery/app/base.py index 71ce9329d81..fe18784809d 100644 --- a/celery/app/base.py +++ b/celery/app/base.py @@ -57,6 +57,24 @@ logger = get_logger(__name__) +if sys.version_info >= (3, 14): + import annotationlib + + def _get_annotations(fun): + # In Python 3.14+, annotations are deferred by default (PEP 649). + # Accessing fun.__annotations__ (or inspect.get_annotations without a + # format) evaluates them and may raise NameError for types only + # available under TYPE_CHECKING. To preserve previous behavior, first + # try to return evaluated annotations; if that fails with NameError, + # fall back to returning stringified annotations instead. + try: + return inspect.get_annotations(fun) + except NameError: + return inspect.get_annotations(fun, format=annotationlib.Format.STRING) +else: + def _get_annotations(fun): + return fun.__annotations__ + BUILTIN_FIXUPS = { 'celery.fixups.django:fixup', } @@ -308,7 +326,11 @@ class name. #: Signal sent after app has prepared the configuration. on_after_configure = None - #: Signal sent after app has been finalized. + #: Signal sent after the app has been finalized (i.e., all pending + #: task decorators have been evaluated, built-in tasks loaded, and + #: every currently registered task has been bound to the app). This is + #: the earliest point at which the task registry is initialized/stable + #: and safe to inspect for tasks currently registered with this app. on_after_finalize = None #: Signal sent by every new process after fork. @@ -590,7 +612,7 @@ def _task_from_fun( '_decorated': True, '__doc__': fun.__doc__, '__module__': fun.__module__, - '__annotations__': fun.__annotations__, + '__annotations__': _get_annotations(fun), '__header__': self.type_checker(fun, bound=bind), '__wrapped__': run}, **options))() # for some reason __qualname__ cannot be set in type() diff --git a/celery/app/control.py b/celery/app/control.py index 603d930a542..00db75d6ddf 100644 --- a/celery/app/control.py +++ b/celery/app/control.py @@ -20,7 +20,7 @@ from kombu.utils.functional import lazy from kombu.utils.objects import cached_property -from celery.exceptions import DuplicateNodenameWarning +from celery.exceptions import DuplicateNodenameWarning, ImproperlyConfigured from celery.utils.log import get_logger from celery.utils.text import pluralize @@ -428,6 +428,12 @@ class Control: def __init__(self, app=None): self.app = app + if (app.conf.control_queue_durable and + app.conf.control_queue_exclusive): + raise ImproperlyConfigured( + "control_queue_durable and control_queue_exclusive cannot both be True " + "(exclusive queues are automatically deleted and cannot be durable).", + ) self.mailbox = self.Mailbox( app.conf.control_exchange, type='fanout', @@ -437,6 +443,8 @@ def __init__(self, app=None): queue_ttl=app.conf.control_queue_ttl, reply_queue_ttl=app.conf.control_queue_ttl, queue_expires=app.conf.control_queue_expires, + queue_exclusive=app.conf.control_queue_exclusive, + queue_durable=app.conf.control_queue_durable, reply_queue_expires=app.conf.control_queue_expires, ) register_after_fork(self, _after_fork_cleanup_control) diff --git a/celery/app/defaults.py b/celery/app/defaults.py index 7fa300dd0dd..480667eeced 100644 --- a/celery/app/defaults.py +++ b/celery/app/defaults.py @@ -150,6 +150,8 @@ def __repr__(self): control=Namespace( queue_ttl=Option(300.0, type='float'), queue_expires=Option(10.0, type='float'), + queue_exclusive=Option(False, type='bool'), + queue_durable=Option(False, type='bool'), exchange=Option('celery', type='string'), ), couchbase=Namespace( @@ -179,6 +181,8 @@ def __repr__(self): queue_expires=Option(60.0, type='float'), queue_ttl=Option(5.0, type='float'), queue_prefix=Option('celeryev'), + queue_exclusive=Option(False, type='bool'), + queue_durable=Option(False, type='bool'), serializer=Option('json'), exchange=Option('celeryev', type='string'), ), @@ -243,6 +247,10 @@ def __repr__(self): database=Namespace( url=Option(old={'celery_result_dburi'}), engine_options=Option( + { + 'pool_pre_ping': True, + 'pool_recycle': 3600, + }, type='dict', old={'celery_result_engine_options'}, ), short_lived_sessions=Option( @@ -260,6 +268,8 @@ def __repr__(self): annotations=Option(type='any'), compression=Option(type='string', old={'celery_message_compression'}), create_missing_queues=Option(True, type='bool'), + create_missing_queue_type=Option('classic', type='string'), + create_missing_queue_exchange_type=Option(None, type='string'), inherit_parent_priority=Option(False, type='bool'), default_delivery_mode=Option(2, type='string'), default_queue=Option('celery'), @@ -336,6 +346,7 @@ def __repr__(self): pool_restarts=Option(False, type='bool'), proc_alive_timeout=Option(4.0, type='float'), prefetch_multiplier=Option(4, type='int'), + eta_task_limit=Option(None, type='int'), enable_prefetch_count_reduction=Option(True, type='bool'), disable_prefetch=Option(False, type='bool'), redirect_stdouts=Option( diff --git a/celery/app/task.py b/celery/app/task.py index 1688eafd01b..79cf9cb7f4a 100644 --- a/celery/app/task.py +++ b/celery/app/task.py @@ -36,6 +36,17 @@ R_UNBOUND_TASK = '' R_INSTANCE = '<@task: {0.name} of {app}{flags}>' +# Filtered headers relating to dead-lettering in RabbitMQ. +X_DEATH_HEADERS = { + 'x-death', + 'x-first-death-exchange', + 'x-first-death-queue', + 'x-first-death-reason', + 'x-last-death-exchange', + 'x-last-death-queue', + 'x-last-death-reason', +} + #: Here for backwards compatibility as tasks no longer use a custom meta-class. TaskType = type @@ -123,6 +134,14 @@ def get(self, key, default=None): def __repr__(self): return f'' + def _filter_x_death_headers(self, headers): + """Filter out X-Death headers to prevent RabbitMQ cycle detection.""" + headers = headers.copy() if headers else {} + for x_death_header in X_DEATH_HEADERS: + headers.pop(x_death_header, None) + + return headers + def as_execution_options(self): limit_hard, limit_soft = self.timelimit or (None, None) execution_options = { @@ -139,7 +158,7 @@ def as_execution_options(self): 'expires': self.expires, 'soft_time_limit': limit_soft, 'time_limit': limit_hard, - 'headers': self.headers, + 'headers': self._filter_x_death_headers(self.headers), 'retries': self.retries, 'reply_to': self.reply_to, 'replaced_task_nesting': self.replaced_task_nesting, @@ -538,6 +557,13 @@ def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, The headers can be used as an overlay for custom labeling using the :ref:`canvas-stamping` feature. + task_id (str): Optional argument to override the default task id. + By default, Celery generates a unique id (UUID4) for every task + submission. You can instead provide your own string identifier. + If supplied, this value will be used as the task’s id instead + of generating one automatically. Be careful to avoid collisions + when overriding task ids. + Returns: celery.result.AsyncResult: Promise of future evaluation. diff --git a/celery/app/trace.py b/celery/app/trace.py index b6289709365..0641415c236 100644 --- a/celery/app/trace.py +++ b/celery/app/trace.py @@ -154,6 +154,28 @@ def get_task_name(request, default): return getattr(request, 'shadow', None) or default +def get_actual_ignore_result(task, req): + """Return the effective ignore_result, with request overriding task. + + If req provides an explicit ignore_result, that value is used; + otherwise task.ignore_result is returned. + """ + if req is None: + return task.ignore_result + + actual = getattr(req, 'ignore_result', None) + + # Context defines `ignore_result = False` at class level (see Context + # in celery/app/task.py). getattr() above would return the class default + # (False) even when the request never set it explicitly, making it + # impossible to distinguish "override=False" from "not set". We check + # __dict__ to detect only instance-level (i.e., explicitly set) values. + if isinstance(req, Context) and 'ignore_result' not in req.__dict__: + actual = None + + return actual if actual is not None else task.ignore_result + + class TraceInfo: """Information about task execution.""" @@ -165,7 +187,9 @@ def __init__(self, state, retval=None): def handle_error_state(self, task, req, eager=False, call_errbacks=True): - if task.ignore_result: + ignore_result = get_actual_ignore_result(task, req) + + if ignore_result: store_errors = task.store_errors_even_if_ignored elif eager and task.store_eager_result: store_errors = True @@ -353,16 +377,6 @@ def build_tracer(name, task, loader=None, hostname=None, store_errors=True, fun = task if task_has_custom(task, '__call__') else task.run loader = loader or app.loader - ignore_result = task.ignore_result - track_started = task.track_started - track_started = not eager and (task.track_started and not ignore_result) - - # #6476 - if eager and not ignore_result and task.store_eager_result: - publish_result = True - else: - publish_result = not eager and not ignore_result - deduplicate_successful_tasks = ((app.conf.task_acks_late or task.acks_late) and app.conf.worker_deduplicate_successful_tasks and app.backend.persistent) @@ -409,6 +423,55 @@ def on_error(request, exc, state=FAILURE, call_errbacks=True): ) return I, R, I.state, I.retval + def _dispatch_callbacks_and_chain( + retval, callbacks, chain, parent_id, root_id, priority, + ): + """Dispatch callbacks and chain for a completed task. + + Dispatches link callbacks and then the next chain step. + Does NOT fire task lifecycle signals (on_success, task_postrun) + or call mark_as_done — callers handle those separately. + + Note: dispatch is not atomic. If callbacks succeed but the + chain step fails (or vice-versa), a Reject + redeliver may + re-dispatch the already-sent callbacks. This is acceptable + under Celery's at-least-once delivery model. + """ + if callbacks: + if len(callbacks) > 1: + sigs, groups = [], [] + for sig in callbacks: + sig = signature(sig, app=app) + if isinstance(sig, group): + groups.append(sig) + else: + sigs.append(sig) + for group_ in groups: + group_.apply_async( + (retval,), + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + if sigs: + group(sigs, app=app).apply_async( + (retval,), + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + else: + signature(callbacks[0], app=app).apply_async( + (retval,), + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + if chain: + _chsig = signature(chain[-1], app=app) + _chsig.apply_async( + (retval,), chain=chain[:-1], + parent_id=parent_id, root_id=root_id, + priority=priority, + ) + def trace_task(uuid, args, kwargs, request=None): # R - is the possibly prepared return value. # I - is the Info object. @@ -434,6 +497,14 @@ def trace_task(uuid, args, kwargs, request=None): task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) + ignore_result = get_actual_ignore_result(task, task_request) + track_started = not eager and (task.track_started and not ignore_result) + # #6476 + if eager and not ignore_result and task.store_eager_result: + publish_result = True + else: + publish_result = not eager and not ignore_result + redelivered = (task_request.delivery_info and task_request.delivery_info.get('redelivered', False)) if deduplicate_successful_tasks and redelivered: @@ -452,6 +523,41 @@ def trace_task(uuid, args, kwargs, request=None): 'name': get_task_name(task_request, name), 'description': 'Task already completed successfully.' }) + _root_id = task_request.root_id or uuid + _priority = task_request.delivery_info.get('priority') if \ + inherit_parent_priority else None + try: + _meta = r._get_task_meta() + stored_retval = _meta.get('result') + # Children are populated by mark_as_done on the + # original execution. If present, callbacks were + # already dispatched — skip to avoid duplicates. + # Requires the backend to persist extended result + # metadata (result_extended=True). + _children = _meta.get('children') + _callbacks = task_request.callbacks + _chain = task_request.chain + if (_callbacks or _chain) and not _children: + _dispatch_callbacks_and_chain( + stored_retval, _callbacks, _chain, + parent_id=uuid, root_id=_root_id, + priority=_priority, + ) + successful_requests.add(task_request.id) + except MemoryError: + raise + except Exception as exc: + # Permanent failures (malformed signature, etc.) + # will requeue indefinitely. Broker-level + # dead-letter / max-delivery-count policies are + # the intended circuit-breaker. + logger.error( + 'Failed to dispatch chain/callbacks for ' + 'deduplicated task %s', + task_request.id, + exc_info=True, + ) + raise Reject(exc, requeue=True) return trace_ok_t(R, I, T, Rstr) push_task(task) @@ -510,43 +616,12 @@ def trace_task(uuid, args, kwargs, request=None): # separately, so need to call them separately # so that the trail's not added multiple times :( # (Issue #1936) - callbacks = task.request.callbacks - if callbacks: - if len(task.request.callbacks) > 1: - sigs, groups = [], [] - for sig in callbacks: - sig = signature(sig, app=app) - if isinstance(sig, group): - groups.append(sig) - else: - sigs.append(sig) - for group_ in groups: - group_.apply_async( - (retval,), - parent_id=uuid, root_id=root_id, - priority=task_priority - ) - if sigs: - group(sigs, app=app).apply_async( - (retval,), - parent_id=uuid, root_id=root_id, - priority=task_priority - ) - else: - signature(callbacks[0], app=app).apply_async( - (retval,), parent_id=uuid, root_id=root_id, - priority=task_priority - ) - - # execute first task in chain - chain = task_request.chain - if chain: - _chsig = signature(chain.pop(), app=app) - _chsig.apply_async( - (retval,), chain=chain, - parent_id=uuid, root_id=root_id, - priority=task_priority - ) + _dispatch_callbacks_and_chain( + retval, task.request.callbacks, + task_request.chain, + parent_id=uuid, root_id=root_id, + priority=task_priority, + ) task.backend.mark_as_done( uuid, retval, task_request, publish_result, ) @@ -597,6 +672,8 @@ def trace_task(uuid, args, kwargs, request=None): exc_info=True) except MemoryError: raise + except Reject: + raise except Exception as exc: _signal_internal_error(task, uuid, args, kwargs, request, exc) if eager: @@ -616,6 +693,8 @@ def trace_task(task, uuid, args, kwargs, request=None, **opts): if task.__trace__ is None: task.__trace__ = build_tracer(task.name, task, **opts) return task.__trace__(uuid, args, kwargs, request) + except Reject: + raise except Exception as exc: _signal_internal_error(task, uuid, args, kwargs, request, exc) return trace_ok_t(report_internal_error(task, exc), TraceInfo(FAILURE, exc), 0.0, None) diff --git a/celery/apps/worker.py b/celery/apps/worker.py index 5558dab8e5f..0a49a878909 100644 --- a/celery/apps/worker.py +++ b/celery/apps/worker.py @@ -17,7 +17,7 @@ from billiard.process import current_process from kombu.utils.encoding import safe_str -from celery import VERSION_BANNER, platforms, signals +from celery import VERSION_BANNER, _original_os_write, platforms, signals from celery.app import trace from celery.loaders.app import AppLoader from celery.platforms import EX_FAILURE, EX_OK, check_privileges, isatty @@ -78,8 +78,14 @@ def active_thread_count(): def safe_say(msg, f=sys.__stderr__): + """ + Uses the original (unpatched) os.write to avoid issues with eventlet/gevent + monkey-patching. When using eventlet>=0.37.0, the patched os.write calls + hubs.trampoline() which raises RuntimeError if called from within the + hub's event loop (e.g., during signal handling). + """ if hasattr(f, 'fileno') and f.fileno() is not None: - os.write(f.fileno(), f'\n{msg}\n'.encode()) + _original_os_write(f.fileno(), f'\n{msg}\n'.encode()) class Worker(WorkController): @@ -350,7 +356,7 @@ def during_soft_shutdown(worker: Worker): install_worker_term_hard_handler(worker, sig='SIGQUIT', callback=on_hard_shutdown) # Cancel all unacked requests and allow the worker to terminate naturally - worker.consumer.cancel_all_unacked_requests() + worker.consumer.cancel_active_requests() # We get here if the worker was in the middle of the soft (cold) shutdown process, # and the matching signal was received. This can typically happen when the worker is @@ -409,11 +415,19 @@ def on_cold_shutdown(worker: Worker): # Initiate soft shutdown process (if enabled and tasks are running) worker.wait_for_soft_shutdown() + # Stop consuming new tasks to prevents requeued messages from being immediately redelivered + if worker.consumer.task_consumer: + worker.consumer.task_consumer.cancel() + # Cancel all unacked requests and allow the worker to terminate naturally - worker.consumer.cancel_all_unacked_requests() + worker.consumer.cancel_active_requests() + + from celery.worker import state + state.should_terminate = True # Stop the pool to allow successful tasks call on_success() - worker.consumer.pool.stop() + if worker.consumer.pool: + worker.consumer.pool.stop() # Allow SIGTERM to be remapped to SIGQUIT to initiate cold shutdown instead of warm shutdown using SIGTERM diff --git a/celery/backends/asynchronous.py b/celery/backends/asynchronous.py index cedae5013a8..c5f292ceb6e 100644 --- a/celery/backends/asynchronous.py +++ b/celery/backends/asynchronous.py @@ -1,8 +1,11 @@ """Async I/O backend support utilities.""" + +import logging import socket import threading import time from collections import deque +from contextlib import contextmanager from queue import Empty from time import sleep from weakref import WeakKeyDictionary @@ -11,13 +14,44 @@ from celery import states from celery.exceptions import TimeoutError +from celery.utils.log import get_logger from celery.utils.threads import THREAD_TIMEOUT_MAX +E_CELERY_RESTART_REQUIRED = "Celery must be restarted because a shutdown signal was detected." + +E_RETRY_LIMIT_EXCEEDED = """ +Retry limit exceeded while trying to reconnect to the Celery result store +backend. The Celery application must be restarted. +""" + +logger = get_logger(__name__) + __all__ = ( 'AsyncBackendMixin', 'BaseResultConsumer', 'Drainer', 'register_drainer', ) + +class EventletAdaptedEvent: + """ + An adapted eventlet event, designed to match the API of `threading.Event` and + `gevent.event.Event`. + """ + + def __init__(self): + import eventlet + self.evt = eventlet.Event() + + def is_set(self): + return self.evt.ready() + + def set(self): + return self.evt.send() + + def wait(self, timeout=None): + return self.evt.wait(timeout) + + drainers = {} @@ -54,6 +88,18 @@ def drain_events_until(self, p, timeout=None, interval=1, on_interval=None, wait yield self.wait_for(p, wait, timeout=interval) except socket.timeout: pass + except OSError: + # Recoverable connection error (e.g. broker restart). + # drain_events handles reconnection internally; if an + # OSError still leaks through, we log, sleep for one + # interval, and continue rather than spinning hot. + logging.warning( + 'Drainer: connection error during drain_events, ' + 'will retry on next loop iteration.', + exc_info=True, + ) + time.sleep(interval) + if on_interval: on_interval() if p.ready: # got event on the wanted channel. @@ -62,46 +108,68 @@ def drain_events_until(self, p, timeout=None, interval=1, on_interval=None, wait def wait_for(self, p, wait, timeout=None): wait(timeout=timeout) + def _event(self): + return threading.Event() + class greenletDrainer(Drainer): spawn = None + _exc = None _g = None _drain_complete_event = None # event, sended (and recreated) after every drain_events iteration - def _create_drain_complete_event(self): - """create new self._drain_complete_event object""" - pass - def _send_drain_complete_event(self): - """raise self._drain_complete_event for wakeup .wait_for""" - pass + self._drain_complete_event.set() + self._drain_complete_event = self._event() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._started = threading.Event() - self._stopped = threading.Event() - self._shutdown = threading.Event() - self._create_drain_complete_event() + + self._started = self._event() + self._stopped = self._event() + self._shutdown = self._event() + self._drain_complete_event = self._event() def run(self): self._started.set() - while not self._stopped.is_set(): + + try: + while not self._stopped.is_set(): + try: + self.result_consumer.drain_events(timeout=1) + self._send_drain_complete_event() + except socket.timeout: + pass + except OSError: + # Recoverable connection errors (e.g. broker restart) + # are handled inside drain_events via reconnection. + # If something still leaks through, we log, back off + # briefly, and retry instead of spinning hot. + logging.warning( + 'Drainer: connection error during drain_events, ' + 'will retry on next loop iteration.', + exc_info=True, + ) + time.sleep(1) + except Exception as e: + self._exc = e + raise + finally: + self._send_drain_complete_event() try: - self.result_consumer.drain_events(timeout=1) - self._send_drain_complete_event() - self._create_drain_complete_event() - except socket.timeout: - pass - self._shutdown.set() + self._shutdown.set() + except RuntimeError as e: + logging.error(f"Failed to set shutdown event: {e}") def start(self): + self._ensure_not_shut_down() + if not self._started.is_set(): self._g = self.spawn(self.run) self._started.wait() def stop(self): self._stopped.set() - self._send_drain_complete_event() self._shutdown.wait(THREAD_TIMEOUT_MAX) def wait_for(self, p, wait, timeout=None): @@ -109,6 +177,23 @@ def wait_for(self, p, wait, timeout=None): if not p.ready: self._drain_complete_event.wait(timeout=timeout) + self._ensure_not_shut_down() + + def _ensure_not_shut_down(self): + """Currently used to ensure the drainer has not run to completion. + + Raises if the shutdown event has been signaled (either due to an exception + or stop() being called). + + The _shutdown event acts as synchronization to ensure _exc is properly + set before it is read from, avoiding need for locks. + """ + if self._shutdown.is_set(): + if self._exc is not None: + raise self._exc + else: + raise Exception(E_CELERY_RESTART_REQUIRED) + @register_drainer('eventlet') class eventletDrainer(greenletDrainer): @@ -119,12 +204,8 @@ def spawn(self, func): sleep(0) return g - def _create_drain_complete_event(self): - from eventlet.event import Event - self._drain_complete_event = Event() - - def _send_drain_complete_event(self): - self._drain_complete_event.send() + def _event(self): + return EventletAdaptedEvent() @register_drainer('gevent') @@ -136,13 +217,9 @@ def spawn(self, func): gevent.sleep(0) return g - def _create_drain_complete_event(self): + def _event(self): from gevent.event import Event - self._drain_complete_event = Event() - - def _send_drain_complete_event(self): - self._drain_complete_event.set() - self._create_drain_complete_event() + return Event() class AsyncBackendMixin: @@ -239,6 +316,11 @@ def is_async(self): class BaseResultConsumer: """Manager responsible for consuming result messages.""" + #: Tuple of transport-layer exceptions that signal a lost connection. + #: Subclasses should override this with the appropriate exception types + #: so that :meth:`reconnect_on_error` can catch and recover from them. + _connection_errors = () + def __init__(self, backend, app, accept, pending_results, pending_messages): self.backend = backend @@ -253,6 +335,34 @@ def __init__(self, backend, app, accept, def start(self, initial_task_id, **kwargs): raise NotImplementedError() + @contextmanager + def reconnect_on_error(self): + """Context manager that catches connection errors and reconnects. + + Wraps a block of code so that any :attr:`_connection_errors` raised + inside it trigger a call to :meth:`_reconnect`. If reconnection + itself raises a connection error the consumer is considered + unrecoverable and a :exc:`RuntimeError` is raised to signal that + the Celery application must be restarted. + """ + try: + yield + except self._connection_errors: + try: + self._reconnect() + except self._connection_errors as exc: + logger.critical(E_RETRY_LIMIT_EXCEEDED) + raise RuntimeError(E_RETRY_LIMIT_EXCEEDED) from exc + + def _reconnect(self): + """Re-establish the backend connection. + + Subclasses must override this method to perform the transport-specific + reconnection logic that should be executed when a connection error is + caught by :meth:`reconnect_on_error`. + """ + pass + def stop(self): pass diff --git a/celery/backends/base.py b/celery/backends/base.py index c80591de19c..69bfa550f6c 100644 --- a/celery/backends/base.py +++ b/celery/backends/base.py @@ -629,6 +629,12 @@ def store_result(self, task_id, result, state, if self.always_retry and self.exception_safe_to_retry(exc): if retries < self.max_retries: retries += 1 + try: + self.on_backend_retryable_error(exc) + except Exception: + logger.exception( + "on_backend_retryable_error hook failed; continuing retry loop", + ) # get_exponential_backoff_interval computes integers # and time.sleep accept floats for sub second sleep @@ -689,6 +695,10 @@ def exception_safe_to_retry(self, exc): """ return False + def on_backend_retryable_error(self, exc): + """Hook called before retrying a recoverable backend exception.""" + return None + def get_task_meta(self, task_id, cache=True): """Get task meta from backend. @@ -710,6 +720,12 @@ def get_task_meta(self, task_id, cache=True): if self.always_retry and self.exception_safe_to_retry(exc): if retries < self.max_retries: retries += 1 + try: + self.on_backend_retryable_error(exc) + except Exception: + logger.exception( + "on_backend_retryable_error hook failed; continuing retry loop", + ) # get_exponential_backoff_interval computes integers # and time.sleep accept floats for sub second sleep diff --git a/celery/backends/couchdb.py b/celery/backends/couchdb.py index a4b040dab75..9cc7d7881f2 100644 --- a/celery/backends/couchdb.py +++ b/celery/backends/couchdb.py @@ -96,4 +96,5 @@ def mget(self, keys): return [self.get(key) for key in keys] def delete(self, key): + key = bytes_to_str(key) self.connection.delete(key) diff --git a/celery/backends/database/__init__.py b/celery/backends/database/__init__.py index df03db56d38..97a96e49813 100644 --- a/celery/backends/database/__init__.py +++ b/celery/backends/database/__init__.py @@ -13,7 +13,7 @@ from .session import SessionManager try: - from sqlalchemy.exc import DatabaseError, InvalidRequestError + from sqlalchemy.exc import DatabaseError, InterfaceError, InvalidRequestError from sqlalchemy.orm.exc import StaleDataError except ImportError: raise ImproperlyConfigured( @@ -24,6 +24,13 @@ __all__ = ('DatabaseBackend',) +RETRYABLE_DB_ERRORS = ( + DatabaseError, + InterfaceError, + InvalidRequestError, + StaleDataError, +) + @contextmanager def session_cleanup(session): @@ -45,7 +52,16 @@ def _inner(*args, **kwargs): for retries in range(max_retries): try: return fun(*args, **kwargs) - except (DatabaseError, InvalidRequestError, StaleDataError): + except RETRYABLE_DB_ERRORS as exc: + backend = args[0] if args else None + on_retryable_error = getattr(backend, 'on_backend_retryable_error', None) + if callable(on_retryable_error): + try: + on_retryable_error(exc) + except Exception: + logger.exception( + "on_backend_retryable_error hook failed; continuing retry loop", + ) logger.warning( 'Failed operation %s. Retrying %s more times.', fun.__name__, max_retries - retries - 1, @@ -77,9 +93,14 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): self.task_cls = TaskExtended self.url = url or dburi or conf.database_url + + # Merge engine options: defaults from config <- constructor overrides + # The defaults (pool_pre_ping=True, pool_recycle=3600) are defined in + # celery/app/defaults.py under database_engine_options self.engine_options = dict( - engine_options or {}, - **conf.database_engine_options or {}) + conf.database_engine_options or {}, + **(engine_options or {}) + ) self.short_lived_sessions = kwargs.get( 'short_lived_sessions', conf.database_short_lived_sessions) @@ -108,6 +129,12 @@ def __init__(self, dburi=None, engine_options=None, url=None, **kwargs): def extended_result(self): return self.app.conf.find_value_for_key('extended', 'result') + def exception_safe_to_retry(self, exc): + return isinstance(exc, RETRYABLE_DB_ERRORS) + + def on_backend_retryable_error(self, exc): + self.session_manager.invalidate(self.url) + def _create_tables(self): """Create the task and taskset tables.""" self.ResultSession() diff --git a/celery/backends/database/models.py b/celery/backends/database/models.py index a5df8f4d341..f8ee6239349 100644 --- a/celery/backends/database/models.py +++ b/celery/backends/database/models.py @@ -11,19 +11,32 @@ __all__ = ('Task', 'TaskExtended', 'TaskSet') +DialectSpecificInteger = sa.Integer().with_variant(sa.BigInteger, 'mssql') + + +def _get_utc_now(): + """Return current UTC datetime. + + This helper is used as a callable for SQLAlchemy column defaults + to ensure the timestamp is evaluated at INSERT/UPDATE time, + not at module import time. + """ + return datetime.now(timezone.utc) + + class Task(ResultModelBase): """Task result/status.""" __tablename__ = 'celery_taskmeta' __table_args__ = {'sqlite_autoincrement': True} - id = sa.Column(sa.Integer, sa.Sequence('task_id_sequence'), + id = sa.Column(DialectSpecificInteger, sa.Sequence('task_id_sequence'), primary_key=True, autoincrement=True) task_id = sa.Column(sa.String(155), unique=True) status = sa.Column(sa.String(50), default=states.PENDING) result = sa.Column(PickleType, nullable=True) - date_done = sa.Column(sa.DateTime, default=datetime.now(timezone.utc), - onupdate=datetime.now(timezone.utc), nullable=True) + date_done = sa.Column(sa.DateTime, default=_get_utc_now, + onupdate=_get_utc_now, nullable=True, index=True) traceback = sa.Column(sa.Text, nullable=True) def __init__(self, task_id): @@ -80,12 +93,12 @@ class TaskSet(ResultModelBase): __tablename__ = 'celery_tasksetmeta' __table_args__ = {'sqlite_autoincrement': True} - id = sa.Column(sa.Integer, sa.Sequence('taskset_id_sequence'), + id = sa.Column(DialectSpecificInteger, sa.Sequence('taskset_id_sequence'), autoincrement=True, primary_key=True) taskset_id = sa.Column(sa.String(155), unique=True) result = sa.Column(PickleType, nullable=True) - date_done = sa.Column(sa.DateTime, default=datetime.now(timezone.utc), - nullable=True) + date_done = sa.Column(sa.DateTime, default=_get_utc_now, + nullable=True, index=True) def __init__(self, taskset_id, result): self.taskset_id = taskset_id diff --git a/celery/backends/database/session.py b/celery/backends/database/session.py index 415d4623e00..d4b6496a7ed 100644 --- a/celery/backends/database/session.py +++ b/celery/backends/database/session.py @@ -48,8 +48,11 @@ def get_engine(self, dburi, **kwargs): engine = self._engines[dburi] = create_engine(dburi, **kwargs) return engine else: - kwargs = {k: v for k, v in kwargs.items() if - not k.startswith('pool')} + unsupported_nullpool_kwargs = {'max_overflow', 'echo_pool'} + kwargs = { + k: v for k, v in kwargs.items() + if not k.startswith('pool') and k not in unsupported_nullpool_kwargs + } return create_engine(dburi, poolclass=NullPool, **kwargs) def create_session(self, dburi, short_lived_sessions=False, **kwargs): @@ -60,6 +63,13 @@ def create_session(self, dburi, short_lived_sessions=False, **kwargs): return engine, self._sessions[dburi] return engine, sessionmaker(bind=engine) + def invalidate(self, dburi): + """Dispose cached engine/session state for a database URI.""" + self._sessions.pop(dburi, None) + engine = self._engines.pop(dburi, None) + if engine is not None: + engine.dispose() + def prepare_models(self, engine): if not self.prepared: # SQLAlchemy will check if the items exist before trying to diff --git a/celery/backends/mongodb.py b/celery/backends/mongodb.py index 1789f6cf0b0..3bdea5ed974 100644 --- a/celery/backends/mongodb.py +++ b/celery/backends/mongodb.py @@ -20,6 +20,7 @@ from bson.binary import Binary except ImportError: from pymongo.binary import Binary + from pymongo import uri_parser from pymongo.errors import InvalidDocument else: # pragma: no cover Binary = None @@ -73,7 +74,7 @@ def __init__(self, app=None, **kwargs): if self.url: self.url = self._ensure_mongodb_uri_compliance(self.url) - uri_data = pymongo.uri_parser.parse_uri(self.url) + uri_data = uri_parser.parse_uri(self.url) # build the hosts list to create a mongo connection hostslist = [ f'{x[0]}:{x[1]}' for x in uri_data['nodelist'] diff --git a/celery/backends/redis.py b/celery/backends/redis.py index e2597be88fd..154285f2a7e 100644 --- a/celery/backends/redis.py +++ b/celery/backends/redis.py @@ -1,13 +1,14 @@ """Redis result store backend.""" import time -from contextlib import contextmanager from functools import partial from ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED from urllib.parse import unquote +from kombu.utils import symbol_by_name from kombu.utils.functional import retry_over_time from kombu.utils.objects import cached_property from kombu.utils.url import _parse_url, maybe_sanitize_url +from redis import CredentialProvider from celery import states from celery._state import task_join_will_block @@ -70,11 +71,6 @@ E_LOST = 'Connection to Redis lost: Retry (%s/%s) %s.' -E_RETRY_LIMIT_EXCEEDED = """ -Retry limit exceeded while trying to reconnect to the Celery redis result \ -store backend. The Celery application must be restarted. -""" - logger = get_logger(__name__) @@ -115,23 +111,14 @@ def _reconnect_pubsub(self): if self.subscribed_to: self._pubsub.subscribe(*self.subscribed_to) else: - self._pubsub.connection = self._pubsub.connection_pool.get_connection( - 'pubsub', self._pubsub.shard_hint - ) + self._pubsub.connection = self._pubsub.connection_pool.get_connection() # even if there is nothing to subscribe, we should not lose the callback after connecting. # The on_connect callback will re-subscribe to any channels we previously subscribed to. self._pubsub.connection.register_connect_callback(self._pubsub.on_connect) - @contextmanager - def reconnect_on_error(self): - try: - yield - except self._connection_errors: - try: - self._ensure(self._reconnect_pubsub, ()) - except self._connection_errors: - logger.critical(E_RETRY_LIMIT_EXCEEDED) - raise + def _reconnect(self): + """Re-establish the Redis pub/sub connection with retry.""" + self._ensure(self._reconnect_pubsub, ()) def _maybe_cancel_ready_task(self, meta): if meta['status'] in states.READY_STATES: @@ -230,6 +217,7 @@ def __init__(self, host=None, port=None, db=None, password=None, retry_on_timeout = _get('redis_retry_on_timeout') socket_keepalive = _get('redis_socket_keepalive') health_check_interval = _get('redis_backend_health_check_interval') + credential_provider = _get('redis_backend_credential_provider') self.connparams = { 'host': _get('redis_host') or 'localhost', @@ -241,6 +229,7 @@ def __init__(self, host=None, port=None, db=None, password=None, 'retry_on_timeout': retry_on_timeout or False, 'socket_connect_timeout': socket_connect_timeout and float(socket_connect_timeout), + 'client_name': _get('redis_client_name'), } username = _get('redis_username') @@ -254,6 +243,23 @@ def __init__(self, host=None, port=None, db=None, password=None, # support for py-redis<3.4.0. self.connparams['username'] = username + if credential_provider: + # if credential provider passed as string or query param + if isinstance(credential_provider, str): + credential_provider_cls = symbol_by_name(credential_provider) + credential_provider = credential_provider_cls() + + if not isinstance(credential_provider, CredentialProvider): + raise ValueError( + "Credential provider is not an instance of a redis.CredentialProvider or a subclass" + ) + + self.connparams['credential_provider'] = credential_provider + + # drop username and password if credential provider is configured + self.connparams.pop("username", None) + self.connparams.pop("password", None) + if health_check_interval: self.connparams["health_check_interval"] = health_check_interval @@ -350,6 +356,23 @@ def _params_from_url(self, url, defaults): db = db.strip('/') if isinstance(db, str) else db connparams['db'] = int(db) + # credential provider as query string + credential_provider = query.pop("credential_provider", None) + if credential_provider: + if isinstance(credential_provider, str): + credential_provider_cls = symbol_by_name(credential_provider) + credential_provider = credential_provider_cls() + + if not isinstance(credential_provider, CredentialProvider): + raise ValueError( + "Credential provider is not an instance of a redis.CredentialProvider or a subclass" + ) + + connparams['credential_provider'] = credential_provider + # drop username and password if credential provider is configured + connparams.pop("username", None) + connparams.pop("password", None) + for key, value in query.items(): if key in redis.connection.URL_QUERY_ARGUMENT_PARSERS: query[key] = redis.connection.URL_QUERY_ARGUMENT_PARSERS[key]( @@ -502,7 +525,7 @@ def on_chord_part_return(self, request, state, result, callback = maybe_signature(request.chord, app=app) total = int(chord_size_bytes) + totaldiff if readycount == total: - header_result = GroupResult.restore(gid) + header_result = GroupResult.restore(gid, app=app) if header_result is not None: # If we manage to restore a `GroupResult`, then it must # have been complex and saved by `apply_chord()` earlier. @@ -645,8 +668,8 @@ def _params_from_url(self, url, defaults): for param in ("host", "port", "db", "password"): connparams.pop(param) - # Adding db/password in connparams to connect to the correct instance - for param in ("db", "password"): + # Adding db/password/username in connparams to connect to the correct instance + for param in ("db", "password", "username"): if connparams['hosts'] and param in connparams['hosts'][0]: connparams[param] = connparams['hosts'][0].get(param) return connparams @@ -671,7 +694,12 @@ def _get_pool(self, **params): master_name = self._transport_options.get("master_name", None) + credentials = { + k: params[k] for k in ("username", "password") if k in params + } + return sentinel_instance.master_for( service_name=master_name, redis_class=self._get_client(), + **credentials, ).connection_pool diff --git a/celery/backends/rpc.py b/celery/backends/rpc.py index 927c7f517fa..577eb7d404f 100644 --- a/celery/backends/rpc.py +++ b/celery/backends/rpc.py @@ -2,6 +2,7 @@ RPC-style result backend, using reply-to and one queue per client. """ +import logging import time import kombu @@ -17,6 +18,8 @@ __all__ = ('BacklogLimitExceeded', 'RPCBackend') +logger = logging.getLogger(__name__) + E_NO_CHORD_SUPPORT = """ The "rpc" result backend does not support chords! @@ -40,13 +43,19 @@ class ResultConsumer(BaseResultConsumer): _connection = None _consumer = None + _no_ack = True def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._create_binding = self.backend._create_binding def start(self, initial_task_id, no_ack=True, **kwargs): + self._no_ack = no_ack self._connection = self.app.connection() + self._connection_errors = ( + self._connection.connection_errors + + self._connection.channel_errors + ) initial_queue = self._create_binding(initial_task_id) self._consumer = self.Consumer( self._connection.default_channel, [initial_queue], @@ -56,10 +65,60 @@ def start(self, initial_task_id, no_ack=True, **kwargs): def drain_events(self, timeout=None): if self._connection: - return self._connection.drain_events(timeout=timeout) + with self.reconnect_on_error(): + return self._connection.drain_events(timeout=timeout) elif timeout: time.sleep(timeout) + def _reconnect(self): + """Close the stale connection and rebuild the consumer. + + Re-subscribes to every queue that the old consumer was listening on + so that pending results can still be drained. + """ + logger.warning( + 'RPC result consumer: connection lost, attempting to reconnect...', + exc_info=True, + ) + old_queues = [] + if self._consumer is not None: + old_queues = list(self._consumer.queues) + try: + self._consumer.cancel() + except Exception: + logger.debug( + 'RPC result consumer: error while cancelling stale ' + 'consumer during reconnect', + exc_info=True, + ) + + if self._connection is not None: + try: + self._connection.close() + except Exception: + logger.debug( + 'RPC result consumer: error while closing stale ' + 'connection during reconnect', + exc_info=True, + ) + self._connection = None + + # Establish a fresh connection and consumer. + self._connection = self.app.connection() + self._connection_errors = ( + self._connection.connection_errors + + self._connection.channel_errors + ) + self._consumer = self.Consumer( + self._connection.default_channel, + old_queues, + callbacks=[self.on_state_change], + no_ack=self._no_ack, + accept=self.accept, + ) + self._consumer.consume() + logger.info('RPC result consumer: reconnected successfully.') + def stop(self): try: self._consumer.cancel() diff --git a/celery/backends/s3.py b/celery/backends/s3.py index ea04ae373d1..e53bd84a7f2 100644 --- a/celery/backends/s3.py +++ b/celery/backends/s3.py @@ -32,7 +32,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) if not boto3 or not botocore: - raise ImproperlyConfigured('You must install boto3' + raise ImproperlyConfigured('You must install boto3 ' 'to use s3 backend') conf = self.app.conf diff --git a/celery/beat.py b/celery/beat.py index 86ad837f0d5..fbda26e9f42 100644 --- a/celery/beat.py +++ b/celery/beat.py @@ -12,6 +12,7 @@ from calendar import timegm from collections import namedtuple from functools import total_ordering +from pickle import UnpicklingError from threading import Event, Thread from billiard import ensure_multiprocessing @@ -569,11 +570,11 @@ def _create_schedule(self): for _ in (1, 2): try: self._store['entries'] - except (KeyError, UnicodeDecodeError, TypeError): + except (KeyError, UnicodeDecodeError, TypeError, UnpicklingError): # new schedule db try: self._store['entries'] = {} - except (KeyError, UnicodeDecodeError, TypeError) + dbm.error as exc: + except (KeyError, UnicodeDecodeError, TypeError, UnpicklingError) + dbm.error as exc: self._store = self._destroy_open_corrupted_schedule(exc) continue else: @@ -626,8 +627,8 @@ def __init__(self, app, max_interval=None, schedule_filename=None, self._is_stopped = Event() def __reduce__(self): - return self.__class__, (self.max_interval, self.schedule_filename, - self.scheduler_cls, self.app) + return self.__class__, (self.app, self.max_interval, + self.schedule_filename, self.scheduler_cls) def start(self, embedded_process=False): info('beat: Starting...') diff --git a/celery/bin/base.py b/celery/bin/base.py index 61cc37a0291..be3999236bc 100644 --- a/celery/bin/base.py +++ b/celery/bin/base.py @@ -8,9 +8,12 @@ import click from click import Context, ParamType +from kombu.exceptions import OperationalError from kombu.utils.objects import cached_property from celery._state import get_current_app +from celery.exceptions import CeleryCommandException +from celery.platforms import EX_UNAVAILABLE from celery.signals import user_preload_options from celery.utils import text from celery.utils.log import mlevel @@ -117,6 +120,26 @@ def say_chat(self, direction, title, body='', show_body=False): self.echo(body) +def handle_remote_command_error(command: str, exc: Exception) -> None: + if isinstance(exc, click.ClickException): + raise + + if isinstance(exc, OperationalError): + raise CeleryCommandException( + message=( + 'Could not connect to the message broker. ' + 'Please make sure your broker (e.g., RabbitMQ or Redis) is running and ' + f'the connection settings are correct. Reason: {exc}' + ), + exit_code=EX_UNAVAILABLE, + ) from exc + + raise CeleryCommandException( + message=f'Unable to run the `{command}` command. Reason: {exc}', + exit_code=EX_UNAVAILABLE, + ) from exc + + def handle_preload_options(f): """Extract preload options and return a wrapped callable.""" def caller(ctx, *args, **kwargs): diff --git a/celery/bin/control.py b/celery/bin/control.py index 38a917ea0f2..48681a9f28e 100644 --- a/celery/bin/control.py +++ b/celery/bin/control.py @@ -5,7 +5,8 @@ import click from kombu.utils.json import dumps -from celery.bin.base import COMMA_SEPARATED_LIST, CeleryCommand, CeleryOption, handle_preload_options +from celery.bin.base import (COMMA_SEPARATED_LIST, CeleryCommand, CeleryOption, handle_preload_options, + handle_remote_command_error) from celery.exceptions import CeleryCommandException from celery.platforms import EX_UNAVAILABLE from celery.utils import text @@ -128,9 +129,12 @@ def _get_commands_of_type(type_: _RemoteControlType) -> dict: def status(ctx, timeout, destination, json, **kwargs): """Show list of workers that are online.""" callback = None if json else partial(_say_remote_command_reply, ctx) - replies = ctx.obj.app.control.inspect(timeout=timeout, - destination=destination, - callback=callback).ping() + try: + replies = ctx.obj.app.control.inspect(timeout=timeout, + destination=destination, + callback=callback).ping() + except Exception as exc: + handle_remote_command_error('status', exc) if not replies: raise CeleryCommandException( @@ -183,7 +187,10 @@ def inspect(ctx, command, timeout, destination, json, **kwargs): inspect = ctx.obj.app.control.inspect(timeout=timeout, destination=destination, callback=callback) - replies = inspect._request(command, **arguments) + try: + replies = inspect._request(command, **arguments) + except Exception as exc: + handle_remote_command_error(f'inspect {command}', exc) if not replies: raise CeleryCommandException( @@ -236,11 +243,14 @@ def control(ctx, command, timeout, destination, json): show_reply=True) args = ctx.args arguments = _compile_arguments(command, args) - replies = ctx.obj.app.control.broadcast(command, timeout=timeout, - destination=destination, - callback=callback, - reply=True, - arguments=arguments) + try: + replies = ctx.obj.app.control.broadcast(command, timeout=timeout, + destination=destination, + callback=callback, + reply=True, + arguments=arguments) + except Exception as exc: + handle_remote_command_error(f'control {command}', exc) if not replies: raise CeleryCommandException( diff --git a/celery/bin/events.py b/celery/bin/events.py index 89470838bcc..c69f4ae7552 100644 --- a/celery/bin/events.py +++ b/celery/bin/events.py @@ -4,7 +4,8 @@ import click -from celery.bin.base import LOG_LEVEL, CeleryDaemonCommand, CeleryOption, handle_preload_options +from celery.bin.base import (LOG_LEVEL, CeleryDaemonCommand, CeleryOption, handle_preload_options, + handle_remote_command_error) from celery.platforms import detached, set_process_title, strargv @@ -82,13 +83,16 @@ def _run_evtop(app): def events(ctx, dump, camera, detach, frequency, maxrate, loglevel, **kwargs): """Event-stream utilities.""" app = ctx.obj.app - if dump: - return _run_evdump(app) + try: + if dump: + return _run_evdump(app) - if camera: - return _run_evcam(camera, app=app, freq=frequency, maxrate=maxrate, - loglevel=loglevel, - detach=detach, - **kwargs) + if camera: + return _run_evcam(camera, app=app, freq=frequency, maxrate=maxrate, + loglevel=loglevel, + detach=detach, + **kwargs) - return _run_evtop(app) + return _run_evtop(app) + except Exception as exc: + handle_remote_command_error('events', exc) diff --git a/celery/bin/graph.py b/celery/bin/graph.py index d4d6f16205f..eb1baaa4aa0 100644 --- a/celery/bin/graph.py +++ b/celery/bin/graph.py @@ -4,7 +4,7 @@ import click -from celery.bin.base import CeleryCommand, handle_preload_options +from celery.bin.base import CeleryCommand, handle_preload_options, handle_remote_command_error from celery.utils.graph import DependencyGraph, GraphFormatter @@ -154,7 +154,10 @@ def maybe_abbr(l, name, max=Wmax): workers = args['nodes'] threads = args.get('threads') or [] except KeyError: - replies = app.control.inspect().stats() or {} + try: + replies = app.control.inspect().stats() or {} + except Exception as exc: + handle_remote_command_error('graph workers', exc) workers, threads = [], [] for worker, reply in replies.items(): workers.append(worker) @@ -171,8 +174,11 @@ def maybe_abbr(l, name, max=Wmax): list(range(int(threads))), 'P', Tmax, ) - broker = Broker(args.get( - 'broker', app.connection_for_read().as_uri())) + try: + broker_uri = args.get('broker', app.connection_for_read().as_uri()) + except Exception as exc: + handle_remote_command_error('graph workers', exc) + broker = Broker(broker_uri) backend = Backend(backend) if backend else None deps = DependencyGraph(formatter=Formatter()) deps.add_arc(broker) diff --git a/celery/bin/worker.py b/celery/bin/worker.py index 52f09f3a83d..f16dafba94f 100644 --- a/celery/bin/worker.py +++ b/celery/bin/worker.py @@ -189,7 +189,8 @@ def detach(path, argv, logfile=None, pidfile=None, uid=None, value: ctx.obj.app.conf.worker_disable_prefetch if value is None else value, cls=CeleryOption, help_group="Worker Options", - help="Disable broker prefetching. The worker will only fetch a task when a process slot is available.") + help="Disable broker prefetching. The worker will only fetch a task when a process slot is available. " + "Only supported with Redis brokers.") @click.option('-c', '--concurrency', type=int, diff --git a/celery/bootsteps.py b/celery/bootsteps.py index 878560624d1..dcd2dc0875a 100644 --- a/celery/bootsteps.py +++ b/celery/bootsteps.py @@ -303,7 +303,7 @@ class Step(metaclass=StepType): #: Set this to true if the step is enabled based on some condition. conditional = False - #: List of other steps that that must be started before this step. + #: List of other steps that must be started before this step. #: Note that all dependencies must be in the same blueprint. requires = () diff --git a/celery/canvas.py b/celery/canvas.py index 1ceeacc166d..5d701839c1f 100644 --- a/celery/canvas.py +++ b/celery/canvas.py @@ -989,13 +989,13 @@ def __or__(self, other): sig.tasks[-2].body = sig.tasks[-2].body | sig.tasks[-1] sig.tasks = sig.tasks[:-1] return sig - elif self.tasks and isinstance(self.tasks[-1], chord): - # CHAIN [last item is chord] -> chain with chord body. + elif self.tasks and isinstance(self.tasks[-1], chord) and not isinstance(other, chord): + # CHAIN [last item is chord] | TASK -> chain with chord body. sig = self.clone() sig.tasks[-1].body = sig.tasks[-1].body | other return sig else: - # chain | task -> chain + # chain | task/chord -> chain # use type(self) for _chain subclasses return type(self)(seq_concat_item( self.unchain_tasks(), other), app=self._app) @@ -1741,7 +1741,7 @@ def _prepared(self, tasks, partial_args, group_id, root_id, app, def _apply_tasks(self, tasks, producer=None, app=None, p=None, add_to_parent=None, chord=None, - args=None, kwargs=None, **options): + args=None, kwargs=None, group_index=None, **options): """Run all the tasks in the group. This is used by :meth:`apply_async` to run all the tasks in the group diff --git a/celery/concurrency/asynpool.py b/celery/concurrency/asynpool.py index dd2f068a215..10783847a96 100644 --- a/celery/concurrency/asynpool.py +++ b/celery/concurrency/asynpool.py @@ -391,6 +391,7 @@ def _flush_outqueue(self, fd, remove, process_index, on_state_change): setblocking(reader, 1) except OSError: return remove(fd) + result = None try: if reader.poll(0): task = reader.recv() @@ -398,7 +399,7 @@ def _flush_outqueue(self, fd, remove, process_index, on_state_change): task = None sleep(0.5) except (OSError, EOFError): - return remove(fd) + result = remove(fd) else: if task: on_state_change(task) @@ -406,7 +407,8 @@ def _flush_outqueue(self, fd, remove, process_index, on_state_change): try: setblocking(reader, 0) except OSError: - return remove(fd) + result = remove(fd) + return result class AsynPool(_pool.Pool): @@ -470,7 +472,7 @@ def __init__(self, processes=None, synack=False, self.write_stats = Counter() - super().__init__(processes, *args, **kwargs) + super().__init__(processes, *args, synack=synack, **kwargs) for proc in self._pool: # create initial mappings, these will be updated @@ -511,10 +513,11 @@ def _track_child_process(self, proc, hub): self._event_process_exit, hub, proc) def _untrack_child_process(self, proc, hub): - if proc._sentinel_poll is not None: - fd, proc._sentinel_poll = proc._sentinel_poll, None - hub.remove(fd) - os.close(fd) + sentinel_poll = getattr(proc, '_sentinel_poll', None) + if sentinel_poll is not None: + proc._sentinel_poll = None + hub.remove(sentinel_poll) + os.close(sentinel_poll) def register_with_event_loop(self, hub): """Register the async pool with the current event loop.""" @@ -997,11 +1000,14 @@ def flush(self): if self._state == TERMINATE: return # cancel all tasks that haven't been accepted so that NACK is sent - # if synack is enabled. - if self.synack: - for job in self._cache.values(): - if not job._accepted: + # if synack is enabled, otherwise discard them from the cache + # since they will be redelivered by the broker. + for job in tuple(self._cache.values()): + if not job._accepted: + if self.synack: job._cancel() + else: + job.discard() # clear the outgoing buffer as the tasks will be redelivered by # the broker anyway. @@ -1026,36 +1032,47 @@ def flush(self): if writer is not None: owned_by[writer] = job - if not self._active_writers: - self._cache.clear() - else: - while self._active_writers: - writers = list(self._active_writers) - for gen in writers: - if (gen.__name__ == '_write_job' and - gen_not_started(gen)): - # hasn't started writing the job so can - # discard the task, but we must also remove - # it from the Pool._cache. - try: - job = owned_by[gen] - except KeyError: - pass - else: - # removes from Pool._cache - job.discard() - self._active_writers.discard(gen) + while self._active_writers: + writers = list(self._active_writers) + for gen in writers: + if (gen.__name__ == '_write_job' and + gen_not_started(gen)): + # hasn't started writing the job so can + # discard the task, but we must also remove + # it from the Pool._cache. + try: + job = owned_by[gen] + except KeyError: + pass else: + # removes from Pool._cache + job.discard() + self._active_writers.discard(gen) + else: + try: + job = owned_by[gen] + except KeyError: + # Generator not in owned_by — not a _write_job + # (e.g. a _write_ack coroutine added by send_ack()). + # These *MUST* complete or the worker process will + # hang waiting for the ack. Advance it one step; + # the generator raises StopIteration/OSError when + # done or when the peer process has already died. try: - job = owned_by[gen] - except KeyError: - pass + next(gen) + except (StopIteration, OSError, EOFError): + self._active_writers.discard(gen) + else: + job_proc = job._write_to + if job_proc._is_alive(): + # _flush_writer calls + # _active_writers.discard(gen) in its finally. + self._flush_writer(job_proc, gen) else: - job_proc = job._write_to - if job_proc._is_alive(): - self._flush_writer(job_proc, gen) - + # Process is dead, job will never + # complete - discard from cache. job.discard() + self._active_writers.discard(gen) # workers may have exited in the meantime. self.maintain_pool() sleep(next(intervals)) # don't busyloop diff --git a/celery/concurrency/prefork.py b/celery/concurrency/prefork.py index b163328d0b3..aed83b8a29a 100644 --- a/celery/concurrency/prefork.py +++ b/celery/concurrency/prefork.py @@ -3,11 +3,14 @@ Pool implementation using :mod:`multiprocessing`. """ import os +import threading +import time from billiard import forking_enable from billiard.common import REMAP_SIGTERM, TERM_SIGNAME from billiard.pool import CLOSE, RUN from billiard.pool import Pool as BlockingPool +from kombu.asynchronous import get_event_loop from celery import platforms, signals from celery._state import _set_task_join_will_block, set_default_app @@ -140,7 +143,48 @@ def on_stop(self): """Gracefully stop the pool.""" if self._pool is not None and self._pool._state in (RUN, CLOSE): self._pool.close() - self._pool.join() + + # Keep firing timers (for heartbeats on async transports) while + # the pool drains. If not using an async transport, no hub exists + # and the timer thread is not created. + hub = get_event_loop() + if hub is not None: + shutdown_event = threading.Event() + + def fire_timers_loop(): + while not shutdown_event.is_set(): + try: + hub.fire_timers() + except Exception: + logger.warning( + "Exception in timer thread during prefork on_stop()", + exc_info=True, + ) + # 0.5 seconds was chosen as a balance between joining quickly + # after the pool join is complete and sleeping long enough to + # avoid excessive CPU usage. + time.sleep(0.5) + + timer_thread = threading.Thread( + target=fire_timers_loop, + daemon=True, + name="prefork-timer-shutdown", + ) + timer_thread.start() + + try: + self._pool.join() + finally: + shutdown_event.set() + timer_thread.join(timeout=1.0) + + if timer_thread.is_alive(): + logger.warning( + "Timer thread in prefork on_stop() did not terminate cleanly" + ) + else: + self._pool.join() + self._pool = None def on_terminate(self): diff --git a/celery/contrib/sphinx.py b/celery/contrib/sphinx.py index a5505ff189a..0b1e6389af9 100644 --- a/celery/contrib/sphinx.py +++ b/celery/contrib/sphinx.py @@ -29,11 +29,27 @@ syntax. Use ``.. autotask::`` to alternatively manually document a task. + +Sphinx 9.0+ Compatibility +------------------------- + +Sphinx 9.0 introduced a rewritten autodoc implementation. The Celery +extension requires the legacy class-based autodoc mode to function +correctly. When using Sphinx 9.0 or later, add the following to your +:file:`conf.py`: + +.. code-block:: python + + autodoc_use_legacy_class_based = True + +The extension will automatically enable this setting if not configured, +but it is recommended to set it explicitly to avoid warnings. """ +import warnings from inspect import signature from docutils import nodes -from sphinx.domains.python import PyFunction +from sphinx.domains.python import PyFunction, PyXRefRole from sphinx.ext.autodoc import FunctionDocumenter from celery.app.task import BaseTask @@ -94,9 +110,27 @@ def autodoc_skip_member_handler(app, what, name, obj, skip, options): def setup(app): """Setup Sphinx extension.""" + import sphinx + app.setup_extension('sphinx.ext.autodoc') + + # Sphinx 9.0+ rewrote autodoc; TaskDocumenter requires legacy mode. + # See: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html + sphinx_version = tuple(int(x) for x in sphinx.__version__.split('.')[:2]) + if sphinx_version >= (9, 0): + if not getattr(app.config, 'autodoc_use_legacy_class_based', False): + warnings.warn( + "Sphinx 9.0+ detected. celery.contrib.sphinx requires " + "'autodoc_use_legacy_class_based = True' in conf.py. " + "Enabling it automatically.", + UserWarning, + stacklevel=2 + ) + app.config.autodoc_use_legacy_class_based = True + app.add_autodocumenter(TaskDocumenter) app.add_directive_to_domain('py', 'task', TaskDirective) + app.add_role_to_domain('py', 'task', PyXRefRole(fix_parens=True)) app.add_config_value('celery_task_prefix', '(task)', True) app.connect('autodoc-skip-member', autodoc_skip_member_handler) diff --git a/celery/contrib/testing/worker.py b/celery/contrib/testing/worker.py index 46eac75fd64..905a6a9ccdc 100644 --- a/celery/contrib/testing/worker.py +++ b/celery/contrib/testing/worker.py @@ -42,7 +42,9 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.pool_cls.__module__.split('.')[-1] == 'prefork': + # Defensive check: pool_cls may be a string (e.g., 'prefork') or a class + pool_module = self.pool_cls if isinstance(self.pool_cls, str) else self.pool_cls.__module__ + if pool_module.split('.')[-1] == 'prefork': from billiard import Queue self.logger_queue = Queue() self.pid = os.getpid() diff --git a/celery/events/dumper.py b/celery/events/dumper.py index 24c7b3e9421..08ee12027ca 100644 --- a/celery/events/dumper.py +++ b/celery/events/dumper.py @@ -4,7 +4,7 @@ as they happen. Think of it like a `tcpdump` for Celery events. """ import sys -from datetime import datetime +from datetime import datetime, timezone from celery.app import app_or_default from celery.utils.functional import LRUCache @@ -48,7 +48,7 @@ def say(self, msg): pass def on_event(self, ev): - timestamp = datetime.utcfromtimestamp(ev.pop('timestamp')) + timestamp = datetime.fromtimestamp(ev.pop('timestamp'), timezone.utc) type = ev.pop('type').lower() hostname = ev.pop('hostname') if type.startswith('task-'): diff --git a/celery/events/receiver.py b/celery/events/receiver.py index 14871073322..bda50a10083 100644 --- a/celery/events/receiver.py +++ b/celery/events/receiver.py @@ -8,6 +8,7 @@ from celery import uuid from celery.app import app_or_default +from celery.exceptions import ImproperlyConfigured from celery.utils.time import adjust_timestamp from .event import get_exchange @@ -34,7 +35,9 @@ class EventReceiver(ConsumerMixin): def __init__(self, channel, handlers=None, routing_key='#', node_id=None, app=None, queue_prefix=None, - accept=None, queue_ttl=None, queue_expires=None): + accept=None, queue_ttl=None, queue_expires=None, + queue_exclusive=None, + queue_durable=None): self.app = app_or_default(app or self.app) self.channel = maybe_channel(channel) self.handlers = {} if handlers is None else handlers @@ -48,11 +51,22 @@ def __init__(self, channel, handlers=None, routing_key='#', queue_ttl = self.app.conf.event_queue_ttl if queue_expires is None: queue_expires = self.app.conf.event_queue_expires + if queue_exclusive is None: + queue_exclusive = self.app.conf.event_queue_exclusive + if queue_durable is None: + queue_durable = self.app.conf.event_queue_durable + if queue_exclusive and queue_durable: + raise ImproperlyConfigured( + 'Queue cannot be both exclusive and durable, ' + 'choose one or the other.' + ) self.queue = Queue( '.'.join([self.queue_prefix, self.node_id]), exchange=self.exchange, routing_key=self.routing_key, - auto_delete=True, durable=False, + auto_delete=not queue_durable, + durable=queue_durable, + exclusive=queue_exclusive, message_ttl=queue_ttl, expires=queue_expires, ) diff --git a/celery/fixups/django.py b/celery/fixups/django.py index b35499493a6..5d78b381607 100644 --- a/celery/fixups/django.py +++ b/celery/fixups/django.py @@ -1,4 +1,5 @@ """Django-specific customization.""" +import contextlib import os import sys import warnings @@ -11,6 +12,7 @@ from celery import _state, signals from celery.exceptions import FixupWarning, ImproperlyConfigured +from celery.worker import WorkController if TYPE_CHECKING: from types import ModuleType @@ -101,6 +103,16 @@ def on_import_modules(self, **kwargs: Any) -> None: self.worker_fixup.validate_models() def on_worker_init(self, **kwargs: Any) -> None: + worker: Optional["WorkController"] = kwargs.get("sender") + if worker: + self.worker_fixup.worker = worker + else: + warnings.warn( + "DjangoFixup.on_worker_init called without a sender (worker instance). " + "This may indicate a misconfiguration or an internal error.", + FixupWarning, + stacklevel=2, + ) self.worker_fixup.install() def now(self, utc: bool = False) -> datetime: @@ -117,6 +129,7 @@ def _now(self) -> datetime: class DjangoWorkerFixup: _db_recycles = 0 + worker = None # Set via on_worker_init callback to avoid recursive WorkController instantiation def __init__(self, app: "Celery") -> None: self.app = app @@ -168,7 +181,7 @@ def on_worker_process_init(self, **kwargs: Any) -> None: self._maybe_close_db_fd(c) # use the _ version to avoid DB_REUSE preventing the conn.close() call - self._close_database(force=True) + self._close_database() self.close_cache() def _maybe_close_db_fd(self, c: "BaseDatabaseWrapper") -> None: @@ -197,13 +210,28 @@ def close_database(self, **kwargs: Any) -> None: self._close_database() self._db_recycles += 1 - def _close_database(self, force: bool = False) -> None: - for conn in self._db.connections.all(): + def _is_prefork(self) -> bool: + if self.worker is None: + return False + pool = self.worker.pool_cls if isinstance(self.worker.pool_cls, str) else self.worker.pool_cls.__module__ + return "prefork" in pool + + def _close_database(self) -> None: + try: + connections = self._db.connections.all(initialized_only=True) + except TypeError: + # Support Django < 4.1 + connections = self._db.connections.all() + + is_prefork = self._is_prefork() + + for conn in connections: try: - if force: - conn.close() - else: - conn.close_if_unusable_or_obsolete() + conn.close() + pool_enabled = self._settings.DATABASES.get(conn.alias, {}).get("OPTIONS", {}).get("pool") + if pool_enabled and is_prefork and hasattr(conn, "close_pool"): + with contextlib.suppress(KeyError): + conn.close_pool() except self.interface_errors: pass except self.DatabaseError as exc: diff --git a/celery/result.py b/celery/result.py index 75512c5aadb..66a9e20aab8 100644 --- a/celery/result.py +++ b/celery/result.py @@ -137,6 +137,8 @@ def forget(self): self._cache = None if self.parent: self.parent.forget() + + self.backend.remove_pending_result(self) self.backend.forget(self.id) def revoke(self, connection=None, terminate=False, signal=None, diff --git a/celery/schedules.py b/celery/schedules.py index 010b3396fa8..dc6cb695394 100644 --- a/celery/schedules.py +++ b/celery/schedules.py @@ -764,7 +764,7 @@ class solar(BaseSchedule): 'sunset': '-0:34', 'dusk_civil': '-6', 'dusk_nautical': '-12', - 'dusk_astronomical': '18', + 'dusk_astronomical': '-18', } _methods = { 'dawn_astronomical': 'next_rising', diff --git a/celery/utils/__init__.py b/celery/utils/__init__.py index e905c247837..0e2e1a33070 100644 --- a/celery/utils/__init__.py +++ b/celery/utils/__init__.py @@ -22,7 +22,6 @@ 'cached_property', 'chunks', 'gen_task_name', - 'gen_task_name', 'gen_unique_id', 'get_cls_by_name', 'get_full_cls_name', diff --git a/celery/utils/functional.py b/celery/utils/functional.py index 5fb0d6339e5..f9a4d4600d5 100644 --- a/celery/utils/functional.py +++ b/celery/utils/functional.py @@ -1,5 +1,6 @@ """Functional-style utilities.""" import inspect +import sys from collections import UserList from functools import partial from itertools import islice, tee, zip_longest @@ -311,6 +312,45 @@ def _argsfromspec(spec, replace_defaults=True): ])) +if sys.version_info >= (3, 14): + import annotationlib as _annotationlib + + def _getfullargspec(fun): + # In Python 3.14+, inspect.getfullargspec evaluates annotations by default + # (PEP 649), raising NameError for TYPE_CHECKING-only types. We don't need + # annotations here, so use Format.STRING to avoid evaluation. + # For bound methods, use __func__ so that 'self' is included in args, + # matching the behaviour of getfullargspec on older Python versions. + target = getattr(fun, '__func__', fun) + sig = inspect.signature(target, annotation_format=_annotationlib.Format.STRING) + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults = [], None, None, [], [], {} + for name, param in sig.parameters.items(): + kind = param.kind + if kind in (param.POSITIONAL_ONLY, param.POSITIONAL_OR_KEYWORD): + args.append(name) + if param.default is not param.empty: + defaults.append(param.default) + elif kind == param.VAR_POSITIONAL: + varargs = name + elif kind == param.KEYWORD_ONLY: + kwonlyargs.append(name) + if param.default is not param.empty: + kwonlydefaults[name] = param.default + elif kind == param.VAR_KEYWORD: + varkw = name + return inspect.FullArgSpec( + args=args, + varargs=varargs, + varkw=varkw, + defaults=tuple(defaults) or None, + kwonlyargs=kwonlyargs, + kwonlydefaults=kwonlydefaults or None, + annotations={}, + ) +else: + _getfullargspec = inspect.getfullargspec + + def head_from_fun(fun: Callable[..., Any], bound: bool = False) -> str: """Generate signature function from actual function.""" # we could use inspect.Signature here, but that implementation @@ -329,7 +369,7 @@ def head_from_fun(fun: Callable[..., Any], bound: bool = False) -> str: name = fun.__name__ definition = FUNHEAD_TEMPLATE.format( fun_name=name, - fun_args=_argsfromspec(inspect.getfullargspec(fun)), + fun_args=_argsfromspec(_getfullargspec(fun)), fun_value=1, ) logger.debug(definition) @@ -359,6 +399,12 @@ def fun_takes_argument(name, fun, position=None): def fun_accepts_kwargs(fun): """Return true if function accepts arbitrary keyword arguments.""" + # inspect.signature evaluates annotations in Python 3.14+ (PEP 649), + # which raises NameError for types only imported under TYPE_CHECKING. + # Check co_flags directly to avoid touching annotations entirely. + code = getattr(fun, '__code__', None) + if code is not None: + return bool(code.co_flags & inspect.CO_VARKEYWORDS) return any( p for p in inspect.signature(fun).parameters.values() if p.kind == p.VAR_KEYWORD diff --git a/celery/utils/time.py b/celery/utils/time.py index 2376bb3b71d..bd9dba1a2e6 100644 --- a/celery/utils/time.py +++ b/celery/utils/time.py @@ -1,6 +1,7 @@ """Utilities related to dates, times, intervals, and timezones.""" from __future__ import annotations +import logging import numbers import os import random @@ -17,6 +18,7 @@ from dateutil.parser import isoparse from kombu.utils.functional import reprcall from kombu.utils.objects import cached_property +from tzlocal import get_localzone from .functional import dictfilter from .text import pluralize @@ -26,6 +28,7 @@ else: from backports.zoneinfo import ZoneInfo +logger = logging.getLogger(__name__) __all__ = ( 'LocalTimezone', 'timezone', 'maybe_timedelta', @@ -117,7 +120,7 @@ def _isdst(self, dt: datetime) -> bool: class _Zone: """Timezone class that provides the timezone for the application. - If `enable_utc` is disabled, LocalTimezone is provided as the timezone provider through local(). + If `enable_utc` is disabled, local system timezone is provided as the timezone provider through local(). Otherwise, this class provides a UTC ZoneInfo instance as the timezone provider for the application. Additionally this class provides a few utility methods for converting datetimes. @@ -158,9 +161,16 @@ def get_timezone(self, zone: str | tzinfo) -> tzinfo: return zone @cached_property - def local(self) -> LocalTimezone: - """Return LocalTimezone instance for the application.""" - return LocalTimezone() + def local(self) -> tzinfo: + """Return the local system timezone for the application.""" + try: + timezone = get_localzone() + except Exception as ex: + timezone = None + logger.warning("Failed to retrieve local timezone (%s): %s", type(ex).__name__, ex) + if timezone is None: + return LocalTimezone() + return timezone @cached_property def utc(self) -> tzinfo: @@ -202,7 +212,7 @@ def delta_resolution(dt: datetime, delta: timedelta) -> datetime: def remaining( - start: datetime, ends_in: timedelta, now: Callable | None = None, + start: datetime, ends_in: timedelta, now: datetime | None = None, relative: bool = False) -> timedelta: """Calculate the real remaining time for a start date and a timedelta. @@ -214,7 +224,7 @@ def remaining( relative (bool): If enabled the end time will be calculated using :func:`delta_resolution` (i.e., rounded to the resolution of `ends_in`). - now (Callable): Function returning the current time and date. + now (~datetime.datetime): Current time and date. Defaults to :func:`datetime.now(timezone.utc)`. Returns: diff --git a/celery/worker/components.py b/celery/worker/components.py index f062affb61f..f60abe98a9c 100644 --- a/celery/worker/components.py +++ b/celery/worker/components.py @@ -75,7 +75,10 @@ def create(self, w): return self def start(self, w): - pass + # Ensure the kombu hub's poller is initialized before the event loop starts. + # Since asynloop() no longer resets the hub on exit (to preserve timers + # during shutdown), we must initialize the poller upfront. + _ = w.hub.poller def stop(self, w): w.hub.close() @@ -191,7 +194,10 @@ def __init__(self, w, beat=False, **kwargs): def create(self, w): from celery.beat import EmbeddedService - if w.pool_cls.__module__.endswith(('gevent', 'eventlet')): + + # Defensive check: pool_cls may be a string (e.g., 'gevent') or a class + pool_module = w.pool_cls if isinstance(w.pool_cls, str) else w.pool_cls.__module__ + if pool_module.endswith(('gevent', 'eventlet')): raise ImproperlyConfigured(ERR_B_GREEN) b = w.beat = EmbeddedService(w.app, schedule_filename=w.schedule_filename, diff --git a/celery/worker/consumer/consumer.py b/celery/worker/consumer/consumer.py index 3e6a66df532..2a5d955fef3 100644 --- a/celery/worker/consumer/consumer.py +++ b/celery/worker/consumer/consumer.py @@ -31,7 +31,8 @@ from celery.utils.text import truncate from celery.utils.time import humanize_seconds, rate from celery.worker import loops -from celery.worker.state import active_requests, maybe_shutdown, requests, reserved_requests, task_reserved +from celery.worker.state import (active_requests, maybe_shutdown, requests, reserved_requests, successful_requests, + task_reserved) __all__ = ('Consumer', 'Evloop', 'dump_body') @@ -157,6 +158,10 @@ class Consumer: #: connection attempt. first_connection_attempt = True + #: Counter to track number of conn retry attempts + #: to broker. Will be reset to 0 once successful + broker_connection_retry_attempt = 0 + class Blueprint(bootsteps.Blueprint): """Consumer blueprint.""" @@ -451,8 +456,6 @@ def on_close(self): # to the current channel. if self.controller and self.controller.semaphore: self.controller.semaphore.clear() - if self.timer: - self.timer.clear() for bucket in self.task_buckets.values(): if bucket: bucket.clear_pending() @@ -488,9 +491,11 @@ def ensure_connected(self, conn): def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP): if getattr(conn, 'alt', None) and interval == 0: next_step = CONNECTION_FAILOVER + elif interval > 0: + self.broker_connection_retry_attempt += 1 next_step = next_step.format( when=humanize_seconds(interval, 'in', ' '), - retries=int(interval / 2), + retries=self.broker_connection_retry_attempt, max_retries=self.app.conf.broker_connection_max_retries) error(CONNECTION_ERROR, conn.as_uri(), exc, next_step) @@ -532,6 +537,7 @@ def _error_handler(exc, interval, next_step=CONNECTION_RETRY_STEP): callback=maybe_shutdown, ) self.first_connection_attempt = False + self.broker_connection_retry_attempt = 0 return conn def _flush_events(self): @@ -733,9 +739,13 @@ def __repr__(self): self=self, state=self.blueprint.human_state(), ) - def cancel_all_unacked_requests(self): - """Cancel all active requests that either do not require late acknowledgments or, + def cancel_active_requests(self): + """Cancel active requests during shutdown. + + Cancels all active requests that either do not require late acknowledgments or, if they do, have not been acknowledged yet. + + Does not cancel successful tasks, even if they have not been acknowledged yet. """ def should_cancel(request): @@ -745,6 +755,9 @@ def should_cancel(request): if not request.acknowledged: # Task is late acknowledged, but it has not been acknowledged yet, cancel it. + if request.id in successful_requests: + # Unless it was successful, in which case we don't want to cancel it. + return False return True # Task is late acknowledged, but it has already been acknowledged. @@ -754,7 +767,10 @@ def should_cancel(request): if requests_to_cancel: for request in requests_to_cancel: - request.cancel(self.pool) + # For acks_late tasks, don't emit RETRY signal since broker will handle redelivery + # For non-acks_late tasks, emit RETRY signal as usual + emit_retry = not request.task.acks_late + request.cancel(self.pool, emit_retry=emit_retry) class Evloop(bootsteps.StartStopStep): diff --git a/celery/worker/consumer/delayed_delivery.py b/celery/worker/consumer/delayed_delivery.py index b9d37a12511..9909be3cf27 100644 --- a/celery/worker/consumer/delayed_delivery.py +++ b/celery/worker/consumer/delayed_delivery.py @@ -3,12 +3,18 @@ This module provides the DelayedDelivery bootstep which handles setup and configuration of native delayed delivery functionality when using quorum queues. """ +import sys from typing import Iterator, List, Optional, Set, Union, ValuesView +if sys.version_info < (3, 11): # pragma: no cover + # Backport of PEP 654 for Python versions < 3.11 + from exceptiongroup import ExceptionGroup + from kombu import Connection, Queue from kombu.transport.native_delayed_delivery import (bind_queue_to_native_delayed_delivery_exchange, declare_native_delayed_delivery_exchanges_and_queues) from kombu.utils.functional import retry_over_time +from kombu.utils.url import maybe_sanitize_url from celery import Celery, bootsteps from celery.utils.log import get_logger @@ -23,7 +29,7 @@ # Default retry settings RETRY_INTERVAL = 1.0 # seconds between retries MAX_RETRIES = 3 # maximum number of retries - +RETRIED_EXCEPTIONS = (ConnectionRefusedError, OSError) # Valid queue types for delayed delivery VALID_QUEUE_TYPES = {'classic', 'quorum'} @@ -84,7 +90,7 @@ def start(self, c: Consumer) -> None: retry_over_time( self._setup_delayed_delivery, args=(c, broker_url), - catch=(ConnectionRefusedError, OSError), + catch=RETRIED_EXCEPTIONS, errback=self._on_retry, interval_start=RETRY_INTERVAL, max_retries=MAX_RETRIES, @@ -92,7 +98,7 @@ def start(self, c: Consumer) -> None: except Exception as e: logger.warning( "Failed to setup delayed delivery for %r: %s", - broker_url, str(e) + maybe_sanitize_url(broker_url), str(e) ) setup_errors.append((broker_url, e)) @@ -118,7 +124,7 @@ def _setup_delayed_delivery(self, c: Consumer, broker_url: str) -> None: queue_type = c.app.conf.broker_native_delayed_delivery_queue_type logger.debug( "Setting up delayed delivery for broker %r with queue type %r", - broker_url, queue_type + maybe_sanitize_url(broker_url), queue_type ) try: @@ -129,7 +135,7 @@ def _setup_delayed_delivery(self, c: Consumer, broker_url: str) -> None: except Exception as e: logger.warning( "Failed to declare exchanges and queues for %r: %s", - broker_url, str(e) + maybe_sanitize_url(broker_url), str(e) ) raise @@ -138,7 +144,7 @@ def _setup_delayed_delivery(self, c: Consumer, broker_url: str) -> None: except Exception as e: logger.warning( "Failed to bind queues for %r: %s", - broker_url, str(e) + maybe_sanitize_url(broker_url), str(e) ) raise @@ -157,6 +163,7 @@ def _bind_queues(self, app: Celery, connection: Connection) -> None: logger.warning("No queues found to bind for delayed delivery") return + exceptions: list[Exception] = [] for queue in queues: try: logger.debug("Binding queue %r to delayed delivery exchange", queue.name) @@ -166,7 +173,27 @@ def _bind_queues(self, app: Celery, connection: Connection) -> None: "Failed to bind queue %r: %s", queue.name, str(e) ) - raise + + # We must re-raise on retried exceptions to ensure they are + # caught with the outer retry_over_time mechanism. + # + # This could be removed if one of: + # * The minimum python version for Celery and Kombu is + # increased to 3.11. Kombu updated to use the `except*` + # clause to catch specific exceptions from an ExceptionGroup. + # * Kombu's retry_over_time utility is updated to use the + # catch utility from agronholm's exceptiongroup backport. + if isinstance(e, RETRIED_EXCEPTIONS): + raise + + exceptions.append(e) + + if exceptions: + raise ExceptionGroup( + ("One or more failures occurred while binding queues to " + "delayed delivery exchanges"), + exceptions, + ) def _on_retry(self, exc: Exception, interval_range: Iterator[float], intervals_count: int) -> float: """Callback for retry attempts. diff --git a/celery/worker/consumer/tasks.py b/celery/worker/consumer/tasks.py index 92e2c51c064..6aabbf620c7 100644 --- a/celery/worker/consumer/tasks.py +++ b/celery/worker/consumer/tasks.py @@ -66,9 +66,22 @@ def set_prefetch_count(prefetch_count): prefetch_count=prefetch_count, apply_global=qos_global, ) - c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) + eta_task_limit = c.app.conf.worker_eta_task_limit + c.qos = QoS( + set_prefetch_count, c.initial_prefetch_count, max_prefetch=eta_task_limit + ) if c.app.conf.worker_disable_prefetch: + # Only apply disable-prefetch for Redis brokers + is_redis_broker = c.connection.transport.driver_type == 'redis' + if not is_redis_broker: + logger.warning( + f"worker_disable_prefetch is only supported for Redis brokers. " + f"Current broker transport: {c.connection.transport.driver_type}. " + f"Ignoring disable_prefetch setting." + ) + return + from types import MethodType from celery.worker import state @@ -76,6 +89,8 @@ def set_prefetch_count(prefetch_count): original_can_consume = channel_qos.can_consume def can_consume(self): + # Gate on workers that have completed the WORKER_UP handshake so a + # recycling/cold-starting slot doesn't pull a task it can't run. if len(state.reserved_requests) >= Tasks.ready_worker_limit(c): return False return original_can_consume() @@ -113,7 +128,9 @@ def qos_global(self, c) -> bool: qos_global = not c.connection.qos_semantics_matches_spec if c.app.conf.worker_detect_quorum_queues: - using_quorum_queues, qname = detect_quorum_queues(c.app, c.connection.transport.driver_type) + using_quorum_queues, _ = detect_quorum_queues( + c.app, c.connection.transport.driver_type + ) if using_quorum_queues: qos_global = False diff --git a/celery/worker/control.py b/celery/worker/control.py index 8f9fc4f92ba..547ce7989b6 100644 --- a/celery/worker/control.py +++ b/celery/worker/control.py @@ -216,6 +216,13 @@ def _revoke(state, task_ids, terminate=False, signal=None, **kwargs): terminated = set() worker_state.revoked.update(task_ids) + + for task_id in task_ids: + try: + state.app.backend.mark_as_revoked(task_id, reason='revoked', store_result=True) + except Exception as exc: + logger.warning('Failed to mark task %s as revoked in backend: %s', task_id, exc) + if terminate: signum = _signals.signum(signal or TERM_SIGNAME) for request in _find_requests_by_id(task_ids): diff --git a/celery/worker/loops.py b/celery/worker/loops.py index 1f9e589eeef..5c36f15da6d 100644 --- a/celery/worker/loops.py +++ b/celery/worker/loops.py @@ -97,12 +97,37 @@ def asynloop(obj, connection, consumer, blueprint, hub, qos, next(loop) except StopIteration: loop = hub.create_loop() - finally: + except Exception: + # Reset the hub on error (e.g. connection loss) to clean up + # stale file descriptors and callbacks from the old connection. + # Also clear the timer queue so that stale periodic entries added by + # register_with_event_loop (e.g. maybe_restore_messages) do not fire + # against the broken connection after reconnect and trigger another + # crash before the new connection is fully established. + # All hub timers are re-registered during blueprint.start() once this + # exception propagates and the consumer reconnects. + # We intentionally do NOT reset on normal exit (graceful shutdown) + # so that timers (e.g. heartbeat) keep firing while the pool drains. + # WorkerShutdown/WorkerTerminate extend SystemExit (not Exception) + # so they won't be caught here. try: hub.reset() except Exception as exc: # pylint: disable=broad-except logger.exception( 'Error cleaning up after event loop: %r', exc) + # Clear stale timer entries accumulated across reconnects (e.g. + # maybe_restore_messages registered via call_repeatedly). Without + # this, each reconnect appends a new entry; all of them fire during + # the reconnect window, raise again, and trigger another restart. + # Use a separate try/except so this always runs even if hub.reset() + # raised above. Timers are re-registered by register_with_event_loop + # when blueprint.start() is called after reconnect. + try: + hub.timer.clear() + except Exception as exc: # pylint: disable=broad-except + logger.exception( + 'Error clearing hub timer after event loop: %r', exc) + raise def synloop(obj, connection, consumer, blueprint, hub, qos, diff --git a/celery/worker/request.py b/celery/worker/request.py index df99b549270..4eaab75675e 100644 --- a/celery/worker/request.py +++ b/celery/worker/request.py @@ -125,7 +125,10 @@ def __init__(self, message, on_ack=noop, self._eventer = eventer self._connection_errors = connection_errors or () self._task = task or self._app.tasks[self._type] - self._ignore_result = self._request_dict.get('ignore_result', False) + ignore_result = self._request_dict.get('ignore_result', None) + if ignore_result is None: + ignore_result = self._task.ignore_result + self._ignore_result = ignore_result # timezone means the message is timezone-aware, and the only timezone # supported at this point is UTC. @@ -287,7 +290,7 @@ def tzlocal(self): @property def store_errors(self): - return (not self.task.ignore_result or + return (not self._ignore_result or self.task.store_errors_even_if_ignored) @property @@ -422,29 +425,34 @@ def terminate(self, pool, signal=None): if obj is not None: obj.terminate(signal) - def cancel(self, pool, signal=None): + def cancel(self, pool, signal=None, emit_retry=True): signal = _signals.signum(signal or TERM_SIGNAME) if self.time_start: pool.terminate_job(self.worker_pid, signal) - self._announce_cancelled() + self._announce_cancelled(emit_retry=emit_retry) if self._apply_result is not None: obj = self._apply_result() # is a weakref if obj is not None: obj.terminate(signal) - def _announce_cancelled(self): + def _announce_cancelled(self, emit_retry=True): task_ready(self) self.send_event('task-cancelled') - reason = 'cancelled by Celery' - exc = Retry(message=reason) - self.task.backend.mark_as_retry(self.id, - exc, - request=self._context) - self.task.on_retry(exc, self.id, self.args, self.kwargs, None) + if emit_retry: + reason = 'cancelled by Celery' + exc = Retry(message=reason) + self.task.backend.mark_as_retry(self.id, + exc, + request=self._context) + + self.task.on_retry(exc, self.id, self.args, self.kwargs, None) + self._already_cancelled = True - send_retry(self.task, request=self._context, einfo=None) + + if emit_retry: + send_retry(self.task, request=self._context, einfo=None) def _announce_revoked(self, reason, terminated, signum, expired): task_ready(self) @@ -525,14 +533,16 @@ def on_timeout(self, soft, timeout): timeout, self.name, self.id) else: task_ready(self) - error('Hard time limit (%ss) exceeded for %s[%s]', - timeout, self.name, self.id) - exc = TimeLimitExceeded(timeout) - - self.task.backend.mark_as_failure( - self.id, exc, request=self._context, - store_result=self.store_errors, - ) + # This is a special case where the task timeout handling is done during + # the cold shutdown process. + if not state.should_terminate: + error('Hard time limit (%ss) exceeded for %s[%s]', timeout, self.name, self.id) + exc = TimeLimitExceeded(timeout) + + self.task.backend.mark_as_failure( + self.id, exc, request=self._context, + store_result=self.store_errors, + ) if self.task.acks_late and self.task.acks_on_failure_or_timeout: self.acknowledge() @@ -617,6 +627,12 @@ def on_failure(self, exc_info, send_failed_event=True, return_ok=False): # need to be removed from prefetched local queue self.reject(requeue=False) + # This is a special case where the task failure handling is done during + # the cold shutdown process. + if state.should_terminate: + return_ok = True + send_failed_event = False + # This is a special case where the process would not have had time # to write the result. if not requeue and (is_worker_lost or not return_ok): diff --git a/celery/worker/strategy.py b/celery/worker/strategy.py index 3fe5fa145ca..6a1c6225b48 100644 --- a/celery/worker/strategy.py +++ b/celery/worker/strategy.py @@ -109,7 +109,6 @@ def default(task, app, consumer, hostname = consumer.hostname connection_errors = consumer.connection_errors _does_info = logger.isEnabledFor(logging.INFO) - # task event related # (optimized to avoid calling request.send_event) eventer = consumer.event_dispatcher @@ -125,7 +124,8 @@ def default(task, app, consumer, limit_task = consumer._limit_task limit_post_eta = consumer._limit_post_eta Request = symbol_by_name(task.Request) - Req = create_request_cls(Request, task, consumer.pool, hostname, eventer, app=app) + Req = create_request_cls(Request, task, consumer.pool, hostname, eventer, + app=app) revoked_tasks = consumer.controller.state.revoked @@ -194,6 +194,7 @@ def task_message_handler(message, body, ack, reject, callbacks, consumer.qos.increment_eventually() return call_at(eta, limit_post_eta, (req, bucket, 1), priority=6) + if eta: consumer.qos.increment_eventually() call_at(eta, apply_eta_task, (req,), priority=6) diff --git a/docker/Dockerfile b/docker/Dockerfile index 479613ac51f..cebba9314ac 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ FROM debian:bookworm-slim -ENV PYTHONUNBUFFERED 1 -ENV PYTHONIOENCODING UTF-8 +ENV PYTHONUNBUFFERED=1 +ENV PYTHONIOENCODING=UTF-8 ARG DEBIAN_FRONTEND=noninteractive @@ -37,9 +37,10 @@ RUN apt-get update && apt-get install -y build-essential \ # Setup variables. Even though changing these may cause unnecessary invalidation of # unrelated elements, grouping them together makes the Dockerfile read better. -ENV PROVISIONING /provisioning +ENV PROVISIONING=/provisioning ENV PIP_NO_CACHE_DIR=off ENV PYTHONDONTWRITEBYTECODE=1 +ENV PIP_PREFER_BINARY=1 ARG CELERY_USER=developer @@ -47,7 +48,7 @@ ARG CELERY_USER=developer # Check for mandatory build arguments RUN : "${CELERY_USER:?CELERY_USER build argument needs to be set and non-empty.}" -ENV HOME /home/$CELERY_USER +ENV HOME=/home/$CELERY_USER ENV PATH="$HOME/.pyenv/bin:$PATH" # Copy and run setup scripts @@ -65,17 +66,16 @@ USER $CELERY_USER RUN curl https://pyenv.run | bash # Install required Python versions -RUN pyenv install 3.13 -RUN pyenv install 3.12 -RUN pyenv install 3.11 -RUN pyenv install 3.10 -RUN pyenv install 3.9 -RUN pyenv install 3.8 -RUN pyenv install pypy3.10 +RUN pyenv install 3.13 && \ + pyenv install 3.12 && \ + pyenv install 3.11 && \ + pyenv install 3.10 && \ + pyenv install 3.9 && \ + pyenv install pypy3.11 # Set global Python versions -RUN pyenv global 3.13 3.12 3.11 3.10 3.9 3.8 pypy3.10 +RUN pyenv global 3.13 3.12 3.11 3.10 3.9 pypy3.11 # Install celery WORKDIR $HOME @@ -84,90 +84,94 @@ COPY --chown=1000:1000 docker/entrypoint /entrypoint RUN chmod gu+x /entrypoint # Define the local pyenvs -RUN pyenv local 3.13 3.12 3.11 3.10 3.9 3.8 pypy3.10 +RUN pyenv local 3.13 3.12 3.11 3.10 3.9 pypy3.11 -RUN pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \ +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.13 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.12 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.11 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.10 -m pip install --upgrade pip setuptools wheel && \ pyenv exec python3.9 -m pip install --upgrade pip setuptools wheel && \ - pyenv exec python3.8 -m pip install --upgrade pip setuptools wheel && \ - pyenv exec pypy3.10 -m pip install --upgrade pip setuptools wheel + pyenv exec pypy3.11 -m pip install --upgrade pip setuptools wheel -COPY --chown=1000:1000 . $HOME/celery +# Install requirements first to leverage Docker layer caching +# Split into separate RUN commands to reduce memory pressure and improve layer caching +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.13 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt -RUN pyenv exec python3.13 -m pip install -e $HOME/celery && \ - pyenv exec python3.12 -m pip install -e $HOME/celery && \ - pyenv exec python3.11 -m pip install -e $HOME/celery && \ - pyenv exec python3.10 -m pip install -e $HOME/celery && \ - pyenv exec python3.9 -m pip install -e $HOME/celery && \ - pyenv exec python3.8 -m pip install -e $HOME/celery && \ - pyenv exec pypy3.10 -m pip install -e $HOME/celery +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.12 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt -# Setup one celery environment for basic development use -RUN pyenv exec python3.13 -m pip install -r requirements/default.txt \ +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.11 -m pip install -r requirements/default.txt \ -r requirements/dev.txt \ -r requirements/docs.txt \ -r requirements/pkgutils.txt \ -r requirements/test-ci-base.txt \ -r requirements/test-ci-default.txt \ -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.12 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.11 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.10 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.9 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec python3.8 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt && \ - pyenv exec pypy3.10 -m pip install -r requirements/default.txt \ - -r requirements/dev.txt \ - -r requirements/docs.txt \ - -r requirements/pkgutils.txt \ - -r requirements/test-ci-base.txt \ - -r requirements/test-ci-default.txt \ - -r requirements/test-integration.txt \ - -r requirements/test-pypy3.txt \ - -r requirements/test.txt + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt + +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.10 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt + +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.9 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt + +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec pypy3.11 -m pip install -r requirements/default.txt \ + -r requirements/dev.txt \ + -r requirements/docs.txt \ + -r requirements/pkgutils.txt \ + -r requirements/test-ci-base.txt \ + -r requirements/test-ci-default.txt \ + -r requirements/test-integration.txt \ + -r requirements/test.txt \ + --build-constraint requirements/constraints.txt + +COPY --chown=1000:1000 . $HOME/celery + +# Install celery in editable mode (dependencies already installed above) +RUN --mount=type=cache,target=/home/$CELERY_USER/.cache/pip \ + pyenv exec python3.13 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.12 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.11 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.10 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec python3.9 -m pip install --no-deps -e $HOME/celery && \ + pyenv exec pypy3.11 -m pip install --no-deps -e $HOME/celery WORKDIR $HOME/celery diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c31138f1942..05bac847c0c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -25,7 +25,7 @@ services: - azurite rabbit: - image: rabbitmq:latest + image: rabbitmq:management redis: image: redis:latest diff --git a/docs/django/first-steps-with-django.rst b/docs/django/first-steps-with-django.rst index 8ac28d342e3..e0c962880a9 100644 --- a/docs/django/first-steps-with-django.rst +++ b/docs/django/first-steps-with-django.rst @@ -216,6 +216,16 @@ However, if your app :ref:`uses a custom task base class `, you'll need inherit from :class:`~celery.contrib.django.task.DjangoTask` instead of :class:`~celery.app.task.Task` to get this behaviour. +Django Connection pool +---------------------- +From Django 5.1+ there is built-in support for database connection pooling. +If you enable it in Django ``DATABASES`` settings Celery will automatically +handle connection pool closing in worker processes via ``close_pool`` +database backend method as +`sharing connections across processes is not possible. `_ + +You can find more about Connection pool at `Django docs. `_ + Extensions ========== diff --git a/docs/faq.rst b/docs/faq.rst index 2a9970c2f17..17e3dd5b338 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -788,9 +788,10 @@ to describe the task prefetching *limit*. There's no actual prefetching involve Disabling the prefetch limits is possible, but that means the worker will consume as many tasks as it can, as fast as possible. -Since version 5.5 you can use the :option:`--disable-prefetch ` +You can use the :option:`--disable-prefetch ` flag (or set :setting:`worker_disable_prefetch` to ``True``) so that a worker -only fetches a task when one of its processes is free. +only fetches a task when one of its processes is free. This feature is currently +only supported when using Redis as the broker. A discussion on prefetch limits, and configuration settings for a worker that only reserves one task at a time is found here: diff --git a/docs/getting-started/backends-and-brokers/kafka.rst b/docs/getting-started/backends-and-brokers/kafka.rst index e5b0ea0b68e..eb6d540532c 100644 --- a/docs/getting-started/backends-and-brokers/kafka.rst +++ b/docs/getting-started/backends-and-brokers/kafka.rst @@ -41,7 +41,7 @@ For celeryconfig.py: "sasl.password": sasl_password, "security.protocol": "SASL_SSL", "sasl.mechanism": "SCRAM-SHA-512", - "bootstrap_servers": "broker:9094", + "bootstrap.servers": "broker:9094", } }) diff --git a/docs/getting-started/backends-and-brokers/rabbitmq.rst b/docs/getting-started/backends-and-brokers/rabbitmq.rst index 2afc3fa3291..b3f3d722a74 100644 --- a/docs/getting-started/backends-and-brokers/rabbitmq.rst +++ b/docs/getting-started/backends-and-brokers/rabbitmq.rst @@ -223,7 +223,7 @@ To migrate from classic mirrored queues to quorum queues, please refer to Rabbit Limitations ----------- -Disabling global QoS means that the the per-channel QoS is now static. +Disabling global QoS means that the per-channel QoS is now static. This means that some Celery features won't work when using Quorum Queues. Autoscaling relies on increasing and decreasing the prefetch count whenever a new process is instantiated diff --git a/docs/getting-started/backends-and-brokers/redis.rst b/docs/getting-started/backends-and-brokers/redis.rst index 11d42544ec2..a618488fd5c 100644 --- a/docs/getting-started/backends-and-brokers/redis.rst +++ b/docs/getting-started/backends-and-brokers/redis.rst @@ -38,6 +38,13 @@ Where the URL is in the format of: all fields after the scheme are optional, and will default to ``localhost`` on port 6379, using database 0. +If redis credential provider should be used, the URL needs to be in the following format: + +.. code-block:: text + + redis://@hostname:port/db_number?credential_provider=mymodule.myfile.myclass + + If a Unix socket connection should be used, the URL needs to be in the format: .. code-block:: text @@ -261,7 +268,7 @@ Group result ordering Versions of Celery up to and including 4.4.6 used an unsorted list to store result objects for groups in the Redis backend. This can cause those results to -be be returned in a different order to their associated tasks in the original +be returned in a different order to their associated tasks in the original group instantiation. Celery 4.4.7 introduced an opt-in behaviour which fixes this issue and ensures that group results are returned in the same order the tasks were defined, matching the behaviour of other backends. In Celery 5.0 diff --git a/docs/getting-started/backends-and-brokers/sqs.rst b/docs/getting-started/backends-and-brokers/sqs.rst index 1e67bc2b58b..d391e790ffc 100644 --- a/docs/getting-started/backends-and-brokers/sqs.rst +++ b/docs/getting-started/backends-and-brokers/sqs.rst @@ -168,6 +168,48 @@ setting:: } } +.. warning:: + + **Important:** When using ``predefined_queues``, do NOT use URL-encoded + credentials (``safequote``) for the ``access_key_id`` and ``secret_access_key`` + values. URL encoding should only be applied to credentials in the broker URL. + + Using URL-encoded credentials in ``predefined_queues`` will cause signature + mismatch errors like: "The request signature we calculated does not match + the signature you provided." + +**Correct example combining broker URL and predefined queues:** + +.. code-block:: python + + import os + from kombu.utils.url import safequote + from celery import Celery + + # Raw credentials from environment + AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") + AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") + + # URL-encode ONLY for broker URL + aws_access_key_encoded = safequote(AWS_ACCESS_KEY_ID) + aws_secret_key_encoded = safequote(AWS_SECRET_ACCESS_KEY) + + # Use encoded credentials in broker URL + broker_url = f"sqs://{aws_access_key_encoded}:{aws_secret_key_encoded}@" + + celery_app = Celery("tasks", broker=broker_url) + celery_app.conf.broker_transport_options = { + "region": "us-east-1", + "predefined_queues": { + "my-queue": { + "url": "https://sqs.us-east-1.amazonaws.com/123456/my-queue", + # Use RAW credentials here (NOT encoded) + "access_key_id": AWS_ACCESS_KEY_ID, + "secret_access_key": AWS_SECRET_ACCESS_KEY, + }, + }, + } + When using this option, the visibility timeout should be set in the SQS queue (in AWS) rather than via the :ref:`visibility timeout ` option. diff --git a/docs/getting-started/introduction.rst b/docs/getting-started/introduction.rst index 7b4f3c0a623..a937a6279a9 100644 --- a/docs/getting-started/introduction.rst +++ b/docs/getting-started/introduction.rst @@ -136,7 +136,6 @@ Celery is… - **Concurrency** - prefork (multiprocessing), - - spawn (multiprocessing using the spawn method), - Eventlet_, gevent_ - thread (multithreaded) - `solo` (single threaded) diff --git a/docs/glossary.rst b/docs/glossary.rst index 0fe3988efad..a145d8ba98b 100644 --- a/docs/glossary.rst +++ b/docs/glossary.rst @@ -106,6 +106,13 @@ Glossary :setting:`worker_prefetch_multiplier` setting, which is multiplied by the number of pool slots (threads/processes/greenthreads). + .. note:: + If you are using eta or countdown tasks, the :setting:`worker_prefetch_multiplier` + still determines the base prefetch count. The :setting:`worker_eta_task_limit` + setting, when enabled, instead caps the total number of unacknowledged + messages the worker will hold (including eta/countdown tasks). See + :setting:`worker_eta_task_limit`. + `prefetch count` Maximum number of unacknowledged messages a consumer can hold and if exceeded the transport shouldn't deliver any more messages to that diff --git a/docs/history/changelog-3.1.rst b/docs/history/changelog-3.1.rst index 4bb58c4f5a4..351faff9e15 100644 --- a/docs/history/changelog-3.1.rst +++ b/docs/history/changelog-3.1.rst @@ -1031,7 +1031,7 @@ News - **Beat**: No longer attempts to upgrade a newly created database file (Issue #1923). -- **Beat**: New setting :setting:``CELERYBEAT_SYNC_EVERY`` can be be used +- **Beat**: New setting :setting:``CELERYBEAT_SYNC_EVERY`` can be used to control file sync by specifying the number of tasks to send between each sync. @@ -1733,4 +1733,4 @@ Fixes :release-date: 2013-11-09 11:00 p.m. UTC :release-by: Ask Solem -See :ref:`whatsnew-3.1`. +See :ref:`whatsnew-3.1`. \ No newline at end of file diff --git a/docs/history/changelog-4.3.rst b/docs/history/changelog-4.3.rst index 0502c1de09e..ad3f6d9e2a6 100644 --- a/docs/history/changelog-4.3.rst +++ b/docs/history/changelog-4.3.rst @@ -339,8 +339,6 @@ Documentation Fixes by: we introduced this new configuration option to specify the accepted content from the backend. - Contributed by **Benjamin Pereto** - - **Canvas**: Fixed error callback processing for class based tasks. Contributed by **Victor Mireyev** @@ -368,8 +366,6 @@ Documentation Fixes by: We now depend on cryptography instead of pyOpenSSL for this serializer. - Contributed by **Benjamin Pereto** - - **Command Line**: :program:`celery report` now reports kernel version along with other platform details. diff --git a/docs/history/changelog-4.4.rst b/docs/history/changelog-4.4.rst index e6a851676cd..4ed3c79a2ac 100644 --- a/docs/history/changelog-4.4.rst +++ b/docs/history/changelog-4.4.rst @@ -556,8 +556,6 @@ Documentation Fixes by: we introduced this new configuration option to specify the accepted content from the backend. - Contributed by **Benjamin Pereto** - - **Canvas**: Fixed error callback processing for class based tasks. Contributed by **Victor Mireyev** @@ -585,8 +583,6 @@ Documentation Fixes by: We now depend on cryptography instead of pyOpenSSL for this serializer. - Contributed by **Benjamin Pereto** - - **Command Line**: :program:`celery report` now reports kernel version along with other platform details. diff --git a/docs/history/changelog-5.6.rst b/docs/history/changelog-5.6.rst new file mode 100644 index 00000000000..a56a1eb531d --- /dev/null +++ b/docs/history/changelog-5.6.rst @@ -0,0 +1,308 @@ +.. _changelog-5.6: + +================ + Change history +================ + +This document contains change notes for bugfix & new features +in the main branch & 5.6.x series, please see :ref:`whatsnew-5.6` for +an overview of what's new in Celery 5.6. + +.. _version-5.6.2: + +5.6.2 +===== + +:release-date: 2026-01-04 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix recursive WorkController instantiation in DjangoWorkerFixup + AttributeError when pool_cls is a string (#10045) +- Bugfix: Revoked tasks now immediately update backend status to REVOKED (#9869) +- Prepare for release: v5.6.2 (#10049) + +.. _version-5.6.1: + +5.6.1 +===== + +:release-date: 2025-12-29 +:release-by: Tomer Nosrati + +What's Changed +~~~~~~~~~~~~~~ + +- Fix Redis Sentinel ACL authentication support (#10013) +- Fix: Broker heartbeats not sent during graceful shutdown (#9986) +- docs #5410 -- Document confirm_publish broker transport option (#10016) +- close DB pools only in prefork mode (#10020) +- Fix: Avoid unnecessary Django database connection creation during cleanup (#10015) +- reliable prefork detection (#10023) +- better coverage (#10029) +- Docs: clarify `result_extended` vs periodic task metadata and show `headers["periodic_task_name"]` example (#10030) +- Stop importing pytest_subtests (#10032) +- Only use exceptiongroup backport for Python < 3.11 (#10033) +- Prepare for release: v5.6.1 (#10037) + +.. _version-5.6.0: + +5.6.0 +===== + +:release-date: 2025-11-30 +:release-by: Tomer Nosrati + +Celery v5.6.0 is now available. + +Key Highlights +~~~~~~~~~~~~~~ + +See :ref:`whatsnew-5.6` for a complete overview or read the main highlights below. + +Python 3.9 Minimum Version +-------------------------- + +Celery 5.6.0 drops support for Python 3.8 (EOL). The minimum required Python +version is now 3.9. Users still on Python 3.8 must upgrade their Python version +before upgrading to Celery 5.6.0. + +Additionally, this release includes initial support for Python 3.14. + +SQS: Reverted to ``pycurl`` from ``urllib3`` +-------------------------------------------- + +The switch from ``pycurl`` to ``urllib3`` for the SQS transport (introduced in +Celery 5.5.0 via Kombu) has been reverted due to critical issues affecting SQS +users: + +- Processing throughput dropped from ~100 tasks/sec to ~3/sec in some environments +- ``UnknownOperationException`` errors causing container crash loops +- Silent message processing failures with no error logs + +Users of the SQS transport must ensure ``pycurl`` is installed. If you removed +``pycurl`` after upgrading to Celery 5.5.0, you will need to reinstall it. + +Contributed by `@auvipy `_ in +`#9620 `_. + +Security Fix: Broker Credential Leak Prevention +------------------------------------------------ + +Fixed a security issue where broker URLs containing passwords were being logged +in plaintext by the delayed delivery mechanism. Broker credentials are now +properly sanitized in all log output. + +Contributed by `@giancarloromeo `_ in +`#9997 `_. + +Memory Leak Fixes +----------------- + +Two significant memory leaks have been fixed in this release: + +**Exception Handling Memory Leak**: Fixed a critical memory leak in task exception +handling that was particularly severe on Python 3.11+ due to enhanced traceback +data. The fix properly breaks reference cycles in tracebacks to allow garbage +collection. + +Contributed by `@jaiganeshs21 `_ in +`#9799 `_. + +**Pending Result Memory Leak**: Fixed a memory leak where ``AsyncResult`` +subscriptions were not being cleaned up when results were forgotten. + +Contributed by `@tsoos99dev `_ in +`#9806 `_. + +ETA Task Memory Limit +--------------------- + +New configuration option :setting:`worker_eta_task_limit` to prevent out-of-memory +crashes when workers fetch large numbers of ETA or countdown tasks. Previously, +workers could exhaust available memory when the broker contained many scheduled tasks. + +Example usage: + +.. code-block:: python + + app.conf.worker_eta_task_limit = 1000 + +Contributed by `@sashu2310 `_ in +`#9853 `_. + +Queue Type Selection for Auto-created Queues +-------------------------------------------- + +New configuration options allow specifying the queue type and exchange type when +Celery auto-creates missing queues. This is particularly useful for RabbitMQ users +who want to use quorum queues with auto-created queues. + +Configuration options: + +- :setting:`task_create_missing_queue_type`: Sets the queue type for auto-created + queues (e.g., ``quorum``, ``classic``) +- :setting:`task_create_missing_queue_exchange_type`: Sets the exchange type for + auto-created queues + +Example usage: + +.. code-block:: python + + app.conf.task_create_missing_queue_type = 'quorum' + +Contributed by `@ghirailghiro `_ in +`#9815 `_. + +What's Changed +~~~~~~~~~~~~~~ + +- Prepare for release: v5.6.0 (#10010) + +.. _version-5.6.0rc2: + +5.6.0rc2 +======== + +:release-date: 2025-11-22 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Remove Python 4.0 version condition for pytest dependencies (#9993) +- Sanitize broker URL in delayed delivery logs (avoid leaking credentials) (#9997) +- Don't fail task on timeout during cold shutdown (#9678) +- Add Py39-314t to CI (#9999) +- asynpool: Don't return from inside a finally block (#10000) +- Prepare for (pre) release: v5.6.0rc2 (#10005) + +.. _version-5.6.0rc1: + +5.6.0rc1 +======== + +:release-date: 2025-11-02 +:release-by: Tomer Nosrati + +Celery v5.6.0 Release Candidate 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- Add support for Django Connection pool (#9953) +- Pin tblib to ==3.1.0 (#9967) +- fix(worker): continue to attempt to bind other queues after a native delayed delivery binding failure has occurred (#9959) +- Handle UnpicklingError in persistent scheduler initialization (#9952) +- Bug Fix: Nested Chords Fail When Using django-celery-results with a Redis Backend (#9950) +- Add support pymongo 4.12 (#9665) +- Make tests compatible with pymongo >= 4.14 (#9968) +- tblib updated from 3.1.0 to 3.2.0 (#9970) +- Fix remaining function typing and docstring (#9971) +- Fix regex pattern in version parsing and remove duplicate entry in __all__ (#9978) +- Bump Kombu to v5.6.0 and removed <5.7 limit on kombu (#9981) +- Prepare for (pre) release: v5.6.0rc1 (#9982) + +.. _version-5.6.0b2: + +5.6.0b2 +======= + +:release-date: 2025-10-20 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 2 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- GitHub Actions: Test on Python 3.14 release candidate 2 (#9891) +- Update pypy to python 3.11 (#9896) +- Feature: Add support credential_provider to Redis Backend (#9879) +- Celery.timezone: try tzlocal.get_localzone() before using LocalTimezone (#9862) +- Run integration tests on Python 3.14 (#9903) +- Fix arithmetic overflow for MSSQL result backend (#9904) +- Add documentation for task_id param for apply_async function (#9906) +- Support redis client name (#9900) +- Bump Kombu to v5.6.0rc1 (#9918) +- Fix broker connection retry attempt counter in the error log (#9911) +- fix: restrict disable-prefetch feature to Redis brokers only (#9919) +- fix(): preserve group order in replaced signature (#9910) +- Remove Python 3.8 from CI workflow (#9930) +- Update default Python versions in integration tests (#9931) +- Update tox.ini to remove Python 3.8 (#9932) +- Remove Python 3.8 from Dockerfile (#9933) +- Update Python version requirement to 3.9 (#9935) +- Update pypy version from 3.10 to 3.11 in Dockerfile (#9934) +- Flake8 fixes (#9955) +- Remove test-pypy3.txt from Dockerfile dependencies (#9939) +- Remove backports.zoneinfo for Python 3.9 compatibility (#9956) +- Update pytest-cov version for Python compatibility (#9957) +- Update pytest-rerunfailures and pre-commit versions (#9958) +- Prepare for (pre) release: v5.6.0b2 (#9938) + +.. _version-5.6.0b1: + +5.6.0b1 +======= + +:release-date: 2025-09-15 +:release-by: Tomer Nosrati + +Celery v5.6.0 Beta 1 is now available for testing. +Please help us test this version and report any issues. + +What's Changed +~~~~~~~~~~~~~~ + +- docs: mention of json serializer recursive reference message size blowup (#5000) (#9743) +- docs: typo in canvas.rst (#9744) +- Makes _on_retry return a float as required to be used as errback on retry_over_time (#9741) +- Update canvas.rst doc calculation order for callback (#9758) +- Updated Blacksmith logo (#9763) +- Made the Sponsors logos link to their website (#9764) +- add missing cloudamqp logo (#9767) +- Improve sponsor visibility (#9768) +- fix: (#9773) task_id must not be empty with chain as body of a chord (#9774) +- Update setup.py to fix deprecation warning (#9771) +- Adds integration test for chord_unlock bug when routed to quorum/topic queue (#9766) +- Add xfail test for default queue/exchange fallback ignoring task_default_* settings (#9765) +- Add xfail test for RabbitMQ quorum queue global QoS race condition (#9770) +- fix: (#8786) time out when chord header fails with group body (#9788) +- Fix #9738 : Add root_id and parent_id to .apply() (#9784) +- Replace DelayedDelivery connection creation to use context manger (#9793) +- Fix #9794: Pydantic integration fails with __future__.annotations. (#9795) +- add go and rust implementation in docs (#9800) +- Fix memory leak in exception handling (Issue #8882) (#9799) +- Fix handlers docs (Issue #9787) (#9804) +- Remove importlib_metadata leftovers (#9791) +- Update timeout minutes for smoke tests CI (#9807) +- Revert "Remove dependency on `pycurl`" (#9620) +- Add Blacksmith Docker layer caching to all Docker builds (#9840) +- Bump Kombu to v5.6.0b1 (#9839) +- Disable pytest-xdist for smoke tests and increase retries (CI ONLY) (#9842) +- Fix Python 3.13 compatibility in events dumper (#9826) +- Dockerfile Build Optimizations (#9733) +- Migrated from useblacksmith/build-push-action@v1 to useblacksmith/setup-docker-builder@v1 in the CI (#9846) +- Remove incorrect example (#9854) +- Revert "Use Django DB max age connection setting" (#9824) +- Fix pending_result memory leak (#9806) +- Update python-package.yml (#9856) +- Bump Kombu to v5.6.0b2 (#9858) +- Refactor integration and smoke tests CI (#9855) +- Fix `AsyncResult.forget()` with couchdb backend method raises `TypeError: a bytes-like object is required, not 'str'` (#9865) +- Improve Docs for SQS Authentication (#9868) +- Added `.github/copilot-instructions.md` for GitHub Copilot (#9874) +- misc: credit removal (#9877) +- Choose queue type and exchange type when creating missing queues (fix #9671) (#9815) +- fix: prevent celery from hanging due to spawned greenlet errors in greenlet drainers (#9371) +- Feature/disable prefetch fixes (#9863) +- Add worker_eta_task_limit configuration to manage ETA task memory usage (#9853) +- Update runner version in Docker workflow (#9884) +- Prepare for (pre) release: v5.6.0b1 (#9890) diff --git a/docs/history/index.rst b/docs/history/index.rst index 22cd146a1f5..fc13c0a0125 100644 --- a/docs/history/index.rst +++ b/docs/history/index.rst @@ -13,6 +13,8 @@ version please visit :ref:`changelog`. .. toctree:: :maxdepth: 2 + whatsnew-5.6 + changelog-5.6 whatsnew-5.5 changelog-5.5 whatsnew-5.4 diff --git a/docs/history/whatsnew-4.3.rst b/docs/history/whatsnew-4.3.rst index 230d751c5f6..27de377998d 100644 --- a/docs/history/whatsnew-4.3.rst +++ b/docs/history/whatsnew-4.3.rst @@ -92,7 +92,6 @@ Artem Vasilyev Asif Saif Uddin (Auvi) aviadatsnyk Axel Haustant -Benjamin Pereto Bojan Jovanovic Brett Jackson Brett Randall diff --git a/docs/history/whatsnew-5.3.rst b/docs/history/whatsnew-5.3.rst index 4ccccb69224..174d31c4082 100644 --- a/docs/history/whatsnew-5.3.rst +++ b/docs/history/whatsnew-5.3.rst @@ -249,9 +249,9 @@ A switch have been made to zoneinfo for handling timezone data instead of pytz. Support for out-of-tree worker pool implementations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Prior to version 5.3, Celery had a fixed notion of the worker pool types it supports. -Celery v5.3.0 introduces the the possibility of an out-of-tree worker pool implementation. -This feature ensure that the current worker pool implementations consistently call into -BasePool._get_info(), and enhance it to report the work pool class in use via the +Celery v5.3.0 introduces the possibility of an out-of-tree worker pool implementation. +This feature ensures that the current worker pool implementations consistently call into +BasePool._get_info(), and enhances it to report the worker pool class in use via the "celery inspect stats" command. For example: $ celery -A ... inspect stats @@ -348,4 +348,3 @@ environment and is not safe for production use at the moment. - diff --git a/docs/history/whatsnew-5.5.rst b/docs/history/whatsnew-5.5.rst index 925629e8b69..120e3a3b5f3 100644 --- a/docs/history/whatsnew-5.5.rst +++ b/docs/history/whatsnew-5.5.rst @@ -358,11 +358,3 @@ actually needed, which can be useful in certain deployment scenarios where you w more control over database schema management. See :ref:`conf-database-result-backend` for complete documentation. - -Spawn Pool Option ------------------ - -Added a new ``spawn`` pool implementation. This pool uses Python's -``spawn`` start method when launching worker processes which is helpful -when libraries are not fork-safe (for example CUDA based frameworks). -Enable it with ``-P spawn`` on the command line. diff --git a/docs/history/whatsnew-5.6.rst b/docs/history/whatsnew-5.6.rst new file mode 100644 index 00000000000..a7ea216b80a --- /dev/null +++ b/docs/history/whatsnew-5.6.rst @@ -0,0 +1,286 @@ +.. _whatsnew-5.6: + +========================================= + What's new in Celery 5.6 (Recovery) +========================================= +:Author: Tomer Nosrati (``tomer.nosrati at gmail.com``). + +.. sidebar:: Change history + + What's new documents describe the changes in major versions, + we also have a :ref:`changelog` that lists the changes in bugfix + releases (0.0.x), while older series are archived under the :ref:`history` + section. + +Celery is a simple, flexible, and reliable distributed programming framework +to process vast amounts of messages, while providing operations with +the tools required to maintain a distributed system with python. + +It's a task queue with focus on real-time processing, while also +supporting task scheduling. + +Celery has a large and diverse community of users and contributors, +you should come join us :ref:`on IRC ` +or :ref:`our mailing-list `. + +.. note:: + + Following the problems with Freenode, we migrated our IRC channel to Libera Chat + as most projects did. + You can also join us using `Gitter `_. + + We're sometimes there to answer questions. We welcome you to join. + +To read more about Celery you should go read the :ref:`introduction `. + +While this version is **mostly** backward compatible with previous versions +it's important that you read the following section as this release +is a new major version. + +This version is officially supported on CPython 3.9, 3.10, 3.11, 3.12 and 3.13, +and is also supported on PyPy3.11+. + +.. _`website`: https://celery.readthedocs.io + +.. topic:: Table of Contents + + Make sure you read the important notes before upgrading to this version. + +.. contents:: + :local: + :depth: 3 + +Preface +======= + +.. note:: + + **This release contains fixes for many long standing bugs & stability issues. + We encourage our users to upgrade to this release as soon as possible.** + +The 5.6.0 release is a new feature release for Celery. + +Releases in the 5.x series are codenamed after songs of `Jon Hopkins `_. +This release has been codenamed `Recovery `_. + +This is the last version to support Python 3.9. +Support for Python 3.8 was removed after v5.6.0b1. + +*— Tomer Nosrati* + +Long Term Support Policy +------------------------ + +We no longer support Celery 4.x as we don't have the resources to do so. +If you'd like to help us, all contributions are welcome. + +Celery 5.x **is not** an LTS release. We will support it until the release +of Celery 6.x. + +We're in the process of defining our Long Term Support policy. +Watch the next "What's New" document for updates. + +Upgrading from Celery 4.x +========================= + +Step 1: Adjust your command line invocation +------------------------------------------- + +Celery 5.0 introduces a new CLI implementation which isn't completely backwards compatible. + +The global options can no longer be positioned after the sub-command. +Instead, they must be positioned as an option for the `celery` command like so:: + + celery --app path.to.app worker + +If you were using our :ref:`daemonizing` guide to deploy Celery in production, +you should revisit it for updates. + +Step 2: Update your configuration with the new setting names +------------------------------------------------------------ + +If you haven't already updated your configuration when you migrated to Celery 4.0, +please do so now. + +We elected to extend the deprecation period until 6.0 since +we did not loudly warn about using these deprecated settings. + +Please refer to the :ref:`migration guide ` for instructions. + +Step 3: Read the important notes in this document +------------------------------------------------- + +Make sure you are not affected by any of the important upgrade notes +mentioned in the :ref:`following section `. + +You should verify that none of the breaking changes in the CLI +do not affect you. Please refer to :ref:`New Command Line Interface ` for details. + +Step 4: Migrate your code to Python 3 +------------------------------------- + +Celery 5.x only supports Python 3. Therefore, you must ensure your code is +compatible with Python 3. + +If you haven't ported your code to Python 3, you must do so before upgrading. + +You can use tools like `2to3 `_ +and `pyupgrade `_ to assist you with +this effort. + +After the migration is done, run your test suite with Celery 5 to ensure +nothing has been broken. + +Step 5: Upgrade to Celery 5.6 +----------------------------- + +At this point you can upgrade your workers and clients with the new version. + +.. _v560-important: + +Important Notes +=============== + +Supported Python Versions +------------------------- + +The supported Python versions are: + +- CPython 3.9 +- CPython 3.10 +- CPython 3.11 +- CPython 3.12 +- CPython 3.13 +- PyPy3.11 (``pypy3``) + +Python 3.9 Support +------------------ + +Python 3.9 will reach EOL in October, 2025. + +Minimum Dependencies +-------------------- + +Kombu +~~~~~ + +Starting from Celery v5.6, the minimum required version is Kombu 5.6. + +Redis +~~~~~ + +redis-py 4.5.2 is the new minimum required version. + + +SQLAlchemy +~~~~~~~~~~ + +SQLAlchemy 1.4.x & 2.0.x is now supported in Celery v5.6. + +Billiard +~~~~~~~~ + +Minimum required version is now 4.2.4. + +Django +~~~~~~ + +Minimum django version is bumped to v2.2.28. +Also added --skip-checks flag to bypass django core checks. + +.. _v560-news: + +News +==== + +SQS: Reverted to ``pycurl`` from ``urllib3`` +-------------------------------------------- + +The switch from ``pycurl`` to ``urllib3`` for the SQS transport (introduced in +Celery 5.5.0 via Kombu) has been reverted due to critical issues affecting SQS +users. + +Security Fix: Broker Credential Leak Prevention +------------------------------------------------ + +Fixed a security issue where broker URLs containing passwords were being logged +in plaintext by the delayed delivery mechanism. Broker credentials are now +properly sanitized in all log output. + +Memory Leak Fixes +----------------- + +Two significant memory leaks have been fixed in this release: + +**Exception Handling Memory Leak**: Fixed a critical memory leak in task exception +handling that was particularly severe on Python 3.11+ due to enhanced traceback +data. The fix properly breaks reference cycles in tracebacks to allow garbage +collection. This resolves a long-standing issue that caused worker memory to grow +unbounded over time. + +**Pending Result Memory Leak**: Fixed a memory leak where ``AsyncResult`` +subscriptions were not being cleaned up when results were forgotten. This affected +users who frequently use ``AsyncResult.forget()`` in their workflows. + +ETA Task Memory Limit +--------------------- + +New configuration option to prevent out-of-memory crashes when workers fetch +large numbers of ETA or countdown tasks. Previously, workers could exhaust +available memory when the broker contained many scheduled tasks. + +Configuration option: + +- :setting:`worker_eta_task_limit`: Sets the maximum number of ETA tasks to hold + in worker memory at once (default: ``None``, unlimited) + +Example usage: + +.. code-block:: python + + app.conf.worker_eta_task_limit = 1000 + +Queue Type Selection for Auto-created Queues +-------------------------------------------- + +New configuration options allow specifying the queue type and exchange type when +Celery auto-creates missing queues. This is particularly useful for RabbitMQ users +who want to use quorum queues with auto-created queues. + +Configuration options: + +- :setting:`task_create_missing_queue_type`: Sets the queue type for auto-created + queues (e.g., ``quorum``, ``classic``) +- :setting:`task_create_missing_queue_exchange_type`: Sets the exchange type for + auto-created queues + +Example usage: + +.. code-block:: python + + app.conf.task_create_missing_queue_type = 'quorum' + +Django Connection Pool Support +------------------------------ + +Fixed an issue where Django applications using psycopg3 connection pooling would +experience ``psycopg_pool.PoolTimeout`` errors after worker forks. Celery now +properly closes Django's connection pools before forking, similar to how Django +itself handles this in its autoreload mechanism. + +Redis Backend Improvements +-------------------------- + +**Credential Provider Support**: Added the :setting:`redis_backend_credential_provider` +setting to the Redis backend. This enables integration with AWS ElastiCache using +IAM authentication and other credential provider mechanisms. + +**Client Name Support**: Added the :setting:`redis_client_name` setting to the Redis +backend, making it easier to identify Celery connections when monitoring Redis servers. + +Cold Shutdown Improvements +-------------------------- + +Fixed an issue where tasks would incorrectly fail with a timeout error during +cold shutdown. The worker now properly skips timeout failure handling during +the cold shutdown phase, allowing tasks to complete or be properly requeued. diff --git a/docs/includes/introduction.txt b/docs/includes/introduction.txt index 4184b38313a..17350312e5a 100644 --- a/docs/includes/introduction.txt +++ b/docs/includes/introduction.txt @@ -1,4 +1,4 @@ -:Version: 5.5.3 (immunity) +:Version: 5.6.3 (recovery) :Web: https://docs.celeryq.dev/en/stable/index.html :Download: https://pypi.org/project/celery/ :Source: https://github.com/celery/celery/ diff --git a/docs/internals/guide.rst b/docs/internals/guide.rst index bbec0a38c7a..731cacbaac4 100644 --- a/docs/internals/guide.rst +++ b/docs/internals/guide.rst @@ -267,7 +267,7 @@ Module Overview - celery.concurrency - Execution pool implementations (prefork, spawn, eventlet, gevent, solo, thread). + Execution pool implementations (prefork, eventlet, gevent, solo, thread). - celery.db diff --git a/docs/internals/protocol.rst b/docs/internals/protocol.rst index 72f461dc936..00d7b4b34e7 100644 --- a/docs/internals/protocol.rst +++ b/docs/internals/protocol.rst @@ -49,7 +49,8 @@ Definition 'argsrepr': str repr(args), 'kwargsrepr': str repr(kwargs), 'origin': str nodename, - 'replaced_task_nesting': int + 'replaced_task_nesting': int, + 'compression': string compression_method (optional; omitted when no compression is used, matches kombu compressor names such as 'zlib', 'bzip2', 'gzip'), } body = ( diff --git a/docs/reference/celery.rst b/docs/reference/celery.rst index 65c778cecd6..962fe0a1f35 100644 --- a/docs/reference/celery.rst +++ b/docs/reference/celery.rst @@ -126,7 +126,14 @@ and creating Celery applications. .. data:: on_after_finalize - Signal sent after app has been finalized. + Signal sent after the app has been finalized — that is, after all + pending task decorators have been evaluated, built-in tasks loaded, + and every task registered at that point has been bound to the app. + At this stage the task registry is initialized and stable enough to + import and inspect task objects reliably. + + See :meth:`~celery.Celery.finalize` for more details on what + finalization does. .. data:: on_after_fork diff --git a/docs/userguide/calling.rst b/docs/userguide/calling.rst index 63b8998f77f..b014357e2b6 100644 --- a/docs/userguide/calling.rst +++ b/docs/userguide/calling.rst @@ -56,6 +56,8 @@ The API defines a standard set of execution options, as well as three methods: - ``T.apply_async(expires=now + timedelta(days=2))`` expires in 2 days, set using :class:`~datetime.datetime`. + - ``T.apply_async(task_id=f'my_own_task_id')`` + sets the id of the task to my_own_task_id instead of a uuid that is normally generated Example ------- @@ -458,6 +460,13 @@ You can handle this error too: ... except add.OperationalError as exc: ... logger.exception('Sending task raised: %r', exc) +.. note:: + + With RabbitMQ, these errors only indicate the broker is unreachable. + Messages can still be silently dropped when the broker hits resource + limits. Enable ``confirm_publish`` in :setting:`broker_transport_options` + to detect this. + .. _calling-serializers: Serializers @@ -803,6 +812,24 @@ setting or by using the ``ignore_result`` option: If you'd like to store additional metadata about the task in the result backend set the :setting:`result_extended` setting to ``True``. +.. note:: + + ``result_extended`` controls what *Celery* includes as extended task metadata, + but it does not automatically add scheduler-specific metadata. + For example, some integrations (e.g. :pypi:`django-celery-beat` together with + :pypi:`django-celery-results`) may record the *periodic task name* in the result + backend only when the scheduler provides it as part of the published message. + + When you call tasks manually using ``apply_async``/``delay``, that periodic task + context is usually not present unless you add it explicitly (e.g. via message + headers/properties in ``apply_async`` options). For example: + + .. code-block:: python + + result = task.apply_async( + headers={"periodic_task_name": "task_name"}, + ) + .. seealso:: For more information on tasks, please see :ref:`guide-tasks`. diff --git a/docs/userguide/canvas.rst b/docs/userguide/canvas.rst index 82b0e1521b6..1383bdb9a25 100644 --- a/docs/userguide/canvas.rst +++ b/docs/userguide/canvas.rst @@ -137,6 +137,24 @@ creates partials: >>> partial.delay(4) # 4 + 2 >>> partial.apply_async((4,)) # same + .. note:: + + Additional args passed to ``delay``/``apply_async`` are **prepended** + to the signature args. Since ``add`` is commutative, the ordering may + not be obvious. A non-commutative task like + ``subtract(x, y) -> x - y`` makes this clear: + + .. code-block:: python + + @app.task + def subtract(x, y): + return x - y + + partial = subtract.s(10) # incomplete: second arg only + partial.delay(30) # -> subtract(30, 10) = 20 + Here ``delay(30)`` prepends ``30`` as the first argument, resulting + in ``subtract(30, 10)`` — not ``subtract(10, 30)``. + - Any keyword arguments added will be merged with the kwargs in the signature, with the new keyword arguments taking precedence: diff --git a/docs/userguide/configuration.rst b/docs/userguide/configuration.rst index d688d66df7e..0c93abd6cab 100644 --- a/docs/userguide/configuration.rst +++ b/docs/userguide/configuration.rst @@ -103,6 +103,8 @@ have been moved into a new ``task_`` prefix. ``CELERY_MONGODB_BACKEND_SETTINGS`` :setting:`mongodb_backend_settings` ``CELERY_EVENT_QUEUE_EXPIRES`` :setting:`event_queue_expires` ``CELERY_EVENT_QUEUE_TTL`` :setting:`event_queue_ttl` +``CELERY_EVENT_QUEUE_DURABLE`` :setting:`event_queue_durable` +``CELERY_EVENT_QUEUE_EXCLUSIVE`` :setting:`event_queue_exclusive` ``CELERY_EVENT_QUEUE_PREFIX`` :setting:`event_queue_prefix` ``CELERY_EVENT_SERIALIZER`` :setting:`event_serializer` ``CELERY_REDIS_DB`` :setting:`redis_db` @@ -112,6 +114,7 @@ have been moved into a new ``task_`` prefix. ``CELERY_REDIS_PASSWORD`` :setting:`redis_password` ``CELERY_REDIS_PORT`` :setting:`redis_port` ``CELERY_REDIS_BACKEND_USE_SSL`` :setting:`redis_backend_use_ssl` +``CELERY_REDIS_BACKEND_CREDENTIAL_PROVIDER`` :setting:`redis_backend_credential_provider` ``CELERY_RESULT_BACKEND`` :setting:`result_backend` ``CELERY_MAX_CACHED_RESULTS`` :setting:`result_cache_max` ``CELERY_MESSAGE_COMPRESSION`` :setting:`result_compression` @@ -134,6 +137,8 @@ have been moved into a new ``task_`` prefix. ``CELERY_ANNOTATIONS`` :setting:`task_annotations` ``CELERY_COMPRESSION`` :setting:`task_compression` ``CELERY_CREATE_MISSING_QUEUES`` :setting:`task_create_missing_queues` +``CELERY_CREATE_MISSING_QUEUE_TYPE`` :setting:`task_create_missing_queue_type` +``CELERY_CREATE_MISSING_QUEUE_EXCHANGE_TYPE`` :setting:`task_create_missing_queue_exchange_type` ``CELERY_DEFAULT_DELIVERY_MODE`` :setting:`task_default_delivery_mode` ``CELERY_DEFAULT_EXCHANGE`` :setting:`task_default_exchange` ``CELERY_DEFAULT_EXCHANGE_TYPE`` :setting:`task_default_exchange_type` @@ -170,6 +175,7 @@ have been moved into a new ``task_`` prefix. ``CELERYD_POOL_PUTLOCKS`` :setting:`worker_pool_putlocks` ``CELERYD_POOL_RESTARTS`` :setting:`worker_pool_restarts` ``CELERYD_PREFETCH_MULTIPLIER`` :setting:`worker_prefetch_multiplier` +``CELERYD_ETA_TASK_LIMIT`` :setting:`worker_eta_task_limit` ``CELERYD_ENABLE_PREFETCH_COUNT_REDUCTION``:setting:`worker_enable_prefetch_count_reduction` ``CELERYD_REDIRECT_STDOUTS`` :setting:`worker_redirect_stdouts` ``CELERYD_REDIRECT_STDOUTS_LEVEL`` :setting:`worker_redirect_stdouts_level` @@ -758,6 +764,7 @@ Can be one of the following: .. _`AzureBlockBlob`: https://azure.microsoft.com/en-us/services/storage/blobs/ .. _`S3`: https://aws.amazon.com/s3/ .. _`GCS`: https://cloud.google.com/storage/ +.. _`RedisCredentialProvider`: https://redis.readthedocs.io/en/stable/examples/connection_examples.html#Connecting-to-a-redis-instance-with-standard-credential-provider .. setting:: result_backend_always_retry @@ -988,6 +995,38 @@ strings (this is the part of the URI that comes after the ``db+`` prefix). .. _`Connection String`: http://www.sqlalchemy.org/docs/core/engines.html#database-urls +.. note:: + + If you are upgrading from Celery 5.6 or earlier, the ``date_done`` column + in ``celery_taskmeta`` and ``celery_tasksetmeta`` tables does not have a + database index. The built-in periodic task ``celery.backend_cleanup`` + queries on ``date_done`` to delete expired task results, so adding an + index significantly improves cleanup performance on large tables. + + Since SQLAlchemy's ``create_all()`` will not alter existing tables, you + will need to update your database schema. If you are using Alembic for + schema migrations, you can generate an empty revision and apply the + following operations: + + .. code-block:: python + + from alembic import op + + def upgrade(): + op.create_index('ix_celery_taskmeta_date_done', 'celery_taskmeta', ['date_done']) + op.create_index('ix_celery_tasksetmeta_date_done', 'celery_tasksetmeta', ['date_done']) + + def downgrade(): + op.drop_index('ix_celery_tasksetmeta_date_done', table_name='celery_tasksetmeta') + op.drop_index('ix_celery_taskmeta_date_done', table_name='celery_taskmeta') + + Otherwise, you can add the indexes manually using SQL: + + .. code-block:: sql + + CREATE INDEX ix_celery_taskmeta_date_done ON celery_taskmeta (date_done); + CREATE INDEX ix_celery_tasksetmeta_date_done ON celery_tasksetmeta (date_done); + .. setting:: database_create_tables_at_setup ``database_create_tables_at_setup`` @@ -1010,7 +1049,14 @@ Default: True by default. ``database_engine_options`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Default: ``{}`` (empty mapping). +Default: ``{'pool_pre_ping': True, 'pool_recycle': 3600}`` + +.. versionchanged:: 5.7 + + The default was changed from ``{}`` to include ``pool_pre_ping=True`` + and ``pool_recycle=3600`` for improved connection health handling. + This helps prevent stale connection errors such as + ``(OperationalError) (2006, 'MySQL server has gone away')``. To specify additional SQLAlchemy database engine options you can use the :setting:`database_engine_options` setting:: @@ -1018,6 +1064,9 @@ the :setting:`database_engine_options` setting:: # echo enables verbose logging from SQLAlchemy. app.conf.database_engine_options = {'echo': True} + # To disable the default pool health options: + app.conf.database_engine_options = {'pool_pre_ping': False, 'pool_recycle': None} + .. setting:: database_short_lived_sessions ``database_short_lived_sessions`` @@ -1201,6 +1250,12 @@ This is a dict supporting the following keys: constructor. See the :mod:`pymongo` docs to see a list of arguments supported. +.. note:: + + With pymongo>=4.14, options are case-sensitive when they were previously + case-insensitive. See :class:`~pymongo.mongo_client.MongoClient` to + determine the correct case. + .. _example-mongodb-result-config: Example configuration @@ -1339,6 +1394,19 @@ the form of a dictionary. The valid key-value pairs are the same as the ones mentioned in the ``redis`` sub-section under :setting:`broker_use_ssl`. +.. setting:: redis_backend_credential_provider + +.. versionadded:: 5.6 + +``redis_backend_credential_provider`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default: Disabled. + +The Redis backend supports credential provider. This value must be set in +the form of a class path string or a class instance. e.g. ``mymodule.myfile.myclass`` +check more details in `RedisCredentialProvider`_ doc. + .. setting:: redis_max_connections ``redis_max_connections`` @@ -1400,6 +1468,18 @@ Default: :const:`False` Socket TCP keepalive to keep connections healthy to the Redis server, used by the redis result backend. +.. setting:: redis_client_name + +``redis_client_name`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.6 + +Default: :const:`None` + +Sets the client name for Redis connections used by the result backend. +This can help identify connections in Redis monitoring tools. + .. _conf-cassandra-result-backend: Cassandra/AstraDB backend settings @@ -2619,6 +2699,51 @@ If enabled (default), any queues specified that aren't defined in :setting:`task_queues` will be automatically created. See :ref:`routing-automatic`. +.. setting:: task_create_missing_queue_type + +``task_create_missing_queue_type`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionadded:: 5.6 + +Default: ``"classic"`` + +When Celery needs to declare a queue that doesn’t exist (i.e., when +``task_create_missing_queues`` is enabled), this setting defines what type +of RabbitMQ queue to create. + +- ``"classic"`` (default): declares a standard classic queue. +- ``"quorum"``: declares a RabbitMQ quorum queue (adds ``x-queue-type: quorum``). + +.. setting:: task_create_missing_queue_exchange_type + +``task_create_missing_queue_exchange_type`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionadded:: 5.6 + +Default: ``None`` + +If this option is None or the empty string (the default), Celery leaves the +exchange exactly as returned by your :attr:`app.amqp.Queues.autoexchange` +hook. + +You can set this to a specific exchange type, such as ``"direct"``, ``"topic"``, or +``"fanout"``, to create the missing queue with that exchange type. + +.. tip:: + +Combine this setting with task_create_missing_queue_type = "quorum" +to create quorum queues bound to a topic exchange, for example:: + + app.conf.task_create_missing_queues=True + app.conf.task_create_missing_queue_type="quorum" + app.conf.task_create_missing_queue_exchange_type="topic" + +.. note:: + +Like the queue-type setting above, this option does not affect queues +that you define explicitly in :setting:`task_queues`; it applies only to +queues created implicitly at runtime. + .. setting:: task_default_queue ``task_default_queue`` @@ -3040,6 +3165,14 @@ won't retry forever if the broker isn't available at the first task execution): broker_transport_options = {'max_retries': 5} +Example enabling publisher confirms (supported by the ``pyamqp`` transport). +Without this, messages can be silently dropped when the broker hits resource +limits: + +.. code-block:: python + + broker_transport_options = {'confirm_publish': True} + .. _conf-worker: Worker @@ -3140,20 +3273,54 @@ early acknowledgments, enable :setting:`worker_disable_prefetch`. When this option is enabled the worker only fetches a task from the broker when one of its processes is available. +.. note:: + + This feature is currently only supported when using Redis as the broker. + You can also enable this via the :option:`--disable-prefetch ` command line flag. For more on prefetching, read :ref:`optimizing-prefetch-limit` -.. note:: +.. setting:: worker_eta_task_limit + +``worker_eta_task_limit`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 5.6 + +Default: No limit (None). - Tasks with ETA/countdown aren't affected by prefetch limits. +The maximum number of ETA/countdown tasks that a worker can hold in memory at once. +When this limit is reached, the worker will not receive new tasks from the broker +until some of the existing ETA tasks are executed. +This setting helps prevent memory exhaustion when a queue contains a large number +of tasks with ETA/countdown values, as these tasks are held in memory until their +execution time. Without this limit, workers may fetch thousands of ETA tasks into +memory, potentially causing out-of-memory issues. + +.. note:: + + Tasks with ETA/countdown are fetched into memory and scheduled on an internal + timer, so they are not constrained by the per-process prefetch window derived + from :setting:`worker_prefetch_multiplier` in the same way as immediately + executed tasks. This is why ``--prefetch-multiplier=1`` can appear to have no + effect when many ETA/countdown tasks are present. + + :setting:`worker_eta_task_limit` configures the maximum number of ETA/countdown + tasks a worker will hold in memory and also sets an overall cap on + unacknowledged messages via kombu's QoS ``max_prefetch``. If the prefetch count + implied by :setting:`worker_prefetch_multiplier` would exceed this cap, the + worker will stop consuming new messages until previously received tasks have + been acknowledged. .. setting:: worker_disable_prefetch ``worker_disable_prefetch`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. versionadded:: 5.6 + Default: ``False``. When enabled, a worker will only consume messages from the broker when it @@ -3161,6 +3328,12 @@ has an available process to execute them. This disables prefetching while still using early acknowledgments, ensuring that tasks are fairly distributed between workers. +.. note:: + + This feature is currently only supported when using Redis as the broker. + Using this setting with other brokers will result in a warning and the + setting will be ignored. + .. setting:: worker_enable_prefetch_count_reduction ``worker_enable_prefetch_count_reduction`` @@ -3430,6 +3603,33 @@ Default: 60.0 seconds. Expiry time in seconds (int/float) for when after a monitor clients event queue will be deleted (``x-expires``). +.. setting:: event_queue_durable + +``event_queue_durable`` +~~~~~~~~~~~~~~~~~~~~~~~~ +:transports supported: ``amqp`` +.. versionadded:: 5.6 + +Default: ``False`` + +If enabled, the event receiver's queue will be marked as *durable*, meaning it will survive broker restarts. + +.. setting:: event_queue_exclusive + +``event_queue_exclusive`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ +:transports supported: ``amqp`` +.. versionadded:: 5.6 + +Default: ``False`` + +If enabled, the event queue will be *exclusive* to the current connection and automatically deleted when the connection closes. + +.. warning:: + + You **cannot** set both ``event_queue_durable`` and ``event_queue_exclusive`` to ``True`` at the same time. + Celery will raise an :exc:`ImproperlyConfigured` error if both are set. + .. setting:: event_queue_prefix ``event_queue_prefix`` @@ -3586,6 +3786,31 @@ Name of the control command exchange. .. _conf-logging: +.. setting:: control_queue_durable + +``control_queue_durable`` +------------------------- + +- **Default:** ``False`` +- **Type:** ``bool`` + +If set to ``True``, the control exchange and queue will be durable — they will survive broker restarts. + +.. setting:: control_queue_exclusive + +``control_queue_exclusive`` +--------------------------- + +- **Default:** ``False`` +- **Type:** ``bool`` + +If set to ``True``, the control queue will be exclusive to a single connection. This is generally not recommended in distributed environments. + +.. warning:: + + Setting both ``control_queue_durable`` and ``control_queue_exclusive`` to ``True`` is not supported and will raise an error. + + Logging ------- diff --git a/docs/userguide/extending.rst b/docs/userguide/extending.rst index ea8c0462598..32c28620376 100644 --- a/docs/userguide/extending.rst +++ b/docs/userguide/extending.rst @@ -220,21 +220,6 @@ Attributes class WorkerStep(bootsteps.StartStopStep): requires = ('celery.worker.autoscaler:Autoscaler',) -.. _extending-worker-autoreloader: - -.. attribute:: autoreloader - - :class:`~celery.worker.autoreloder.Autoreloader` used to automatically - reload use code when the file-system changes. - - This is only defined if the ``autoreload`` argument is enabled. - Your worker bootstep must require the `Autoreloader` bootstep to use this; - - .. code-block:: python - - class WorkerStep(bootsteps.StartStopStep): - requires = ('celery.worker.autoreloader:Autoreloader',) - Example worker bootstep ----------------------- diff --git a/docs/userguide/monitoring.rst b/docs/userguide/monitoring.rst index b542633ec9d..361cfe06172 100644 --- a/docs/userguide/monitoring.rst +++ b/docs/userguide/monitoring.rst @@ -460,6 +460,18 @@ The default queue is named `celery`. To get all available queues, invoke: hosts), but this won't affect the monitoring events used by for example Flower as Redis pub/sub commands are global rather than database based. +.. _monitoring-prometheus: + +Prometheus +========= + +While Prometheus monitoring is not a native part of Celery, +you can easily monitor your Celery workers using Prometheus via Flower. +Flower also provides pre-made Grafana dashboards to easily graph the amount +of tasks, workers and other instrumental statistics. + +To set up Prometheus, refer to the Flower documentation: https://flower.readthedocs.io/en/latest/prometheus-integration.html + .. _monitoring-munin: Munin @@ -814,3 +826,24 @@ worker-offline :signature: ``worker-offline(hostname, timestamp, freq, sw_ident, sw_ver, sw_sys)`` The worker has disconnected from the broker. + +Mailbox Configuration (Advanced) +-------------------------------- + +Celery uses `kombu.pidbox.Mailbox` internally to send control and broadcast commands +to workers. + +.. versionadded:: Kombu 5.6.0 + +Advanced users can configure the behavior of this mailbox by customizing how it is created. +The following parameters are now supported by `Mailbox`: + +- ``durable`` (default: ``False``): If set to ``True``, the control exchanges will survive broker restarts. +- ``exclusive`` (default: ``False``): If set to ``True``, the exchanges will be usable by only one connection. + +.. warning:: + + Setting both ``durable=True`` and ``exclusive=True`` is not permitted and will + raise an error, as these two options are mutually incompatible in AMQP. + +See :setting:`event_queue_durable` and :setting:`event_queue_exclusive` for advanced configuration. diff --git a/docs/userguide/optimizing.rst b/docs/userguide/optimizing.rst index 42cfdda33ad..7cc7b635751 100644 --- a/docs/userguide/optimizing.rst +++ b/docs/userguide/optimizing.rst @@ -186,7 +186,8 @@ prefetching by enabling :setting:`worker_disable_prefetch`. With this setting the worker fetches a new task only when an execution slot is free, preventing tasks from waiting behind long running ones on busy workers. This can also be set from the command line using -:option:`--disable-prefetch `. +:option:`--disable-prefetch `. This feature +is currently only supported when using Redis as the broker. Memory Usage ------------ diff --git a/docs/userguide/routing.rst b/docs/userguide/routing.rst index a5d58755427..f88d64b0f29 100644 --- a/docs/userguide/routing.rst +++ b/docs/userguide/routing.rst @@ -276,7 +276,7 @@ This means that even though there are 10 (0-9) priority levels, these are consolidated into 4 levels by default to save resources. This means that a queue named celery will really be split into 4 queues. -The highest priority queue will be named celery, and the the other queues will +The highest priority queue will be named celery, and the other queues will have a separator (by default `\x06\x16`) and their priority number appended to the queue name. diff --git a/docs/userguide/signals.rst b/docs/userguide/signals.rst index 7aeea8adbf8..28bc988c481 100644 --- a/docs/userguide/signals.rst +++ b/docs/userguide/signals.rst @@ -239,6 +239,12 @@ Provides arguments: Detailed exception information, including traceback (a :class:`billiard.einfo.ExceptionInfo` object). +.. note:: + + Only the ``request`` argument is guaranteed to be provided in all cases. + The ``reason`` and ``einfo`` arguments may be ``None`` or not provided + in certain scenarios, such as when a task is cancelled and retried. + Signal handlers should not assume these arguments are always present. .. signal:: task_success @@ -820,22 +826,25 @@ It can be used to add additional command-line arguments to the .. code-block:: python - from celery import Celery - from celery import signals - from celery.bin.base import Option + from celery import Celery, signals + from click import Option app = Celery() + + # Celery 5.0+ uses click for its command-line interface. + # Use click.option to add new command-line arguments. app.user_options['preload'].add(Option( - '--monitoring', action='store_true', + ('--monitoring',), is_flag=True, help='Enable our external monitoring utility, blahblah', )) @signals.user_preload_options.connect def handle_preload_options(options, **kwargs): - if options['monitoring']: + if options.get('monitoring'): enable_monitoring() + Sender is the :class:`~celery.bin.base.Command` instance, and the value depends on the program that was called (e.g., for the umbrella command it'll be a :class:`~celery.bin.celery.CeleryCommand`) object). diff --git a/docs/userguide/tasks.rst b/docs/userguide/tasks.rst index 3dfdbd58093..0a29e0982d9 100644 --- a/docs/userguide/tasks.rst +++ b/docs/userguide/tasks.rst @@ -1042,6 +1042,13 @@ General If :const:`True`, errors will be stored even if the task is configured to ignore results. + .. versionchanged:: 5.7 + Previously, if the ``ignore_result`` key was missing from the request + message, ``store_errors`` would default to ``True``, ignoring the + task's own ``ignore_result`` setting. The worker now correctly + falls back to ``Task.ignore_result`` when no per-request override + is present. + .. attribute:: Task.serializer A string identifying the default serialization @@ -1596,7 +1603,8 @@ The following diagram shows the exact order of execution: │ 4. on_success() OR ← Outcome-specific handler │ │ on_retry() OR │ │ │ on_failure() │ │ - │ 5. after_return() ← Always runs last │ + │ 5. after_return() ← Runs last on terminal states │ + │ (skipped for RETRY/REJECTED/IGNORED) │ └───────────────────────────────────────────────────────────────┘ .. important:: @@ -1606,7 +1614,9 @@ The following diagram shows the exact order of execution: - All handlers run in the **same worker process** as your task - ``before_start`` **blocks** the task - ``run()`` won't start until it completes - Result backend is updated **before** ``on_success``/``on_failure`` - other clients can see the task as finished while handlers are still running - - ``after_return`` **always** executes, regardless of task outcome + - ``after_return`` executes when the task reaches a terminal state. + It does not run for ``RETRY``, ``REJECTED``, or ``IGNORED``. If you need + a hook that fires on every attempt, use the :signal:`task_postrun` signal. Available handlers ~~~~~~~~~~~~~~~~~~ @@ -1687,8 +1697,13 @@ Available handlers Handler called after the task returns. .. note:: - Executes **after** ``on_success``/``on_retry``/``on_failure``. This is the - final hook in the task lifecycle and **always** runs, regardless of outcome. + Executes after the outcome-specific handler when the task reaches a + terminal state. + + In practice, this means it runs after ``on_success`` or ``on_failure``. + It is not executed for ``RETRY``, ``REJECTED``, or ``IGNORED`` states. + If a hook is needed for every attempt, consider using the + :signal:`task_postrun` signal. :param status: Current task state. :param retval: Task return value/exception. diff --git a/docs/userguide/testing.rst b/docs/userguide/testing.rst index 5b2a5761818..1a7f353830c 100644 --- a/docs/userguide/testing.rst +++ b/docs/userguide/testing.rst @@ -121,10 +121,9 @@ use in your integration (or unit) test suites. Enabling -------- -Celery initially ships the plugin in a disabled state, to enable it you can either: +Celery initially ships the plugin in a disabled state. To enable it, you can either: * ``pip install celery[pytest]`` - * ``pip install pytest-celery`` * or add an environment variable ``PYTEST_PLUGINS=celery.contrib.pytest`` * or add ``pytest_plugins = ("celery.contrib.pytest", )`` to your root conftest.py diff --git a/docs/userguide/workers.rst b/docs/userguide/workers.rst index 01d6491d72b..024b3aa7078 100644 --- a/docs/userguide/workers.rst +++ b/docs/userguide/workers.rst @@ -108,7 +108,7 @@ Worker Shutdown We will use the terms *Warm, Soft, Cold, Hard* to describe the different stages of worker shutdown. The worker will initiate the shutdown process when it receives the :sig:`TERM` or :sig:`QUIT` signal. -The :sig:`INT` (Ctrl-C) signal is also handled during the shutdown process and always triggers the +The :sig:`INT` (Ctrl-C) signal is also handled during the shutdown process and always triggers the next stage of the shutdown process. .. _worker-warm-shutdown: @@ -126,6 +126,12 @@ and will call :func:`WorkController.stop() =82.0.0, see +# https://github.com/apache/cassandra-python-driver/pull/1268 +setuptools<82.0.0 diff --git a/requirements/default.txt b/requirements/default.txt index fc85b911128..eddb0ca7229 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -1,9 +1,10 @@ billiard>=4.2.1,<5.0 -kombu>=5.5.2,<5.6 +kombu>=5.6.0 vine>=5.1.0,<6.0 click>=8.1.2,<9.0 click-didyoumean>=0.3.0 click-repl>=0.2.0 click-plugins>=1.1.1 -backports.zoneinfo[tzdata]>=0.2.1; python_version < '3.9' python-dateutil>=2.8.2 +exceptiongroup>=1.3.0; python_version < '3.11' +tzlocal diff --git a/requirements/dev.txt b/requirements/dev.txt index fae13c00951..44d852a369c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,4 +2,4 @@ git+https://github.com/celery/py-amqp.git git+https://github.com/celery/kombu.git git+https://github.com/celery/billiard.git vine>=5.0.0 -isort==5.13.2 +isort==7.0.0 diff --git a/requirements/extras/auth.txt b/requirements/extras/auth.txt index e9a03334287..ed207ea0cdb 100644 --- a/requirements/extras/auth.txt +++ b/requirements/extras/auth.txt @@ -1 +1 @@ -cryptography==44.0.2 +cryptography==46.0.5 diff --git a/requirements/extras/elasticsearch.txt b/requirements/extras/elasticsearch.txt index 58cdcae1836..5362f230427 100644 --- a/requirements/extras/elasticsearch.txt +++ b/requirements/extras/elasticsearch.txt @@ -1,2 +1,2 @@ -elasticsearch<=8.17.2 -elastic-transport<=8.17.1 +elasticsearch<=9.3.0 +elastic-transport<=9.2.1 diff --git a/requirements/extras/gcs.txt b/requirements/extras/gcs.txt index 7a724e51b15..64d2ff93e10 100644 --- a/requirements/extras/gcs.txt +++ b/requirements/extras/gcs.txt @@ -1,3 +1,5 @@ google-cloud-storage>=2.10.0 -google-cloud-firestore==2.20.1 -grpcio==1.67.0 +grpcio==1.76.0 +google-cloud-firestore==2.23.0 + + diff --git a/requirements/extras/pydantic.txt b/requirements/extras/pydantic.txt index 29ac1fa96c9..99167f63c19 100644 --- a/requirements/extras/pydantic.txt +++ b/requirements/extras/pydantic.txt @@ -1 +1,2 @@ -pydantic>=2.4 +pydantic>=2.4 ; python_version < "3.14" +pydantic>=2.12.0a1 ; python_version >= "3.14" diff --git a/requirements/extras/pytest.txt b/requirements/extras/pytest.txt index 01fe3ab8c5e..0dff7152678 100644 --- a/requirements/extras/pytest.txt +++ b/requirements/extras/pytest.txt @@ -1 +1 @@ -pytest-celery[all]>=1.2.0,<1.3.0 +pytest-celery[all]>=1.3.0 diff --git a/requirements/extras/sqs.txt b/requirements/extras/sqs.txt index a7be017ff2f..78ba57ff78c 100644 --- a/requirements/extras/sqs.txt +++ b/requirements/extras/sqs.txt @@ -1,3 +1,5 @@ boto3>=1.26.143 +pycurl>=7.43.0.5,<7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version < "3.9" +pycurl>=7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version >= "3.9" urllib3>=1.26.16 kombu[sqs]>=5.5.0 diff --git a/requirements/extras/tblib.txt b/requirements/extras/tblib.txt index 5a837d19198..81d957704c6 100644 --- a/requirements/extras/tblib.txt +++ b/requirements/extras/tblib.txt @@ -1,2 +1 @@ -tblib>=1.5.0;python_version>='3.8.0' -tblib>=1.3.0;python_version<'3.8.0' +tblib==3.2.2 diff --git a/requirements/pkgutils.txt b/requirements/pkgutils.txt index eefe5d34af0..cb8932c8439 100644 --- a/requirements/pkgutils.txt +++ b/requirements/pkgutils.txt @@ -8,4 +8,4 @@ sphinx2rst>=1.0 # Disable cyanide until it's fully updated. # cyanide>=1.0.1 bumpversion==0.6.0 -pyperclip==1.9.0 +pyperclip==1.11.0 diff --git a/requirements/test-ci-base.txt b/requirements/test-ci-base.txt index b5649723471..a25e4f7b130 100644 --- a/requirements/test-ci-base.txt +++ b/requirements/test-ci-base.txt @@ -1,5 +1,4 @@ -pytest-cov==5.0.0; python_version<"3.9" -pytest-cov==6.0.0; python_version>="3.9" +pytest-cov==7.0.0; python_version>="3.9" pytest-github-actions-annotate-failures==0.3.0 -r extras/redis.txt -r extras/sqlalchemy.txt diff --git a/requirements/test-ci-default.txt b/requirements/test-ci-default.txt index e689866e245..78994fa8e45 100644 --- a/requirements/test-ci-default.txt +++ b/requirements/test-ci-default.txt @@ -21,4 +21,5 @@ git+https://github.com/celery/kombu.git # SQS dependencies other than boto -urllib3>=1.26.16 +pycurl>=7.43.0.5,<7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version < "3.9" +pycurl>=7.45.4; sys_platform != 'win32' and platform_python_implementation=="CPython" and python_version >= "3.9" diff --git a/requirements/test-integration.txt b/requirements/test-integration.txt index 50f5fdd9dcf..b915556332f 100644 --- a/requirements/test-integration.txt +++ b/requirements/test-integration.txt @@ -2,5 +2,6 @@ -r extras/azureblockblob.txt -r extras/auth.txt -r extras/memcache.txt +-r extras/django.txt pytest-rerunfailures>=11.1.2 git+https://github.com/celery/kombu.git diff --git a/requirements/test.txt b/requirements/test.txt index 527d975f617..cdba2da75f9 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,20 +1,19 @@ -pytest==8.3.5 -pytest-celery[all]>=1.2.0,<1.3.0 -pytest-rerunfailures>=14.0,<15.0; python_version >= "3.8" and python_version < "3.9" -pytest-rerunfailures>=15.0; python_version >= "3.9" and python_version < "4.0" -pytest-subtests<0.14.0; python_version < "3.9" +pytest==8.4.2 +pytest-celery[all]>=1.3.0 +pytest-rerunfailures>=15.0; python_version >= "3.9" pytest-subtests>=0.14.1; python_version >= "3.9" -pytest-timeout==2.3.1 +pytest-timeout==2.4.0 pytest-click==1.1.0 pytest-order==1.3.0 boto3>=1.26.143 moto>=4.1.11,<5.1.0 -# typing extensions -mypy==1.14.1; platform_python_implementation=="CPython" -pre-commit>=3.5.0,<3.8.0; python_version < '3.9' +# type checking +mypy==1.19.1; platform_python_implementation=="CPython" pre-commit>=4.0.1; python_version >= '3.9' -r extras/yaml.txt -r extras/msgpack.txt -r extras/mongodb.txt -r extras/gcs.txt -r extras/pydantic.txt +-r extras/azureblockblob.txt +-r extras/gevent.txt diff --git a/setup.cfg b/setup.cfg index a74a438d952..775ec0cc776 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,6 +34,7 @@ per-file-ignores = [bdist_rpm] requires = backports.zoneinfo>=0.2.1;python_version<'3.9' tzdata>=2022.7 + tzlocal billiard >=4.1.0,<5.0 kombu >= 5.3.4,<6.0.0 diff --git a/setup.py b/setup.py index aa10be2d3de..f9074260871 100755 --- a/setup.py +++ b/setup.py @@ -147,7 +147,7 @@ def long_description(): license='BSD-3-Clause', platforms=['any'], install_requires=install_requires(), - python_requires=">=3.8", + python_requires=">=3.9", tests_require=reqs('test.txt'), extras_require=extras_require(), include_package_data=True, @@ -170,7 +170,6 @@ def long_description(): "Framework :: Celery", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/t/integration/conftest.py b/t/integration/conftest.py index 2383cb2d9b6..1e05302b75e 100644 --- a/t/integration/conftest.py +++ b/t/integration/conftest.py @@ -8,6 +8,7 @@ from celery.contrib.pytest import celery_app, celery_session_worker from celery.contrib.testing.manager import Manager +from celery.exceptions import TimeoutError from t.integration.tasks import get_redis_connection # we have to import the pytest plugin fixtures here, @@ -20,9 +21,25 @@ TEST_BROKER = os.environ.get('TEST_BROKER', 'pyamqp://') TEST_BACKEND = os.environ.get('TEST_BACKEND', 'redis://') +RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) + + +def is_retryable_exception(exc): + return isinstance(exc, RETRYABLE_EXCEPTIONS) + + +_flaky = pytest.mark.flaky(reruns=5, reruns_delay=1, cause=is_retryable_exception) +_timeout = pytest.mark.timeout(timeout=300) + + +def flaky(fn): + return _timeout(_flaky(fn)) + + __all__ = ( 'celery_app', 'celery_session_worker', + 'flaky', 'get_active_redis_channels', ) diff --git a/t/integration/tasks.py b/t/integration/tasks.py index ff823b96cbc..4bd874a8882 100644 --- a/t/integration/tasks.py +++ b/t/integration/tasks.py @@ -6,7 +6,7 @@ from celery import Signature, Task, chain, chord, group, shared_task from celery.canvas import signature -from celery.exceptions import SoftTimeLimitExceeded +from celery.exceptions import Reject, SoftTimeLimitExceeded from celery.utils.log import get_task_logger LEGACY_TASKS_DISABLED = True @@ -522,6 +522,27 @@ def replace_with_stamped_task(self: StampedTaskOnReplace, replace_with=None): self.replace(signature(replace_with)) +@shared_task(bind=True, acks_late=True) +def store_success_then_reject(self): + """First delivery: store SUCCESS manually, then Reject to trigger redelivery. + Second delivery: dedup finds SUCCESS, dispatches chain.""" + from celery.backends.base import states + if not self.request.delivery_info.get('redelivered'): + self.backend.store_result(self.request.id, 'first-pass', states.SUCCESS) + raise Reject(requeue=True) + # When dedup is enabled the fast-path intercepts before reaching here, + # so 'dedup-pass' is only returned when dedup is disabled. + return 'dedup-pass' + + +@shared_task(bind=True, acks_late=True) +def reject_then_succeed(self): + """First delivery: Reject(requeue=True). Second delivery: succeed normally.""" + if not self.request.delivery_info.get('redelivered'): + raise Reject(requeue=True) + return 'second-pass' + + @shared_task(soft_time_limit=2, time_limit=1) def soft_time_limit_must_exceed_time_limit(): pass diff --git a/t/integration/test_canvas.py b/t/integration/test_canvas.py index d7b47362440..b1daaae8619 100644 --- a/t/integration/test_canvas.py +++ b/t/integration/test_canvas.py @@ -6,7 +6,6 @@ from time import monotonic, sleep import pytest -import pytest_subtests # noqa from celery import chain, chord, group, signature from celery.backends.base import BaseKeyValueStoreBackend @@ -16,7 +15,7 @@ from celery.signals import before_task_publish, task_received from . import tasks -from .conftest import TEST_BACKEND, check_for_logs, get_active_redis_channels, get_redis_connection +from .conftest import TEST_BACKEND, check_for_logs, flaky, get_active_redis_channels, get_redis_connection from .tasks import (ExpectedException, StampOnReplace, add, add_chord_to_chord, add_replaced, add_to_all, add_to_all_to_chord, build_chain_inside_task, collect_ids, delayed_sum, delayed_sum_with_soft_guard, errback_new_style, errback_old_style, fail, fail_replaced, identity, @@ -25,22 +24,8 @@ replace_with_stamped_task, retry_once, return_exception, return_priority, second_order_replace1, tsum, write_to_file_and_return_int, xsum) -RETRYABLE_EXCEPTIONS = (OSError, ConnectionError, TimeoutError) - - -def is_retryable_exception(exc): - return isinstance(exc, RETRYABLE_EXCEPTIONS) - - TIMEOUT = 60 -_flaky = pytest.mark.flaky(reruns=5, reruns_delay=1, cause=is_retryable_exception) -_timeout = pytest.mark.timeout(timeout=300) - - -def flaky(fn): - return _timeout(_flaky(fn)) - def await_redis_echo(expected_msgs, redis_key="redis-echo", timeout=TIMEOUT): """ @@ -441,6 +426,48 @@ def test_chain_of_chords_with_two_tasks(self, manager): res = c() assert res.get(timeout=TIMEOUT) == 12 + @flaky + def test_chain_of_explicit_chords(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + c1 = chain( + chord(group(add.si(1, 0), add.si(1, 0)), tsum.s()), + chord(group(add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(0), add.s(0)), tsum.s()), + ) + c2 = chain( + chord(group(add.s(10), add.s(10)), tsum.s()), + chord(group(add.s(0), add.s(0)), tsum.s()), + chord(group(add.s(1), add.s(1)), tsum.s()), + ) + c = c1 | c2 + res = c() + assert res.get(timeout=TIMEOUT) == 178 + + @flaky + def test_chain_of_nine_chords(self, manager): + try: + manager.app.backend.ensure_chords_allowed() + except NotImplementedError as e: + raise pytest.skip(e.args[0]) + + c = chain( + chord(group(add.si(1, 0), add.si(1, 0), add.si(1, 0)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(1), add.s(1), add.s(1)), tsum.s()), + chord(group(add.s(0), add.s(0), add.s(0)), tsum.s()), + ) + res = c() + assert res.get(timeout=TIMEOUT) == 29520 + @flaky def test_chain_of_a_chord_and_a_group_with_two_tasks(self, manager): try: @@ -1575,6 +1602,13 @@ def test_group_child_replaced_with_chain_last(self, manager): res_obj = orig_sig.delay() assert res_obj.get(timeout=TIMEOUT) == [42, 1337] + def test_task_replace_with_group_preserves_group_order(self, manager): + if manager.app.conf.result_backend.startswith("rpc"): + raise pytest.skip("RPC result backend does not support replacing with a group") + orig_sig = group([add_to_all.s([2, 1], 1), add_to_all.s([4, 3], 1)] * 10) + res_obj = orig_sig.delay() + assert res_obj.get(timeout=TIMEOUT) == [[3, 2], [5, 4]] * 10 + def assert_ids(r, expected_value, expected_root_id, expected_parent_id): root_id, parent_id, value = r.get(timeout=TIMEOUT) diff --git a/t/integration/test_dedup_chain_dispatch.py b/t/integration/test_dedup_chain_dispatch.py new file mode 100644 index 00000000000..013288cc589 --- /dev/null +++ b/t/integration/test_dedup_chain_dispatch.py @@ -0,0 +1,87 @@ +"""Integration tests for chain/callback dispatch on the dedup fast-path. + +When ``worker_deduplicate_successful_tasks=True`` and +``task_acks_late=True``, a redelivered task that hits the dedup +fast-path in ``trace.py`` must still dispatch its chain and link +callbacks. + +See https://github.com/celery/celery/issues/9835 +""" + +import pytest + +from celery import chain +from celery.contrib.testing.worker import start_worker +from celery.result import AsyncResult + +from .conftest import flaky +from .tasks import add, identity, reject_then_succeed, store_success_then_reject + +TIMEOUT = 60 + + +@pytest.fixture() +def dedup_worker(celery_session_app): + """Solo worker with dedup enabled. + + Temporarily enables ``worker_deduplicate_successful_tasks`` on the + session app, starts a solo worker, and restores the original + setting on teardown. + """ + if not celery_session_app.backend.persistent: + raise pytest.skip('Requires a persistent result backend.') + + orig_dedup = celery_session_app.conf.worker_deduplicate_successful_tasks + orig_acks_late = celery_session_app.conf.task_acks_late + celery_session_app.conf.worker_deduplicate_successful_tasks = True + celery_session_app.conf.task_acks_late = True + + try: + with start_worker( + celery_session_app, + pool='solo', + concurrency=1, + perform_ping_check=False, + shutdown_timeout=TIMEOUT, + ) as worker: + yield worker + finally: + celery_session_app.conf.worker_deduplicate_successful_tasks = orig_dedup + celery_session_app.conf.task_acks_late = orig_acks_late + + +class test_dedup_chain_dispatch: + """Test chain/callback dispatch on the dedup fast-path.""" + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_chain_completes_with_dedup_enabled(self): + """Smoke test: a normal chain works when dedup is on.""" + c = chain(add.s(2, 3), add.s(5)) + assert c().get(timeout=TIMEOUT) == 10 + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_reject_requeue_completes_chain(self): + """Reject passthrough: chain completes after rejection + redelivery.""" + c = chain(reject_then_succeed.s(), identity.s()) + assert c().get(timeout=TIMEOUT) == 'second-pass' + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_dedup_dispatches_chain_on_redelivery(self): + """Core test: dedup fast-path dispatches the chain.""" + c = chain(store_success_then_reject.s(), identity.s()) + assert c().get(timeout=TIMEOUT) == 'first-pass' + + @flaky + @pytest.mark.usefixtures('dedup_worker') + def test_dedup_dispatches_callback_on_redelivery(self, celery_session_app): + """Dedup fast-path dispatches link callbacks.""" + import uuid as _uuid + cb_id = _uuid.uuid4().hex + sig = store_success_then_reject.s() + sig.link(identity.s().set(task_id=cb_id)) + sig.apply_async() + cb_result = AsyncResult(cb_id, app=celery_session_app) + assert cb_result.get(timeout=TIMEOUT) == 'first-pass' diff --git a/t/integration/test_django_settings.py b/t/integration/test_django_settings.py new file mode 100644 index 00000000000..93de7f81139 --- /dev/null +++ b/t/integration/test_django_settings.py @@ -0,0 +1,19 @@ +import logging + +logging.info("Initializing Django settings for Celery integration tests") + +DEBUG = True +INSTALLED_APPS = [ + 'django.contrib.contenttypes', + 'django.contrib.auth', +] + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': ':memory:', + } +} + +CELERY_BROKER_URL = 'memory://' +CELERY_RESULT_BACKEND = 'cache+memory://' diff --git a/t/integration/test_native_delayed_delivery_binding.py b/t/integration/test_native_delayed_delivery_binding.py new file mode 100644 index 00000000000..f738810203c --- /dev/null +++ b/t/integration/test_native_delayed_delivery_binding.py @@ -0,0 +1,190 @@ +"""Integration tests for native delayed delivery queue binding. + +Tests that verify queue bindings are created correctly for native delayed +delivery, especially when some queues in task_queues fail to bind. +""" +import os +import uuid +from urllib.parse import quote + +import pytest +import requests +from kombu import Exchange, Queue +from requests.auth import HTTPBasicAuth + +from celery import Celery +from celery.contrib.testing.worker import start_worker + + +def get_rabbitmq_credentials(): + """Get RabbitMQ credentials from environment.""" + user = os.environ.get("RABBITMQ_DEFAULT_USER", "guest") + password = os.environ.get("RABBITMQ_DEFAULT_PASSWORD", "guest") + return user, password + + +def get_rabbitmq_url(): + """Get RabbitMQ broker URL from environment.""" + user, password = get_rabbitmq_credentials() + return os.environ.get( + "TEST_BROKER", f"pyamqp://{user}:{password}@localhost:5672//") + + +def get_management_api_url(): + """Get RabbitMQ Management API base URL.""" + return "http://localhost:15672/api" + + +def get_bindings_for_exchange(exchange_name, vhost='/'): + """Fetch bindings where the given exchange is the source. + + Args: + exchange_name: Name of the exchange + vhost: Virtual host (default '/') + + Returns: + List of binding dictionaries + """ + user, password = get_rabbitmq_credentials() + vhost_encoded = quote(vhost, safe='') + exchange_encoded = quote(exchange_name, safe='') + api_url = ( + f"{get_management_api_url()}/exchanges/{vhost_encoded}/" + f"{exchange_encoded}/bindings/source" + ) + response = requests.get(api_url, auth=HTTPBasicAuth(user, password)) + response.raise_for_status() + return response.json() + + +def get_bindings_for_queue(queue_name, vhost='/'): + """Fetch bindings for a specific queue. + + Args: + queue_name: Name of the queue + vhost: Virtual host (default '/') + + Returns: + List of binding dictionaries + """ + user, password = get_rabbitmq_credentials() + vhost_encoded = quote(vhost, safe='') + queue_encoded = quote(queue_name, safe='') + api_url = ( + f"{get_management_api_url()}/queues/{vhost_encoded}/{queue_encoded}/" + "bindings" + ) + response = requests.get(api_url, auth=HTTPBasicAuth(user, password)) + response.raise_for_status() + return response.json() + + +def create_test_app(unique_id): + """Create Celery app configured for native delayed delivery testing. + + Args: + unique_id: Unique identifier to ensure queue/exchange names don't + conflict + + Returns: + Tuple of (app, exchange_name, queue_a_name, queue_b_name) + """ + broker_url = get_rabbitmq_url() + + # Get Redis backend URL from environment + redis_host = os.environ.get("REDIS_HOST", "localhost") + redis_port = os.environ.get("REDIS_PORT", "6379") + backend_url = os.environ.get( + "TEST_BACKEND", f"redis://{redis_host}:{redis_port}/0") + + app = Celery( + "test_native_delayed_delivery_binding", + broker=broker_url, + backend=backend_url, + ) + + # Configure topic exchange with unique name + exchange_name = f'celery.topic_{unique_id}' + default_exchange = Exchange(exchange_name, type='topic') + + # Define task queues with queue-a first, queue-b second + queue_a_name = f'queue-a_{unique_id}' + queue_b_name = f'queue-b_{unique_id}' + app.conf.task_queues = [ + Queue(queue_a_name, exchange=default_exchange, + routing_key=queue_a_name, + queue_arguments={'x-queue-type': 'quorum'}), + Queue(queue_b_name, exchange=default_exchange, + routing_key=queue_b_name, + queue_arguments={'x-queue-type': 'quorum'}), + ] + + # Recommended setting for using celery with quorum queues + app.conf.broker_transport_options = {"confirm_publish": True} + + # Enable quorum queue detection to disable global QoS + app.conf.worker_detect_quorum_queues = True + + return app, exchange_name, queue_a_name, queue_b_name + + +@pytest.mark.amqp +@pytest.mark.timeout(90) +def test_worker_binds_consumed_queue_despite_earlier_queue_failure(): + """Test that queue binding continues even when earlier queues fail to bind. + + This test reproduces the scenario from + https://github.com/celery/celery/issues/9960 + """ + unique_id = uuid.uuid4().hex + app, exchange_name, queue_a_name, queue_b_name = create_test_app(unique_id) + + # Set default queue to queue-b so the start_worker ping task is received + # by our worker + app.conf.task_default_queue = queue_b_name + + # Start worker that only consumes from queue-b + # queue-a is NOT consumed, so it won't be declared by this worker + with start_worker( + app, + queues=[queue_b_name], + loglevel="INFO", + perform_ping_check=True, + shutdown_timeout=15, + ): + # Check celery_delayed_delivery → exchange bindings + delayed_delivery_bindings = \ + get_bindings_for_exchange('celery_delayed_delivery') + queue_b_delayed_binding = [ + b for b in delayed_delivery_bindings + if b.get('destination') == exchange_name + and b.get('routing_key') == f'#.{queue_b_name}' + ] + assert len(queue_b_delayed_binding) >= 1, ( + f"Expected delayed delivery binding for {queue_b_name!r}, but " + f"got bindings: {delayed_delivery_bindings!r}" + ) + + # Check celery.topic → queue-b bindings + # Should have bindings from the topic exchange to queue-b for both + # immediate and delayed delivery + queue_b_bindings = get_bindings_for_queue(queue_b_name) + topic_to_queue_bindings = [ + b for b in queue_b_bindings + if b.get('source') == exchange_name + ] + topic_to_queue_routing_keys = { + b.get('routing_key') for b in topic_to_queue_bindings + } + + # Check the routing key for immediate delivery + assert queue_b_name in topic_to_queue_routing_keys, ( + f"Expected routing key {queue_b_name!r} in bindings, but got: " + f"{topic_to_queue_bindings!r}" + ) + + # Check the routing key for delayed delivery + assert f"#.{queue_b_name}" in topic_to_queue_routing_keys, ( + f"Expected at least one binding from {exchange_name!r} to " + f"{queue_b_name!r}, but got: {topic_to_queue_bindings!r}" + ) diff --git a/t/integration/test_prefork_shutdown.py b/t/integration/test_prefork_shutdown.py new file mode 100644 index 00000000000..6ae5c1fffd2 --- /dev/null +++ b/t/integration/test_prefork_shutdown.py @@ -0,0 +1,89 @@ +"""Integration tests for prefork pool shutdown behaviour. + +These tests verify that the prefork pool gracefully shuts down and maintains +heartbeats during the shutdown process, preventing connection loss during +worker drain. +""" + +from time import sleep + +import pytest + +from celery.contrib.testing.worker import start_worker +from celery.worker import state + +from .tasks import sleeping + +TEST_HEARTBEAT = 2 + +# Exceeds AMQP connection timeout (~4 seconds: broker closes after missing +# 2 consecutive 2-second heartbeats) +LONG_TASK_DURATION = 10 + +TIMEOUT = LONG_TASK_DURATION * 2 + + +@pytest.fixture +def heartbeat_worker(celery_session_app): + """Worker with short heartbeat for testing purposes.""" + + # Temporarily lower heartbeat for this test + original_heartbeat = celery_session_app.conf.broker_heartbeat + celery_session_app.conf.broker_heartbeat = TEST_HEARTBEAT + + original_acks_late = celery_session_app.conf.task_acks_late + celery_session_app.conf.task_acks_late = True + + with start_worker( + celery_session_app, + pool="prefork", + without_heartbeat=False, + concurrency=4, + shutdown_timeout=TIMEOUT, + perform_ping_check=False, + ) as worker: + # Verify that low heartbeat is configured correctly + assert worker.consumer.amqheartbeat == TEST_HEARTBEAT + + yield worker + + celery_session_app.conf.broker_heartbeat = original_heartbeat + celery_session_app.conf.task_acks_late = original_acks_late + + +class test_prefork_shutdown: + """Test prefork shutdown with heartbeat maintenance.""" + + # Test timeout should be longer than worker timeout + @pytest.mark.timeout(timeout=TIMEOUT * 2) + @pytest.mark.usefixtures("heartbeat_worker") + def test_shutdown_with_long_running_tasks(self): + """Test that graceful shutdown completes long-running tasks without + connection loss. + + This test verifies that when the prefork pool is shutting down with + long-running tasks, heartbeats continue to be sent to maintain the + broker connection. + + - Heartbeat frames sent every 2 seconds + - Connection closes after 4 seconds (two missed frames) without heartbeats + - Tasks run 10 seconds to exceed 4-second threshold + """ + + # Submit multiple long-running tasks that will be active during shutdown + num_tasks = 3 + results = [] + for _ in range(num_tasks): + results.append(sleeping.delay(LONG_TASK_DURATION)) + + # Give time for tasks to start executing + sleep(1) + + state.should_stop = True + + # Wait for all tasks to complete. If heartbeats aren't maintained during + # shutdown, this will fail with `ConnectionResetError`, `BrokenPipeError` + # and `celery.exceptions.TimeoutError`. + for result in results: + result.get(timeout=TIMEOUT) + assert result.status == "SUCCESS" diff --git a/t/integration/test_rabbitmq_quorum_queue_cycle_detection.py b/t/integration/test_rabbitmq_quorum_queue_cycle_detection.py new file mode 100644 index 00000000000..f0bed95afc5 --- /dev/null +++ b/t/integration/test_rabbitmq_quorum_queue_cycle_detection.py @@ -0,0 +1,148 @@ +""" +Integration tests for RabbitMQ cycle detection bug with quorum queues. + +This reproduces the conditions for +https://github.com/celery/celery/issues/9867 when running against RabbitMQ +<4.0.1. +""" + +import os + +import pytest +from kombu import Exchange, Queue + +from celery import Celery, exceptions +from celery.contrib.testing.worker import start_worker + + +class TaskFailedException(Exception): + """Test exception for tasks that should fail and retry.""" + pass + + +def get_rabbitmq_url(): + """Get RabbitMQ broker URL from environment.""" + user = os.environ.get("RABBITMQ_DEFAULT_USER", "guest") + password = os.environ.get("RABBITMQ_DEFAULT_PASSWORD", "guest") + return os.environ.get( + "TEST_BROKER", f"pyamqp://{user}:{password}@localhost:5672//") + + +def get_backend_url(): + """Get backend URL from environment.""" + return os.environ.get("TEST_BACKEND", "rpc") + + +def create_test_app_with_quorum_queues(): + """ + Create Celery app configured for quorum queue testing. + + Returns: + Tuple of (app, queue_name) + """ + broker_url = get_rabbitmq_url() + backend_url = get_backend_url() + + app = Celery( + "test_cycle_detection", + broker=broker_url, + backend=backend_url, + ) + + # Configure queue with quorum queue type + queue_name = 'test_cycle_queue' + default_exchange = Exchange('celery.topic', type='topic') + + app.conf.task_queues = [ + Queue( + queue_name, + exchange=default_exchange, + routing_key=queue_name, + queue_arguments={'x-queue-type': 'quorum'} + ), + ] + + app.conf.task_default_queue = queue_name + + # Recommended settings for quorum queues + app.conf.broker_transport_options = {"confirm_publish": True} + + # Enable result extended to track retries + app.conf.result_extended = True + + return app + + +@pytest.mark.amqp +@pytest.mark.timeout(35) +def test_countdown_task_with_retry(): + """Test where a countdown task gets retried with the same delay.""" + app = create_test_app_with_quorum_queues() + + @app.task(bind=True, default_retry_delay=3, max_retries=1) + def countdown_task_with_retry(self): + """Task that fails once then succeeds.""" + # First attempt: fail to trigger retry + if self.request.retries < 1: + raise self.retry(exc=TaskFailedException("Simulated failure")) + + return self.request.retries + + with start_worker( + app, + loglevel="INFO", + shutdown_timeout=15, + ): + # Execute task with countdown=3 (same as default_retry_delay) + # We need to hit the same delayed buckets to trigger the cycle detection + # in RabbitMQ. + result = countdown_task_with_retry.apply_async(countdown=3) + + try: + # Wait for the task to complete, we must wait at least long enough + # for the countdown to be reached and the retry to occur (6s). + value = result.get(timeout=15) + except exceptions.TimeoutError as e: + pytest.fail( + f"Task was silently dropped by RabbitMQ due to cycle detection. " + f"Exception: {e!r}." + ) + + assert value == 1, \ + f"Expected task to succeed after 1 retry, got: {value}" + + +@pytest.mark.amqp +@pytest.mark.timeout(35) +def test_non_eta_task_with_multiple_retries(): + """Test where a task gets retried twice with the same delay.""" + app = create_test_app_with_quorum_queues() + + @app.task(bind=True, max_retries=2) + def non_eta_task_with_retries(self): + """Task that fails twice then succeeds.""" + # First and second attempts: fail to trigger retry + if self.request.retries < 2: + raise self.retry(exc=TaskFailedException("Simulated failure"), countdown=3) + + return self.request.retries + + with start_worker( + app, + loglevel="INFO", + shutdown_timeout=15, + ): + result = non_eta_task_with_retries.apply_async() + + try: + # Wait for the task to complete, we must wait at least long enough + # for the two retries to occur (6s). + value = result.get(timeout=15) + except exceptions.TimeoutError as e: + pytest.fail( + f"Task was silently dropped by RabbitMQ due to cycle detection. " + f"Exception: {e!r}" + ) + + assert value == 2, \ + f"Expected task to succeed after 2 retries, got: {value}" diff --git a/t/integration/test_spawn_pool.py b/t/integration/test_spawn_pool.py new file mode 100644 index 00000000000..912f2fdd259 --- /dev/null +++ b/t/integration/test_spawn_pool.py @@ -0,0 +1,20 @@ +import pytest + +from .tasks import add + + +@pytest.fixture(scope="session") +def celery_worker_pool(): + return "spawn" + + +@pytest.mark.flaky(reruns=5, reruns_delay=2) +def test_basic_task_spawn(manager): + results = [] + for i in range(5): + results.append([i + i, add.delay(i, i)]) + for expected, result in results: + assert result.get(timeout=10) == expected + assert result.status == "SUCCESS" + assert result.ready() is True + assert result.successful() is True diff --git a/t/integration/test_tasks.py b/t/integration/test_tasks.py index 1f6a0499018..74e15b5913c 100644 --- a/t/integration/test_tasks.py +++ b/t/integration/test_tasks.py @@ -98,6 +98,7 @@ def test_basic_task(self, manager): assert result.successful() is True @flaky + @pytest.mark.skip(reason="Broken test") def test_multiprocess_producer(self, manager): """Testing multiple processes calling tasks.""" set_multiprocessing_start_method() @@ -108,6 +109,7 @@ def test_multiprocess_producer(self, manager): assert list(ret) == list(range(120)) @flaky + @pytest.mark.skip(reason="Broken test") def test_multithread_producer(self, manager): """Testing multiple threads calling tasks.""" set_multiprocessing_start_method() @@ -436,10 +438,16 @@ def test_retry_with_unpickleable_exception(self, manager): res = job.result assert job.status == 'RETRY' # make sure that it wasn't completed yet - # Check it - assert isinstance(res, UnpickleableExceptionWrapper) - assert res.exc_cls_name == "UnpickleableException" - assert res.exc_args == ("foo",) + # Check it. Accept both the dedicated wrapper and plain Exception + # (some environments may return a stringified Exception instead). + if isinstance(res, UnpickleableExceptionWrapper): + assert res.exc_cls_name == "UnpickleableException" + assert res.exc_args == ("foo",) + else: + # Fallback: ensure the exception string mentions the class and argument + res_str = str(res) + assert "UnpickleableException" in res_str + assert "foo" in res_str job.revoke() @@ -450,12 +458,20 @@ def test_fail_with_unpickleable_exception(self, manager): """ result = fail_unpickleable.delay("foo", "bar") - with pytest.raises(UnpickleableExceptionWrapper) as exc_info: + # Accept either the dedicated wrapper exception or a plain Exception + # whose string contains the class name and args (some backends + # may return a stringified exception). + try: result.get() - - exc_wrapper = exc_info.value - assert exc_wrapper.exc_cls_name == "UnpickleableException" - assert exc_wrapper.exc_args == ("foo",) + pytest.fail("Expected an exception when getting result") + except UnpickleableExceptionWrapper as exc_wrapper: + assert exc_wrapper.exc_cls_name == "UnpickleableException" + assert exc_wrapper.exc_args == ("foo",) + except Exception as exc: + # Fallback: ensure the exception string mentions the class and argument + exc_str = str(exc) + assert "UnpickleableException" in exc_str + assert "foo" in exc_str assert result.status == 'FAILURE' diff --git a/t/integration/test_worker.py b/t/integration/test_worker.py index 9487753f4a5..23abe032ba5 100644 --- a/t/integration/test_worker.py +++ b/t/integration/test_worker.py @@ -2,6 +2,8 @@ import pytest +from celery import Celery + def test_run_worker(): with pytest.raises(subprocess.CalledProcessError) as exc_info: @@ -16,3 +18,49 @@ def test_run_worker(): "Retrying to establish a connection to the message broker after a connection " "loss has been disabled (app.conf.broker_connection_retry_on_startup=False). " "Shutting down...") != -1, output + + +def test_django_fixup_direct_worker(caplog, monkeypatch): + """Test Django fixup by directly instantiating Celery worker without subprocess.""" + import logging + + import django + + # Set logging level to capture debug messages + caplog.set_level(logging.DEBUG) + + # Configure Django settings + monkeypatch.setenv('DJANGO_SETTINGS_MODULE', 't.integration.test_django_settings') + django.setup() + + # Create Celery app with Django integration + app = Celery('test_django_direct') + app.config_from_object('django.conf:settings', namespace='CELERY') + app.autodiscover_tasks() + + # Test that we can access worker configuration without recursion errors + # This should trigger the Django fixup initialization + worker = app.Worker( + pool='solo', + concurrency=1, + loglevel='debug' + ) + + # Accessing pool_cls should not cause AttributeError + pool_cls = worker.pool_cls + assert pool_cls is not None + + # Verify pool_cls has __module__ attribute (should be a class, not a string) + assert hasattr(pool_cls, '__module__'), \ + f"pool_cls should be a class with __module__, got {type(pool_cls)}: {pool_cls}" + + # Capture and check logs + log_output = caplog.text + + # Verify no recursion-related errors in logs + assert "RecursionError" not in log_output, f"RecursionError found in logs:\n{log_output}" + assert "maximum recursion depth exceeded" not in log_output, \ + f"Recursion depth error found in logs:\n{log_output}" + + assert "AttributeError: 'str' object has no attribute '__module__'." not in log_output, \ + f"AttributeError found in logs:\n{log_output}" diff --git a/t/smoke/tests/test_canvas.py b/t/smoke/tests/test_canvas.py index e0886d56e49..b6c69e76397 100644 --- a/t/smoke/tests/test_canvas.py +++ b/t/smoke/tests/test_canvas.py @@ -179,3 +179,15 @@ def test_chord_error_propagation_with_different_body_types( # The chord should fail with the expected exception from the failing task with pytest.raises(ExpectedException): result.get(timeout=RESULT_TIMEOUT) + + +class test_complex_workflow: + def test_pending_tasks_released_on_forget(self, celery_setup: CeleryTestSetup): + sig = add.si(1, 1) | group( + add.s(1) | group(add.si(1, 1), add.si(2, 2)) | add.si(2, 2), + add.s(1) | group(add.si(1, 1), add.si(2, 2)) | add.si(2, 2) + ) | add.si(1, 1) + res = sig.apply_async(queue=celery_setup.worker.worker_queue) + assert not all(len(mapping) == 0 for mapping in res.backend._pending_results) + res.forget() + assert all(len(mapping) == 0 for mapping in res.backend._pending_results) diff --git a/t/smoke/tests/test_thread_safe.py b/t/smoke/tests/test_thread_safe.py index ceab993e24d..47563e6cbec 100644 --- a/t/smoke/tests/test_thread_safe.py +++ b/t/smoke/tests/test_thread_safe.py @@ -56,7 +56,6 @@ def test_multithread_task_publish( @after_task_publish.connect def after_task_publish_handler(*args, **kwargs): - nonlocal signal_was_called signal_was_called(True) def thread_worker(): diff --git a/t/smoke/tests/test_worker.py b/t/smoke/tests/test_worker.py index 2165f4296af..617576d1860 100644 --- a/t/smoke/tests/test_worker.py +++ b/t/smoke/tests/test_worker.py @@ -215,6 +215,68 @@ def test_hard_shutdown_from_soft(self, celery_setup: CeleryTestSetup): assert_container_exited(worker) + def test_task_completes_during_soft_shutdown(self, celery_setup: CeleryTestSetup): + app = celery_setup.app + queue = celery_setup.worker.worker_queue + worker = celery_setup.worker + + task_duration = app.conf.worker_soft_shutdown_timeout // 2 + sig = long_running_task.si(task_duration, verbose=True).set(queue=queue) + res = sig.delay() + + worker.assert_log_exists("Starting long running task") + self.kill_worker(worker, WorkerKill.Method.SIGQUIT) + worker.assert_log_exists( + f"Initiating Soft Shutdown, terminating in {app.conf.worker_soft_shutdown_timeout} seconds" + ) + worker.assert_log_exists("worker: Cold shutdown (MainProcess)") + + assert_container_exited(worker) + assert res.get(RESULT_TIMEOUT) + assert res.state == 'SUCCESS' + + class test_time_limit(SuiteOperations): + @pytest.fixture + def default_worker_app(self, default_worker_app: Celery) -> Celery: + app = default_worker_app + app.conf.worker_soft_shutdown_timeout = 16 + app.conf.task_time_limit = 15 + app.conf.task_soft_time_limit = 10 + return app + + def test_task_completes_during_soft_shutdown_with_time_limit(self, celery_setup: CeleryTestSetup): + app = celery_setup.app + queue = celery_setup.worker.worker_queue + worker = celery_setup.worker + + task_duration = 8 + sig = long_running_task.si(task_duration, verbose=True).set(queue=queue) + res = sig.delay() + + worker.assert_log_exists("Starting long running task") + self.kill_worker(worker, WorkerKill.Method.SIGQUIT) + + worker.assert_log_exists( + f"Initiating Soft Shutdown, terminating in {app.conf.worker_soft_shutdown_timeout} seconds" + ) + + worker.assert_log_exists("Finished long running task") + worker.assert_log_exists(f"long_running_task[{res.id}] succeeded") + + worker.assert_log_does_not_exist( + f"Task handler raised error: TimeLimitExceeded({app.conf.task_time_limit})", + timeout=RESULT_TIMEOUT, + ) + worker.assert_log_does_not_exist( + f"Hard time limit ({app.conf.task_time_limit}s) exceeded for {long_running_task.name}[{res.id}]", + timeout=RESULT_TIMEOUT, + ) + worker.assert_log_exists("worker: Cold shutdown (MainProcess)") + + assert_container_exited(worker) + assert res.get(RESULT_TIMEOUT) + assert res.state == 'SUCCESS' + class test_REMAP_SIGTERM(SuiteOperations): @pytest.fixture def default_worker_env(self, default_worker_env: dict) -> dict: diff --git a/t/smoke/workers/docker/dev b/t/smoke/workers/docker/dev index 015be6deebb..51041dbb0c6 100644 --- a/t/smoke/workers/docker/dev +++ b/t/smoke/workers/docker/dev @@ -23,6 +23,7 @@ RUN apt-get update && apt-get install -y build-essential \ ARG CELERY_LOG_LEVEL=INFO ARG CELERY_WORKER_NAME=celery_dev_worker ARG CELERY_WORKER_QUEUE=celery +ARG PYTEST_CELERY_PKG="pytest-celery" ENV LOG_LEVEL=$CELERY_LOG_LEVEL ENV WORKER_NAME=$CELERY_WORKER_NAME ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE @@ -39,7 +40,7 @@ COPY --chown=test_user:test_user . /celery RUN pip install --no-cache-dir --upgrade \ pip \ -e /celery[redis,pymemcache,pydantic,sqs] \ - pytest-celery>=1.1.3 + "${PYTEST_CELERY_PKG}" # The workdir must be /app WORKDIR /app diff --git a/t/smoke/workers/docker/pypi b/t/smoke/workers/docker/pypi index d0b2c21aa48..9e1f94d64d5 100644 --- a/t/smoke/workers/docker/pypi +++ b/t/smoke/workers/docker/pypi @@ -24,6 +24,7 @@ ARG CELERY_VERSION="" ARG CELERY_LOG_LEVEL=INFO ARG CELERY_WORKER_NAME=celery_tests_worker ARG CELERY_WORKER_QUEUE=celery +ARG PYTEST_CELERY_PKG="pytest-celery" ENV PIP_VERSION=$CELERY_VERSION ENV LOG_LEVEL=$CELERY_LOG_LEVEL ENV WORKER_NAME=$CELERY_WORKER_NAME @@ -38,7 +39,7 @@ EXPOSE 5678 RUN pip install --no-cache-dir --upgrade \ pip \ celery[redis,pymemcache]${CELERY_VERSION:+==$CELERY_VERSION} \ - pytest-celery[sqs]>=1.1.3 \ + "${PYTEST_CELERY_PKG}" \ pydantic>=2.4 # The workdir must be /app diff --git a/t/unit/app/test_amqp.py b/t/unit/app/test_amqp.py index 4b46148d144..db15c343a99 100644 --- a/t/unit/app/test_amqp.py +++ b/t/unit/app/test_amqp.py @@ -134,6 +134,15 @@ def test_with_max_priority(self, queues_kwargs, qname, q, expected): queues.add(q) assert queues[qname].queue_arguments == expected + def test_missing_queue_quorum(self): + queues = Queues(create_missing_queue_type="quorum", + create_missing_queue_exchange_type="topic") + + q = queues.new_missing("spontaneous") + assert q.name == "spontaneous" + assert q.queue_arguments == {"x-queue-type": "quorum"} + assert q.exchange.type == "topic" + class test_default_queues: @@ -360,6 +369,22 @@ def update_conf_runtime_for_tasks_queues(self): router = self.app.amqp.router assert router != router_was + def test_create_missing_queue_type_from_conf(self): + self.app.conf.task_create_missing_queue_type = "quorum" + self.app.conf.task_create_missing_queue_exchange_type = "topic" + self.app.amqp.__dict__.pop("queues", None) + q = self.app.amqp.queues["auto"] + assert q.queue_arguments == {"x-queue-type": "quorum"} + assert q.exchange.type == "topic" + + def test_create_missing_queue_type_explicit_param(self): + qmap = self.app.amqp.Queues({}, create_missing=True, + create_missing_queue_type="quorum", + create_missing_queue_exchange_type="topic") + q = qmap["auto"] + assert q.queue_arguments == {"x-queue-type": "quorum"} + assert q.exchange.type == "topic" + class test_as_task_v2(test_AMQP_Base): diff --git a/t/unit/app/test_app.py b/t/unit/app/test_app.py index ca2dd2b4bf1..dc5a4cc417e 100644 --- a/t/unit/app/test_app.py +++ b/t/unit/app/test_app.py @@ -32,13 +32,13 @@ from celery.utils.collections import DictAttribute from celery.utils.objects import Bunch from celery.utils.serialization import pickle -from celery.utils.time import localize, timezone, to_utc +from celery.utils.time import LocalTimezone, localize, timezone, to_utc from t.unit import conftest if sys.version_info >= (3, 9): from zoneinfo import ZoneInfo else: - from backports.zoneinfo import ZoneInfo # noqa + from backports.zoneinfo import ZoneInfo THIS_IS_A_KEY = 'this is a value' @@ -831,6 +831,26 @@ def foo(parameter: int) -> None: assert typing.get_type_hints(foo) == { 'parameter': int, 'return': type(None)} + @pytest.mark.skipif(sys.version_info < (3, 14), reason="PEP 649 deferred annotations require Python 3.14+") + def test_task_with_type_checking_annotation(self): + # Regression test for https://github.com/celery/celery/discussions/10099 + # On Python 3.14+, annotations are deferred (PEP 649). Registering a task + # whose annotations reference TYPE_CHECKING-only types must not raise NameError. + local = {} + exec( + 'def foo(args: Sequence[str], x: int = 0): return args', + {'app': None}, + local, + ) + raw_fun = local['foo'] + + with self.Celery() as app: + task = app.task(raw_fun) + result = task.apply(args=(['hello'],)) + assert result.result == ['hello'] + # Annotations should be stored as strings, not evaluated + assert task.__annotations__['args'] == 'Sequence[str]' + def test_annotate_decorator(self): from celery.app.task import Task @@ -1176,7 +1196,7 @@ def test_signature(self): sig = self.app.signature('foo', (1, 2)) assert sig.app is self.app - def test_timezone__none_set(self): + def test_timezone_none_set(self): self.app.conf.timezone = None self.app.conf.enable_utc = True assert self.app.timezone == timezone.utc @@ -1184,6 +1204,32 @@ def test_timezone__none_set(self): self.app.conf.enable_utc = False assert self.app.timezone == timezone.local + def test_use_local_timezone(self): + self.app.conf.timezone = None + self.app.conf.enable_utc = False + + self._clear_timezone_cache() + try: + assert isinstance(self.app.timezone, ZoneInfo) + finally: + self._clear_timezone_cache() + + @patch("celery.utils.time.get_localzone") + def test_use_local_timezone_failure(self, mock_get_localzone): + mock_get_localzone.side_effect = Exception("Failed to get local timezone") + self.app.conf.timezone = None + self.app.conf.enable_utc = False + + self._clear_timezone_cache() + try: + assert isinstance(self.app.timezone, LocalTimezone) + finally: + self._clear_timezone_cache() + + def _clear_timezone_cache(self): + del self.app.timezone + del timezone.local + def test_uses_utc_timezone(self): self.app.conf.timezone = None self.app.conf.enable_utc = True diff --git a/t/unit/app/test_beat.py b/t/unit/app/test_beat.py index b81a11426e1..55feb9d7c9c 100644 --- a/t/unit/app/test_beat.py +++ b/t/unit/app/test_beat.py @@ -1,5 +1,6 @@ import dbm import errno +import pickle import sys from datetime import datetime, timedelta, timezone from pickle import dumps, loads @@ -708,6 +709,26 @@ def test_create_schedule_corrupted_dbm_error(self): s._create_schedule() s._destroy_open_corrupted_schedule.assert_called_with(expected_error) + def test_create_schedule_corrupted_pickle_error(self): + """ + Test that any UnpicklingError that might happen when opening beat-schedule.db is caught + """ + s = create_persistent_scheduler()[0](app=self.app, + schedule_filename='schedule') + s._store = MagicMock() + s._destroy_open_corrupted_schedule = Mock() + s._destroy_open_corrupted_schedule.return_value = MagicMock() + + # self._store['entries'] = {} will throw a pickle.UnpicklingError + s._store.__getitem__.side_effect = pickle.UnpicklingError("test") + # then, when _create_schedule tries to reset _store['entries'], + # throw another error, specifically pickle.UnpicklingError + expected_error = pickle.UnpicklingError("test") + s._store.__setitem__.side_effect = expected_error + + s._create_schedule() + s._destroy_open_corrupted_schedule.assert_called_with(expected_error) + def test_create_schedule_missing_entries(self): """ Test that if _create_schedule can't find the key "entries" in _store it will recreate it diff --git a/t/unit/app/test_control.py b/t/unit/app/test_control.py index 0908491a9ee..4916880a431 100644 --- a/t/unit/app/test_control.py +++ b/t/unit/app/test_control.py @@ -4,7 +4,7 @@ from celery import uuid from celery.app import control -from celery.exceptions import DuplicateNodenameWarning +from celery.exceptions import DuplicateNodenameWarning, ImproperlyConfigured from celery.utils.collections import LimitedSet @@ -291,6 +291,7 @@ def test_time_limit__with_destination(self): self.mytask.name, soft=10, hard=20, destination='a@q.com', limit=99, ) + self.assert_control_called_with_args( 'time_limit', destination='a@q.com', @@ -564,3 +565,18 @@ def test_control_exchange__setting(self): self.app.conf.control_exchange = 'test_exchange' c = control.Control(self.app) assert c.mailbox.namespace == 'test_exchange' + + def test_control_mailbox_queue_options(self): + self.app.conf.control_queue_durable = True + self.app.conf.control_queue_exclusive = False + + c = control.Control(self.app) + assert c.mailbox.queue_durable is True + assert c.mailbox.queue_exclusive is False + + def test_control_mailbox_invalid_combination(self): + self.app.conf.control_queue_durable = True + self.app.conf.control_queue_exclusive = True + + with pytest.raises(ImproperlyConfigured): + control.Control(self.app) diff --git a/t/unit/app/test_schedules.py b/t/unit/app/test_schedules.py index 63689831bdf..924a72d3360 100644 --- a/t/unit/app/test_schedules.py +++ b/t/unit/app/test_schedules.py @@ -76,6 +76,16 @@ def test_invalid_event(self): with pytest.raises(ValueError): solar('asdqwewqew', 60, 60, app=self.app) + def test_dusk_horizons_are_negative(self): + """All dusk events should have negative horizons (sun below horizon).""" + for event in ('dusk_civil', 'dusk_nautical', 'dusk_astronomical'): + s = solar(event, 50, 10, app=self.app) + horizon = float(s.cal.horizon) + assert horizon < 0, ( + f"{event} horizon should be negative (below horizon), " + f"got {s.cal.horizon}" + ) + def test_event_uses_center(self): s = solar('solar_noon', 60, 60, app=self.app) for ev, is_center in s._use_center_l.items(): diff --git a/t/unit/backends/test_asynchronous.py b/t/unit/backends/test_asynchronous.py index 479fd855838..05a0557379f 100644 --- a/t/unit/backends/test_asynchronous.py +++ b/t/unit/backends/test_asynchronous.py @@ -1,3 +1,4 @@ +import logging import os import socket import sys @@ -8,17 +9,325 @@ import pytest from vine import promise -from celery.backends.asynchronous import BaseResultConsumer +from celery.backends.asynchronous import E_CELERY_RESTART_REQUIRED, BaseResultConsumer, greenletDrainer from celery.backends.base import Backend from celery.utils import cached_property -pytest.importorskip('gevent') -pytest.importorskip('eventlet') +# ---- helpers --------------------------------------------------------------- + + +def _make_consumer(app, environment='default'): + """Create a BaseResultConsumer with a mocked drainer environment.""" + with patch('celery.backends.asynchronous.detect_environment') as det: + det.return_value = environment + backend = Backend(app) + consumer = BaseResultConsumer( + backend, app, backend.accept, + pending_results={}, pending_messages={}, + ) + return consumer + + +# --------------------------------------------------------------------------- +# 1. Drainer (default / synchronous) -- no gevent / eventlet needed +# --------------------------------------------------------------------------- + +class test_Drainer_without_greenlets: + + # -- drain_events_until: normal flow ------------------------------------ + + def test_drain_fulfils_promise(self, app): + """Loop exits once the promise is fulfilled.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] >= 3: + p('done') + + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5): + pass + + assert p.ready + assert calls[0] >= 3 + + def test_drain_calls_on_interval(self, app): + """on_interval callback is invoked every iteration.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + on_interval = Mock() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] >= 3: + p('done') + + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5, + on_interval=on_interval): + pass + + assert on_interval.call_count >= 2 + + def test_drain_raises_timeout(self, app): + """socket.timeout raised when total elapsed time exceeds *timeout*.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + + def wait(timeout=None): + time.sleep(0.02) + + with pytest.raises(socket.timeout): + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=0.05): + pass + + assert not p.ready + + def test_drain_uses_result_consumer_drain_events_by_default(self, app): + """When *wait* is None, result_consumer.drain_events is used.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def mock_drain(timeout=None): + calls[0] += 1 + if calls[0] >= 2: + p('done') + + consumer.drain_events = mock_drain + + for _ in drainer.drain_events_until(p, interval=0.01, timeout=5): + pass + + assert p.ready + assert calls[0] >= 2 + + # -- drain_events_until: socket.timeout from wait ----------------------- + + def test_drain_swallows_socket_timeout_from_wait(self, app): + """socket.timeout raised inside wait() must be silently caught.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] <= 2: + raise socket.timeout('idle') + p('done') + + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5): + pass + + assert p.ready + + # -- drain_events_until: OSError from wait ------------------------------ + + def test_drain_catches_oserror_and_logs(self, app): + """OSError from wait() must be caught, logged, loop continues.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + calls = [0] + + def wait(timeout=None): + calls[0] += 1 + if calls[0] <= 2: + raise OSError('broker away') + p('done') + + with patch.object(logging, 'warning') as mock_warn: + for _ in drainer.drain_events_until( + p, wait=wait, interval=0.01, timeout=5): + pass + + assert p.ready + assert mock_warn.call_count >= 2 + + # -- wait_for ----------------------------------------------------------- + + def test_wait_for_calls_wait_with_timeout(self, app): + """Drainer.wait_for delegates to the wait callback.""" + consumer = _make_consumer(app) + drainer = consumer.drainer + p = promise() + wait = Mock() + drainer.wait_for(p, wait, timeout=0.5) + wait.assert_called_once_with(timeout=0.5) + + +# --------------------------------------------------------------------------- +# 2. greenletDrainer -- tested synchronously (no real greenlet spawning) +# --------------------------------------------------------------------------- + +class test_greenletDrainer: + + def _make_greenlet_drainer(self, app): + consumer = _make_consumer(app) + drainer = greenletDrainer(consumer) + return drainer + + # -- run: normal stop --------------------------------------------------- + + def test_run_exits_when_stopped(self, app): + """run() exits cleanly when _stopped is set.""" + drainer = self._make_greenlet_drainer(app) + calls = [0] + + def drain(timeout=None): + calls[0] += 1 + if calls[0] >= 3: + drainer._stopped.set() + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + drainer.run() + + assert drainer._shutdown.is_set() + assert drainer._exc is None + + # -- run: socket.timeout is swallowed ----------------------------------- + + def test_run_swallows_socket_timeout(self, app): + """socket.timeout inside run() must be silently caught.""" + drainer = self._make_greenlet_drainer(app) + calls = [0] + + def drain(timeout=None): + calls[0] += 1 + if calls[0] <= 3: + raise socket.timeout('idle') + drainer._stopped.set() + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + drainer.run() + + assert calls[0] >= 4 + assert drainer._exc is None + + # -- run: OSError is caught and logged ---------------------------------- + + def test_run_catches_oserror_and_logs(self, app): + """OSError in run() must be caught/logged, loop continues.""" + drainer = self._make_greenlet_drainer(app) + calls = [0] + + def drain(timeout=None): + calls[0] += 1 + if calls[0] <= 3: + raise OSError('connection reset') + drainer._stopped.set() + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + + with patch.object(logging, 'warning') as mock_warn, \ + patch('celery.backends.asynchronous.time.sleep') as mock_sleep: + drainer.run() + + assert calls[0] >= 4 + assert mock_warn.call_count >= 3 + # backoff sleep should have been called once per OSError + assert mock_sleep.call_count >= 3 + assert drainer._exc is None + + # -- run: unexpected Exception is stored and re-raised ------------------ + + def test_run_stores_and_reraises_unexpected_exception(self, app): + """Non-OSError / non-timeout exceptions must propagate and be stored.""" + drainer = self._make_greenlet_drainer(app) + + def drain(timeout=None): + raise RuntimeError('unexpected') + + drainer.result_consumer.drain_events = Mock(side_effect=drain) + + with pytest.raises(RuntimeError, match='unexpected'): + drainer.run() + + assert drainer._exc is not None + assert drainer._shutdown.is_set() + + # -- _ensure_not_shut_down ---------------------------------------------- + + def test_ensure_not_shut_down_raises_stored_exc(self, app): + """If run() failed, _ensure_not_shut_down re-raises the exception.""" + drainer = self._make_greenlet_drainer(app) + drainer._shutdown.set() + drainer._exc = ValueError('boom') + + with pytest.raises(ValueError, match='boom'): + drainer._ensure_not_shut_down() + + def test_ensure_not_shut_down_raises_restart_msg(self, app): + """If stopped cleanly, _ensure_not_shut_down raises restart msg.""" + drainer = self._make_greenlet_drainer(app) + drainer._shutdown.set() + drainer._exc = None + + with pytest.raises(Exception, match=E_CELERY_RESTART_REQUIRED): + drainer._ensure_not_shut_down() + + def test_ensure_not_shut_down_noop_when_running(self, app): + """No-op when _shutdown is not set.""" + drainer = self._make_greenlet_drainer(app) + # Should not raise + drainer._ensure_not_shut_down() + + # -- start / stop ------------------------------------------------------- + + def test_start_spawns_and_waits(self, app): + """start() calls spawn(run) and waits for _started.""" + drainer = self._make_greenlet_drainer(app) + + def fake_spawn(func): + # Run synchronously with immediate stop. + drainer._stopped.set() + func() + + drainer.spawn = fake_spawn + drainer.result_consumer.drain_events = Mock( + side_effect=lambda timeout=None: drainer._stopped.set() + ) + drainer.start() + + assert drainer._started.is_set() + assert drainer._shutdown.is_set() + + def test_start_raises_if_already_shut_down(self, app): + """start() raises if drainer already completed.""" + drainer = self._make_greenlet_drainer(app) + drainer._shutdown.set() + + with pytest.raises(Exception, match=E_CELERY_RESTART_REQUIRED): + drainer.start() + + def test_stop_signals_and_waits(self, app): + """stop() sets _stopped and waits for _shutdown.""" + drainer = self._make_greenlet_drainer(app) + # Pre-set _shutdown so wait returns immediately. + drainer._shutdown.set() + drainer.stop() + + assert drainer._stopped.is_set() + + +# --------------------------------------------------------------------------- +# 3. Integration tests with real greenlet runtimes (gevent + eventlet) +# --------------------------------------------------------------------------- @pytest.fixture(autouse=True) def setup_eventlet(): - # By default eventlet will patch the DNS resolver when imported. os.environ.update(EVENTLET_NO_GREENDNS='yes') @@ -141,14 +450,109 @@ def test_drain_timeout(self): assert not p.ready, 'Promise should remain un-fulfilled' assert on_interval.call_count < 20, 'Should have limited number of calls to on_interval' + def test_drain_catches_and_logs_oserror(self): + p = promise() + + def fulfill(): + self.sleep(self.interval * 2) + p('done') + + t = self.schedule_thread(fulfill) + + state = {'n': 0} + + def flaky(*args, **kwargs): + state['n'] += 1 + if state['n'] == 1: + raise OSError('simulated broker restart') + # Yield to hub so the promise thread can run. + self.result_consumer_drain_events( + timeout=kwargs.get('timeout', None), + ) + + with patch.object( + self.drainer.result_consumer, 'drain_events', + side_effect=flaky, + ): + with patch('logging.warning') as mock_warn: + for _ in self.drainer.drain_events_until( + p, interval=self.interval, + timeout=self.MAX_TIMEOUT): + pass + + self.teardown_thread(t) + assert p.ready + assert mock_warn.called + + +class GreenletDrainerTests(DrainerTests): + def test_drain_raises_when_greenlet_already_exited(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=Exception("Test Exception")): + thread = self.schedule_thread(self.drainer.run) + + with pytest.raises(Exception, match="Test Exception"): + p = promise() + + for _ in self.drainer.drain_events_until(p, interval=self.interval): + pass + + self.teardown_thread(thread) + + def test_drain_raises_while_waiting_on_exiting_greenlet(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=Exception("Test Exception")): + with pytest.raises(Exception, match="Test Exception"): + p = promise() + + for _ in self.drainer.drain_events_until(p, interval=self.interval): + pass + + def test_start_raises_if_previous_error_in_run(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=Exception("Test Exception")): + thread = self.schedule_thread(self.drainer.run) + + with pytest.raises(Exception, match="Test Exception"): + self.drainer.start() + + self.teardown_thread(thread) + + def test_start_raises_if_drainer_already_stopped(self): + with patch.object(self.drainer.result_consumer, 'drain_events', side_effect=lambda **_: self.sleep(0)): + thread = self.schedule_thread(self.drainer.run) + self.drainer.stop() + + with pytest.raises(Exception, match=E_CELERY_RESTART_REQUIRED): + self.drainer.start() + + self.teardown_thread(thread) + + def test_run_catches_and_logs_oserror(self): + def flaky(*args, **kwargs): + if not hasattr(flaky, '_raised'): + flaky._raised = True + raise OSError('simulated broker restart in greenlet') + self.drainer._stopped.set() + + with patch.object( + self.drainer.result_consumer, 'drain_events', + side_effect=flaky, + ): + with patch('logging.warning') as mock_warn: + t = self.schedule_thread(self.drainer.run) + self.teardown_thread(t) + + assert mock_warn.called + assert 'connection error during drain_events' in mock_warn.call_args[0][0] + assert self.drainer._exc is None + @pytest.mark.skipif( sys.platform == "win32", reason="hangs forever intermittently on windows" ) -class test_EventletDrainer(DrainerTests): +class test_EventletDrainer(GreenletDrainerTests): @pytest.fixture(autouse=True) def setup_drainer(self): + pytest.importorskip('eventlet') self.drainer = self.get_drainer('eventlet') @cached_property @@ -171,7 +575,12 @@ def schedule_thread(self, thread): return g def teardown_thread(self, thread): - thread.wait() + try: + # eventlet's wait() propagates any errors on the green thread, unlike + # similar methods in gevent or python's threading library + thread.wait() + except Exception: + pass class test_Drainer(DrainerTests): @@ -196,9 +605,10 @@ def teardown_thread(self, thread): thread.join() -class test_GeventDrainer(DrainerTests): +class test_GeventDrainer(GreenletDrainerTests): @pytest.fixture(autouse=True) def setup_drainer(self): + pytest.importorskip('gevent') self.drainer = self.get_drainer('gevent') @cached_property @@ -223,3 +633,81 @@ def schedule_thread(self, thread): def teardown_thread(self, thread): import gevent gevent.wait([thread]) + + +class test_BaseResultConsumer_reconnect: + + def _make_consumer(self, app): + return _make_consumer(app) + + def test_reconnect_on_error_no_exception_passes_through(self, app): + consumer = self._make_consumer(app) + result = [] + with consumer.reconnect_on_error(): + result.append('ok') + assert result == ['ok'] + + def test_reconnect_on_error_ignores_non_connection_error(self, app): + consumer = self._make_consumer(app) + with pytest.raises(ValueError): + with consumer.reconnect_on_error(): + raise ValueError('unrelated') + + def test_reconnect_on_error_default_connection_errors_empty(self, app): + consumer = self._make_consumer(app) + assert consumer._connection_errors == () + + class FakeConnError(Exception): + pass + + with pytest.raises(FakeConnError): + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + def test_reconnect_on_error_calls_reconnect_on_connection_error(self, app): + consumer = self._make_consumer(app) + + class FakeConnError(Exception): + pass + + consumer._connection_errors = (FakeConnError,) + consumer._reconnect = Mock() + + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + consumer._reconnect.assert_called_once_with() + + def test_reconnect_on_error_raises_runtime_when_reconnect_also_fails(self, app): + consumer = self._make_consumer(app) + + class FakeConnError(Exception): + pass + + consumer._connection_errors = (FakeConnError,) + consumer._reconnect = Mock(side_effect=FakeConnError('still down')) + + with pytest.raises(RuntimeError, match='Retry limit exceeded'): + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + def test_reconnect_on_error_runtime_chained_from_connection_error(self, app): + consumer = self._make_consumer(app) + + class FakeConnError(Exception): + pass + + consumer._connection_errors = (FakeConnError,) + original = FakeConnError('still down') + consumer._reconnect = Mock(side_effect=original) + + with pytest.raises(RuntimeError) as exc_info: + with consumer.reconnect_on_error(): + raise FakeConnError('dropped') + + assert exc_info.value.__cause__ is original + + def test_reconnect_base_implementation_is_noop(self, app): + consumer = self._make_consumer(app) + + assert consumer._reconnect() is None diff --git a/t/unit/backends/test_base.py b/t/unit/backends/test_base.py index ce25ff72ad8..44f3f0cbdc7 100644 --- a/t/unit/backends/test_base.py +++ b/t/unit/backends/test_base.py @@ -1492,6 +1492,7 @@ def test_get_with_retries(self): b = BaseBackend(app=self.app) b.exception_safe_to_retry = lambda exc: True b._sleep = Mock() + b.on_backend_retryable_error = Mock() b._get_task_meta_for = Mock() b._get_task_meta_for.side_effect = [ Exception("failed"), @@ -1500,6 +1501,7 @@ def test_get_with_retries(self): res = b.get_task_meta(sentinel.task_id) assert res == {'status': states.SUCCESS, 'result': 42} assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() finally: self.app.conf.result_backend_always_retry = prev @@ -1554,6 +1556,7 @@ def test_store_result_never_retries(self): b = BaseBackend(app=self.app) b.exception_safe_to_retry = lambda exc: True b._sleep = Mock() + b.on_backend_retryable_error = Mock() b._get_task_meta_for = Mock() b._get_task_meta_for.return_value = { 'status': states.RETRY, @@ -1583,6 +1586,7 @@ def test_store_result_with_retries(self): b = BaseBackend(app=self.app) b.exception_safe_to_retry = lambda exc: True b._sleep = Mock() + b.on_backend_retryable_error = Mock() b._get_task_meta_for = Mock() b._get_task_meta_for.return_value = { 'status': states.RETRY, @@ -1600,6 +1604,56 @@ def test_store_result_with_retries(self): res = b.store_result(sentinel.task_id, 42, states.SUCCESS) assert res == 42 assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() + finally: + self.app.conf.result_backend_always_retry = prev + + def test_get_with_retries_hook_failure_continues(self): + self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry + + try: + b = BaseBackend(app=self.app) + b.exception_safe_to_retry = lambda exc: True + b._sleep = Mock() + b.on_backend_retryable_error = Mock(side_effect=RuntimeError("hook failed")) + b._get_task_meta_for = Mock() + b._get_task_meta_for.side_effect = [ + Exception("failed"), + {'status': states.SUCCESS, 'result': 42} + ] + res = b.get_task_meta(sentinel.task_id) + assert res == {'status': states.SUCCESS, 'result': 42} + assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() + finally: + self.app.conf.result_backend_always_retry = prev + + def test_store_result_with_retries_hook_failure_continues(self): + self.app.conf.result_backend_always_retry, prev = True, self.app.conf.result_backend_always_retry + + try: + b = BaseBackend(app=self.app) + b.exception_safe_to_retry = lambda exc: True + b._sleep = Mock() + b.on_backend_retryable_error = Mock(side_effect=RuntimeError("hook failed")) + b._get_task_meta_for = Mock() + b._get_task_meta_for.return_value = { + 'status': states.RETRY, + 'result': { + "exc_type": "Exception", + "exc_message": ["failed"], + "exc_module": "builtins", + }, + } + b._store_result = Mock() + b._store_result.side_effect = [ + Exception("failed"), + 42 + ] + res = b.store_result(sentinel.task_id, 42, states.SUCCESS) + assert res == 42 + assert b._sleep.call_count == 1 + b.on_backend_retryable_error.assert_called_once() finally: self.app.conf.result_backend_always_retry = prev diff --git a/t/unit/backends/test_couchdb.py b/t/unit/backends/test_couchdb.py index 07497b18cec..bdae58f339a 100644 --- a/t/unit/backends/test_couchdb.py +++ b/t/unit/backends/test_couchdb.py @@ -1,8 +1,10 @@ from unittest.mock import MagicMock, Mock, sentinel +from urllib.parse import urlparse import pytest +from kombu.utils.encoding import str_to_bytes -from celery import states +from celery import states, uuid from celery.app import backends from celery.backends import couchdb as module from celery.backends.couchdb import CouchBackend @@ -115,3 +117,97 @@ def test_backend_params_by_url(self): assert x.username == 'johndoe' assert x.password == 'mysecret' assert x.port == 123 + + +class CouchSessionMock: + """ + Mock for `requests.session` that emulates couchdb storage. + """ + + _store = {} + + def request(self, method, url, stream=False, data=None, params=None, + headers=None, **kw): + tid = urlparse(url).path.split("/")[-1] + + response = Mock() + response.headers = {"content-type": "application/json"} + response.status_code = 200 + response.content = b'' + + if method == "GET": + if tid not in self._store: + return self._not_found_response() + response.content = self._store.get(tid) + elif method == "PUT": + self._store[tid] = data + response.content = str_to_bytes(f'{{"ok":true,"id":"{tid}","rev":"1-revid"}}') + elif method == "HEAD": + if tid not in self._store: + return self._not_found_response() + response.headers.update({"etag": "1-revid"}) + elif method == "DELETE": + if tid not in self._store: + return self._not_found_response() + del self._store[tid] + response.content = str_to_bytes(f'{{"ok":true,"id":"{tid}","rev":"1-revid"}}') + else: + raise NotImplementedError(f"CouchSessionMock.request() does not handle {method} method") + + return response + + def _not_found_response(self): + response = Mock() + response.headers = {"content-type": "application/json"} + response.status_code = 404 + response.content = str_to_bytes('{"error":"not_found","reason":"missing"}') + return response + + +class test_CouchBackend_result: + def setup_method(self): + self.backend = CouchBackend(app=self.app) + resource = pycouchdb.resource.Resource("resource-url", session=CouchSessionMock()) + self.backend._connection = pycouchdb.client.Database(resource, "container") + + def test_get_set_forget(self): + tid = uuid() + self.backend.store_result(tid, "successful-result", states.SUCCESS) + assert self.backend.get_state(tid) == states.SUCCESS + assert self.backend.get_result(tid) == "successful-result" + self.backend.forget(tid) + assert self.backend.get_state(tid) == states.PENDING + + def test_mark_as_started(self): + tid = uuid() + self.backend.mark_as_started(tid) + assert self.backend.get_state(tid) == states.STARTED + + def test_mark_as_revoked(self): + tid = uuid() + self.backend.mark_as_revoked(tid) + assert self.backend.get_state(tid) == states.REVOKED + + def test_mark_as_retry(self): + tid = uuid() + try: + raise KeyError('foo') + except KeyError as exception: + import traceback + trace = '\n'.join(traceback.format_stack()) + self.backend.mark_as_retry(tid, exception, traceback=trace) + assert self.backend.get_state(tid) == states.RETRY + assert isinstance(self.backend.get_result(tid), KeyError) + assert self.backend.get_traceback(tid) == trace + + def test_mark_as_failure(self): + tid = uuid() + try: + raise KeyError('foo') + except KeyError as exception: + import traceback + trace = '\n'.join(traceback.format_stack()) + self.backend.mark_as_failure(tid, exception, traceback=trace) + assert self.backend.get_state(tid) == states.FAILURE + assert isinstance(self.backend.get_result(tid), KeyError) + assert self.backend.get_traceback(tid) == trace diff --git a/t/unit/backends/test_database.py b/t/unit/backends/test_database.py index 328ee0c9c02..ffff028a755 100644 --- a/t/unit/backends/test_database.py +++ b/t/unit/backends/test_database.py @@ -45,6 +45,72 @@ def test_context_raises(self): session.close.assert_called_with() +@skip.if_pypy +class test_ModelsIdFieldTypeVariations: + + def test_for_mssql_dialect(self): + """Test that ID columns use BigInteger for MSSQL and Integer for other dialects.""" + from sqlalchemy import BigInteger, Integer + from sqlalchemy.dialects import mssql, mysql, oracle, postgresql, sqlite + + models = [Task, TaskSet] + id_columns = [m.__table__.columns['id'] for m in models] + + for dialect in [mssql, postgresql, mysql, sqlite, oracle]: + for id_column in id_columns: + compiled_type = id_column.type.dialect_impl(dialect.dialect()) + if dialect == mssql: + assert isinstance(compiled_type, BigInteger) + else: + assert isinstance(compiled_type, Integer) + + +@skip.if_pypy +class test_DateDoneIndex: + """Test that date_done columns have index=True on Task and TaskSet models.""" + + def test_task_date_done_has_index(self): + col = Task.__table__.columns['date_done'] + assert col.index is True, "Task.date_done should have index=True" + + def test_taskset_date_done_has_index(self): + col = TaskSet.__table__.columns['date_done'] + assert col.index is True, "TaskSet.date_done should have index=True" + + +class test_DateDoneColumnDefaults: + """Test that date_done column defaults are callables, not fixed values. + + The default and onupdate values must be callables (lambdas) so that + datetime.now() is evaluated per-row at INSERT/UPDATE time, not once + at module import time. + """ + + def test_task_date_done_default_is_callable(self): + """Task.date_done default should be a callable.""" + col = Task.__table__.columns['date_done'] + assert col.default is not None, \ + "Task.date_done should have a default" + assert callable(col.default.arg), \ + "Task.date_done default should be a callable, not a fixed datetime" + + def test_task_date_done_onupdate_is_callable(self): + """Task.date_done onupdate should be a callable (lambda).""" + col = Task.__table__.columns['date_done'] + assert col.onupdate is not None, \ + "Task.date_done should have an onupdate" + assert callable(col.onupdate.arg), \ + "Task.date_done onupdate should be a callable, not a fixed datetime" + + def test_taskset_date_done_default_is_callable(self): + """TaskSet.date_done default should be a callable.""" + col = TaskSet.__table__.columns['date_done'] + assert col.default is not None, \ + "TaskSet.date_done should have a default" + assert callable(col.default.arg), \ + "TaskSet.date_done default should be a callable, not a fixed datetime" + + @skip.if_pypy class test_DatabaseBackend: @@ -72,11 +138,115 @@ def raises(): raises(max_retries=5) assert calls[0] == 5 + def test_retry_helper_calls_on_backend_retryable_error(self): + from celery.backends.database import DatabaseError + + calls = [0] + hook_calls = [] + + mock_backend = Mock() + mock_backend.on_backend_retryable_error = Mock(side_effect=lambda exc: hook_calls.append(exc)) + + @retry + def raises_with_backend(backend): + calls[0] += 1 + raise DatabaseError(1, 2, 3) + + with pytest.raises(DatabaseError): + raises_with_backend(mock_backend, max_retries=3) + + assert calls[0] == 3 + assert mock_backend.on_backend_retryable_error.call_count == 3 + for exc in hook_calls: + assert isinstance(exc, DatabaseError) + + def test_retry_helper_without_hook(self): + from celery.backends.database import DatabaseError + + calls = [0] + + mock_backend = Mock(spec=[]) + + @retry + def raises_with_backend(backend): + calls[0] += 1 + raise DatabaseError(1, 2, 3) + + with pytest.raises(DatabaseError): + raises_with_backend(mock_backend, max_retries=3) + + assert calls[0] == 3 + + def test_retry_helper_hook_failure_continues(self): + from celery.backends.database import DatabaseError + + calls = [0] + + mock_backend = Mock() + mock_backend.on_backend_retryable_error = Mock(side_effect=RuntimeError("hook failed")) + + @retry + def raises_with_backend(backend): + calls[0] += 1 + raise DatabaseError(1, 2, 3) + + with pytest.raises(DatabaseError): + raises_with_backend(mock_backend, max_retries=3) + + assert calls[0] == 3 + assert mock_backend.on_backend_retryable_error.call_count == 3 + def test_missing_dburi_raises_ImproperlyConfigured(self): self.app.conf.database_url = None with pytest.raises(ImproperlyConfigured): DatabaseBackend(app=self.app) + def test_engine_options_include_pool_health_defaults(self): + tb = DatabaseBackend(self.uri, app=self.app) + assert tb.engine_options["pool_pre_ping"] is True + assert tb.engine_options["pool_recycle"] == 3600 + + def test_engine_options_explicit_values_override_defaults(self): + self.app.conf.database_engine_options = {"pool_pre_ping": False} + tb = DatabaseBackend( + self.uri, + app=self.app, + engine_options={"pool_recycle": 15}, + ) + assert tb.engine_options["pool_pre_ping"] is False + assert tb.engine_options["pool_recycle"] == 15 + + def test_exception_safe_to_retry(self): + from celery.backends.database import DatabaseError, InvalidRequestError, StaleDataError + + tb = DatabaseBackend(self.uri, app=self.app) + assert tb.exception_safe_to_retry(DatabaseError("", "", Exception("db error"))) + assert tb.exception_safe_to_retry(InvalidRequestError()) + assert tb.exception_safe_to_retry(StaleDataError()) + assert not tb.exception_safe_to_retry(RuntimeError("not retryable")) + + def test_exception_safe_to_retry_with_interface_error(self): + from celery.backends.database import InterfaceError + + tb = DatabaseBackend(self.uri, app=self.app) + assert tb.exception_safe_to_retry(InterfaceError("", None, Exception("connection lost"))) + + def test_on_backend_retryable_error_invalidates_session(self): + tb = DatabaseBackend(self.uri, app=self.app) + tb.session_manager.invalidate = Mock() + + tb.on_backend_retryable_error(RuntimeError("retryable")) + tb.session_manager.invalidate.assert_called_once_with(tb.url) + + def test_on_backend_retryable_error_called_with_exception(self): + tb = DatabaseBackend(self.uri, app=self.app) + tb.session_manager.invalidate = Mock() + mock_exc = RuntimeError("connection lost") + + tb.on_backend_retryable_error(mock_exc) + + tb.session_manager.invalidate.assert_called_once_with(tb.url) + def test_table_schema_config(self): self.app.conf.database_table_schemas = { 'task': 'foo', @@ -391,6 +561,34 @@ def test_get_engine_kwargs(self, create_engine): engine2 = s.get_engine('dburi', foo=1) assert engine2 is engine + def test_invalidate_disposes_cached_engine(self): + s = SessionManager() + engine = Mock() + s._engines['dburi'] = engine + s._sessions['dburi'] = Mock() + + s.invalidate('dburi') + + assert 'dburi' not in s._engines + assert 'dburi' not in s._sessions + engine.dispose.assert_called_once_with() + + def test_invalidate_nonexistent_dburi_is_noop(self): + s = SessionManager() + s.invalidate('nonexistent-dburi') + assert 'nonexistent-dburi' not in s._engines + assert 'nonexistent-dburi' not in s._sessions + + def test_invalidate_only_engine_cached(self): + s = SessionManager() + engine = Mock() + s._engines['dburi'] = engine + + s.invalidate('dburi') + + assert 'dburi' not in s._engines + engine.dispose.assert_called_once_with() + @patch('celery.backends.database.session.sessionmaker') def test_create_session_forked(self, sessionmaker): s = SessionManager() @@ -447,3 +645,21 @@ def raise_err(bind): manager.prepare_models(engine) assert mock_create_all.call_count == PREPARE_MODELS_MAX_RETRIES + 1 + + @patch('celery.backends.database.session.create_engine') + def test_get_engine_filters_nullpool_unsupported_kwargs(self, mock_create_engine): + """ + Test that QueuePool-specific kwargs (like pool_size and max_overflow) + are filtered out when creating an engine with NullPool. + """ + from celery.backends.database.session import NullPool + + s = SessionManager() + s.forked = False # Ensure we're in the non-forked code path + + s.get_engine('dburi', echo_pool=True, pool_size=10, max_overflow=5) + + mock_create_engine.assert_called_once_with( + 'dburi', + poolclass=NullPool, + ) diff --git a/t/unit/backends/test_mongodb.py b/t/unit/backends/test_mongodb.py index 9ae340ee149..075ce3d4862 100644 --- a/t/unit/backends/test_mongodb.py +++ b/t/unit/backends/test_mongodb.py @@ -130,17 +130,21 @@ def test_init_with_settings(self): 'celerydatabase?replicaSet=rs0') mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == MONGODB_BACKEND_HOST - assert mb.options == dict( - mb._prepare_client_options(), - replicaset='rs0', - ) + if 'replicaSet' in mb.options: # pragma: no cover # pymongo >= 4.14 + replicaset_option = 'replicaSet' + else: # pragma: no cover # pymongo < 4.14 + replicaset_option = 'replicaset' + assert mb.options == { + **mb._prepare_client_options(), + replicaset_option: 'rs0', + } assert mb.user == CELERY_USER assert mb.password == CELERY_PASSWORD assert mb.database_name == CELERY_DATABASE # same uri, change some parameters in backend settings self.app.conf.mongodb_backend_settings = { - 'replicaset': 'rs1', + replicaset_option: 'rs1', 'user': 'backenduser', 'database': 'another_db', 'options': { @@ -149,11 +153,11 @@ def test_init_with_settings(self): } mb = MongoBackend(app=self.app, url=uri) assert mb.mongo_host == MONGODB_BACKEND_HOST - assert mb.options == dict( - mb._prepare_client_options(), - replicaset='rs1', - socketKeepAlive=True, - ) + assert mb.options == { + **mb._prepare_client_options(), + replicaset_option: 'rs1', + 'socketKeepAlive': True, + } assert mb.user == 'backenduser' assert mb.password == CELERY_PASSWORD assert mb.database_name == 'another_db' @@ -222,11 +226,15 @@ def test_init_mongodb_dnspython2_pymongo4_seedlist(self): with patch('dns.resolver.resolve', side_effect=resolver): mb = self.perform_seedlist_assertions() - assert mb.options == dict( - mb._prepare_client_options(), - replicaset='rs0', - tls=True - ) + if 'replicaSet' in mb.options: # pragma: no cover # pymongo >= 4.14 + replicaset_option = 'replicaSet' + else: # pragma: no cover # pymongo < 4.14 + replicaset_option = 'replicaset' + assert mb.options == { + **mb._prepare_client_options(), + replicaset_option: 'rs0', + 'tls': True, + } def perform_seedlist_assertions(self): mb = MongoBackend(app=self.app, url=MONGODB_SEEDLIST_URI) @@ -299,12 +307,15 @@ def test_get_connection_with_authmechanism(self): mb = MongoBackend(app=self.app, url=uri) mock_Connection.return_value = sentinel.connection connection = mb._get_connection() + if 'authMechanism' in mb.options: # pragma: no cover # pymongo >= 4.14 + authmechanism_option = 'authMechanism' + else: # pragma: no cover # pymongo < 4.14 + authmechanism_option = 'authmechanism' mock_Connection.assert_called_once_with( host=['localhost:27017'], username=CELERY_USER, password=CELERY_PASSWORD, - authmechanism='SCRAM-SHA-256', - **mb._prepare_client_options() + **{**mb._prepare_client_options(), authmechanism_option: 'SCRAM-SHA-256'} ) assert sentinel.connection == connection @@ -319,10 +330,13 @@ def test_get_connection_with_authmechanism_no_username(self): 'SCRAM-SHA-256 requires a username.') with pytest.raises(ConfigurationError): mb._get_connection() + if 'authMechanism' in mb.options: # pragma: no cover # pymongo >= 4.14 + authmechanism_option = 'authMechanism' + else: # pragma: no cover # pymongo < 4.14 + authmechanism_option = 'authmechanism' mock_Connection.assert_called_once_with( host=['localhost:27017'], - authmechanism='SCRAM-SHA-256', - **mb._prepare_client_options() + **{**mb._prepare_client_options(), authmechanism_option: 'SCRAM-SHA-256'} ) @patch('celery.backends.mongodb.MongoBackend._get_connection') @@ -376,7 +390,7 @@ def test_store_result(self, mock_get_database): upsert=True) assert sentinel.result == ret_val - mock_collection.replace_one.side_effect = InvalidDocument() + mock_collection.replace_one.side_effect = InvalidDocument("bad") with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) @@ -403,7 +417,7 @@ def test_store_result_with_request(self, mock_get_database): assert parameters['parent_id'] == sentinel.parent_id assert sentinel.result == ret_val - mock_collection.replace_one.side_effect = InvalidDocument() + mock_collection.replace_one.side_effect = InvalidDocument("bad") with pytest.raises(EncodeError): self.backend._store_result( sentinel.task_id, sentinel.result, sentinel.status) diff --git a/t/unit/backends/test_redis.py b/t/unit/backends/test_redis.py index 314327ef174..c10495e2e45 100644 --- a/t/unit/backends/test_redis.py +++ b/t/unit/backends/test_redis.py @@ -10,9 +10,10 @@ import pytest try: - from redis import exceptions + from redis import CredentialProvider, exceptions except ImportError: exceptions = None + CredentialProvider = None from celery import signature, states, uuid from celery.canvas import Signature @@ -172,7 +173,8 @@ def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None, self.min_other_sentinels = min_other_sentinels self.connection_kwargs = connection_kwargs - def master_for(self, service_name, redis_class): + def master_for(self, service_name, redis_class, **kwargs): + self.master_for_kwargs = kwargs return random.choice(self.sentinels) @@ -369,6 +371,14 @@ def setup_method(self): self.b = self.Backend(app=self.app) +class MyCredentialProvider(CredentialProvider): + pass + + +class NonCredentialProvider: + pass + + class test_RedisBackend(basetest_RedisBackend): @pytest.mark.usefixtures('depends_on_current_app') def test_reduce(self): @@ -397,6 +407,33 @@ def test_username_password_from_redis_conf(self): assert x.connparams['username'] == 'username' assert x.connparams['password'] == 'password' + def test_credential_provider_from_redis_conf(self): + self.app.conf.redis_backend_credential_provider = "redis.CredentialProvider" + x = self.Backend(app=self.app) + + assert x.connparams + assert 'credential_provider' in x.connparams + assert 'username' not in x.connparams + assert 'password' not in x.connparams + + # with local credential provider + self.app.conf.redis_backend_credential_provider = MyCredentialProvider() + x = self.Backend(app=self.app) + assert x.connparams + assert 'credential_provider' in x.connparams + assert 'username' not in x.connparams + assert 'password' not in x.connparams + + # raise ImportError + self.app.conf.redis_backend_credential_provider = "not_exist.CredentialProvider" + with pytest.raises(ImportError): + self.Backend(app=self.app) + + # raise value Error + self.app.conf.redis_backend_credential_provider = NonCredentialProvider() + with pytest.raises(ValueError): + self.Backend(app=self.app) + def test_url(self): self.app.conf.redis_socket_timeout = 30.0 self.app.conf.redis_socket_connect_timeout = 100.0 @@ -424,6 +461,47 @@ def test_url(self): assert x.connparams['socket_timeout'] == 30.0 assert x.connparams['socket_connect_timeout'] == 100.0 + def test_url_with_credential_provider(self): + self.app.conf.redis_socket_timeout = 30.0 + self.app.conf.redis_socket_connect_timeout = 100.0 + x = self.Backend( + 'redis://:bosco@vandelay.com:123/1?credential_provider=redis.CredentialProvider', app=self.app, + ) + + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['socket_timeout'] == 30.0 + assert x.connparams['socket_connect_timeout'] == 100.0 + assert isinstance(x.connparams['credential_provider'], CredentialProvider) + assert "username" not in x.connparams + assert "password" not in x.connparams + + # without username and password + x = self.Backend( + 'redis://@vandelay.com:123/1?credential_provider=redis.UsernamePasswordCredentialProvider', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert isinstance(x.connparams['credential_provider'], CredentialProvider) + + # raise importError + with pytest.raises(ImportError): + self.Backend( + 'redis://@vandelay.com:123/1?credential_provider=not_exist.CredentialProvider', app=self.app, + ) + + # raise valueError + with pytest.raises(ValueError): + # some non-credential provider class + # not ideal but serve purpose + self.Backend( + 'redis://@vandelay.com:123/1?credential_provider=abc.ABC', app=self.app, + ) + def test_timeouts_in_url_coerced(self): pytest.importorskip('redis') @@ -536,6 +614,31 @@ def test_backend_health_check_interval_not_set(self): assert x.connparams['port'] == 123 assert "health_check_interval" not in x.connparams + def test_backend_redis_client_name(self): + pytest.importorskip('redis') + + self.app.conf.redis_client_name = 'celery-worker' + x = self.Backend( + 'redis://vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['client_name'] == 'celery-worker' + + def test_backend_redis_client_name_not_set(self): + pytest.importorskip('redis') + + x = self.Backend( + 'redis://vandelay.com:123//1', app=self.app, + ) + assert x.connparams + assert x.connparams['host'] == 'vandelay.com' + assert x.connparams['db'] == 1 + assert x.connparams['port'] == 123 + assert x.connparams['client_name'] is None + @pytest.mark.parametrize('cert_str', [ "required", "CERT_REQUIRED", @@ -1183,7 +1286,7 @@ def test_on_chord_part_return( self.b.client.zrange.assert_not_called() self.b.client.lrange.assert_not_called() # Confirm that the `GroupResult.restore` mock was called - complex_header_result.assert_called_once_with(request.group) + complex_header_result.assert_called_once_with(request.group, app=self.b.app) # Confirm that the callback was called with the `join()`ed group result if supports_native_join: expected_join = mock_result_obj.join_native @@ -1315,3 +1418,56 @@ def test_backend_ssl(self): from celery.backends.redis import SentinelManagedSSLConnection assert x.connparams['connection_class'] is SentinelManagedSSLConnection + + def test_url_with_acl_credentials(self): + x = self.Backend( + 'sentinel://myuser:mypass@github.com:123/1;' + 'sentinel://myuser:mypass@github.com:124/1', + app=self.app, + ) + assert x.connparams + assert "host" not in x.connparams + assert x.connparams['db'] == 1 + assert "port" not in x.connparams + assert x.connparams['password'] == "mypass" + assert x.connparams['username'] == "myuser" + assert len(x.connparams['hosts']) == 2 + + expected_usernames = ["myuser", "myuser"] + found_usernames = [cp['username'] for cp in x.connparams['hosts']] + assert found_usernames == expected_usernames + + def test_get_pool_with_acl_credentials(self): + x = self.Backend( + 'sentinel://myuser:mypass@github.com:123/1;' + 'sentinel://myuser:mypass@github.com:124/1', + app=self.app, + ) + with patch.object(x, '_get_sentinel_instance') as mock_get_sentinel: + mock_sentinel = Mock() + mock_sentinel.master_for.return_value = Mock(connection_pool=Mock()) + mock_get_sentinel.return_value = mock_sentinel + + x._get_pool(**x.connparams) + + mock_sentinel.master_for.assert_called_once() + call_kwargs = mock_sentinel.master_for.call_args[1] + assert call_kwargs.get('username') == 'myuser' + assert call_kwargs.get('password') == 'mypass' + + def test_get_pool_with_password_only(self): + x = self.Backend( + 'sentinel://:mypass@github.com:123/1', + app=self.app, + ) + with patch.object(x, '_get_sentinel_instance') as mock_get_sentinel: + mock_sentinel = Mock() + mock_sentinel.master_for.return_value = Mock(connection_pool=Mock()) + mock_get_sentinel.return_value = mock_sentinel + + x._get_pool(**x.connparams) + + mock_sentinel.master_for.assert_called_once() + call_kwargs = mock_sentinel.master_for.call_args[1] + assert 'username' not in call_kwargs + assert call_kwargs.get('password') == 'mypass' diff --git a/t/unit/backends/test_rpc.py b/t/unit/backends/test_rpc.py index 5d37689a31d..1c09f347b25 100644 --- a/t/unit/backends/test_rpc.py +++ b/t/unit/backends/test_rpc.py @@ -20,6 +20,155 @@ def test_drain_events_before_start(self): # drain_events shouldn't crash when called before start consumer.drain_events(0.001) + def test_drain_events_reconnects_on_connection_error(self): + consumer = self.get_consumer() + # Simulate a started consumer with a live connection. + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (OSError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = OSError( + 'Server unexpectedly closed connection' + ) + consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors + + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [Mock(name='queue1')] + consumer._consumer = mock_consumer + + # Patch app.connection() to return a fresh mock connection + # and Consumer to return a mock consumer. + new_conn = Mock(name='new_connection') + new_conn.connection_errors = (OSError,) + new_conn.channel_errors = () + new_kombu_consumer = Mock(name='new_kombu_consumer') + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=new_kombu_consumer) + + # drain_events should NOT raise; it should reconnect instead. + consumer.drain_events(timeout=1) + + # Old connection should be closed. + mock_conn.close.assert_called_once() + # New connection should be established. + consumer.app.connection.assert_called_once() + assert consumer._connection is new_conn + # New consumer should be consuming. + assert consumer._consumer is new_kombu_consumer + new_kombu_consumer.consume.assert_called_once() + + def test_drain_events_reconnect_preserves_queues(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (ConnectionError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = ConnectionError('reset') + consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors + + queue1, queue2 = Mock(name='q1'), Mock(name='q2') + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [queue1, queue2] + consumer._consumer = mock_consumer + + new_conn = Mock(name='new_connection') + new_conn.connection_errors = (ConnectionError,) + new_conn.channel_errors = () + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=Mock(name='new_kombu_consumer')) + + consumer.drain_events(timeout=1) + + # The new Consumer should have been created with both old queues. + new_consumer_call = consumer.Consumer.call_args + assert list(new_consumer_call[0][1]) == [queue1, queue2] + + def test_drain_events_no_reconnect_on_other_errors(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (OSError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = RuntimeError('unexpected') + consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors + + with pytest.raises(RuntimeError, match='unexpected'): + consumer.drain_events(timeout=1) + + def test_reconnect_handles_close_failures_gracefully(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.close.side_effect = OSError('already closed') + consumer._connection = mock_conn + + mock_consumer = Mock(name='consumer') + mock_consumer.cancel.side_effect = OSError('channel gone') + mock_consumer.queues = [Mock(name='queue1')] + consumer._consumer = mock_consumer + + new_conn = Mock(name='new_connection') + new_conn.connection_errors = (OSError,) + new_conn.channel_errors = () + new_kombu_consumer = Mock(name='new_kombu_consumer') + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=new_kombu_consumer) + + # _reconnect should NOT raise even if cancel/close fail + consumer._reconnect() + + assert consumer._connection is new_conn + new_kombu_consumer.consume.assert_called_once() + + def test_drain_events_channel_error_triggers_reconnect(self): + consumer = self.get_consumer() + mock_conn = Mock(name='connection') + mock_conn.connection_errors = () + mock_conn.channel_errors = (KeyError,) + mock_conn.drain_events.side_effect = KeyError('channel closed') + consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors + + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [] + consumer._consumer = mock_consumer + + new_conn = Mock(name='new_connection') + new_conn.connection_errors = () + new_conn.channel_errors = (KeyError,) + consumer.app = Mock() + consumer.app.connection.return_value = new_conn + consumer.Consumer = Mock(return_value=Mock(name='new_kombu_consumer')) + + consumer.drain_events(timeout=1) + + assert consumer._connection is new_conn + + def test_drain_events_raises_runtime_when_reconnect_also_fails(self): + consumer = self.get_consumer() + + class FakeConnError(Exception): + pass + + mock_conn = Mock(name='connection') + mock_conn.connection_errors = (FakeConnError,) + mock_conn.channel_errors = () + mock_conn.drain_events.side_effect = FakeConnError('dropped') + consumer._connection = mock_conn + consumer._connection_errors = mock_conn.connection_errors + mock_conn.channel_errors + + mock_consumer = Mock(name='consumer') + mock_consumer.queues = [] + consumer._consumer = mock_consumer + + consumer.app = Mock() + consumer.app.connection.side_effect = FakeConnError('still down') + + with pytest.raises(RuntimeError, match='Retry limit exceeded'): + consumer.drain_events(timeout=1) + class test_RPCBackend: diff --git a/t/unit/bin/test_control.py b/t/unit/bin/test_control.py index 74f6e4fb1ca..217b37a8c0d 100644 --- a/t/unit/bin/test_control.py +++ b/t/unit/bin/test_control.py @@ -4,6 +4,7 @@ import pytest from click.testing import CliRunner +from kombu.exceptions import OperationalError from celery.bin.celery import celery from celery.platforms import EX_UNAVAILABLE @@ -80,3 +81,157 @@ def test_listing_remote_commands(celery_cmd, expected_regex, isolated_cli_runner ) assert res.exit_code == 0, (res, res.stdout) assert expected_regex.search(res.stdout) + + +def test_status_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.app.control.Inspect.ping', + side_effect=OperationalError('[Errno 61] Connection refused')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'status'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: [Errno 61] Connection refused' in res.output + assert 'Traceback' not in res.output + + +def test_status_unexpected_error_is_summarized(cli_runner: CliRunner): + with patch('celery.app.control.Inspect.ping', + side_effect=RuntimeError('boom')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'status'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Unable to run the `status` command. Reason: boom' in res.output + assert 'Traceback' not in res.output + + +def test_graph_workers_shows_friendly_error_when_broker_unreachable( + cli_runner: CliRunner, +): + with patch('celery.app.control.Inspect.stats', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'graph', 'workers'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + + +def test_events_dump_shows_friendly_error_when_broker_unreachable( + cli_runner: CliRunner, +): + with patch('celery.bin.events._run_evdump', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'events', '--dump'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + + +def test_handle_remote_command_error_reraises_click_exception(): + """base.py: bare ``raise`` inside the ClickException branch must be covered.""" + import click + + from celery.bin.base import handle_remote_command_error + + original = click.ClickException('original click error') + with pytest.raises(click.ClickException) as exc_info: + try: + raise original + except Exception as exc: + handle_remote_command_error('any', exc) + assert exc_info.value is original + + +def test_inspect_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.app.control.Inspect._request', + side_effect=OperationalError('connection refused')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'inspect', *_INSPECT_OPTIONS, 'custom_inspect_cmd', '1'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection refused' in res.output + assert 'Traceback' not in res.output + + +def test_inspect_unexpected_error_is_summarized(cli_runner: CliRunner): + with patch('celery.app.control.Inspect._request', + side_effect=RuntimeError('inspect boom')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'inspect', *_INSPECT_OPTIONS, 'custom_inspect_cmd', '1'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Unable to run the `inspect custom_inspect_cmd` command. Reason: inspect boom' in res.output + assert 'Traceback' not in res.output + + +def test_control_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.app.control.Control.broadcast', + side_effect=OperationalError('connection refused')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'control', *_INSPECT_OPTIONS, 'custom_control_cmd', '1', '2'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection refused' in res.output + assert 'Traceback' not in res.output + + +def test_control_unexpected_error_is_summarized(cli_runner: CliRunner): + with patch('celery.app.control.Control.broadcast', + side_effect=RuntimeError('control boom')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'control', *_INSPECT_OPTIONS, 'custom_control_cmd', '1', '2'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Unable to run the `control custom_control_cmd` command. Reason: control boom' in res.output + assert 'Traceback' not in res.output + + +def test_events_camera_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.bin.events._run_evcam', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'events', '--camera', 'myapp.MyCameraClass'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + assert 'Traceback' not in res.output + + +def test_events_evtop_shows_friendly_error_when_broker_unreachable(cli_runner: CliRunner): + with patch('celery.bin.events._run_evtop', + side_effect=OperationalError('connection failed')): + res = cli_runner.invoke( + celery, + [*_GLOBAL_OPTIONS, 'events'], + catch_exceptions=False, + ) + assert res.exit_code == EX_UNAVAILABLE, (res, res.output) + assert 'Error: Could not connect to the message broker.' in res.output + assert 'Reason: connection failed' in res.output + assert 'Traceback' not in res.output diff --git a/t/unit/bin/test_worker.py b/t/unit/bin/test_worker.py index b63a2a03306..baa73385d6c 100644 --- a/t/unit/bin/test_worker.py +++ b/t/unit/bin/test_worker.py @@ -1,11 +1,12 @@ import os -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest from click.testing import CliRunner from celery.app.log import Logging from celery.bin.celery import celery +from celery.worker.consumer.tasks import Tasks @pytest.fixture(scope='session') @@ -13,6 +14,41 @@ def use_celery_app_trap(): return False +@pytest.fixture +def mock_app(): + app = Mock() + app.conf = Mock() + app.conf.worker_disable_prefetch = False + app.conf.worker_detect_quorum_queues = False + return app + + +@pytest.fixture +def mock_consumer(mock_app): + consumer = Mock() + consumer.app = mock_app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.connection = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' # Default to Redis for existing tests + consumer.connection.qos_semantics_matches_spec = True + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + consumer.app.amqp.queues = {} # Empty dict for quorum queue detection + return consumer + + def test_cli(isolated_cli_runner: CliRunner): Logging._setup = True # To avoid hitting the logging sanity checks res = isolated_cli_runner.invoke( @@ -33,3 +69,59 @@ def test_cli_skip_checks(isolated_cli_runner: CliRunner): ) assert res.exit_code == 1, (res, res.stdout) assert os.environ["CELERY_SKIP_CHECKS"] == "true", "should set CELERY_SKIP_CHECKS" + + +def test_cli_disable_prefetch_flag(isolated_cli_runner: CliRunner): + Logging._setup = True + with patch('celery.bin.worker.worker.callback') as worker_callback_mock: + res = isolated_cli_runner.invoke( + celery, + ["-A", "t.unit.bin.proj.app", "worker", "--pool", "solo", "--disable-prefetch"], + catch_exceptions=False, + ) + assert res.exit_code == 0 + _, kwargs = worker_callback_mock.call_args + assert kwargs['disable_prefetch'] is True + + +def test_disable_prefetch_affects_qos_behavior(mock_app, mock_consumer): + mock_app.conf.worker_disable_prefetch = True + original_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + with patch('celery.worker.state.reserved_requests', []): + tasks_instance = Tasks(mock_consumer) + tasks_instance.start(mock_consumer) + assert mock_consumer.task_consumer.channel.qos.can_consume != original_can_consume + modified_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + with patch('celery.worker.state.reserved_requests', list(range(4))): + assert not modified_can_consume() + with patch('celery.worker.state.reserved_requests', list(range(2))): + original_can_consume.return_value = True + assert modified_can_consume() + original_can_consume.return_value = False + assert not modified_can_consume() + + +def test_disable_prefetch_none_preserves_behavior(mock_app, mock_consumer): + mock_app.conf.worker_disable_prefetch = False + kwargs_with_none = {'disable_prefetch': None} + if 'disable_prefetch' in kwargs_with_none and kwargs_with_none['disable_prefetch'] is not None: + mock_app.conf.worker_disable_prefetch = kwargs_with_none.pop('disable_prefetch') + assert mock_app.conf.worker_disable_prefetch is False + assert 'disable_prefetch' in kwargs_with_none + original_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + tasks_instance = Tasks(mock_consumer) + tasks_instance.start(mock_consumer) + assert mock_consumer.task_consumer.channel.qos.can_consume == original_can_consume + + +def test_disable_prefetch_ignored_for_non_redis_brokers(mock_app, mock_consumer): + """Test that disable_prefetch is ignored for non-Redis brokers.""" + mock_app.conf.worker_disable_prefetch = True + mock_consumer.connection.transport.driver_type = 'amqp' # RabbitMQ + original_can_consume = mock_consumer.task_consumer.channel.qos.can_consume + + tasks_instance = Tasks(mock_consumer) + tasks_instance.start(mock_consumer) + + # Should not modify can_consume method for non-Redis brokers + assert mock_consumer.task_consumer.channel.qos.can_consume == original_can_consume diff --git a/t/unit/concurrency/test_prefork.py b/t/unit/concurrency/test_prefork.py index ea42c09bad9..8f266f78302 100644 --- a/t/unit/concurrency/test_prefork.py +++ b/t/unit/concurrency/test_prefork.py @@ -3,7 +3,7 @@ import socket import tempfile from itertools import cycle -from unittest.mock import Mock, patch +from unittest.mock import MagicMock, Mock, patch import pytest from billiard.pool import ApplyResult @@ -488,12 +488,252 @@ def test_before_create_process_signal(self, create_process): sender=pool, ) + def test_untrack_child_process_without_sentinel_poll(self): + """_untrack_child_process must not raise when proc lacks _sentinel_poll. -@t.skip.if_win32 -class test_ResultHandler: + Race condition during cold shutdown can cause _untrack_child_process to + be called with a process that never had _sentinel_poll set or had it + cleared. Use getattr for safe access. + """ + pytest.importorskip('multiprocessing') + pool = asynpool.AsynPool(processes=1, threads=False) + hub = Mock(name='hub') + proc = object() # No _sentinel_poll attribute + pool._untrack_child_process(proc, hub) # Should not raise AttributeError + hub.remove.assert_not_called() - def setup_method(self): + def test_untrack_child_process_with_sentinel_poll(self): + """_untrack_child_process cleans up when proc has _sentinel_poll set.""" pytest.importorskip('multiprocessing') + pool = asynpool.AsynPool(processes=1, threads=False) + hub = Mock(name='hub') + fd = os.open(os.devnull, os.O_RDONLY) + proc = Mock(_sentinel_poll=fd) + pool._untrack_child_process(proc, hub) + hub.remove.assert_called_once_with(fd) + assert proc._sentinel_poll is None + + @t.skip.if_pypy + def test_flush_no_synack_discards_unaccepted_jobs(self): + """flush() should discard unaccepted jobs when synack is disabled. + + Previously, flush() only handled the synack case. Without synack, + unaccepted jobs were never cleaned from the cache, leading to stale + entries. + """ + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + job1 = Mock(name='job1') + job1._accepted = False + job1._writer.return_value = None + job2 = Mock(name='job2') + job2._accepted = True + job2._writer.return_value = None + + pool._cache = {1: job1, 2: job2} + pool.outbound_buffer.clear() + pool._active_writers.clear() + + pool.flush() + + job1.discard.assert_called_once() + job2.discard.assert_not_called() + + @t.skip.if_pypy + def test_flush_synack_cancels_unaccepted_jobs(self): + """flush() should call _cancel() on unaccepted jobs when synack is enabled.""" + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + job1 = Mock(name='job1') + job1._accepted = False + job1._writer.return_value = None + job2 = Mock(name='job2') + job2._accepted = True + job2._writer.return_value = None + + pool._cache = {1: job1, 2: job2} + pool.outbound_buffer.clear() + pool._active_writers.clear() + + pool.flush() + + job1._cancel.assert_called_once() + job1.discard.assert_not_called() + job2._cancel.assert_not_called() + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_dead_process_discards_active_writer(self, _create_worker_process): + """flush() must discard generator from _active_writers when process is dead. + + Previously, when a process was dead, the generator was never removed + from _active_writers, causing an infinite loop in the while loop. + """ + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + # Create a mock generator (already started, so not gen_not_started) + gen = Mock(name='gen') + gen.__name__ = '_write_job' + # Simulate a started generator + with patch.object(asynpool, 'gen_not_started', return_value=False): + proc = Mock(name='proc') + proc._is_alive.return_value = False # Process is dead + + job = Mock(name='job') + job._accepted = True + job._write_to = proc + job._writer.return_value = gen + + pool._cache = {1: job} + pool._active_writers = {gen} + pool.outbound_buffer.clear() + + pool.flush() + + # Generator should have been removed from active_writers + assert gen not in pool._active_writers + # Job should have been discarded since process is dead + job.discard.assert_called() + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_alive_process_flushes_writer(self, _create_worker_process): + """flush() should call _flush_writer when process is still alive.""" + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + gen = Mock(name='gen') + gen.__name__ = '_write_job' + + with patch.object(asynpool, 'gen_not_started', return_value=False): + proc = Mock(name='proc') + proc._is_alive.return_value = True + + job = Mock(name='job') + job._accepted = True + job._write_to = proc + job._writer.return_value = gen + + pool._cache = {1: job} + pool._active_writers = {gen} + pool.outbound_buffer.clear() + + with patch.object(pool, '_flush_writer') as mock_flush: + # _flush_writer removes from _active_writers in its finally + def side_effect(p, g): + pool._active_writers.discard(g) + mock_flush.side_effect = side_effect + + pool.flush() + + mock_flush.assert_called_once_with(proc, gen) + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_write_ack_coroutine_is_advanced_not_dropped(self, _create_worker_process): + """flush() must not silently drop _write_ack generators (synack mode). + + _write_ack coroutines are added to _active_writers by send_ack() but + are NOT mapped in owned_by (which is built from _cache job writers only). + Dropping them mid-write leaves a partially-written ack on the synq pipe + and hangs the worker process waiting for the ack that never arrives. + flush() must advance them to completion instead. + """ + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + # Simulate a _write_ack generator (name != '_write_job', not in owned_by) + ack_gen = Mock(name='ack_gen') + ack_gen.__name__ = '_write_ack' + # First call to next() returns normally (still writing), + # second raises StopIteration (write complete). + ack_gen.__next__ = Mock(side_effect=[None, StopIteration()]) + + pool._cache = {} + pool._active_writers = {ack_gen} + pool.outbound_buffer.clear() + + pool.flush() + + # Generator should have been advanced (not just silently discarded) + assert ack_gen.__next__.called + # And removed once it signalled completion + assert ack_gen not in pool._active_writers + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_write_ack_coroutine_handles_oserror(self, _create_worker_process): + """flush() should discard the coroutine if OSError is raised during next().""" + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + ack_gen = MagicMock(name='ack_gen') + ack_gen.__name__ = '_write_ack' + ack_gen.__next__.side_effect = OSError() + + pool._cache = {} + pool._active_writers = {ack_gen} + pool.outbound_buffer.clear() + pool.flush() + + assert ack_gen not in pool._active_writers + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_write_ack_coroutine_handles_eoferror(self, _create_worker_process): + """flush() should discard the coroutine if EOFError is raised during next().""" + pool = asynpool.AsynPool(processes=1, synack=True, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + ack_gen = MagicMock(name='ack_gen') + ack_gen.__name__ = '_write_ack' + ack_gen.__next__.side_effect = EOFError() + + pool._cache = {} + pool._active_writers = {ack_gen} + pool.outbound_buffer.clear() + pool.flush() + + assert ack_gen not in pool._active_writers + + @t.skip.if_pypy + @patch('billiard.pool.Pool._create_worker_process') + def test_flush_not_started_write_job_is_discarded(self, _create_worker_process): + """flush() should discard a _write_job generator that has not started yet. + + When gen_not_started() returns True the job has not been written to the + pipe at all, so it is safe to discard it and let the broker redeliver. + """ + pool = asynpool.AsynPool(processes=1, synack=False, threads=False) + pool._state = asynpool.RUN + pool.maintain_pool = Mock(name='maintain_pool') + + gen = Mock(name='gen') + gen.__name__ = '_write_job' + + with patch.object(asynpool, 'gen_not_started', return_value=True): + job = Mock(name='job') + job._accepted = True + job._writer.return_value = gen + + pool._cache = {1: job} + pool._active_writers = {gen} + pool.outbound_buffer.clear() + + pool.flush() + + job.discard.assert_called_once() + assert gen not in pool._active_writers def test_process_result(self): x = asynpool.ResultHandler( @@ -572,6 +812,85 @@ def test_on_close__pool_not_running(self): pool.on_close() pool._pool.close.assert_not_called() + @patch('celery.concurrency.prefork.get_event_loop') + @patch('celery.concurrency.prefork.threading.Thread') + def test_on_stop_with_hub_fires_timers(self, mock_thread, mock_get_event_loop): + pool = TaskPool(10) + mock_pool = Mock(name='pool') + mock_pool._state = mp.RUN + pool._pool = mock_pool + + mock_hub = Mock(name='hub') + mock_get_event_loop.return_value = mock_hub + mock_timer_thread = Mock(name='timer_thread') + mock_thread.return_value = mock_timer_thread + + pool.on_stop() + + mock_pool.close.assert_called_with() + mock_pool.join.assert_called_with() + mock_get_event_loop.assert_called_once() + mock_thread.assert_called_once() + assert mock_thread.call_args[1]['daemon'] is True + mock_timer_thread.start.assert_called_once() + mock_timer_thread.join.assert_called_once_with(timeout=1.0) + + @patch('celery.concurrency.prefork.get_event_loop') + @patch('celery.concurrency.prefork.threading.Thread') + @patch('celery.concurrency.prefork.threading.Event') + def test_on_stop_timer_thread_handles_exceptions( + self, + mock_event_class, + mock_thread, + mock_get_event_loop, + ): + pool = TaskPool(10) + mock_pool = Mock(name='pool') + mock_pool._state = mp.RUN + pool._pool = mock_pool + + mock_hub = Mock(name='hub') + mock_hub.fire_timers.side_effect = [Exception("Hub error"), None] + mock_get_event_loop.return_value = mock_hub + + mock_shutdown_event = Mock(name='shutdown_event') + # Simulate two loop iterations and then shutdown + mock_shutdown_event.is_set.side_effect = [False, False, True] + mock_event_class.return_value = mock_shutdown_event + + thread_target = None + + def capture_thread(*args, **kwargs): + nonlocal thread_target + thread_target = kwargs['target'] + mock_timer_thread = Mock(name='timer_thread') + return mock_timer_thread + + mock_thread.side_effect = capture_thread + + pool.on_stop() + + with patch('celery.concurrency.prefork.time.sleep'): + thread_target() + + # Should match number of loop iterations allowed by mock_shutdown_event.is_set.side_effect + assert mock_hub.fire_timers.call_count == 2 + + @patch('celery.concurrency.prefork.get_event_loop') + def test_on_stop_no_hub(self, mock_get_event_loop): + pool = TaskPool(10) + mock_pool = Mock(name='pool') + mock_pool._state = mp.RUN + pool._pool = mock_pool + + mock_get_event_loop.return_value = None + + pool.on_stop() + + mock_pool.close.assert_called_with() + mock_pool.join.assert_called_with() + mock_get_event_loop.assert_called_once() + def test_apply_async(self): pool = TaskPool(10) pool.start() diff --git a/t/unit/contrib/proj/contents.rst b/t/unit/contrib/proj/contents.rst index 5ba93e82eba..43701b1a21b 100644 --- a/t/unit/contrib/proj/contents.rst +++ b/t/unit/contrib/proj/contents.rst @@ -5,3 +5,5 @@ Documentation .. automodule:: foo :members: + +Cross-reference test: :task:`foo.bar` diff --git a/t/unit/contrib/test_sphinx.py b/t/unit/contrib/test_sphinx.py index 0a5abceab91..dcdceb16562 100644 --- a/t/unit/contrib/test_sphinx.py +++ b/t/unit/contrib/test_sphinx.py @@ -28,3 +28,6 @@ def test_sphinx(): 'This task is in a different module!' not in contents ) + # Verify :task: cross-reference role resolves to a link + assert 'foo.bar' in contents + assert 'Cross-reference test' in contents diff --git a/t/unit/contrib/test_worker.py b/t/unit/contrib/test_worker.py index 4534317ae83..fe4ac2ef8e5 100644 --- a/t/unit/contrib/test_worker.py +++ b/t/unit/contrib/test_worker.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock, patch + import pytest # this import adds a @shared_task, which uses connect_on_app_finalize @@ -57,3 +59,38 @@ def test_start_worker_with_hostname_config(self): result = self.add.s(1, 2).apply_async() val = result.get(timeout=5) assert val == 3 + + +class test_TestWorkController: + + @patch('celery.contrib.testing.worker.worker.WorkController.__init__') + def test_init_with_string_pool_cls_prefork(self, mock_super_init): + mock_super_init.return_value = None + controller = object.__new__(TestWorkController) + controller._on_started = None + controller.pool_cls = 'prefork' + with patch('celery.contrib.testing.worker.logging.handlers.QueueListener') as mock_listener: + with patch('billiard.Queue') as mock_queue: + with patch.dict('sys.modules', {'tblib': None, 'tblib.pickling_support': None}): + mock_queue.return_value = Mock() + controller.__init__(app=Mock()) + mock_listener.assert_called_once() + assert controller.logger_queue is not None + + @patch('celery.contrib.testing.worker.worker.WorkController.__init__') + def test_init_with_string_pool_cls_solo(self, mock_super_init): + mock_super_init.return_value = None + controller = object.__new__(TestWorkController) + controller._on_started = None + controller.pool_cls = 'solo' + controller.__init__(app=Mock()) + assert controller.logger_queue is None + + @patch('celery.contrib.testing.worker.worker.WorkController.__init__') + def test_init_with_string_pool_cls_gevent(self, mock_super_init): + mock_super_init.return_value = None + controller = object.__new__(TestWorkController) + controller._on_started = None + controller.pool_cls = 'gevent' + controller.__init__(app=Mock()) + assert controller.logger_queue is None diff --git a/t/unit/events/test_dumper.py b/t/unit/events/test_dumper.py index e6f8a577e99..eb259db49d3 100644 --- a/t/unit/events/test_dumper.py +++ b/t/unit/events/test_dumper.py @@ -1,5 +1,5 @@ import io -from datetime import datetime +from datetime import datetime, timezone from celery.events import dumper @@ -39,7 +39,7 @@ def test_on_event_task_received(): buf = io.StringIO() d = dumper.Dumper(out=buf) event = { - 'timestamp': datetime(2024, 1, 1, 12, 0, 0).timestamp(), + 'timestamp': datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc).timestamp(), 'type': 'task-received', 'hostname': 'worker1', 'uuid': 'abc', @@ -49,7 +49,7 @@ def test_on_event_task_received(): } d.on_event(event.copy()) output = buf.getvalue() - assert 'worker1 [2024-01-01 12:00:00]' in output + assert 'worker1 [2024-01-01 12:00:00+00:00]' in output assert 'task received' in output assert 'mytask(abc) args=(1,) kwargs={}' in output @@ -58,13 +58,13 @@ def test_on_event_non_task(): buf = io.StringIO() d = dumper.Dumper(out=buf) event = { - 'timestamp': datetime(2024, 1, 1, 12, 0, 0).timestamp(), + 'timestamp': datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc).timestamp(), 'type': 'worker-online', 'hostname': 'worker1', 'foo': 'bar', } d.on_event(event.copy()) output = buf.getvalue() - assert 'worker1 [2024-01-01 12:00:00]' in output + assert 'worker1 [2024-01-01 12:00:00+00:00]' in output assert 'started' in output assert 'foo=bar' in output diff --git a/t/unit/events/test_events.py b/t/unit/events/test_events.py index 21fcc5003f1..ae2c4e4930c 100644 --- a/t/unit/events/test_events.py +++ b/t/unit/events/test_events.py @@ -5,6 +5,7 @@ from celery.events import Event from celery.events.receiver import CLIENT_CLOCK_SKEW +from celery.exceptions import ImproperlyConfigured class MockProducer: @@ -327,6 +328,39 @@ def handler(event): channel.close() connection.close() + def test_event_queue_exclusive(self): + self.app.conf.update( + event_queue_exclusive=True, + event_queue_durable=False + ) + + ev_recv = self.app.events.Receiver(Mock(name='connection')) + q = ev_recv.queue + + assert q.exclusive is True + assert q.durable is False + assert q.auto_delete is True + + def test_event_queue_durable_and_validation(self): + self.app.conf.update( + event_queue_exclusive=False, + event_queue_durable=True + ) + ev_recv = self.app.events.Receiver(Mock(name='connection')) + q = ev_recv.queue + + assert q.durable is True + assert q.exclusive is False + assert q.auto_delete is False + + self.app.conf.update( + event_queue_exclusive=True, + event_queue_durable=True + ) + + with pytest.raises(ImproperlyConfigured): + self.app.events.Receiver(Mock(name='connection')) + def test_State(app): state = app.events.State() diff --git a/t/unit/fixups/test_django.py b/t/unit/fixups/test_django.py index c09ba61642c..a75dbd90c1e 100644 --- a/t/unit/fixups/test_django.py +++ b/t/unit/fixups/test_django.py @@ -3,6 +3,7 @@ import pytest +from celery.concurrency.thread import TaskPool as ThreadTaskPool from celery.fixups.django import DjangoFixup, DjangoWorkerFixup, FixupWarning, _maybe_close_fd, fixup from t.unit import conftest @@ -11,11 +12,14 @@ class FixupCase: Fixup = None @contextmanager - def fixup_context(self, app): + def fixup_context(self, app, **kwargs): with patch('celery.fixups.django.DjangoWorkerFixup.validate_models'): with patch('celery.fixups.django.symbol_by_name') as symbyname: with patch('celery.fixups.django.import_module') as impmod: - f = self.Fixup(app) + worker = Mock() + worker.pool_cls = Mock(__module__='celery.concurrency.prefork') + f = self.Fixup(app, **kwargs) + f.worker = worker yield f, impmod, symbyname @@ -150,11 +154,20 @@ def test_now(self): def test_on_worker_init(self): with self.fixup_context(self.app) as (f, _, _): with patch('celery.fixups.django.DjangoWorkerFixup') as DWF: - f.on_worker_init() + mock_worker = Mock(name="worker") + f.on_worker_init(sender=mock_worker) + assert DWF.return_value.worker == mock_worker + DWF.assert_called_with(f.app) DWF.return_value.install.assert_called_with() assert f._worker_fixup is DWF.return_value + def test_on_worker_init_warns_without_sender(self): + with self.fixup_context(self.app) as (f, _, _): + with patch("celery.fixups.django.DjangoWorkerFixup"): + with pytest.warns(FixupWarning, match="called without a sender"): + f.on_worker_init(sender=None) + class InterfaceError(Exception): pass @@ -196,7 +209,7 @@ def test_on_worker_process_init(self, patching): f.on_worker_process_init() mcf.assert_called_with(conns[1].connection) f.close_cache.assert_called_with() - f._close_database.assert_called_with(force=True) + f._close_database.assert_called_with() f.validate_models = Mock(name='validate_models') patching.setenv('FORKED_BY_MULTIPROCESSING', '1') @@ -262,40 +275,139 @@ def test__close_database(self): f.interface_errors = () f._db.connections = Mock() # ConnectionHandler - f._db.connections.all.side_effect = lambda: conns + f._db.connections.all.side_effect = lambda initialized_only: conns - f._close_database(force=True) + f._close_database() conns[0].close.assert_called_with() - conns[0].close_if_unusable_or_obsolete.assert_not_called() conns[1].close.assert_called_with() - conns[1].close_if_unusable_or_obsolete.assert_not_called() conns[2].close.assert_called_with() - conns[2].close_if_unusable_or_obsolete.assert_not_called() - - for conn in conns: - conn.reset_mock() - - f._close_database() - conns[0].close.assert_not_called() - conns[0].close_if_unusable_or_obsolete.assert_called_with() - conns[1].close.assert_not_called() - conns[1].close_if_unusable_or_obsolete.assert_called_with() - conns[2].close.assert_not_called() - conns[2].close_if_unusable_or_obsolete.assert_called_with() conns[1].close.side_effect = KeyError( 'omg') - f._close_database() - with pytest.raises(KeyError): - f._close_database(force=True) - - conns[1].close.side_effect = None - conns[1].close_if_unusable_or_obsolete.side_effect = KeyError( - 'omg') - f._close_database(force=True) with pytest.raises(KeyError): f._close_database() + def test__close_database_django_pre_41(self): + """Test that Django < 4.1 (without initialized_only parameter) is handled.""" + with self.fixup_context(self.app) as (f, _, _): + conns = [Mock(), Mock()] + f.DatabaseError = KeyError + f.interface_errors = () + + # Mock Django < 4.1 behavior: connections.all() doesn't accept initialized_only + f._db.connections = Mock() + + def all_without_initialized_only(**kwargs): + if 'initialized_only' in kwargs: + raise TypeError("all() got an unexpected keyword argument 'initialized_only'") + return conns + + f._db.connections.all = Mock(side_effect=all_without_initialized_only) + + # Should fall back to calling all() without initialized_only + f._close_database() + + # Verify it was called twice: first with initialized_only (raises), then without + assert f._db.connections.all.call_count == 2 + # First call with initialized_only=True + f._db.connections.all.assert_any_call(initialized_only=True) + # Second call without arguments (fallback) + f._db.connections.all.assert_any_call() + + # Verify connections were closed + conns[0].close.assert_called_with() + conns[1].close.assert_called_with() + + def test_close_database_always_closes_connections(self): + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + f._db.connections.all = Mock(return_value=[conn]) + f.close_database() + conn.close.assert_called_once_with() + # close_if_unusable_or_obsolete is not safe to call in all conditions, so avoid using + # it to optimize connection handling. + conn.close_if_unusable_or_obsolete.assert_not_called() + + def test_close_database_skip_conn_pool(self): + class Connection: + """Mock connection without `close_pool` method.""" + alias = 'default' + + def close(self): + pass + + with self.fixup_context(self.app) as (f, _, _): + conn = Mock(spec=Connection) + f._db.connections.all = Mock(return_value=[conn]) + f.close_database() + assert not hasattr(conn, "close_pool") + conn.close.assert_called_once_with() + + def test_close_database_suppresses_close_pool_keyerror(self): + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + conn.close_pool = Mock(side_effect=KeyError("pool already closed")) + f._db.connections.all = Mock(return_value=[conn]) + f.close_database() # should not raise + conn.close.assert_called_once_with() + conn.close_pool.assert_called_once_with() + + def test_close_database_conn_pool_based_on_settings(self): + class DJSettings: + DATABASES = {} + + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + conn.alias = "default" + conn.close_pool = Mock() + f._db.connections.all = Mock(return_value=[conn]) + f._settings = DJSettings + + f._settings.DATABASES["default"] = {"OPTIONS": {}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_not_called() + + conn.reset_mock() + f._settings.DATABASES["default"] = {"OPTIONS": {"pool": True}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_called_once_with() + + conn.reset_mock() + f._settings.DATABASES["default"] = {"OPTIONS": {"pool": False}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_not_called() + + def test_close_database_conn_pool_thread_pool(self): + class DJSettings: + DATABASES = {} + + with self.fixup_context(self.app) as (f, _, _): + conn = Mock() + conn.alias = "default" + conn.close_pool = Mock() + f._db.connections.all = Mock(return_value=[conn]) + f._settings = DJSettings + + f._settings.DATABASES["default"] = {"OPTIONS": {"pool": True}} + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_called_once_with() + + conn.reset_mock() + f.worker.pool_cls = ThreadTaskPool + assert "prefork" not in ThreadTaskPool.__module__ + f.close_database() + conn.close.assert_called_once_with() + conn.close_pool.assert_not_called() + + def test_close_cache_raises_error(self): + with self.fixup_context(self.app) as (f, _, _): + f._cache.close_caches.side_effect = AttributeError + f.close_cache() + def test_close_cache(self): with self.fixup_context(self.app) as (f, _, _): f.close_cache() @@ -330,3 +442,55 @@ def test_django_setup(self, patching): f = self.Fixup(self.app) f.django_setup() django.setup.assert_called_with() + + def test__is_prefork(self): + with self.fixup_context(self.app) as (f, _, _): + f.worker.pool_cls = Mock(__module__='celery.concurrency.prefork') + assert f._is_prefork() + + f.worker.pool_cls = "prefork" + assert f._is_prefork() + + f.worker.pool_cls = Mock(__module__='celery.concurrency.thread') + assert not f._is_prefork() + + f.worker = None + assert not f._is_prefork() + + def test_no_recursive_worker_instantiation(self, patching): + """Regression test: DjangoWorkerFixup must not create a WorkController in __init__. + + Historically, DjangoWorkerFixup.__init__ instantiated a WorkController when + called with worker=None, which could cause recursive instantiation when + invoked from worker lifecycle signals. + + This test verifies the fixed behavior: + - DjangoWorkerFixup(app, worker=None) must not create a WorkController + - It should instead leave self.worker unset and rely on on_worker_init + to attach the actual worker instance later + """ + from celery.worker import WorkController + + patching('celery.fixups.django.symbol_by_name') + patching('celery.fixups.django.import_module') + patching.modules('django', 'django.db', 'django.core.checks') + + # Track WorkController instantiations + instantiation_count = {'count': 0} + original_init = WorkController.__init__ + + def tracking_init(self_worker, *args, **kwargs): + instantiation_count['count'] += 1 + return original_init(self_worker, *args, **kwargs) + + with patch.object(WorkController, '__init__', tracking_init): + # Creating DjangoWorkerFixup without a worker argument + # should NOT create a WorkController instance + DjangoWorkerFixup(self.app) + + # EXPECTED: 0 WorkController instances created + assert instantiation_count['count'] == 0, ( + f"DjangoWorkerFixup(app) should NOT create a WorkController, " + f"but {instantiation_count['count']} instance(s) were created. " + f"This is the root cause of the recursion bug." + ) diff --git a/t/unit/tasks/test_canvas.py b/t/unit/tasks/test_canvas.py index 1eb088f0c51..144ee625be7 100644 --- a/t/unit/tasks/test_canvas.py +++ b/t/unit/tasks/test_canvas.py @@ -4,7 +4,6 @@ from unittest.mock import ANY, MagicMock, Mock, call, patch, sentinel import pytest -import pytest_subtests # noqa from celery._state import _task_stack from celery.canvas import (Signature, _chain, _maybe_group, _merge_dictionaries, chain, chord, chunks, group, @@ -593,6 +592,40 @@ def test_chain_of_chord_upgrade_on_chaining__protocol_3(self): ), "Chord followed by a group should be upgraded to a single chord with chained body." assert len(c.tasks) == 6 + def test_chain_of_chords_stays_flat(self): + c = chain( + chord([signature('h1'), signature('h2')], signature('b1'), app=self.app), + chord([signature('h3'), signature('h4')], signature('b2'), app=self.app), + chord([signature('h5'), signature('h6')], signature('b3'), app=self.app), + ) + assert isinstance(c, _chain) + assert len(c.tasks) == 3 + for task in c.tasks: + assert isinstance(task, chord) + assert not isinstance(c.tasks[0].body, _chain) + assert not isinstance(c.tasks[1].body, _chain) + assert not isinstance(c.tasks[2].body, _chain) + + def test_chain_of_chords_serialized_size_constant(self): + chords = [ + chord([signature(f'h{i}_{j}') for j in range(3)], + signature(f'b{i}'), app=self.app) + for i in range(6) + ] + c = chain(*chords) + assert isinstance(c, _chain) + sizes = [len(json.dumps(task.__json__())) for task in c.tasks] + assert max(sizes) == min(sizes), ( + f"Chord sizes not constant across chain: {sizes}" + ) + + def test_chord_or_task_still_nests(self): + c = chord([signature('h1')], signature('b1'), app=self.app) + t = signature('t1') + result = chain(c) | t + assert isinstance(result, _chain) + assert isinstance(result.tasks[0].body, _chain) + def test_apply_options(self): class static(Signature): @@ -1243,6 +1276,12 @@ def test_group_prepared(self): assert isinstance(result, AsyncResult) assert group_id is not None + def test_task_replace_with_group_preserves_group_order(self): + self.app.conf.task_always_eager = True + sig = self.replace_with_group.s(1, 2) + res = self.helper_test_get_delay(sig.delay()) + assert res == [3, 2] + class test_chord(CanvasCase): def test__get_app_does_not_exhaust_generator(self): diff --git a/t/unit/tasks/test_result.py b/t/unit/tasks/test_result.py index 062c0695427..d5aaa481926 100644 --- a/t/unit/tasks/test_result.py +++ b/t/unit/tasks/test_result.py @@ -449,6 +449,20 @@ def test_date_done(self, utc_datetime_mock, timezone, date): result = Backend(app=self.app)._get_result_meta(None, states.SUCCESS, None, None) assert result.get('date_done') == date + def test_forget_remove_pending_result(self): + with patch('celery.result.AsyncResult.backend') as backend: + result = self.app.AsyncResult(self.task1['id']) + result.backend = backend + result_clone = copy.copy(result) + result.forget() + backend.remove_pending_result.assert_called_once_with( + result_clone + ) + + result = self.app.AsyncResult(self.task1['id']) + result.backend = None + del result + class test_ResultSet: diff --git a/t/unit/tasks/test_stamping.py b/t/unit/tasks/test_stamping.py index 1c8da859dd7..2161d314fd3 100644 --- a/t/unit/tasks/test_stamping.py +++ b/t/unit/tasks/test_stamping.py @@ -663,7 +663,6 @@ def test_on_signature_gets_the_signature(self): class CustomStampingVisitor(StampingVisitor): def on_signature(self, actual_sig, **headers) -> dict: - nonlocal expected_sig assert actual_sig == expected_sig return {"header": "value"} diff --git a/t/unit/tasks/test_tasks.py b/t/unit/tasks/test_tasks.py index 720394641c8..13b39739e5f 100644 --- a/t/unit/tasks/test_tasks.py +++ b/t/unit/tasks/test_tasks.py @@ -424,6 +424,37 @@ def test_signature_from_request__passes_headers(self): sig = self.retry_task.signature_from_request() assert sig.options['headers']['custom'] == 10.1 + def test_signature_from_request__filters_x_death_headers(self): + """ + Test that X-Death headers are filtered out during retries to prevent + RabbitMQ cycle detection. + """ + + self.retry_task.push_request() + self.retry_task.request.headers = { + 'custom': 10.1, + 'x-death': [{'count': 1, 'queue': 'celery_delayed_0'}], + 'x-first-death-exchange': 'celery_delayed_0', + 'x-first-death-queue': 'celery_delayed_0', + 'x-first-death-reason': 'expired', + 'x-last-death-exchange': 'celery_delayed_0', + 'x-last-death-queue': 'celery_delayed_0', + 'x-last-death-reason': 'expired', + } + sig = self.retry_task.signature_from_request() + + # Custom headers should be preserved + assert sig.options['headers']['custom'] == 10.1 + + # X-Death related headers should be filtered out + assert 'x-death' not in sig.options['headers'] + assert 'x-first-death-exchange' not in sig.options['headers'] + assert 'x-first-death-queue' not in sig.options['headers'] + assert 'x-first-death-reason' not in sig.options['headers'] + assert 'x-last-death-exchange' not in sig.options['headers'] + assert 'x-last-death-queue' not in sig.options['headers'] + assert 'x-last-death-reason' not in sig.options['headers'] + def test_signature_from_request__delivery_info(self): self.retry_task.push_request() self.retry_task.request.delivery_info = { diff --git a/t/unit/tasks/test_trace.py b/t/unit/tasks/test_trace.py index cd0c8c6901e..5f7172383dd 100644 --- a/t/unit/tasks/test_trace.py +++ b/t/unit/tasks/test_trace.py @@ -14,6 +14,7 @@ from celery.backends.base import BaseDictBackend from celery.backends.cache import CacheBackend from celery.exceptions import BackendGetMetaError, Ignore, Reject, Retry +from celery.result import AsyncResult from celery.states import PENDING from celery.worker.state import successful_requests @@ -564,6 +565,569 @@ def add(x, y): successful_requests.clear() self.app.conf.worker_deduplicate_successful_tasks = False + def test_deduplicate_successful_tasks__backend_dedup_dispatches_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_apply.assert_called_once() + call_args = mock_apply.call_args + assert call_args[0] == ((2,),) + assert call_args[1]['parent_id'] == task_id + assert call_args[1]['root_id'] == task_id + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_multi_element_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + step2 = self.add.s(20) + step3 = self.add.s(30) + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [step3, step2], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_apply.assert_called_once() + call_args = mock_apply.call_args + assert call_args[1]['chain'] == [step3] + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_adds_to_successful_requests(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + successful_requests.discard(task_id) + + request_dedup = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + } + with patch('celery.canvas.maybe_signature'): + trace(self.app, add, (1, 1), task_id=task_id, request=request_dedup) + + assert task_id in successful_requests + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_dispatch_failure_skips_successful_requests(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + successful_requests.discard(task_id) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = RuntimeError('broker down') + with patch('celery.app.trace.logger'): + with pytest.raises(Reject): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + + assert task_id not in successful_requests + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__inmemory_dedup_skips_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + + task_id = str(uuid4()) + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + successful_requests.add(task_id) + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_apply.assert_not_called() + + successful_requests.clear() + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_chain_dispatch_does_not_mutate_request_chain(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + + chain_list = [self.add.s(10), self.add.s(20)] + original_length = len(chain_list) + task_id = str(uuid4()) + request = { + 'id': task_id, + 'delivery_info': {'redelivered': False}, + 'chain': chain_list, + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async = Mock() + trace(self.app, add, (1, 1), task_id=task_id, request=request) + call_args = mock_signature.return_value.apply_async.call_args + assert call_args[1]['chain'] == chain_list[:-1] + assert len(chain_list) == original_length + + def test_deduplicate_successful_tasks__backend_dedup_dispatches_callbacks(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_callbacks = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'callbacks': [self.add.s(99)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_callbacks) + mock_apply.assert_called_once() + call_args = mock_apply.call_args + assert call_args[0] == ((2,),) + assert call_args[1]['parent_id'] == task_id + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_chain_and_callbacks(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_both = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + 'callbacks': [self.add.s(99)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_both) + assert mock_apply.call_count == 2 + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_skips_when_children_present(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + 'callbacks': [self.add.s(99)], + } + + meta_with_children = { + 'status': 'SUCCESS', 'result': 2, + 'children': [('some-child-id', None)], + } + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + with patch('celery.result.AsyncResult._get_task_meta', + return_value=meta_with_children): + trace(self.app, add, (1, 1), task_id=task_id, + request=request_with_chain) + mock_apply.assert_not_called() + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_dispatch_failure_logged(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = RuntimeError('broker down') + with patch('celery.app.trace.logger') as mock_logger: + with pytest.raises(Reject): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + mock_logger.error.assert_called_once() + assert 'deduplicated task' in mock_logger.error.call_args[0][0] + + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_dedup_memory_error_propagates(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = MemoryError() + with pytest.raises(MemoryError): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__reject_propagates_through_trace_task(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + add.__trace__ = None + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_signature.return_value.apply_async.side_effect = RuntimeError('broker down') + with patch('celery.app.trace.logger'): + with pytest.raises(Reject): + trace_task(add, task_id, (1, 1), {}, request=request_with_chain, app=self.app) + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__root_id_fallback(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_no_root_id = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_no_root_id) + call_args = mock_apply.call_args + assert call_args[1]['root_id'] == task_id + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__empty_chain_skips_dispatch(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + request_empty_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [], + 'callbacks': [], + } + + with patch('celery.canvas.maybe_signature') as mock_signature: + mock_apply = Mock() + mock_signature.return_value.apply_async = mock_apply + trace(self.app, add, (1, 1), task_id=task_id, request=request_empty_chain) + mock_apply.assert_not_called() + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_deduplicate_successful_tasks__backend_read_failure_rejects(self): + """When _get_task_meta() fails after state==SUCCESS, the exception + is caught and re-raised as Reject(requeue=True).""" + @self.app.task(shared=False) + def add(x, y): + return x + y + + backend = CacheBackend(app=self.app, backend='memory') + add.backend = backend + add.store_eager_result = True + add.ignore_result = False + add.acks_late = True + + self.app.conf.worker_deduplicate_successful_tasks = True + task_id = str(uuid4()) + request = {'id': task_id, 'delivery_info': {'redelivered': True}} + + trace(self.app, add, (1, 1), task_id=task_id, request=request) + + successful_requests.discard(task_id) + + request_with_chain = { + 'id': task_id, + 'delivery_info': {'redelivered': True}, + 'chain': [self.add.s(10)], + } + + # First call to _get_task_meta (from r.state) returns normally; + # second call (line 508 in trace.py) raises to simulate a + # transient backend failure during dispatch. + original = AsyncResult._get_task_meta + call_count = 0 + + def fail_on_second_call(self_): + nonlocal call_count + call_count += 1 + if call_count >= 2: + raise ConnectionError('redis gone') + return original(self_) + + with patch.object(AsyncResult, '_get_task_meta', fail_on_second_call): + with patch('celery.app.trace.logger'): + with pytest.raises(Reject): + trace(self.app, add, (1, 1), task_id=task_id, request=request_with_chain) + + assert task_id not in successful_requests + + successful_requests.discard(task_id) + self.app.conf.worker_deduplicate_successful_tasks = False + + def test_ignore_result_priority__request_overrides_task_true(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + add.ignore_result = True + request = {'ignore_result': False} + + self.trace(add, (2, 2), {}, request=request, eager=False) + + add.backend.mark_as_done.assert_called_with(ANY, 4, ANY, True) + + def test_ignore_result_priority__request_overrides_task_false(self): + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + add.ignore_result = False + request = {'ignore_result': True} + + self.trace(add, (2, 2), {}, request=request, eager=False) + + add.backend.mark_as_done.assert_called_with(ANY, 4, ANY, False) + + def test_ignore_result_priority__request_overrides_app_config(self): + prev_ignore = self.app.conf.task_ignore_result + + try: + self.app.conf.task_ignore_result = True + + @self.app.task(shared=False) + def add(x, y): + return x + y + + add.backend = Mock(name='backend') + + assert add.ignore_result is True + + request = {'ignore_result': False} + self.trace(add, (2, 2), {}, request=request, eager=False) + + add.backend.mark_as_done.assert_called_with(ANY, 4, ANY, True) + finally: + self.app.conf.task_ignore_result = prev_ignore + class test_TraceInfo(TraceCase): class TI(TraceInfo): @@ -606,6 +1170,35 @@ def test_handle_error_for_eager_saved_to_backend(self): call_errbacks=True, ) + def test_handle_error_state_missing_request_store_errors_false_while_task_ignore_result_true(self): + x = self.TI(states.FAILURE) + x.handle_failure = Mock() + + self.add.ignore_result = True + self.add.store_errors_even_if_ignored = False + + x.handle_error_state(self.add, None) + x.handle_failure.assert_called_once_with( + self.add, + None, + store_errors=False, + call_errbacks=True, + ) + + def test_handle_error_state_missing_request_store_errors_true_while_task_ignore_result_false(self): + x = self.TI(states.FAILURE) + x.handle_failure = Mock() + + self.add.ignore_result = False + + x.handle_error_state(self.add, None) + x.handle_failure.assert_called_once_with( + self.add, + None, + store_errors=True, + call_errbacks=True, + ) + @patch('celery.app.trace.ExceptionInfo') def test_handle_reject(self, ExceptionInfo): x = self.TI(states.FAILURE) diff --git a/t/unit/utils/test_collections.py b/t/unit/utils/test_collections.py index 2f183899017..9f5b0b6a492 100644 --- a/t/unit/utils/test_collections.py +++ b/t/unit/utils/test_collections.py @@ -128,10 +128,6 @@ def test_len(self): self.view.clear() assert len(self.view) == 2 - def test_isa_mapping(self): - from collections.abc import Mapping - assert issubclass(ConfigurationView, Mapping) - def test_isa_mutable_mapping(self): from collections.abc import MutableMapping assert issubclass(ConfigurationView, MutableMapping) diff --git a/t/unit/utils/test_functional.py b/t/unit/utils/test_functional.py index a8c9dc1e893..3b97a12b2b9 100644 --- a/t/unit/utils/test_functional.py +++ b/t/unit/utils/test_functional.py @@ -1,7 +1,7 @@ import collections +import sys import pytest -import pytest_subtests # noqa from kombu.utils.functional import lazy from celery.utils.functional import (DummyContext, first, firstmethod, fun_accepts_kwargs, fun_takes_argument, @@ -369,6 +369,21 @@ def test_kwonly_required_args(self): g(b=3) + @pytest.mark.skipif(sys.version_info < (3, 14), reason="PEP 649 deferred annotations require Python 3.14+") + def test_type_checking_annotation(self): + # Regression test for https://github.com/celery/celery/discussions/10099 + # On Python 3.14+, annotations are deferred (PEP 649). Functions with + # annotations referencing TYPE_CHECKING-only types must not raise NameError. + local = {} + exec('def f(args: Sequence[str], x: int = 0): return args', {}, local) + f = local['f'] + + g = head_from_fun(f) + with pytest.raises(TypeError): + g() + g(1) + g(1, 2) + class test_fun_takes_argument: @@ -472,6 +487,25 @@ def test_accepts(self, fun): def test_rejects(self, fun): assert not fun_accepts_kwargs(fun) + @pytest.mark.skipif(sys.version_info < (3, 14), reason="PEP 649 deferred annotations require Python 3.14+") + def test_type_checking_annotation(self): + # Regression test for https://github.com/celery/celery/discussions/10099 + # On Python 3.14+, annotations are deferred (PEP 649). Calling + # fun_accepts_kwargs on a function whose annotations reference + # TYPE_CHECKING-only types must not raise NameError. + # + # This reproduces the failure seen with on_after_finalize.connect: + # def setup_periodic_tasks(sender: Celery, **kwargs: object) -> None: ... + # where 'Celery' is only imported under TYPE_CHECKING. + local = {} + exec('def f(sender: Celery, **kwargs: object) -> None: pass', {}, local) + f = local['f'] + assert fun_accepts_kwargs(f) is True + + exec('def g(sender: Celery) -> None: pass', {}, local) + g = local['g'] + assert fun_accepts_kwargs(g) is False + @pytest.mark.parametrize('value,expected', [ (5, True), diff --git a/t/unit/worker/test_autoscale.py b/t/unit/worker/test_autoscale.py index c4a2a75ed73..c5f459b5ff0 100644 --- a/t/unit/worker/test_autoscale.py +++ b/t/unit/worker/test_autoscale.py @@ -236,3 +236,53 @@ def test_no_negative_scale(self, sleepdeprived): assert all(x.min_concurrency <= i <= x.max_concurrency for i in total_num_processes) + + def test_disable_prefetch_respects_max_concurrency(self): + """Test that disable_prefetch respects autoscale max_concurrency setting""" + from celery.worker.consumer.tasks import Tasks + + # Create a mock consumer with autoscale and disable_prefetch enabled + consumer = Mock() + consumer.app = Mock() + consumer.app.conf.worker_disable_prefetch = True + consumer.pool = Mock() + consumer.pool.num_processes = 10 + consumer.controller = Mock() + consumer.controller.max_concurrency = 5 # Lower than pool processes + + # Mock task consumer setup + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + consumer.task_consumer.channel.qos.can_consume = Mock(return_value=True) + + # Mock the connection and other required attributes + consumer.connection = Mock() + consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' + consumer.initial_prefetch_count = 20 + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock the amqp TaskConsumer + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + # Mock 5 reserved requests (at autoscale limit of 5) + mock_requests = [Mock() for _ in range(5)] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should not be able to consume when at autoscale limit + assert consumer.task_consumer.channel.qos.can_consume() is False + + # Test with 4 reserved requests (under autoscale limit of 5) + mock_requests = [Mock() for _ in range(4)] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should be able to consume when under autoscale limit + assert consumer.task_consumer.channel.qos.can_consume() is True diff --git a/t/unit/worker/test_bootsteps.py b/t/unit/worker/test_bootsteps.py index 4a33f44da35..6443d660626 100644 --- a/t/unit/worker/test_bootsteps.py +++ b/t/unit/worker/test_bootsteps.py @@ -122,6 +122,8 @@ def test_interface(self): def test_start_stop_shutdown(self): consumer = Mock() self.connection = Mock() + self.connection.connection_errors = () + self.connection.channel_errors = () class Step(bootsteps.ConsumerStep): @@ -141,6 +143,8 @@ def get_consumers(self, c): def test_start_no_consumers(self): self.connection = Mock() + self.connection.connection_errors = () + self.connection.channel_errors = () class Step(bootsteps.ConsumerStep): diff --git a/t/unit/worker/test_components.py b/t/unit/worker/test_components.py index 739808e4311..ea43d3d5f75 100644 --- a/t/unit/worker/test_components.py +++ b/t/unit/worker/test_components.py @@ -89,3 +89,17 @@ def test_create__green(self): w.pool_cls.__module__ = 'foo_gevent' with pytest.raises(ImproperlyConfigured): Beat(w).create(w) + + def test_create__green_string_pool_cls(self): + """Test Beat.create raises ImproperlyConfigured when pool_cls is a string like 'gevent'.""" + w = Mock(name='w') + w.pool_cls = 'gevent' # pool_cls can be a string instead of a class + with pytest.raises(ImproperlyConfigured): + Beat(w).create(w) + + def test_create__green_string_pool_cls_eventlet(self): + """Test Beat.create raises ImproperlyConfigured when pool_cls is 'eventlet'.""" + w = Mock(name='w') + w.pool_cls = 'eventlet' + with pytest.raises(ImproperlyConfigured): + Beat(w).create(w) diff --git a/t/unit/worker/test_consumer.py b/t/unit/worker/test_consumer.py index 7f4e5f94cf3..8cdb908795f 100644 --- a/t/unit/worker/test_consumer.py +++ b/t/unit/worker/test_consumer.py @@ -19,7 +19,7 @@ from celery.worker.consumer.heart import Heart from celery.worker.consumer.mingle import Mingle from celery.worker.consumer.tasks import Tasks -from celery.worker.state import active_requests +from celery.worker.state import active_requests, successful_requests class ConsumerTestCase: @@ -376,12 +376,11 @@ def test_register_with_event_loop(self): c = self.get_consumer() c.register_with_event_loop(Mock(name='loop')) - def test_on_close_clears_semaphore_timer_and_reqs(self): + def test_on_close_clears_semaphore_and_reqs(self): with patch('celery.worker.consumer.consumer.reserved_requests') as res: c = self.get_consumer() c.on_close() c.controller.semaphore.clear.assert_called_with() - c.timer.clear.assert_called_with() res.clear.assert_called_with() c.pool.flush.assert_called_with() @@ -405,6 +404,8 @@ def test_connect_error_handler_progress(self, error): self.app.conf.broker_connection_max_retries = 3 self.app._connection = _amqp_connection() conn = self.app._connection.return_value + # Placeholder alt connection to satisfy failover condition + conn.alt = [conn] c = self.get_consumer() assert c.connect() errback = conn.ensure_connection.call_args[0][0] @@ -412,8 +413,10 @@ def test_connect_error_handler_progress(self, error): assert error.call_args[0][3] == 'Trying again in 2.00 seconds... (1/3)' errback(Mock(), 4) assert error.call_args[0][3] == 'Trying again in 4.00 seconds... (2/3)' - errback(Mock(), 6) - assert error.call_args[0][3] == 'Trying again in 6.00 seconds... (3/3)' + errback(Mock(), 12) + assert error.call_args[0][3] == 'Trying again in 12.00 seconds... (3/3)' + errback(Mock(), 0) + assert getattr(c, 'broker_connection_retry_attempt', 0) == 3 def test_cancel_long_running_tasks_on_connection_loss(self): c = self.get_consumer() @@ -449,7 +452,7 @@ def test_cancel_long_running_tasks_on_connection_loss__warning(self): c.on_connection_error_after_connected(Mock()) @pytest.mark.usefixtures('depends_on_current_app') - def test_cancel_all_unacked_requests(self): + def test_cancel_active_requests(self): c = self.get_consumer() mock_request_acks_late_not_acknowledged = Mock(id='1') @@ -465,14 +468,36 @@ def test_cancel_all_unacked_requests(self): active_requests.add(mock_request_acks_late_acknowledged) active_requests.add(mock_request_acks_early) - c.cancel_all_unacked_requests() + c.cancel_active_requests() - mock_request_acks_late_not_acknowledged.cancel.assert_called_once_with(c.pool) + # acks_late unacknowledged tasks should be cancelled without RETRY + mock_request_acks_late_not_acknowledged.cancel.assert_called_once_with(c.pool, emit_retry=False) + # acks_late acknowledged tasks should NOT be cancelled mock_request_acks_late_acknowledged.cancel.assert_not_called() - mock_request_acks_early.cancel.assert_called_once_with(c.pool) + # Non-acks_late tasks should be cancelled normally (with RETRY) + mock_request_acks_early.cancel.assert_called_once_with(c.pool, emit_retry=True) active_requests.clear() + @pytest.mark.usefixtures('depends_on_current_app') + def test_cancel_active_requests_preserves_successful_tasks(self): + c = self.get_consumer() + + mock_successful_request = Mock(id='successful-task') + mock_successful_request.task.acks_late = True + mock_successful_request.acknowledged = False + + active_requests.add(mock_successful_request) + + successful_requests.add('successful-task') + + try: + c.cancel_active_requests() + mock_successful_request.cancel.assert_not_called() + finally: + active_requests.clear() + successful_requests.clear() + @pytest.mark.parametrize("broker_connection_retry", [True, False]) @pytest.mark.parametrize("broker_connection_retry_on_startup", [None, False]) @pytest.mark.parametrize("first_connection_attempt", [True, False]) @@ -495,6 +520,212 @@ def test_ensure_connected(self, subtests, broker_connection_retry, broker_connec with pytest.raises(ConnectionError): c.ensure_connected(conn) + def test_disable_prefetch_not_enabled(self): + """Test that disable_prefetch doesn't affect behavior when disabled""" + self.app.conf.worker_disable_prefetch = False + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () + consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + tasks_instance.start(consumer) + + # Should not modify can_consume method when disabled + assert consumer.task_consumer.channel.qos.can_consume == original_can_consume + + def test_disable_prefetch_enabled_basic(self): + """Test that disable_prefetch modifies can_consume when enabled""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () + consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + with patch('celery.worker.state.reserved_requests', []): + tasks_instance.start(consumer) + + # Should modify can_consume method when enabled + assert callable(consumer.task_consumer.channel.qos.can_consume) + assert consumer.task_consumer.channel.qos.can_consume != original_can_consume + + def test_disable_prefetch_respects_reserved_requests_limit(self): + """Test that disable_prefetch respects reserved requests limit""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () + consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + consumer.task_consumer.channel.qos.can_consume = Mock(return_value=True) + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + # Mock 4 reserved requests (at limit of 4) + mock_requests = [Mock(), Mock(), Mock(), Mock()] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should not be able to consume when at limit + assert consumer.task_consumer.channel.qos.can_consume() is False + + def test_disable_prefetch_respects_autoscale_max_concurrency(self): + """Test that disable_prefetch respects autoscale max_concurrency limit""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = 2 # Lower than pool processes + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () + consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'redis' + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + consumer.task_consumer.channel.qos.can_consume = Mock(return_value=True) + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + + tasks_instance = Tasks(consumer) + + # Mock 2 reserved requests (at autoscale limit of 2) + mock_requests = [Mock(), Mock()] + with patch('celery.worker.state.reserved_requests', mock_requests): + tasks_instance.start(consumer) + + # Should not be able to consume when at autoscale limit + assert consumer.task_consumer.channel.qos.can_consume() is False + + def test_disable_prefetch_ignored_for_non_redis_brokers(self): + """Test that disable_prefetch is ignored for non-Redis brokers.""" + self.app.conf.worker_disable_prefetch = True + + # Test the core logic by creating a mock consumer and Tasks instance + from celery.worker.consumer.tasks import Tasks + consumer = Mock() + consumer.app = self.app + consumer.pool = Mock() + consumer.pool.num_processes = 4 + consumer.controller = Mock() + consumer.controller.max_concurrency = None + consumer.initial_prefetch_count = 16 + consumer.connection = Mock() + consumer.connection.connection_errors = () + consumer.connection.channel_errors = () + consumer.connection.default_channel = Mock() + consumer.connection.transport = Mock() + consumer.connection.transport.driver_type = 'amqp' # RabbitMQ + consumer.connection.qos_semantics_matches_spec = True + consumer.update_strategies = Mock() + consumer.on_decode_error = Mock() + + # Mock task consumer + consumer.task_consumer = Mock() + consumer.task_consumer.channel = Mock() + consumer.task_consumer.channel.qos = Mock() + original_can_consume = Mock(return_value=True) + consumer.task_consumer.channel.qos.can_consume = original_can_consume + consumer.task_consumer.qos = Mock() + + consumer.app.amqp = Mock() + consumer.app.amqp.TaskConsumer = Mock(return_value=consumer.task_consumer) + consumer.app.amqp.queues = {} # Empty dict for quorum queue detection + + tasks_instance = Tasks(consumer) + tasks_instance.start(consumer) + + # Should not modify can_consume method for non-Redis brokers + assert consumer.task_consumer.channel.qos.can_consume == original_can_consume + @pytest.mark.parametrize( "broker_connection_retry_on_startup,is_connection_loss_on_startup", @@ -694,6 +925,7 @@ def test_log_when_qos_is_false(self, caplog): c = self.c c.connection.transport.driver_type = 'amqp' c.app.conf.broker_native_delayed_delivery = True + c.app.conf.worker_disable_prefetch = False # Prevent our warning from interfering c.app.amqp.queues = {"celery": Mock(queue_arguments={"x-queue-type": "quorum"})} tasks = Tasks(c) @@ -706,6 +938,72 @@ def test_log_when_qos_is_false(self, caplog): assert record.levelname == "INFO" assert record.msg == "Global QoS is disabled. Prefetch count in now static." + def test_qos_with_worker_eta_task_limit(self): + """Test QoS is instantiated with worker_eta_task_limit as max_prefetch.""" + c = self.c + c.app.conf.worker_eta_task_limit = 100 + c.initial_prefetch_count = 10 + c.task_consumer = Mock() + c.app.amqp.TaskConsumer = Mock(return_value=c.task_consumer) + c.connection.default_channel.basic_qos = Mock() + c.update_strategies = Mock() + c.on_decode_error = Mock() + + tasks = Tasks(c) + + with patch('celery.worker.consumer.tasks.QoS') as mock_qos: + tasks.start(c) + + # Verify QoS was called with max_prefetch set to worker_eta_task_limit + mock_qos.assert_called_once() + args, kwargs = mock_qos.call_args + assert len(args) == 2 # callback and initial_value + assert kwargs.get('max_prefetch') == 100 + + def test_qos_without_worker_eta_task_limit(self): + """Test QoS is instantiated with None max_prefetch when worker_eta_task_limit is None.""" + c = self.c + c.app.conf.worker_eta_task_limit = None + c.initial_prefetch_count = 10 + c.task_consumer = Mock() + c.app.amqp.TaskConsumer = Mock(return_value=c.task_consumer) + c.connection.default_channel.basic_qos = Mock() + c.update_strategies = Mock() + c.on_decode_error = Mock() + + tasks = Tasks(c) + + with patch('celery.worker.consumer.tasks.QoS') as mock_qos: + tasks.start(c) + + # Verify QoS was called with max_prefetch set to None + mock_qos.assert_called_once() + args, kwargs = mock_qos.call_args + assert len(args) == 2 # callback and initial_value + assert kwargs.get('max_prefetch') is None + + def test_qos_with_zero_worker_eta_task_limit(self): + """Test that QoS respects zero as a valid worker_eta_task_limit value.""" + c = self.c + c.app.conf.worker_eta_task_limit = 0 + c.initial_prefetch_count = 10 + c.task_consumer = Mock() + c.app.amqp.TaskConsumer = Mock(return_value=c.task_consumer) + c.connection.default_channel.basic_qos = Mock() + c.update_strategies = Mock() + c.on_decode_error = Mock() + + tasks = Tasks(c) + + with patch('celery.worker.consumer.tasks.QoS') as mock_qos: + tasks.start(c) + + # Verify QoS was called with max_prefetch set to 0 + mock_qos.assert_called_once() + args, kwargs = mock_qos.call_args + assert len(args) == 2 # callback and initial_value + assert kwargs.get('max_prefetch') == 0 + def test_ready_worker_limit_caps_at_ready_workers(self): # Concurrency 3 but only 2 workers have completed the WORKER_UP # handshake (one recycling), so the limit is capped at 2. @@ -737,21 +1035,6 @@ def test_ready_worker_limit_falls_back_on_introspection_error(self): c.pool._pool._fileno_to_inq = object() # len() raises assert Tasks.ready_worker_limit(c) == 3 - def _start_disable_prefetch(self, c): - c.app.conf.worker_disable_prefetch = True - c.app.conf.worker_detect_quorum_queues = False - Tasks(c).start(c) - return c.task_consumer.channel.qos.can_consume - - def test_disable_prefetch_does_not_fetch_into_recycling_slot(self): - # can_consume gates on ready_worker_limit: 2 ready, 2 reserved -> refuse. - c = self.c - c.controller.max_concurrency = 3 - c.pool._pool._fileno_to_inq = {10: 'w1', 11: 'w2'} - can_consume = self._start_disable_prefetch(c) - with patch('celery.worker.state.reserved_requests', [0, 1]): - assert can_consume() is False - class test_Agent: diff --git a/t/unit/worker/test_control.py b/t/unit/worker/test_control.py index 6d7e923d2db..5f4c690f4b1 100644 --- a/t/unit/worker/test_control.py +++ b/t/unit/worker/test_control.py @@ -4,7 +4,7 @@ from collections import defaultdict from datetime import datetime, timedelta from queue import Queue as FastQueue -from unittest.mock import Mock, call, patch +from unittest.mock import Mock, PropertyMock, call, patch import pytest from kombu import pidbox @@ -815,3 +815,56 @@ def test_query_task(self): assert ret[req1.id][0] == 'reserved' finally: worker_state.reserved_requests.clear() + + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) + def test_revoke_backend_status_update(self): + state = self.create_state() + task_ids = ['task-1', 'task-2'] + + control._revoke(state, task_ids) + + assert self.app.backend.mark_as_revoked.call_count == 2 + calls = self.app.backend.mark_as_revoked.call_args_list + assert calls[0] == (('task-1',), {'reason': 'revoked', 'store_result': True}) + assert calls[1] == (('task-2',), {'reason': 'revoked', 'store_result': True}) + + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) + def test_revoke_backend_failure_defensive(self): + self.app.backend.mark_as_revoked.side_effect = Exception("Backend error") + state = self.create_state() + + control._revoke(state, ['task-1']) + + assert 'task-1' in worker_state.revoked + + @patch('celery.Celery.backend', new=PropertyMock(name='backend')) + def test_revoke_terminate_backend_update(self): + state = self.create_state() + + with patch('celery.worker.control._find_requests_by_id', return_value=[]): + control._revoke(state, ['task-1'], terminate=True) + + self.app.backend.mark_as_revoked.assert_called_once_with( + 'task-1', reason='revoked', store_result=True + ) + + def test_revoke_by_stamped_headers_terminates_matching_request(self): + state = self.create_state() + state.consumer = Mock() + + request = Mock() + request.id = 'task-with-stamp' + request.stamps = {'monitoring_id': 'test-123'} + request.terminate = Mock() + + worker_state.active_requests.add(request) + + headers = {'monitoring_id': 'test-123'} + + with patch('celery.worker.control._signals.signum', return_value=15): + control.revoke_by_stamped_headers(state, headers, terminate=True) + + request.terminate.assert_called_once() + assert 'test-123' in worker_state.revoked_stamps.get('monitoring_id', []) + + worker_state.active_requests.clear() diff --git a/t/unit/worker/test_loops.py b/t/unit/worker/test_loops.py index 754a3a119c7..26e682a6485 100644 --- a/t/unit/worker/test_loops.py +++ b/t/unit/worker/test_loops.py @@ -453,6 +453,143 @@ def test_no_heartbeat_support(self): x.hub.timer.call_repeatedly.assert_not_called() + def test_hub_reset_on_connection_error(self): + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()') + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + poller = x.hub.poller + poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.reset.assert_called_once() + + def test_hub_timer_cleared_on_connection_error(self): + # Stale timer entries (e.g. maybe_restore_messages) must be cleared + # when the event loop exits due to a connection error. Without this, + # entries accumulated across reconnects can fire against the broken + # connection and crash the loop again before the new connection is + # fully established, causing a rapid restart loop. + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()') + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + x.hub.poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.timer.clear.assert_called_once() + + def test_hub_timer_not_cleared_on_graceful_shutdown(self): + # On graceful shutdown the timer queue must be left intact so that + # periodic timers (e.g. heartbeat) keep firing while the pool drains. + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + x.hub.on_tick.add(x.closer(mod=2)) + asynloop(*x.args) + x.hub.timer.clear.assert_not_called() + + def test_hub_timer_not_cleared_on_worker_shutdown(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_stop = 303 + try: + with pytest.raises(WorkerShutdown): + asynloop(*x.args) + finally: + state.should_stop = None + x.hub.timer.clear.assert_not_called() + + def test_hub_timer_not_cleared_on_worker_terminate(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_terminate = True + try: + with pytest.raises(WorkerTerminate): + asynloop(*x.args) + finally: + state.should_terminate = None + x.hub.timer.clear.assert_not_called() + + def test_hub_timer_clear_error_still_reraises_original(self): + # If hub.timer.clear() itself raises, the original connection error + # must still be propagated, not the cleanup error. + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()') + x.hub.timer.clear = Mock( + name='hub.timer.clear()', side_effect=RuntimeError('clear failed') + ) + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + x.hub.poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.timer.clear.assert_called_once() + + def test_hub_timer_cleared_even_when_reset_raises(self): + # hub.timer.clear() must still be called even if hub.reset() raises. + # The two cleanup calls are in separate try/except blocks so that a + # failure in hub.reset() does not prevent stale timer entries from + # being discarded, avoiding stale timers persisting after a reset error. + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()', side_effect=RuntimeError('reset failed')) + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + x.hub.poller.poll.return_value = [] + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.timer.clear.assert_called_once() + + def test_hub_not_reset_on_graceful_shutdown(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + x.hub.on_tick.add(x.closer(mod=2)) + asynloop(*x.args) + x.hub.reset.assert_not_called() + + def test_hub_not_reset_on_worker_shutdown(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_stop = 303 + try: + with pytest.raises(WorkerShutdown): + asynloop(*x.args) + finally: + state.should_stop = None + x.hub.reset.assert_not_called() + + def test_hub_not_reset_on_worker_terminate(self): + x = X(self.app) + x.hub.reset = Mock(name='hub.reset()') + state.should_terminate = True + try: + with pytest.raises(WorkerTerminate): + asynloop(*x.args) + finally: + state.should_terminate = None + x.hub.reset.assert_not_called() + + def test_hub_reset_error_still_reraises_original(self): + x = X(self.app) + x.hub.readers = {6: Mock()} + x.hub.timer._queue = [1] + x.hub.reset = Mock(name='hub.reset()', side_effect=RuntimeError('reset failed')) + x.close_then_error(x.hub.poller.poll) + x.hub.fire_timers.return_value = 33.37 + poller = x.hub.poller + poller.poll.return_value = [] + # The original socket.error should still be raised, not the RuntimeError from reset() + with pytest.raises(socket.error): + asynloop(*x.args) + x.hub.reset.assert_called_once() + class test_synloop: diff --git a/t/unit/worker/test_native_delayed_delivery.py b/t/unit/worker/test_native_delayed_delivery.py index 654d7c15ab7..83b7d2888b7 100644 --- a/t/unit/worker/test_native_delayed_delivery.py +++ b/t/unit/worker/test_native_delayed_delivery.py @@ -1,9 +1,14 @@ import itertools +import sys from logging import LogRecord from typing import Iterator from unittest.mock import MagicMock, Mock, patch +if sys.version_info < (3, 11): # pragma: no cover + from exceptiongroup import ExceptionGroup + import pytest +from amqp import NotFound from kombu import Exchange, Queue from kombu.utils.functional import retry_over_time @@ -306,3 +311,161 @@ def test_setup_bind_error(self, mock_bind, caplog): assert len([r for r in caplog.records if r.levelname == "CRITICAL"]) == 1 assert any("Failed to bind queue" in r.message for r in caplog.records) assert any("Failed to setup delayed delivery for all broker URLs" in r.message for r in caplog.records) + + @patch('celery.worker.consumer.delayed_delivery.bind_queue_to_native_delayed_delivery_exchange') + def test_bind_queues_continues_after_failure(self, mock_bind, caplog): + """ + Test that binding continues for remaining queues after one fails. + """ + consumer_mock = MagicMock() + consumer_mock.app.conf.broker_native_delayed_delivery_queue_type = \ + 'classic' + consumer_mock.app.conf.broker_url = 'amqp://' + + # Create three queues + queue1 = Queue('queue1', exchange=Exchange('exchange1', type='topic')) + queue2 = Queue('queue2', exchange=Exchange('exchange2', type='topic')) + queue3 = Queue('queue3', exchange=Exchange('exchange3', type='topic')) + + consumer_mock.app.amqp.queues = { + 'queue1': queue1, + 'queue2': queue2, + 'queue3': queue3, + } + + # Make the second queue fail to bind + def bind_side_effect(connection, queue): + if queue.name == 'queue2': + raise NotFound( + reply_text="NOT_FOUND - no queue 'queue2' in vhost '/'", + method_name="Queue.bind", + reply_code=404, + ) + + mock_bind.side_effect = bind_side_effect + + delayed_delivery = DelayedDelivery(consumer_mock) + delayed_delivery.start(consumer_mock) + + # Verify that bind was called for all three queues + assert mock_bind.call_count == 3 + + # Verify error was logged for queue2 + error_logs = [r for r in caplog.records if r.levelname == "ERROR"] + expected_msg = \ + "Queue.bind: (404) NOT_FOUND - no queue 'queue2' in vhost '/'" + assert any(expected_msg in r.message for r in error_logs) + + @patch('celery.worker.consumer.delayed_delivery.bind_queue_to_native_delayed_delivery_exchange') + def test_bind_queues_raises_exceptions_on_failures(self, mock_bind): + """Test that Exceptions are raised with all binding failures.""" + consumer_mock = MagicMock() + consumer_mock.app.conf.broker_native_delayed_delivery_queue_type = \ + 'classic' + consumer_mock.app.conf.broker_url = 'amqp://' + + # Create three queues + queue1 = Queue('queue1', exchange=Exchange('exchange1', type='topic')) + queue2 = Queue('queue2', exchange=Exchange('exchange2', type='topic')) + queue3 = Queue('queue3', exchange=Exchange('exchange3', type='topic')) + + consumer_mock.app.amqp.queues = { + 'queue1': queue1, + 'queue2': queue2, + 'queue3': queue3, + } + + # Make queue1 and queue3 fail with different errors + def bind_side_effect(connection, queue): + if queue.name == 'queue1': + raise ValueError("Queue1 binding failed") + elif queue.name == 'queue3': + raise RuntimeError("Queue3 binding failed") + + mock_bind.side_effect = bind_side_effect + + delayed_delivery = DelayedDelivery(consumer_mock) + + # Should raise RuntimeError containing both exceptions messages + with pytest.raises(ExceptionGroup) as exc_info: + delayed_delivery._setup_delayed_delivery(consumer_mock, 'amqp://') + + # Verify the RuntimeError message contains both exceptions + exception_group = exc_info.value + assert str(exception_group) == ( + "One or more failures occurred while binding queues to delayed " + "delivery exchanges (2 sub-exceptions)" + ) + + # Verify the ExceptionGroup contains both exceptions + exceptions = exception_group.exceptions + assert len(exceptions) == 2 + assert any( + isinstance(ex, ValueError) and str(ex) == "Queue1 binding failed" + for ex in exceptions + ) + assert any( + isinstance(ex, RuntimeError) and str(ex) == "Queue3 binding failed" + for ex in exceptions + ) + + # Verify bind was called for all three queues + assert mock_bind.call_count == 3 + + @patch('celery.worker.consumer.delayed_delivery.bind_queue_to_native_delayed_delivery_exchange') + def test_bind_retries_on_retried_exception(self, mock_bind, caplog): + """ + Test that retried exceptions from + bind_queue_to_native_delayed_delivery_exchange trigger the retry + mechanism. + """ + consumer_mock = MagicMock() + consumer_mock.app.conf.broker_native_delayed_delivery_queue_type = \ + 'classic' + consumer_mock.app.conf.broker_url = 'amqp://' + + # Create a queue + queue1 = Queue('queue1', exchange=Exchange('exchange1', type='topic')) + consumer_mock.app.amqp.queues = {'queue1': queue1} + + # Track bind attempts + bind_attempts = [0] + + # Make bind raise a ConnectionRefusedError twice, then succeed + # This simulates a transient connection issue that resolves on retry + def bind_side_effect(connection, queue): + bind_attempts[0] += 1 + if bind_attempts[0] <= 2: + # ConnectionRefusedError is one of the RETRIED_EXCEPTIONS + raise ConnectionRefusedError("Connection refused") + # Succeed on third attempt + + mock_bind.side_effect = bind_side_effect + + delayed_delivery = DelayedDelivery(consumer_mock) + delayed_delivery.start(consumer_mock) + + # Verify bind was attempted multiple times (indicating retries + # occurred) + assert bind_attempts[0] == 3, \ + "Expected 3 bind attempts (2 failures + 1 success), got " + \ + f"{bind_attempts[0]}" + + # Verify retry warnings were logged + warning_logs = [r for r in caplog.records if r.levelname == "WARNING"] + retry_warnings = [ + r for r in warning_logs + if "Retrying delayed delivery setup" in r.message + ] + + # Should have 2 retry warnings (one for each failed attempt) + assert len(retry_warnings) == 2, \ + f"Expected 2 retry warnings, got {len(retry_warnings)}. " + \ + f"All warnings: {[r.message for r in warning_logs]}" + + # Verify the retry messages contain the expected information and + # correct attempt numbers + assert "Connection refused" in retry_warnings[0].message + assert "attempt 1/" in retry_warnings[0].message + assert "Connection refused" in retry_warnings[1].message + assert "attempt 2/" in retry_warnings[1].message diff --git a/t/unit/worker/test_request.py b/t/unit/worker/test_request.py index 172ca5162ac..1a57f2d4212 100644 --- a/t/unit/worker/test_request.py +++ b/t/unit/worker/test_request.py @@ -188,13 +188,14 @@ def get_request(self, sig, Request=Request, exclude_headers=None, + headers=None, **kwargs): msg = self.task_message_from_sig(self.app, sig) - headers = None - if exclude_headers: - headers = msg.headers - for header in exclude_headers: - headers.pop(header) + if headers is None: + headers = msg.headers.copy() + if exclude_headers: + for header in exclude_headers: + headers.pop(header, None) return Request( msg, on_ack=Mock(name='on_ack'), @@ -443,6 +444,34 @@ def test_tzlocal_is_cached(self): req._tzlocal = 'foo' assert req.tzlocal == 'foo' + def test_ignore_result_from_request_true(self): + req = self.get_request(self.add.s(2, 2).set(ignore_result=True)) + assert req.ignore_result is True + + def test_ignore_result_from_request_false(self): + req = self.get_request(self.add.s(2, 2).set(ignore_result=False)) + assert req.ignore_result is False + + def test_ignore_result_default_from_task_false(self): + req = self.get_request(self.add.s(2, 2)) + assert req.ignore_result is False + + def test_ignore_result_default_from_task_true(self): + self.add.ignore_result = True + try: + req = self.get_request(self.add.s(2, 2)) + assert req.ignore_result is True + finally: + self.add.ignore_result = False + + def test_ignore_result_request_overrides_task_true(self): + self.add.ignore_result = True + try: + req = self.get_request(self.add.s(2, 2).set(ignore_result=False)) + assert req.ignore_result is False + finally: + self.add.ignore_result = False + def test_task_wrapper_repr(self): assert repr(self.xRequest()) @@ -455,6 +484,35 @@ def test_sets_store_errors(self): job = self.xRequest() assert job.store_errors + def test_store_errors_default(self): + self.mytask.ignore_result = False + job = self.xRequest() + assert job.store_errors + + def test_store_errors_task_ignore_result(self): + self.mytask.ignore_result = True + job = self.xRequest() + assert not job.store_errors + + def test_store_errors_request_overrides_task_ignore_result(self): + self.mytask.ignore_result = True + msg = self.task_message_from_sig(self.app, self.mytask.s()) + headers = msg.headers.copy() + headers['ignore_result'] = False + job = self.get_request( + self.mytask.s(), + headers=headers, + ) + assert job.store_errors + + def test_ignore_result_from_request_none(self): + self.mytask.ignore_result = True + msg = self.task_message_from_sig(self.app, self.mytask.s()) + headers = msg.headers.copy() + headers['ignore_result'] = None + job = self.get_request(self.mytask.s(), headers=headers) + assert job._ignore_result is True + def test_send_event(self): job = self.xRequest() job.eventer = Mock(name='.eventer') @@ -575,6 +633,39 @@ def test_cancel__task_reserved(self): pool.terminate_job.assert_not_called() assert job._terminate_on_ack is None + def test_cancel__emit_retry_true(self): + pool = Mock() + signum = signal.SIGTERM + job = self.get_request(self.mytask.s(1, f='x')) + job._apply_result = Mock(name='_apply_result') + job.task.backend.mark_as_retry = Mock(name='mark_as_retry') + job.task.on_retry = Mock(name='on_retry') + with self.assert_signal_called( + task_retry, sender=job.task, request=job._context, + einfo=None): + job.time_start = monotonic() + job.worker_pid = 314 + job.cancel(pool, signal='TERM', emit_retry=True) + pool.terminate_job.assert_called_with(job.worker_pid, signum) + job.task.backend.mark_as_retry.assert_called_once() + job.task.on_retry.assert_called_once() + assert job._already_cancelled is True + + def test_cancel__emit_retry_false(self): + pool = Mock() + signum = signal.SIGTERM + job = self.get_request(self.mytask.s(1, f='x')) + job._apply_result = Mock(name='_apply_result') + job.task.backend.mark_as_retry = Mock(name='mark_as_retry') + job.task.on_retry = Mock(name='on_retry') + job.time_start = monotonic() + job.worker_pid = 314 + job.cancel(pool, signal='TERM', emit_retry=False) + pool.terminate_job.assert_called_with(job.worker_pid, signum) + job.task.backend.mark_as_retry.assert_not_called() + job.task.on_retry.assert_not_called() + assert job._already_cancelled is True + def test_revoked_expires_expired(self): job = self.get_request(self.mytask.s(1, f='x').set( expires=datetime.now(timezone.utc) - timedelta(days=1) @@ -870,6 +961,26 @@ def test_on_failure_task_cancelled(self): job.on_failure(exc_info) assert not job.eventer.send.called + def test_on_failure_should_terminate(self): + from celery.worker import state + original_should_terminate = state.should_terminate + state.should_terminate = True + job = self.xRequest() + job.send_event = Mock(name='send_event') + job.task.backend = Mock(name='backend') + + try: + try: + raise KeyError('foo') + except KeyError: + exc_info = ExceptionInfo() + job.on_failure(exc_info) + + job.send_event.assert_not_called() + job.task.backend.mark_as_failure.assert_not_called() + finally: + state.should_terminate = original_should_terminate + def test_from_message_invalid_kwargs(self): m = self.TaskMessage(self.mytask.name, args=(), kwargs='foo') req = Request(m, app=self.app) @@ -937,6 +1048,21 @@ def test_on_soft_timeout(self, patching): job.on_timeout(soft=True, timeout=1336) assert self.mytask.backend.get_status(job.id) == states.PENDING + def test_on_timeout_should_terminate(self, patching): + from celery.worker import state + warn = patching('celery.worker.request.warn') + error = patching('celery.worker.request.error') + + original_should_terminate = state.should_terminate + try: + state.should_terminate = True + job = self.xRequest() + job.on_timeout(None, None) + warn.assert_not_called() + error.assert_not_called() + finally: + state.should_terminate = original_should_terminate + def test_fast_trace_task(self): assert self.app.use_fast_trace_task is False setup_worker_optimizations(self.app) diff --git a/t/unit/worker/test_strategy.py b/t/unit/worker/test_strategy.py index 30c50b98455..b2b829c4f45 100644 --- a/t/unit/worker/test_strategy.py +++ b/t/unit/worker/test_strategy.py @@ -1,5 +1,4 @@ import logging -from collections import defaultdict from contextlib import contextmanager from unittest.mock import ANY, Mock, patch @@ -99,8 +98,8 @@ def was_limited_with_eta(self): assert not self.was_reserved() called = self.consumer.timer.call_at.called if called: - assert self.consumer.timer.call_at.call_args[0][1] == \ - self.consumer._limit_post_eta + callback = self.consumer.timer.call_at.call_args[0][1] + assert callback == self.consumer._limit_post_eta return called def was_scheduled(self): @@ -128,10 +127,15 @@ def _context(self, sig, reserved = Mock() consumer = Mock() - consumer.task_buckets = defaultdict(lambda: None) + # Create a proper mock for task_buckets that supports __getitem__ + task_buckets_mock = Mock() + task_buckets_mock.__getitem__ = Mock(side_effect=lambda key: None) + consumer.task_buckets = task_buckets_mock if limit: bucket = TokenBucket(rate(limit), capacity=1) - consumer.task_buckets[sig.task] = bucket + task_buckets_mock.__getitem__.side_effect = ( + lambda key: bucket if key == sig.task else None + ) consumer.controller.state.revoked = set() consumer.disable_rate_limits = not rate_limits consumer.event_dispatcher.enabled = events diff --git a/t/unit/worker/test_worker.py b/t/unit/worker/test_worker.py index c14c3c89f55..d0459adf9ba 100644 --- a/t/unit/worker/test_worker.py +++ b/t/unit/worker/test_worker.py @@ -111,6 +111,8 @@ def LoopConsumer(self, buffer=None, controller=None, timer=None, app=None, c.task_consumer = Mock(name='.task_consumer') c.qos = QoS(c.task_consumer.qos, 10) c.connection = Mock(name='.connection') + c.connection.connection_errors = () + c.connection.channel_errors = () c.controller = c.app.WorkController() c.heart = Mock(name='.heart') c.controller.consumer = c @@ -249,6 +251,8 @@ def _get_on_message(self, c): c.task_consumer = Mock() c.event_dispatcher = mock_event_dispatcher() c.connection = Mock(name='.connection') + c.connection.connection_errors = () + c.connection.channel_errors = () c.connection.get_heartbeat_interval.return_value = 0 c.connection.drain_events.side_effect = WorkerShutdown() @@ -1242,3 +1246,26 @@ def test_safe_say_writes_to_std_out(self, capfd): captured = capfd.readouterr() assert "\nout\n" == captured.out assert "" == captured.err + + def test_safe_say_uses_original_os_write(self): + from celery import _original_os_write + from celery.apps.worker import _original_os_write as worker_os_write + + assert _original_os_write is not None + assert callable(_original_os_write) + assert worker_os_write is _original_os_write + assert _original_os_write.__name__ == 'write' + + def test_safe_say_works_with_patched_os_write(self, capfd): + original_write = os.write + + def patched_write(fd, data): + raise RuntimeError("do not call blocking functions from the mainloop") + + try: + os.write = patched_write + safe_say("test message") + captured = capfd.readouterr() + assert "\ntest message\n" == captured.err + finally: + os.write = original_write diff --git a/tox.ini b/tox.ini index 2b5fdfcfb57..4cf856837d6 100644 --- a/tox.ini +++ b/tox.ini @@ -2,9 +2,9 @@ requires = tox-gh-actions envlist = - {3.8,3.9,3.10,3.11,3.12,3.13,pypy3}-unit - {3.8,3.9,3.10,3.11,3.12,3.13,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch,docker} - {3.8,3.9,3.10,3.11,3.12,3.13,pypy3}-smoke + {3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-unit + {3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-integration-{rabbitmq_redis,rabbitmq,redis,dynamodb,azureblockblob,cache,cassandra,elasticsearch,docker} + {3.9,3.10,3.11,3.12,3.13,3.14,pypy3}-smoke flake8 apicheck @@ -14,12 +14,12 @@ envlist = [gh-actions] python = - 3.8: 3.8-unit 3.9: 3.9-unit 3.10: 3.10-unit 3.11: 3.11-unit 3.12: 3.12-unit 3.13: 3.13-unit + 3.14: 3.14-unit pypy-3: pypy3-unit [testenv] @@ -32,8 +32,8 @@ deps= -r{toxinidir}/requirements/test.txt -r{toxinidir}/requirements/pkgutils.txt - 3.8,3.9,3.10,3.11,3.12,3.13: -r{toxinidir}/requirements/test-ci-default.txt - 3.8,3.9,3.10,3.11,3.12,3.13: -r{toxinidir}/requirements/docs.txt + 3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/test-ci-default.txt + 3.9,3.10,3.11,3.12,3.13,3.14: -r{toxinidir}/requirements/docs.txt pypy3: -r{toxinidir}/requirements/test-ci-default.txt integration: -r{toxinidir}/requirements/test-integration.txt @@ -43,10 +43,14 @@ deps= lint: pre-commit bandit: bandit +install_command = python -I -m pip install {opts} {packages} --build-constraint {toxinidir}/requirements/constraints.txt + commands = - unit: pytest -vv --maxfail=10 --capture=no -v --cov=celery --cov-report=xml --junitxml=junit.xml -o junit_family=legacy --cov-report term {posargs} + unit: coverage run --source=celery -m pytest -vv --maxfail=10 --capture=no -v --junitxml=junit.xml -o junit_family=legacy {posargs} + unit: coverage xml + unit: coverage report integration: pytest -xsvv t/integration {posargs} - smoke: pytest -xsvv t/smoke --dist=loadscope --reruns 5 --reruns-delay 10 {posargs} + smoke: pytest -xsvv t/smoke --reruns 5 --reruns-delay 10 {posargs} setenv = PIP_EXTRA_INDEX_URL=https://celery.github.io/celery-wheelhouse/repo/simple/ BOTO_CONFIG = /dev/null @@ -82,15 +86,15 @@ setenv = dynamodb: AWS_SECRET_ACCESS_KEY=test_aws_secret_key azureblockblob: TEST_BROKER=redis:// - azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1; + azureblockblob: TEST_BACKEND="azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" basepython = - 3.8: python3.8 3.9: python3.9 3.10: python3.10 3.11: python3.11 3.12: python3.12 3.13: python3.13 + 3.14: python3.14 pypy3: pypy3 mypy: python3.13 lint,apicheck,linkcheck,configcheck,bandit: python3.13