diff --git a/airflow-core/docs/administration-and-deployment/web-stack.rst b/airflow-core/docs/administration-and-deployment/web-stack.rst index f043789d5e553..a4ef9bd159c80 100644 --- a/airflow-core/docs/administration-and-deployment/web-stack.rst +++ b/airflow-core/docs/administration-and-deployment/web-stack.rst @@ -142,8 +142,8 @@ The following configuration options are available in the ``[api]`` section: - ``server_type``: ``uvicorn`` (default) or ``gunicorn`` - ``worker_refresh_interval``: Seconds between worker refresh cycles (0 = disabled, default) - ``worker_refresh_batch_size``: Number of workers to refresh per cycle (default: 1) -- ``dag_cache_size``: Max cached SerializedDAG versions in the API server (default: 64, 0 = unbounded) -- ``dag_cache_ttl``: TTL in seconds for cached DAGs (default: 3600, 0 = LRU only) +- ``dag_cache_size``: Max cached SerializedDAG versions in the API server (default: 64, 0 = no size cap) +- ``dag_cache_ttl``: TTL in seconds for cached DAGs (default: 3600, 0 = LRU only, both 0 = no eviction) When to Use Gunicorn ^^^^^^^^^^^^^^^^^^^^ diff --git a/airflow-core/docs/faq.rst b/airflow-core/docs/faq.rst index ce89ecac393ce..7418b5aab3962 100644 --- a/airflow-core/docs/faq.rst +++ b/airflow-core/docs/faq.rst @@ -725,9 +725,11 @@ in memory. Configure this in the ``[api]`` section: .. code-block:: ini [api] - dag_cache_size = 64 ; max cached versions (0 = unbounded, pre-3.2 behavior) + dag_cache_size = 64 ; max cached versions (0 = no size cap, TTL still applies) dag_cache_ttl = 3600 ; seconds before a cached entry expires (0 = LRU only) +Setting both to 0 disables eviction entirely, matching pre-3.2 behavior. + The cache is keyed by Dag version ID. After a Dag is updated, the API server may serve the previous version until the cached entry expires (controlled by ``dag_cache_ttl``). diff --git a/airflow-core/src/airflow/api_fastapi/common/dagbag.py b/airflow-core/src/airflow/api_fastapi/common/dagbag.py index d87aca49a524a..72ab77a673501 100644 --- a/airflow-core/src/airflow/api_fastapi/common/dagbag.py +++ b/airflow-core/src/airflow/api_fastapi/common/dagbag.py @@ -39,16 +39,12 @@ def create_dag_bag() -> DBDagBag: cache_ttl_config = conf.getint("api", "dag_cache_ttl", fallback=3600) if cache_size < 0: - log.warning("dag_cache_size must be >= 0, using unbounded dict") + log.warning("dag_cache_size must be >= 0, using no size cap") cache_size = 0 if cache_ttl_config < 0: log.warning("dag_cache_ttl must be >= 0, disabling TTL") cache_ttl_config = 0 - # Use unbounded dict (no eviction) if cache_size is 0 - if cache_size <= 0: - return DBDagBag(cache_size=0) - # Disable TTL if cache_ttl is 0 cache_ttl: int | None = cache_ttl_config if cache_ttl_config > 0 else None diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index c065b277716c4..22b64dc9a0a84 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -1704,7 +1704,8 @@ api: dag_cache_size: description: | Size of the LRU cache for SerializedDAG objects in the API server. - Set to 0 to use an unbounded dict (no eviction, matching pre-3.2 behavior). + Set to 0 to remove the size cap. Cached entries are then evicted only by + ``dag_cache_ttl``, or never if that is also 0 (matching pre-3.2 behavior). The cache is keyed by Dag version ID, so lookups by Dag ID (e.g., viewing a Dag's details) always query the database for the latest version, but the deserialized result is cached for subsequent @@ -1717,7 +1718,9 @@ api: description: | Time-to-live (seconds) for cached SerializedDAG objects in the API server. After this time, cached DAGs will be re-fetched from the database on next access. - Set to 0 to disable TTL (cache entries will only be evicted by LRU policy). + Applies whether or not ``dag_cache_size`` sets a size cap. Set to 0 to disable TTL, + leaving eviction to the ``dag_cache_size`` LRU policy, or no eviction at all if + ``dag_cache_size`` is also 0. Note: After a DAG is updated, the API server may serve the previous version until the cached entry expires. Lower values reduce staleness but increase diff --git a/airflow-core/src/airflow/models/dagbag.py b/airflow-core/src/airflow/models/dagbag.py index c4bd8eceea102..d36dbe5d3824f 100644 --- a/airflow-core/src/airflow/models/dagbag.py +++ b/airflow-core/src/airflow/models/dagbag.py @@ -18,6 +18,7 @@ from __future__ import annotations import hashlib +import math import time from collections.abc import MutableMapping from contextlib import nullcontext @@ -62,9 +63,9 @@ class DBDagBag: """ Internal class for retrieving dags from the database. - Optionally supports LRU+TTL caching when cache_size is provided. - The scheduler uses this without caching, while the API server can - enable caching via configuration. + Optionally caches deserialized dags. A size cap enables LRU eviction and a TTL + enables age-based eviction, with or without a size cap. The scheduler uses this + without caching, while the API server can enable caching via configuration. :meta private: """ @@ -79,8 +80,9 @@ def __init__( Initialize DBDagBag. :param load_op_links: Should the extra operator link be loaded when de-serializing the DAG? - :param cache_size: Size of LRU cache. If None or 0, uses unbounded dict (no eviction). - :param cache_ttl: Time-to-live for cache entries in seconds. If None or 0, no TTL (LRU only). + :param cache_size: Max cached entries; 0 or None means no size cap. + :param cache_ttl: Seconds until a cached entry expires. If > 0, entries are evicted by + age regardless of ``cache_size`` (with no size cap this gives TTL-only eviction). """ self.load_op_links = load_op_links self._dags: MutableMapping[UUID | str, _CacheEntry] = {} @@ -88,12 +90,13 @@ def __init__( self._revalidation_interval = conf.getint("core", "min_serialized_dag_update_interval") - # Initialize bounded cache if cache_size is provided and > 0 - if cache_size and cache_size > 0: - if cache_ttl and cache_ttl > 0: - self._dags = TTLCache(maxsize=cache_size, ttl=cache_ttl) - else: - self._dags = LRUCache(maxsize=cache_size) + # Initialize a TTL cache if configured, otherwise a bounded LRU cache. + if cache_ttl and cache_ttl > 0: + maxsize = cache_size if cache_size and cache_size > 0 else math.inf + self._dags = TTLCache(maxsize=maxsize, ttl=cache_ttl) + self._use_cache = True + elif cache_size and cache_size > 0: + self._dags = LRUCache(maxsize=cache_size) self._use_cache = True # Lock required for bounded caches: cachetools caches are NOT thread-safe diff --git a/airflow-core/tests/unit/api_fastapi/common/test_dagbag.py b/airflow-core/tests/unit/api_fastapi/common/test_dagbag.py index 48c6f706ba7e2..b908ad09a5fa3 100644 --- a/airflow-core/tests/unit/api_fastapi/common/test_dagbag.py +++ b/airflow-core/tests/unit/api_fastapi/common/test_dagbag.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import math from unittest import mock import pytest @@ -92,8 +93,9 @@ class TestCreateDagBag: ("cache_size", "cache_ttl", "expected_use_cache", "expected_dags_type"), [ pytest.param(64, 3600, True, TTLCache, id="default_ttl_cache"), - pytest.param(0, 3600, False, dict, id="size_zero_unbounded"), + pytest.param(0, 3600, True, TTLCache, id="size_zero_ttl_only"), pytest.param(64, 0, True, LRUCache, id="ttl_zero_lru_only"), + pytest.param(0, 0, False, dict, id="size_zero_ttl_zero_unbounded"), ], ) @mock.patch("airflow.api_fastapi.common.dagbag.conf") @@ -110,3 +112,14 @@ def test_create_dag_bag_cache_modes( dag_bag = create_dag_bag() assert dag_bag._use_cache is expected_use_cache assert isinstance(dag_bag._dags, expected_dags_type) + + @mock.patch("airflow.api_fastapi.common.dagbag.conf") + def test_create_dag_bag_ttl_only_has_no_size_cap(self, mock_conf): + from airflow.api_fastapi.common.dagbag import create_dag_bag + + mock_conf.getint.side_effect = lambda section, key, fallback: { + "dag_cache_size": 0, + "dag_cache_ttl": 3600, + }.get(key, fallback) + + assert create_dag_bag()._dags.maxsize == math.inf diff --git a/airflow-core/tests/unit/models/test_dagbag.py b/airflow-core/tests/unit/models/test_dagbag.py index 79668d4fe54f4..770cc3b6980d4 100644 --- a/airflow-core/tests/unit/models/test_dagbag.py +++ b/airflow-core/tests/unit/models/test_dagbag.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import math import time from concurrent.futures import ThreadPoolExecutor from unittest.mock import MagicMock, patch @@ -259,14 +260,24 @@ def test_lru_cache_enabled_with_cache_size(self): assert isinstance(dag_bag._dags, LRUCache) def test_ttl_cache_enabled_with_cache_size_and_ttl(self): - """Test that TTL cache is enabled when both cache_size and cache_ttl are provided.""" + """Test that a bounded TTL cache is used when both cache_size and cache_ttl are provided.""" dag_bag = DBDagBag(cache_size=10, cache_ttl=60) assert dag_bag._use_cache is True assert isinstance(dag_bag._dags, TTLCache) + assert dag_bag._dags.maxsize == 10 - def test_zero_cache_size_uses_unbounded_dict(self): - """Test that cache_size=0 uses unbounded dict (same as no caching).""" - dag_bag = DBDagBag(cache_size=0, cache_ttl=60) + @pytest.mark.parametrize("cache_size", [0, None]) + def test_ttl_only_without_size_cap(self, cache_size): + """Test that a positive cache_ttl with no size cap gives a TTL cache with maxsize=inf.""" + dag_bag = DBDagBag(cache_size=cache_size, cache_ttl=60) + assert dag_bag._use_cache is True + assert isinstance(dag_bag._dags, TTLCache) + assert dag_bag._dags.maxsize == math.inf + + @pytest.mark.parametrize("cache_ttl", [None, 0]) + def test_zero_cache_size_uses_unbounded_dict(self, cache_ttl): + """Test that cache_size=0 without a TTL uses an unbounded dict (same as no caching).""" + dag_bag = DBDagBag(cache_size=0, cache_ttl=cache_ttl) assert dag_bag._use_cache is False assert isinstance(dag_bag._dags, dict) @@ -310,6 +321,21 @@ def test_ttl_cache_expiry(self): with time_machine.travel("2025-01-01 00:00:02", tick=False): assert dag_bag._dags.get("test_version_id") is None + def test_ttl_only_evicts_by_ttl_not_size(self): + """An unbounded (maxsize=inf) TTL cache keeps every entry until it expires by age.""" + dag_bag = DBDagBag(cache_size=0, cache_ttl=1) + assert dag_bag._dags.maxsize == math.inf + dag_bag._dags = TTLCache(maxsize=math.inf, ttl=1, timer=time.time) + + with time_machine.travel("2025-01-01 00:00:00", tick=False): + for i in range(500): + dag_bag._dags[f"version_{i}"] = MagicMock() + assert len(dag_bag._dags) == 500 + + with time_machine.travel("2025-01-01 00:00:02", tick=False): + assert dag_bag._dags.get("version_0") is None + assert len(dag_bag._dags) == 0 + def test_lru_eviction(self): """Test that LRU eviction works when cache is full.""" dag_bag = DBDagBag(cache_size=2)