Preserve food search results and respect provider availability

backend/app/main.py

@@ -107,6 +107,7 @@
     validate_origin_login_handoff,
 )
 from .services.open_food_facts import search_food_products
+from .services.food_search_availability import FoodSearchUnavailable
 
 logger = logging.getLogger(__name__)
 
@@ -1690,6 +1691,21 @@ async def search_food(q: str = Query(..., min_length=1, max_length=120)) -> Food
 
     try:
         results = await search_food_products(query)
+    except FoodSearchUnavailable as exc:
+        logger.warning("Open Food Facts unavailable (status=%s)", exc.status_code)
+        raise HTTPException(
+            status_code=exc.status_code,
+            detail=(
+                "Food search rate limit reached"
+                if exc.status_code == 429
+                else "Food search temporarily unavailable"
+            ),
+            headers={
+                "Retry-After": str(exc.retry_after_seconds),
+                "Cache-Control": "no-store",
+                "Pragma": "no-cache",
+            },
+        ) from exc
     except AdapterAdmissionRejected as exc:
         logger.warning(
             "Open Food Facts admission rejected (reason=%s)",

backend/app/services/food_search_availability.py

@@ -0,0 +1,115 @@
+"""Short-lived public search results and provider-directed cooldowns."""
+
+import hashlib
+import math
+import time
+from collections import OrderedDict
+from collections.abc import Callable
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+from threading import Lock
+
+from app.schemas import FoodSearchResult
+
+
+DEFAULT_PROVIDER_RETRY_SECONDS = 30
+
+
+def provider_retry_seconds(value: str | None) -> int:
+    """Honor Retry-After without treating a missing header as immediate retry."""
+    if value:
+        value = value.strip()
+        if value.isascii() and value.isdigit() and len(value) <= 10:
+            return max(1, int(value))
+        try:
+            retry_at = parsedate_to_datetime(value)
+            if retry_at.tzinfo is None:
+                retry_at = retry_at.replace(tzinfo=UTC)
+            seconds = math.ceil((retry_at - datetime.now(UTC)).total_seconds())
+            if seconds > 0:
+                return seconds
+        except (TypeError, ValueError, OverflowError):
+            pass
+    return DEFAULT_PROVIDER_RETRY_SECONDS
+
+
+class FoodSearchUnavailable(Exception):
+    """A provider status safe to expose without request text or response bodies."""
+
+    def __init__(self, status_code: int, retry_after_seconds: int) -> None:
+        super().__init__("Food search temporarily unavailable")
+        self.status_code = status_code
+        self.retry_after_seconds = retry_after_seconds
+
+
+class FoodSearchAvailability:
+    def __init__(
+        self,
+        *,
+        max_entries: int = 64,
+        ttl_seconds: float = 300,
+        clock: Callable[[], float] = time.monotonic,
+    ) -> None:
+        self.max_entries = max_entries
+        self.ttl_seconds = ttl_seconds
+        self.clock = clock
+        self._lock = Lock()
+        self._cache: OrderedDict[bytes, tuple[float, list[FoodSearchResult]]] = OrderedDict()
+        self._retry_at = 0.0
+        self._status_code = 503
+
+    @staticmethod
+    def _key(query: str, page_size: int) -> bytes:
+        return hashlib.sha256(f"{page_size}\0{query}".encode("utf-8")).digest()
+
+    def _expire(self, now: float) -> None:
+        for key, (expires_at, _) in list(self._cache.items()):
+            if expires_at <= now:
+                del self._cache[key]
+
+    def get(self, query: str, page_size: int) -> list[FoodSearchResult] | None:
+        with self._lock:
+            self._expire(self.clock())
+            key = self._key(query, page_size)
+            entry = self._cache.get(key)
+            if entry is None:
+                return None
+            self._cache.move_to_end(key)
+            return [item.model_copy(deep=True) for item in entry[1]]
+
+    def remember(self, query: str, page_size: int, results: list[FoodSearchResult]) -> None:
+        # Do not turn a transient empty provider response into a cached absence.
+        if not results:
+            return
+        with self._lock:
+            now = self.clock()
+            self._expire(now)
+            key = self._key(query, page_size)
+            self._cache[key] = (
+                now + self.ttl_seconds,
+                [item.model_copy(deep=True) for item in results],
+            )
+            self._cache.move_to_end(key)
+            while len(self._cache) > self.max_entries:
+                self._cache.popitem(last=False)
+
+    def check_provider(self) -> None:
+        with self._lock:
+            remaining = math.ceil(self._retry_at - self.clock())
+            if remaining > 0:
+                raise FoodSearchUnavailable(self._status_code, remaining)
+
+    def pause_provider(self, status_code: int, seconds: int) -> None:
+        with self._lock:
+            retry_at = self.clock() + seconds
+            # An overlapping request must never shorten a provider's pause.
+            if retry_at >= self._retry_at:
+                self._retry_at = retry_at
+                self._status_code = status_code
+
+    def reset(self) -> None:
+        """Clear process-local state for isolated tests."""
+        with self._lock:
+            self._cache.clear()
+            self._retry_at = 0.0
+            self._status_code = 503

backend/app/services/open_food_facts.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import math
+import re
 import shutil
 import subprocess
 from collections.abc import Awaitable, Callable
@@ -17,6 +18,11 @@
 from app.database import engine
 from app.provider_rate_governor import build_provider_rate_governor
 from app.schemas import FoodSearchResult
+from app.services.food_search_availability import (
+    FoodSearchAvailability,
+    FoodSearchUnavailable,
+    provider_retry_seconds,
+)
 from app.source_admission import (
     AdapterAdmissionController,
     AdapterAdmissionRejected,
@@ -55,11 +61,15 @@
     DuplicateRequestCoalescer()
 )
 _OPEN_FOOD_FACTS_RATE_GOVERNOR = build_provider_rate_governor(engine)
+_OPEN_FOOD_FACTS_AVAILABILITY = FoodSearchAvailability()
 
 
 async def _governed_attempt(operation: Callable[[], Awaitable[T]]) -> T:
     """Reserve shared egress capacity immediately before an upstream attempt."""
+    _OPEN_FOOD_FACTS_AVAILABILITY.check_provider()
     await _OPEN_FOOD_FACTS_RATE_GOVERNOR.acquire()
+    # Another request can start a provider pause while this reservation waits.
+    _OPEN_FOOD_FACTS_AVAILABILITY.check_provider()
     return await operation()
 
 
@@ -133,6 +143,10 @@ def _extract_nutri_score(product: dict[str, Any]) -> str | None:
 
 async def search_food_products(query: str, page_size: int = 10) -> list[FoodSearchResult]:
     safe_query = query.strip()
+    cached = _OPEN_FOOD_FACTS_AVAILABILITY.get(safe_query, page_size)
+    if cached is not None:
+        return cached
+    _OPEN_FOOD_FACTS_AVAILABILITY.check_provider()
     return await _OPEN_FOOD_FACTS_COALESCER.run(
         (safe_query, page_size),
         lambda: _search_food_products_once(safe_query, page_size),
@@ -182,6 +196,13 @@ async def _search_food_products_once(
                 ) from fallback_exc
 
         results = _normalize_products(payload)
+    except httpx.HTTPStatusError as exc:
+        _OPEN_FOOD_FACTS_ADMISSION.record_failure(permit)
+        if exc.response.status_code in {429, 503}:
+            seconds = provider_retry_seconds(exc.response.headers.get("Retry-After"))
+            _OPEN_FOOD_FACTS_AVAILABILITY.pause_provider(exc.response.status_code, seconds)
+            raise FoodSearchUnavailable(exc.response.status_code, seconds) from exc
+        raise
     except AdapterAdmissionRejected:
         if permit.half_open_probe:
             _OPEN_FOOD_FACTS_ADMISSION.record_failure(permit)
@@ -195,6 +216,7 @@ async def _search_food_products_once(
         raise
     else:
         _OPEN_FOOD_FACTS_ADMISSION.record_success(permit)
+        _OPEN_FOOD_FACTS_AVAILABILITY.remember(safe_query, page_size, results)
         return results
 
 
@@ -273,6 +295,10 @@ def _curl_fetch(params: dict[str, Any]) -> dict[str, Any]:
                 "--silent",
                 "--show-error",
                 "--fail",
+                "--dump-header",
+                "-",
+                "--write-out",
+                "\n%{http_code}",
                 "-L",
                 "--connect-timeout",
                 "5",
@@ -284,24 +310,41 @@ def _curl_fetch(params: dict[str, Any]) -> dict[str, Any]:
                 "Accept: application/json",
                 url,
             ],
-            check=True,
+            check=False,
             capture_output=True,
             timeout=15,
         )
     except FileNotFoundError as exc:
         raise ValueError(f"{curl_cmd} not found on system; please ensure curl is installed") from exc
     except subprocess.TimeoutExpired as exc:
         raise ValueError("curl request timed out") from exc
-    except subprocess.CalledProcessError as exc:
-        stderr_text = (exc.stderr or b"").decode("utf-8", errors="replace").strip()
-        if stderr_text:
-            raise ValueError(
-                f"curl command failed (exit {exc.returncode}): {stderr_text}"
-            ) from exc
-        raise ValueError(f"curl command failed (exit {exc.returncode})") from exc
-
+    response_bytes, separator, status_bytes = completed.stdout.rpartition(b"\n")
+    status_code = int(status_bytes) if separator and re.fullmatch(rb"\d{3}", status_bytes) else None
+    if completed.returncode != 0 and (status_code is None or status_code < 400):
+        # stderr can contain the private search URL. Keep only the diagnostic
+        # exit code, and preserve known HTTP errors for provider pause handling.
+        logger.warning("Open Food Facts curl transport failed (exit=%s)", completed.returncode)
+        raise ValueError(f"curl transport failed (exit {completed.returncode})")
+    if status_code is None:
+        raise ValueError("curl response is missing its HTTP status")
+    response_headers: dict[str, str] = {}
+    # curl can print CONNECT and redirect headers before the final response.
+    # Keep only the final headers, and keep all transfer data in memory.
+    while response_bytes.startswith(b"HTTP/"):
+        parts = re.split(rb"\r?\n\r?\n", response_bytes, maxsplit=1)
+        if len(parts) != 2:
+            raise ValueError("curl response headers are incomplete")
+        header_block, response_bytes = parts
+        response_headers = {}
+        for line in header_block.splitlines()[1:]:
+            key, colon, value = line.partition(b":")
+            if colon and key.lower() == b"retry-after":
+                response_headers["Retry-After"] = value.decode("ascii", errors="replace").strip()
+    if status_code >= 400:
+        _raise_fallback_http_status(status_code, response_headers.get("Retry-After"))
+
     # Decode subprocess bytes explicitly to avoid Windows locale mojibake.
-    response_text = completed.stdout.decode("utf-8", errors="replace")
+    response_text = response_bytes.decode("utf-8", errors="replace")
     if not response_text.strip():
         raise ValueError("curl returned empty response")
     try:
@@ -325,6 +368,18 @@ def _resolve_curl_command() -> str | None:
     return shutil.which("curl")
 
 
+def _raise_fallback_http_status(status_code: int, retry_after: str | None) -> None:
+    # Preserve status and Retry-After through the same handler as httpx, without
+    # attaching private query text or copying the provider's response body.
+    request = httpx.Request("GET", OPEN_FOOD_FACTS_SEARCH_URL)
+    response = httpx.Response(
+        status_code,
+        request=request,
+        headers={"Retry-After": retry_after} if retry_after else {},
+    )
+    response.raise_for_status()
+
+
 def _urllib_fetch(params: dict[str, Any]) -> dict[str, Any]:
     """Portable fallback using Python stdlib only (no external binaries required)."""
     query_string = urlencode(params)
@@ -334,7 +389,10 @@ def _urllib_fetch(params: dict[str, Any]) -> dict[str, Any]:
         with urlopen(request, timeout=15) as response:
             response_bytes = response.read()
     except UrllibHTTPError as exc:
-        raise ValueError(f"urllib request failed with HTTP {exc.code}") from exc
+        _raise_fallback_http_status(
+            exc.code, exc.headers.get("Retry-After") if exc.headers else None
+        )
+        raise ValueError("urllib returned an unexpected status") from exc
     except (URLError, TimeoutError) as exc:
         raise ValueError(f"urllib request failed: {exc}") from exc
 

backend/tests/test_endpoints.py

@@ -220,6 +220,25 @@ def test_search_food_upstream_failure_returns_502(mock_search: AsyncMock, client
     assert response.status_code == 502
 
 
+@patch("app.main.search_food_products", new_callable=AsyncMock)
+@pytest.mark.parametrize("status", [429, 503])
+def test_search_food_preserves_provider_pause_without_exposing_request_text(
+    mock_search: AsyncMock, client: TestClient, status: int, caplog: pytest.LogCaptureFixture,
+) -> None:
+    from app.services.food_search_availability import FoodSearchUnavailable
+
+    mock_search.side_effect = FoodSearchUnavailable(status, 120)
+    response = client.get("/search-food?q=private-search-term")
+    assert response.status_code == status
+    assert response.headers["retry-after"] == "120"
+    assert response.headers["cache-control"] == "no-store"
+    assert response.headers["pragma"] == "no-cache"
+    assert "private-search-term" not in response.text
+    messages = [record.getMessage() for record in caplog.records if record.name == "app.main"]
+    assert messages
+    assert all("private-search-term" not in message for message in messages)
+
+
 @patch("app.main.search_food_products", new_callable=AsyncMock)
 def test_search_food_admission_rejection_returns_bounded_503(
     mock_search: AsyncMock,

backend/tests/test_food_search_availability.py

@@ -0,0 +1,56 @@
+from datetime import UTC, datetime, timedelta
+from email.utils import format_datetime
+
+import pytest
+
+from app.schemas import FoodSearchResult
+from app.services.food_search_availability import (
+    FoodSearchAvailability,
+    FoodSearchUnavailable,
+    provider_retry_seconds,
+)
+
+
+def food(name: str) -> list[FoodSearchResult]:
+    return [FoodSearchResult(product_name=name, calories=100, protein=5, fat=2, carbohydrates=20)]
+
+
+def test_cache_is_bounded_and_keeps_queries_and_page_sizes_separate() -> None:
+    state = FoodSearchAvailability(max_entries=2)
+    original = food("Oats")
+    state.remember("oats", 10, original)
+    original[0].product_name = "changed"
+    state.remember("oats", 20, food("More oats"))
+    assert state.get("oats", 10)[0].product_name == "Oats"
+    state.remember("apple", 10, food("Apple"))
+    assert state.get("oats", 20) is None  # least recently used
+    assert state.get("oats", 10)[0].product_name == "Oats"
+    assert state.get("apple", 10)[0].product_name == "Apple"
+    assert state.get("apple", 20) is None
+
+
+def test_overlapping_failure_cannot_shorten_provider_cooldown() -> None:
+    now = [0.0]
+    state = FoodSearchAvailability(clock=lambda: now[0])
+    state.pause_provider(429, 120)
+    now[0] = 10
+    state.pause_provider(503, 30)
+    with pytest.raises(FoodSearchUnavailable) as error:
+        state.check_provider()
+    assert error.value.status_code == 429
+    assert error.value.retry_after_seconds == 110
+    now[0] = 120
+    state.check_provider()
+
+
+@pytest.mark.parametrize("value", [None, "", "invalid", "-1", "NaN", "Infinity", "\r\nbad"])
+def test_missing_or_invalid_retry_after_uses_a_safe_default(value: str | None) -> None:
+    assert provider_retry_seconds(value) == 30
+
+
+def test_retry_after_preserves_long_pauses_and_accepts_http_dates() -> None:
+    assert provider_retry_seconds(" 7200 ") == 7200
+    future = format_datetime(datetime.now(UTC) + timedelta(seconds=180), usegmt=True)
+    assert 179 <= provider_retry_seconds(future) <= 180
+    past = format_datetime(datetime.now(UTC) - timedelta(seconds=10), usegmt=True)
+    assert provider_retry_seconds(past) == 30